From b9a4581d606e33f085c136a4906732222f58a1a2 Mon Sep 17 00:00:00 2001 From: Anton Lundin Date: Wed, 6 Nov 2019 12:48:44 +0100 Subject: [PATCH 01/17] Pass in HOME env-var in container mode. To get ~/-paths to work as expected in contaier mode, env-var HOME must be the same outside the container as inside the docker-compose container, otherwise HOME inside the container points to /root which might not be what the user expects. Signed-off-by: Anton Lundin --- script/run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run/run.sh b/script/run/run.sh index f3456720f..21b837329 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -43,7 +43,7 @@ if [ -n "$compose_dir" ]; then VOLUMES="$VOLUMES -v $compose_dir:$compose_dir" fi if [ -n "$HOME" ]; then - VOLUMES="$VOLUMES -v $HOME:$HOME -v $HOME:/root" # mount $HOME in /root to share docker.config + VOLUMES="$VOLUMES -v $HOME:$HOME -e HOME" # Pass in HOME to share docker.config and allow ~/-relative paths to work. fi # Only allocate tty if we detect one From aeddfd41d6bd4d44577689eaef6756f2a28ce605 Mon Sep 17 00:00:00 2001 From: Anton Lundin Date: Wed, 6 Nov 2019 13:50:48 +0100 Subject: [PATCH 02/17] Resolve shellcheck warnings in container-script This fixes a couple of small potential issues pointed out by shellcheck. Signed-off-by: Anton Lundin --- script/run/run.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/script/run/run.sh b/script/run/run.sh index f3456720f..dc28eeb7e 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -36,7 +36,7 @@ if [ "$(pwd)" != '/' ]; then fi if [ -n "$COMPOSE_FILE" ]; then COMPOSE_OPTIONS="$COMPOSE_OPTIONS -e COMPOSE_FILE=$COMPOSE_FILE" - compose_dir=$(realpath $(dirname $COMPOSE_FILE)) + compose_dir=$(realpath "$(dirname "$COMPOSE_FILE")") fi # TODO: also check --file argument if [ -n "$compose_dir" ]; then @@ -47,7 +47,7 @@ if [ -n "$HOME" ]; then fi # Only allocate tty if we detect one -if [ -t 0 -a -t 1 ]; then +if [ -t 0 ] && [ -t 1 ]; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" fi @@ -56,8 +56,9 @@ DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" # Handle userns security -if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then +if docker info 2>/dev/null | grep -q userns; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host" fi +# shellcheck disable=SC2086 exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@" From 962421d0197db71db037a9c82a0d9bc356547d05 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 20 Nov 2019 08:56:55 +0100 Subject: [PATCH 03/17] config_detail.filename is None when passed by stdin Fix #7032 Signed-off-by: Nicolas De Loof --- compose/cli/command.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index c3a10a043..bdab67ccf 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -170,7 +170,11 @@ def execution_context_labels(config_details, environment_file): def config_files_label(config_details): return ",".join( - map(str, (os.path.normpath(c.filename) for c in config_details.config_files))) + map(str, (config_file_path(c.filename) for c in config_details.config_files))) + + +def config_file_path(config_file): + return os.path.normpath(config_file) if config_file else 'stdin' def get_project_name(working_dir, project_name=None, environment=None): From fe2b69254797309c28ef95f15991e971dc3d7511 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Wed, 20 Nov 2019 13:32:07 +0100 Subject: [PATCH 04/17] testcase for compose file read from stdin Signed-off-by: Nicolas De Loof --- tests/acceptance/cli_test.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a03d56567..b1790fdcf 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -48,6 +48,7 @@ BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:1.27.2' def start_process(base_dir, options): proc = subprocess.Popen( ['docker-compose'] + options, + stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=base_dir) @@ -55,8 +56,8 @@ def start_process(base_dir, options): return proc -def wait_on_process(proc, returncode=0): - stdout, stderr = proc.communicate() +def wait_on_process(proc, returncode=0, stdin=None): + stdout, stderr = proc.communicate(input=stdin) if proc.returncode != returncode: print("Stderr: {}".format(stderr)) print("Stdout: {}".format(stdout)) @@ -64,10 +65,10 @@ def wait_on_process(proc, returncode=0): return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) -def dispatch(base_dir, options, project_options=None, returncode=0): +def dispatch(base_dir, options, project_options=None, returncode=0, stdin=None): project_options = project_options or [] proc = start_process(base_dir, project_options + options) - return wait_on_process(proc, returncode=returncode) + return wait_on_process(proc, returncode=returncode, stdin=stdin) def wait_on_condition(condition, delay=0.1, timeout=40): @@ -156,8 +157,8 @@ class CLITestCase(DockerClientTestCase): self._project = get_project(self.base_dir, override_dir=self.override_dir) return self._project - def dispatch(self, options, project_options=None, returncode=0): - return dispatch(self.base_dir, options, project_options, returncode) + def dispatch(self, options, project_options=None, returncode=0, stdin=None): + return dispatch(self.base_dir, options, project_options, returncode, stdin) def execute(self, container, cmd): # Remove once Hijack and CloseNotifier sign a peace treaty @@ -241,6 +242,17 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '--quiet']).stdout == '' + def test_config_stdin(self): + config = b"""version: "3.7" +services: + web: + image: nginx + other: + image: alpine +""" + result = self.dispatch(['-f', '-', 'config', '--services'], stdin=config) + assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} + def test_config_with_hash_option(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--hash=*']) From e13a7213f13f4e7993f33736e75e64473153719e Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Fri, 8 Nov 2019 15:05:40 +0100 Subject: [PATCH 05/17] Build OSX binary as a directory OSX Catalina otherwise do scan the temporary executable files created by the single-file packaging. Signed-off-by: Nicolas De Loof --- .circleci/config.yml | 4 +- docker-compose_darwin.spec | 108 +++++++++++++++++++++++++++ script/build/osx | 6 ++ script/release/release/downloader.py | 1 + 4 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 docker-compose_darwin.spec diff --git a/.circleci/config.yml b/.circleci/config.yml index 906b1c0dc..9b95b7e9e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,8 +30,8 @@ jobs: name: build script command: ./script/build/osx - store_artifacts: - path: dist/docker-compose-Darwin-x86_64 - destination: docker-compose-Darwin-x86_64 + path: dist/docker-compose-Darwin-x86_64.tgz + destination: docker-compose-Darwin-x86_64.tgz - deploy: name: Deploy binary to bintray command: | diff --git a/docker-compose_darwin.spec b/docker-compose_darwin.spec new file mode 100644 index 000000000..344c070d5 --- /dev/null +++ b/docker-compose_darwin.spec @@ -0,0 +1,108 @@ +# -*- mode: python ; coding: utf-8 -*- + +block_cipher = None + +a = Analysis(['bin/docker-compose'], + pathex=['.'], + hiddenimports=[], + hookspath=[], + runtime_hooks=[], + cipher=block_cipher) + +pyz = PYZ(a.pure, a.zipped_data, + cipher=block_cipher) + +exe = EXE(pyz, + a.scripts, + exclude_binaries=True, + name='docker-compose', + debug=False, + strip=False, + upx=True, + console=True, + bootloader_ignore_signals=True) +coll = COLLECT(exe, + a.binaries, + a.zipfiles, + a.datas, + [ + ( + 'compose/config/config_schema_v1.json', + 'compose/config/config_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.0.json', + 'compose/config/config_schema_v2.0.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.1.json', + 'compose/config/config_schema_v2.1.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.2.json', + 'compose/config/config_schema_v2.2.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.3.json', + 'compose/config/config_schema_v2.3.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v2.4.json', + 'compose/config/config_schema_v2.4.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.0.json', + 'compose/config/config_schema_v3.0.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.1.json', + 'compose/config/config_schema_v3.1.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.2.json', + 'compose/config/config_schema_v3.2.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.3.json', + 'compose/config/config_schema_v3.3.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.4.json', + 'compose/config/config_schema_v3.4.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.5.json', + 'compose/config/config_schema_v3.5.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.6.json', + 'compose/config/config_schema_v3.6.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.7.json', + 'compose/config/config_schema_v3.7.json', + 'DATA' + ), + ( + 'compose/GITSHA', + 'compose/GITSHA', + 'DATA' + ) + ], + strip=False, + upx=True, + upx_exclude=[], + name='docker-compose-Darwin-x86_64') diff --git a/script/build/osx b/script/build/osx index 529914586..bd501ee55 100755 --- a/script/build/osx +++ b/script/build/osx @@ -11,6 +11,12 @@ venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA + venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version + +# Also build as a folder, required on osx Catalina +venv/bin/pyinstaller docker-compose_darwin.spec +dist/docker-compose-Darwin-x86_64/docker-compose version +cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz . diff --git a/script/release/release/downloader.py b/script/release/release/downloader.py index d92ae78b5..0e9b80130 100644 --- a/script/release/release/downloader.py +++ b/script/release/release/downloader.py @@ -55,6 +55,7 @@ class BinaryDownloader(requests.Session): def download_all(self, version): files = { + 'docker-compose-Darwin-x86_64.tgz': None, 'docker-compose-Darwin-x86_64': None, 'docker-compose-Linux-x86_64': None, 'docker-compose-Windows-x86_64.exe': None, From fa9e8bd6412f01597288795df850a0320419acc5 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 21 Nov 2019 08:31:48 +0100 Subject: [PATCH 06/17] Skip label if some config file was passed by stdin com.docker.compose.project.config_files must contain valid file paths Signed-off-by: Nicolas De Loof --- compose/cli/command.py | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index bdab67ccf..1fa8a17a2 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -159,22 +159,28 @@ def get_project(project_dir, config_path=None, project_name=None, verbose=False, def execution_context_labels(config_details, environment_file): extra_labels = [ - '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)), - '{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details)), + '{0}={1}'.format(LABEL_WORKING_DIR, os.path.abspath(config_details.working_dir)) ] + + if not use_config_from_stdin(config_details): + extra_labels.append('{0}={1}'.format(LABEL_CONFIG_FILES, config_files_label(config_details))) + if environment_file is not None: extra_labels.append('{0}={1}'.format(LABEL_ENVIRONMENT_FILE, os.path.normpath(environment_file))) return extra_labels +def use_config_from_stdin(config_details): + for c in config_details.config_files: + if not c.filename: + return True + return False + + def config_files_label(config_details): return ",".join( - map(str, (config_file_path(c.filename) for c in config_details.config_files))) - - -def config_file_path(config_file): - return os.path.normpath(config_file) if config_file else 'stdin' + map(str, (os.path.normpath(c.filename) for c in config_details.config_files))) def get_project_name(working_dir, project_name=None, environment=None): From f4cf7a939ef656f25bf4e68cd2bb95388c8b57ed Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 21 Nov 2019 12:00:30 +0100 Subject: [PATCH 07/17] Announce drop py2 Signed-off-by: Ulysses Souza --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index fd643f174..f1ae7e1ed 100644 --- a/README.md +++ b/README.md @@ -2,6 +2,8 @@ Docker Compose ============== ![Docker Compose](logo.png?raw=true "Docker Compose Logo") +## :exclamation: The docker-compose project announces that as Python 2 reaches it's EOL, versions 1.25.x will be the last to support it. For more information, please refer to this [issue](https://github.com/docker/compose/issues/6890). + Compose is a tool for defining and running multi-container Docker applications. With Compose, you use a Compose file to configure your application's services. Then, using a single command, you create and start all the services From e6ec77047bf50abc34d5f08781bb36f75d424b59 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Thu, 21 Nov 2019 11:09:49 +0100 Subject: [PATCH 08/17] Revert "only pull images that can't build" This reverts commit c6dd7da15eb3d85a1f7634e8ded9fe42c9035669. Signed-off-by: Nicolas De Loof --- compose/project.py | 7 ++----- tests/acceptance/cli_test.py | 7 ------- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/compose/project.py b/compose/project.py index 094ce4d7a..d7dcb6bd6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -619,9 +619,6 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, include_deps=False): services = self.get_services(service_names, include_deps) - images_to_build = {service.image_name for service in services if service.can_be_built()} - services_to_pull = [service for service in services if service.image_name not in images_to_build] - msg = not silent and 'Pulling' or None if parallel_pull: @@ -647,7 +644,7 @@ class Project(object): ) _, errors = parallel.parallel_execute( - services_to_pull, + services, pull_service, operator.attrgetter('name'), msg, @@ -660,7 +657,7 @@ class Project(object): raise ProjectError(combined_errors) else: - for service in services_to_pull: + for service in services: service.pull(ignore_pull_failures, silent=silent) def push(self, service_names=None, ignore_push_failures=False): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a03d56567..25f426d5b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -661,13 +661,6 @@ class CLITestCase(DockerClientTestCase): 'image library/nonexisting-image:latest not found' in result.stderr or 'pull access denied for nonexisting-image' in result.stderr) - def test_pull_with_build(self): - result = self.dispatch(['-f', 'pull-with-build.yml', 'pull']) - - assert 'Pulling simple' not in result.stderr - assert 'Pulling from_simple' not in result.stderr - assert 'Pulling another ...' in result.stderr - def test_pull_with_quiet(self): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' From d837d27ad7dd4ed8b00d45106af00f251352d634 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 22 Nov 2019 14:12:39 +0100 Subject: [PATCH 09/17] Update dev version Signed-off-by: Ulysses Souza --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index d35e818c7..69c4e0e49 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.25.0' +__version__ = '1.26.0dev' From a7e8356651c6caac6097c1f9b4ec02fd4bc14830 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 22 Nov 2019 15:13:22 +0100 Subject: [PATCH 10/17] Update maintainers and add CODEOWNERS for github Signed-off-by: Ulysses Souza --- .github/CODEOWNERS | 6 ++++++ MAINTAINERS | 6 ++++++ 2 files changed, 12 insertions(+) create mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS new file mode 100644 index 000000000..13fb9bac0 --- /dev/null +++ b/.github/CODEOWNERS @@ -0,0 +1,6 @@ +# GitHub code owners +# See https://help.github.com/articles/about-codeowners/ +# +# KEEP THIS FILE SORTED. Order is important. Last match takes precedence. + +* @ndeloof @rumpl @ulyssessouza diff --git a/MAINTAINERS b/MAINTAINERS index 5d4bd6a63..27bde5bb3 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,6 +11,7 @@ [Org] [Org."Core maintainers"] people = [ + "ndeloof", "rumpl", "ulyssessouza", ] @@ -77,6 +78,11 @@ Email = "mazz@houseofmnowster.com" GitHub = "mnowster" + [people.ndeloof] + Name = "Nicolas De Loof" + Email = "nicolas.deloof@gmail.com" + GitHub = "ndeloof" + [people.rumpl] Name = "Djordje Lukic" Email = "djordje.lukic@docker.com" From c8cfc590cc8252a3130c40f5257c32c9a3cd7a82 Mon Sep 17 00:00:00 2001 From: Anton Lundin Date: Fri, 22 Nov 2019 16:24:09 +0100 Subject: [PATCH 11/17] Better userns detection in container-script MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This uses formatting to have docker info just emit the information we're interested in. Based-on-code-by: Sebastiaan van Stijn Signed-off-by: Anton Lundin Co-Authored-By: Sebastiaan van Stijn --- script/run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run/run.sh b/script/run/run.sh index dc28eeb7e..5bed51cac 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -56,7 +56,7 @@ DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" # Handle userns security -if docker info 2>/dev/null | grep -q userns; then +if docker info --format '{{json .SecurityOptions}}' 2>/dev/null | grep -q 'name=userns'; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host" fi From 55c5c8e8ac35d8c069ec8dd2ebeb919e39efca41 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 25 Nov 2019 11:09:42 +0100 Subject: [PATCH 12/17] Report image we can't pull and must be built Signed-off-by: Nicolas De Loof --- compose/progress_stream.py | 10 +++ compose/project.py | 81 ++++++++++++------- tests/acceptance/cli_test.py | 8 ++ .../can-build-pull-failures.yml | 6 ++ 4 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 tests/fixtures/simple-composefile/can-build-pull-failures.yml diff --git a/compose/progress_stream.py b/compose/progress_stream.py index c4281cb4c..522ddf75d 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -114,3 +114,13 @@ def get_digest_from_push(events): if digest: return digest return None + + +def read_status(event): + status = event['status'].lower() + if 'progressDetail' in event: + detail = event['progressDetail'] + if 'current' in detail and 'total' in detail: + percentage = float(detail['current']) / float(detail['total']) + status = '{} ({:.1%})'.format(status, percentage) + return status diff --git a/compose/project.py b/compose/project.py index d7dcb6bd6..d7405defd 100644 --- a/compose/project.py +++ b/compose/project.py @@ -11,6 +11,8 @@ from os import path import enum import six from docker.errors import APIError +from docker.errors import ImageNotFound +from docker.errors import NotFound from docker.utils import version_lt from . import parallel @@ -25,6 +27,7 @@ from .container import Container from .network import build_networks from .network import get_networks from .network import ProjectNetworks +from .progress_stream import read_status from .service import BuildAction from .service import ContainerNetworkMode from .service import ContainerPidMode @@ -619,46 +622,68 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, include_deps=False): services = self.get_services(service_names, include_deps) - msg = not silent and 'Pulling' or None if parallel_pull: - def pull_service(service): - strm = service.pull(ignore_pull_failures, True, stream=True) - if strm is None: # Attempting to pull service with no `image` key is a no-op - return + self.parallel_pull(services, silent=silent) + else: + must_build = [] + for service in services: + try: + service.pull(ignore_pull_failures, silent=silent) + except (ImageNotFound, NotFound): + if service.can_be_built(): + must_build.append(service.name) + else: + raise + + if len(must_build): + log.warning('Some service image(s) must be built from source by running:\n' + ' docker-compose build {}' + .format(' '.join(must_build))) + + def parallel_pull(self, services, ignore_pull_failures=False, silent=False): + msg = 'Pulling' if not silent else None + must_build = [] + + def pull_service(service): + strm = service.pull(ignore_pull_failures, True, stream=True) + + if strm is None: # Attempting to pull service with no `image` key is a no-op + return + + try: writer = parallel.get_stream_writer() - for event in strm: if 'status' not in event: continue - status = event['status'].lower() - if 'progressDetail' in event: - detail = event['progressDetail'] - if 'current' in detail and 'total' in detail: - percentage = float(detail['current']) / float(detail['total']) - status = '{} ({:.1%})'.format(status, percentage) - + status = read_status(event) writer.write( msg, service.name, truncate_string(status), lambda s: s ) + except (ImageNotFound, NotFound): + if service.can_be_built(): + must_build.append(service.name) + else: + raise - _, errors = parallel.parallel_execute( - services, - pull_service, - operator.attrgetter('name'), - msg, - limit=5, - ) - if len(errors): - combined_errors = '\n'.join([ - e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() - ]) - raise ProjectError(combined_errors) + _, errors = parallel.parallel_execute( + services, + pull_service, + operator.attrgetter('name'), + msg, + limit=5, + ) - else: - for service in services: - service.pull(ignore_pull_failures, silent=silent) + if len(must_build): + log.warning('Some service image(s) must be built from source by running:\n' + ' docker-compose build {}' + .format(' '.join(must_build))) + if len(errors): + combined_errors = '\n'.join([ + e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() + ]) + raise ProjectError(combined_errors) def push(self, service_names=None, ignore_push_failures=False): unique_images = set() diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 41e51a304..b729e7d76 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -694,6 +694,14 @@ services: result.stderr ) + def test_pull_can_build(self): + result = self.dispatch([ + '-f', 'can-build-pull-failures.yml', 'pull'], + returncode=0 + ) + assert 'Some service image(s) must be built from source' in result.stderr + assert 'docker-compose build can_build' in result.stderr + def test_pull_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' result = self.dispatch(['pull', '--no-parallel', 'web']) diff --git a/tests/fixtures/simple-composefile/can-build-pull-failures.yml b/tests/fixtures/simple-composefile/can-build-pull-failures.yml new file mode 100644 index 000000000..1ffe8e0fb --- /dev/null +++ b/tests/fixtures/simple-composefile/can-build-pull-failures.yml @@ -0,0 +1,6 @@ +version: '3' +services: + can_build: + image: nonexisting-image-but-can-build:latest + build: . + command: top From d6b5d324e2b08587c3e4b99ac456276bc4d8d5f7 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 28 Nov 2019 10:50:21 +0100 Subject: [PATCH 13/17] Use Python3 for macOS build environment With the deprecation of Python 2 coming soon, explicitly use Python 3. Signed-off-by: Christopher Crone --- script/setup/osx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/setup/osx b/script/setup/osx index 69280f8a2..08420fb23 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -36,7 +36,7 @@ if ! [ -x "$(command -v python3)" ]; then brew install python3 fi if ! [ -x "$(command -v virtualenv)" ]; then - pip install virtualenv==16.2.0 + pip3 install virtualenv==16.2.0 fi # From fedc8f71adef0eb33f6697fb16c3c4f023f83617 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Fri, 29 Nov 2019 18:27:35 +0100 Subject: [PATCH 14/17] Build single binary and folder format for macOS Previously we were overwriting the single binary with the folder format. Signed-off-by: Christopher Crone --- script/build/osx | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/script/build/osx b/script/build/osx index bd501ee55..66868756b 100755 --- a/script/build/osx +++ b/script/build/osx @@ -12,11 +12,13 @@ venv/bin/pip install --no-deps . DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA +# Build as a folder for macOS Catalina. +venv/bin/pyinstaller docker-compose_darwin.spec +dist/docker-compose-Darwin-x86_64/docker-compose version +(cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz .) +rm -rf dist/docker-compose-Darwin-x86_64 + +# Build static binary for legacy. venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version - -# Also build as a folder, required on osx Catalina -venv/bin/pyinstaller docker-compose_darwin.spec -dist/docker-compose-Darwin-x86_64/docker-compose version -cd dist/docker-compose-Darwin-x86_64/ && tar zcvf ../docker-compose-Darwin-x86_64.tgz . From 882034388245b2a852a90ac4d1ffaa5daa37f751 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Fri, 29 Nov 2019 18:28:17 +0100 Subject: [PATCH 15/17] Stash all macOS build artifacts Signed-off-by: Christopher Crone --- .circleci/config.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.circleci/config.yml b/.circleci/config.yml index 9b95b7e9e..36dd8d57e 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -29,6 +29,9 @@ jobs: - run: name: build script command: ./script/build/osx + - store_artifacts: + path: dist/docker-compose-Darwin-x86_64 + destination: docker-compose-Darwin-x86_64 - store_artifacts: path: dist/docker-compose-Darwin-x86_64.tgz destination: docker-compose-Darwin-x86_64.tgz From b7a675b1c047dcf53da0076e074271e2af8e4d00 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Fri, 29 Nov 2019 18:28:47 +0100 Subject: [PATCH 16/17] Upload macOS folder format to bintray Signed-off-by: Christopher Crone --- script/circle/bintray-deploy.sh | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/script/circle/bintray-deploy.sh b/script/circle/bintray-deploy.sh index d508da365..a7cce726e 100755 --- a/script/circle/bintray-deploy.sh +++ b/script/circle/bintray-deploy.sh @@ -25,3 +25,11 @@ curl -f -T dist/docker-compose-${OS_NAME}-x86_64 -u$BINTRAY_USERNAME:$BINTRAY_AP -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \ -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \ https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64 || exit 1 + +# Upload folder format of docker-compose for macOS in addition to binary. +if [ "${OS_NAME}" == "Darwin" ]; then + curl -f -T dist/docker-compose-${OS_NAME}-x86_64.tgz -u$BINTRAY_USERNAME:$BINTRAY_API_KEY \ + -H "X-Bintray-Package: ${PKG_NAME}" -H "X-Bintray-Version: $CIRCLE_BRANCH" \ + -H "X-Bintray-Override: 1" -H "X-Bintray-Publish: 1" -X PUT \ + https://api.bintray.com/content/docker-compose/${CIRCLE_BRANCH}/docker-compose-${OS_NAME}-x86_64.tgz || exit 1 +fi From d92e9beec1b405df52ff745d8128c72b5bc35833 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 29 Nov 2019 19:21:16 +0100 Subject: [PATCH 17/17] "Bump 1.25.1-rc1" Signed-off-by: Ulysses Souza --- CHANGELOG.md | 13 +++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 15 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index d1a6fae32..ea0ddf802 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,19 @@ Change log ========== +1.25.1-rc1 (2019-11-29) +----------------------- + +### Bugfixes + +- Discard label `com.docker.compose.filepaths` having `None` as value. Typically, when coming from stdin + +- Add OSX binary as a directory to solve slow start up time caused by MacOS Catalina binary scan + +- Pass in HOME env-var in container mode (running with `script/run/run.sh`) + +- Revert behavior of "only pull images that we can't build" and replace by a warning informing the image we can't pull and must be built + 1.25.0 (2019-11-18) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 69c4e0e49..cda820978 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.26.0dev' +__version__ = '1.25.1-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 77f245c13..744766a06 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.25.0" +VERSION="1.25.1-rc1" IMAGE="docker/compose:$VERSION"