From f4ed9b2ef59513fe09392e8386b8eafc19b5e578 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 22 Nov 2018 18:26:25 +0100 Subject: [PATCH 01/62] Detects the execution on an`exec` command and sets the environment to `silent` mode. Signed-off-by: Ulysses Souza --- compose/cli/command.py | 16 +++++++++ compose/config/environment.py | 3 +- requirements-dev.txt | 1 + tests/integration/environment_test.py | 52 +++++++++++++++++++++++++++ 4 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 tests/integration/environment_test.py diff --git a/compose/cli/command.py b/compose/cli/command.py index 339a65c53..c1f6c2925 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -21,10 +21,26 @@ from .utils import get_version_info log = logging.getLogger(__name__) +SILENT_COMMANDS = set(( + 'events', + 'exec', + 'kill', + 'logs', + 'pause', + 'ps', + 'restart', + 'rm', + 'start', + 'stop', + 'top', + 'unpause', +)) + def project_from_options(project_dir, options): override_dir = options.get('--project-directory') environment = Environment.from_env_file(override_dir or project_dir) + environment.silent = options.get('COMMAND', None) in SILENT_COMMANDS set_parallel_limit(environment) host = options.get('--host') diff --git a/compose/config/environment.py b/compose/config/environment.py index bd52758f2..62c40a4bc 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -56,6 +56,7 @@ class Environment(dict): def __init__(self, *args, **kwargs): super(Environment, self).__init__(*args, **kwargs) self.missing_keys = [] + self.silent = False @classmethod def from_env_file(cls, base_dir): @@ -95,7 +96,7 @@ class Environment(dict): return super(Environment, self).__getitem__(key.upper()) except KeyError: pass - if key not in self.missing_keys: + if not self.silent and key not in self.missing_keys: log.warn( "The {} variable is not set. Defaulting to a blank string." .format(key) diff --git a/requirements-dev.txt b/requirements-dev.txt index 4d74f6d15..d49ce49c4 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ coverage==4.4.2 +ddt==1.2.0 flake8==3.5.0 mock>=1.0.1 pytest==3.6.3 diff --git a/tests/integration/environment_test.py b/tests/integration/environment_test.py new file mode 100644 index 000000000..07619eec1 --- /dev/null +++ b/tests/integration/environment_test.py @@ -0,0 +1,52 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import tempfile + +from ddt import data +from ddt import ddt + +from .. import mock +from compose.cli.command import project_from_options +from tests.integration.testcases import DockerClientTestCase + + +@ddt +class EnvironmentTest(DockerClientTestCase): + @classmethod + def setUpClass(cls): + super(EnvironmentTest, cls).setUpClass() + cls.compose_file = tempfile.NamedTemporaryFile(mode='w+b') + cls.compose_file.write(bytes("""version: '3.2' +services: + svc: + image: busybox:latest + environment: + TEST_VARIABLE: ${TEST_VARIABLE}""", encoding='utf-8')) + cls.compose_file.flush() + + @classmethod + def tearDownClass(cls): + super(EnvironmentTest, cls).tearDownClass() + cls.compose_file.close() + + @data('events', + 'exec', + 'kill', + 'logs', + 'pause', + 'ps', + 'restart', + 'rm', + 'start', + 'stop', + 'top', + 'unpause') + def _test_no_warning_on_missing_host_environment_var_on_silent_commands(self, cmd): + options = {'COMMAND': cmd, '--file': [EnvironmentTest.compose_file.name]} + with mock.patch('compose.config.environment.log') as fake_log: + # Note that the warning silencing and the env variables check is + # done in `project_from_options` + # So no need to have a proper options map, the `COMMAND` key is enough + project_from_options('.', options) + assert fake_log.warn.call_count == 0 From 693343500499d6052dff6e7db2291a1bb6ecb611 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 14 Jan 2019 15:22:12 -0800 Subject: [PATCH 02/62] Update maintainers file Signed-off-by: Joffrey F --- MAINTAINERS | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 7aedd46e9..5d4bd6a63 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -11,9 +11,8 @@ [Org] [Org."Core maintainers"] people = [ - "mefyl", - "mnottale", - "shin-", + "rumpl", + "ulyssessouza", ] [Org.Alumni] people = [ @@ -34,6 +33,10 @@ # including muti-file support, variable interpolation, secrets # emulation and many more "dnephin", + + "shin-", + "mefyl", + "mnottale", ] [people] @@ -74,7 +77,17 @@ Email = "mazz@houseofmnowster.com" GitHub = "mnowster" - [People.shin-] + [people.rumpl] + Name = "Djordje Lukic" + Email = "djordje.lukic@docker.com" + GitHub = "rumpl" + + [people.shin-] Name = "Joffrey F" - Email = "joffrey@docker.com" + Email = "f.joffrey@gmail.com" GitHub = "shin-" + + [people.ulyssessouza] + Name = "Ulysses Domiciano Souza" + Email = "ulysses.souza@docker.com" + GitHub = "ulyssessouza" From 14a1a0c020e70b99bc5b54f2098be52c66c45fb6 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 15 Jan 2019 09:01:49 +0100 Subject: [PATCH 03/62] Add bash completion for `ps --all|-a` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 395888d34..5938ff9e2 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -361,7 +361,7 @@ _docker_compose_ps() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --quiet -q --services --filter" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--all -a --filter --help --quiet -q --services" -- "$cur" ) ) ;; *) __docker_compose_complete_services From 0c20fc5d914367816387dbf149077835117daa14 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Fri, 11 Jan 2019 10:04:57 +0100 Subject: [PATCH 04/62] Resolve digests without pulling image If there is no image locally `docker-compose --resolve-image-digests` will try and get the digest from the repository. Fixes https://github.com/docker/compose/issues/5818 Signed-off-by: Djordje Lukic --- compose/bundle.py | 41 +++++++++++++++++++++++++++------------ compose/service.py | 6 ++++++ tests/unit/bundle_test.py | 11 +++++++++++ 3 files changed, 46 insertions(+), 12 deletions(-) diff --git a/compose/bundle.py b/compose/bundle.py index 937a3708a..efc455b72 100644 --- a/compose/bundle.py +++ b/compose/bundle.py @@ -95,19 +95,10 @@ def get_image_digest(service, allow_push=False): if separator == '@': return service.options['image'] - try: - image = service.image() - except NoSuchImageError: - action = 'build' if 'build' in service.options else 'pull' - raise UserError( - "Image not found for service '{service}'. " - "You might need to run `docker-compose {action} {service}`." - .format(service=service.name, action=action)) + digest = get_digest(service) - if image['RepoDigests']: - # TODO: pick a digest based on the image tag if there are multiple - # digests - return image['RepoDigests'][0] + if digest: + return digest if 'build' not in service.options: raise NeedsPull(service.image_name, service.name) @@ -118,6 +109,32 @@ def get_image_digest(service, allow_push=False): return push_image(service) +def get_digest(service): + digest = None + try: + image = service.image() + # TODO: pick a digest based on the image tag if there are multiple + # digests + if image['RepoDigests']: + digest = image['RepoDigests'][0] + except NoSuchImageError: + try: + # Fetch the image digest from the registry + distribution = service.get_image_registry_data() + + if distribution['Descriptor']['digest']: + digest = '{image_name}@{digest}'.format( + image_name=service.image_name, + digest=distribution['Descriptor']['digest'] + ) + except NoSuchImageError: + raise UserError( + "Digest not found for service '{service}'. " + "Repository does not exist or may require 'docker login'" + .format(service=service.name)) + return digest + + def push_image(service): try: digest = service.push() diff --git a/compose/service.py b/compose/service.py index 3c5e356a3..59f05f8dc 100644 --- a/compose/service.py +++ b/compose/service.py @@ -363,6 +363,12 @@ class Service(object): "rebuild this image you must use `docker-compose build` or " "`docker-compose up --build`.".format(self.name)) + def get_image_registry_data(self): + try: + return self.client.inspect_distribution(self.image_name) + except APIError: + raise NoSuchImageError("Image '{}' not found".format(self.image_name)) + def image(self): try: return self.client.inspect_image(self.image_name) diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index 88f75405a..065cdd00b 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -10,6 +10,7 @@ from compose import service from compose.cli.errors import UserError from compose.config.config import Config from compose.const import COMPOSEFILE_V2_0 as V2_0 +from compose.service import NoSuchImageError @pytest.fixture @@ -35,6 +36,16 @@ def test_get_image_digest_image_uses_digest(mock_service): assert not mock_service.image.called +def test_get_image_digest_from_repository(mock_service): + mock_service.options['image'] = 'abcd' + mock_service.image_name = 'abcd' + mock_service.image.side_effect = NoSuchImageError(None) + mock_service.get_image_registry_data.return_value = {'Descriptor': {'digest': 'digest'}} + + digest = bundle.get_image_digest(mock_service) + assert digest == 'abcd@digest' + + def test_get_image_digest_no_image(mock_service): with pytest.raises(UserError) as exc: bundle.get_image_digest(service.Service(name='theservice')) From ae0f3c74a0a986f0b5f1c0b546d6fb88e617aa76 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Wed, 16 Jan 2019 13:47:23 +0100 Subject: [PATCH 05/62] Support for credential_spec Signed-off-by: Djordje Lukic --- compose/cli/main.py | 4 ++-- compose/config/config.py | 23 ++++++++++++++++++++++- compose/config/validation.py | 12 ++++++++++++ contrib/completion/zsh/_docker-compose | 2 +- tests/unit/config/config_test.py | 6 +++++- 5 files changed, 42 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 950e5055d..789601792 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -206,8 +206,8 @@ class TopLevelCommand(object): name specified in the client certificate --project-directory PATH Specify an alternate working directory (default: the path of the Compose file) - --compatibility If set, Compose will attempt to convert deploy - keys in v3 files to their non-Swarm equivalent + --compatibility If set, Compose will attempt to convert keys + in v3 files to their non-Swarm equivalent Commands: build Build or rebuild services diff --git a/compose/config/config.py b/compose/config/config.py index 3df211f73..e2ed29a47 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -51,6 +51,7 @@ from .validation import match_named_volumes from .validation import validate_against_config_schema from .validation import validate_config_section from .validation import validate_cpu +from .validation import validate_credential_spec from .validation import validate_depends_on from .validation import validate_extends_file_path from .validation import validate_healthcheck @@ -369,7 +370,6 @@ def check_swarm_only_config(service_dicts, compatibility=False): ) if not compatibility: check_swarm_only_key(service_dicts, 'deploy') - check_swarm_only_key(service_dicts, 'credential_spec') check_swarm_only_key(service_dicts, 'configs') @@ -706,6 +706,7 @@ def validate_service(service_config, service_names, config_file): validate_depends_on(service_config, service_names) validate_links(service_config, service_names) validate_healthcheck(service_config) + validate_credential_spec(service_config) if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( @@ -894,6 +895,7 @@ def finalize_service(service_config, service_names, version, environment, compat normalize_build(service_dict, service_config.working_dir, environment) if compatibility: + service_dict = translate_credential_spec_to_security_opt(service_dict) service_dict, ignored_keys = translate_deploy_keys_to_container_config( service_dict ) @@ -930,6 +932,25 @@ def convert_restart_policy(name): raise ConfigurationError('Invalid restart policy "{}"'.format(name)) +def convert_credential_spec_to_security_opt(credential_spec): + if 'file' in credential_spec: + return 'file://{file}'.format(file=credential_spec['file']) + return 'registry://{registry}'.format(registry=credential_spec['registry']) + + +def translate_credential_spec_to_security_opt(service_dict): + result = [] + + if 'credential_spec' in service_dict: + spec = convert_credential_spec_to_security_opt(service_dict['credential_spec']) + result.append('credentialspec={spec}'.format(spec=spec)) + + if result: + service_dict['security_opt'] = result + + return service_dict + + def translate_deploy_keys_to_container_config(service_dict): if 'credential_spec' in service_dict: del service_dict['credential_spec'] diff --git a/compose/config/validation.py b/compose/config/validation.py index 039569551..1cceb71f0 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -240,6 +240,18 @@ def validate_depends_on(service_config, service_names): ) +def validate_credential_spec(service_config): + credential_spec = service_config.config.get('credential_spec') + if not credential_spec: + return + + if 'registry' not in credential_spec and 'file' not in credential_spec: + raise ConfigurationError( + "Service '{s.name}' is missing 'credential_spec.file' or " + "credential_spec.registry'".format(s=service_config) + ) + + def get_unsupported_config_msg(path, error_key): msg = "Unsupported config option for {}: '{}'".format(path_string(path), error_key) if error_key in DOCKER_CONFIG_HINTS: diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index e55c91964..5fd418a69 100755 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -339,7 +339,7 @@ _docker-compose() { '(- :)'{-h,--help}'[Get help]' \ '*'{-f,--file}"[${file_description}]:file:_files -g '*.yml'" \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ - "--compatibility[If set, Compose will attempt to convert deploy keys in v3 files to their non-Swarm equivalent]" \ + "--compatibility[If set, Compose will attempt to convert keys in v3 files to their non-Swarm equivalent]" \ '(- :)'{-v,--version}'[Print version and exit]' \ '--verbose[Show more output]' \ '--log-level=[Set log level]:level:(DEBUG INFO WARNING ERROR CRITICAL)' \ diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8baf8e4ee..c2e6b7b07 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3593,6 +3593,9 @@ class InterpolationTest(unittest.TestCase): 'reservations': {'memory': '100M'}, }, }, + 'credential_spec': { + 'file': 'spec.json' + }, }, }, }) @@ -3610,7 +3613,8 @@ class InterpolationTest(unittest.TestCase): 'mem_limit': '300M', 'mem_reservation': '100M', 'cpus': 0.7, - 'name': 'foo' + 'name': 'foo', + 'security_opt': ['credentialspec=file://spec.json'], } @mock.patch.dict(os.environ) From 698ea33b1568b81dd7fe5234d26f616e66b6fa27 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 21 Jan 2019 19:13:45 +0100 Subject: [PATCH 06/62] Add `--parallel` to `docker build`'s options in `bash` and `zsh` completion Signed-off-by: Ulysses Souza --- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 5938ff9e2..2add0c9cd 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -114,7 +114,7 @@ _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory --no-cache --pull --parallel" -- "$cur" ) ) ;; *) __docker_compose_complete_services --filter source=build diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 5fd418a69..d25256c14 100755 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -117,6 +117,7 @@ __docker-compose_subcommand() { '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ '--compress[Compress the build context using gzip.]' \ + '--parallel[Build images in parallel.]' \ '*:services:__docker-compose_services_from_build' && ret=0 ;; (bundle) From c27132afad5cb39a84bf3cb349ba993ea8c738a6 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Thu, 15 Nov 2018 15:19:08 +0300 Subject: [PATCH 07/62] remove stopped containers on --remove-orphans Signed-off-by: Collins Abitekaniza kill orphan containers, catch APIError Exception Signed-off-by: Collins Abitekaniza test remove orphans with --no-start Signed-off-by: Collins Abitekaniza --- compose/project.py | 7 +++++-- tests/acceptance/cli_test.py | 17 +++++++++++++++++ tests/fixtures/v2-simple/one-container.yml | 5 +++++ 3 files changed, 27 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/v2-simple/one-container.yml diff --git a/compose/project.py b/compose/project.py index 92c352050..78b5a3569 100644 --- a/compose/project.py +++ b/compose/project.py @@ -627,7 +627,7 @@ class Project(object): def find_orphan_containers(self, remove_orphans): def _find(): - containers = self._labeled_containers() + containers = set(self._labeled_containers() + self._labeled_containers(stopped=True)) for ctnr in containers: service_name = ctnr.labels.get(LABEL_SERVICE) if service_name not in self.service_names: @@ -638,7 +638,10 @@ class Project(object): if remove_orphans: for ctnr in orphans: log.info('Removing orphan container "{0}"'.format(ctnr.name)) - ctnr.kill() + try: + ctnr.kill() + except APIError: + pass ctnr.remove(force=True) else: log.warning( diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5b0a0e0fd..1844b757a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -12,6 +12,7 @@ import subprocess import time from collections import Counter from collections import namedtuple +from functools import reduce from operator import attrgetter import pytest @@ -1099,6 +1100,22 @@ class CLITestCase(DockerClientTestCase): ] assert len(remote_volumes) > 0 + @v2_only() + def test_up_no_start_remove_orphans(self): + self.base_dir = 'tests/fixtures/v2-simple' + self.dispatch(['up', '--no-start'], None) + + services = self.project.get_services() + + stopped = reduce((lambda prev, next: prev.containers( + stopped=True) + next.containers(stopped=True)), services) + assert len(stopped) == 2 + + self.dispatch(['-f', 'one-container.yml', 'up', '--no-start', '--remove-orphans'], None) + stopped2 = reduce((lambda prev, next: prev.containers( + stopped=True) + next.containers(stopped=True)), services) + assert len(stopped2) == 1 + @v2_only() def test_up_no_ansi(self): self.base_dir = 'tests/fixtures/v2-simple' diff --git a/tests/fixtures/v2-simple/one-container.yml b/tests/fixtures/v2-simple/one-container.yml new file mode 100644 index 000000000..22cd9863c --- /dev/null +++ b/tests/fixtures/v2-simple/one-container.yml @@ -0,0 +1,5 @@ +version: "2" +services: + simple: + image: busybox:latest + command: top From 8ad4c08109f4ec45985b10fa9cf0482d9361c886 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 5 Feb 2019 10:40:03 +0100 Subject: [PATCH 08/62] macOS: Bump Python and OpenSSL Signed-off-by: Christopher Crone --- script/setup/osx | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/script/setup/osx b/script/setup/osx index 08491b6e5..1b546816d 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -13,13 +13,13 @@ if ! [ ${DEPLOYMENT_TARGET} == "$(macos_version)" ]; then SDK_SHA1=dd228a335194e3392f1904ce49aff1b1da26ca62 fi -OPENSSL_VERSION=1.1.0h +OPENSSL_VERSION=1.1.0j OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -OPENSSL_SHA1=0fc39f6aa91b6e7f4d05018f7c5e991e1d2491fd +OPENSSL_SHA1=dcad1efbacd9a4ed67d4514470af12bbe2a1d60a -PYTHON_VERSION=3.6.6 +PYTHON_VERSION=3.6.8 PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -PYTHON_SHA1=ae1fc9ddd29ad8c1d5f7b0d799ff0787efeb9652 +PYTHON_SHA1=09fcc4edaef0915b4dedbfb462f1cd15f82d3a6f # # Install prerequisites. From b572b3299995c54189c0ec66ed39b6fc288cb98f Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 5 Feb 2019 10:50:25 +0100 Subject: [PATCH 09/62] requirements-dev: Fix version of mock to 2.0.0 Signed-off-by: Christopher Crone --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index d49ce49c4..b19fc2e01 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,6 +1,6 @@ coverage==4.4.2 ddt==1.2.0 flake8==3.5.0 -mock>=1.0.1 +mock==2.0.0 pytest==3.6.3 pytest-cov==2.5.1 From f1f0894c1bee49585166d0dc6045ca9742eeb83b Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 5 Feb 2019 10:50:55 +0100 Subject: [PATCH 10/62] script.build.linux: Do not tail image build logs Signed-off-by: Christopher Crone --- script/build/linux | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build/linux b/script/build/linux index 1a4cd4d9b..056940ad0 100755 --- a/script/build/linux +++ b/script/build/linux @@ -5,7 +5,7 @@ set -ex ./script/clean TAG="docker-compose" -docker build -t "$TAG" . | tail -n 200 +docker build -t "$TAG" . docker run \ --rm --entrypoint="script/build/linux-entrypoint" \ -v $(pwd)/dist:/code/dist \ From f472fd545be79860a3c9e03b6e63fbacb86977d1 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 5 Feb 2019 10:51:26 +0100 Subject: [PATCH 11/62] Dockerfile: Force version of virtualenv to 16.2.0 Signed-off-by: Christopher Crone --- Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/Dockerfile b/Dockerfile index a14be492e..c5e7c815a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -17,6 +17,8 @@ ENV LANG en_US.UTF-8 RUN useradd -d /home/user -m -s /bin/bash user WORKDIR /code/ +# FIXME(chris-crone): virtualenv 16.3.0 breaks build, force 16.2.0 until fixed +RUN pip install virtualenv==16.2.0 RUN pip install tox==2.1.1 ADD requirements.txt /code/ @@ -25,6 +27,7 @@ ADD .pre-commit-config.yaml /code/ ADD setup.py /code/ ADD tox.ini /code/ ADD compose /code/compose/ +ADD README.md /code/ RUN tox --notest ADD . /code/ From c8a621b637a7574b621e919b849d56fa66c3d996 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 30 Jan 2019 14:28:00 +0100 Subject: [PATCH 12/62] Fix Flake8 lint This removes extra indentation and replace the use of `is` by `==` when comparing strings Signed-off-by: Ulysses Souza --- compose/service.py | 80 +++++++++++++++--------------- tests/unit/cli/log_printer_test.py | 2 +- 2 files changed, 41 insertions(+), 41 deletions(-) diff --git a/compose/service.py b/compose/service.py index 59f05f8dc..401efa7eb 100644 --- a/compose/service.py +++ b/compose/service.py @@ -291,7 +291,7 @@ class Service(object): c for c in stopped_containers if self._containers_have_diverged([c]) ] for c in divergent_containers: - c.remove() + c.remove() all_containers = list(set(all_containers) - set(divergent_containers)) @@ -467,50 +467,50 @@ class Service(object): def _execute_convergence_recreate(self, containers, scale, timeout, detached, start, renew_anonymous_volumes): - if scale is not None and len(containers) > scale: - self._downscale(containers[scale:], timeout) - containers = containers[:scale] + if scale is not None and len(containers) > scale: + self._downscale(containers[scale:], timeout) + containers = containers[:scale] - def recreate(container): - return self.recreate_container( - container, timeout=timeout, attach_logs=not detached, - start_new_container=start, renew_anonymous_volumes=renew_anonymous_volumes - ) - containers, errors = parallel_execute( - containers, - recreate, - lambda c: c.name, - "Recreating", + def recreate(container): + return self.recreate_container( + container, timeout=timeout, attach_logs=not detached, + start_new_container=start, renew_anonymous_volumes=renew_anonymous_volumes ) + containers, errors = parallel_execute( + containers, + recreate, + lambda c: c.name, + "Recreating", + ) + for error in errors.values(): + raise OperationFailedError(error) + + if scale is not None and len(containers) < scale: + containers.extend(self._execute_convergence_create( + scale - len(containers), detached, start + )) + return containers + + def _execute_convergence_start(self, containers, scale, timeout, detached, start): + if scale is not None and len(containers) > scale: + self._downscale(containers[scale:], timeout) + containers = containers[:scale] + if start: + _, errors = parallel_execute( + containers, + lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True), + lambda c: c.name, + "Starting", + ) + for error in errors.values(): raise OperationFailedError(error) - if scale is not None and len(containers) < scale: - containers.extend(self._execute_convergence_create( - scale - len(containers), detached, start - )) - return containers - - def _execute_convergence_start(self, containers, scale, timeout, detached, start): - if scale is not None and len(containers) > scale: - self._downscale(containers[scale:], timeout) - containers = containers[:scale] - if start: - _, errors = parallel_execute( - containers, - lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True), - lambda c: c.name, - "Starting", - ) - - for error in errors.values(): - raise OperationFailedError(error) - - if scale is not None and len(containers) < scale: - containers.extend(self._execute_convergence_create( - scale - len(containers), detached, start - )) - return containers + if scale is not None and len(containers) < scale: + containers.extend(self._execute_convergence_create( + scale - len(containers), detached, start + )) + return containers def _downscale(self, containers, timeout=None): def stop_and_remove(container): diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index d0c4b56be..6db24e464 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -193,7 +193,7 @@ class TestConsumeQueue(object): queue.put(item) generator = consume_queue(queue, True) - assert next(generator) is 'foobar-1' + assert next(generator) == 'foobar-1' def test_item_is_none_when_timeout_is_hit(self): queue = Queue() From d9ffec400224256cd06dfc6dd71ceb909285aa2d Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 5 Feb 2019 12:13:19 +0100 Subject: [PATCH 13/62] circleci: Fix virtualenv version to 16.2.0 Signed-off-by: Christopher Crone --- .circleci/config.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index f4e90d6de..08f8c42c3 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -10,7 +10,7 @@ jobs: command: ./script/setup/osx - run: name: install tox - command: sudo pip install --upgrade tox==2.1.1 + command: sudo pip install --upgrade tox==2.1.1 virtualenv==16.2.0 - run: name: unit tests command: tox -e py27,py36,py37 -- tests/unit @@ -22,7 +22,7 @@ jobs: - checkout - run: name: upgrade python tools - command: sudo pip install --upgrade pip virtualenv + command: sudo pip install --upgrade pip virtualenv==16.2.0 - run: name: setup script command: DEPLOYMENT_TARGET=10.11 ./script/setup/osx From 436a343a185ef10efc88dcf2eebda102e260324b Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Mon, 11 Feb 2019 13:50:41 +0100 Subject: [PATCH 14/62] Fix bash completion for `build --memory` - the option requires an argument - adds missing short form `-m` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2add0c9cd..b0ff484fc 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -110,11 +110,14 @@ _docker_compose_build() { __docker_compose_nospace return ;; + --memory|-m) + return + ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory --no-cache --pull --parallel" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory -m --no-cache --pull --parallel" -- "$cur" ) ) ;; *) __docker_compose_complete_services --filter source=build From fbbf78d3da278f2844c769461bfdbec823532608 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Tue, 5 Feb 2019 12:47:51 +0100 Subject: [PATCH 15/62] macOS: Bump OpenSSL to 1.1.1a Signed-off-by: Christopher Crone --- script/setup/osx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/setup/osx b/script/setup/osx index 1b546816d..b0c34e5bf 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -13,9 +13,9 @@ if ! [ ${DEPLOYMENT_TARGET} == "$(macos_version)" ]; then SDK_SHA1=dd228a335194e3392f1904ce49aff1b1da26ca62 fi -OPENSSL_VERSION=1.1.0j +OPENSSL_VERSION=1.1.1a OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz -OPENSSL_SHA1=dcad1efbacd9a4ed67d4514470af12bbe2a1d60a +OPENSSL_SHA1=8fae27b4f34445a5500c9dc50ae66b4d6472ce29 PYTHON_VERSION=3.6.8 PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz From a35aef4953954759e8970fd5163d96d998048939 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 20 Feb 2019 10:36:59 +0100 Subject: [PATCH 16/62] Add --no-rm to command build - When present, build does not remove intermediate containers after a successful build. Signed-off-by: Ulysses Souza --- compose/cli/main.py | 2 ++ compose/project.py | 4 ++-- compose/service.py | 4 ++-- contrib/completion/bash/docker-compose | 2 +- tests/acceptance/cli_test.py | 17 +++++++++++++++++ 5 files changed, 24 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 789601792..08976347e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -260,6 +260,7 @@ class TopLevelCommand(object): --compress Compress the build context using gzip. --force-rm Always remove intermediate containers. --no-cache Do not use cache when building the image. + --no-rm Do not remove intermediate containers after a successful build. --pull Always attempt to pull a newer version of the image. -m, --memory MEM Sets memory limit for the build container. --build-arg key=val Set build-time variables for services. @@ -282,6 +283,7 @@ class TopLevelCommand(object): pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False)), memory=options.get('--memory'), + rm=not bool(options.get('--no-rm', False)), build_args=build_args, gzip=options.get('--compress', False), parallel_build=options.get('--parallel', False), diff --git a/compose/project.py b/compose/project.py index a7f2aa057..a2307fa05 100644 --- a/compose/project.py +++ b/compose/project.py @@ -355,7 +355,7 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False, parallel_build=False): + build_args=None, gzip=False, parallel_build=False, rm=True): services = [] for service in self.get_services(service_names): @@ -365,7 +365,7 @@ class Project(object): log.info('%s uses an image, skipping' % service.name) def build_service(service): - service.build(no_cache, pull, force_rm, memory, build_args, gzip) + service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm) if parallel_build: _, errors = parallel.parallel_execute( diff --git a/compose/service.py b/compose/service.py index 401efa7eb..6483f4f31 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1049,7 +1049,7 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, - gzip=False): + gzip=False, rm=True): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) @@ -1070,7 +1070,7 @@ class Service(object): build_output = self.client.build( path=path, tag=self.image_name, - rm=True, + rm=rm, forcerm=force_rm, pull=pull, nocache=no_cache, diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index b0ff484fc..e330e576b 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -117,7 +117,7 @@ _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory -m --no-cache --pull --parallel" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory -m --no-cache --no-rm --pull --parallel" -- "$cur" ) ) ;; *) __docker_compose_complete_services --filter source=build diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5142f96eb..8a6415b40 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -747,6 +747,23 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_build_rm(self): + containers = [ + Container.from_ps(self.project.client, c) + for c in self.project.client.containers(all=True) + ] + + assert not containers + + self.base_dir = 'tests/fixtures/simple-dockerfile' + self.dispatch(['build', '--no-rm', 'simple'], returncode=0) + + containers = [ + Container.from_ps(self.project.client, c) + for c in self.project.client.containers(all=True) + ] + assert containers + def test_build_shm_size_build_option(self): pull_busybox(self.client) self.base_dir = 'tests/fixtures/build-shm-size' From a734371e7fc27f07bbd5f3b16d7f19d49053716d Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 14 Feb 2019 12:08:00 +0100 Subject: [PATCH 17/62] Bump python version from 3.6.8 to 3.7.2 Signed-off-by: Ulysses Souza --- .circleci/config.yml | 2 +- Dockerfile | 4 ++-- Dockerfile.armhf | 4 ++-- Jenkinsfile | 3 +-- appveyor.yml | 4 ++-- requirements-build.txt | 2 +- script/build/linux-entrypoint | 2 +- script/build/windows.ps1 | 4 ++-- script/setup/osx | 4 ++-- script/test/all | 2 +- tox.ini | 2 +- 11 files changed, 16 insertions(+), 17 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 08f8c42c3..906b1c0dc 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,7 @@ jobs: command: sudo pip install --upgrade tox==2.1.1 virtualenv==16.2.0 - run: name: unit tests - command: tox -e py27,py36,py37 -- tests/unit + command: tox -e py27,py37 -- tests/unit build-osx-binary: macos: diff --git a/Dockerfile b/Dockerfile index c5e7c815a..d3edfcbeb 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ FROM docker:18.06.1 as docker -FROM python:3.6 +FROM python:3.7.2-stretch RUN set -ex; \ apt-get update -qq; \ @@ -33,4 +33,4 @@ RUN tox --notest ADD . /code/ RUN chown -R user /code/ -ENTRYPOINT ["/code/.tox/py36/bin/docker-compose"] +ENTRYPOINT ["/code/.tox/py37/bin/docker-compose"] diff --git a/Dockerfile.armhf b/Dockerfile.armhf index ee2ce8941..9c02dbc76 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -1,4 +1,4 @@ -FROM python:3.6 +FROM python:3.7.2-stretch RUN set -ex; \ apt-get update -qq; \ @@ -36,4 +36,4 @@ RUN tox --notest ADD . /code/ RUN chown -R user /code/ -ENTRYPOINT ["/code/.tox/py36/bin/docker-compose"] +ENTRYPOINT ["/code/.tox/py37/bin/docker-compose"] diff --git a/Jenkinsfile b/Jenkinsfile index 04f5cfbda..a19e82273 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -37,7 +37,7 @@ def runTests = { Map settings -> def pythonVersions = settings.get("pythonVersions", null) if (!pythonVersions) { - throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py36')`") + throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py37')`") } if (!dockerVersions) { throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`") @@ -77,7 +77,6 @@ def docker_versions = get_versions(2) for (int i = 0; i < docker_versions.length; i++) { def dockerVersion = docker_versions[i] testMatrix["${dockerVersion}_py27"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py27"]) - testMatrix["${dockerVersion}_py36"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py36"]) testMatrix["${dockerVersion}_py37"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py37"]) } diff --git a/appveyor.yml b/appveyor.yml index da80d01d9..98a128a8a 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -2,7 +2,7 @@ version: '{branch}-{build}' install: - - "SET PATH=C:\\Python36-x64;C:\\Python36-x64\\Scripts;%PATH%" + - "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%" - "python --version" - "pip install tox==2.9.1 virtualenv==15.1.0" @@ -10,7 +10,7 @@ install: build: false test_script: - - "tox -e py27,py36,py37 -- tests/unit" + - "tox -e py27,py37 -- tests/unit" - ps: ".\\script\\build\\windows.ps1" artifacts: diff --git a/requirements-build.txt b/requirements-build.txt index e5a77e794..9161fadf9 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.3.1 +pyinstaller==3.4 diff --git a/script/build/linux-entrypoint b/script/build/linux-entrypoint index 0e3c7ec1e..34c16ac69 100755 --- a/script/build/linux-entrypoint +++ b/script/build/linux-entrypoint @@ -3,7 +3,7 @@ set -ex TARGET=dist/docker-compose-$(uname -s)-$(uname -m) -VENV=/code/.tox/py36 +VENV=/code/.tox/py37 mkdir -p `pwd`/dist chmod 777 `pwd`/dist diff --git a/script/build/windows.ps1 b/script/build/windows.ps1 index 41dc51e31..14c75762b 100644 --- a/script/build/windows.ps1 +++ b/script/build/windows.ps1 @@ -6,11 +6,11 @@ # # http://git-scm.com/download/win # -# 2. Install Python 3.6.4: +# 2. Install Python 3.7.2: # # https://www.python.org/downloads/ # -# 3. Append ";C:\Python36;C:\Python36\Scripts" to the "Path" environment variable: +# 3. Append ";C:\Python37;C:\Python37\Scripts" to the "Path" environment variable: # # https://www.microsoft.com/resources/documentation/windows/xp/all/proddocs/en-us/sysdm_advancd_environmnt_addchange_variable.mspx?mfr=true # diff --git a/script/setup/osx b/script/setup/osx index b0c34e5bf..ff460d392 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -17,9 +17,9 @@ OPENSSL_VERSION=1.1.1a OPENSSL_URL=https://www.openssl.org/source/openssl-${OPENSSL_VERSION}.tar.gz OPENSSL_SHA1=8fae27b4f34445a5500c9dc50ae66b4d6472ce29 -PYTHON_VERSION=3.6.8 +PYTHON_VERSION=3.7.2 PYTHON_URL=https://www.python.org/ftp/python/${PYTHON_VERSION}/Python-${PYTHON_VERSION}.tgz -PYTHON_SHA1=09fcc4edaef0915b4dedbfb462f1cd15f82d3a6f +PYTHON_SHA1=0cd8e52d8ed1d0be12ac8e87a623a15df3a3b418 # # Install prerequisites. diff --git a/script/test/all b/script/test/all index e48f73bba..5c911bba4 100755 --- a/script/test/all +++ b/script/test/all @@ -24,7 +24,7 @@ fi BUILD_NUMBER=${BUILD_NUMBER-$USER} -PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py36} +PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py37} for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" diff --git a/tox.ini b/tox.ini index 08efd4e68..57e57bc63 100644 --- a/tox.ini +++ b/tox.ini @@ -1,5 +1,5 @@ [tox] -envlist = py27,py36,py37,pre-commit +envlist = py27,py37,pre-commit [testenv] usedevelop=True From bb0bd3b26b91a8fdbcaf242265a29b9fffd9f904 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 14 Feb 2019 12:20:15 +0100 Subject: [PATCH 18/62] Harmonize tox and virtualenv versions - Set all tox versions to 2.9.1 - Set all virtualenv version to 16.2.0 Signed-off-by: Ulysses Souza --- Dockerfile | 2 +- appveyor.yml | 2 +- script/build/windows.ps1 | 2 +- script/setup/osx | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index d3edfcbeb..b887bcc4e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -19,7 +19,7 @@ WORKDIR /code/ # FIXME(chris-crone): virtualenv 16.3.0 breaks build, force 16.2.0 until fixed RUN pip install virtualenv==16.2.0 -RUN pip install tox==2.1.1 +RUN pip install tox==2.9.1 ADD requirements.txt /code/ ADD requirements-dev.txt /code/ diff --git a/appveyor.yml b/appveyor.yml index 98a128a8a..04a40e9c2 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -4,7 +4,7 @@ version: '{branch}-{build}' install: - "SET PATH=C:\\Python37-x64;C:\\Python37-x64\\Scripts;%PATH%" - "python --version" - - "pip install tox==2.9.1 virtualenv==15.1.0" + - "pip install tox==2.9.1 virtualenv==16.2.0" # Build the binary after tests build: false diff --git a/script/build/windows.ps1 b/script/build/windows.ps1 index 14c75762b..4c7a8bed5 100644 --- a/script/build/windows.ps1 +++ b/script/build/windows.ps1 @@ -16,7 +16,7 @@ # # 4. In Powershell, run the following commands: # -# $ pip install 'virtualenv>=15.1.0' +# $ pip install 'virtualenv==16.2.0' # $ Set-ExecutionPolicy -Scope CurrentUser RemoteSigned # # 5. Clone the repository: diff --git a/script/setup/osx b/script/setup/osx index ff460d392..d8613e864 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -36,7 +36,7 @@ if ! [ -x "$(command -v python3)" ]; then brew install python3 fi if ! [ -x "$(command -v virtualenv)" ]; then - pip install virtualenv + pip install virtualenv==16.2.0 fi # From dbc229dc3790976055d17a9f14a6364f7c949002 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 21 Feb 2019 13:57:19 +0100 Subject: [PATCH 19/62] Fix macOS build for Python 3.7 - Specify --with-openssl directory for Python build - Better checks for downloaded SDK, OpenSSL, and Python - Fix missing slash for Python build CPPFLAGS Signed-off-by: Christopher Crone --- script/setup/osx | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/script/setup/osx b/script/setup/osx index d8613e864..2a1277822 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -50,7 +50,7 @@ mkdir -p ${TOOLCHAIN_PATH} # # Set macOS SDK. # -if [ ${SDK_FETCH} ]; then +if [[ ${SDK_FETCH} && ! -f ${TOOLCHAIN_PATH}/MacOSX${DEPLOYMENT_TARGET}.sdk/SDKSettings.plist ]]; then SDK_PATH=${TOOLCHAIN_PATH}/MacOSX${DEPLOYMENT_TARGET}.sdk fetch_tarball ${SDK_URL} ${SDK_PATH} ${SDK_SHA1} else @@ -61,7 +61,7 @@ fi # Build OpenSSL. # OPENSSL_SRC_PATH=${TOOLCHAIN_PATH}/openssl-${OPENSSL_VERSION} -if ! [ -f ${TOOLCHAIN_PATH}/bin/openssl ]; then +if ! [[ $(${TOOLCHAIN_PATH}/bin/openssl version) == *"${OPENSSL_VERSION}"* ]]; then rm -rf ${OPENSSL_SRC_PATH} fetch_tarball ${OPENSSL_URL} ${OPENSSL_SRC_PATH} ${OPENSSL_SHA1} ( @@ -77,7 +77,7 @@ fi # Build Python. # PYTHON_SRC_PATH=${TOOLCHAIN_PATH}/Python-${PYTHON_VERSION} -if ! [ -f ${TOOLCHAIN_PATH}/bin/python3 ]; then +if ! [[ $(${TOOLCHAIN_PATH}/bin/python3 --version) == *"${PYTHON_VERSION}"* ]]; then rm -rf ${PYTHON_SRC_PATH} fetch_tarball ${PYTHON_URL} ${PYTHON_SRC_PATH} ${PYTHON_SHA1} ( @@ -87,9 +87,10 @@ if ! [ -f ${TOOLCHAIN_PATH}/bin/python3 ]; then --datarootdir=${TOOLCHAIN_PATH}/share \ --datadir=${TOOLCHAIN_PATH}/share \ --enable-framework=${TOOLCHAIN_PATH}/Frameworks \ + --with-openssl=${TOOLCHAIN_PATH} \ MACOSX_DEPLOYMENT_TARGET=${DEPLOYMENT_TARGET} \ CFLAGS="-isysroot ${SDK_PATH} -I${TOOLCHAIN_PATH}/include" \ - CPPFLAGS="-I${SDK_PATH}/usr/include -I${TOOLCHAIN_PATH}include" \ + CPPFLAGS="-I${SDK_PATH}/usr/include -I${TOOLCHAIN_PATH}/include" \ LDFLAGS="-isysroot ${SDK_PATH} -L ${TOOLCHAIN_PATH}/lib" make -j 4 make install PYTHONAPPSDIR=${TOOLCHAIN_PATH} From 133df6310851888067c42e1b938cf51de46ebc03 Mon Sep 17 00:00:00 2001 From: Christopher Crone Date: Thu, 21 Feb 2019 16:21:03 +0100 Subject: [PATCH 20/62] Add built Python smoke test to macOS setup script Prior to this smoke test, the macOS setup step wouldn't fail if the Python that it built wasn't functional. This will make debugging Python build issues easier in the future. Signed-off-by: Christopher Crone --- script/setup/osx | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/script/setup/osx b/script/setup/osx index 2a1277822..1fb91edca 100755 --- a/script/setup/osx +++ b/script/setup/osx @@ -98,6 +98,11 @@ if ! [[ $(${TOOLCHAIN_PATH}/bin/python3 --version) == *"${PYTHON_VERSION}"* ]]; ) fi +# +# Smoke test built Python. +# +openssl_version ${TOOLCHAIN_PATH} + echo "" echo "*** Targeting macOS: ${DEPLOYMENT_TARGET}" echo "*** Using SDK ${SDK_PATH}" From 572032fc0b2c35f12787a170a891da6c59b4bc90 Mon Sep 17 00:00:00 2001 From: tuttieee Date: Wed, 6 Feb 2019 20:39:19 +0900 Subject: [PATCH 21/62] Fix Project#build_container_operation_with_timeout_func not to mutate a 'option' dict over multiple containers Signed-off-by: Yuichiro Tsuchiya --- compose/project.py | 7 ++++--- tests/unit/project_test.py | 30 ++++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 3 deletions(-) diff --git a/compose/project.py b/compose/project.py index a2307fa05..bdeff19e6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -725,10 +725,11 @@ class Project(object): def build_container_operation_with_timeout_func(self, operation, options): def container_operation_with_timeout(container): - if options.get('timeout') is None: + _options = options.copy() + if _options.get('timeout') is None: service = self.get_service(container.service) - options['timeout'] = service.stop_timeout(None) - return getattr(container, operation)(**options) + _options['timeout'] = service.stop_timeout(None) + return getattr(container, operation)(**_options) return container_operation_with_timeout diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 4aea91a0d..89b080d20 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -15,6 +15,8 @@ from compose.config.types import VolumeFromSpec from compose.const import COMPOSEFILE_V1 as V1 from compose.const import COMPOSEFILE_V2_0 as V2_0 from compose.const import COMPOSEFILE_V2_4 as V2_4 +from compose.const import COMPOSEFILE_V3_7 as V3_7 +from compose.const import DEFAULT_TIMEOUT from compose.const import LABEL_SERVICE from compose.container import Container from compose.errors import OperationFailedError @@ -765,6 +767,34 @@ class ProjectTest(unittest.TestCase): ) assert project.get_service('web').platform == 'linux/s390x' + def test_build_container_operation_with_timeout_func_does_not_mutate_options_with_timeout(self): + config_data = Config( + version=V3_7, + services=[ + {'name': 'web', 'image': 'busybox:latest'}, + {'name': 'db', 'image': 'busybox:latest', 'stop_grace_period': '1s'}, + ], + networks={}, volumes={}, secrets=None, configs=None, + ) + + project = Project.from_config(name='test', client=self.mock_client, config_data=config_data) + + stop_op = project.build_container_operation_with_timeout_func('stop', options={}) + + web_container = mock.create_autospec(Container, service='web') + db_container = mock.create_autospec(Container, service='db') + + # `stop_grace_period` is not set to 'web' service, + # then it is stopped with the default timeout. + stop_op(web_container) + web_container.stop.assert_called_once_with(timeout=DEFAULT_TIMEOUT) + + # `stop_grace_period` is set to 'db' service, + # then it is stopped with the specified timeout and + # the value is not overridden by the previous function call. + stop_op(db_container) + db_container.stop.assert_called_once_with(timeout=1) + @mock.patch('compose.parallel.ParallelStreamWriter._write_noansi') def test_error_parallel_pull(self, mock_write): project = Project.from_config( From b09d8802edb498a91a73bcd63603774c9b027b47 Mon Sep 17 00:00:00 2001 From: slowr Date: Fri, 22 Feb 2019 18:40:28 -0800 Subject: [PATCH 22/62] Added additional argument (--env-file) for docker-compose to import environment variables from a given PATH. Signed-off-by: Dimitrios Mavrommatis --- compose/cli/command.py | 6 ++++-- compose/cli/main.py | 19 ++++++++++++++----- compose/config/environment.py | 7 +++++-- tests/acceptance/cli_test.py | 15 +++++++++++++++ tests/fixtures/default-env-file/.env2 | 4 ++++ tests/unit/config/config_test.py | 19 +++++++++++++++++++ 6 files changed, 61 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/default-env-file/.env2 diff --git a/compose/cli/command.py b/compose/cli/command.py index c1f6c2925..f9e698e17 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -39,7 +39,8 @@ SILENT_COMMANDS = set(( def project_from_options(project_dir, options): override_dir = options.get('--project-directory') - environment = Environment.from_env_file(override_dir or project_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(override_dir or project_dir, environment_file) environment.silent = options.get('COMMAND', None) in SILENT_COMMANDS set_parallel_limit(environment) @@ -77,7 +78,8 @@ def set_parallel_limit(environment): def get_config_from_options(base_dir, options): override_dir = options.get('--project-directory') - environment = Environment.from_env_file(override_dir or base_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(override_dir or base_dir, environment_file) config_path = get_config_path_from_options( base_dir, options, environment ) diff --git a/compose/cli/main.py b/compose/cli/main.py index 08976347e..d8793c85b 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -208,6 +208,7 @@ class TopLevelCommand(object): (default: the path of the Compose file) --compatibility If set, Compose will attempt to convert keys in v3 files to their non-Swarm equivalent + --env-file PATH Specify an alternate environment file Commands: build Build or rebuild services @@ -274,7 +275,8 @@ class TopLevelCommand(object): '--build-arg is only supported when services are specified for API version < 1.25.' ' Please use a Compose file version > 2.2 or specify which services to build.' ) - environment = Environment.from_env_file(self.project_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(self.project_dir, environment_file) build_args = resolve_build_args(build_args, environment) self.project.build( @@ -423,8 +425,10 @@ class TopLevelCommand(object): Compose file -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) + --env-file PATH Specify an alternate environment file """ - environment = Environment.from_env_file(self.project_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(self.project_dir, environment_file) ignore_orphans = environment.get_boolean('COMPOSE_IGNORE_ORPHANS') if ignore_orphans and options['--remove-orphans']: @@ -481,8 +485,10 @@ class TopLevelCommand(object): -e, --env KEY=VAL Set environment variables (can be used multiple times, not supported in API < 1.25) -w, --workdir DIR Path to workdir directory for this command. + --env-file PATH Specify an alternate environment file """ - environment = Environment.from_env_file(self.project_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(self.project_dir, environment_file) use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') index = int(options.get('--index')) service = self.project.get_service(options['SERVICE']) @@ -1038,6 +1044,7 @@ class TopLevelCommand(object): container. Implies --abort-on-container-exit. --scale SERVICE=NUM Scale SERVICE to NUM instances. Overrides the `scale` setting in the Compose file if present. + --env-file PATH Specify an alternate environment file """ start_deps = not options['--no-deps'] always_recreate_deps = options['--always-recreate-deps'] @@ -1052,7 +1059,8 @@ class TopLevelCommand(object): if detached and (cascade_stop or exit_value_from): raise UserError("--abort-on-container-exit and -d cannot be combined.") - environment = Environment.from_env_file(self.project_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(self.project_dir, environment_file) ignore_orphans = environment.get_boolean('COMPOSE_IGNORE_ORPHANS') if ignore_orphans and remove_orphans: @@ -1345,7 +1353,8 @@ def run_one_off_container(container_options, project, service, options, toplevel if options['--rm']: project.client.remove_container(container.id, force=True, v=True) - environment = Environment.from_env_file(project_dir) + environment_file = options.get('--env-file') + environment = Environment.from_env_file(project_dir, environment_file) use_cli = not environment.get_boolean('COMPOSE_INTERACTIVE_NO_CLI') signals.set_signal_handler_to_shutdown() diff --git a/compose/config/environment.py b/compose/config/environment.py index 62c40a4bc..e2db343d7 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -59,12 +59,15 @@ class Environment(dict): self.silent = False @classmethod - def from_env_file(cls, base_dir): + def from_env_file(cls, base_dir, env_file=None): def _initialize(): result = cls() if base_dir is None: return result - env_file_path = os.path.join(base_dir, '.env') + if env_file: + env_file_path = os.path.join(base_dir, env_file) + else: + env_file_path = os.path.join(base_dir, '.env') try: return cls(env_vars_from_file(env_file_path)) except EnvFileNotFound: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8a6415b40..6a9a392a5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -324,6 +324,21 @@ class CLITestCase(DockerClientTestCase): 'version': '2.4' } + def test_config_with_env_file(self): + self.base_dir = 'tests/fixtures/default-env-file' + result = self.dispatch(['--env-file', '.env2', 'config']) + json_result = yaml.load(result.stdout) + assert json_result == { + 'services': { + 'web': { + 'command': 'false', + 'image': 'alpine:latest', + 'ports': ['5644/tcp', '9998/tcp'] + } + }, + 'version': '2.4' + } + def test_config_with_dot_env_and_override_dir(self): self.base_dir = 'tests/fixtures/default-env-file' result = self.dispatch(['--project-directory', 'alt/', 'config']) diff --git a/tests/fixtures/default-env-file/.env2 b/tests/fixtures/default-env-file/.env2 new file mode 100644 index 000000000..d754523fc --- /dev/null +++ b/tests/fixtures/default-env-file/.env2 @@ -0,0 +1,4 @@ +IMAGE=alpine:latest +COMMAND=false +PORT1=5644 +PORT2=9998 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c2e6b7b07..888c0ae58 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3465,6 +3465,25 @@ class InterpolationTest(unittest.TestCase): 'command': 'true' } + @mock.patch.dict(os.environ) + def test_config_file_with_options_environment_file(self): + project_dir = 'tests/fixtures/default-env-file' + service_dicts = config.load( + config.find( + project_dir, None, Environment.from_env_file(project_dir, '.env2') + ) + ).services + + assert service_dicts[0] == { + 'name': 'web', + 'image': 'alpine:latest', + 'ports': [ + types.ServicePort.parse('5644')[0], + types.ServicePort.parse('9998')[0] + ], + 'command': 'false' + } + @mock.patch.dict(os.environ) def test_config_file_with_environment_variable(self): project_dir = 'tests/fixtures/environment-interpolation' From 1f97a572fe35f2ff2e557b2f6ca371c67f7c2a82 Mon Sep 17 00:00:00 2001 From: Akshit Grover Date: Sat, 2 Mar 2019 13:07:23 +0530 Subject: [PATCH 23/62] Add --quiet build flag Signed-off-by: Akshit Grover --- compose/cli/main.py | 2 ++ compose/project.py | 7 +++---- compose/service.py | 10 ++++++---- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + tests/acceptance/cli_test.py | 7 +++++++ 6 files changed, 20 insertions(+), 9 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index d8793c85b..2349568c9 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -266,6 +266,7 @@ class TopLevelCommand(object): -m, --memory MEM Sets memory limit for the build container. --build-arg key=val Set build-time variables for services. --parallel Build images in parallel. + -q, --quiet Don't print anything to STDOUT """ service_names = options['SERVICE'] build_args = options.get('--build-arg', None) @@ -289,6 +290,7 @@ class TopLevelCommand(object): build_args=build_args, gzip=options.get('--compress', False), parallel_build=options.get('--parallel', False), + silent=options.get('--quiet', False) ) def bundle(self, options): diff --git a/compose/project.py b/compose/project.py index bdeff19e6..a74033729 100644 --- a/compose/project.py +++ b/compose/project.py @@ -355,18 +355,17 @@ class Project(object): return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False, memory=None, - build_args=None, gzip=False, parallel_build=False, rm=True): + build_args=None, gzip=False, parallel_build=False, rm=True, silent=False): services = [] for service in self.get_services(service_names): if service.can_be_built(): services.append(service) - else: + elif not silent: log.info('%s uses an image, skipping' % service.name) def build_service(service): - service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm) - + service.build(no_cache, pull, force_rm, memory, build_args, gzip, rm, silent) if parallel_build: _, errors = parallel.parallel_execute( services, diff --git a/compose/service.py b/compose/service.py index 6483f4f31..b5a6b392d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -59,7 +59,6 @@ from .utils import parse_seconds_float from .utils import truncate_id from .utils import unique_everseen - log = logging.getLogger(__name__) @@ -1049,8 +1048,11 @@ class Service(object): return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False, memory=None, build_args_override=None, - gzip=False, rm=True): - log.info('Building %s' % self.name) + gzip=False, rm=True, silent=False): + output_stream = open(os.devnull, 'w') + if not silent: + output_stream = sys.stdout + log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) @@ -1091,7 +1093,7 @@ class Service(object): ) try: - all_events = list(stream_output(build_output, sys.stdout)) + all_events = list(stream_output(build_output, output_stream)) except StreamOutputError as e: raise BuildError(self, six.text_type(e)) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e330e576b..941f25a3b 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -117,7 +117,7 @@ _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory -m --no-cache --no-rm --pull --parallel" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--build-arg --compress --force-rm --help --memory -m --no-cache --no-rm --pull --parallel -q --quiet" -- "$cur" ) ) ;; *) __docker_compose_complete_services --filter source=build diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index d25256c14..808b068a3 100755 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -113,6 +113,7 @@ __docker-compose_subcommand() { $opts_help \ "*--build-arg=[Set build-time variables for one service.]:=: " \ '--force-rm[Always remove intermediate containers.]' \ + '(--quiet -q)'{--quiet,-q}'[Curb build output]' \ '(--memory -m)'{--memory,-m}'[Memory limit for the build container.]' \ '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6a9a392a5..dbd822115 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -170,6 +170,13 @@ class CLITestCase(DockerClientTestCase): # Prevent tearDown from trying to create a project self.base_dir = None + def test_quiet_build(self): + self.base_dir = 'tests/fixtures/build-args' + result = self.dispatch(['build'], None) + quietResult = self.dispatch(['build', '-q'], None) + assert result.stdout != "" + assert quietResult.stdout == "" + def test_help_nonexistent(self): self.base_dir = 'tests/fixtures/no-composefile' result = self.dispatch(['help', 'foobar'], returncode=1) From e34d3292273447e4c7ecca4cb85609034f7c7468 Mon Sep 17 00:00:00 2001 From: "Peter Nagy (NPE)" Date: Fri, 23 Nov 2018 11:30:49 +0100 Subject: [PATCH 24/62] adds --no-interpolate to docker-compose config Signed-off-by: Peter Nagy --- compose/cli/command.py | 12 ++++---- compose/cli/main.py | 10 ++++--- compose/config/config.py | 49 +++++++++++++++++++++++--------- compose/config/serialize.py | 20 +++++++++---- tests/unit/config/config_test.py | 41 ++++++++++++++++++++++++++ 5 files changed, 103 insertions(+), 29 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index f9e698e17..21ab9a39f 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -37,7 +37,7 @@ SILENT_COMMANDS = set(( )) -def project_from_options(project_dir, options): +def project_from_options(project_dir, options, additional_options={}): override_dir = options.get('--project-directory') environment_file = options.get('--env-file') environment = Environment.from_env_file(override_dir or project_dir, environment_file) @@ -57,6 +57,7 @@ def project_from_options(project_dir, options): environment=environment, override_dir=override_dir, compatibility=options.get('--compatibility'), + interpolate=(not additional_options.get('--no-interpolate')) ) @@ -76,7 +77,7 @@ def set_parallel_limit(environment): parallel.GlobalLimit.set_global_limit(parallel_limit) -def get_config_from_options(base_dir, options): +def get_config_from_options(base_dir, options, additional_options={}): override_dir = options.get('--project-directory') environment_file = options.get('--env-file') environment = Environment.from_env_file(override_dir or base_dir, environment_file) @@ -85,7 +86,8 @@ def get_config_from_options(base_dir, options): ) return config.load( config.find(base_dir, config_path, environment, override_dir), - options.get('--compatibility') + options.get('--compatibility'), + not additional_options.get('--no-interpolate') ) @@ -123,14 +125,14 @@ def get_client(environment, verbose=False, version=None, tls_config=None, host=N def get_project(project_dir, config_path=None, project_name=None, verbose=False, host=None, tls_config=None, environment=None, override_dir=None, - compatibility=False): + compatibility=False, interpolate=True): if not environment: environment = Environment.from_env_file(project_dir) config_details = config.find(project_dir, config_path, environment, override_dir) project_name = get_project_name( config_details.working_dir, project_name, environment ) - config_data = config.load(config_details, compatibility) + config_data = config.load(config_details, compatibility, interpolate) api_version = environment.get( 'COMPOSE_API_VERSION', diff --git a/compose/cli/main.py b/compose/cli/main.py index d8793c85b..1f8c58ca4 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -331,6 +331,7 @@ class TopLevelCommand(object): Options: --resolve-image-digests Pin image tags to digests. + --no-interpolate Don't interpolate environment variables -q, --quiet Only validate the configuration, don't print anything. --services Print the service names, one per line. @@ -340,11 +341,12 @@ class TopLevelCommand(object): or use the wildcard symbol to display all services """ - compose_config = get_config_from_options('.', self.toplevel_options) + additional_options = {'--no-interpolate': options.get('--no-interpolate')} + compose_config = get_config_from_options('.', self.toplevel_options, additional_options) image_digests = None if options['--resolve-image-digests']: - self.project = project_from_options('.', self.toplevel_options) + self.project = project_from_options('.', self.toplevel_options, additional_options) with errors.handle_connection_errors(self.project.client): image_digests = image_digests_for_project(self.project) @@ -361,14 +363,14 @@ class TopLevelCommand(object): if options['--hash'] is not None: h = options['--hash'] - self.project = project_from_options('.', self.toplevel_options) + self.project = project_from_options('.', self.toplevel_options, additional_options) services = [svc for svc in options['--hash'].split(',')] if h != '*' else None with errors.handle_connection_errors(self.project.client): for service in self.project.get_services(services): print('{} {}'.format(service.name, service.config_hash)) return - print(serialize_config(compose_config, image_digests)) + print(serialize_config(compose_config, image_digests, not options['--no-interpolate'])) def create(self, options): """ diff --git a/compose/config/config.py b/compose/config/config.py index e2ed29a47..c91c5aeb2 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -373,7 +373,7 @@ def check_swarm_only_config(service_dicts, compatibility=False): check_swarm_only_key(service_dicts, 'configs') -def load(config_details, compatibility=False): +def load(config_details, compatibility=False, interpolate=True): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top of each other to create the final configuration. @@ -383,7 +383,7 @@ def load(config_details, compatibility=False): validate_config_version(config_details.config_files) processed_files = [ - process_config_file(config_file, config_details.environment) + process_config_file(config_file, config_details.environment, interpolate=interpolate) for config_file in config_details.config_files ] config_details = config_details._replace(config_files=processed_files) @@ -505,7 +505,6 @@ def load_services(config_details, config_file, compatibility=False): def interpolate_config_section(config_file, config, section, environment): - validate_config_section(config_file.filename, config, section) return interpolate_environment_variables( config_file.version, config, @@ -514,38 +513,60 @@ def interpolate_config_section(config_file, config, section, environment): ) -def process_config_file(config_file, environment, service_name=None): - services = interpolate_config_section( +def process_config_section(config_file, config, section, environment, interpolate): + validate_config_section(config_file.filename, config, section) + if interpolate: + return interpolate_environment_variables( + config_file.version, + config, + section, + environment + ) + else: + return config + + +def process_config_file(config_file, environment, service_name=None, interpolate=True): + services = process_config_section( config_file, config_file.get_service_dicts(), 'service', - environment) + environment, + interpolate, + ) if config_file.version > V1: processed_config = dict(config_file.config) processed_config['services'] = services - processed_config['volumes'] = interpolate_config_section( + processed_config['volumes'] = process_config_section( config_file, config_file.get_volumes(), 'volume', - environment) - processed_config['networks'] = interpolate_config_section( + environment, + interpolate, + ) + processed_config['networks'] = process_config_section( config_file, config_file.get_networks(), 'network', - environment) + environment, + interpolate, + ) if config_file.version >= const.COMPOSEFILE_V3_1: - processed_config['secrets'] = interpolate_config_section( + processed_config['secrets'] = process_config_section( config_file, config_file.get_secrets(), 'secret', - environment) + environment, + interpolate, + ) if config_file.version >= const.COMPOSEFILE_V3_3: - processed_config['configs'] = interpolate_config_section( + processed_config['configs'] = process_config_section( config_file, config_file.get_configs(), 'config', - environment + environment, + interpolate, ) else: processed_config = services diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 8cb8a2808..5776ce957 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -24,14 +24,12 @@ def serialize_dict_type(dumper, data): def serialize_string(dumper, data): - """ Ensure boolean-like strings are quoted in the output and escape $ characters """ + """ Ensure boolean-like strings are quoted in the output """ representer = dumper.represent_str if six.PY3 else dumper.represent_unicode if isinstance(data, six.binary_type): data = data.decode('utf-8') - data = data.replace('$', '$$') - if data.lower() in ('y', 'n', 'yes', 'no', 'on', 'off', 'true', 'false'): # Empirically only y/n appears to be an issue, but this might change # depending on which PyYaml version is being used. Err on safe side. @@ -39,6 +37,12 @@ def serialize_string(dumper, data): return representer(data) +def serialize_string_escape_dollar(dumper, data): + """ Ensure boolean-like strings are quoted in the output and escape $ characters """ + data = data.replace('$', '$$') + return serialize_string(dumper, data) + + yaml.SafeDumper.add_representer(types.MountSpec, serialize_dict_type) yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) @@ -46,8 +50,6 @@ yaml.SafeDumper.add_representer(types.SecurityOpt, serialize_config_type) yaml.SafeDumper.add_representer(types.ServiceSecret, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServiceConfig, serialize_dict_type) yaml.SafeDumper.add_representer(types.ServicePort, serialize_dict_type) -yaml.SafeDumper.add_representer(str, serialize_string) -yaml.SafeDumper.add_representer(six.text_type, serialize_string) def denormalize_config(config, image_digests=None): @@ -93,7 +95,13 @@ def v3_introduced_name_key(key): return V3_5 -def serialize_config(config, image_digests=None): +def serialize_config(config, image_digests=None, escape_dollar=True): + if escape_dollar: + yaml.SafeDumper.add_representer(str, serialize_string_escape_dollar) + yaml.SafeDumper.add_representer(six.text_type, serialize_string_escape_dollar) + else: + yaml.SafeDumper.add_representer(str, serialize_string) + yaml.SafeDumper.add_representer(six.text_type, serialize_string) return yaml.safe_dump( denormalize_config(config, image_digests), default_flow_style=False, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 888c0ae58..d9d8496af 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -613,6 +613,25 @@ class ConfigTest(unittest.TestCase): excinfo.exconly() ) + def test_config_integer_service_name_raise_validation_error_v2_when_no_interpolate(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': '2', + 'services': {1: {'image': 'busybox'}} + }, + 'working_dir', + 'filename.yml' + ), + interpolate=False + ) + + assert ( + "In file 'filename.yml', the service name 1 must be a quoted string, i.e. '1'." in + excinfo.exconly() + ) + def test_config_integer_service_property_raise_validation_error(self): with pytest.raises(ConfigurationError) as excinfo: config.load( @@ -5328,6 +5347,28 @@ class SerializeTest(unittest.TestCase): assert serialized_service['command'] == 'echo $$FOO' assert serialized_service['entrypoint'][0] == '$$SHELL' + def test_serialize_escape_dont_interpolate(self): + cfg = { + 'version': '2.2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': 'echo $FOO', + 'environment': { + 'CURRENCY': '$' + }, + 'entrypoint': ['$SHELL', '-c'], + } + } + } + config_dict = config.load(build_config_details(cfg), interpolate=False) + + serialized_config = yaml.load(serialize_config(config_dict, escape_dollar=False)) + serialized_service = serialized_config['services']['web'] + assert serialized_service['environment']['CURRENCY'] == '$' + assert serialized_service['command'] == 'echo $FOO' + assert serialized_service['entrypoint'][0] == '$SHELL' + def test_serialize_unicode_values(self): cfg = { 'version': '2.3', From 42c965935f93aff8961517cd7c1803afcb621a2c Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Tue, 5 Mar 2019 11:28:15 +0000 Subject: [PATCH 25/62] Fix scale attribute to accept 0 as a value Signed-off-by: Jonathan Cremin --- compose/service.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 6483f4f31..2754572fe 100644 --- a/compose/service.py +++ b/compose/service.py @@ -177,7 +177,7 @@ class Service(object): network_mode=None, networks=None, secrets=None, - scale=None, + scale=1, pid_mode=None, default_platform=None, **options @@ -192,7 +192,7 @@ class Service(object): self.pid_mode = pid_mode or PidMode(None) self.networks = networks or {} self.secrets = secrets or [] - self.scale_num = scale or 1 + self.scale_num = scale self.default_platform = default_platform self.options = options From 0b039202ac3c78c12857e464350225b8c10e43ef Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Tue, 5 Mar 2019 11:00:05 +0000 Subject: [PATCH 26/62] docs/README.md: update since `vnext-compose` branch is no longer used. All PRs should be made to `master` now. Also: - Template seems to exist now[0] so remove the "coming soon". - The labels used seem different now, but labelling seems more like a docs maintainer thing than a contributor thing, so just drop that paragraph. [0] https://raw.githubusercontent.com/docker/docker.github.io/master/.github/PULL_REQUEST_TEMPLATE.md Signed-off-by: Ian Campbell --- docs/README.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/docs/README.md b/docs/README.md index 50c91d207..accc7c23e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -6,11 +6,9 @@ The documentation for Compose has been merged into The docs for Compose are now here: https://github.com/docker/docker.github.io/tree/master/compose -Please submit pull requests for unpublished features on the `vnext-compose` branch (https://github.com/docker/docker.github.io/tree/vnext-compose). +Please submit pull requests for unreleased features/changes on the `master` branch (https://github.com/docker/docker.github.io/tree/master), please prefix the PR title with `[WIP]` to indicate that it relates to an unreleased change. -If you submit a PR to this codebase that has a docs impact, create a second docs PR on `docker.github.io`. Use the docs PR template provided (coming soon - watch this space). - -PRs for typos, additional information, etc. for already-published features should be labeled as `okay-to-publish` (we are still settling on a naming convention, will provide a label soon). You can submit these PRs either to `vnext-compose` or directly to `master` on `docker.github.io` +If you submit a PR to this codebase that has a docs impact, create a second docs PR on `docker.github.io`. Use the docs PR template provided. As always, the docs remain open-source and we appreciate your feedback and pull requests! From 087bef4f95da6007a3a17ce1f796ee521050be7b Mon Sep 17 00:00:00 2001 From: Jonathan Cremin Date: Wed, 6 Mar 2019 12:57:14 +0000 Subject: [PATCH 27/62] Add tests for compose file 'scale: 0' Signed-off-by: Jonathan Cremin --- tests/acceptance/cli_test.py | 14 +++++++++++--- tests/fixtures/scale/docker-compose.yml | 8 ++++++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6a9a392a5..4fbc535d0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2505,10 +2505,12 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 + assert len(project.get_service('worker').containers()) == 0 - self.dispatch(['up', '-d', '--scale', 'web=3']) + self.dispatch(['up', '-d', '--scale', 'web=3', '--scale', 'worker=1']) assert len(project.get_service('web').containers()) == 3 assert len(project.get_service('db').containers()) == 1 + assert len(project.get_service('worker').containers()) == 1 def test_up_scale_scale_down(self): self.base_dir = 'tests/fixtures/scale' @@ -2517,22 +2519,26 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 + assert len(project.get_service('worker').containers()) == 0 self.dispatch(['up', '-d', '--scale', 'web=1']) assert len(project.get_service('web').containers()) == 1 assert len(project.get_service('db').containers()) == 1 + assert len(project.get_service('worker').containers()) == 0 def test_up_scale_reset(self): self.base_dir = 'tests/fixtures/scale' project = self.project - self.dispatch(['up', '-d', '--scale', 'web=3', '--scale', 'db=3']) + self.dispatch(['up', '-d', '--scale', 'web=3', '--scale', 'db=3', '--scale', 'worker=3']) assert len(project.get_service('web').containers()) == 3 assert len(project.get_service('db').containers()) == 3 + assert len(project.get_service('worker').containers()) == 3 self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 + assert len(project.get_service('worker').containers()) == 0 def test_up_scale_to_zero(self): self.base_dir = 'tests/fixtures/scale' @@ -2541,10 +2547,12 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d']) assert len(project.get_service('web').containers()) == 2 assert len(project.get_service('db').containers()) == 1 + assert len(project.get_service('worker').containers()) == 0 - self.dispatch(['up', '-d', '--scale', 'web=0', '--scale', 'db=0']) + self.dispatch(['up', '-d', '--scale', 'web=0', '--scale', 'db=0', '--scale', 'worker=0']) assert len(project.get_service('web').containers()) == 0 assert len(project.get_service('db').containers()) == 0 + assert len(project.get_service('worker').containers()) == 0 def test_port(self): self.base_dir = 'tests/fixtures/ports-composefile' diff --git a/tests/fixtures/scale/docker-compose.yml b/tests/fixtures/scale/docker-compose.yml index a0d3b771f..53ae1342d 100644 --- a/tests/fixtures/scale/docker-compose.yml +++ b/tests/fixtures/scale/docker-compose.yml @@ -5,5 +5,9 @@ services: command: top scale: 2 db: - image: busybox - command: top + image: busybox + command: top + worker: + image: busybox + command: top + scale: 0 From 3f1d41a97eb8c73f390009a99cb802899daaed90 Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Tue, 5 Mar 2019 10:17:09 -0500 Subject: [PATCH 28/62] Fix merging of compose files when network has None config Signed-off-by: Michael Irwin Resolves #6525 --- compose/config/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index c91c5aeb2..c110e2cfa 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1214,7 +1214,7 @@ def merge_networks(base, override): base = {k: {} for k in base} if isinstance(base, list) else base override = {k: {} for k in override} if isinstance(override, list) else override for network_name in all_network_names: - md = MergeDict(base.get(network_name, {}), override.get(network_name, {})) + md = MergeDict(base.get(network_name) or {}, override.get(network_name) or {}) md.merge_field('aliases', merge_unique_items_lists, []) md.merge_field('link_local_ips', merge_unique_items_lists, []) md.merge_scalar('priority') From d8e390eb9fd843d2d232ba08bae02381e1236e54 Mon Sep 17 00:00:00 2001 From: Michael Irwin Date: Tue, 5 Mar 2019 22:18:22 -0500 Subject: [PATCH 29/62] Added test case to verify fix for #6525 Signed-off-by: Michael Irwin --- tests/unit/config/config_test.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d9d8496af..e27427ca5 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3970,6 +3970,24 @@ class MergeNetworksTest(unittest.TestCase, MergeListsTest): } } + def test_network_has_none_value(self): + service_dict = config.merge_service_dicts( + {self.config_name: { + 'default': None + }}, + {self.config_name: { + 'default': { + 'aliases': [] + } + }}, + DEFAULT_VERSION) + + assert service_dict[self.config_name] == { + 'default': { + 'aliases': [] + } + } + def test_all_properties(self): service_dict = config.merge_service_dicts( {self.config_name: { From 76d0406fab3ee17bcd71afd98e05a5e6f2731ff4 Mon Sep 17 00:00:00 2001 From: Henke Adolfsson Date: Sat, 2 Mar 2019 15:02:47 +0100 Subject: [PATCH 30/62] Add test and implementation for secret added after container has been created The issue is that if a secret is added to the compose file, then it will not notice that containers have diverged since last run, because secrets are not part of the config_hash, which determines if the configuration of a service is the same or not. Signed-off-by: Henke Adolfsson --- compose/service.py | 1 + tests/integration/project_test.py | 42 +++++++++++++++++++++++++++++++ 2 files changed, 43 insertions(+) diff --git a/compose/service.py b/compose/service.py index 0aaf3cf37..e989d4877 100644 --- a/compose/service.py +++ b/compose/service.py @@ -685,6 +685,7 @@ class Service(object): 'links': self.get_link_names(), 'net': self.network_mode.id, 'networks': self.networks, + 'secrets': self.secrets, 'volumes_from': [ (v.source.name, v.mode) for v in self.volumes_from if isinstance(v.source, Service) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 57f3b7074..c6949cdc9 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1496,6 +1496,48 @@ class ProjectTest(DockerClientTestCase): output = container.logs() assert output == b"This is the secret\n" + @v3_only() + def test_project_up_with_added_secrets(self): + node = create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) + + config_data = build_config( + version=V3_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'cat /run/secrets/special', + # 'secrets': [ + # types.ServiceSecret.parse({'source': 'super', 'target': 'special'}), + # ], + 'environment': ['constraint:node=={}'.format(node if node is not None else '*')] + }], + secrets={ + 'super': { + 'file': os.path.abspath('tests/fixtures/secrets/default'), + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + project.stop() + project.services[0].secrets = [ + types.ServiceSecret.parse({'source': 'super', 'target': 'special'}) + ] + project.up() + project.stop() + + containers = project.containers(stopped=True) + assert len(containers) == 1 + container, = containers + + output = container.logs() + assert output == b"This is the secret\n" + @v2_only() def test_initialize_volumes_invalid_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) From aa79fb2473da535f758c8c4562857844e4ed1710 Mon Sep 17 00:00:00 2001 From: Henke Adolfsson Date: Tue, 5 Mar 2019 10:32:05 +0100 Subject: [PATCH 31/62] Ensure test passes Signed-off-by: Henke Adolfsson --- tests/integration/project_test.py | 56 +++++++++++++++++++------------ 1 file changed, 35 insertions(+), 21 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index c6949cdc9..f915640e3 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import copy import json import os import random @@ -1500,34 +1501,47 @@ class ProjectTest(DockerClientTestCase): def test_project_up_with_added_secrets(self): node = create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) - config_data = build_config( - version=V3_1, - services=[{ - 'name': 'web', - 'image': 'busybox:latest', - 'command': 'cat /run/secrets/special', - # 'secrets': [ - # types.ServiceSecret.parse({'source': 'super', 'target': 'special'}), - # ], - 'environment': ['constraint:node=={}'.format(node if node is not None else '*')] - }], - secrets={ - 'super': { - 'file': os.path.abspath('tests/fixtures/secrets/default'), - }, - }, - ) + config_input1 = { + 'version': V3_1, + 'services': [ + { + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'cat /run/secrets/special', + 'environment': ['constraint:node=={}'.format(node if node is not None else '')] + } + ], + 'secrets': { + 'super': { + 'file': os.path.abspath('tests/fixtures/secrets/default') + } + } + } + config_input2 = copy.deepcopy(config_input1) + # Add the secret + config_input2['services'][0]['secrets'] = [ + types.ServiceSecret.parse({'source': 'super', 'target': 'special'}) + ] + + config_data1 = build_config(**config_input1) + config_data2 = build_config(**config_input2) + + # First up with non-secret project = Project.from_config( client=self.client, name='composetest', - config_data=config_data, + config_data=config_data1, ) project.up() project.stop() - project.services[0].secrets = [ - types.ServiceSecret.parse({'source': 'super', 'target': 'special'}) - ] + + # Then up with secret + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data2, + ) project.up() project.stop() From 87935893fcdf9d5ea5bfa661836fa95c98743705 Mon Sep 17 00:00:00 2001 From: Henke Adolfsson Date: Tue, 5 Mar 2019 15:50:05 +0100 Subject: [PATCH 32/62] Update data for unit tests Signed-off-by: Henke Adolfsson --- tests/unit/service_test.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8b3352fcb..3d7c4987a 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -333,7 +333,7 @@ class ServiceTest(unittest.TestCase): assert service.options['environment'] == environment assert opts['labels'][LABEL_CONFIG_HASH] == \ - '2524a06fcb3d781aa2c981fc40bcfa08013bb318e4273bfa388df22023e6f2aa' + '689149e6041a85f6fb4945a2146a497ed43c8a5cbd8991753d875b165f1b4de4' assert opts['environment'] == ['also=real'] def test_get_container_create_options_sets_affinity_with_binds(self): @@ -676,6 +676,7 @@ class ServiceTest(unittest.TestCase): 'options': {'image': 'example.com/foo'}, 'links': [('one', 'one')], 'net': 'other', + 'secrets': [], 'networks': {'default': None}, 'volumes_from': [('two', 'rw')], } @@ -698,6 +699,7 @@ class ServiceTest(unittest.TestCase): 'options': {'image': 'example.com/foo'}, 'links': [], 'networks': {}, + 'secrets': [], 'net': 'aaabbb', 'volumes_from': [], } From 853215acf619cadda4f5b892a2fb8402de4ba28f Mon Sep 17 00:00:00 2001 From: Henke Adolfsson Date: Tue, 5 Mar 2019 17:53:13 +0100 Subject: [PATCH 33/62] Remove project.stop() in test Signed-off-by: Henke Adolfsson --- tests/integration/project_test.py | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index f915640e3..fe6ace90e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1534,7 +1534,6 @@ class ProjectTest(DockerClientTestCase): config_data=config_data1, ) project.up() - project.stop() # Then up with secret project = Project.from_config( From 0863785e96903a8610f25171d67cf063c6f6cfea Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Tue, 12 Mar 2019 10:35:09 -0400 Subject: [PATCH 34/62] Enable bootloader_ignore_signals in pyinstaller Fixes #3347 Signed-off-by: Ben Firshman --- docker-compose.spec | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docker-compose.spec b/docker-compose.spec index 70db5c29e..5ca1e4c24 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -98,4 +98,5 @@ exe = EXE(pyz, debug=False, strip=None, upx=True, - console=True) + console=True, + bootloader_ignore_signals=True) From dc712bfa23aac08f6503f616095e6576b06b3c20 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 20 Mar 2019 18:03:41 +0100 Subject: [PATCH 35/62] Bump docker-py version to 3.7.1 This docker-py version includes ssh fixes Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9333ec60d..2487f92c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==3.7.0 +docker==3.7.1 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 From 1e4fde8aa7258c3e4c33cbdaf1e71381a5e43c27 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 20 Mar 2019 18:03:41 +0100 Subject: [PATCH 36/62] Bump docker-py version to 3.7.1 This docker-py version includes ssh fixes Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 9333ec60d..2487f92c0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==3.7.0 +docker==3.7.1 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 From cd1fcd3ea52a0453ae26253bbe48e3f24ed722e2 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 22 Mar 2019 10:32:59 +0100 Subject: [PATCH 37/62] Use os.system() instead of run_setup() Use `os.system()` instead of `run_setup()` because the last is not taking any effect Signed-off-by: Ulysses Souza --- script/release/release.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 63bf863df..9db1a49d9 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -7,7 +7,6 @@ import os import shutil import sys import time -from distutils.core import run_setup from jinja2 import Template from release.bintray import BintrayAPI @@ -276,7 +275,8 @@ def finalize(args): repository.checkout_branch(br_name) - run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['sdist', 'bdist_wheel']) + os.system('python {setup_script} sdist bdist_wheel'.format( + setup_script=os.path.join(REPO_ROOT, 'setup.py'))) merge_status = pr_data.merge() if not merge_status.merged and not args.finalize_resume: From 15f8c30a51e7e614a22f66244b06fd2f736ac356 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 22 Mar 2019 10:58:32 +0100 Subject: [PATCH 38/62] Fix typo on finalize Signed-off-by: Ulysses Souza --- script/release/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/README.md b/script/release/README.md index f7f911e53..a0ce5d2a8 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -129,7 +129,7 @@ assets public), proceed to the "Finalize a release" section of this guide. Once you're ready to make your release public, you may execute the following command from the root of the Compose repository: ``` -./script/release/release.sh -b finalize RELEAE_VERSION +./script/release/release.sh -b finalize RELEASE_VERSION ``` Note that this command will create and publish versioned assets to the public. From 2948c396a6abc853ffcb16051f3593a70e43625d Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 22 Mar 2019 11:00:23 +0100 Subject: [PATCH 39/62] Fix bintray docker-compose link Signed-off-by: Ulysses Souza --- script/release/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/README.md b/script/release/README.md index a0ce5d2a8..0c6f12cbe 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -40,7 +40,7 @@ This API token should be exposed to the release script through the ### A Bintray account and Bintray API key Your Bintray account will need to be an admin member of the -[docker-compose organization](https://github.com/settings/tokens). +[docker-compose organization](https://bintray.com/docker-compose). Additionally, you should generate a personal API key. To do so, click your username in the top-right hand corner and select "Edit profile" ; on the new page, select "API key" in the left-side menu. From 154d7c17228cf9196ea4ee6b5e13a5268cc69407 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 22 Mar 2019 17:10:41 +0100 Subject: [PATCH 40/62] Fix script for release file already present case This avoids a: "AttributeError: 'HTTPError' object has no attribute 'message'" Signed-off-by: Ulysses Souza --- script/release/release/pypi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/release/pypi.py b/script/release/release/pypi.py index a40e17544..dc0b0cb97 100644 --- a/script/release/release/pypi.py +++ b/script/release/release/pypi.py @@ -18,7 +18,7 @@ def pypi_upload(args): 'dist/docker-compose-{}*.tar.gz'.format(rel) ]) except HTTPError as e: - if e.response.status_code == 400 and 'File already exists' in e.message: + if e.response.status_code == 400 and 'File already exists' in str(e): if not args.finalize_resume: raise ScriptError( 'Package already uploaded on PyPi.' From c6dd7da15eb3d85a1f7634e8ded9fe42c9035669 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Sun, 3 Feb 2019 04:29:20 +0300 Subject: [PATCH 41/62] only pull images that can't build Signed-off-by: Collins Abitekaniza --- compose/project.py | 7 +++++-- tests/acceptance/cli_test.py | 7 +++++++ tests/fixtures/simple-composefile/pull-with-build.yml | 11 +++++++++++ 3 files changed, 23 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/simple-composefile/pull-with-build.yml diff --git a/compose/project.py b/compose/project.py index a7f2aa057..c6a802509 100644 --- a/compose/project.py +++ b/compose/project.py @@ -602,6 +602,9 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, include_deps=False): services = self.get_services(service_names, include_deps) + images_to_build = {service.image_name for service in services if service.can_be_built()} + services_to_pull = [service for service in services if service.image_name not in images_to_build] + msg = not silent and 'Pulling' or None if parallel_pull: @@ -627,7 +630,7 @@ class Project(object): ) _, errors = parallel.parallel_execute( - services, + services_to_pull, pull_service, operator.attrgetter('name'), msg, @@ -640,7 +643,7 @@ class Project(object): raise ProjectError(combined_errors) else: - for service in services: + for service in services_to_pull: service.pull(ignore_pull_failures, silent=silent) def push(self, service_names=None, ignore_push_failures=False): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5142f96eb..8a7c0febc 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -633,6 +633,13 @@ class CLITestCase(DockerClientTestCase): 'image library/nonexisting-image:latest not found' in result.stderr or 'pull access denied for nonexisting-image' in result.stderr) + def test_pull_with_build(self): + result = self.dispatch(['-f', 'pull-with-build.yml', 'pull']) + + assert 'Pulling simple' not in result.stderr + assert 'Pulling from_simple' not in result.stderr + assert 'Pulling another ...' in result.stderr + def test_pull_with_quiet(self): assert self.dispatch(['pull', '--quiet']).stderr == '' assert self.dispatch(['pull', '--quiet']).stdout == '' diff --git a/tests/fixtures/simple-composefile/pull-with-build.yml b/tests/fixtures/simple-composefile/pull-with-build.yml new file mode 100644 index 000000000..261dc44df --- /dev/null +++ b/tests/fixtures/simple-composefile/pull-with-build.yml @@ -0,0 +1,11 @@ +version: "3" +services: + build_simple: + image: simple + build: . + command: top + from_simple: + image: simple + another: + image: busybox:latest + command: top From 8a339946fa77871e23919af0be322a4aba1f8747 Mon Sep 17 00:00:00 2001 From: joeweoj Date: Tue, 19 Mar 2019 11:16:51 +0000 Subject: [PATCH 42/62] Fixed depends_on recreation behaviour for issue #6589 Previously any containers which did *not* have any links were always recreated. In order to fix depends_on and preserve expected links recreation behaviour, we now only use the ConvergenceStrategy.always recreation strategy for a service if any of the the following conditions are true: * --always-recreate-deps flag provided * service container is stopped * service defines links but the container does not have any * container has links but the service definition does not Signed-off-by: joeweoj --- compose/project.py | 6 +- tests/integration/state_test.py | 139 ++++++++++++++++++++++++++++++++ 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index a74033729..e3b14d627 100644 --- a/compose/project.py +++ b/compose/project.py @@ -586,8 +586,10 @@ class Project(object): ", ".join(updated_dependencies)) containers_stopped = any( service.containers(stopped=True, filters={'status': ['created', 'exited']})) - has_links = any(c.get('HostConfig.Links') for c in service.containers()) - if always_recreate_deps or containers_stopped or not has_links: + service_has_links = any(service.get_link_names()) + container_has_links = any(c.get('HostConfig.Links') for c in service.containers()) + should_recreate_for_links = service_has_links ^ container_has_links + if always_recreate_deps or containers_stopped or should_recreate_for_links: plan = service.convergence_plan(ConvergenceStrategy.always) else: plan = service.convergence_plan(strategy) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index b7d38a4ba..0d69217cf 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -5,6 +5,8 @@ by `docker-compose up`. from __future__ import absolute_import from __future__ import unicode_literals +import copy + import py from docker.errors import ImageNotFound @@ -209,6 +211,143 @@ class ProjectWithDependenciesTest(ProjectTestCase): } +class ProjectWithDependsOnDependenciesTest(ProjectTestCase): + def setUp(self): + super(ProjectWithDependsOnDependenciesTest, self).setUp() + + self.cfg = { + 'version': '2', + 'services': { + 'db': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + }, + 'web': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + 'depends_on': ['db'], + }, + 'nginx': { + 'image': 'busybox:latest', + 'command': 'tail -f /dev/null', + 'depends_on': ['web'], + }, + } + } + + def test_up(self): + local_cfg = copy.deepcopy(self.cfg) + containers = self.run_up(local_cfg) + assert set(c.service for c in containers) == set(['db', 'web', 'nginx']) + + def test_change_leaf(self): + local_cfg = copy.deepcopy(self.cfg) + old_containers = self.run_up(local_cfg) + + local_cfg['services']['nginx']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(local_cfg) + + assert set(c.service for c in new_containers - old_containers) == set(['nginx']) + + def test_change_middle(self): + local_cfg = copy.deepcopy(self.cfg) + old_containers = self.run_up(local_cfg) + + local_cfg['services']['web']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(local_cfg) + + assert set(c.service for c in new_containers - old_containers) == set(['web']) + + def test_change_middle_always_recreate_deps(self): + local_cfg = copy.deepcopy(self.cfg) + old_containers = self.run_up(local_cfg, always_recreate_deps=True) + + local_cfg['services']['web']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(local_cfg, always_recreate_deps=True) + + assert set(c.service for c in new_containers - old_containers) == set(['web', 'nginx']) + + def test_change_root(self): + local_cfg = copy.deepcopy(self.cfg) + old_containers = self.run_up(local_cfg) + + local_cfg['services']['db']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(local_cfg) + + assert set(c.service for c in new_containers - old_containers) == set(['db']) + + def test_change_root_always_recreate_deps(self): + local_cfg = copy.deepcopy(self.cfg) + old_containers = self.run_up(local_cfg, always_recreate_deps=True) + + local_cfg['services']['db']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up(local_cfg, always_recreate_deps=True) + + assert set(c.service for c in new_containers - old_containers) == set(['db', 'web', 'nginx']) + + def test_change_root_no_recreate(self): + local_cfg = copy.deepcopy(self.cfg) + old_containers = self.run_up(local_cfg) + + local_cfg['services']['db']['environment'] = {'NEW_VAR': '1'} + new_containers = self.run_up( + local_cfg, + strategy=ConvergenceStrategy.never) + + assert new_containers - old_containers == set() + + def test_service_removed_while_down(self): + local_cfg = copy.deepcopy(self.cfg) + next_cfg = copy.deepcopy(self.cfg) + del next_cfg['services']['db'] + del next_cfg['services']['web']['depends_on'] + + containers = self.run_up(local_cfg) + assert set(c.service for c in containers) == set(['db', 'web', 'nginx']) + + project = self.make_project(local_cfg) + project.stop(timeout=1) + + next_containers = self.run_up(next_cfg) + assert set(c.service for c in next_containers) == set(['web', 'nginx']) + + def test_service_removed_while_up(self): + local_cfg = copy.deepcopy(self.cfg) + containers = self.run_up(local_cfg) + assert set(c.service for c in containers) == set(['db', 'web', 'nginx']) + + del local_cfg['services']['db'] + del local_cfg['services']['web']['depends_on'] + + containers = self.run_up(local_cfg) + assert set(c.service for c in containers) == set(['web', 'nginx']) + + def test_dependency_removed(self): + local_cfg = copy.deepcopy(self.cfg) + next_cfg = copy.deepcopy(self.cfg) + del next_cfg['services']['nginx']['depends_on'] + + containers = self.run_up(local_cfg, service_names=['nginx']) + assert set(c.service for c in containers) == set(['db', 'web', 'nginx']) + + project = self.make_project(local_cfg) + project.stop(timeout=1) + + next_containers = self.run_up(next_cfg, service_names=['nginx']) + assert set(c.service for c in next_containers if c.is_running) == set(['nginx']) + + def test_dependency_added(self): + local_cfg = copy.deepcopy(self.cfg) + + del local_cfg['services']['nginx']['depends_on'] + containers = self.run_up(local_cfg, service_names=['nginx']) + assert set(c.service for c in containers) == set(['nginx']) + + local_cfg['services']['nginx']['depends_on'] = ['db'] + containers = self.run_up(local_cfg, service_names=['nginx']) + assert set(c.service for c in containers) == set(['nginx', 'db']) + + class ServiceStateTest(DockerClientTestCase): """Test cases for Service.convergence_plan.""" From ac148bc1ca4d3e1f367fac36dbdf7485e093280f Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 28 Mar 2019 17:45:03 +0100 Subject: [PATCH 43/62] Bump docker-py 3.7.2 Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 2487f92c0..5e8ec2ed7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==3.7.1 +docker==3.7.2 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 From ef10c1803fa7857a955585a62f77449e608d07dc Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Fri, 11 Jan 2019 17:43:47 +0100 Subject: [PATCH 44/62] "Bump 1.24.0" Signed-off-by: Ulysses Souza --- CHANGELOG.md | 52 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 54 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7a2ffe9..7a3ad8bfd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,58 @@ Change log ========== +1.24.0 (2019-03-28) +------------------- + +### Features + +- Added support for connecting to the Docker Engine using the `ssh` protocol. + +- Added a `--all` flag to `docker-compose ps` to include stopped one-off containers + in the command's output. + +- Add bash completion for `ps --all|-a` + +- Support for credential_spec + +- Add `--parallel` to `docker build`'s options in `bash` and `zsh` completion + +### Bugfixes + +- Fixed a bug where some valid credential helpers weren't properly handled by Compose + when attempting to pull images from private registries. + +- Fixed an issue where the output of `docker-compose start` before containers were created + was misleading + +- To match the Docker CLI behavior and to avoid confusing issues, Compose will no longer + accept whitespace in variable names sourced from environment files. + +- Compose will now report a configuration error if a service attempts to declare + duplicate mount points in the volumes section. + +- Fixed an issue with the containerized version of Compose that prevented users from + writing to stdin during interactive sessions started by `run` or `exec`. + +- One-off containers started by `run` no longer adopt the restart policy of the service, + and are instead set to never restart. + +- Fixed an issue that caused some container events to not appear in the output of + the `docker-compose events` command. + +- Missing images will no longer stop the execution of `docker-compose down` commands + (a warning will be displayed instead). + +- Force `virtualenv` version for macOS CI + +- Fix merging of compose files when network has `None` config + +- Fix `CTRL+C` issues by enabling `bootloader_ignore_signals` in `pyinstaller` + +- Bump `docker-py` version to `3.7.2` to fix SSH and proxy config issues + +- Fix release script and some typos on release documentation + 1.23.2 (2018-11-28) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 652e1fad9..0042896b6 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.24.0dev' +__version__ = '1.25.0dev' diff --git a/script/run/run.sh b/script/run/run.sh index cc36e4751..a8690cad1 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.2" +VERSION="1.24.0" IMAGE="docker/compose:$VERSION" From 9e3d9f66811f0d21dfca65874a2937e8cf210089 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 4 Apr 2019 10:08:04 +0200 Subject: [PATCH 45/62] Update release process on updating docs Signed-off-by: Ulysses Souza --- script/release/README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/release/README.md b/script/release/README.md index 0c6f12cbe..97168d376 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -192,6 +192,8 @@ be handled manually by the operator: - Bump the version in `compose/__init__.py` to the *next* minor version number with `dev` appended. For example, if you just released `1.4.0`, update it to `1.5.0dev` + - Update compose_version in [github.com/docker/docker.github.io/blob/master/_config.yml](https://github.com/docker/docker.github.io/blob/master/_config.yml) and [github.com/docker/docker.github.io/blob/master/_config_authoring.yml](https://github.com/docker/docker.github.io/blob/master/_config_authoring.yml) + - Update the release note in [github.com/docker/docker.github.io](https://github.com/docker/docker.github.io/blob/master/release-notes/docker-compose.md) ## Advanced options From c217bab7f6123de80dbd55c99c2254666d766fb3 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 10 Apr 2019 21:05:02 +0200 Subject: [PATCH 46/62] Refactor Dockerfiles for generating musl binaries - Refactor Dockerfile to be used for tests and distribution on docker hub on debian and alpine to use for final usage and also tests - Adapt test scripts to the new Dockerfiles' structure - Adapt Jenkinsfile to add alpine to the test matrix Signed-off-by: Ulysses Souza --- Dockerfile | 85 ++++++++++++++++++++++++----------- Dockerfile.run | 19 -------- Jenkinsfile | 46 ++++++++++--------- docker-compose-entrypoint.sh | 20 +++++++++ pyinstaller/ldd | 13 ++++++ script/build/linux | 19 +++++--- script/build/linux-entrypoint | 37 +++++++++++---- script/build/test-image | 15 ++++--- script/test/ci | 3 -- script/test/default | 9 ++-- 10 files changed, 173 insertions(+), 93 deletions(-) delete mode 100644 Dockerfile.run create mode 100755 docker-compose-entrypoint.sh create mode 100755 pyinstaller/ldd diff --git a/Dockerfile b/Dockerfile index b887bcc4e..1a3c501ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,36 +1,71 @@ -FROM docker:18.06.1 as docker -FROM python:3.7.2-stretch +ARG DOCKER_VERSION=18.09.5 +ARG PYTHON_VERSION=3.7.3 +ARG BUILD_ALPINE_VERSION=3.9 +ARG BUILD_DEBIAN_VERSION=slim-stretch +ARG RUNTIME_ALPINE_VERSION=3.9.3 +ARG RUNTIME_DEBIAN_VERSION=stretch-20190326-slim -RUN set -ex; \ - apt-get update -qq; \ - apt-get install -y \ - locales \ - python-dev \ - git +ARG BUILD_PLATFORM=alpine -COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker +FROM docker:${DOCKER_VERSION} AS docker-cli -# Python3 requires a valid locale -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 +FROM python:${PYTHON_VERSION}-alpine${BUILD_ALPINE_VERSION} AS build-alpine +RUN apk add --no-cache \ + bash \ + build-base \ + ca-certificates \ + curl \ + gcc \ + git \ + libc-dev \ + libffi-dev \ + libgcc \ + make \ + musl-dev \ + openssl \ + openssl-dev \ + python2 \ + python2-dev \ + zlib-dev +ENV BUILD_BOOTLOADER=1 -RUN useradd -d /home/user -m -s /bin/bash user +FROM python:${PYTHON_VERSION}-${BUILD_DEBIAN_VERSION} AS build-debian +RUN apt-get update && apt-get install -y \ + curl \ + gcc \ + git \ + libc-dev \ + libgcc-6-dev \ + make \ + openssl \ + python2.7-dev + +FROM build-${BUILD_PLATFORM} AS build +COPY docker-compose-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["sh", "/usr/local/bin/docker-compose-entrypoint.sh"] +COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker WORKDIR /code/ - # FIXME(chris-crone): virtualenv 16.3.0 breaks build, force 16.2.0 until fixed RUN pip install virtualenv==16.2.0 RUN pip install tox==2.9.1 -ADD requirements.txt /code/ -ADD requirements-dev.txt /code/ -ADD .pre-commit-config.yaml /code/ -ADD setup.py /code/ -ADD tox.ini /code/ -ADD compose /code/compose/ -ADD README.md /code/ +COPY requirements.txt . +COPY requirements-dev.txt . +COPY .pre-commit-config.yaml . +COPY tox.ini . +COPY setup.py . +COPY README.md . +COPY compose compose/ RUN tox --notest +COPY . . +ARG GIT_COMMIT=unknown +ENV DOCKER_COMPOSE_GITSHA=$GIT_COMMIT +RUN script/build/linux-entrypoint -ADD . /code/ -RUN chown -R user /code/ - -ENTRYPOINT ["/code/.tox/py37/bin/docker-compose"] +FROM alpine:${RUNTIME_ALPINE_VERSION} AS runtime-alpine +FROM debian:${RUNTIME_DEBIAN_VERSION} AS runtime-debian +FROM runtime-${BUILD_PLATFORM} AS runtime +COPY docker-compose-entrypoint.sh /usr/local/bin/ +ENTRYPOINT ["sh", "/usr/local/bin/docker-compose-entrypoint.sh"] +COPY --from=docker-cli /usr/local/bin/docker /usr/local/bin/docker +COPY --from=build /usr/local/bin/docker-compose /usr/local/bin/docker-compose diff --git a/Dockerfile.run b/Dockerfile.run deleted file mode 100644 index ccc86ea96..000000000 --- a/Dockerfile.run +++ /dev/null @@ -1,19 +0,0 @@ -FROM docker:18.06.1 as docker -FROM alpine:3.8 - -ENV GLIBC 2.28-r0 - -RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ - curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ - curl -fsSL -o glibc-$GLIBC.apk https://github.com/sgerrand/alpine-pkg-glibc/releases/download/$GLIBC/glibc-$GLIBC.apk && \ - apk add --no-cache glibc-$GLIBC.apk && \ - ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ - ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ - ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib && \ - rm /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ - apk del curl - -COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker -COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose - -ENTRYPOINT ["docker-compose"] diff --git a/Jenkinsfile b/Jenkinsfile index a19e82273..51fecf99f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,29 +1,32 @@ #!groovy -def image - -def buildImage = { -> +def buildImage = { String baseImage -> + def image wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { - stage("build image") { + stage("build image for \"${baseImage}\"") { checkout(scm) - def imageName = "dockerbuildbot/compose:${gitCommit()}" + def imageName = "dockerbuildbot/compose:${baseImage}-${gitCommit()}" image = docker.image(imageName) try { image.pull() } catch (Exception exc) { - image = docker.build(imageName, ".") - image.push() + sh "docker build -t ${imageName} --target build --build-arg BUILD_PLATFORM=${baseImage} ." + sh "docker push ${imageName}" + echo "${imageName}" + return imageName } } } + echo "image.id: ${image.id}" + return image.id } -def get_versions = { int number -> +def get_versions = { String imageId, int number -> def docker_versions wrappedNode(label: "ubuntu && !zfs") { def result = sh(script: """docker run --rm \\ --entrypoint=/code/.tox/py27/bin/python \\ - ${image.id} \\ + ${imageId} \\ /code/script/test/versions.py -n ${number} docker/docker-ce recent """, returnStdout: true ) @@ -35,6 +38,8 @@ def get_versions = { int number -> def runTests = { Map settings -> def dockerVersions = settings.get("dockerVersions", null) def pythonVersions = settings.get("pythonVersions", null) + def baseImage = settings.get("baseImage", null) + def imageName = settings.get("image", null) if (!pythonVersions) { throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py37')`") @@ -45,7 +50,7 @@ def runTests = { Map settings -> { -> wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { - stage("test python=${pythonVersions} / docker=${dockerVersions}") { + stage("test python=${pythonVersions} / docker=${dockerVersions} / baseImage=${baseImage}") { checkout(scm) def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() echo "Using local system's storage driver: ${storageDriver}" @@ -55,13 +60,13 @@ def runTests = { Map settings -> --privileged \\ --volume="\$(pwd)/.git:/code/.git" \\ --volume="/var/run/docker.sock:/var/run/docker.sock" \\ - -e "TAG=${image.id}" \\ + -e "TAG=${imageName}" \\ -e "STORAGE_DRIVER=${storageDriver}" \\ -e "DOCKER_VERSIONS=${dockerVersions}" \\ -e "BUILD_NUMBER=\$BUILD_TAG" \\ -e "PY_TEST_VERSIONS=${pythonVersions}" \\ --entrypoint="script/test/ci" \\ - ${image.id} \\ + ${imageName} \\ --verbose """ } @@ -69,15 +74,16 @@ def runTests = { Map settings -> } } -buildImage() - def testMatrix = [failFast: true] -def docker_versions = get_versions(2) - -for (int i = 0; i < docker_versions.length; i++) { - def dockerVersion = docker_versions[i] - testMatrix["${dockerVersion}_py27"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py27"]) - testMatrix["${dockerVersion}_py37"] = runTests([dockerVersions: dockerVersion, pythonVersions: "py37"]) +def baseImages = ['alpine', 'debian'] +def pythonVersions = ['py27', 'py37'] +baseImages.each { baseImage -> + def imageName = buildImage(baseImage) + get_versions(imageName, 2).each { dockerVersion -> + pythonVersions.each { pyVersion -> + testMatrix["${baseImage}_${dockerVersion}_${pyVersion}"] = runTests([baseImage: baseImage, image: imageName, dockerVersions: dockerVersion, pythonVersions: pyVersion]) + } + } } parallel(testMatrix) diff --git a/docker-compose-entrypoint.sh b/docker-compose-entrypoint.sh new file mode 100755 index 000000000..84436fa07 --- /dev/null +++ b/docker-compose-entrypoint.sh @@ -0,0 +1,20 @@ +#!/bin/sh +set -e + +# first arg is `-f` or `--some-option` +if [ "${1#-}" != "$1" ]; then + set -- docker-compose "$@" +fi + +# if our command is a valid Docker subcommand, let's invoke it through Docker instead +# (this allows for "docker run docker ps", etc) +if docker-compose help "$1" > /dev/null 2>&1; then + set -- docker-compose "$@" +fi + +# if we have "--link some-docker:docker" and not DOCKER_HOST, let's set DOCKER_HOST automatically +if [ -z "$DOCKER_HOST" -a "$DOCKER_PORT_2375_TCP" ]; then + export DOCKER_HOST='tcp://docker:2375' +fi + +exec "$@" diff --git a/pyinstaller/ldd b/pyinstaller/ldd new file mode 100755 index 000000000..3f10ad275 --- /dev/null +++ b/pyinstaller/ldd @@ -0,0 +1,13 @@ +#!/bin/sh + +# From http://wiki.musl-libc.org/wiki/FAQ#Q:_where_is_ldd_.3F +# +# Musl's dynlinker comes with ldd functionality built in. just create a +# symlink from ld-musl-$ARCH.so to /bin/ldd. If the dynlinker was started +# as "ldd", it will detect that and print the appropriate DSO information. +# +# Instead, this string replaced "ldd" with the package so that pyinstaller +# can find the actual lib. +exec /usr/bin/ldd "$@" | \ + sed -r 's/([^[:space:]]+) => ldd/\1 => \/lib\/\1/g' | \ + sed -r 's/ldd \(.*\)//g' diff --git a/script/build/linux b/script/build/linux index 056940ad0..8de7218db 100755 --- a/script/build/linux +++ b/script/build/linux @@ -4,10 +4,15 @@ set -ex ./script/clean -TAG="docker-compose" -docker build -t "$TAG" . -docker run \ - --rm --entrypoint="script/build/linux-entrypoint" \ - -v $(pwd)/dist:/code/dist \ - -v $(pwd)/.git:/code/.git \ - "$TAG" +TMP_CONTAINER="tmpcontainer" +TAG="docker/compose:tmp-glibc-linux-binary" +DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) + +docker build -t "${TAG}" . \ + --build-arg BUILD_PLATFORM=debian \ + --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} +docker create --name ${TMP_CONTAINER} ${TAG} +mkdir -p dist +docker cp ${TMP_CONTAINER}:/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 +docker container rm -f ${TMP_CONTAINER} +docker image rm -f ${TAG} diff --git a/script/build/linux-entrypoint b/script/build/linux-entrypoint index 34c16ac69..1556bbf20 100755 --- a/script/build/linux-entrypoint +++ b/script/build/linux-entrypoint @@ -2,14 +2,35 @@ set -ex -TARGET=dist/docker-compose-$(uname -s)-$(uname -m) -VENV=/code/.tox/py37 +CODE_PATH=/code +VENV=${CODE_PATH}/.tox/py37 -mkdir -p `pwd`/dist -chmod 777 `pwd`/dist +cd ${CODE_PATH} +mkdir -p dist +chmod 777 dist -$VENV/bin/pip install -q -r requirements-build.txt +${VENV}/bin/pip3 install -q -r requirements-build.txt + +# TODO(ulyssessouza) To check if really needed ./script/build/write-git-sha -su -c "$VENV/bin/pyinstaller docker-compose.spec" user -mv dist/docker-compose $TARGET -$TARGET version + +export PATH="${CODE_PATH}/pyinstaller:${PATH}" + +if [ ! -z "${BUILD_BOOTLOADER}" ]; then + # Build bootloader for alpine + git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller + cd /tmp/pyinstaller/bootloader + git checkout v3.4 + ${VENV}/bin/python3 ./waf configure --no-lsb all + ${VENV}/bin/pip3 install .. + cd ${CODE_PATH} + rm -Rf /tmp/pyinstaller +else + echo "NOT compiling bootloader!!!" +fi + +${VENV}/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec +ls -la dist/ +ldd dist/docker-compose +mv dist/docker-compose /usr/local/bin +docker-compose version diff --git a/script/build/test-image b/script/build/test-image index a2eb62cdf..9d880c27f 100755 --- a/script/build/test-image +++ b/script/build/test-image @@ -7,11 +7,12 @@ if [ -z "$1" ]; then exit 1 fi -TAG=$1 +TAG="$1" +IMAGE="docker/compose-tests" -docker build -t docker-compose-tests:tmp . -ctnr_id=$(docker create --entrypoint=tox docker-compose-tests:tmp) -docker commit $ctnr_id docker/compose-tests:latest -docker tag docker/compose-tests:latest docker/compose-tests:$TAG -docker rm -f $ctnr_id -docker rmi -f docker-compose-tests:tmp +DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) +docker build -t "${IMAGE}:${TAG}" . \ + --target build \ + --build-arg BUILD_PLATFORM=debian \ + --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} +docker tag ${IMAGE}:${TAG} ${IMAGE}:latest diff --git a/script/test/ci b/script/test/ci index 8d3aa56cb..bbcedac47 100755 --- a/script/test/ci +++ b/script/test/ci @@ -20,6 +20,3 @@ export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" GIT_VOLUME="--volumes-from=$(hostname)" . script/test/all - ->&2 echo "Building Linux binary" -. script/build/linux-entrypoint diff --git a/script/test/default b/script/test/default index cbb6a67cb..d24b41b0d 100755 --- a/script/test/default +++ b/script/test/default @@ -3,17 +3,18 @@ set -ex -TAG="docker-compose:$(git rev-parse --short HEAD)" +TAG="docker-compose:alpine-$(git rev-parse --short HEAD)" -# By default use the Dockerfile, but can be overridden to use an alternative file +# By default use the Dockerfile.alpine, but can be overridden to use an alternative file # e.g DOCKERFILE=Dockerfile.armhf script/test/default -DOCKERFILE="${DOCKERFILE:-Dockerfile}" +DOCKERFILE="${DOCKERFILE:-Dockerfile.alpine}" +DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}" rm -rf coverage-html # Create the host directory so it's owned by $USER mkdir -p coverage-html -docker build -f ${DOCKERFILE} -t "$TAG" . +docker build -f ${DOCKERFILE} -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" . GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" . script/test/all From 2b24eb693c62ce8fd5f274f0fcb2837132a6a0b8 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 12 Apr 2019 18:46:06 +0200 Subject: [PATCH 47/62] Refactor release and build scripts - Make use of the same Dockerfile when producing an image for testing and for deploying to DockerHub Signed-off-by: Ulysses Souza --- Jenkinsfile | 8 ++- script/build/image | 11 +-- script/build/linux | 15 ++-- script/build/linux-entrypoint | 19 ++--- script/build/osx | 5 +- script/build/test-image | 8 +-- script/build/write-git-sha | 2 +- script/release/release.py | 13 +++- script/release/release/images.py | 101 +++++++++++++++++++-------- script/release/release/repository.py | 1 + script/run/run.sh | 2 +- script/test/all | 3 +- script/test/default | 6 +- 13 files changed, 126 insertions(+), 68 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 51fecf99f..4de276ada 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -10,7 +10,13 @@ def buildImage = { String baseImage -> try { image.pull() } catch (Exception exc) { - sh "docker build -t ${imageName} --target build --build-arg BUILD_PLATFORM=${baseImage} ." + sh """GIT_COMMIT=\$(script/build/write-git-sha) && \\ + docker build -t ${imageName} \\ + --target build \\ + --build-arg BUILD_PLATFORM="${baseImage}" \\ + --build-arg GIT_COMMIT="${GIT_COMMIT}" \\ + .\\ + """ sh "docker push ${imageName}" echo "${imageName}" return imageName diff --git a/script/build/image b/script/build/image index a3198c99f..fb3f856ee 100755 --- a/script/build/image +++ b/script/build/image @@ -7,11 +7,14 @@ if [ -z "$1" ]; then exit 1 fi -TAG=$1 +TAG="$1" VERSION="$(python setup.py --version)" -./script/build/write-git-sha +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA python setup.py sdist bdist_wheel -./script/build/linux -docker build -t docker/compose:$TAG -f Dockerfile.run . + +docker build \ + --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" \ + -t "${TAG}" . diff --git a/script/build/linux b/script/build/linux index 8de7218db..28065da08 100755 --- a/script/build/linux +++ b/script/build/linux @@ -4,15 +4,14 @@ set -ex ./script/clean -TMP_CONTAINER="tmpcontainer" -TAG="docker/compose:tmp-glibc-linux-binary" -DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +TAG="docker/compose:tmp-glibc-linux-binary-${DOCKER_COMPOSE_GITSHA}" docker build -t "${TAG}" . \ --build-arg BUILD_PLATFORM=debian \ - --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} -docker create --name ${TMP_CONTAINER} ${TAG} + --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" +TMP_CONTAINER=$(docker create "${TAG}") mkdir -p dist -docker cp ${TMP_CONTAINER}:/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 -docker container rm -f ${TMP_CONTAINER} -docker image rm -f ${TAG} +docker cp "${TMP_CONTAINER}":/usr/local/bin/docker-compose dist/docker-compose-Linux-x86_64 +docker container rm -f "${TMP_CONTAINER}" +docker image rm -f "${TAG}" diff --git a/script/build/linux-entrypoint b/script/build/linux-entrypoint index 1556bbf20..1c5438d8e 100755 --- a/script/build/linux-entrypoint +++ b/script/build/linux-entrypoint @@ -3,16 +3,19 @@ set -ex CODE_PATH=/code -VENV=${CODE_PATH}/.tox/py37 +VENV="${CODE_PATH}"/.tox/py37 -cd ${CODE_PATH} +cd "${CODE_PATH}" mkdir -p dist chmod 777 dist -${VENV}/bin/pip3 install -q -r requirements-build.txt +"${VENV}"/bin/pip3 install -q -r requirements-build.txt # TODO(ulyssessouza) To check if really needed -./script/build/write-git-sha +if [ -z "${DOCKER_COMPOSE_GITSHA}" ]; then + DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +fi +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA export PATH="${CODE_PATH}/pyinstaller:${PATH}" @@ -21,15 +24,15 @@ if [ ! -z "${BUILD_BOOTLOADER}" ]; then git clone --single-branch --branch master https://github.com/pyinstaller/pyinstaller.git /tmp/pyinstaller cd /tmp/pyinstaller/bootloader git checkout v3.4 - ${VENV}/bin/python3 ./waf configure --no-lsb all - ${VENV}/bin/pip3 install .. - cd ${CODE_PATH} + "${VENV}"/bin/python3 ./waf configure --no-lsb all + "${VENV}"/bin/pip3 install .. + cd "${CODE_PATH}" rm -Rf /tmp/pyinstaller else echo "NOT compiling bootloader!!!" fi -${VENV}/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec +"${VENV}"/bin/pyinstaller --exclude-module pycrypto --exclude-module PyInstaller docker-compose.spec ls -la dist/ ldd dist/docker-compose mv dist/docker-compose /usr/local/bin diff --git a/script/build/osx b/script/build/osx index c62b27024..529914586 100755 --- a/script/build/osx +++ b/script/build/osx @@ -5,11 +5,12 @@ TOOLCHAIN_PATH="$(realpath $(dirname $0)/../../build/toolchain)" rm -rf venv -virtualenv -p ${TOOLCHAIN_PATH}/bin/python3 venv +virtualenv -p "${TOOLCHAIN_PATH}"/bin/python3 venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . -./script/build/write-git-sha +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version diff --git a/script/build/test-image b/script/build/test-image index 9d880c27f..4964a5f9d 100755 --- a/script/build/test-image +++ b/script/build/test-image @@ -10,9 +10,9 @@ fi TAG="$1" IMAGE="docker/compose-tests" -DOCKER_COMPOSE_GITSHA=$(script/build/write-git-sha) +DOCKER_COMPOSE_GITSHA="$(script/build/write-git-sha)" docker build -t "${IMAGE}:${TAG}" . \ --target build \ - --build-arg BUILD_PLATFORM=debian \ - --build-arg GIT_COMMIT=${DOCKER_COMPOSE_GITSHA} -docker tag ${IMAGE}:${TAG} ${IMAGE}:latest + --build-arg BUILD_PLATFORM="debian" \ + --build-arg GIT_COMMIT="${DOCKER_COMPOSE_GITSHA}" +docker tag "${IMAGE}":"${TAG}" "${IMAGE}":latest diff --git a/script/build/write-git-sha b/script/build/write-git-sha index be87f5058..cac4b6fd3 100755 --- a/script/build/write-git-sha +++ b/script/build/write-git-sha @@ -9,4 +9,4 @@ if [[ "${?}" != "0" ]]; then echo "Couldn't get revision of the git repository. Setting to 'unknown' instead" DOCKER_COMPOSE_GITSHA="unknown" fi -echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA +echo "${DOCKER_COMPOSE_GITSHA}" diff --git a/script/release/release.py b/script/release/release.py index 9db1a49d9..9fdd92dae 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -204,7 +204,7 @@ def resume(args): delete_assets(gh_release) upload_assets(gh_release, files) img_manager = ImageManager(args.release) - img_manager.build_images(repository, files) + img_manager.build_images(repository) except ScriptError as e: print(e) return 1 @@ -244,7 +244,7 @@ def start(args): gh_release = create_release_draft(repository, args.release, pr_data, files) upload_assets(gh_release, files) img_manager = ImageManager(args.release) - img_manager.build_images(repository, files) + img_manager.build_images(repository) except ScriptError as e: print(e) return 1 @@ -258,7 +258,8 @@ def finalize(args): try: check_pypirc() repository = Repository(REPO_ROOT, args.repo) - img_manager = ImageManager(args.release) + tag_as_latest = _check_if_tag_latest(args.release) + img_manager = ImageManager(args.release, tag_as_latest) pr_data = repository.find_release_pr(args.release) if not pr_data: raise ScriptError('No PR found for {}'.format(args.release)) @@ -314,6 +315,12 @@ EPILOG = '''Example uses: ''' +# Checks if this version respects the GA version format ('x.y.z') and not an RC +def _check_if_tag_latest(version): + ga_version = all(n.isdigit() for n in version.split('.')) and version.count('.') == 2 + return ga_version and yesno('Should this release be tagged as \"latest\"? Y/n', default=True) + + def main(): if 'GITHUB_TOKEN' not in os.environ: print('GITHUB_TOKEN environment variable must be set') diff --git a/script/release/release/images.py b/script/release/release/images.py index df6eeda4f..796a4d825 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -5,18 +5,29 @@ from __future__ import unicode_literals import base64 import json import os -import shutil import docker +from enum import Enum +from .const import NAME from .const import REPO_ROOT from .utils import ScriptError +class Platform(Enum): + ALPINE = 'alpine' + DEBIAN = 'debian' + + def __str__(self): + return self.value + + class ImageManager(object): - def __init__(self, version): + def __init__(self, version, latest=False): + self.built_tags = [] self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.version = version + self.latest = latest if 'HUB_CREDENTIALS' in os.environ: print('HUB_CREDENTIALS found in environment, issuing login') credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS'])) @@ -24,16 +35,31 @@ class ImageManager(object): username=credentials['Username'], password=credentials['Password'] ) - def build_images(self, repository, files): - print("Building release images...") - repository.write_git_sha() - distdir = os.path.join(REPO_ROOT, 'dist') - os.makedirs(distdir, exist_ok=True) - shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) - os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755) - print('Building docker/compose image') + def _tag(self, image, existing_tag, new_tag): + existing_repo_tag = '{image}:{tag}'.format(image=image, tag=existing_tag) + new_repo_tag = '{image}:{tag}'.format(image=image, tag=new_tag) + self.docker_client.tag(existing_repo_tag, new_repo_tag) + self.built_tags.append(new_repo_tag) + + def build_runtime_image(self, repository, platform): + git_sha = repository.write_git_sha() + compose_image_base_name = NAME + print('Building {image} image ({platform} based)'.format( + image=compose_image_base_name, + platform=platform + )) + full_version = '{version}-{platform}'.format(version=self.version, platform=platform) + build_tag = '{image_base_image}:{full_version}'.format( + image_base_image=compose_image_base_name, + full_version=full_version + ) logstream = self.docker_client.build( - REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', + REPO_ROOT, + tag=build_tag, + buildargs={ + 'BUILD_PLATFORM': platform.value, + 'GIT_COMMIT': git_sha, + }, decode=True ) for chunk in logstream: @@ -42,9 +68,32 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - print('Building test image (for UCP e2e)') + self.built_tags.append(build_tag) + if platform == Platform.ALPINE: + self._tag(compose_image_base_name, full_version, self.version) + if self.latest: + self._tag(compose_image_base_name, full_version, platform) + if platform == Platform.ALPINE: + self._tag(compose_image_base_name, full_version, 'latest') + + # Used for producing a test image for UCP + def build_ucp_test_image(self, repository): + print('Building test image (debian based for UCP e2e)') + git_sha = repository.write_git_sha() + compose_tests_image_base_name = NAME + '-tests' + ucp_test_image_tag = '{image}:{tag}'.format( + image=compose_tests_image_base_name, + tag=self.version + ) logstream = self.docker_client.build( - REPO_ROOT, tag='docker-compose-tests:tmp', decode=True + REPO_ROOT, + tag=ucp_test_image_tag, + target='build', + buildargs={ + 'BUILD_PLATFORM': Platform.DEBIAN.value, + 'GIT_COMMIT': git_sha, + }, + decode=True ) for chunk in logstream: if 'error' in chunk: @@ -52,26 +101,16 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - container = self.docker_client.create_container( - 'docker-compose-tests:tmp', entrypoint='tox' - ) - self.docker_client.commit(container, 'docker/compose-tests', 'latest') - self.docker_client.tag( - 'docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version) - ) - self.docker_client.remove_container(container, force=True) - self.docker_client.remove_image('docker-compose-tests:tmp', force=True) + self.built_tags.append(ucp_test_image_tag) + self._tag(compose_tests_image_base_name, self.version, 'latest') - @property - def image_names(self): - return [ - 'docker/compose-tests:latest', - 'docker/compose-tests:{}'.format(self.version), - 'docker/compose:{}'.format(self.version) - ] + def build_images(self, repository): + self.build_runtime_image(repository, Platform.ALPINE) + self.build_runtime_image(repository, Platform.DEBIAN) + self.build_ucp_test_image(repository) def check_images(self): - for name in self.image_names: + for name in self.built_tags: try: self.docker_client.inspect_image(name) except docker.errors.ImageNotFound: @@ -80,7 +119,7 @@ class ImageManager(object): return True def push_images(self): - for name in self.image_names: + for name in self.built_tags: print('Pushing {} to Docker Hub'.format(name)) logstream = self.docker_client.push(name, stream=True, decode=True) for chunk in logstream: diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 9a5d432c0..0dc724f80 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -175,6 +175,7 @@ class Repository(object): def write_git_sha(self): with open(os.path.join(REPO_ROOT, 'compose', 'GITSHA'), 'w') as f: f.write(self.git_repo.head.commit.hexsha[:7]) + return self.git_repo.head.commit.hexsha[:7] def cherry_pick_prs(self, release_branch, ids): if not ids: diff --git a/script/run/run.sh b/script/run/run.sh index a8690cad1..f3456720f 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -48,7 +48,7 @@ fi # Only allocate tty if we detect one if [ -t 0 -a -t 1 ]; then - DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" + DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" fi # Always set -i to support piped and terminal input in run/exec diff --git a/script/test/all b/script/test/all index 5c911bba4..f929a57ee 100755 --- a/script/test/all +++ b/script/test/all @@ -8,8 +8,7 @@ set -e docker run --rm \ --tty \ ${GIT_VOLUME} \ - --entrypoint="tox" \ - "$TAG" -e pre-commit + "$TAG" tox -e pre-commit get_versions="docker run --rm --entrypoint=/code/.tox/py27/bin/python diff --git a/script/test/default b/script/test/default index d24b41b0d..4d973d1d0 100755 --- a/script/test/default +++ b/script/test/default @@ -5,16 +5,16 @@ set -ex TAG="docker-compose:alpine-$(git rev-parse --short HEAD)" -# By default use the Dockerfile.alpine, but can be overridden to use an alternative file +# By default use the Dockerfile, but can be overridden to use an alternative file # e.g DOCKERFILE=Dockerfile.armhf script/test/default -DOCKERFILE="${DOCKERFILE:-Dockerfile.alpine}" +DOCKERFILE="${DOCKERFILE:-Dockerfile}" DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}" rm -rf coverage-html # Create the host directory so it's owned by $USER mkdir -p coverage-html -docker build -f ${DOCKERFILE} -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" . +docker build -f "${DOCKERFILE}" -t "${TAG}" --target "${DOCKER_BUILD_TARGET}" . GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" . script/test/all From e047169315d3ca7fa62de8a4bba436a058aa3e94 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Mon, 15 Apr 2019 14:14:38 +0200 Subject: [PATCH 48/62] Workaround race conditions on tests Signed-off-by: Ulysses Souza --- tests/acceptance/cli_test.py | 2 ++ tests/fixtures/logs-composefile/docker-compose.yml | 4 ++-- tests/fixtures/logs-restart-composefile/docker-compose.yml | 2 +- 3 files changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index c15a99e09..8c8286f5d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2347,6 +2347,7 @@ class CLITestCase(DockerClientTestCase): assert 'another' in result.stdout assert 'exited with code 0' in result.stdout + @pytest.mark.skip(reason="race condition between up and logs") def test_logs_follow_logs_from_new_containers(self): self.base_dir = 'tests/fixtures/logs-composefile' self.dispatch(['up', '-d', 'simple']) @@ -2393,6 +2394,7 @@ class CLITestCase(DockerClientTestCase): ) == 3 assert result.stdout.count('world') == 3 + @pytest.mark.skip(reason="race condition between up and logs") def test_logs_default(self): self.base_dir = 'tests/fixtures/logs-composefile' self.dispatch(['up', '-d']) diff --git a/tests/fixtures/logs-composefile/docker-compose.yml b/tests/fixtures/logs-composefile/docker-compose.yml index b719c91e0..ea18f162d 100644 --- a/tests/fixtures/logs-composefile/docker-compose.yml +++ b/tests/fixtures/logs-composefile/docker-compose.yml @@ -1,6 +1,6 @@ simple: image: busybox:latest - command: sh -c "echo hello && tail -f /dev/null" + command: sh -c "sleep 1 && echo hello && tail -f /dev/null" another: image: busybox:latest - command: sh -c "echo test" + command: sh -c "sleep 1 && echo test" diff --git a/tests/fixtures/logs-restart-composefile/docker-compose.yml b/tests/fixtures/logs-restart-composefile/docker-compose.yml index c662a1e71..6be8b9079 100644 --- a/tests/fixtures/logs-restart-composefile/docker-compose.yml +++ b/tests/fixtures/logs-restart-composefile/docker-compose.yml @@ -3,5 +3,5 @@ simple: command: sh -c "echo hello && tail -f /dev/null" another: image: busybox:latest - command: sh -c "sleep 0.5 && echo world && /bin/false" + command: sh -c "sleep 2 && echo world && /bin/false" restart: "on-failure:2" From f2dc9230843020a70d5bf99166ada325f4e1cb79 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 19 Apr 2019 15:53:02 +0200 Subject: [PATCH 49/62] Avoid race condition on test Signed-off-by: Ulysses Souza --- tests/acceptance/cli_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8c8286f5d..7f45fc199 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2374,6 +2374,7 @@ class CLITestCase(DockerClientTestCase): assert '{} exited with code 0'.format(another_name) in result.stdout assert '{} exited with code 137'.format(simple_name) in result.stdout + @pytest.mark.skip(reason="race condition between up and logs") def test_logs_follow_logs_from_restarted_containers(self): self.base_dir = 'tests/fixtures/logs-restart-composefile' proc = start_process(self.base_dir, ['up']) From 482bca9519af4947e8d61396d37483ee0be62108 Mon Sep 17 00:00:00 2001 From: Joakim Roubert Date: Tue, 23 Apr 2019 09:52:35 +0200 Subject: [PATCH 50/62] Purge Dockerfile.armhf which is no longer needed Current Dockerfile builds fine for armhf and thus the outdated Dockerfile.armhf is unnecessary. Change-Id: Idafdb9fbddedd622c2c0aaddb1d5331d81cfe57d Signed-off-by: Joakim Roubert --- Dockerfile.armhf | 39 --------------------------------------- script/test/default | 2 +- 2 files changed, 1 insertion(+), 40 deletions(-) delete mode 100644 Dockerfile.armhf diff --git a/Dockerfile.armhf b/Dockerfile.armhf deleted file mode 100644 index 9c02dbc76..000000000 --- a/Dockerfile.armhf +++ /dev/null @@ -1,39 +0,0 @@ -FROM python:3.7.2-stretch - -RUN set -ex; \ - apt-get update -qq; \ - apt-get install -y \ - locales \ - curl \ - python-dev \ - git - -RUN curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/armhf/docker-17.12.0-ce.tgz" && \ - SHA256=f8de6378dad825b9fd5c3c2f949e791d22f918623c27a72c84fd6975a0e5d0a2; \ - echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ - tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ - mv docker /usr/local/bin/docker && \ - chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz - -# Python3 requires a valid locale -RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen -ENV LANG en_US.UTF-8 - -RUN useradd -d /home/user -m -s /bin/bash user -WORKDIR /code/ - -RUN pip install tox==2.1.1 - -ADD requirements.txt /code/ -ADD requirements-dev.txt /code/ -ADD .pre-commit-config.yaml /code/ -ADD setup.py /code/ -ADD tox.ini /code/ -ADD compose /code/compose/ -RUN tox --notest - -ADD . /code/ -RUN chown -R user /code/ - -ENTRYPOINT ["/code/.tox/py37/bin/docker-compose"] diff --git a/script/test/default b/script/test/default index 4d973d1d0..4f307f2e9 100755 --- a/script/test/default +++ b/script/test/default @@ -6,7 +6,7 @@ set -ex TAG="docker-compose:alpine-$(git rev-parse --short HEAD)" # By default use the Dockerfile, but can be overridden to use an alternative file -# e.g DOCKERFILE=Dockerfile.armhf script/test/default +# e.g DOCKERFILE=Dockerfile.s390x script/test/default DOCKERFILE="${DOCKERFILE:-Dockerfile}" DOCKER_BUILD_TARGET="${DOCKER_BUILD_TARGET:-build}" From 3a47000e71c98a0916d735215e12efd778f7b737 Mon Sep 17 00:00:00 2001 From: ulyssessouza Date: Sun, 12 May 2019 13:33:38 +0200 Subject: [PATCH 51/62] Bump urllib3 Signed-off-by: ulyssessouza --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 5e8ec2ed7..f2bfe6e1a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,5 +20,5 @@ PyYAML==4.2b1 requests==2.20.0 six==1.10.0 texttable==0.9.1 -urllib3==1.21.1; python_version == '3.3' +urllib3==1.24.2; python_version == '3.3' websocket-client==0.32.0 From c2783d6f886db0f14c40dc07ea52f685dbe763d7 Mon Sep 17 00:00:00 2001 From: noname Date: Thu, 18 Apr 2019 00:15:29 -0700 Subject: [PATCH 52/62] fix #6579 cli ps --all Signed-off-by: Seedf --- compose/cli/main.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 78f596753..443435b10 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -721,7 +721,8 @@ class TopLevelCommand(object): if options['--all']: containers = sorted(self.project.containers(service_names=options['SERVICE'], - one_off=OneOffFilter.include, stopped=True)) + one_off=OneOffFilter.include, stopped=True), + key=attrgetter('name')) else: containers = sorted( self.project.containers(service_names=options['SERVICE'], stopped=True) + From 8a9575bd0d6ca19f67f89c2881d950b40a93a83a Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Tue, 14 May 2019 16:53:52 +0200 Subject: [PATCH 53/62] Remove remaining containers on test_build_run Signed-off-by: Ulysses Souza --- tests/acceptance/cli_test.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 7f45fc199..49b1d0021 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -793,6 +793,9 @@ class CLITestCase(DockerClientTestCase): ] assert containers + for c in self.project.client.containers(all=True): + self.addCleanup(self.project.client.remove_container, c, force=True) + def test_build_shm_size_build_option(self): pull_busybox(self.client) self.base_dir = 'tests/fixtures/build-shm-size' From 99e67d0c061fa3d9b9793391f3b7c8bdf8e841fc Mon Sep 17 00:00:00 2001 From: Inconnu08 Date: Wed, 15 May 2019 23:45:40 +0600 Subject: [PATCH 54/62] fix warning method is deprecated with tests Signed-off-by: Taufiq Rahman --- compose/bundle.py | 8 +++---- compose/cli/docker_client.py | 2 +- compose/cli/main.py | 12 +++++----- compose/config/config.py | 14 +++++------ compose/config/environment.py | 2 +- compose/network.py | 6 ++--- compose/project.py | 8 +++---- compose/service.py | 24 +++++++++---------- compose/volume.py | 4 ++-- .../migrate-compose-file-v1-to-v2.py | 6 ++--- tests/integration/service_test.py | 6 ++--- tests/unit/bundle_test.py | 8 +++---- tests/unit/cli/docker_client_test.py | 2 +- tests/unit/cli/main_test.py | 2 +- tests/unit/config/config_test.py | 12 +++++----- tests/unit/network_test.py | 4 ++-- tests/unit/service_test.py | 20 ++++++++-------- 17 files changed, 70 insertions(+), 70 deletions(-) diff --git a/compose/bundle.py b/compose/bundle.py index efc455b72..77cb37aa9 100644 --- a/compose/bundle.py +++ b/compose/bundle.py @@ -164,10 +164,10 @@ def push_image(service): def to_bundle(config, image_digests): if config.networks: - log.warn("Unsupported top level key 'networks' - ignoring") + log.warning("Unsupported top level key 'networks' - ignoring") if config.volumes: - log.warn("Unsupported top level key 'volumes' - ignoring") + log.warning("Unsupported top level key 'volumes' - ignoring") config = denormalize_config(config) @@ -192,7 +192,7 @@ def convert_service_to_bundle(name, service_dict, image_digest): continue if key not in SUPPORTED_KEYS: - log.warn("Unsupported key '{}' in services.{} - ignoring".format(key, name)) + log.warning("Unsupported key '{}' in services.{} - ignoring".format(key, name)) continue if key == 'environment': @@ -239,7 +239,7 @@ def make_service_networks(name, service_dict): for network_name, network_def in get_network_defs_for_service(service_dict).items(): for key in network_def.keys(): - log.warn( + log.warning( "Unsupported key '{}' in services.{}.networks.{} - ignoring" .format(key, name, network_name)) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index a01704fd2..a57a69b50 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -31,7 +31,7 @@ def get_tls_version(environment): tls_attr_name = "PROTOCOL_{}".format(compose_tls_version) if not hasattr(ssl, tls_attr_name): - log.warn( + log.warning( 'The "{}" protocol is unavailable. You may need to update your ' 'version of Python or OpenSSL. Falling back to TLSv1 (default).' .format(compose_tls_version) diff --git a/compose/cli/main.py b/compose/cli/main.py index 78f596753..d697dc3b1 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -391,7 +391,7 @@ class TopLevelCommand(object): """ service_names = options['SERVICE'] - log.warn( + log.warning( 'The create command is deprecated. ' 'Use the up command with the --no-start flag instead.' ) @@ -765,7 +765,7 @@ class TopLevelCommand(object): --include-deps Also pull services declared as dependencies """ if options.get('--parallel'): - log.warn('--parallel option is deprecated and will be removed in future versions.') + log.warning('--parallel option is deprecated and will be removed in future versions.') self.project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures'), @@ -806,7 +806,7 @@ class TopLevelCommand(object): -a, --all Deprecated - no effect. """ if options.get('--all'): - log.warn( + log.warning( '--all flag is obsolete. This is now the default behavior ' 'of `docker-compose rm`' ) @@ -916,7 +916,7 @@ class TopLevelCommand(object): 'Use the up command with the --scale flag instead.' ) else: - log.warn( + log.warning( 'The scale command is deprecated. ' 'Use the up command with the --scale flag instead.' ) @@ -1250,7 +1250,7 @@ def exitval_from_opts(options, project): exit_value_from = options.get('--exit-code-from') if exit_value_from: if not options.get('--abort-on-container-exit'): - log.warn('using --exit-code-from implies --abort-on-container-exit') + log.warning('using --exit-code-from implies --abort-on-container-exit') options['--abort-on-container-exit'] = True if exit_value_from not in [s.name for s in project.get_services()]: log.error('No service named "%s" was found in your compose file.', @@ -1580,7 +1580,7 @@ def warn_for_swarm_mode(client): # UCP does multi-node scheduling with traditional Compose files. return - log.warn( + log.warning( "The Docker Engine you're using is running in swarm mode.\n\n" "Compose does not use swarm mode to deploy services to multiple nodes in a swarm. " "All containers will be scheduled on the current node.\n\n" diff --git a/compose/config/config.py b/compose/config/config.py index c110e2cfa..5202d0025 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -198,9 +198,9 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): version = self.config['version'] if isinstance(version, dict): - log.warn('Unexpected type for "version" key in "{}". Assuming ' - '"version" is the name of a service, and defaulting to ' - 'Compose file version 1.'.format(self.filename)) + log.warning('Unexpected type for "version" key in "{}". Assuming ' + '"version" is the name of a service, and defaulting to ' + 'Compose file version 1.'.format(self.filename)) return V1 if not isinstance(version, six.string_types): @@ -318,8 +318,8 @@ def get_default_config_files(base_dir): winner = candidates[0] if len(candidates) > 1: - log.warn("Found multiple config files with supported names: %s", ", ".join(candidates)) - log.warn("Using %s\n", winner) + log.warning("Found multiple config files with supported names: %s", ", ".join(candidates)) + log.warning("Using %s\n", winner) return [os.path.join(path, winner)] + get_default_override_file(path) @@ -362,7 +362,7 @@ def check_swarm_only_config(service_dicts, compatibility=False): def check_swarm_only_key(service_dicts, key): services = [s for s in service_dicts if s.get(key)] if services: - log.warn( + log.warning( warning_template.format( services=", ".join(sorted(s['name'] for s in services)), key=key @@ -921,7 +921,7 @@ def finalize_service(service_config, service_names, version, environment, compat service_dict ) if ignored_keys: - log.warn( + log.warning( 'The following deploy sub-keys are not supported in compatibility mode and have' ' been ignored: {}'.format(', '.join(ignored_keys)) ) diff --git a/compose/config/environment.py b/compose/config/environment.py index e2db343d7..e72c88231 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -100,7 +100,7 @@ class Environment(dict): except KeyError: pass if not self.silent and key not in self.missing_keys: - log.warn( + log.warning( "The {} variable is not set. Defaulting to a blank string." .format(key) ) diff --git a/compose/network.py b/compose/network.py index 2491a5989..e0d711ff7 100644 --- a/compose/network.py +++ b/compose/network.py @@ -231,7 +231,7 @@ def check_remote_network_config(remote, local): if k.startswith('com.docker.'): # We are only interested in user-specified labels continue if remote_labels.get(k) != local_labels.get(k): - log.warn( + log.warning( 'Network {}: label "{}" has changed. It may need to be' ' recreated.'.format(local.true_name, k) ) @@ -276,7 +276,7 @@ class ProjectNetworks(object): } unused = set(networks) - set(service_networks) - {'default'} if unused: - log.warn( + log.warning( "Some networks were defined but are not used by any service: " "{}".format(", ".join(unused))) return cls(service_networks, use_networking) @@ -288,7 +288,7 @@ class ProjectNetworks(object): try: network.remove() except NotFound: - log.warn("Network %s not found.", network.true_name) + log.warning("Network %s not found.", network.true_name) def initialize(self): if not self.use_networking: diff --git a/compose/project.py b/compose/project.py index 6965c6251..996c4cbac 100644 --- a/compose/project.py +++ b/compose/project.py @@ -776,13 +776,13 @@ def get_secrets(service, service_secrets, secret_defs): .format(service=service, secret=secret.source)) if secret_def.get('external'): - log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " - "External secrets are not available to containers created by " - "docker-compose.".format(service=service, secret=secret.source)) + log.warning("Service \"{service}\" uses secret \"{secret}\" which is external. " + "External secrets are not available to containers created by " + "docker-compose.".format(service=service, secret=secret.source)) continue if secret.uid or secret.gid or secret.mode: - log.warn( + log.warning( "Service \"{service}\" uses secret \"{secret}\" with uid, " "gid, or mode. These fields are not supported by this " "implementation of the Compose file".format( diff --git a/compose/service.py b/compose/service.py index e989d4877..af9b10baf 100644 --- a/compose/service.py +++ b/compose/service.py @@ -240,15 +240,15 @@ class Service(object): def show_scale_warnings(self, desired_num): if self.custom_container_name and desired_num > 1: - log.warn('The "%s" service is using the custom container name "%s". ' - 'Docker requires each container to have a unique name. ' - 'Remove the custom name to scale the service.' - % (self.name, self.custom_container_name)) + log.warning('The "%s" service is using the custom container name "%s". ' + 'Docker requires each container to have a unique name. ' + 'Remove the custom name to scale the service.' + % (self.name, self.custom_container_name)) if self.specifies_host_port() and desired_num > 1: - log.warn('The "%s" service specifies a port on the host. If multiple containers ' - 'for this service are created on a single host, the port will clash.' - % self.name) + log.warning('The "%s" service specifies a port on the host. If multiple containers ' + 'for this service are created on a single host, the port will clash.' + % self.name) def scale(self, desired_num, timeout=None): """ @@ -357,7 +357,7 @@ class Service(object): raise NeedsBuildError(self) self.build() - log.warn( + log.warning( "Image for service {} was built because it did not already exist. To " "rebuild this image you must use `docker-compose build` or " "`docker-compose up --build`.".format(self.name)) @@ -1325,7 +1325,7 @@ class ServicePidMode(PidMode): if containers: return 'container:' + containers[0].id - log.warn( + log.warning( "Service %s is trying to use reuse the PID namespace " "of another service that is not running." % (self.service_name) ) @@ -1388,8 +1388,8 @@ class ServiceNetworkMode(object): if containers: return 'container:' + containers[0].id - log.warn("Service %s is trying to use reuse the network stack " - "of another service that is not running." % (self.id)) + log.warning("Service %s is trying to use reuse the network stack " + "of another service that is not running." % (self.id)) return None @@ -1540,7 +1540,7 @@ def warn_on_masked_volume(volumes_option, container_volumes, service): volume.internal in container_volumes and container_volumes.get(volume.internal) != volume.external ): - log.warn(( + log.warning(( "Service \"{service}\" is using volume \"{volume}\" from the " "previous container. Host mapping \"{host_path}\" has no effect. " "Remove the existing containers (with `docker-compose rm {service}`) " diff --git a/compose/volume.py b/compose/volume.py index 60c1e0fe8..b02fc5d80 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -127,7 +127,7 @@ class ProjectVolumes(object): try: volume.remove() except NotFound: - log.warn("Volume %s not found.", volume.true_name) + log.warning("Volume %s not found.", volume.true_name) def initialize(self): try: @@ -209,7 +209,7 @@ def check_remote_volume_config(remote, local): if k.startswith('com.docker.'): # We are only interested in user-specified labels continue if remote_labels.get(k) != local_labels.get(k): - log.warn( + log.warning( 'Volume {}: label "{}" has changed. It may need to be' ' recreated.'.format(local.name, k) ) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py index c1785b0da..274b499b9 100755 --- a/contrib/migration/migrate-compose-file-v1-to-v2.py +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -44,7 +44,7 @@ def warn_for_links(name, service): links = service.get('links') if links: example_service = links[0].partition(':')[0] - log.warn( + log.warning( "Service {name} has links, which no longer create environment " "variables such as {example_service_upper}_PORT. " "If you are using those in your application code, you should " @@ -57,7 +57,7 @@ def warn_for_links(name, service): def warn_for_external_links(name, service): external_links = service.get('external_links') if external_links: - log.warn( + log.warning( "Service {name} has external_links: {ext}, which now work " "slightly differently. In particular, two containers must be " "connected to at least one network in common in order to " @@ -107,7 +107,7 @@ def rewrite_volumes_from(service, service_names): def create_volumes_section(data): named_volumes = get_named_volumes(data['services']) if named_volumes: - log.warn( + log.warning( "Named volumes ({names}) must be explicitly declared. Creating a " "'volumes' section with declarations.\n\n" "For backwards-compatibility, they've been declared as external. " diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 000f6838c..b49ae7106 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -695,8 +695,8 @@ class ServiceTest(DockerClientTestCase): new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) - mock_log.warn.assert_called_once_with(mock.ANY) - _, args, kwargs = mock_log.warn.mock_calls[0] + mock_log.warning.assert_called_once_with(mock.ANY) + _, args, kwargs = mock_log.warning.mock_calls[0] assert "Service \"db\" is using volume \"/data\" from the previous container" in args[0] assert [mount['Destination'] for mount in new_container.get('Mounts')] == ['/data'] @@ -1382,7 +1382,7 @@ class ServiceTest(DockerClientTestCase): with pytest.raises(OperationFailedError): service.scale(3) - captured_output = mock_log.warn.call_args[0][0] + captured_output = mock_log.warning.call_args[0][0] assert len(service.containers()) == 1 assert "Remove the custom name to scale the service." in captured_output diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index 065cdd00b..8faebb7f1 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -94,7 +94,7 @@ def test_to_bundle(): configs={} ) - with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log: output = bundle.to_bundle(config, image_digests) assert mock_log.mock_calls == [ @@ -128,7 +128,7 @@ def test_convert_service_to_bundle(): 'privileged': True, } - with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log: config = bundle.convert_service_to_bundle(name, service_dict, image_digest) mock_log.assert_called_once_with( @@ -177,7 +177,7 @@ def test_make_service_networks_default(): name = 'theservice' service_dict = {} - with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log: networks = bundle.make_service_networks(name, service_dict) assert not mock_log.called @@ -195,7 +195,7 @@ def test_make_service_networks(): }, } - with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + with mock.patch('compose.bundle.log.warning', autospec=True) as mock_log: networks = bundle.make_service_networks(name, service_dict) mock_log.assert_called_once_with( diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index be91ea31d..772c136ee 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -247,5 +247,5 @@ class TestGetTlsVersion(object): environment = {'COMPOSE_TLS_VERSION': 'TLSv5_5'} with mock.patch('compose.cli.docker_client.log') as mock_log: tls_version = get_tls_version(environment) - mock_log.warn.assert_called_once_with(mock.ANY) + mock_log.warning.assert_called_once_with(mock.ANY) assert tls_version is None diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index 2e97f2c87..eb6a99d72 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -63,7 +63,7 @@ class TestCLIMainTestCase(object): with mock.patch('compose.cli.main.log') as fake_log: warn_for_swarm_mode(mock_client) - assert fake_log.warn.call_count == 1 + assert fake_log.warning.call_count == 1 class TestSetupConsoleHandlerTestCase(object): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e27427ca5..163c9cd16 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -329,7 +329,7 @@ class ConfigTest(unittest.TestCase): ) assert 'Unexpected type for "version" key in "filename.yml"' \ - in mock_logging.warn.call_args[0][0] + in mock_logging.warning.call_args[0][0] service_dicts = config_data.services assert service_sort(service_dicts) == service_sort([ @@ -3570,8 +3570,8 @@ class InterpolationTest(unittest.TestCase): with mock.patch('compose.config.environment.log') as log: config.load(config_details) - assert 2 == log.warn.call_count - warnings = sorted(args[0][0] for args in log.warn.call_args_list) + assert 2 == log.warning.call_count + warnings = sorted(args[0][0] for args in log.warning.call_args_list) assert 'BAR' in warnings[0] assert 'FOO' in warnings[1] @@ -3601,8 +3601,8 @@ class InterpolationTest(unittest.TestCase): with mock.patch('compose.config.config.log') as log: config.load(config_details, compatibility=True) - assert log.warn.call_count == 1 - warn_message = log.warn.call_args[0][0] + assert log.warning.call_count == 1 + warn_message = log.warning.call_args[0][0] assert warn_message.startswith( 'The following deploy sub-keys are not supported in compatibility mode' ) @@ -3641,7 +3641,7 @@ class InterpolationTest(unittest.TestCase): with mock.patch('compose.config.config.log') as log: cfg = config.load(config_details, compatibility=True) - assert log.warn.call_count == 0 + assert log.warning.call_count == 0 service_dict = cfg.services[0] assert service_dict == { diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index d7ffa2894..82cfb3be2 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -165,6 +165,6 @@ class NetworkTest(unittest.TestCase): with mock.patch('compose.network.log') as mock_log: check_remote_network_config(remote, net) - mock_log.warn.assert_called_once_with(mock.ANY) - _, args, kwargs = mock_log.warn.mock_calls[0] + mock_log.warning.assert_called_once_with(mock.ANY) + _, args, kwargs = mock_log.warning.mock_calls[0] assert 'label "com.project.touhou.character" has changed' in args[0] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 3d7c4987a..8c381f15d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -516,8 +516,8 @@ class ServiceTest(unittest.TestCase): with mock.patch('compose.service.log', autospec=True) as mock_log: service.create_container() - assert mock_log.warn.called - _, args, _ = mock_log.warn.mock_calls[0] + assert mock_log.warning.called + _, args, _ = mock_log.warning.mock_calls[0] assert 'was built because it did not already exist' in args[0] assert self.mock_client.build.call_count == 1 @@ -546,7 +546,7 @@ class ServiceTest(unittest.TestCase): with mock.patch('compose.service.log', autospec=True) as mock_log: service.ensure_image_exists(do_build=BuildAction.force) - assert not mock_log.warn.called + assert not mock_log.warning.called assert self.mock_client.build.call_count == 1 self.mock_client.build.call_args[1]['tag'] == 'default_foo' @@ -847,13 +847,13 @@ class ServiceTest(unittest.TestCase): ports=["8080:80"]) service.scale(0) - assert not mock_log.warn.called + assert not mock_log.warning.called service.scale(1) - assert not mock_log.warn.called + assert not mock_log.warning.called service.scale(2) - mock_log.warn.assert_called_once_with( + mock_log.warning.assert_called_once_with( 'The "{}" service specifies a port on the host. If multiple containers ' 'for this service are created on a single host, the port will clash.'.format(name)) @@ -1391,7 +1391,7 @@ class ServiceVolumesTest(unittest.TestCase): with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) - assert not mock_log.warn.called + assert not mock_log.warning.called def test_warn_on_masked_volume_when_masked(self): volumes_option = [VolumeSpec('/home/user', '/path', 'rw')] @@ -1404,7 +1404,7 @@ class ServiceVolumesTest(unittest.TestCase): with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) - mock_log.warn.assert_called_once_with(mock.ANY) + mock_log.warning.assert_called_once_with(mock.ANY) def test_warn_on_masked_no_warning_with_same_path(self): volumes_option = [VolumeSpec('/home/user', '/path', 'rw')] @@ -1414,7 +1414,7 @@ class ServiceVolumesTest(unittest.TestCase): with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) - assert not mock_log.warn.called + assert not mock_log.warning.called def test_warn_on_masked_no_warning_with_container_only_option(self): volumes_option = [VolumeSpec(None, '/path', 'rw')] @@ -1426,7 +1426,7 @@ class ServiceVolumesTest(unittest.TestCase): with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) - assert not mock_log.warn.called + assert not mock_log.warning.called def test_create_with_special_volume_mode(self): self.mock_client.inspect_image.return_value = {'Id': 'imageid'} From 51ee6093df9947b4e77ff679d9d587b1d0164fa2 Mon Sep 17 00:00:00 2001 From: Nao YONASHIRO Date: Mon, 1 Apr 2019 19:09:10 +0900 Subject: [PATCH 55/62] feat: drop empty tag on cache_from Signed-off-by: Nao YONASHIRO --- compose/service.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index e989d4877..6bb755767 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1078,7 +1078,7 @@ class Service(object): pull=pull, nocache=no_cache, dockerfile=build_opts.get('dockerfile', None), - cache_from=build_opts.get('cache_from', None), + cache_from=self.get_cache_from(build_opts), labels=build_opts.get('labels', None), buildargs=build_args, network_mode=build_opts.get('network', None), @@ -1116,6 +1116,12 @@ class Service(object): return image_id + def get_cache_from(self, build_opts): + cache_from = build_opts.get('cache_from', None) + if cache_from is not None: + cache_from = [tag for tag in cache_from if tag] + return cache_from + def can_be_built(self): return 'build' in self.options From a857be3f7ed3882bc291df928621d5e61d0d7081 Mon Sep 17 00:00:00 2001 From: Sergey Fursov Date: Mon, 20 May 2019 16:54:49 +0300 Subject: [PATCH 56/62] support requests up to 2.22.x version Signed-off-by: Sergey Fursov --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index f2bfe6e1a..aac374aac 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==4.2b1 -requests==2.20.0 +requests==2.22.0 six==1.10.0 texttable==0.9.1 urllib3==1.24.2; python_version == '3.3' diff --git a/setup.py b/setup.py index 8371cc756..e0ab41bd0 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4.3', - 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', + 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.23', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', 'docker[ssh] >= 3.7.0, < 4.0', From e4b4babc24a49b5875cbdbe64c6c9c0c3e128e86 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 22 May 2019 11:55:37 +0200 Subject: [PATCH 57/62] Bump docker-py 4.0.1 Signed-off-by: Ulysses Souza --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index aac374aac..ff23516e1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==3.7.2 +docker==4.0.1 docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index e0ab41bd0..c6d07a86a 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.23', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker[ssh] >= 3.7.0, < 4.0', + 'docker[ssh] >= 3.7.0, < 4.0.2', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From c15e8af7f86932f8a064a076b606fd2028b8c7be Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Wed, 22 May 2019 19:00:16 +0200 Subject: [PATCH 58/62] Fix release script for null user Signed-off-by: Ulysses Souza --- script/release/release/repository.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 0dc724f80..a0281eaa3 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -220,6 +220,8 @@ def get_contributors(pr_data): commits = pr_data.get_commits() authors = {} for commit in commits: + if not commit or not commit.author or not commit.author.login: + continue author = commit.author.login authors[author] = authors.get(author, 0) + 1 return [x[0] for x in sorted(list(authors.items()), key=lambda x: x[1])] From 2d2b0bd9a8764cd40877c95bb758badda9413de7 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 23 May 2019 21:38:20 +0200 Subject: [PATCH 59/62] Fix 'finalize' command on release script Signed-off-by: Ulysses Souza --- script/release/release.py | 9 ++--- script/release/release/const.py | 1 + script/release/release/images.py | 61 +++++++++++++++++++++++--------- 3 files changed, 47 insertions(+), 24 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 9fdd92dae..861461f67 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -17,6 +17,7 @@ from release.downloader import BinaryDownloader from release.images import ImageManager from release.pypi import check_pypirc from release.pypi import pypi_upload +from release.images import is_tag_latest from release.repository import delete_assets from release.repository import get_contributors from release.repository import Repository @@ -258,7 +259,7 @@ def finalize(args): try: check_pypirc() repository = Repository(REPO_ROOT, args.repo) - tag_as_latest = _check_if_tag_latest(args.release) + tag_as_latest = is_tag_latest(args.release) img_manager = ImageManager(args.release, tag_as_latest) pr_data = repository.find_release_pr(args.release) if not pr_data: @@ -315,12 +316,6 @@ EPILOG = '''Example uses: ''' -# Checks if this version respects the GA version format ('x.y.z') and not an RC -def _check_if_tag_latest(version): - ga_version = all(n.isdigit() for n in version.split('.')) and version.count('.') == 2 - return ga_version and yesno('Should this release be tagged as \"latest\"? Y/n', default=True) - - def main(): if 'GITHUB_TOKEN' not in os.environ: print('GITHUB_TOKEN environment variable must be set') diff --git a/script/release/release/const.py b/script/release/release/const.py index 5a72bde41..52458ea14 100644 --- a/script/release/release/const.py +++ b/script/release/release/const.py @@ -6,4 +6,5 @@ import os REPO_ROOT = os.path.join(os.path.dirname(__file__), '..', '..', '..') NAME = 'docker/compose' +COMPOSE_TESTS_IMAGE_BASE_NAME = NAME + '-tests' BINTRAY_ORG = 'docker-compose' diff --git a/script/release/release/images.py b/script/release/release/images.py index 796a4d825..d8752b5d2 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -9,9 +9,12 @@ import os import docker from enum import Enum +from script.release.release.const import COMPOSE_TESTS_IMAGE_BASE_NAME + from .const import NAME from .const import REPO_ROOT from .utils import ScriptError +from .utils import yesno class Platform(Enum): @@ -22,9 +25,14 @@ class Platform(Enum): return self.value +# Checks if this version respects the GA version format ('x.y.z') and not an RC +def is_tag_latest(version): + ga_version = all(n.isdigit() for n in version.split('.')) and version.count('.') == 2 + return ga_version and yesno('Should this release be tagged as \"latest\"? [Y/n]: ', default=True) + + class ImageManager(object): def __init__(self, version, latest=False): - self.built_tags = [] self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.version = version self.latest = latest @@ -39,7 +47,15 @@ class ImageManager(object): existing_repo_tag = '{image}:{tag}'.format(image=image, tag=existing_tag) new_repo_tag = '{image}:{tag}'.format(image=image, tag=new_tag) self.docker_client.tag(existing_repo_tag, new_repo_tag) - self.built_tags.append(new_repo_tag) + + def get_full_version(self, platform=None): + return self.version + '-' + platform.__str__() if platform else self.version + + def get_runtime_image_tag(self, tag): + return '{image_base_image}:{tag}'.format( + image_base_image=NAME, + tag=self.get_full_version(tag) + ) def build_runtime_image(self, repository, platform): git_sha = repository.write_git_sha() @@ -48,11 +64,8 @@ class ImageManager(object): image=compose_image_base_name, platform=platform )) - full_version = '{version}-{platform}'.format(version=self.version, platform=platform) - build_tag = '{image_base_image}:{full_version}'.format( - image_base_image=compose_image_base_name, - full_version=full_version - ) + full_version = self.get_full_version(self, platform) + build_tag = self.get_runtime_image_tag(platform) logstream = self.docker_client.build( REPO_ROOT, tag=build_tag, @@ -68,7 +81,6 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - self.built_tags.append(build_tag) if platform == Platform.ALPINE: self._tag(compose_image_base_name, full_version, self.version) if self.latest: @@ -76,15 +88,17 @@ class ImageManager(object): if platform == Platform.ALPINE: self._tag(compose_image_base_name, full_version, 'latest') + def get_ucp_test_image_tag(self, tag=None): + return '{image}:{tag}'.format( + image=COMPOSE_TESTS_IMAGE_BASE_NAME, + tag=tag or self.version + ) + # Used for producing a test image for UCP def build_ucp_test_image(self, repository): print('Building test image (debian based for UCP e2e)') git_sha = repository.write_git_sha() - compose_tests_image_base_name = NAME + '-tests' - ucp_test_image_tag = '{image}:{tag}'.format( - image=compose_tests_image_base_name, - tag=self.version - ) + ucp_test_image_tag = self.get_ucp_test_image_tag() logstream = self.docker_client.build( REPO_ROOT, tag=ucp_test_image_tag, @@ -101,8 +115,7 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - self.built_tags.append(ucp_test_image_tag) - self._tag(compose_tests_image_base_name, self.version, 'latest') + self._tag(COMPOSE_TESTS_IMAGE_BASE_NAME, self.version, 'latest') def build_images(self, repository): self.build_runtime_image(repository, Platform.ALPINE) @@ -110,7 +123,7 @@ class ImageManager(object): self.build_ucp_test_image(repository) def check_images(self): - for name in self.built_tags: + for name in self.get_images_to_push(): try: self.docker_client.inspect_image(name) except docker.errors.ImageNotFound: @@ -118,8 +131,22 @@ class ImageManager(object): return False return True + def get_images_to_push(self): + tags_to_push = { + "{}:{}".format(NAME, self.version), + self.get_runtime_image_tag(Platform.ALPINE), + self.get_runtime_image_tag(Platform.DEBIAN), + self.get_ucp_test_image_tag(), + self.get_ucp_test_image_tag('latest'), + } + if is_tag_latest(self.version): + tags_to_push.add("{}:latest".format(NAME)) + return tags_to_push + def push_images(self): - for name in self.built_tags: + tags_to_push = self.get_images_to_push() + print('Build tags to push {}'.format(tags_to_push)) + for name in tags_to_push: print('Pushing {} to Docker Hub'.format(name)) logstream = self.docker_client.push(name, stream=True, decode=True) for chunk in logstream: From a2516c48d99496816d47ae6c2d518499c7b4f8ed Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 23 May 2019 22:04:02 +0200 Subject: [PATCH 60/62] Fix imports ordering Signed-off-by: Ulysses Souza --- script/release/release.py | 2 +- script/release/release/images.py | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 861461f67..a9c05eb78 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -15,9 +15,9 @@ from release.const import NAME from release.const import REPO_ROOT from release.downloader import BinaryDownloader from release.images import ImageManager +from release.images import is_tag_latest from release.pypi import check_pypirc from release.pypi import pypi_upload -from release.images import is_tag_latest from release.repository import delete_assets from release.repository import get_contributors from release.repository import Repository diff --git a/script/release/release/images.py b/script/release/release/images.py index d8752b5d2..0ce7a0c2d 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -9,12 +9,11 @@ import os import docker from enum import Enum -from script.release.release.const import COMPOSE_TESTS_IMAGE_BASE_NAME - from .const import NAME from .const import REPO_ROOT from .utils import ScriptError from .utils import yesno +from script.release.release.const import COMPOSE_TESTS_IMAGE_BASE_NAME class Platform(Enum): From 1f55b533c492eb40f189a1e91480a5c05420bb13 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 23 May 2019 23:35:57 +0200 Subject: [PATCH 61/62] Fix release script get_full_version() Signed-off-by: Ulysses Souza --- script/release/release/images.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/release/images.py b/script/release/release/images.py index 0ce7a0c2d..17d572df3 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -63,7 +63,7 @@ class ImageManager(object): image=compose_image_base_name, platform=platform )) - full_version = self.get_full_version(self, platform) + full_version = self.get_full_version(platform) build_tag = self.get_runtime_image_tag(platform) logstream = self.docker_client.build( REPO_ROOT, From 8552e8e2833f377103843bbc31df282cce299a77 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 23 May 2019 23:39:09 +0200 Subject: [PATCH 62/62] "Bump 1.25.0-rc1" Signed-off-by: Ulysses Souza --- CHANGELOG.md | 69 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 71 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a3ad8bfd..0a512c353 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,75 @@ Change log ========== +1.25.0 (2019-05-22) +------------------- + +### Features + +- Add tag `docker-compose:latest` + +- Add `docker-compose:-alpine` image/tag + +- Add `docker-compose:-debian` image/tag + +- Bumped `docker-py` 4.0.1 + +- Supports `requests` up to 2.22.0 version + +- Drops empty tag on `build:cache_from` + +- `Dockerfile` now generates `libmusl` binaries for alpine + +- Only pull images that can't be built + +- Attribute `scale` can now accept `0` as a value + +- Added `--quiet` build flag + +- Added `--no-interpolate` to `docker-compose config` + +- Bump OpenSSL for macOS build (`1.1.0j` to `1.1.1a`) + +- Added `--no-rm` to `build` command + +- Added support for `credential_spec` + +- Resolve digests without pulling image + +- Upgrade `pyyaml` to `4.2b1` + +- Lowered severity to `warning` if `down` tries to remove nonexisting image + +- Use improved API fields for project events when possible + +- Update `setup.py` for modern `pypi/setuptools` and remove `pandoc` dependencies + +- Removed `Dockerfile.armhf` which is no longer needed + +### Bugfixes + +- Fixed `--remove-orphans` when used with `up --no-start` + +- Fixed `docker-compose ps --all` + +- Fixed `depends_on` dependency recreation behavior + +- Fixed bash completion for `build --memory` + +- Fixed misleading warning concerning env vars when performing an `exec` command + +- Fixed failure check in parallel_execute_watch + +- Fixed race condition after pulling image + +- Fixed error on duplicate mount points. + +- Fixed merge on networks section + +- Always connect Compose container to `stdin` + +- Fixed the presentation of failed services on 'docker-compose start' when containers are not available + 1.24.0 (2019-03-28) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 0042896b6..55060583e 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.25.0dev' +__version__ = '1.25.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index f3456720f..58caf3612 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.24.0" +VERSION="1.25.0-rc1" IMAGE="docker/compose:$VERSION"