From 4b4c250638eb1649e3672b829c47f9e5e2ef6191 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Sep 2018 20:05:40 -0700 Subject: [PATCH 01/79] Fix some release script issues Signed-off-by: Joffrey F --- script/release/release.py | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index c6dc146a7..749ea49d3 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -77,19 +77,24 @@ def monitor_pr_status(pr_data): 'pending': 0, 'success': 0, 'failure': 0, + 'error': 0, } for detail in status.statuses: if detail.context == 'dco-signed': # dco-signed check breaks on merge remote-tracking ; ignore it continue - summary[detail.state] += 1 - print('{pending} pending, {success} successes, {failure} failures'.format(**summary)) - if summary['pending'] == 0 and summary['failure'] == 0 and summary['success'] > 0: + if detail.state in summary: + summary[detail.state] += 1 + print( + '{pending} pending, {success} successes, {failure} failures, ' + '{error} errors'.format(**summary) + ) + if summary['failure'] > 0 or summary['error'] > 0: + raise ScriptError('CI failures detected!') + elif summary['pending'] == 0 and summary['success'] > 0: # This check assumes at least 1 non-DCO CI check to avoid race conditions. # If testing on a repo without CI, use --skip-ci-check to avoid looping eternally return True - elif summary['failure'] > 0: - raise ScriptError('CI failures detected!') time.sleep(30) elif status.state == 'success': print('{} successes: all clear!'.format(status.total_count)) @@ -97,12 +102,14 @@ def monitor_pr_status(pr_data): def check_pr_mergeable(pr_data): - if not pr_data.mergeable: + if pr_data.mergeable is False: + # mergeable can also be null, in which case the warning would be a false positive. print( 'WARNING!! PR #{} can not currently be merged. You will need to ' 'resolve the conflicts manually before finalizing the release.'.format(pr_data.number) ) - return pr_data.mergeable + + return pr_data.mergeable is True def create_release_draft(repository, version, pr_data, files): From cc2462e6f4e475bc18bd6cc3be01d9a2ef1cb1d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 25 Sep 2018 09:13:12 -0700 Subject: [PATCH 02/79] Don't rely on container names containing the db string to identify them Signed-off-by: Joffrey F --- tests/integration/project_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 858a8dfd7..63939676e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -432,7 +432,7 @@ class ProjectTest(DockerClientTestCase): project.up(strategy=ConvergenceStrategy.always) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.service == 'db'][0] assert db_container.id != old_db_id assert db_container.get('Volumes./etc') == db_volume_path @@ -452,7 +452,7 @@ class ProjectTest(DockerClientTestCase): project.up(strategy=ConvergenceStrategy.always) assert len(project.containers()) == 2 - db_container = [c for c in project.containers() if 'db' in c.name][0] + db_container = [c for c in project.containers() if c.service == 'db'][0] assert db_container.id != old_db_id assert db_container.get_mount('/etc')['Source'] == db_volume_path @@ -499,7 +499,7 @@ class ProjectTest(DockerClientTestCase): assert len(new_containers) == 2 assert [c.is_running for c in new_containers] == [True, True] - db_container = [c for c in new_containers if 'db' in c.name][0] + db_container = [c for c in new_containers if c.service == 'db'][0] assert db_container.id == old_db_id assert db_container.get_mount('/var/db')['Source'] == db_volume_path From 879f7cb1edf8b15b393d7e65c00ee4852f0ffcb0 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 15:55:55 +0200 Subject: [PATCH 03/79] tests.unit.config: Make make_service_dict working dir argument optional. Signed-off-by: Antony MECHIN --- tests/unit/config/config_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1d42c10d5..c054c388e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -42,7 +42,7 @@ from tests import unittest DEFAULT_VERSION = V2_0 -def make_service_dict(name, service_dict, working_dir, filename=None): +def make_service_dict(name, service_dict, working_dir='.', filename=None): """Test helper function to construct a ServiceExtendsResolver """ resolver = config.ServiceExtendsResolver( From bbcfce40290a42de4f9658e8463d605b1242edd2 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 15:59:02 +0200 Subject: [PATCH 04/79] tests.unit.config: Make sure volume order is preserved. Signed-off-by: Antony MECHIN --- tests/unit/config/config_test.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c054c388e..52c89a9e0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -8,6 +8,7 @@ import os import shutil import tempfile from operator import itemgetter +from random import shuffle import py import pytest @@ -3536,6 +3537,13 @@ class VolumeConfigTest(unittest.TestCase): ).services[0] assert d['volumes'] == [VolumeSpec.parse('/host/path:/container/path')] + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') + def test_volumes_order_is_preserved(self): + volumes = ['/{0}:/{0}'.format(i) for i in range(0, 6)] + shuffle(volumes) + cfg = make_service_dict('foo', {'build': '.', 'volumes': volumes}) + assert cfg['volumes'] == volumes + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) def test_volume_binding_with_home(self): From de1958c5ff74b811b2766501eababee765d56677 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 18:08:17 +0200 Subject: [PATCH 05/79] utils: Add unique_everseen (from itertools recipies). Signed-off-by: Antony MECHIN --- compose/utils.py | 9 +++++++++ tests/unit/utils_test.py | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/compose/utils.py b/compose/utils.py index 8f0b3e549..b9b6ab9bd 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -170,3 +170,12 @@ def truncate_id(value): if len(value) > 12: return value[:12] return value + + +def unique_everseen(iterable, key=lambda x: x): + "List unique elements, preserving order. Remember all elements ever seen." + seen = set() + for element in iterable: + if key(element) not in seen: + seen.add(element) + yield element diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 84becb975..186b6b14e 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -68,3 +68,9 @@ class TestParseBytes(object): assert utils.parse_bytes(123) == 123 assert utils.parse_bytes('foobar') is None assert utils.parse_bytes('123') == 123 + + +class TestMoreItertools(object): + def test_unique_everseen(self): + assert list(utils.unique_everseen([2, 1, 2, 1])) == [2, 1] + assert list(utils.unique_everseen([2, 1, 2, 1], hash)) == [2, 1] From 39b051885052c556e1c1fe7e6c8de2147d906742 Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Mon, 24 Sep 2018 16:57:49 +0200 Subject: [PATCH 06/79] tests.unity.service: Make sure volumes order is preserved. Signed-off-by: Antony MECHIN --- compose/service.py | 6 ++++-- tests/unit/service_test.py | 17 +++++++++++++++++ 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index aca24ce17..8df061b9e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -56,6 +56,7 @@ from .utils import json_hash from .utils import parse_bytes from .utils import parse_seconds_float from .utils import truncate_id +from .utils import unique_everseen log = logging.getLogger(__name__) @@ -940,8 +941,9 @@ class Service(object): override_options['mounts'] = override_options.get('mounts') or [] override_options['mounts'].extend([build_mount(v) for v in secret_volumes]) - # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885) - override_options['binds'] = list(set(binds)) + # Remove possible duplicates (see e.g. https://github.com/docker/compose/issues/5885). + # unique_everseen preserves order. (see https://github.com/docker/compose/issues/6091). + override_options['binds'] = list(unique_everseen(binds)) return container_options, override_options def _get_container_host_config(self, override_options, one_off=False): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d5dbcbea6..af1cd1bea 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -1037,6 +1037,23 @@ class ServiceTest(unittest.TestCase): assert len(override_opts['binds']) == 1 assert override_opts['binds'][0] == 'vol:/data:rw' + def test_volumes_order_is_preserved(self): + service = Service('foo', client=self.mock_client) + volumes = [ + VolumeSpec.parse(cfg) for cfg in [ + '/v{0}:/v{0}:rw'.format(i) for i in range(6) + ] + ] + ctnr_opts, override_opts = service._build_container_volume_options( + previous_container=None, + container_options={ + 'volumes': volumes, + 'environment': {}, + }, + override_options={}, + ) + assert override_opts['binds'] == [vol.repr() for vol in volumes] + class TestServiceNetwork(unittest.TestCase): def setUp(self): From bf46a6cc600828c3aab55ed51a267bfb0fc0c35f Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Wed, 26 Sep 2018 15:15:59 +0200 Subject: [PATCH 07/79] service: Use OrderedDict to preserve volumes order on versions prior 3.6. Signed-off-by: Antony MECHIN --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 8df061b9e..3327c77f8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1429,7 +1429,7 @@ def merge_volume_bindings(volumes, tmpfs, previous_container, mounts): """ affinity = {} - volume_bindings = dict( + volume_bindings = OrderedDict( build_volume_binding(volume) for volume in volumes if volume.external From 772a3071922d3ed6055eb0083026c82ee0d7f195 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 26 Sep 2018 13:44:42 -0700 Subject: [PATCH 08/79] Avoid cred helpers errors in release script Signed-off-by: Joffrey F --- script/release/README.md | 6 ++++++ script/release/release.sh | 14 ++++++++++++-- script/release/release/images.py | 8 ++++++++ 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/script/release/README.md b/script/release/README.md index c5136c764..65883f5d3 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -20,6 +20,12 @@ following repositories: - docker/compose - docker/compose-tests +### A local Python environment + +While most of the release script is running inside a Docker container, +fetching local Docker credentials depends on the `docker` Python package +being available locally. + ### A Github account and Github API token Your Github account needs to have write access on the `docker/compose` repo. diff --git a/script/release/release.sh b/script/release/release.sh index 201182657..ee75b13a6 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -15,9 +15,19 @@ if test -z $BINTRAY_TOKEN; then exit 1 fi -docker run -e GITHUB_TOKEN=$GITHUB_TOKEN -e BINTRAY_TOKEN=$BINTRAY_TOKEN -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK -it \ +if test -z $(python -c "import docker; print(docker.version)" 2>/dev/null); then + echo "This script requires the 'docker' Python package to be installed locally" + exit 1 +fi + +hub_credentials=$(python -c "from docker import auth; cfg = auth.load_config(); print(auth.encode_header(auth.resolve_authconfig(cfg, 'docker.io')).decode('ascii'))") + +docker run -it \ + -e GITHUB_TOKEN=$GITHUB_TOKEN \ + -e BINTRAY_TOKEN=$BINTRAY_TOKEN \ + -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK \ + -e HUB_CREDENTIALS=$hub_credentials \ --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$HOME/.docker,target=/root/.docker \ --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ diff --git a/script/release/release/images.py b/script/release/release/images.py index b8f7ed3d6..e247f596d 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -2,6 +2,8 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import base64 +import json import os import shutil @@ -15,6 +17,12 @@ class ImageManager(object): def __init__(self, version): self.docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) self.version = version + if 'HUB_CREDENTIALS' in os.environ: + print('HUB_CREDENTIALS found in environment, issuing login') + credentials = json.loads(base64.urlsafe_b64decode(os.environ['HUB_CREDENTIALS'])) + self.docker_client.login( + username=credentials['Username'], password=credentials['Password'] + ) def build_images(self, repository, files): print("Building release images...") From b29ffb49e9405135768a39d7c96ec667d8fa18d6 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 27 Sep 2018 08:46:37 +0200 Subject: [PATCH 09/79] Fix bash completion for `config --hash` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index f4c42362c..395888d34 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -136,7 +136,18 @@ _docker_compose_bundle() { _docker_compose_config() { - COMPREPLY=( $( compgen -W "--help --quiet -q --resolve-image-digests --services --volumes --hash" -- "$cur" ) ) + case "$prev" in + --hash) + if [[ $cur == \\* ]] ; then + COMPREPLY=( '\*' ) + else + COMPREPLY=( $(compgen -W "$(__docker_compose_services) \\\* " -- "$cur") ) + fi + return + ;; + esac + + COMPREPLY=( $( compgen -W "--hash --help --quiet -q --resolve-image-digests --services --volumes" -- "$cur" ) ) } From 5b9b519e8a7c6441dcd980405c3b105944ff5a1a Mon Sep 17 00:00:00 2001 From: Antony MECHIN Date: Thu, 27 Sep 2018 13:58:38 +0200 Subject: [PATCH 10/79] utils: Fix typo in unique_everseen. Signed-off-by: Antony MECHIN --- compose/utils.py | 5 +++-- tests/unit/utils_test.py | 6 ++++-- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index b9b6ab9bd..72e6ced17 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -176,6 +176,7 @@ def unique_everseen(iterable, key=lambda x: x): "List unique elements, preserving order. Remember all elements ever seen." seen = set() for element in iterable: - if key(element) not in seen: - seen.add(element) + unique_key = key(element) + if unique_key not in seen: + seen.add(unique_key) yield element diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 186b6b14e..21b88d962 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -72,5 +72,7 @@ class TestParseBytes(object): class TestMoreItertools(object): def test_unique_everseen(self): - assert list(utils.unique_everseen([2, 1, 2, 1])) == [2, 1] - assert list(utils.unique_everseen([2, 1, 2, 1], hash)) == [2, 1] + unique = utils.unique_everseen + assert list(unique([2, 1, 2, 1])) == [2, 1] + assert list(unique([2, 1, 2, 1], hash)) == [2, 1] + assert list(unique([2, 1, 2, 1], lambda x: 'key_%s' % x)) == [2, 1] From 15089886c2b261d75545bfc853d55804596fe13a Mon Sep 17 00:00:00 2001 From: Emil Hessman Date: Sat, 29 Sep 2018 18:30:52 +0200 Subject: [PATCH 11/79] Avoid modifying mutable default value Rationale: http://effbot.org/zone/default-values.htm Signed-off-by: Emil Hessman --- compose/service.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 3327c77f8..5ae2ac3af 100644 --- a/compose/service.py +++ b/compose/service.py @@ -197,7 +197,9 @@ class Service(object): def __repr__(self): return ''.format(self.name) - def containers(self, stopped=False, one_off=False, filters={}, labels=None): + def containers(self, stopped=False, one_off=False, filters=None, labels=None): + if filters is None: + filters = {} filters.update({'label': self.labels(one_off=one_off) + (labels or [])}) result = list(filter(None, [ From 8493540a1c6b010ec743cc0c4297c366491b96ac Mon Sep 17 00:00:00 2001 From: Gabriel Machado Date: Sat, 29 Sep 2018 20:08:00 -0300 Subject: [PATCH 12/79] Reffer Docker for Mac and Windows as Docker Desktop Signed-off-by: Gabriel Machado --- script/release/release.md.tmpl | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/release/release.md.tmpl b/script/release/release.md.tmpl index ee97ef104..4d0ebe926 100644 --- a/script/release/release.md.tmpl +++ b/script/release/release.md.tmpl @@ -1,6 +1,6 @@ -If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. +If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker Desktop for Mac and Windows](https://www.docker.com/products/docker-desktop)**. -Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. +Docker Desktop will automatically install the latest version of Docker Engine for you. Alternatively, you can use the usual commands to install or upgrade Compose: From abf67565f63a7dad1c0e51ceb55d203b9d18d069 Mon Sep 17 00:00:00 2001 From: Heath Milligan Date: Tue, 2 Oct 2018 14:25:33 -0400 Subject: [PATCH 13/79] Show more helpful error message when Docker is not running. Fixes #6175 Signed-off-by: Heath Milligan --- compose/cli/errors.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 82768970b..b48ccf4d7 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -67,7 +67,9 @@ def handle_connection_errors(client): def log_windows_pipe_error(exc): - if exc.winerror == 232: # https://github.com/docker/compose/issues/5005 + if exc.winerror == 2: + log.error("Couldn't connect to Docker daemon. You might need to start Docker for Windows.") + elif exc.winerror == 232: # https://github.com/docker/compose/issues/5005 log.error( "The current Compose file version is not compatible with your engine version. " "Please upgrade your Compose file to a more recent version, or set " From 25e419c763128bbf3b9104acc1b322bab809ee68 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 00:48:53 -0700 Subject: [PATCH 14/79] Fix twine upload for RC versions Signed-off-by: Joffrey F --- script/release/release.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 749ea49d3..9a5af3aa5 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -173,9 +173,10 @@ def distclean(): def pypi_upload(args): print('Uploading to PyPi') try: + rel = args.release.replace('-rc', 'rc') twine_upload([ - 'dist/docker_compose-{}*.whl'.format(args.release), - 'dist/docker-compose-{}*.tar.gz'.format(args.release) + 'dist/docker_compose-{}*.whl'.format(rel), + 'dist/docker-compose-{}*.tar.gz'.format(rel) ]) except HTTPError as e: if e.response.status_code == 400 and 'File already exists' in e.message: From cc595a65f04487b77d4779b9580b573b50ddcdb6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 01:09:48 -0700 Subject: [PATCH 15/79] Don't attempt iterating on None during parallel pull Signed-off-by: Joffrey F --- compose/project.py | 11 +++++------ compose/utils.py | 6 ++++++ tests/integration/project_test.py | 17 +++++++++++++++++ 3 files changed, 28 insertions(+), 6 deletions(-) diff --git a/compose/project.py b/compose/project.py index 4340577c9..92c352050 100644 --- a/compose/project.py +++ b/compose/project.py @@ -34,6 +34,7 @@ from .service import Service from .service import ServiceNetworkMode from .service import ServicePidMode from .utils import microseconds_from_time_nano +from .utils import truncate_string from .volume import ProjectVolumes @@ -554,12 +555,10 @@ class Project(object): if parallel_pull: def pull_service(service): strm = service.pull(ignore_pull_failures, True, stream=True) - writer = parallel.get_stream_writer() + if strm is None: # Attempting to pull service with no `image` key is a no-op + return - def trunc(s): - if len(s) > 35: - return s[:33] + '...' - return s + writer = parallel.get_stream_writer() for event in strm: if 'status' not in event: @@ -572,7 +571,7 @@ class Project(object): status = '{} ({:.1%})'.format(status, percentage) writer.write( - msg, service.name, trunc(status), lambda s: s + msg, service.name, truncate_string(status), lambda s: s ) _, errors = parallel.parallel_execute( diff --git a/compose/utils.py b/compose/utils.py index 72e6ced17..9f0441d08 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -180,3 +180,9 @@ def unique_everseen(iterable, key=lambda x: x): if unique_key not in seen: seen.add(unique_key) yield element + + +def truncate_string(s, max_chars=35): + if len(s) > max_chars: + return s[:max_chars - 2] + '...' + return s diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 63939676e..57f3b7074 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -105,6 +105,23 @@ class ProjectTest(DockerClientTestCase): project = Project('composetest', [web, db], self.client) assert set(project.containers(stopped=True)) == set([web_1, db_1]) + def test_parallel_pull_with_no_image(self): + config_data = build_config( + version=V2_3, + services=[{ + 'name': 'web', + 'build': {'context': '.'}, + }], + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, + client=self.client + ) + + project.pull(parallel_pull=True) + def test_volumes_from_service(self): project = Project.from_config( name='composetest', From b21a06cd6f657e94193581fcc9a58006254a4b41 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 4 Oct 2018 01:40:39 -0700 Subject: [PATCH 16/79] Re-enable testing of TP and beta releases Signed-off-by: Joffrey F --- script/test/versions.py | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/script/test/versions.py b/script/test/versions.py index 6d273a9e6..a06c49f20 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -36,6 +36,8 @@ import requests GITHUB_API = 'https://api.github.com/repos' +STAGES = ['tp', 'beta', 'rc'] + class Version(namedtuple('_Version', 'major minor patch stage edition')): @@ -45,7 +47,7 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): version = version.lstrip('v') version, _, stage = version.partition('-') if stage: - if not any(marker in stage for marker in ['rc', 'tp', 'beta']): + if not any(marker in stage for marker in STAGES): edition = stage stage = None elif '-' in stage: @@ -62,8 +64,16 @@ class Version(namedtuple('_Version', 'major minor patch stage edition')): """Return a representation that allows this object to be sorted correctly with the default comparator. """ - # rc releases should appear before official releases - stage = (0, self.stage) if self.stage else (1, ) + # non-GA releases should appear before GA releases + # Order: tp -> beta -> rc -> GA + if self.stage: + for st in STAGES: + if st in self.stage: + stage = (STAGES.index(st), self.stage) + break + else: + stage = (len(STAGES),) + return (int(self.major), int(self.minor), int(self.patch)) + stage def __str__(self): @@ -124,9 +134,6 @@ def get_versions(tags): v = Version.parse(tag['name']) if v in BLACKLIST: continue - # FIXME: Temporary. Remove once these versions are built on dockerswarm/dind - if v.stage and 'rc' not in v.stage: - continue yield v except ValueError: print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) From 9d7202d12256d8672b0ddfa6840877e551323f08 Mon Sep 17 00:00:00 2001 From: Alexander Date: Fri, 5 Oct 2018 14:52:56 +0600 Subject: [PATCH 17/79] Squashed commit of the following: commit d3fbd3d630099dc0d34cb1a93b0a664f633a1c25 Author: zasca Date: Wed Oct 3 11:27:43 2018 +0600 Fix typo in function name, path separator updated commit bc3f03cd9a7702b3f2d96b18380d75e10f18def0 Author: zasca Date: Tue Oct 2 11:12:28 2018 +0600 Fix endswith arg in the test commit 602d2977b4e881850c99c7555bc284690a802815 Author: zasca Date: Mon Oct 1 12:24:17 2018 +0600 Update test commit 6cd7a4a2c411ddf9b8e7d91194c60fb2238db8d7 Author: zasca Date: Fri Sep 28 11:13:36 2018 +0600 Fix last test commit 0d37343433caceec18ea15babf924b5975b83c80 Author: zasca Date: Fri Sep 28 10:58:57 2018 +0600 Unit test added commit fc086e544677dd33bad798c773cb92600aaefc51 Author: zasca Date: Thu Sep 27 20:28:03 2018 +0600 Improved expanding source paths of volumes defined with long syntax when paths starts with '~' Signed-off-by: Alexander --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 23 +++++++++++++++++++++++ 2 files changed, 24 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7abab2546..fb2c742f4 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1281,7 +1281,7 @@ def resolve_volume_paths(working_dir, service_dict): def resolve_volume_path(working_dir, volume): if isinstance(volume, dict): - if volume.get('source', '').startswith('.') and volume['type'] == 'bind': + if volume.get('source', '').startswith(('.', '~')) and volume['type'] == 'bind': volume['source'] = expand_path(working_dir, volume['source']) return volume diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1d42c10d5..8f98a7515 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1322,6 +1322,29 @@ class ConfigTest(unittest.TestCase): assert mount.type == 'bind' assert mount.source == expected_source + def test_load_bind_mount_relative_path_with_tilde(self): + base_file = config.ConfigFile( + 'base.yaml', { + 'version': '3.4', + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': [ + {'type': 'bind', 'source': '~/web', 'target': '/web'}, + ], + }, + }, + }, + ) + + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + mount = config_data.services[0].get('volumes')[0] + assert mount.target == '/web' + assert mount.type == 'bind' + assert (not mount.source.startswith('~') + and mount.source.endswith('{}web'.format(os.path.sep))) + def test_config_invalid_ipam_config(self): with pytest.raises(ConfigurationError) as excinfo: config.load( From 6a35663781779f298c2032fa620c08c2b890ef36 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 5 Oct 2018 08:21:39 -0700 Subject: [PATCH 18/79] Decontainerize release script Credentials management inside containers is a mess. Let's work on the host instead. Signed-off-by: Joffrey F --- script/release/Dockerfile | 15 -------------- script/release/README.md | 21 +++++++++++++------ script/release/release.sh | 39 +++++++++--------------------------- script/release/setup-venv.sh | 30 +++++++++++++++++++++++++++ 4 files changed, 54 insertions(+), 51 deletions(-) delete mode 100644 script/release/Dockerfile create mode 100755 script/release/setup-venv.sh diff --git a/script/release/Dockerfile b/script/release/Dockerfile deleted file mode 100644 index e5af676a5..000000000 --- a/script/release/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM python:3.6 -RUN mkdir -p /src && pip install -U Jinja2==2.10 \ - PyGithub==1.39 \ - pypandoc==1.4 \ - GitPython==2.1.9 \ - requests==2.18.4 \ - twine==1.11.0 && \ - apt-get update && apt-get install -y pandoc - -VOLUME /src/script/release -WORKDIR /src -COPY . /src -RUN python setup.py develop -ENTRYPOINT ["python", "script/release/release.py"] -CMD ["--help"] diff --git a/script/release/README.md b/script/release/README.md index 65883f5d3..f7f911e53 100644 --- a/script/release/README.md +++ b/script/release/README.md @@ -9,8 +9,7 @@ The following things are required to bring a release to a successful conclusion ### Local Docker engine (Linux Containers) -The release script runs inside a container and builds images that will be part -of the release. +The release script builds images that will be part of the release. ### Docker Hub account @@ -20,11 +19,9 @@ following repositories: - docker/compose - docker/compose-tests -### A local Python environment +### Python -While most of the release script is running inside a Docker container, -fetching local Docker credentials depends on the `docker` Python package -being available locally. +The release script is written in Python and requires Python 3.3 at minimum. ### A Github account and Github API token @@ -59,6 +56,18 @@ Said account needs to be a member of the maintainers group for the Moreover, the `~/.pypirc` file should exist on your host and contain the relevant pypi credentials. +The following is a sample `.pypirc` provided as a guideline: + +``` +[distutils] +index-servers = + pypi + +[pypi] +username = user +password = pass +``` + ## Start a feature release A feature release is a release that includes all changes present in the diff --git a/script/release/release.sh b/script/release/release.sh index ee75b13a6..7947316e0 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -1,36 +1,15 @@ #!/bin/sh -docker image inspect compose/release-tool > /dev/null -if test $? -ne 0; then - docker build -t compose/release-tool -f $(pwd)/script/release/Dockerfile $(pwd) +if test -d ./.release-venv; then + true +else + ./script/release/setup-venv.sh fi -if test -z $GITHUB_TOKEN; then - echo "GITHUB_TOKEN environment variable must be set" - exit 1 +args=$* + +if test -z $args; then + args="--help" fi -if test -z $BINTRAY_TOKEN; then - echo "BINTRAY_TOKEN environment variable must be set" - exit 1 -fi - -if test -z $(python -c "import docker; print(docker.version)" 2>/dev/null); then - echo "This script requires the 'docker' Python package to be installed locally" - exit 1 -fi - -hub_credentials=$(python -c "from docker import auth; cfg = auth.load_config(); print(auth.encode_header(auth.resolve_authconfig(cfg, 'docker.io')).decode('ascii'))") - -docker run -it \ - -e GITHUB_TOKEN=$GITHUB_TOKEN \ - -e BINTRAY_TOKEN=$BINTRAY_TOKEN \ - -e SSH_AUTH_SOCK=$SSH_AUTH_SOCK \ - -e HUB_CREDENTIALS=$hub_credentials \ - --mount type=bind,source=$(pwd),target=/src \ - --mount type=bind,source=$HOME/.gitconfig,target=/root/.gitconfig \ - --mount type=bind,source=/var/run/docker.sock,target=/var/run/docker.sock \ - --mount type=bind,source=$HOME/.ssh,target=/root/.ssh \ - --mount type=bind,source=/tmp,target=/tmp \ - -v $HOME/.pypirc:/root/.pypirc \ - compose/release-tool $* +./.release-venv/bin/python ./script/release/release.py $args diff --git a/script/release/setup-venv.sh b/script/release/setup-venv.sh new file mode 100755 index 000000000..d3d3f9a42 --- /dev/null +++ b/script/release/setup-venv.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +if test -z $PYTHONBIN; then + PYTHONBIN=$(which python3) + if test -z $PYTHONBIN; then + PYTHONBIN=$(which python) + fi +fi + +VERSION=$($PYTHONBIN -c "import sys; print('{}.{}'.format(*sys.version_info[0:2]))") +if test $(echo $VERSION | cut -d. -f1) -lt 3; then + echo "Python 3.3 or above is required" +fi + +if test $(echo $VERSION | cut -d. -f2) -lt 3; then + echo "Python 3.3 or above is required" +fi + +$PYTHONBIN -m venv ./.release-venv + +VENVBINS=./.release-venv/bin + +$VENVBINS/pip install -U Jinja2==2.10 \ + PyGithub==1.39 \ + pypandoc==1.4 \ + GitPython==2.1.9 \ + requests==2.18.4 \ + twine==1.11.0 + +$VENVBINS/python setup.py develop From 21a51bcd600408768e8c15efd71808d5d7da8df5 Mon Sep 17 00:00:00 2001 From: Andrew Rabert Date: Fri, 5 Oct 2018 14:01:35 -0400 Subject: [PATCH 19/79] Use Docker binary from official Docker image Signed-off-by: Andrew Rabert --- Dockerfile.run | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index e9ba19fd4..bf87fc335 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,7 +1,7 @@ +FROM docker:17.12.1 as docker FROM alpine:3.6 ENV GLIBC 2.27-r0 -ENV DOCKERBINS_SHA 1270dce1bd7e1838d62ae21d2505d87f16efc1d9074645571daaefdfd0c14054 RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ @@ -10,14 +10,10 @@ RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ ln -s /lib/libz.so.1 /usr/glibc-compat/lib/ && \ ln -s /lib/libc.musl-x86_64.so.1 /usr/glibc-compat/lib && \ ln -s /usr/lib/libgcc_s.so.1 /usr/glibc-compat/lib && \ - curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.1-ce.tgz" && \ - echo "${DOCKERBINS_SHA} dockerbins.tgz" | sha256sum -c - && \ - tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ - mv docker /usr/local/bin/docker && \ - chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ + rm /etc/apk/keys/sgerrand.rsa.pub glibc-$GLIBC.apk && \ apk del curl +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker COPY dist/docker-compose-Linux-x86_64 /usr/local/bin/docker-compose ENTRYPOINT ["docker-compose"] From c7c5b5e8c436a6f1f7a3ca576d3c22931383572a Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 10 Oct 2018 22:04:33 -0400 Subject: [PATCH 20/79] Upgrade Windows-specific dependency colorama Signed-off-by: Ofek Lev --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2819810c2..a0093d8dd 100644 --- a/setup.py +++ b/setup.py @@ -55,7 +55,7 @@ extras_require = { ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], ':python_version < "3.3"': ['ipaddress >= 1.0.16'], - ':sys_platform == "win32"': ['colorama >= 0.3.9, < 0.4'], + ':sys_platform == "win32"': ['colorama >= 0.4, < 0.5'], 'socks': ['PySocks >= 1.5.6, != 1.5.7, < 2'], } From be324d57a20be9280e8bdbce3eed12f7b9feecfa Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Oct 2018 06:13:55 -0700 Subject: [PATCH 21/79] Add pypirc check Signed-off-by: Joffrey F --- script/release/release.py | 24 +++---------------- script/release/release/pypi.py | 44 ++++++++++++++++++++++++++++++++++ 2 files changed, 47 insertions(+), 21 deletions(-) create mode 100644 script/release/release/pypi.py diff --git a/script/release/release.py b/script/release/release.py index 9a5af3aa5..15c74c775 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -17,6 +17,8 @@ from release.const import NAME from release.const import REPO_ROOT from release.downloader import BinaryDownloader from release.images import ImageManager +from release.pypi import check_pypirc +from release.pypi import pypi_upload from release.repository import delete_assets from release.repository import get_contributors from release.repository import Repository @@ -28,8 +30,6 @@ from release.utils import ScriptError from release.utils import update_init_py_version from release.utils import update_run_sh_version from release.utils import yesno -from requests.exceptions import HTTPError -from twine.commands.upload import main as twine_upload def create_initial_branch(repository, args): @@ -170,25 +170,6 @@ def distclean(): shutil.rmtree(folder, ignore_errors=True) -def pypi_upload(args): - print('Uploading to PyPi') - try: - rel = args.release.replace('-rc', 'rc') - twine_upload([ - 'dist/docker_compose-{}*.whl'.format(rel), - 'dist/docker-compose-{}*.tar.gz'.format(rel) - ]) - except HTTPError as e: - if e.response.status_code == 400 and 'File already exists' in e.message: - if not args.finalize_resume: - raise ScriptError( - 'Package already uploaded on PyPi.' - ) - print('Skipping PyPi upload - package already uploaded') - else: - raise ScriptError('Unexpected HTTP error uploading package to PyPi: {}'.format(e)) - - def resume(args): try: distclean() @@ -277,6 +258,7 @@ def start(args): def finalize(args): distclean() try: + check_pypirc() repository = Repository(REPO_ROOT, args.repo) img_manager = ImageManager(args.release) pr_data = repository.find_release_pr(args.release) diff --git a/script/release/release/pypi.py b/script/release/release/pypi.py new file mode 100644 index 000000000..a40e17544 --- /dev/null +++ b/script/release/release/pypi.py @@ -0,0 +1,44 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from configparser import Error +from requests.exceptions import HTTPError +from twine.commands.upload import main as twine_upload +from twine.utils import get_config + +from .utils import ScriptError + + +def pypi_upload(args): + print('Uploading to PyPi') + try: + rel = args.release.replace('-rc', 'rc') + twine_upload([ + 'dist/docker_compose-{}*.whl'.format(rel), + 'dist/docker-compose-{}*.tar.gz'.format(rel) + ]) + except HTTPError as e: + if e.response.status_code == 400 and 'File already exists' in e.message: + if not args.finalize_resume: + raise ScriptError( + 'Package already uploaded on PyPi.' + ) + print('Skipping PyPi upload - package already uploaded') + else: + raise ScriptError('Unexpected HTTP error uploading package to PyPi: {}'.format(e)) + + +def check_pypirc(): + try: + config = get_config() + except Error as e: + raise ScriptError('Failed to parse .pypirc file: {}'.format(e)) + + if config is None: + raise ScriptError('Failed to parse .pypirc file') + + if 'pypi' not in config: + raise ScriptError('Missing [pypi] section in .pypirc file') + + if not (config['pypi'].get('username') and config['pypi'].get('password')): + raise ScriptError('Missing login/password pair for pypi repo') From 297bee897b251a791fc3612225bf1e93505fe957 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Oct 2018 06:14:35 -0700 Subject: [PATCH 22/79] Fix arg checks in release.sh Signed-off-by: Joffrey F --- script/release/release.sh | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/script/release/release.sh b/script/release/release.sh index 7947316e0..c10f8aba5 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -6,10 +6,8 @@ else ./script/release/setup-venv.sh fi -args=$* - -if test -z $args; then +if test -z "$*"; then args="--help" fi -./.release-venv/bin/python ./script/release/release.py $args +./.release-venv/bin/python ./script/release/release.py "$@" From bd67b90869051db4f7fd5c222b538ee07c58ef56 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 12 Oct 2018 06:39:56 -0700 Subject: [PATCH 23/79] Fix ImageManager inconsistencies Signed-off-by: Joffrey F --- script/release/release.py | 2 +- script/release/release/images.py | 27 ++++++++++++--------------- 2 files changed, 13 insertions(+), 16 deletions(-) diff --git a/script/release/release.py b/script/release/release.py index 15c74c775..6574bfddd 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -266,7 +266,7 @@ def finalize(args): raise ScriptError('No PR found for {}'.format(args.release)) if not check_pr_mergeable(pr_data): raise ScriptError('Can not finalize release with an unmergeable PR') - if not img_manager.check_images(args.release): + if not img_manager.check_images(): raise ScriptError('Missing release image') br_name = branch_name(args.release) if not repository.branch_exists(br_name): diff --git a/script/release/release/images.py b/script/release/release/images.py index e247f596d..df6eeda4f 100644 --- a/script/release/release/images.py +++ b/script/release/release/images.py @@ -27,13 +27,12 @@ class ImageManager(object): def build_images(self, repository, files): print("Building release images...") repository.write_git_sha() - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) distdir = os.path.join(REPO_ROOT, 'dist') os.makedirs(distdir, exist_ok=True) shutil.copy(files['docker-compose-Linux-x86_64'][0], distdir) os.chmod(os.path.join(distdir, 'docker-compose-Linux-x86_64'), 0o755) print('Building docker/compose image') - logstream = docker_client.build( + logstream = self.docker_client.build( REPO_ROOT, tag='docker/compose:{}'.format(self.version), dockerfile='Dockerfile.run', decode=True ) @@ -44,7 +43,7 @@ class ImageManager(object): print(chunk['stream'], end='') print('Building test image (for UCP e2e)') - logstream = docker_client.build( + logstream = self.docker_client.build( REPO_ROOT, tag='docker-compose-tests:tmp', decode=True ) for chunk in logstream: @@ -53,13 +52,15 @@ class ImageManager(object): if 'stream' in chunk: print(chunk['stream'], end='') - container = docker_client.create_container( + container = self.docker_client.create_container( 'docker-compose-tests:tmp', entrypoint='tox' ) - docker_client.commit(container, 'docker/compose-tests', 'latest') - docker_client.tag('docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version)) - docker_client.remove_container(container, force=True) - docker_client.remove_image('docker-compose-tests:tmp', force=True) + self.docker_client.commit(container, 'docker/compose-tests', 'latest') + self.docker_client.tag( + 'docker/compose-tests:latest', 'docker/compose-tests:{}'.format(self.version) + ) + self.docker_client.remove_container(container, force=True) + self.docker_client.remove_image('docker-compose-tests:tmp', force=True) @property def image_names(self): @@ -69,23 +70,19 @@ class ImageManager(object): 'docker/compose:{}'.format(self.version) ] - def check_images(self, version): - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) - + def check_images(self): for name in self.image_names: try: - docker_client.inspect_image(name) + self.docker_client.inspect_image(name) except docker.errors.ImageNotFound: print('Expected image {} was not found'.format(name)) return False return True def push_images(self): - docker_client = docker.APIClient(**docker.utils.kwargs_from_env()) - for name in self.image_names: print('Pushing {} to Docker Hub'.format(name)) - logstream = docker_client.push(name, stream=True, decode=True) + logstream = self.docker_client.push(name, stream=True, decode=True) for chunk in logstream: if 'status' in chunk: print(chunk['status']) From 402060e41975cafee88982eab74b63301fa6acd2 Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Fri, 12 Oct 2018 11:35:27 -0400 Subject: [PATCH 24/79] Update requirements.txt Signed-off-by: Ofek Lev --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 41d21172e..0ea046589 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,13 +2,13 @@ backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 +colorama==0.4.0; sys_platform == 'win32' docker==3.5.0 docker-pycreds==0.3.0 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' -git+git://github.com/tartley/colorama.git@bd378c725b45eba0b8e5cc091c3ca76a954c92ff; sys_platform == 'win32' idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 From 3844ff2fde0a840e28fd05ea0031b19e040b7327 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 15 Oct 2018 19:14:58 -0700 Subject: [PATCH 25/79] Update versions in Dockerfiles Signed-off-by: Joffrey F --- Dockerfile | 9 ++------- Dockerfile.run | 6 +++--- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9df78a826..aa3e1d87b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,4 @@ +FROM docker:18.06.1 as docker FROM python:3.6 RUN set -ex; \ @@ -8,13 +9,7 @@ RUN set -ex; \ python-dev \ git -RUN curl -fsSL -o dockerbins.tgz "https://download.docker.com/linux/static/stable/x86_64/docker-17.12.0-ce.tgz" && \ - SHA256=692e1c72937f6214b1038def84463018d8e320c8eaf8530546c84c2f8f9c767d; \ - echo "${SHA256} dockerbins.tgz" | sha256sum -c - && \ - tar xvf dockerbins.tgz docker/docker --strip-components 1 && \ - mv docker /usr/local/bin/docker && \ - chmod +x /usr/local/bin/docker && \ - rm dockerbins.tgz +COPY --from=docker /usr/local/bin/docker /usr/local/bin/docker # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen diff --git a/Dockerfile.run b/Dockerfile.run index bf87fc335..ccc86ea96 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,7 +1,7 @@ -FROM docker:17.12.1 as docker -FROM alpine:3.6 +FROM docker:18.06.1 as docker +FROM alpine:3.8 -ENV GLIBC 2.27-r0 +ENV GLIBC 2.28-r0 RUN apk update && apk add --no-cache openssl ca-certificates curl libgcc && \ curl -fsSL -o /etc/apk/keys/sgerrand.rsa.pub https://alpine-pkgs.sgerrand.com/sgerrand.rsa.pub && \ From 9df0a4f3a974a24512e6b7e13053075cff0ffb3d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 15 Oct 2018 19:22:25 -0700 Subject: [PATCH 26/79] Remove obsolete curl dependency Signed-off-by: Joffrey F --- Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index aa3e1d87b..a14be492e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -5,7 +5,6 @@ RUN set -ex; \ apt-get update -qq; \ apt-get install -y \ locales \ - curl \ python-dev \ git From 4cb92294a34524c4501d7c12046c9391f46b1f55 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 Oct 2018 13:57:01 -0700 Subject: [PATCH 27/79] Avoid creating duplicate mount points when recreating a service Signed-off-by: Joffrey F --- compose/service.py | 5 +++++ tests/integration/service_test.py | 16 ++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/compose/service.py b/compose/service.py index 5ae2ac3af..cc6d16a82 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1491,6 +1491,11 @@ def get_container_data_volumes(container, volumes_option, tmpfs_option, mounts_o if not mount.get('Name'): continue + # Volume (probably an image volume) is overridden by a mount in the service's config + # and would cause a duplicate mountpoint error + if volume.internal in [m.target for m in mounts_option]: + continue + # Copy existing volume from old container volume = volume._replace(external=mount['Name']) volumes.append(volume) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index db40409f8..edc195287 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -425,6 +425,22 @@ class ServiceTest(DockerClientTestCase): new_container = service.recreate_container(old_container) assert new_container.get_mount('/data')['Source'] == volume_path + def test_recreate_volume_to_mount(self): + # https://github.com/docker/compose/issues/6280 + service = Service( + project='composetest', + name='db', + client=self.client, + build={'context': 'tests/fixtures/dockerfile-with-volume'}, + volumes=[MountSpec.parse({ + 'type': 'volume', + 'target': '/data', + })] + ) + old_container = create_and_start_container(service) + new_container = service.recreate_container(old_container) + assert new_container.get_mount('/data')['Source'] + def test_duplicate_volume_trailing_slash(self): """ When an image specifies a volume, and the Compose file specifies a host path From 62057d098f3e563313d84e928f48962e75cb3807 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 Oct 2018 17:21:57 -0700 Subject: [PATCH 28/79] Don't use dot as a path separator as it is a valid character in resource identifiers Signed-off-by: Joffrey F --- compose/config/interpolation.py | 10 ++++---- tests/unit/config/interpolation_test.py | 31 +++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 5 deletions(-) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 4f56dff59..0f878be14 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -48,7 +48,7 @@ def interpolate_environment_variables(version, config, section, environment): def get_config_path(config_key, section, name): - return '{}.{}.{}'.format(section, name, config_key) + return '{}/{}/{}'.format(section, name, config_key) def interpolate_value(name, config_key, value, section, interpolator): @@ -75,7 +75,7 @@ def interpolate_value(name, config_key, value, section, interpolator): def recursive_interpolate(obj, interpolator, config_path): def append(config_path, key): - return '{}.{}'.format(config_path, key) + return '{}/{}'.format(config_path, key) if isinstance(obj, six.string_types): return converter.convert(config_path, interpolator.interpolate(obj)) @@ -160,12 +160,12 @@ class UnsetRequiredSubstitution(Exception): self.err = custom_err_msg -PATH_JOKER = '[^.]+' +PATH_JOKER = '[^/]+' FULL_JOKER = '.+' def re_path(*args): - return re.compile('^{}$'.format('\.'.join(args))) + return re.compile('^{}$'.format('/'.join(args))) def re_path_basic(section, name): @@ -288,7 +288,7 @@ class ConversionMap(object): except ValueError as e: raise ConfigurationError( 'Error while attempting to convert {} to appropriate type: {}'.format( - path, e + path.replace('/', '.'), e ) ) return value diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 0d0e7d28d..91fc3e69d 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -332,6 +332,37 @@ def test_interpolate_environment_external_resource_convert_types(mock_env): assert value == expected +def test_interpolate_service_name_uses_dot(mock_env): + entry = { + 'service.1': { + 'image': 'busybox', + 'ulimits': { + 'nproc': '${POSINT}', + 'nofile': { + 'soft': '${POSINT}', + 'hard': '${DEFAULT:-40000}' + }, + }, + } + } + + expected = { + 'service.1': { + 'image': 'busybox', + 'ulimits': { + 'nproc': 50, + 'nofile': { + 'soft': 50, + 'hard': 40000 + }, + }, + } + } + + value = interpolate_environment_variables(V3_4, entry, 'service', mock_env) + assert value == expected + + def test_escaped_interpolation(defaults_interpolator): assert defaults_interpolator('$${foo}') == '${foo}' From 5017b25f149603a02fdcae45611d753a698f6cfb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 16 Oct 2018 18:12:34 -0700 Subject: [PATCH 29/79] Update issue templates Signed-off-by: Joffrey F --- .github/ISSUE_TEMPLATE/bug_report.md | 60 +++++++++++++++++++ .github/ISSUE_TEMPLATE/feature_request.md | 29 +++++++++ .../question-about-using-compose.md | 9 +++ 3 files changed, 98 insertions(+) create mode 100644 .github/ISSUE_TEMPLATE/bug_report.md create mode 100644 .github/ISSUE_TEMPLATE/feature_request.md create mode 100644 .github/ISSUE_TEMPLATE/question-about-using-compose.md diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 000000000..49d4691fb --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,60 @@ +--- +name: Bug report +about: Report a bug encountered while using docker-compose + +--- + + + +## Description of the issue + +## Context information (for bug reports) + +**Output of `docker-compose version`** +``` +(paste here) +``` + +**Output of `docker version`** +``` +(paste here) +``` + +**Output of `docker-compose config`** +(Make sure to add the relevant `-f` and other flags) +``` +(paste here) +``` + + +## Steps to reproduce the issue + +1. +2. +3. + +### Observed result + +### Expected result + +### Stacktrace / full error message + +``` +(paste here) +``` + +## Additional information + +OS version / distribution, `docker-compose` install method, etc. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 000000000..d53c49a79 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,29 @@ +--- +name: Feature request +about: Suggest an idea to improve Compose + +--- + + + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +**Describe the solution you'd like** +A clear and concise description of what you want to happen. + +**Describe alternatives you've considered** +A clear and concise description of any alternative solutions or features you've considered. + +**Additional context** +Add any other context or screenshots about the feature request here. diff --git a/.github/ISSUE_TEMPLATE/question-about-using-compose.md b/.github/ISSUE_TEMPLATE/question-about-using-compose.md new file mode 100644 index 000000000..11ef65ccf --- /dev/null +++ b/.github/ISSUE_TEMPLATE/question-about-using-compose.md @@ -0,0 +1,9 @@ +--- +name: Question about using Compose +about: This is not the appropriate channel + +--- + +Please post on our forums: https://forums.docker.com for questions about using `docker-compose`. + +Posts that are not a bug report or a feature/enhancement request will not be addressed on this issue tracker. From 7712d19b32cb6f8302ee3f8c1a24e951b784dfd7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 12:10:08 -0700 Subject: [PATCH 30/79] Add workaround for Debian/Ubuntu venv setup failure Signed-off-by: Joffrey F --- script/release/release.sh | 4 ++-- script/release/setup-venv.sh | 25 +++++++++++++++++++++---- 2 files changed, 23 insertions(+), 6 deletions(-) diff --git a/script/release/release.sh b/script/release/release.sh index c10f8aba5..5f853808b 100755 --- a/script/release/release.sh +++ b/script/release/release.sh @@ -1,6 +1,6 @@ #!/bin/sh -if test -d ./.release-venv; then +if test -d ${VENV_DIR:-./.release-venv}; then true else ./script/release/setup-venv.sh @@ -10,4 +10,4 @@ if test -z "$*"; then args="--help" fi -./.release-venv/bin/python ./script/release/release.py "$@" +${VENV_DIR:-./.release-venv}/bin/python ./script/release/release.py "$@" diff --git a/script/release/setup-venv.sh b/script/release/setup-venv.sh index d3d3f9a42..780fc800f 100755 --- a/script/release/setup-venv.sh +++ b/script/release/setup-venv.sh @@ -1,5 +1,11 @@ #!/bin/bash +debian_based() { test -f /etc/debian_version; } + +if test -z $VENV_DIR; then + VENV_DIR=./.release-venv +fi + if test -z $PYTHONBIN; then PYTHONBIN=$(which python3) if test -z $PYTHONBIN; then @@ -16,15 +22,26 @@ if test $(echo $VERSION | cut -d. -f2) -lt 3; then echo "Python 3.3 or above is required" fi -$PYTHONBIN -m venv ./.release-venv +# Debian / Ubuntu workaround: +# https://askubuntu.com/questions/879437/ensurepip-is-disabled-in-debian-ubuntu-for-the-system-python +if debian_based; then + VENV_FLAGS="$VENV_FLAGS --without-pip" +fi -VENVBINS=./.release-venv/bin +$PYTHONBIN -m venv $VENV_DIR $VENV_FLAGS -$VENVBINS/pip install -U Jinja2==2.10 \ +VENV_PYTHONBIN=$VENV_DIR/bin/python + +if debian_based; then + curl https://bootstrap.pypa.io/get-pip.py -o $VENV_DIR/get-pip.py + $VENV_PYTHONBIN $VENV_DIR/get-pip.py +fi + +$VENV_PYTHONBIN -m pip install -U Jinja2==2.10 \ PyGithub==1.39 \ pypandoc==1.4 \ GitPython==2.1.9 \ requests==2.18.4 \ twine==1.11.0 -$VENVBINS/python setup.py develop +$VENV_PYTHONBIN setup.py develop From ca8ab06571933f867d2c4ec633ef4c2d6562533b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 17 Oct 2018 13:39:11 -0700 Subject: [PATCH 31/79] Some additional exclusions in .gitignore / .dockerignore Signed-off-by: Joffrey F --- .dockerignore | 4 +++- .gitignore | 18 ++++++++++-------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/.dockerignore b/.dockerignore index eccd86dda..65ad588d9 100644 --- a/.dockerignore +++ b/.dockerignore @@ -1,11 +1,13 @@ *.egg-info .coverage .git +.github .tox build +binaries coverage-html docs/_site -venv +*venv .tox **/__pycache__ *.pyc diff --git a/.gitignore b/.gitignore index 18afd643d..798882748 100644 --- a/.gitignore +++ b/.gitignore @@ -1,16 +1,18 @@ *.egg-info *.pyc +*.swo +*.swp +.cache .coverage* +.DS_Store +.idea + /.tox +/binaries /build +/compose/GITSHA /coverage-html /dist /docs/_site -/venv -README.rst -compose/GITSHA -*.swo -*.swp -.DS_Store -.cache -.idea +/README.rst +/*venv From 98bb68e404f70cb164125f1f4f8464971d453617 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 24 Oct 2018 15:06:04 -0700 Subject: [PATCH 32/79] Fix new flake8 errors/warnings Signed-off-by: Joffrey F --- compose/cli/errors.py | 2 +- compose/config/types.py | 2 +- compose/config/validation.py | 8 ++++---- tests/acceptance/cli_test.py | 2 +- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 82768970b..8c89da6c5 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -54,7 +54,7 @@ def handle_connection_errors(client): except APIError as e: log_api_error(e, client.api_version) raise ConnectionError() - except (ReadTimeout, socket.timeout) as e: + except (ReadTimeout, socket.timeout): log_timeout_error(client.timeout) raise ConnectionError() except Exception as e: diff --git a/compose/config/types.py b/compose/config/types.py index 838fb9f58..ab8f34e3d 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -125,7 +125,7 @@ def parse_extra_hosts(extra_hosts_config): def normalize_path_for_engine(path): - """Windows paths, c:\my\path\shiny, need to be changed to be compatible with + """Windows paths, c:\\my\\path\\shiny, need to be changed to be compatible with the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ """ drive, tail = splitdrive(path) diff --git a/compose/config/validation.py b/compose/config/validation.py index 0fdcb37e7..87c1f2345 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -41,15 +41,15 @@ DOCKER_CONFIG_HINTS = { } -VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' +VALID_NAME_CHARS = r'[a-zA-Z0-9\._\-]' VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' VALID_IPV4_SEG = r'(\d{1,2}|1\d{2}|2[0-4]\d|25[0-5])' -VALID_IPV4_ADDR = "({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) -VALID_REGEX_IPV4_CIDR = "^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) +VALID_IPV4_ADDR = r"({IPV4_SEG}\.){{3}}{IPV4_SEG}".format(IPV4_SEG=VALID_IPV4_SEG) +VALID_REGEX_IPV4_CIDR = r"^{IPV4_ADDR}/(\d|[1-2]\d|3[0-2])$".format(IPV4_ADDR=VALID_IPV4_ADDR) VALID_IPV6_SEG = r'[0-9a-fA-F]{1,4}' -VALID_REGEX_IPV6_CIDR = "".join(""" +VALID_REGEX_IPV6_CIDR = "".join(r""" ^ ( (({IPV6_SEG}:){{7}}{IPV6_SEG})| diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3d063d853..5b0a0e0fd 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2361,7 +2361,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d']) result = self.dispatch(['logs', '-f', '-t']) - assert re.search('(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})', result.stdout) + assert re.search(r'(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})', result.stdout) def test_logs_tail(self): self.base_dir = 'tests/fixtures/logs-tail-composefile' From 4368b8ac0541bd3d42cf818b04f772f618362790 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 24 Oct 2018 16:08:56 -0700 Subject: [PATCH 33/79] Only use supported protocols when starting engine CLI subprocess Signed-off-by: Joffrey F --- compose/cli/main.py | 4 +++- tests/unit/cli/main_test.py | 8 ++++++++ 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index f2e76c1ad..46b547b00 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1452,7 +1452,9 @@ def call_docker(args, dockeropts): if verify: tls_options.append('--tlsverify') if host: - tls_options.extend(['--host', host.lstrip('=')]) + tls_options.extend( + ['--host', re.sub(r'^https?://', 'tcp://', host.lstrip('='))] + ) args = [executable_path] + tls_options + args log.debug(" ".join(map(pipes.quote, args))) diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index 1a2dfbcf3..2e97f2c87 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -155,6 +155,14 @@ class TestCallDocker(object): 'docker', '--host', 'tcp://mydocker.net:2333', 'ps' ] + def test_with_http_host(self): + with mock.patch('subprocess.call') as fake_call: + call_docker(['ps'], {'--host': 'http://mydocker.net:2333'}) + + assert fake_call.call_args[0][0] == [ + 'docker', '--host', 'tcp://mydocker.net:2333', 'ps', + ] + def test_with_host_option_shorthand_equal(self): with mock.patch('subprocess.call') as fake_call: call_docker(['ps'], {'--host': '=tcp://mydocker.net:2333'}) From e008db5c975cbaaf1a5e7ce2b59b5d37e754aa4a Mon Sep 17 00:00:00 2001 From: Ofek Lev Date: Wed, 17 Oct 2018 17:11:36 -0400 Subject: [PATCH 34/79] Allow requests 2.20.x Signed-off-by: Ofek Lev --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a0093d8dd..8260ebc69 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.20', + 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', 'docker >= 3.5.0, < 4.0', From fd83791d55dd3074fb9c784517198af0d9c43cee Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 29 Oct 2018 14:38:50 -0700 Subject: [PATCH 35/79] Bump requests version in requirements.txt Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 0ea046589..024b671cc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 PyYAML==3.12 -requests==2.19.1 +requests==2.20.0 six==1.10.0 texttable==0.9.1 urllib3==1.21.1; python_version == '3.3' From 147a8e9ab86f83f8bd167157a966b250c5dbafdc Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Oct 2018 14:29:07 -0700 Subject: [PATCH 36/79] Bump next dev version Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index b9088474f..aeca7923f 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.23.0' +__version__ = '1.24.0-dev' From 7925f8cfa805d20487b95dd141208e92ed4111b6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 30 Oct 2018 14:30:11 -0700 Subject: [PATCH 37/79] Fix version Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index aeca7923f..652e1fad9 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.24.0-dev' +__version__ = '1.24.0dev' From 03bdd67eb54780f76753f9f98737b8e5fcf90257 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 13:55:20 -0700 Subject: [PATCH 38/79] Don't attempt to truncate a None value in Container.slug Signed-off-by: Joffrey F --- compose/container.py | 2 ++ tests/unit/container_test.py | 9 ++++++++- 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/compose/container.py b/compose/container.py index 3ee45c8f3..026306866 100644 --- a/compose/container.py +++ b/compose/container.py @@ -96,6 +96,8 @@ class Container(object): @property def slug(self): + if not self.full_slug: + return None return truncate_id(self.full_slug) @property diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 64c9cc344..66c6c1578 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -5,6 +5,7 @@ import docker from .. import mock from .. import unittest +from compose.const import LABEL_SLUG from compose.container import Container from compose.container import get_container_name @@ -87,7 +88,7 @@ class ContainerTest(unittest.TestCase): assert container.name == "composetest_db_1" def test_name_without_project(self): - self.container_dict['Name'] = "/composetest_web_7" + self.container_dict['Name'] = "/composetest_web_7_092cd63296fd" container = Container(None, self.container_dict, has_been_inspected=True) assert container.name_without_project == "web_7_092cd63296fd" @@ -96,6 +97,12 @@ class ContainerTest(unittest.TestCase): container = Container(None, self.container_dict, has_been_inspected=True) assert container.name_without_project == "custom_name_of_container" + def test_name_without_project_noslug(self): + self.container_dict['Name'] = "/composetest_web_7" + del self.container_dict['Config']['Labels'][LABEL_SLUG] + container = Container(None, self.container_dict, has_been_inspected=True) + assert container.name_without_project == 'web_7' + def test_inspect_if_not_inspected(self): mock_client = mock.create_autospec(docker.APIClient) container = Container(mock_client, dict(Id="the_id")) From 8f4d56a6489ef0fa685e6b57e2b3eb286cd9050f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 31 Oct 2018 14:24:35 -0700 Subject: [PATCH 39/79] Impose consistent behavior across command for --project-directory flag Signed-off-by: Joffrey F --- compose/cli/command.py | 10 ++++++---- compose/cli/main.py | 4 ++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 8a32a93a2..339a65c53 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -23,7 +23,8 @@ log = logging.getLogger(__name__) def project_from_options(project_dir, options): - environment = Environment.from_env_file(project_dir) + override_dir = options.get('--project-directory') + environment = Environment.from_env_file(override_dir or project_dir) set_parallel_limit(environment) host = options.get('--host') @@ -37,7 +38,7 @@ def project_from_options(project_dir, options): host=host, tls_config=tls_config_from_options(options, environment), environment=environment, - override_dir=options.get('--project-directory'), + override_dir=override_dir, compatibility=options.get('--compatibility'), ) @@ -59,12 +60,13 @@ def set_parallel_limit(environment): def get_config_from_options(base_dir, options): - environment = Environment.from_env_file(base_dir) + override_dir = options.get('--project-directory') + environment = Environment.from_env_file(override_dir or base_dir) config_path = get_config_path_from_options( base_dir, options, environment ) return config.load( - config.find(base_dir, config_path, environment), + config.find(base_dir, config_path, environment, override_dir), options.get('--compatibility') ) diff --git a/compose/cli/main.py b/compose/cli/main.py index 46b547b00..afe813ee5 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -306,7 +306,7 @@ class TopLevelCommand(object): -o, --output PATH Path to write the bundle file to. Defaults to ".dab". """ - compose_config = get_config_from_options(self.project_dir, self.toplevel_options) + compose_config = get_config_from_options('.', self.toplevel_options) output = options["--output"] if not output: @@ -336,7 +336,7 @@ class TopLevelCommand(object): or use the wildcard symbol to display all services """ - compose_config = get_config_from_options(self.project_dir, self.toplevel_options) + compose_config = get_config_from_options('.', self.toplevel_options) image_digests = None if options['--resolve-image-digests']: From db819bf0b212eb6edb751119c84dcc1aa3b44f94 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Nov 2018 11:35:34 -0700 Subject: [PATCH 40/79] Fix config merging for isolation and storage_opt keys Signed-off-by: Joffrey F --- compose/config/config.py | 2 ++ compose/service.py | 1 + tests/unit/config/config_test.py | 46 ++++++++++++++++++++++++++++++-- 3 files changed, 47 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index fb2c742f4..0298b4e2d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -91,6 +91,7 @@ DOCKER_CONFIG_KEYS = [ 'healthcheck', 'image', 'ipc', + 'isolation', 'labels', 'links', 'mac_address', @@ -1042,6 +1043,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('networks', parse_networks) md.merge_mapping('sysctls', parse_sysctls) md.merge_mapping('depends_on', parse_depends_on) + md.merge_mapping('storage_opt', parse_flat_dict) md.merge_sequence('links', ServiceLink.parse) md.merge_sequence('secrets', types.ServiceSecret.parse) md.merge_sequence('configs', types.ServiceConfig.parse) diff --git a/compose/service.py b/compose/service.py index cc6d16a82..7be2d8feb 100644 --- a/compose/service.py +++ b/compose/service.py @@ -85,6 +85,7 @@ HOST_CONFIG_KEYS = [ 'group_add', 'init', 'ipc', + 'isolation', 'read_only', 'log_driver', 'log_opt', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index bcff21c92..203dfbec8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1343,8 +1343,11 @@ class ConfigTest(unittest.TestCase): mount = config_data.services[0].get('volumes')[0] assert mount.target == '/web' assert mount.type == 'bind' - assert (not mount.source.startswith('~') - and mount.source.endswith('{}web'.format(os.path.sep))) + assert ( + not mount.source.startswith('~') and mount.source.endswith( + '{}web'.format(os.path.sep) + ) + ) def test_config_invalid_ipam_config(self): with pytest.raises(ConfigurationError) as excinfo: @@ -2667,6 +2670,45 @@ class ConfigTest(unittest.TestCase): ['c 7:128 rwm', 'x 3:244 rw', 'f 0:128 n'] ) + def test_merge_isolation(self): + base = { + 'image': 'bar', + 'isolation': 'default', + } + + override = { + 'isolation': 'hyperv', + } + + actual = config.merge_service_dicts(base, override, V2_3) + assert actual == { + 'image': 'bar', + 'isolation': 'hyperv', + } + + def test_merge_storage_opt(self): + base = { + 'image': 'bar', + 'storage_opt': { + 'size': '1G', + 'readonly': 'false', + } + } + + override = { + 'storage_opt': { + 'size': '2G', + 'encryption': 'aes', + } + } + + actual = config.merge_service_dicts(base, override, V2_3) + assert actual['storage_opt'] == { + 'size': '2G', + 'readonly': 'false', + 'encryption': 'aes', + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 5b0292245586ee114342a2b1e48e99ba9c541e39 Mon Sep 17 00:00:00 2001 From: Alex Puschinsky Date: Sat, 3 Nov 2018 18:09:28 +0200 Subject: [PATCH 41/79] Fix ZSH autocomplete for multiple -f flags Signed-off-by: Alex Puschinsky --- contrib/completion/zsh/_docker-compose | 21 +++++++++++++++++---- 1 file changed, 17 insertions(+), 4 deletions(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index eb6199831..e55c91964 100755 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -354,7 +354,7 @@ _docker-compose() { '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 - local -a relevant_compose_flags relevant_docker_flags compose_options docker_options + local -a relevant_compose_flags relevant_compose_repeatable_flags relevant_docker_flags compose_options docker_options relevant_compose_flags=( "--file" "-f" @@ -368,6 +368,10 @@ _docker-compose() { "--skip-hostname-check" ) + relevant_compose_repeatable_flags=( + "--file" "-f" + ) + relevant_docker_flags=( "--host" "-H" "--tls" @@ -385,9 +389,18 @@ _docker-compose() { fi fi if [[ -n "${relevant_compose_flags[(r)$k]}" ]]; then - compose_options+=$k - if [[ -n "$opt_args[$k]" ]]; then - compose_options+=$opt_args[$k] + if [[ -n "${relevant_compose_repeatable_flags[(r)$k]}" ]]; then + values=("${(@s/:/)opt_args[$k]}") + for value in $values + do + compose_options+=$k + compose_options+=$value + done + else + compose_options+=$k + if [[ -n "$opt_args[$k]" ]]; then + compose_options+=$opt_args[$k] + fi fi fi done From d5eb209be04a733fbe118a27b888c2eff3f40c57 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Nov 2018 13:45:15 -0800 Subject: [PATCH 42/79] Fix parse_key_from_error_msg to not error out on non-string keys Signed-off-by: Joffrey F --- compose/config/validation.py | 5 ++++- tests/unit/config/config_test.py | 13 +++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 87c1f2345..039569551 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -330,7 +330,10 @@ def handle_generic_error(error, path): def parse_key_from_error_msg(error): - return error.message.split("'")[1] + try: + return error.message.split("'")[1] + except IndexError: + return error.message.split('(')[1].split(' ')[0].strip("'") def path_string(path): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 203dfbec8..787d8ff4a 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -613,6 +613,19 @@ class ConfigTest(unittest.TestCase): excinfo.exconly() ) + def test_config_integer_service_property_raise_validation_error(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details({ + 'version': '2.1', + 'services': {'foobar': {'image': 'busybox', 1234: 'hah'}} + }, 'working_dir', 'filename.yml') + ) + + assert ( + "Unsupported config option for services.foobar: '1234'" in excinfo.exconly() + ) + def test_config_invalid_service_name_raise_validation_error(self): with pytest.raises(ConfigurationError) as excinfo: config.load( From ba1e0311a7e9c02ff5a2751a62062aedba151314 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Tue, 6 Nov 2018 14:39:53 +0300 Subject: [PATCH 43/79] add option to list all processes Signed-off-by: Collins Abitekaniza --- compose/cli/main.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index afe813ee5..e96aac031 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -694,6 +694,7 @@ class TopLevelCommand(object): -q, --quiet Only display IDs --services Display services --filter KEY=VAL Filter services by a property + -a, --all Shows all stopped containers """ if options['--quiet'] and options['--services']: raise UserError('--quiet and --services cannot be combined') @@ -706,10 +707,14 @@ class TopLevelCommand(object): print('\n'.join(service.name for service in services)) return - containers = sorted( - self.project.containers(service_names=options['SERVICE'], stopped=True) + - self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), - key=attrgetter('name')) + if options['--all']: + containers = sorted(self.project.containers(service_names=options['SERVICE'], + one_off=OneOffFilter.include, stopped=True)) + else: + containers = sorted( + self.project.containers(service_names=options['SERVICE'], stopped=True) + + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), + key=attrgetter('name')) if options['--quiet']: for container in containers: From 05efe52ccd1c1d64a87e906b389ef31d948067e9 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Tue, 6 Nov 2018 14:49:56 +0300 Subject: [PATCH 44/79] test --all flag Signed-off-by: Collins Abitekaniza --- compose/cli/main.py | 2 +- tests/acceptance/cli_test.py | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index e96aac031..f64af8948 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -694,7 +694,7 @@ class TopLevelCommand(object): -q, --quiet Only display IDs --services Display services --filter KEY=VAL Filter services by a property - -a, --all Shows all stopped containers + -a, --all Show all stopped containers """ if options['--quiet'] and options['--services']: raise UserError('--quiet and --services cannot be combined') diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5b0a0e0fd..f42268f2c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -599,6 +599,14 @@ class CLITestCase(DockerClientTestCase): assert 'with_build' in running.stdout assert 'with_image' in running.stdout + def test_ps_all(self): + self.project.get_service('simple').create_container(one_off='blahblah') + result = self.dispatch(['ps']) + assert 'simple-composefile_simple_run_1' not in result.stdout + + result2 = self.dispatch(['ps', '--all']) + assert 'simple-composefile_simple_run_1' in result2.stdout + def test_pull(self): result = self.dispatch(['pull']) assert 'Pulling simple' in result.stderr From e0e06a4b5627cdccf2408f2a2e91125c4d0ea8b8 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Thu, 15 Nov 2018 15:24:50 +0300 Subject: [PATCH 45/79] add detail to description for --all flag Signed-off-by: Collins Abitekaniza --- compose/cli/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index f64af8948..e9c7dbb43 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -694,7 +694,7 @@ class TopLevelCommand(object): -q, --quiet Only display IDs --services Display services --filter KEY=VAL Filter services by a property - -a, --all Show all stopped containers + -a, --all Show all stopped containers (including those created by the run command) """ if options['--quiet'] and options['--services']: raise UserError('--quiet and --services cannot be combined') From 1affc55b17cb68c748ef6f5192705b0a0d536ea0 Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 22 Nov 2018 15:58:41 +0100 Subject: [PATCH 46/79] Adopts 'unknown' as build revision in case git cannot retrieve it. Signed-off-by: Ulysses Souza --- script/build/write-git-sha | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/script/build/write-git-sha b/script/build/write-git-sha index d16743c6f..be87f5058 100755 --- a/script/build/write-git-sha +++ b/script/build/write-git-sha @@ -2,6 +2,11 @@ # # Write the current commit sha to the file GITSHA. This file is included in # packaging so that `docker-compose version` can include the git sha. -# -set -e -git rev-parse --short HEAD > compose/GITSHA +# sets to 'unknown' and echoes a message if the command is not successful + +DOCKER_COMPOSE_GITSHA="$(git rev-parse --short HEAD)" +if [[ "${?}" != "0" ]]; then + echo "Couldn't get revision of the git repository. Setting to 'unknown' instead" + DOCKER_COMPOSE_GITSHA="unknown" +fi +echo "${DOCKER_COMPOSE_GITSHA}" > compose/GITSHA From 6559af7660fd157a21d0abf4d0e1708201a5c5de Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Mon, 19 Nov 2018 15:01:32 +0100 Subject: [PATCH 47/79] Fix one-off commands for "restart: unless-stopped" (fixes #6302) Signed-off-by: Sebastian Pipping --- compose/cli/main.py | 4 ++-- tests/unit/cli_test.py | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index e9c7dbb43..0dc39d660 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1288,8 +1288,8 @@ def build_container_options(options, detach, command): [""] if options['--entrypoint'] == '' else options['--entrypoint'] ) - if options['--rm']: - container_options['restart'] = None + # Ensure that run command remains one-off (issue #6302) + container_options['restart'] = None if options['--user']: container_options['user'] = options.get('--user') diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 7c8a1423c..a7522f939 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -171,7 +171,10 @@ class CLITestCase(unittest.TestCase): '--workdir': None, }) - assert mock_client.create_host_config.call_args[1]['restart_policy']['Name'] == 'always' + # NOTE: The "run" command is supposed to be a one-off tool; therefore restart policy "no" + # (the default) is enforced despite explicit wish for "always" in the project + # configuration file + assert not mock_client.create_host_config.call_args[1].get('restart_policy') command = TopLevelCommand(project) command.run({ From e7f82d298919639980f72d8a85acd20da042940d Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Mon, 26 Nov 2018 23:21:26 +0100 Subject: [PATCH 48/79] Rename build_container_options to build_one_off_container_options .. to better reflect that its scope is limited to one-off execution (i.e. the "run" command) Signed-off-by: Sebastian Pipping --- compose/cli/main.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 0dc39d660..950e5055d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -872,7 +872,7 @@ class TopLevelCommand(object): else: command = service.options.get('command') - container_options = build_container_options(options, detach, command) + container_options = build_one_off_container_options(options, detach, command) run_one_off_container( container_options, self.project, service, options, self.toplevel_options, self.project_dir @@ -1267,7 +1267,7 @@ def build_action_from_opts(options): return BuildAction.none -def build_container_options(options, detach, command): +def build_one_off_container_options(options, detach, command): container_options = { 'command': command, 'tty': not (detach or options['-T'] or not sys.stdin.isatty()), From ccc777831c658c52ce9b32cd3e99833cf82571bf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 26 Nov 2018 15:26:27 -0800 Subject: [PATCH 49/79] Don't add long path prefix to build context URLs Signed-off-by: Joffrey F --- compose/config/__init__.py | 1 + compose/service.py | 3 ++- tests/unit/service_test.py | 27 +++++++++++++++++++++++++++ 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/compose/config/__init__.py b/compose/config/__init__.py index e1032f3de..2b40666f1 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -6,6 +6,7 @@ from . import environment from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find +from .config import is_url from .config import load from .config import merge_environment from .config import merge_labels diff --git a/compose/service.py b/compose/service.py index 7be2d8feb..240ca9cd3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -27,6 +27,7 @@ from . import __version__ from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS +from .config import is_url from .config import merge_environment from .config import merge_labels from .config.errors import DependencyError @@ -1676,7 +1677,7 @@ def rewrite_build_path(path): if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') - if IS_WINDOWS_PLATFORM and not path.startswith(WINDOWS_LONGPATH_PREFIX): + if IS_WINDOWS_PLATFORM and not is_url(path) and not path.startswith(WINDOWS_LONGPATH_PREFIX): path = WINDOWS_LONGPATH_PREFIX + os.path.normpath(path) return path diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index af1cd1bea..6c9ea1511 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -21,6 +21,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import SECRETS_PATH +from compose.const import WINDOWS_LONGPATH_PREFIX from compose.container import Container from compose.errors import OperationFailedError from compose.parallel import ParallelStreamWriter @@ -38,6 +39,7 @@ from compose.service import NeedsBuildError from compose.service import NetworkMode from compose.service import NoSuchImageError from compose.service import parse_repository_tag +from compose.service import rewrite_build_path from compose.service import Service from compose.service import ServiceNetworkMode from compose.service import warn_on_masked_volume @@ -1486,3 +1488,28 @@ class ServiceSecretTest(unittest.TestCase): assert volumes[0].source == secret1['file'] assert volumes[0].target == '{}/{}'.format(SECRETS_PATH, secret1['secret'].source) + + +class RewriteBuildPathTest(unittest.TestCase): + @mock.patch('compose.service.IS_WINDOWS_PLATFORM', True) + def test_rewrite_url_no_prefix(self): + urls = [ + 'http://test.com', + 'https://test.com', + 'git://test.com', + 'github.com/test/test', + 'git@test.com', + ] + for u in urls: + assert rewrite_build_path(u) == u + + @mock.patch('compose.service.IS_WINDOWS_PLATFORM', True) + def test_rewrite_windows_path(self): + assert rewrite_build_path('C:\\context') == WINDOWS_LONGPATH_PREFIX + 'C:\\context' + assert rewrite_build_path( + rewrite_build_path('C:\\context') + ) == rewrite_build_path('C:\\context') + + @mock.patch('compose.service.IS_WINDOWS_PLATFORM', False) + def test_rewrite_unix_path(self): + assert rewrite_build_path('/context') == '/context' From 6ea20e43f68ad6783df1206b45956368c01ca65d Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Tue, 27 Nov 2018 00:24:30 +0100 Subject: [PATCH 50/79] README.md: Drop reference to IRC channel Signed-off-by: Sebastian Pipping --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index ea07f6a7d..a1e391a07 100644 --- a/README.md +++ b/README.md @@ -48,7 +48,6 @@ Installation and documentation ------------------------------ - Full documentation is available on [Docker's website](https://docs.docker.com/compose/). -- If you have any questions, you can talk in real-time with other developers in the #docker-compose IRC channel on Freenode. [Click here to join using IRCCloud.](https://www.irccloud.com/invite?hostname=irc.freenode.net&channel=%23docker-compose) - Code repository for Compose is on [GitHub](https://github.com/docker/compose) - If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new) From 6421ae5ea3eee7a88a88feb5eb89b885e43c9679 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Sun, 18 Nov 2018 18:05:04 +0100 Subject: [PATCH 51/79] README.md: Add a few missing full stops One full stop is moved out of a link and a "Thank you!" is added as well. Signed-off-by: Sebastian Pipping --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index a1e391a07..2750b890f 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file/compose-versioning.md) +[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file/compose-versioning.md). Compose has commands for managing the whole lifecycle of your application: @@ -48,8 +48,8 @@ Installation and documentation ------------------------------ - Full documentation is available on [Docker's website](https://docs.docker.com/compose/). -- Code repository for Compose is on [GitHub](https://github.com/docker/compose) -- If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new) +- Code repository for Compose is on [GitHub](https://github.com/docker/compose). +- If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new). Thank you! Contributing ------------ From 10864ba68733482dbdd1311b0bcfd25c72414382 Mon Sep 17 00:00:00 2001 From: Sebastian Pipping Date: Tue, 27 Nov 2018 00:16:23 +0100 Subject: [PATCH 52/79] README.md: Update bug report link Signed-off-by: Sebastian Pipping --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2750b890f..dd4003048 100644 --- a/README.md +++ b/README.md @@ -49,7 +49,7 @@ Installation and documentation - Full documentation is available on [Docker's website](https://docs.docker.com/compose/). - Code repository for Compose is on [GitHub](https://github.com/docker/compose). -- If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new). Thank you! +- If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new/choose). Thank you! Contributing ------------ From 61bb1ea4849a4d7d2b98a6689d5e1813b06639bd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 27 Nov 2018 17:09:36 -0800 Subject: [PATCH 53/79] Don't append slugs to containers created by "up" This change reverts the new naming convention introduced in 1.23 for service containers. One-off containers will now use a slug instead of a sequential number as they do not present addressability concerns and benefit from being capable of running in parallel. Signed-off-by: Joffrey F --- compose/container.py | 11 +++++++++- compose/service.py | 36 +++++++++++++++++-------------- tests/acceptance/cli_test.py | 23 ++++++++++---------- tests/integration/service_test.py | 9 +++----- tests/integration/state_test.py | 8 +++---- tests/unit/container_test.py | 17 +++++++++------ tests/unit/service_test.py | 4 ++-- 7 files changed, 60 insertions(+), 48 deletions(-) diff --git a/compose/container.py b/compose/container.py index 026306866..8a2fb240e 100644 --- a/compose/container.py +++ b/compose/container.py @@ -7,6 +7,7 @@ import six from docker.errors import ImageNotFound from .const import LABEL_CONTAINER_NUMBER +from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_SLUG @@ -82,12 +83,16 @@ class Container(object): @property def name_without_project(self): if self.name.startswith('{0}_{1}'.format(self.project, self.service)): - return '{0}_{1}{2}'.format(self.service, self.number, '_' + self.slug if self.slug else '') + return '{0}_{1}'.format(self.service, self.number if self.number is not None else self.slug) else: return self.name @property def number(self): + if self.one_off: + # One-off containers are no longer assigned numbers and use slugs instead. + return None + number = self.labels.get(LABEL_CONTAINER_NUMBER) if not number: raise ValueError("Container {0} does not have a {1} label".format( @@ -104,6 +109,10 @@ class Container(object): def full_slug(self): return self.labels.get(LABEL_SLUG) + @property + def one_off(self): + return self.labels.get(LABEL_ONE_OFF) == 'True' + @property def ports(self): self.inspect_if_not_inspected() diff --git a/compose/service.py b/compose/service.py index 240ca9cd3..17d631e8a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -738,16 +738,18 @@ class Service(object): return [s.source.name for s in self.volumes_from if isinstance(s.source, Service)] def _next_container_number(self, one_off=False): + if one_off: + return None containers = itertools.chain( self._fetch_containers( all=True, - filters={'label': self.labels(one_off=one_off)} + filters={'label': self.labels(one_off=False)} ), self._fetch_containers( all=True, - filters={'label': self.labels(one_off=one_off, legacy=True)} + filters={'label': self.labels(one_off=False, legacy=True)} ) ) - numbers = [c.number for c in containers] + numbers = [c.number for c in containers if c.number is not None] return 1 if not numbers else max(numbers) + 1 def _fetch_containers(self, **fetch_options): @@ -825,7 +827,7 @@ class Service(object): one_off=False, previous_container=None): add_config_hash = (not one_off and not override_options) - slug = generate_random_id() if previous_container is None else previous_container.full_slug + slug = generate_random_id() if one_off else None container_options = dict( (k, self.options[k]) @@ -834,7 +836,7 @@ class Service(object): container_options.update(override_options) if not container_options.get('name'): - container_options['name'] = self.get_container_name(self.name, number, slug, one_off) + container_options['name'] = self.get_container_name(self.name, number, slug) container_options.setdefault('detach', True) @@ -1122,12 +1124,12 @@ class Service(object): def custom_container_name(self): return self.options.get('container_name') - def get_container_name(self, service_name, number, slug, one_off=False): - if self.custom_container_name and not one_off: + def get_container_name(self, service_name, number, slug=None): + if self.custom_container_name and slug is None: return self.custom_container_name container_name = build_container_name( - self.project, service_name, number, slug, one_off, + self.project, service_name, number, slug, ) ext_links_origins = [l.split(':')[0] for l in self.options.get('external_links', [])] if container_name in ext_links_origins: @@ -1384,13 +1386,13 @@ class ServiceNetworkMode(object): # Names -def build_container_name(project, service, number, slug, one_off=False): +def build_container_name(project, service, number, slug=None): bits = [project.lstrip('-_'), service] - if one_off: - bits.append('run') - return '_'.join( - bits + ([str(number), truncate_id(slug)] if slug else [str(number)]) - ) + if slug: + bits.extend(['run', truncate_id(slug)]) + else: + bits.append(str(number)) + return '_'.join(bits) # Images @@ -1579,8 +1581,10 @@ def build_mount(mount_spec): def build_container_labels(label_options, service_labels, number, config_hash, slug): labels = dict(label_options or {}) labels.update(label.split('=', 1) for label in service_labels) - labels[LABEL_CONTAINER_NUMBER] = str(number) - labels[LABEL_SLUG] = slug + if number is not None: + labels[LABEL_CONTAINER_NUMBER] = str(number) + if slug is not None: + labels[LABEL_SLUG] = slug labels[LABEL_VERSION] = __version__ if config_hash: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f42268f2c..d49c16073 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -602,10 +602,10 @@ class CLITestCase(DockerClientTestCase): def test_ps_all(self): self.project.get_service('simple').create_container(one_off='blahblah') result = self.dispatch(['ps']) - assert 'simple-composefile_simple_run_1' not in result.stdout + assert 'simple-composefile_simple_run_' not in result.stdout result2 = self.dispatch(['ps', '--all']) - assert 'simple-composefile_simple_run_1' in result2.stdout + assert 'simple-composefile_simple_run_' in result2.stdout def test_pull(self): result = self.dispatch(['pull']) @@ -973,11 +973,11 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['down', '--rmi=local', '--volumes']) assert 'Stopping v2-full_web_1' in result.stderr assert 'Stopping v2-full_other_1' in result.stderr - assert 'Stopping v2-full_web_run_2' in result.stderr + assert 'Stopping v2-full_web_run_' in result.stderr assert 'Removing v2-full_web_1' in result.stderr assert 'Removing v2-full_other_1' in result.stderr - assert 'Removing v2-full_web_run_1' in result.stderr - assert 'Removing v2-full_web_run_2' in result.stderr + assert 'Removing v2-full_web_run_' in result.stderr + assert 'Removing v2-full_web_run_' in result.stderr assert 'Removing volume v2-full_data' in result.stderr assert 'Removing image v2-full_web' in result.stderr assert 'Removing image busybox' not in result.stderr @@ -1039,8 +1039,8 @@ class CLITestCase(DockerClientTestCase): stopped=True )[0].name_without_project - assert '{} | simple'.format(simple_name) in result.stdout - assert '{} | another'.format(another_name) in result.stdout + assert '{} | simple'.format(simple_name) in result.stdout + assert '{} | another'.format(another_name) in result.stdout assert '{} exited with code 0'.format(simple_name) in result.stdout assert '{} exited with code 0'.format(another_name) in result.stdout @@ -2340,10 +2340,9 @@ class CLITestCase(DockerClientTestCase): result = wait_on_process(proc) - assert len(re.findall( - r'logs-restart-composefile_another_1_[a-f0-9]{12} exited with code 1', - result.stdout - )) == 3 + assert result.stdout.count( + r'logs-restart-composefile_another_1 exited with code 1' + ) == 3 assert result.stdout.count('world') == 3 def test_logs_default(self): @@ -2714,7 +2713,7 @@ class CLITestCase(DockerClientTestCase): ) result = wait_on_process(proc, returncode=1) - assert re.findall(r'exit-code-from_another_1_[a-f0-9]{12} exited with code 1', result.stdout) + assert 'exit-code-from_another_1 exited with code 1' in result.stdout def test_exit_code_from_signal_stop(self): self.base_dir = 'tests/fixtures/exit-code-from' diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index edc195287..000f6838c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -32,7 +32,6 @@ from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE -from compose.const import LABEL_SLUG from compose.const import LABEL_VERSION from compose.container import Container from compose.errors import OperationFailedError @@ -1269,16 +1268,15 @@ class ServiceTest(DockerClientTestCase): test that those containers are restarted and not removed/recreated. """ service = self.create_service('web') - valid_numbers = [service._next_container_number(), service._next_container_number()] - service.create_container(number=valid_numbers[0]) - service.create_container(number=valid_numbers[1]) + service.create_container(number=1) + service.create_container(number=2) ParallelStreamWriter.instance = None with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) for container in service.containers(): assert container.is_running - assert container.number in valid_numbers + assert container.number in [1, 2] captured_output = mock_stderr.getvalue() assert 'Creating' not in captured_output @@ -1610,7 +1608,6 @@ class ServiceTest(DockerClientTestCase): labels = ctnr.labels.items() for pair in expected.items(): assert pair in labels - assert ctnr.labels[LABEL_SLUG] == ctnr.full_slug def test_empty_labels(self): labels_dict = {'foo': '', 'bar': ''} diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index a41986f46..b7d38a4ba 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -198,14 +198,14 @@ class ProjectWithDependenciesTest(ProjectTestCase): db, = [c for c in containers if c.service == 'db'] assert set(get_links(web)) == { - 'composetest_db_{}_{}'.format(db.number, db.slug), + 'composetest_db_1', 'db', - 'db_{}_{}'.format(db.number, db.slug) + 'db_1', } assert set(get_links(nginx)) == { - 'composetest_web_{}_{}'.format(web.number, web.slug), + 'composetest_web_1', 'web', - 'web_{}_{}'.format(web.number, web.slug) + 'web_1', } diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 66c6c1578..fde17847a 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -5,6 +5,7 @@ import docker from .. import mock from .. import unittest +from compose.const import LABEL_ONE_OFF from compose.const import LABEL_SLUG from compose.container import Container from compose.container import get_container_name @@ -32,7 +33,6 @@ class ContainerTest(unittest.TestCase): "com.docker.compose.project": "composetest", "com.docker.compose.service": "web", "com.docker.compose.container-number": "7", - "com.docker.compose.slug": "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52" }, } } @@ -88,20 +88,23 @@ class ContainerTest(unittest.TestCase): assert container.name == "composetest_db_1" def test_name_without_project(self): - self.container_dict['Name'] = "/composetest_web_7_092cd63296fd" + self.container_dict['Name'] = "/composetest_web_7" container = Container(None, self.container_dict, has_been_inspected=True) - assert container.name_without_project == "web_7_092cd63296fd" + assert container.name_without_project == "web_7" def test_name_without_project_custom_container_name(self): self.container_dict['Name'] = "/custom_name_of_container" container = Container(None, self.container_dict, has_been_inspected=True) assert container.name_without_project == "custom_name_of_container" - def test_name_without_project_noslug(self): - self.container_dict['Name'] = "/composetest_web_7" - del self.container_dict['Config']['Labels'][LABEL_SLUG] + def test_name_without_project_one_off(self): + self.container_dict['Name'] = "/composetest_web_092cd63296f" + self.container_dict['Config']['Labels'][LABEL_SLUG] = ( + "092cd63296fdc446ad432d3905dd1fcbe12a2ba6b52" + ) + self.container_dict['Config']['Labels'][LABEL_ONE_OFF] = 'True' container = Container(None, self.container_dict, has_been_inspected=True) - assert container.name_without_project == 'web_7' + assert container.name_without_project == 'web_092cd63296fd' def test_inspect_if_not_inspected(self): mock_client = mock.create_autospec(docker.APIClient) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 6c9ea1511..99adea34b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -175,10 +175,10 @@ class ServiceTest(unittest.TestCase): def test_self_reference_external_link(self): service = Service( name='foo', - external_links=['default_foo_1_bdfa3ed91e2c'] + external_links=['default_foo_1'] ) with pytest.raises(DependencyError): - service.get_container_name('foo', 1, 'bdfa3ed91e2c') + service.get_container_name('foo', 1) def test_mem_reservation(self): self.mock_client.create_host_config.return_value = {} From dbe3a6e9a938cc10e9543e17285cc0045c24b965 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Fri, 16 Nov 2018 18:33:54 +0300 Subject: [PATCH 54/79] stdout failed for failing services Signed-off-by: Collins Abitekaniza --- compose/parallel.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/compose/parallel.py b/compose/parallel.py index 34a498ca7..32ee602f4 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -50,7 +50,11 @@ def parallel_execute_watch(events, writer, errors, results, msg, get_name): error_to_reraise = None for obj, result, exception in events: if exception is None: - writer.write(msg, get_name(obj), 'done', green) + if callable(getattr(obj, 'containers', None)) and not obj.containers(): + # If service has no containers started + writer.write(msg, get_name(obj), 'failed', red) + else: + writer.write(msg, get_name(obj), 'done', green) results.append(result) elif isinstance(exception, ImageNotFound): # This is to bubble up ImageNotFound exceptions to the client so we From b8b6199958eaa77c7364b07431d1f6b2e8c1e220 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Fri, 16 Nov 2018 18:34:18 +0300 Subject: [PATCH 55/79] refactor cli tests Signed-off-by: Collins Abitekaniza --- tests/acceptance/cli_test.py | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f42268f2c..d808da9c5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2229,6 +2229,7 @@ class CLITestCase(DockerClientTestCase): def test_start_no_containers(self): result = self.dispatch(['start'], returncode=1) + assert 'failed' in result.stderr assert 'No containers to start' in result.stderr @v2_only() From d1bf27e73a59a6d66434dfdd7231965a3a101880 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 11:53:26 -0800 Subject: [PATCH 56/79] Bump SDK version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 024b671cc..45ed9049d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==3.5.0 +docker==3.6.0 docker-pycreds==0.3.0 dockerpty==0.4.1 docopt==0.6.2 diff --git a/setup.py b/setup.py index 8260ebc69..22dafdb22 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.5.0, < 4.0', + 'docker >= 3.6.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From d9e05f262fd98880b3809fe3b45c998b325ad6cf Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Thu, 22 Nov 2018 11:34:34 +0100 Subject: [PATCH 57/79] Avoids pushing the same image more than once. Signed-off-by: Ulysses Souza --- compose/project.py | 10 +++++++++- tests/integration/project_test.py | 18 ++++++++++++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 92c352050..8fab09e54 100644 --- a/compose/project.py +++ b/compose/project.py @@ -29,6 +29,7 @@ from .service import ContainerNetworkMode from .service import ContainerPidMode from .service import ConvergenceStrategy from .service import NetworkMode +from .service import parse_repository_tag from .service import PidMode from .service import Service from .service import ServiceNetworkMode @@ -592,8 +593,15 @@ class Project(object): service.pull(ignore_pull_failures, silent=silent) def push(self, service_names=None, ignore_push_failures=False): + unique_images = set() for service in self.get_services(service_names, include_deps=False): - service.push(ignore_push_failures) + # Considering and as the same + repo, tag, sep = parse_repository_tag(service.image_name) + service_image_name = sep.join((repo, tag)) if tag else sep.join((repo, 'latest')) + + if service_image_name not in unique_images: + service.push(ignore_push_failures) + unique_images.add(service_image_name) def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude): ctnrs = list(filter(None, [ diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 57f3b7074..203b7bb3d 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1995,3 +1995,21 @@ class ProjectTest(DockerClientTestCase): net_data = self.client.inspect_network(full_net_name) assert net_data assert net_data['Labels'][LABEL_PROJECT] == '-dashtest' + + def test_avoid_multiple_push(self): + service_config_latest = {'image': 'busybox:latest', 'build': '.'} + service_config_default = {'image': 'busybox', 'build': '.'} + service_config_sha = { + 'image': 'busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d', + 'build': '.' + } + svc1 = self.create_service('busy1', **service_config_latest) + svc1_1 = self.create_service('busy11', **service_config_latest) + svc2 = self.create_service('busy2', **service_config_default) + svc2_1 = self.create_service('busy21', **service_config_default) + svc3 = self.create_service('busy3', **service_config_sha) + svc3_1 = self.create_service('busy31', **service_config_sha) + project = Project('composetest', [svc1, svc1_1, svc2, svc2_1, svc3, svc3_1], self.client) + with mock.patch('compose.service.Service.push') as fake_push: + project.push(ignore_push_failures=True) + assert fake_push.call_count == 2 From a7894ddfea6298ee1d4e07fe703f0edfe1996842 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 28 Nov 2018 14:19:21 -0800 Subject: [PATCH 58/79] Fix incorrect pre-create container name in up logs Signed-off-by: Joffrey F --- compose/service.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index 17d631e8a..f6dfa7c72 100644 --- a/compose/service.py +++ b/compose/service.py @@ -129,7 +129,7 @@ class NoSuchImageError(Exception): pass -ServiceName = namedtuple('ServiceName', 'project service number slug') +ServiceName = namedtuple('ServiceName', 'project service number') ConvergencePlan = namedtuple('ConvergencePlan', 'action containers') @@ -447,13 +447,11 @@ class Service(object): containers, errors = parallel_execute( [ - ServiceName(self.project, self.name, index, generate_random_id()) + ServiceName(self.project, self.name, index) for index in range(i, i + scale) ], lambda service_name: create_and_start(self, service_name.number), - lambda service_name: self.get_container_name( - service_name.service, service_name.number, service_name.slug - ), + lambda service_name: self.get_container_name(service_name.service, service_name.number), "Creating" ) for error in errors.values(): From d563a6640539ad6395d69561c719d886c0d1861c Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Fri, 30 Nov 2018 23:30:55 +0100 Subject: [PATCH 59/79] Update `reorder_python_imports` version to fix Unicode problems Signed-off-by: Ulysses Souza --- .pre-commit-config.yaml | 2 +- compose/utils.py | 1 - tests/acceptance/cli_test.py | 1 - 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index b7bcc8466..e447294eb 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,7 @@ - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports - sha: v0.3.5 + sha: v1.3.4 hooks: - id: reorder-python-imports language_version: 'python2.7' diff --git a/compose/utils.py b/compose/utils.py index 9f0441d08..a1e5e6435 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import codecs import hashlib -import json import json.decoder import logging import ntpath diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index d49c16073..b429e3567 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -4,7 +4,6 @@ from __future__ import unicode_literals import datetime import json -import os import os.path import re import signal From 7b82b2e8c721010b73f664e9d4657746a1fcd92b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 16:24:38 -0800 Subject: [PATCH 60/79] Add SSH-enabled docker SDK to requirements Signed-off-by: Joffrey F --- requirements.txt | 1 + setup.py | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 45ed9049d..fbb285b6b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ functools32==3.2.3.post2; python_version < '3.2' idna==2.5 ipaddress==1.0.18 jsonschema==2.6.0 +paramiko==2.4.2 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 diff --git a/setup.py b/setup.py index 22dafdb22..9efc642c4 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 3.6.0, < 4.0', + 'docker[ssh] >= 3.6.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From fc3df83d39b63d3a67db7650e858d92803cb1033 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Nov 2018 17:59:55 -0800 Subject: [PATCH 61/79] Update setup.py for modern pypi /setuptools Remove pandoc dependencies Signed-off-by: Joffrey F --- MANIFEST.in | 3 +-- script/release/push-release | 8 -------- script/release/release.py | 4 ---- script/release/setup-venv.sh | 2 +- setup.py | 17 ++++++++++++----- 5 files changed, 14 insertions(+), 20 deletions(-) diff --git a/MANIFEST.in b/MANIFEST.in index 8c6f932ba..fca685eaa 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,8 +4,7 @@ include requirements.txt include requirements-dev.txt include tox.ini include *.md -exclude README.md -include README.rst +include README.md include compose/config/*.json include compose/GITSHA recursive-include contrib/completion * diff --git a/script/release/push-release b/script/release/push-release index 0578aaff8..f28c1d4fe 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -26,12 +26,6 @@ if [ -z "$(command -v jq 2> /dev/null)" ]; then fi -if [ -z "$(command -v pandoc 2> /dev/null)" ]; then - >&2 echo "$0 requires http://pandoc.org/" - >&2 echo "Please install it and make sure it is available on your \$PATH." - exit 2 -fi - API=https://api.github.com/repos REPO=docker/compose GITHUB_REPO=git@github.com:$REPO @@ -59,8 +53,6 @@ docker push docker/compose-tests:latest docker push docker/compose-tests:$VERSION echo "Uploading package to PyPI" -pandoc -f markdown -t rst README.md -o README.rst -sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst ./script/build/write-git-sha python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then diff --git a/script/release/release.py b/script/release/release.py index 6574bfddd..63bf863df 100755 --- a/script/release/release.py +++ b/script/release/release.py @@ -9,7 +9,6 @@ import sys import time from distutils.core import run_setup -import pypandoc from jinja2 import Template from release.bintray import BintrayAPI from release.const import BINTRAY_ORG @@ -277,9 +276,6 @@ def finalize(args): repository.checkout_branch(br_name) - pypandoc.convert_file( - os.path.join(REPO_ROOT, 'README.md'), 'rst', outputfile=os.path.join(REPO_ROOT, 'README.rst') - ) run_setup(os.path.join(REPO_ROOT, 'setup.py'), script_args=['sdist', 'bdist_wheel']) merge_status = pr_data.merge() diff --git a/script/release/setup-venv.sh b/script/release/setup-venv.sh index 780fc800f..ab419be0c 100755 --- a/script/release/setup-venv.sh +++ b/script/release/setup-venv.sh @@ -39,9 +39,9 @@ fi $VENV_PYTHONBIN -m pip install -U Jinja2==2.10 \ PyGithub==1.39 \ - pypandoc==1.4 \ GitPython==2.1.9 \ requests==2.18.4 \ + setuptools==40.6.2 \ twine==1.11.0 $VENV_PYTHONBIN setup.py develop diff --git a/setup.py b/setup.py index 9efc642c4..4c49bab7b 100644 --- a/setup.py +++ b/setup.py @@ -77,19 +77,26 @@ setup( name='docker-compose', version=find_version("compose", "__init__.py"), description='Multi-container orchestration for Docker', + long_description=read('README.md'), + long_description_content_type='text/markdown', url='https://www.docker.com/', + project_urls={ + 'Documentation': 'https://docs.docker.com/compose/overview', + 'Changelog': 'https://github.com/docker/compose/blob/release/CHANGELOG.md', + 'Source': 'https://github.com/docker/compose', + 'Tracker': 'https://github.com/docker/compose/issues', + }, author='Docker, Inc.', license='Apache License 2.0', packages=find_packages(exclude=['tests.*', 'tests']), include_package_data=True, - test_suite='nose.collector', install_requires=install_requires, extras_require=extras_require, tests_require=tests_require, - entry_points=""" - [console_scripts] - docker-compose=compose.cli.main:main - """, + python_requires='>=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*', + entry_points={ + 'console_scripts': ['docker-compose=compose.cli.main:main'], + }, classifiers=[ 'Development Status :: 5 - Production/Stable', 'Environment :: Console', From d3933cd34a88e811b9612d728569fc9be6d560d4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 4 Dec 2018 17:13:40 -0800 Subject: [PATCH 62/79] Move multi-push test to unit tests Signed-off-by: Joffrey F --- tests/integration/project_test.py | 18 ------------------ tests/unit/project_test.py | 20 ++++++++++++++++++++ 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 203b7bb3d..57f3b7074 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1995,21 +1995,3 @@ class ProjectTest(DockerClientTestCase): net_data = self.client.inspect_network(full_net_name) assert net_data assert net_data['Labels'][LABEL_PROJECT] == '-dashtest' - - def test_avoid_multiple_push(self): - service_config_latest = {'image': 'busybox:latest', 'build': '.'} - service_config_default = {'image': 'busybox', 'build': '.'} - service_config_sha = { - 'image': 'busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d', - 'build': '.' - } - svc1 = self.create_service('busy1', **service_config_latest) - svc1_1 = self.create_service('busy11', **service_config_latest) - svc2 = self.create_service('busy2', **service_config_default) - svc2_1 = self.create_service('busy21', **service_config_default) - svc3 = self.create_service('busy3', **service_config_sha) - svc3_1 = self.create_service('busy31', **service_config_sha) - project = Project('composetest', [svc1, svc1_1, svc2, svc2_1, svc3, svc3_1], self.client) - with mock.patch('compose.service.Service.push') as fake_push: - project.push(ignore_push_failures=True) - assert fake_push.call_count == 2 diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 1cc841814..f17bc571e 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -620,3 +620,23 @@ class ProjectTest(unittest.TestCase): self.mock_client.pull.side_effect = OperationFailedError(b'pull error') with pytest.raises(ProjectError): project.pull(parallel_pull=True) + + def test_avoid_multiple_push(self): + service_config_latest = {'image': 'busybox:latest', 'build': '.'} + service_config_default = {'image': 'busybox', 'build': '.'} + service_config_sha = { + 'image': 'busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d', + 'build': '.' + } + svc1 = Service('busy1', **service_config_latest) + svc1_1 = Service('busy11', **service_config_latest) + svc2 = Service('busy2', **service_config_default) + svc2_1 = Service('busy21', **service_config_default) + svc3 = Service('busy3', **service_config_sha) + svc3_1 = Service('busy31', **service_config_sha) + project = Project( + 'composetest', [svc1, svc1_1, svc2, svc2_1, svc3, svc3_1], self.mock_client + ) + with mock.patch('compose.service.Service.push') as fake_push: + project.push() + assert fake_push.call_count == 2 From afc161a0b1ce74cdcbc9be84d1dca58b10f0daf1 Mon Sep 17 00:00:00 2001 From: Hiroshi Ioka Date: Fri, 7 Dec 2018 17:12:40 +0900 Subject: [PATCH 63/79] reject environment variable that contains white spaces Signed-off-by: Hiroshi Ioka --- compose/config/environment.py | 29 ++++++++++++++++++++------- compose/config/errors.py | 4 ++++ tests/unit/config/environment_test.py | 10 +++++++++ 3 files changed, 36 insertions(+), 7 deletions(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 0087b6128..675ab10eb 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -5,22 +5,33 @@ import codecs import contextlib import logging import os +import string import six from ..const import IS_WINDOWS_PLATFORM from .errors import ConfigurationError +from .errors import EnvFileNotFound log = logging.getLogger(__name__) +whitespace = set(string.whitespace) + def split_env(env): if isinstance(env, six.binary_type): env = env.decode('utf-8', 'replace') + key = value = None if '=' in env: - return env.split('=', 1) + key, value = env.split('=', 1) else: - return env, None + key = env + for k in key: + if k in whitespace: + raise ConfigurationError( + "environment variable name '%s' may not contains white spaces." % key + ) + return key, value def env_vars_from_file(filename): @@ -28,16 +39,19 @@ def env_vars_from_file(filename): Read in a line delimited file of environment variables. """ if not os.path.exists(filename): - raise ConfigurationError("Couldn't find env file: %s" % filename) + raise EnvFileNotFound("Couldn't find env file: %s" % filename) elif not os.path.isfile(filename): - raise ConfigurationError("%s is not a file." % (filename)) + raise EnvFileNotFound("%s is not a file." % (filename)) env = {} with contextlib.closing(codecs.open(filename, 'r', 'utf-8-sig')) as fileobj: for line in fileobj: line = line.strip() if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v + try: + k, v = split_env(line) + env[k] = v + except ConfigurationError as e: + raise ConfigurationError('In file {}: {}'.format(filename, e.msg)) return env @@ -55,9 +69,10 @@ class Environment(dict): env_file_path = os.path.join(base_dir, '.env') try: return cls(env_vars_from_file(env_file_path)) - except ConfigurationError: + except EnvFileNotFound: pass return result + instance = _initialize() instance.update(os.environ) return instance diff --git a/compose/config/errors.py b/compose/config/errors.py index f5c038088..9b2078f2c 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -19,6 +19,10 @@ class ConfigurationError(Exception): return self.msg +class EnvFileNotFound(ConfigurationError): + pass + + class DependencyError(ConfigurationError): pass diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py index 854aee5a3..88eb0d6e1 100644 --- a/tests/unit/config/environment_test.py +++ b/tests/unit/config/environment_test.py @@ -9,6 +9,7 @@ import pytest from compose.config.environment import env_vars_from_file from compose.config.environment import Environment +from compose.config.errors import ConfigurationError from tests import unittest @@ -52,3 +53,12 @@ class EnvironmentTest(unittest.TestCase): assert env_vars_from_file(str(tmpdir.join('bom.env'))) == { 'PARK_BOM': '박봄' } + + def test_env_vars_from_file_whitespace(self): + tmpdir = pytest.ensuretemp('env_file') + self.addCleanup(tmpdir.remove) + with codecs.open('{}/whitespace.env'.format(str(tmpdir)), 'w', encoding='utf-8') as f: + f.write('WHITESPACE =yes\n') + with pytest.raises(ConfigurationError) as exc: + env_vars_from_file(str(tmpdir.join('whitespace.env'))) + assert 'environment variable' in exc.exconly() From a2bcf526652164a8e1d440052a56a9e5f190f18e Mon Sep 17 00:00:00 2001 From: Ulysses Souza Date: Sat, 1 Dec 2018 03:15:05 +0100 Subject: [PATCH 64/79] Fix merge on networks section Signed-off-by: Ulysses Souza --- compose/config/config.py | 18 ++++- tests/unit/config/config_test.py | 112 +++++++++++++++++++++++++++++-- 2 files changed, 125 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0298b4e2d..f8c9773b3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1040,7 +1040,6 @@ def merge_service_dicts(base, override, version): md.merge_mapping('environment', parse_environment) md.merge_mapping('labels', parse_labels) md.merge_mapping('ulimits', parse_flat_dict) - md.merge_mapping('networks', parse_networks) md.merge_mapping('sysctls', parse_sysctls) md.merge_mapping('depends_on', parse_depends_on) md.merge_mapping('storage_opt', parse_flat_dict) @@ -1050,6 +1049,7 @@ def merge_service_dicts(base, override, version): md.merge_sequence('security_opt', types.SecurityOpt.parse) md.merge_mapping('extra_hosts', parse_extra_hosts) + md.merge_field('networks', merge_networks, default={}) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) @@ -1154,6 +1154,22 @@ def merge_deploy(base, override): return dict(md) +def merge_networks(base, override): + merged_networks = {} + all_network_names = set(base) | set(override) + base = {k: {} for k in base} if isinstance(base, list) else base + override = {k: {} for k in override} if isinstance(override, list) else override + for network_name in all_network_names: + md = MergeDict(base.get(network_name, {}), override.get(network_name, {})) + md.merge_field('aliases', merge_unique_items_lists, []) + md.merge_field('link_local_ips', merge_unique_items_lists, []) + md.merge_scalar('priority') + md.merge_scalar('ipv4_address') + md.merge_scalar('ipv6_address') + merged_networks[network_name] = dict(md) + return merged_networks + + def merge_reservations(base, override): md = MergeDict(base, override) md.merge_scalar('cpus') diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 787d8ff4a..39ff374d4 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1085,8 +1085,43 @@ class ConfigTest(unittest.TestCase): details = config.ConfigDetails('.', [base_file, override_file]) web_service = config.load(details).services[0] assert web_service['networks'] == { - 'foobar': {'aliases': ['foo', 'bar']}, - 'baz': None + 'foobar': {'aliases': ['bar', 'foo']}, + 'baz': {} + } + + def test_load_with_multiple_files_mismatched_networks_format_inverse_order(self): + base_file = config.ConfigFile( + 'override.yaml', + { + 'version': '2', + 'services': { + 'web': { + 'networks': ['baz'] + } + } + } + ) + override_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2', + 'services': { + 'web': { + 'image': 'example/web', + 'networks': { + 'foobar': {'aliases': ['foo', 'bar']} + } + } + }, + 'networks': {'foobar': {}, 'baz': {}} + } + ) + + details = config.ConfigDetails('.', [base_file, override_file]) + web_service = config.load(details).services[0] + assert web_service['networks'] == { + 'foobar': {'aliases': ['bar', 'foo']}, + 'baz': {} } def test_load_with_multiple_files_v2(self): @@ -3843,8 +3878,77 @@ class MergePortsTest(unittest.TestCase, MergeListsTest): class MergeNetworksTest(unittest.TestCase, MergeListsTest): config_name = 'networks' - base_config = ['frontend', 'backend'] - override_config = ['monitoring'] + base_config = {'default': {'aliases': ['foo.bar', 'foo.baz']}} + override_config = {'default': {'ipv4_address': '123.234.123.234'}} + + def test_no_network_overrides(self): + service_dict = config.merge_service_dicts( + {self.config_name: self.base_config}, + {self.config_name: self.override_config}, + DEFAULT_VERSION) + assert service_dict[self.config_name] == { + 'default': { + 'aliases': ['foo.bar', 'foo.baz'], + 'ipv4_address': '123.234.123.234' + } + } + + def test_all_properties(self): + service_dict = config.merge_service_dicts( + {self.config_name: { + 'default': { + 'aliases': ['foo.bar', 'foo.baz'], + 'link_local_ips': ['192.168.1.10', '192.168.1.11'], + 'ipv4_address': '111.111.111.111', + 'ipv6_address': 'FE80:CD00:0000:0CDE:1257:0000:211E:729C-first' + } + }}, + {self.config_name: { + 'default': { + 'aliases': ['foo.baz', 'foo.baz2'], + 'link_local_ips': ['192.168.1.11', '192.168.1.12'], + 'ipv4_address': '123.234.123.234', + 'ipv6_address': 'FE80:CD00:0000:0CDE:1257:0000:211E:729C-second' + } + }}, + DEFAULT_VERSION) + + assert service_dict[self.config_name] == { + 'default': { + 'aliases': ['foo.bar', 'foo.baz', 'foo.baz2'], + 'link_local_ips': ['192.168.1.10', '192.168.1.11', '192.168.1.12'], + 'ipv4_address': '123.234.123.234', + 'ipv6_address': 'FE80:CD00:0000:0CDE:1257:0000:211E:729C-second' + } + } + + def test_no_network_name_overrides(self): + service_dict = config.merge_service_dicts( + { + self.config_name: { + 'default': { + 'aliases': ['foo.bar', 'foo.baz'], + 'ipv4_address': '123.234.123.234' + } + } + }, + { + self.config_name: { + 'another_network': { + 'ipv4_address': '123.234.123.234' + } + } + }, + DEFAULT_VERSION) + assert service_dict[self.config_name] == { + 'default': { + 'aliases': ['foo.bar', 'foo.baz'], + 'ipv4_address': '123.234.123.234' + }, + 'another_network': { + 'ipv4_address': '123.234.123.234' + } + } class MergeStringsOrListsTest(unittest.TestCase): From 8b293d486e6276555bf94ded4e8559503502dff3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 10 Dec 2018 17:52:52 -0800 Subject: [PATCH 65/79] Use improved API fields for project events when possible Signed-off-by: Joffrey F --- compose/cli/log_printer.py | 3 +- compose/const.py | 1 - compose/project.py | 70 ++++++++++++--- tests/unit/project_test.py | 172 ++++++++++++++++++++++++++++++++++++- 4 files changed, 233 insertions(+), 13 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index bd6723ef2..8aa93a844 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -236,7 +236,8 @@ def watch_events(thread_map, event_stream, presenters, thread_args): thread_map[event['id']] = build_thread( event['container'], next(presenters), - *thread_args) + *thread_args + ) def consume_queue(queue, cascade_stop): diff --git a/compose/const.py b/compose/const.py index 0e66a297a..46d81ae71 100644 --- a/compose/const.py +++ b/compose/const.py @@ -7,7 +7,6 @@ from .version import ComposeVersion DEFAULT_TIMEOUT = 10 HTTP_TIMEOUT = 60 -IMAGE_EVENTS = ['delete', 'import', 'load', 'pull', 'push', 'save', 'tag', 'untag'] IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' diff --git a/compose/project.py b/compose/project.py index 8fab09e54..5c4ce6e17 100644 --- a/compose/project.py +++ b/compose/project.py @@ -10,13 +10,13 @@ from functools import reduce import enum import six from docker.errors import APIError +from docker.utils import version_lt from . import parallel from .config import ConfigurationError from .config.config import V1 from .config.sort_services import get_container_name_from_network_mode from .config.sort_services import get_service_name_from_network_mode -from .const import IMAGE_EVENTS from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE @@ -402,11 +402,13 @@ class Project(object): detached=True, start=False) - def events(self, service_names=None): + def _legacy_event_processor(self, service_names): + # Only for v1 files or when Compose is forced to use an older API version def build_container_event(event, container): time = datetime.datetime.fromtimestamp(event['time']) time = time.replace( - microsecond=microseconds_from_time_nano(event['timeNano'])) + microsecond=microseconds_from_time_nano(event['timeNano']) + ) return { 'time': time, 'type': 'container', @@ -425,17 +427,15 @@ class Project(object): filters={'label': self.labels()}, decode=True ): - # The first part of this condition is a guard against some events - # broadcasted by swarm that don't have a status field. + # This is a guard against some events broadcasted by swarm that + # don't have a status field. # See https://github.com/docker/compose/issues/3316 - if 'status' not in event or event['status'] in IMAGE_EVENTS: - # We don't receive any image events because labels aren't applied - # to images + if 'status' not in event: continue - # TODO: get labels from the API v1.22 , see github issue 2618 try: - # this can fail if the container has been removed + # this can fail if the container has been removed or if the event + # refers to an image container = Container.from_id(self.client, event['id']) except APIError: continue @@ -443,6 +443,56 @@ class Project(object): continue yield build_container_event(event, container) + def events(self, service_names=None): + if version_lt(self.client.api_version, '1.22'): + # New, better event API was introduced in 1.22. + return self._legacy_event_processor(service_names) + + def build_container_event(event): + container_attrs = event['Actor']['Attributes'] + time = datetime.datetime.fromtimestamp(event['time']) + time = time.replace( + microsecond=microseconds_from_time_nano(event['timeNano']) + ) + + container = None + try: + container = Container.from_id(self.client, event['id']) + except APIError: + # Container may have been removed (e.g. if this is a destroy event) + pass + + return { + 'time': time, + 'type': 'container', + 'action': event['status'], + 'id': event['Actor']['ID'], + 'service': container_attrs.get(LABEL_SERVICE), + 'attributes': dict([ + (k, v) for k, v in container_attrs.items() + if not k.startswith('com.docker.compose.') + ]), + 'container': container, + } + + def yield_loop(service_names): + for event in self.client.events( + filters={'label': self.labels()}, + decode=True + ): + # TODO: support other event types + if event.get('Type') != 'container': + continue + + try: + if event['Actor']['Attributes'][LABEL_SERVICE] not in service_names: + continue + except KeyError: + continue + yield build_container_event(event) + + return yield_loop(set(service_names) if service_names else self.service_names) + def up(self, service_names=None, start_deps=True, diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f17bc571e..4aea91a0d 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -254,9 +254,10 @@ class ProjectTest(unittest.TestCase): [container_ids[0] + ':rw'] ) - def test_events(self): + def test_events_legacy(self): services = [Service(name='web'), Service(name='db')] project = Project('test', services, self.mock_client) + self.mock_client.api_version = '1.21' self.mock_client.events.return_value = iter([ { 'status': 'create', @@ -362,6 +363,175 @@ class ProjectTest(unittest.TestCase): }, ] + def test_events(self): + services = [Service(name='web'), Service(name='db')] + project = Project('test', services, self.mock_client) + self.mock_client.api_version = '1.35' + self.mock_client.events.return_value = iter([ + { + 'status': 'create', + 'from': 'example/image', + 'Type': 'container', + 'Actor': { + 'ID': 'abcde', + 'Attributes': { + 'com.docker.compose.project': 'test', + 'com.docker.compose.service': 'web', + 'image': 'example/image', + 'name': 'test_web_1', + } + }, + 'id': 'abcde', + 'time': 1420092061, + 'timeNano': 14200920610000002000, + }, + { + 'status': 'attach', + 'from': 'example/image', + 'Type': 'container', + 'Actor': { + 'ID': 'abcde', + 'Attributes': { + 'com.docker.compose.project': 'test', + 'com.docker.compose.service': 'web', + 'image': 'example/image', + 'name': 'test_web_1', + } + }, + 'id': 'abcde', + 'time': 1420092061, + 'timeNano': 14200920610000003000, + }, + { + 'status': 'create', + 'from': 'example/other', + 'Type': 'container', + 'Actor': { + 'ID': 'bdbdbd', + 'Attributes': { + 'image': 'example/other', + 'name': 'shrewd_einstein', + } + }, + 'id': 'bdbdbd', + 'time': 1420092061, + 'timeNano': 14200920610000005000, + }, + { + 'status': 'create', + 'from': 'example/db', + 'Type': 'container', + 'Actor': { + 'ID': 'ababa', + 'Attributes': { + 'com.docker.compose.project': 'test', + 'com.docker.compose.service': 'db', + 'image': 'example/db', + 'name': 'test_db_1', + } + }, + 'id': 'ababa', + 'time': 1420092061, + 'timeNano': 14200920610000004000, + }, + { + 'status': 'destroy', + 'from': 'example/db', + 'Type': 'container', + 'Actor': { + 'ID': 'eeeee', + 'Attributes': { + 'com.docker.compose.project': 'test', + 'com.docker.compose.service': 'db', + 'image': 'example/db', + 'name': 'test_db_1', + } + }, + 'id': 'eeeee', + 'time': 1420092061, + 'timeNano': 14200920610000004000, + }, + ]) + + def dt_with_microseconds(dt, us): + return datetime.datetime.fromtimestamp(dt).replace(microsecond=us) + + def get_container(cid): + if cid == 'eeeee': + raise NotFound(None, None, "oops") + if cid == 'abcde': + name = 'web' + labels = {LABEL_SERVICE: name} + elif cid == 'ababa': + name = 'db' + labels = {LABEL_SERVICE: name} + else: + labels = {} + name = '' + return { + 'Id': cid, + 'Config': {'Labels': labels}, + 'Name': '/project_%s_1' % name, + } + + self.mock_client.inspect_container.side_effect = get_container + + events = project.events() + + events_list = list(events) + # Assert the return value is a generator + assert not list(events) + assert events_list == [ + { + 'type': 'container', + 'service': 'web', + 'action': 'create', + 'id': 'abcde', + 'attributes': { + 'name': 'test_web_1', + 'image': 'example/image', + }, + 'time': dt_with_microseconds(1420092061, 2), + 'container': Container(None, get_container('abcde')), + }, + { + 'type': 'container', + 'service': 'web', + 'action': 'attach', + 'id': 'abcde', + 'attributes': { + 'name': 'test_web_1', + 'image': 'example/image', + }, + 'time': dt_with_microseconds(1420092061, 3), + 'container': Container(None, get_container('abcde')), + }, + { + 'type': 'container', + 'service': 'db', + 'action': 'create', + 'id': 'ababa', + 'attributes': { + 'name': 'test_db_1', + 'image': 'example/db', + }, + 'time': dt_with_microseconds(1420092061, 4), + 'container': Container(None, get_container('ababa')), + }, + { + 'type': 'container', + 'service': 'db', + 'action': 'destroy', + 'id': 'eeeee', + 'attributes': { + 'name': 'test_db_1', + 'image': 'example/db', + }, + 'time': dt_with_microseconds(1420092061, 4), + 'container': None, + }, + ] + def test_net_unset(self): project = Project.from_config( name='test', From 0323920957f42f134e0690f30915a881d7f12986 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 14 Dec 2018 14:36:40 -0800 Subject: [PATCH 66/79] Style and language fixes Signed-off-by: Joffrey F --- compose/config/environment.py | 17 +++++++---------- 1 file changed, 7 insertions(+), 10 deletions(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 675ab10eb..bd52758f2 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -5,7 +5,7 @@ import codecs import contextlib import logging import os -import string +import re import six @@ -15,8 +15,6 @@ from .errors import EnvFileNotFound log = logging.getLogger(__name__) -whitespace = set(string.whitespace) - def split_env(env): if isinstance(env, six.binary_type): @@ -26,11 +24,10 @@ def split_env(env): key, value = env.split('=', 1) else: key = env - for k in key: - if k in whitespace: - raise ConfigurationError( - "environment variable name '%s' may not contains white spaces." % key - ) + if re.search(r'\s', key): + raise ConfigurationError( + "environment variable name '{}' may not contains whitespace.".format(key) + ) return key, value @@ -39,9 +36,9 @@ def env_vars_from_file(filename): Read in a line delimited file of environment variables. """ if not os.path.exists(filename): - raise EnvFileNotFound("Couldn't find env file: %s" % filename) + raise EnvFileNotFound("Couldn't find env file: {}".format(filename)) elif not os.path.isfile(filename): - raise EnvFileNotFound("%s is not a file." % (filename)) + raise EnvFileNotFound("{} is not a file.".format(filename)) env = {} with contextlib.closing(codecs.open(filename, 'r', 'utf-8-sig')) as fileobj: for line in fileobj: From fee5261014ff1f0a867e2f871a82cc3ade78e3cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 19 Dec 2018 00:24:33 -0800 Subject: [PATCH 67/79] Always connect Compose container to stdin Signed-off-by: Joffrey F --- script/run/run.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/script/run/run.sh b/script/run/run.sh index d3069ff78..cc36e4751 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -47,14 +47,14 @@ if [ -n "$HOME" ]; then fi # Only allocate tty if we detect one -if [ -t 0 ]; then - if [ -t 1 ]; then +if [ -t 0 -a -t 1 ]; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -t" - fi -else - DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" fi +# Always set -i to support piped and terminal input in run/exec +DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" + + # Handle userns security if [ ! -z "$(docker info 2>/dev/null | grep userns)" ]; then DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS --userns=host" From 01eb4b625045100f7aeeafd69ef4e305cfe605ef Mon Sep 17 00:00:00 2001 From: Andriy Maletsky Date: Fri, 26 Jan 2018 13:26:31 +0200 Subject: [PATCH 68/79] Lower severity to "warning" if `down` tries to remove nonexisting image Signed-off-by: Andriy Maletsky --- compose/service.py | 3 +++ tests/unit/service_test.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/compose/service.py b/compose/service.py index f6dfa7c72..3c5e356a3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -1148,6 +1148,9 @@ class Service(object): try: self.client.remove_image(self.image_name) return True + except ImageNotFound: + log.warning("Image %s not found.", self.image_name) + return False except APIError as e: log.error("Failed to remove image for service %s: %s", self.name, e) return False diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 99adea34b..8b3352fcb 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -5,6 +5,7 @@ import docker import pytest from docker.constants import DEFAULT_DOCKER_API_VERSION from docker.errors import APIError +from docker.errors import ImageNotFound from docker.errors import NotFound from .. import mock @@ -755,6 +756,13 @@ class ServiceTest(unittest.TestCase): mock_log.error.assert_called_once_with( "Failed to remove image for service %s: %s", web.name, error) + def test_remove_non_existing_image(self): + self.mock_client.remove_image.side_effect = ImageNotFound('image not found') + web = Service('web', image='example', client=self.mock_client) + with mock.patch('compose.service.log', autospec=True) as mock_log: + assert not web.remove_image(ImageType.all) + mock_log.warning.assert_called_once_with("Image %s not found.", web.image_name) + def test_specifies_host_port_with_no_ports(self): service = Service( 'foo', From d980d170a6a58df573c0b78211fd35a9d33286a2 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Sat, 8 Dec 2018 13:48:05 +0300 Subject: [PATCH 69/79] error on duplicate mount points Signed-off-by: Collins Abitekaniza --- compose/config/config.py | 12 ++++++++++++ compose/service.py | 1 + 2 files changed, 13 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 0298b4e2d..59c76680b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -8,6 +8,7 @@ import os import string import sys from collections import namedtuple +from operator import itemgetter, attrgetter import six import yaml @@ -835,6 +836,17 @@ def finalize_service_volumes(service_dict, environment): finalized_volumes.append(MountSpec.parse(v, normalize, win_host)) else: finalized_volumes.append(VolumeSpec.parse(v, normalize, win_host)) + + duplicate_mounts = [] + mounts = [v.as_volume_spec() if isinstance(v, MountSpec) else v for v in finalized_volumes] + for mount in mounts: + if list(map(attrgetter('internal'), mounts)).count(mount.internal) > 1: + duplicate_mounts.append(mount.repr()) + + if duplicate_mounts: + raise ConfigurationError("Duplicate mount points: volumes [%s]" % ( + ', '.join(duplicate_mounts))) + service_dict['volumes'] = finalized_volumes return service_dict diff --git a/compose/service.py b/compose/service.py index f6dfa7c72..964ab0193 100644 --- a/compose/service.py +++ b/compose/service.py @@ -9,6 +9,7 @@ import sys from collections import namedtuple from collections import OrderedDict from operator import attrgetter +from operator import itemgetter import enum import six From 47ff8d710c62a2c58e359ac41a90d7b6c411de90 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Sun, 9 Dec 2018 00:43:06 +0300 Subject: [PATCH 70/79] test create from config with duplicate mount points Signed-off-by: Collins Abitekaniza --- compose/config/config.py | 6 +++--- compose/service.py | 1 - tests/unit/config/config_test.py | 35 ++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 59c76680b..d9ee158d5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -8,7 +8,7 @@ import os import string import sys from collections import namedtuple -from operator import itemgetter, attrgetter +from operator import attrgetter import six import yaml @@ -844,9 +844,9 @@ def finalize_service_volumes(service_dict, environment): duplicate_mounts.append(mount.repr()) if duplicate_mounts: - raise ConfigurationError("Duplicate mount points: volumes [%s]" % ( + raise ConfigurationError("Duplicate mount points: [%s]" % ( ', '.join(duplicate_mounts))) - + service_dict['volumes'] = finalized_volumes return service_dict diff --git a/compose/service.py b/compose/service.py index 964ab0193..f6dfa7c72 100644 --- a/compose/service.py +++ b/compose/service.py @@ -9,7 +9,6 @@ import sys from collections import namedtuple from collections import OrderedDict from operator import attrgetter -from operator import itemgetter import enum import six diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 787d8ff4a..f95c46d89 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3071,6 +3071,41 @@ class ConfigTest(unittest.TestCase): ) config.load(config_details) + def test_config_duplicate_mount_points(self): + config1 = build_config_details( + { + 'version': '3.5', + 'services': { + 'web': { + 'image': 'busybox', + 'volumes': ['/tmp/foo:/tmp/foo', '/tmp/foo:/tmp/foo:rw'] + } + } + } + ) + + config2 = build_config_details( + { + 'version': '3.5', + 'services': { + 'web': { + 'image': 'busybox', + 'volumes': ['/x:/y', '/z:/y'] + } + } + } + ) + + with self.assertRaises(ConfigurationError) as e: + config.load(config1) + self.assertEquals(str(e.exception), 'Duplicate mount points: [%s]' % ( + ', '.join(['/tmp/foo:/tmp/foo:rw']*2))) + + with self.assertRaises(ConfigurationError) as e: + config.load(config2) + self.assertEquals(str(e.exception), 'Duplicate mount points: [%s]' % ( + ', '.join(['/x:/y:rw', '/z:/y:rw']))) + class NetworkModeTest(unittest.TestCase): From 8419a670aed3364c39b86a0608782aaeae3ce5df Mon Sep 17 00:00:00 2001 From: Quentin Brunet Date: Tue, 8 Jan 2019 14:04:54 +0100 Subject: [PATCH 71/79] Upgrade pyyaml to 4.2b1 Signed-off-by: Quentin Brunet --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index fbb285b6b..8883d55b7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,7 +16,7 @@ paramiko==2.4.2 pypiwin32==219; sys_platform == 'win32' and python_version < '3.6' pypiwin32==223; sys_platform == 'win32' and python_version >= '3.6' PySocks==1.6.7 -PyYAML==3.12 +PyYAML==4.2b1 requests==2.20.0 six==1.10.0 texttable==0.9.1 diff --git a/setup.py b/setup.py index 4c49bab7b..8b5f9d99b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ def find_version(*file_paths): install_requires = [ 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', - 'PyYAML >= 3.10, < 4', + 'PyYAML >= 3.10, < 4.3', 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', From 56fbd22825794b488894b9b589f7d17d2337d1d5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Stephan=20M=C3=BCller?= Date: Wed, 9 Jan 2019 23:14:12 +0100 Subject: [PATCH 72/79] fix race condition after pulling image MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stephan Müller --- compose/progress_stream.py | 8 +++---- tests/unit/progress_stream_test.py | 36 ++++++++++++++++-------------- 2 files changed, 23 insertions(+), 21 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 4cd311432..c4281cb4c 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -98,14 +98,14 @@ def print_output_event(event, stream, is_terminal): def get_digest_from_pull(events): + digest = None for event in events: status = event.get('status') if not status or 'Digest' not in status: continue - - _, digest = status.split(':', 1) - return digest.strip() - return None + else: + digest = status.split(':', 1)[1].strip() + return digest def get_digest_from_push(events): diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index d29227458..6fdb7d927 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -97,22 +97,24 @@ class ProgressStreamTestCase(unittest.TestCase): tf.seek(0) assert tf.read() == '???' + def test_get_digest_from_push(self): + digest = "sha256:abcd" + events = [ + {"status": "..."}, + {"status": "..."}, + {"progressDetail": {}, "aux": {"Digest": digest}}, + ] + assert progress_stream.get_digest_from_push(events) == digest -def test_get_digest_from_push(): - digest = "sha256:abcd" - events = [ - {"status": "..."}, - {"status": "..."}, - {"progressDetail": {}, "aux": {"Digest": digest}}, - ] - assert progress_stream.get_digest_from_push(events) == digest + def test_get_digest_from_pull(self): + events = list() + assert progress_stream.get_digest_from_pull(events) is None - -def test_get_digest_from_pull(): - digest = "sha256:abcd" - events = [ - {"status": "..."}, - {"status": "..."}, - {"status": "Digest: %s" % digest}, - ] - assert progress_stream.get_digest_from_pull(events) == digest + digest = "sha256:abcd" + events = [ + {"status": "..."}, + {"status": "..."}, + {"status": "Digest: %s" % digest}, + {"status": "..."}, + ] + assert progress_stream.get_digest_from_pull(events) == digest From ab0a0d69d98af05afb8edf9293088dc29cbe764a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 10 Jan 2019 13:50:28 -0800 Subject: [PATCH 73/79] Bump SDK version -> 3.7.0 Signed-off-by: Joffrey F --- requirements.txt | 4 ++-- setup.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/requirements.txt b/requirements.txt index fbb285b6b..cf569ed23 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,8 +3,8 @@ cached-property==1.3.0 certifi==2017.4.17 chardet==3.0.4 colorama==0.4.0; sys_platform == 'win32' -docker==3.6.0 -docker-pycreds==0.3.0 +docker==3.7.0 +docker-pycreds==0.4.0 dockerpty==0.4.1 docopt==0.6.2 enum34==1.1.6; python_version < '3.4' diff --git a/setup.py b/setup.py index 4c49bab7b..1ef0da5c5 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, != 2.12.2, != 2.18.0, < 2.21', 'texttable >= 0.9.0, < 0.10', 'websocket-client >= 0.32.0, < 1.0', - 'docker[ssh] >= 3.6.0, < 4.0', + 'docker[ssh] >= 3.7.0, < 4.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From bab8b3985e02a71373f5d6f753485ac7a08c3092 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Thu, 10 Jan 2019 13:48:42 +0300 Subject: [PATCH 74/79] check for started containers only on service_start Signed-off-by: Collins Abitekaniza --- compose/parallel.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index 32ee602f4..dbf4845b8 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -43,14 +43,15 @@ class GlobalLimit(object): cls.global_limiter = Semaphore(value) -def parallel_execute_watch(events, writer, errors, results, msg, get_name): +def parallel_execute_watch(events, writer, errors, results, msg, get_name, func_name): """ Watch events from a parallel execution, update status and fill errors and results. Returns exception to re-raise. """ error_to_reraise = None for obj, result, exception in events: if exception is None: - if callable(getattr(obj, 'containers', None)) and not obj.containers(): + if func_name == 'start_service' and ( + callable(getattr(obj, 'containers', None)) and not obj.containers()): # If service has no containers started writer.write(msg, get_name(obj), 'failed', red) else: @@ -100,7 +101,8 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): errors = {} results = [] - error_to_reraise = parallel_execute_watch(events, writer, errors, results, msg, get_name) + error_to_reraise = parallel_execute_watch( + events, writer, errors, results, msg, get_name, func.__name__) for obj_name, error in errors.items(): stream.write("\nERROR: for {} {}\n".format(obj_name, error)) From 325637d9d5939a82418a25360934b525dd601c20 Mon Sep 17 00:00:00 2001 From: Collins Abitekaniza Date: Thu, 10 Jan 2019 14:49:30 +0300 Subject: [PATCH 75/79] test image pull done Signed-off-by: Collins Abitekaniza --- compose/parallel.py | 2 +- tests/acceptance/cli_test.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/parallel.py b/compose/parallel.py index dbf4845b8..d6af51694 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -102,7 +102,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): errors = {} results = [] error_to_reraise = parallel_execute_watch( - events, writer, errors, results, msg, get_name, func.__name__) + events, writer, errors, results, msg, get_name, getattr(func, '__name__', None)) for obj_name, error in errors.items(): stream.write("\nERROR: for {} {}\n".format(obj_name, error)) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 1dc9616a5..9334a29fb 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -611,6 +611,11 @@ class CLITestCase(DockerClientTestCase): assert 'Pulling simple' in result.stderr assert 'Pulling another' in result.stderr + def test_pull_done(self): + result = self.dispatch(['pull']) + assert 'Pulling simple' in result.stderr + assert 'done' in result.stderr + def test_pull_with_digest(self): result = self.dispatch(['-f', 'digest.yml', 'pull', '--no-parallel']) From 2ed171cae94ef5ac9d2eeeb683b217e380e93b81 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 10 Jan 2019 15:47:44 -0800 Subject: [PATCH 76/79] Bring zero container check up in the call stack Signed-off-by: Joffrey F --- compose/parallel.py | 13 +++++++------ compose/project.py | 1 + tests/acceptance/cli_test.py | 5 +---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index d6af51694..e242a318a 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -43,16 +43,14 @@ class GlobalLimit(object): cls.global_limiter = Semaphore(value) -def parallel_execute_watch(events, writer, errors, results, msg, get_name, func_name): +def parallel_execute_watch(events, writer, errors, results, msg, get_name, fail_check): """ Watch events from a parallel execution, update status and fill errors and results. Returns exception to re-raise. """ error_to_reraise = None for obj, result, exception in events: if exception is None: - if func_name == 'start_service' and ( - callable(getattr(obj, 'containers', None)) and not obj.containers()): - # If service has no containers started + if fail_check is not None and fail_check(obj): writer.write(msg, get_name(obj), 'failed', red) else: writer.write(msg, get_name(obj), 'done', green) @@ -77,12 +75,14 @@ def parallel_execute_watch(events, writer, errors, results, msg, get_name, func_ return error_to_reraise -def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): +def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None, fail_check=None): """Runs func on objects in parallel while ensuring that func is ran on object only after it is ran on all its dependencies. get_deps called on object must return a collection with its dependencies. get_name called on object must return its name. + fail_check is an additional failure check for cases that should display as a failure + in the CLI logs, but don't raise an exception (such as attempting to start 0 containers) """ objects = list(objects) stream = get_output_stream(sys.stderr) @@ -102,7 +102,8 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None, limit=None): errors = {} results = [] error_to_reraise = parallel_execute_watch( - events, writer, errors, results, msg, get_name, getattr(func, '__name__', None)) + events, writer, errors, results, msg, get_name, fail_check + ) for obj_name, error in errors.items(): stream.write("\nERROR: for {} {}\n".format(obj_name, error)) diff --git a/compose/project.py b/compose/project.py index 5c4ce6e17..a7f2aa057 100644 --- a/compose/project.py +++ b/compose/project.py @@ -280,6 +280,7 @@ class Project(object): operator.attrgetter('name'), 'Starting', get_deps, + fail_check=lambda obj: not obj.containers(), ) return containers diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9334a29fb..5142f96eb 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -610,11 +610,8 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['pull']) assert 'Pulling simple' in result.stderr assert 'Pulling another' in result.stderr - - def test_pull_done(self): - result = self.dispatch(['pull']) - assert 'Pulling simple' in result.stderr assert 'done' in result.stderr + assert 'failed' not in result.stderr def test_pull_with_digest(self): result = self.dispatch(['-f', 'digest.yml', 'pull', '--no-parallel']) From 60f8ce09f9b512536ce731a693446f6279ecd939 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Fri, 11 Jan 2019 17:43:47 +0100 Subject: [PATCH 77/79] "Bump 1.24.0-rc1" Signed-off-by: Djordje Lukic --- CHANGELOG.md | 36 ++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- script/run/run.sh | 2 +- 3 files changed, 38 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a7a2ffe9..e294a209a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,42 @@ Change log ========== +1.24.0 (2019-01-25) +------------------- + +### Features + +- Added support for connecting to the Docker Engine using the `ssh` protocol. + +- Added a `--all` flag to `docker-compose ps` to include stopped one-off containers + in the command's output. + +### Bugfixes + +- Fixed a bug where some valid credential helpers weren't properly handled by Compose + when attempting to pull images from private registries. + +- Fixed an issue where the output of `docker-compose start` before containers were created + was misleading + +- To match the Docker CLI behavior and to avoid confusing issues, Compose will no longer + accept whitespace in variable names sourced from environment files. + +- Compose will now report a configuration error if a service attempts to declare + duplicate mount points in the volumes section. + +- Fixed an issue with the containerized version of Compose that prevented users from + writing to stdin during interactive sessions started by `run` or `exec`. + +- One-off containers started by `run` no longer adopt the restart policy of the service, + and are instead set to never restart. + +- Fixed an issue that caused some container events to not appear in the output of + the `docker-compose events` command. + +- Missing images will no longer stop the execution of `docker-compose down` commands + (a warning will be displayed instead). + 1.23.2 (2018-11-28) ------------------- diff --git a/compose/__init__.py b/compose/__init__.py index 652e1fad9..bc5e6b116 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.24.0dev' +__version__ = '1.24.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index cc36e4751..df3f2298f 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.23.2" +VERSION="1.24.0-rc1" IMAGE="docker/compose:$VERSION" From 200795173168aeb5aeccaa9184d972a7a28ad600 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Fri, 11 Jan 2019 17:57:05 +0100 Subject: [PATCH 78/79] "Bump 1.24.0-rc1" Signed-off-by: Djordje Lukic --- CHANGELOG.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e294a209a..c5eb1bb4d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,7 +18,7 @@ Change log - Fixed an issue where the output of `docker-compose start` before containers were created was misleading - + - To match the Docker CLI behavior and to avoid confusing issues, Compose will no longer accept whitespace in variable names sourced from environment files. @@ -31,7 +31,7 @@ Change log - One-off containers started by `run` no longer adopt the restart policy of the service, and are instead set to never restart. -- Fixed an issue that caused some container events to not appear in the output of +- Fixed an issue that caused some container events to not appear in the output of the `docker-compose events` command. - Missing images will no longer stop the execution of `docker-compose down` commands From 0f3d4ddaa7b8262f964f2c42d502e5a65d28cc64 Mon Sep 17 00:00:00 2001 From: Djordje Lukic Date: Fri, 11 Jan 2019 18:28:30 +0100 Subject: [PATCH 79/79] "Bump 1.24.0-rc1" Signed-off-by: Djordje Lukic --- script/release/release/repository.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/script/release/release/repository.py b/script/release/release/repository.py index 9a5d432c0..bb8f4fbeb 100644 --- a/script/release/release/repository.py +++ b/script/release/release/repository.py @@ -219,6 +219,8 @@ def get_contributors(pr_data): commits = pr_data.get_commits() authors = {} for commit in commits: + if not commit.author: + continue author = commit.author.login authors[author] = authors.get(author, 0) + 1 return [x[0] for x in sorted(list(authors.items()), key=lambda x: x[1])]