From 55c5c8e8ac35d8c069ec8dd2ebeb919e39efca41 Mon Sep 17 00:00:00 2001 From: Nicolas De Loof Date: Mon, 25 Nov 2019 11:09:42 +0100 Subject: [PATCH] Report image we can't pull and must be built Signed-off-by: Nicolas De Loof --- compose/progress_stream.py | 10 +++ compose/project.py | 81 ++++++++++++------- tests/acceptance/cli_test.py | 8 ++ .../can-build-pull-failures.yml | 6 ++ 4 files changed, 77 insertions(+), 28 deletions(-) create mode 100644 tests/fixtures/simple-composefile/can-build-pull-failures.yml diff --git a/compose/progress_stream.py b/compose/progress_stream.py index c4281cb4c..522ddf75d 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -114,3 +114,13 @@ def get_digest_from_push(events): if digest: return digest return None + + +def read_status(event): + status = event['status'].lower() + if 'progressDetail' in event: + detail = event['progressDetail'] + if 'current' in detail and 'total' in detail: + percentage = float(detail['current']) / float(detail['total']) + status = '{} ({:.1%})'.format(status, percentage) + return status diff --git a/compose/project.py b/compose/project.py index d7dcb6bd6..d7405defd 100644 --- a/compose/project.py +++ b/compose/project.py @@ -11,6 +11,8 @@ from os import path import enum import six from docker.errors import APIError +from docker.errors import ImageNotFound +from docker.errors import NotFound from docker.utils import version_lt from . import parallel @@ -25,6 +27,7 @@ from .container import Container from .network import build_networks from .network import get_networks from .network import ProjectNetworks +from .progress_stream import read_status from .service import BuildAction from .service import ContainerNetworkMode from .service import ContainerPidMode @@ -619,46 +622,68 @@ class Project(object): def pull(self, service_names=None, ignore_pull_failures=False, parallel_pull=False, silent=False, include_deps=False): services = self.get_services(service_names, include_deps) - msg = not silent and 'Pulling' or None if parallel_pull: - def pull_service(service): - strm = service.pull(ignore_pull_failures, True, stream=True) - if strm is None: # Attempting to pull service with no `image` key is a no-op - return + self.parallel_pull(services, silent=silent) + else: + must_build = [] + for service in services: + try: + service.pull(ignore_pull_failures, silent=silent) + except (ImageNotFound, NotFound): + if service.can_be_built(): + must_build.append(service.name) + else: + raise + + if len(must_build): + log.warning('Some service image(s) must be built from source by running:\n' + ' docker-compose build {}' + .format(' '.join(must_build))) + + def parallel_pull(self, services, ignore_pull_failures=False, silent=False): + msg = 'Pulling' if not silent else None + must_build = [] + + def pull_service(service): + strm = service.pull(ignore_pull_failures, True, stream=True) + + if strm is None: # Attempting to pull service with no `image` key is a no-op + return + + try: writer = parallel.get_stream_writer() - for event in strm: if 'status' not in event: continue - status = event['status'].lower() - if 'progressDetail' in event: - detail = event['progressDetail'] - if 'current' in detail and 'total' in detail: - percentage = float(detail['current']) / float(detail['total']) - status = '{} ({:.1%})'.format(status, percentage) - + status = read_status(event) writer.write( msg, service.name, truncate_string(status), lambda s: s ) + except (ImageNotFound, NotFound): + if service.can_be_built(): + must_build.append(service.name) + else: + raise - _, errors = parallel.parallel_execute( - services, - pull_service, - operator.attrgetter('name'), - msg, - limit=5, - ) - if len(errors): - combined_errors = '\n'.join([ - e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() - ]) - raise ProjectError(combined_errors) + _, errors = parallel.parallel_execute( + services, + pull_service, + operator.attrgetter('name'), + msg, + limit=5, + ) - else: - for service in services: - service.pull(ignore_pull_failures, silent=silent) + if len(must_build): + log.warning('Some service image(s) must be built from source by running:\n' + ' docker-compose build {}' + .format(' '.join(must_build))) + if len(errors): + combined_errors = '\n'.join([ + e.decode('utf-8') if isinstance(e, six.binary_type) else e for e in errors.values() + ]) + raise ProjectError(combined_errors) def push(self, service_names=None, ignore_push_failures=False): unique_images = set() diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 41e51a304..b729e7d76 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -694,6 +694,14 @@ services: result.stderr ) + def test_pull_can_build(self): + result = self.dispatch([ + '-f', 'can-build-pull-failures.yml', 'pull'], + returncode=0 + ) + assert 'Some service image(s) must be built from source' in result.stderr + assert 'docker-compose build can_build' in result.stderr + def test_pull_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' result = self.dispatch(['pull', '--no-parallel', 'web']) diff --git a/tests/fixtures/simple-composefile/can-build-pull-failures.yml b/tests/fixtures/simple-composefile/can-build-pull-failures.yml new file mode 100644 index 000000000..1ffe8e0fb --- /dev/null +++ b/tests/fixtures/simple-composefile/can-build-pull-failures.yml @@ -0,0 +1,6 @@ +version: '3' +services: + can_build: + image: nonexisting-image-but-can-build:latest + build: . + command: top