From 204655be137ecf1d901ab0c875ba217447f9fa57 Mon Sep 17 00:00:00 2001 From: aiordache Date: Thu, 10 Sep 2020 16:07:03 +0200 Subject: [PATCH 01/10] Update changelog for 1.27.2 Signed-off-by: aiordache --- CHANGELOG.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index a319cbb49..4b98799b5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,26 @@ Change log ========== +1.27.2 (2020-09-10) +------------------- + +### Bugs + +- Fix bug on `docker-compose run` container attach + +1.27.1 (2020-09-10) +------------------- + +### Bugs + +- Fix `docker-compose run` when `service.scale` is specified + +- Allow `driver` property for external networks as temporary workaround for swarm network propagation issue + +- Pin new internal schema version to `3.9` as the default + +- Preserve the version when configured in the compose file + 1.27.0 (2020-09-07) ------------------- From d811500fa0702c30310289f736c66874171b0657 Mon Sep 17 00:00:00 2001 From: Kevin Clark Date: Thu, 10 Sep 2020 17:05:11 -0400 Subject: [PATCH 02/10] Added merge for max_replicas_per_node Signed-off-by: Kevin Clark --- compose/config/config.py | 1 + tests/unit/config/config_test.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 243529bd6..7b3969b6c 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1114,6 +1114,7 @@ def merge_deploy(base, override): md['resources'] = dict(resources_md) if md.needs_merge('placement'): placement_md = MergeDict(md.base.get('placement') or {}, md.override.get('placement') or {}) + placement_md.merge_scalar('max_replicas_per_node') placement_md.merge_field('constraints', merge_unique_items_lists, default=[]) placement_md.merge_field('preferences', merge_unique_objects_lists, default=[]) md['placement'] = dict(placement_md) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8b0d37526..63eece17e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2543,6 +2543,7 @@ web: 'labels': ['com.docker.compose.a=1', 'com.docker.compose.b=2'], 'mode': 'replicated', 'placement': { + 'max_replicas_per_node': 1, 'constraints': [ 'node.role == manager', 'engine.labels.aws == true' ], @@ -2599,6 +2600,7 @@ web: 'com.docker.compose.c': '3' }, 'placement': { + 'max_replicas_per_node': 1, 'constraints': [ 'engine.labels.aws == true', 'engine.labels.dev == true', 'node.role == manager', 'node.role == worker' From a75b6249f83be2e1b8fadc40b2c29aa7d09921ef Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 14 Sep 2020 11:41:30 +0200 Subject: [PATCH 03/10] Fix depends_on serialisation on `docker-compose config` Signed-off-by: aiordache --- compose/config/serialize.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 2dd2c47f1..2d9493a03 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -121,11 +121,6 @@ def denormalize_service_dict(service_dict, version, image_digest=None): if version == V1 and 'network_mode' not in service_dict: service_dict['network_mode'] = 'bridge' - if 'depends_on' in service_dict: - service_dict['depends_on'] = sorted([ - svc for svc in service_dict['depends_on'].keys() - ]) - if 'healthcheck' in service_dict: if 'interval' in service_dict['healthcheck']: service_dict['healthcheck']['interval'] = serialize_ns_time_value( From fa720787d62cf833ed7648ff55dbd248267a5b74 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 14 Sep 2020 18:01:35 +0200 Subject: [PATCH 04/10] update depends_on tests Signed-off-by: aiordache --- tests/unit/config/config_test.py | 24 ++---------------------- 1 file changed, 2 insertions(+), 22 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 63eece17e..b1586ae1f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5269,7 +5269,7 @@ def get_config_filename_for_files(filenames, subdir=None): class SerializeTest(unittest.TestCase): - def test_denormalize_depends_on_v3(self): + def test_denormalize_depends(self): service_dict = { 'image': 'busybox', 'command': 'true', @@ -5279,27 +5279,7 @@ class SerializeTest(unittest.TestCase): } } - assert denormalize_service_dict(service_dict, VERSION) == { - 'image': 'busybox', - 'command': 'true', - 'depends_on': ['service2', 'service3'] - } - - def test_denormalize_depends_on_v2_1(self): - service_dict = { - 'image': 'busybox', - 'command': 'true', - 'depends_on': { - 'service2': {'condition': 'service_started'}, - 'service3': {'condition': 'service_started'}, - } - } - - assert denormalize_service_dict(service_dict, VERSION) == { - 'image': 'busybox', - 'command': 'true', - 'depends_on': ['service2', 'service3'] - } + assert denormalize_service_dict(service_dict, VERSION) == service_dict def test_serialize_time(self): data = { From 50a4afaf1743d908088576e554c5040ea93d1456 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 11 Sep 2020 17:23:40 +0200 Subject: [PATCH 05/10] Fix scaling when some containers are not running Signed-off-by: aiordache --- compose/service.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index 70939cac7..a1a500cb2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -411,7 +411,7 @@ class Service: stopped = [c for c in containers if not c.is_running] if stopped: - return ConvergencePlan('start', stopped) + return ConvergencePlan('start', containers) return ConvergencePlan('noop', containers) @@ -514,8 +514,9 @@ class Service: self._downscale(containers[scale:], timeout) containers = containers[:scale] if start: + stopped = [c for c in containers if not c.is_running] _, errors = parallel_execute( - containers, + stopped, lambda c: self.start_container_if_stopped(c, attach_logs=not detached, quiet=True), lambda c: c.name, "Starting", From a85d2bc64ccd79960e1f7c744f0702710e3887f0 Mon Sep 17 00:00:00 2001 From: aiordache Date: Fri, 11 Sep 2020 18:06:31 +0200 Subject: [PATCH 06/10] update test for start trigger Signed-off-by: aiordache --- tests/integration/state_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 5258e310c..8168cddf0 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -375,7 +375,7 @@ class ServiceStateTest(DockerClientTestCase): assert [c.is_running for c in containers] == [False, True] - assert ('start', containers[0:1]) == web.convergence_plan() + assert ('start', containers) == web.convergence_plan() def test_trigger_recreate_with_config_change(self): web = self.create_service('web', command=["top"]) From 5340a6d760c463c759cdf3514357dfe5eb11a31e Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 14 Sep 2020 19:11:13 +0200 Subject: [PATCH 07/10] Add test for scale with stopped containers Signed-off-by: aiordache --- tests/integration/project_test.py | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 879701076..96929f209 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1347,6 +1347,36 @@ class ProjectTest(DockerClientTestCase): project.up() assert len(project.containers()) == 3 + def test_project_up_scale_with_stopped_containers(self): + config_data = build_config( + services=[{ + 'name': 'web', + 'image': BUSYBOX_IMAGE_WITH_TAG, + 'command': 'top', + 'scale': 2 + }] + ) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + + project.up() + containers = project.containers() + assert len(containers) == 2 + + self.client.stop(containers[0].id) + project.up(scale_override={'web': 2}) + containers = project.containers() + assert len(containers) == 2 + + self.client.stop(containers[0].id) + project.up(scale_override={'web': 3}) + assert len(project.containers()) == 3 + + self.client.stop(containers[0].id) + project.up(scale_override={'web': 1}) + assert len(project.containers()) == 1 + def test_initialize_volumes(self): vol_name = '{:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{}'.format(vol_name) From 8c81a9da7a846c1f0b1fe3b40e7c7ab2e70c2215 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 14 Sep 2020 17:02:19 +0200 Subject: [PATCH 08/10] Enable relative paths for driver_opts.device Signed-off-by: aiordache --- compose/config/config.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7b3969b6c..69d0d902d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -423,17 +423,31 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): elif not config.get('name'): config['name'] = name - if 'driver_opts' in config: - config['driver_opts'] = build_string_dict( - config['driver_opts'] - ) - if 'labels' in config: config['labels'] = parse_labels(config['labels']) if 'file' in config: config['file'] = expand_path(working_dir, config['file']) + if 'driver_opts' in config: + config['driver_opts'] = build_string_dict( + config['driver_opts'] + ) + if entity_type != 'Volume': + continue + # default driver is 'local' + driver = config.get('driver', 'local') + if driver != 'local': + continue + o = config['driver_opts'].get('o') + device = config['driver_opts'].get('device') + if o and o == 'bind' and device: + fullpath = os.path.abspath(os.path.expanduser(device)) + if not os.path.exists(fullpath): + raise ConfigurationError( + "Device path {} does not exist.".format(fullpath)) + config['driver_opts']['device'] = fullpath + return mapping From c960b028b96970ebedac981bfbd64cfd0a04c848 Mon Sep 17 00:00:00 2001 From: aiordache Date: Mon, 14 Sep 2020 18:25:12 +0200 Subject: [PATCH 09/10] fix flake8 complexity Signed-off-by: aiordache --- compose/config/config.py | 35 ++++++++++++++++++++--------------- 1 file changed, 20 insertions(+), 15 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 69d0d902d..55e8c2757 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -433,24 +433,29 @@ def load_mapping(config_files, get_func, entity_type, working_dir=None): config['driver_opts'] = build_string_dict( config['driver_opts'] ) - if entity_type != 'Volume': - continue - # default driver is 'local' - driver = config.get('driver', 'local') - if driver != 'local': - continue - o = config['driver_opts'].get('o') - device = config['driver_opts'].get('device') - if o and o == 'bind' and device: - fullpath = os.path.abspath(os.path.expanduser(device)) - if not os.path.exists(fullpath): - raise ConfigurationError( - "Device path {} does not exist.".format(fullpath)) - config['driver_opts']['device'] = fullpath - + device = format_device_option(entity_type, config) + if device: + config['driver_opts']['device'] = device return mapping +def format_device_option(entity_type, config): + if entity_type != 'Volume': + return + # default driver is 'local' + driver = config.get('driver', 'local') + if driver != 'local': + return + o = config['driver_opts'].get('o') + device = config['driver_opts'].get('device') + if o and o == 'bind' and device: + fullpath = os.path.abspath(os.path.expanduser(device)) + if not os.path.exists(fullpath): + raise ConfigurationError( + "Device path {} does not exist.".format(fullpath)) + return fullpath + + def validate_external(entity_type, name, config, version): for k in config.keys(): if entity_type == 'Network' and k == 'driver': From 60514c1adbeee1ce829f7394a0c6ee2176ac9405 Mon Sep 17 00:00:00 2001 From: aiordache Date: Wed, 16 Sep 2020 14:06:16 +0200 Subject: [PATCH 10/10] Allow strings for cpus fields Signed-off-by: aiordache --- compose/config/config_schema_compose_spec.json | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/compose/config/config_schema_compose_spec.json b/compose/config/config_schema_compose_spec.json index 8af7faa63..43d3a3edf 100644 --- a/compose/config/config_schema_compose_spec.json +++ b/compose/config/config_schema_compose_spec.json @@ -153,7 +153,7 @@ "cpu_period": {"type": ["number", "string"]}, "cpu_rt_period": {"type": ["number", "string"]}, "cpu_rt_runtime": {"type": ["number", "string"]}, - "cpus": {"type": "number", "minimum": 0}, + "cpus": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, "credential_spec": { "type": "object", @@ -503,7 +503,7 @@ "limits": { "type": "object", "properties": { - "cpus": {"type": "number", "minimum": 0}, + "cpus": {"type": ["number", "string"]}, "memory": {"type": "string"} }, "additionalProperties": false, @@ -512,7 +512,7 @@ "reservations": { "type": "object", "properties": { - "cpus": {"type": "number", "minimum": 0}, + "cpus": {"type": ["number", "string"]}, "memory": {"type": "string"}, "generic_resources": {"$ref": "#/definitions/generic_resources"} },