diff --git a/CHANGELOG.md b/CHANGELOG.md index 6699f880..14197cc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,82 @@ Change log ========== +1.11.2 (2017-02-17) +------------------- + +### Bugfixes + +- Fixed a bug that was preventing secrets configuration from being + loaded properly + +- Fixed a bug where the `docker-compose config` command would fail + if the config file contained secrets definitions + +- Fixed an issue where Compose on some linux distributions would + pick up and load an outdated version of the requests library + +- Fixed an issue where socket-type files inside a build folder + would cause `docker-compose` to crash when trying to build that + service + +- Fixed an issue where recursive wildcard patterns `**` were not being + recognized in `.dockerignore` files. + +1.11.1 (2017-02-09) +------------------- + +### Bugfixes + +- Fixed a bug where the 3.1 file format was not being recognized as valid + by the Compose parser + +1.11.0 (2017-02-08) +------------------- + +### New Features + +#### Compose file version 3.1 + +- Introduced version 3.1 of the `docker-compose.yml` specification. This + version requires Docker Engine 1.13.0 or above. It introduces support + for secrets. See the documentation for more information + +#### Compose file version 2.0 and up + +- Introduced the `docker-compose top` command that displays processes running + for the different services managed by Compose. + +### Bugfixes + +- Fixed a bug where extending a service defining a healthcheck dictionary + would cause `docker-compose` to error out. + +- Fixed an issue where the `pid` entry in a service definition was being + ignored when using multiple Compose files. + +1.10.1 (2017-02-01) +------------------ + +### Bugfixes + +- Fixed an issue where presence of older versions of the docker-py + package would cause unexpected crashes while running Compose + +- Fixed an issue where healthcheck dependencies would be lost when + using multiple compose files for a project + +- Fixed a few issues that made the output of the `config` command + invalid + +- Fixed an issue where adding volume labels to v3 Compose files would + result in an error + +- Fixed an issue on Windows where build context paths containing unicode + characters were being improperly encoded + +- Fixed a bug where Compose would occasionally crash while streaming logs + when containers would stop or restart + 1.10.0 (2017-01-18) ------------------- diff --git a/Dockerfile b/Dockerfile index 63fac3eb..a03e1510 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN set -ex; \ ca-certificates \ curl \ libsqlite3-dev \ + libbz2-dev \ ; \ rm -rf /var/lib/apt/lists/* @@ -20,40 +21,32 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker -# Build Python 2.7.9 from source +# Build Python 2.7.13 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \ - cd Python-2.7.9; \ + curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + cd Python-2.7.13; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.9 + rm -rf /Python-2.7.13 # Build python 3.4 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \ - cd Python-3.4.3; \ + curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + cd Python-3.4.6; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.3 + rm -rf /Python-3.4.6 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib -# Install setuptools -RUN set -ex; \ - curl -L https://bootstrap.pypa.io/ez_setup.py | python - # Install pip RUN set -ex; \ - curl -L https://pypi.python.org/packages/source/p/pip/pip-8.1.1.tar.gz | tar -xz; \ - cd pip-8.1.1; \ - python setup.py install; \ - cd ..; \ - rm -rf pip-8.1.1 + curl -L https://bootstrap.pypa.io/get-pip.py | python # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen diff --git a/compose/__init__.py b/compose/__init__.py index 38417836..a6647198 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0dev' +__version__ = '1.11.2' diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index e69de29b..c5db4455 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import subprocess +import sys + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # We don't try importing pip because it messes with package imports + # on some Linux distros (Ubuntu, Fedora) + # https://github.com/docker/compose/issues/4425 + # https://github.com/docker/compose/issues/4481 + # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py + s_cmd = subprocess.Popen( + ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + packages = s_cmd.communicate()[0].splitlines() + dockerpy_installed = len( + list(filter(lambda p: p.startswith(b'docker-py=='), packages)) + ) > 0 + if dockerpy_installed: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) + +except OSError: + # pip command is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass diff --git a/compose/cli/main.py b/compose/cli/main.py index e2ebce48..51ba36a0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,30 +14,6 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter - -# Attempt to detect https://github.com/docker/compose/issues/4344 -try: - # A regular import statement causes PyInstaller to freak out while - # trying to load pip. This way it is simply ignored. - pip = __import__('pip') - pip_packages = pip.get_installed_distributions() - if 'docker-py' in [pkg.project_name for pkg in pip_packages]: - from .colors import red - print( - red('ERROR:'), - "Dependency conflict: an older version of the 'docker-py' package " - "is polluting the namespace. " - "Run the following command to remedy the issue:\n" - "pip uninstall docker docker-py; pip install docker", - file=sys.stderr - ) - sys.exit(1) -except ImportError: - # pip is not available, which indicates it's probably the binary - # distribution of Compose which is not affected - pass - - from . import errors from . import signals from .. import __version__ diff --git a/compose/config/config.py b/compose/config/config.py index ae85674b..4c9cf423 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -186,11 +186,6 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): if version == '3': version = V3_0 - if version not in (V2_0, V2_1, V3_0): - raise ConfigurationError( - 'Version in "{}" is unsupported. {}' - .format(self.filename, VERSION_EXPLANATION)) - return version def get_service(self, name): @@ -479,7 +474,7 @@ def process_config_file(config_file, environment, service_name=None): 'service', environment) - if config_file.version in (V2_0, V2_1, V3_0): + if config_file.version in (V2_0, V2_1, V3_0, V3_1): processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( @@ -495,7 +490,9 @@ def process_config_file(config_file, environment, service_name=None): elif config_file.version == V1: processed_config = services else: - raise Exception("Unsupported version: {}".format(repr(config_file.version))) + raise ConfigurationError( + 'Version in "{}" is unsupported. {}' + .format(config_file.filename, VERSION_EXPLANATION)) config_file = config_file._replace(config=processed_config) validate_against_config_schema(config_file) @@ -766,6 +763,11 @@ def finalize_service(service_config, service_names, version, environment): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + if 'secrets' in service_dict: + service_dict['secrets'] = [ + types.ServiceSecret.parse(s) for s in service_dict['secrets'] + ] + normalize_build(service_dict, service_config.working_dir, environment) service_dict['name'] = service_config.name diff --git a/compose/config/environment.py b/compose/config/environment.py index 7b926930..4ba228c8 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import contextlib import logging import os @@ -31,11 +32,12 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise ConfigurationError("%s is not a file." % (filename)) env = {} - for line in codecs.open(filename, 'r', 'utf-8'): - line = line.strip() - if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v + with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + for line in fileobj: + line = line.strip() + if line and not line.startswith('#'): + k, v = split_env(line) + env[k] = v return env diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 3745de82..46d283f0 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -102,4 +102,7 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) + if 'secrets' in service_dict: + service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets']) + return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index 17d5c8b3..811e6c1f 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -253,3 +253,8 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): @property def merge_field(self): return self.source + + def repr(self): + return dict( + [(k, v) for k, v in self._asdict().items() if v is not None] + ) diff --git a/docker-compose.spec b/docker-compose.spec index ec5a2039..ef0e2593 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -37,6 +37,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.0.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.1.json', + 'compose/config/config_schema_v3.1.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/requirements-build.txt b/requirements-build.txt index 3f1dbd75..27f610ca 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.1.1 +pyinstaller==3.2.1 diff --git a/requirements.txt b/requirements.txt index 3b06bff4..53b9294c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.2 +docker==2.1.0 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/script/run/run.sh b/script/run/run.sh index 5872b081..f5d9ae86 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.8.0" +VERSION="1.11.2" IMAGE="docker/compose:$VERSION" diff --git a/setup.py b/setup.py index 0b1d4e08..13fe59b2 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import +from __future__ import print_function from __future__ import unicode_literals import codecs -import logging import os import re import sys @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.2, < 3.0', + 'docker >= 2.1.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', @@ -64,11 +64,9 @@ try: for key, value in extras_require.items(): if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): install_requires.extend(value) -except Exception: - logging.getLogger(__name__).exception( - 'Failed to compute platform dependencies. All dependencies will be ' - 'installed as a result.' - ) +except Exception as e: + print("Failed to compute platform dependencies: {}. ".format(e) + + "All dependencies will be installed as a result.", file=sys.stderr) for key, value in extras_require.items(): if key.startswith(':'): install_requires.extend(value) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 666b21f2..c26272d9 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -13,12 +13,14 @@ import pytest from ...helpers import build_config_details from compose.config import config +from compose.config import types from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 from compose.config.config import V3_0 +from compose.config.config import V3_1 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION @@ -52,6 +54,10 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) +def secret_sort(secrets): + return sorted(secrets, key=itemgetter('source')) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( @@ -168,6 +174,9 @@ class ConfigTest(unittest.TestCase): cfg = config.load(build_config_details({'version': version})) assert cfg.version == V3_0 + cfg = config.load(build_config_details({'version': '3.1'})) + assert cfg.version == V3_1 + def test_v1_file_version(self): cfg = config.load(build_config_details({'web': {'image': 'busybox'}})) assert cfg.version == V1 @@ -1766,6 +1775,38 @@ class ConfigTest(unittest.TestCase): 'labels': {'com.docker.compose.test': 'yes'} } + def test_merge_different_secrets(self): + base = { + 'image': 'busybox', + 'secrets': [ + {'source': 'src.txt'} + ] + } + override = {'secrets': ['other-src.txt']} + + actual = config.merge_service_dicts(base, override, V3_1) + assert secret_sort(actual['secrets']) == secret_sort([ + {'source': 'src.txt'}, + {'source': 'other-src.txt'} + ]) + + def test_merge_secrets_override(self): + base = { + 'image': 'busybox', + 'secrets': ['src.txt'], + } + override = { + 'secrets': [ + { + 'source': 'src.txt', + 'target': 'data.txt', + 'mode': 0o400 + } + ] + } + actual = config.merge_service_dicts(base, override, V3_1) + assert actual['secrets'] == override['secrets'] + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', @@ -1845,6 +1886,91 @@ class ConfigTest(unittest.TestCase): config.load(config_details) assert 'has neither an image nor a build context' in exc.exconly() + def test_load_secrets(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': [ + 'one', + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + details = config.ConfigDetails('.', [base_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + + def test_load_secrets_multi_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': ['one'], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + override_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'secrets': [ + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + class NetworkModeTest(unittest.TestCase): def test_network_mode_standard(self): @@ -3401,3 +3527,24 @@ class SerializeTest(unittest.TestCase): denormalized_service = denormalize_service_dict(processed_service, V2_1) assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' + + def test_denormalize_secrets(self): + service_dict = { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + } + denormalized_service = denormalize_service_dict(service_dict, V3_1) + assert secret_sort(denormalized_service['secrets']) == secret_sort([ + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ])