Merge pull request #4499 from xulike666/function-name-modification

function-name-modification for tests/*

CHANGELOG.md

@@ -1,6 +1,53 @@
 Change log
 ==========
 
+1.11.0 (2017-02-08)
+-------------------
+
+### New Features
+
+#### Compose file version 3.1
+
+- Introduced version 3.1 of the `docker-compose.yml` specification. This
+  version requires Docker Engine 1.13.0 or above. It introduces support
+  for secrets. See the documentation for more information
+
+#### Compose file version 2.0 and up
+
+- Introduced the `docker-compose top` command that displays processes running
+  for the different services managed by Compose.
+
+### Bugfixes
+
+- Fixed a bug where extending a service defining a healthcheck dictionary
+  would cause `docker-compose` to error out.
+
+- Fixed an issue where the `pid` entry in a service definition was being
+  ignored when using multiple Compose files.
+
+1.10.1 (2017-02-01)
+------------------
+
+### Bugfixes
+
+- Fixed an issue where presence of older versions of the docker-py
+  package would cause unexpected crashes while running Compose
+
+- Fixed an issue where healthcheck dependencies would be lost when
+  using multiple compose files for a project
+
+- Fixed a few issues that made the output of the `config` command
+  invalid
+
+- Fixed an issue where adding volume labels to v3 Compose files would
+  result in an error
+
+- Fixed an issue on Windows where build context paths containing unicode
+  characters were being improperly encoded
+
+- Fixed a bug where Compose would occasionally crash while streaming logs
+  when containers would stop or restart
+
 1.10.0 (2017-01-18)
 -------------------
 
@@ -573,7 +620,7 @@ Bug Fixes:
     if at least one container is using the network.
 
 -   When printings logs during `up` or `logs`, flush the output buffer after
-    each line to prevent buffering issues from hideing logs.
+    each line to prevent buffering issues from hiding logs.
 
 -   Recreate a container if one of its dependencies is being created.
     Previously a container was only recreated if it's dependencies already

Dockerfile

@@ -13,6 +13,7 @@ RUN set -ex; \
         ca-certificates \
         curl \
         libsqlite3-dev \
+        libbz2-dev \
     ; \
     rm -rf /var/lib/apt/lists/*
 
@@ -20,40 +21,32 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \
         -o /usr/local/bin/docker && \
     chmod +x /usr/local/bin/docker
 
-# Build Python 2.7.9 from source
+# Build Python 2.7.13 from source
 RUN set -ex; \
-    curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \
+    curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \
-    cd Python-2.7.9; \
+    cd Python-2.7.13; \
     ./configure --enable-shared; \
     make; \
     make install; \
     cd ..; \
-    rm -rf /Python-2.7.9
+    rm -rf /Python-2.7.13
 
 # Build python 3.4 from source
 RUN set -ex; \
-    curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \
+    curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \
-    cd Python-3.4.3; \
+    cd Python-3.4.6; \
     ./configure --enable-shared; \
     make; \
     make install; \
     cd ..; \
-    rm -rf /Python-3.4.3
+    rm -rf /Python-3.4.6
 
 # Make libpython findable
 ENV LD_LIBRARY_PATH /usr/local/lib
 
-# Install setuptools
-RUN set -ex; \
-    curl -L https://bootstrap.pypa.io/ez_setup.py | python
-
 # Install pip
 RUN set -ex; \
-    curl -L https://pypi.python.org/packages/source/p/pip/pip-8.1.1.tar.gz | tar -xz; \
+    curl -L https://bootstrap.pypa.io/get-pip.py | python
-    cd pip-8.1.1; \
-    python setup.py install; \
-    cd ..; \
-    rm -rf pip-8.1.1
 
 # Python3 requires a valid locale
 RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen

README.md

@@ -35,7 +35,7 @@ A `docker-compose.yml` looks like this:
         image: redis
 
 For more information about the Compose file, see the
-[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file.md)
+[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file/compose-versioning.md)
 
 Compose has commands for managing the whole lifecycle of your application:
 

compose/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-__version__ = '1.11.0dev'
+__version__ = '1.12.0dev'

compose/bundle.py

@@ -202,7 +202,7 @@ def convert_service_to_bundle(name, service_dict, image_digest):
     return container_config
 
 
-# See https://github.com/docker/swarmkit/blob//agent/exec/container/container.go#L95
+# See https://github.com/docker/swarmkit/blob/agent/exec/container/container.go#L95
 def set_command_and_args(config, entrypoint, command):
     if isinstance(entrypoint, six.string_types):
         entrypoint = split_command(entrypoint)

compose/cli/__init__.py

@@ -0,0 +1,37 @@
+from __future__ import absolute_import
+from __future__ import print_function
+from __future__ import unicode_literals
+
+import subprocess
+import sys
+
+# Attempt to detect https://github.com/docker/compose/issues/4344
+try:
+    # We don't try importing pip because it messes with package imports
+    # on some Linux distros (Ubuntu, Fedora)
+    # https://github.com/docker/compose/issues/4425
+    # https://github.com/docker/compose/issues/4481
+    # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py
+    s_cmd = subprocess.Popen(
+        ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE
+    )
+    packages = s_cmd.communicate()[0].splitlines()
+    dockerpy_installed = len(
+        list(filter(lambda p: p.startswith(b'docker-py=='), packages))
+    ) > 0
+    if dockerpy_installed:
+        from .colors import red
+        print(
+            red('ERROR:'),
+            "Dependency conflict: an older version of the 'docker-py' package "
+            "is polluting the namespace. "
+            "Run the following command to remedy the issue:\n"
+            "pip uninstall docker docker-py; pip install docker",
+            file=sys.stderr
+        )
+        sys.exit(1)
+
+except OSError:
+    # pip command is not available, which indicates it's probably the binary
+    # distribution of Compose which is not affected
+    pass

compose/cli/main.py

@@ -14,30 +14,6 @@ from distutils.spawn import find_executable
 from inspect import getdoc
 from operator import attrgetter
 
-
-# Attempt to detect https://github.com/docker/compose/issues/4344
-try:
-    # A regular import statement causes PyInstaller to freak out while
-    # trying to load pip. This way it is simply ignored.
-    pip = __import__('pip')
-    pip_packages = pip.get_installed_distributions()
-    if 'docker-py' in [pkg.project_name for pkg in pip_packages]:
-        from .colors import red
-        print(
-            red('ERROR:'),
-            "Dependency conflict: an older version of the 'docker-py' package "
-            "is polluting the namespace. "
-            "Run the following command to remedy the issue:\n"
-            "pip uninstall docker docker-py; pip install docker",
-            file=sys.stderr
-        )
-        sys.exit(1)
-except ImportError:
-    # pip is not available, which indicates it's probably the binary
-    # distribution of Compose which is not affected
-    pass
-
-
 from . import errors
 from . import signals
 from .. import __version__

compose/config/config.py

@@ -186,11 +186,6 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')):
         if version == '3':
             version = V3_0
 
-        if version not in (V2_0, V2_1, V3_0):
-            raise ConfigurationError(
-                'Version in "{}" is unsupported. {}'
-                .format(self.filename, VERSION_EXPLANATION))
-
         return version
 
     def get_service(self, name):
@@ -479,7 +474,7 @@ def process_config_file(config_file, environment, service_name=None):
         'service',
         environment)
 
-    if config_file.version in (V2_0, V2_1, V3_0):
+    if config_file.version in (V2_0, V2_1, V3_0, V3_1):
         processed_config = dict(config_file.config)
         processed_config['services'] = services
         processed_config['volumes'] = interpolate_config_section(
@@ -495,7 +490,9 @@ def process_config_file(config_file, environment, service_name=None):
     elif config_file.version == V1:
         processed_config = services
     else:
-        raise Exception("Unsupported version: {}".format(repr(config_file.version)))
+        raise ConfigurationError(
+            'Version in "{}" is unsupported. {}'
+            .format(config_file.filename, VERSION_EXPLANATION))
 
     config_file = config_file._replace(config=processed_config)
     validate_against_config_schema(config_file)
@@ -766,6 +763,11 @@ def finalize_service(service_config, service_names, version, environment):
     if 'restart' in service_dict:
         service_dict['restart'] = parse_restart_spec(service_dict['restart'])
 
+    if 'secrets' in service_dict:
+        service_dict['secrets'] = [
+            types.ServiceSecret.parse(s) for s in service_dict['secrets']
+        ]
+
     normalize_build(service_dict, service_config.working_dir, environment)
 
     service_dict['name'] = service_config.name

compose/config/environment.py

@@ -2,6 +2,7 @@ from __future__ import absolute_import
 from __future__ import unicode_literals
 
 import codecs
+import contextlib
 import logging
 import os
 
@@ -31,11 +32,12 @@ def env_vars_from_file(filename):
     elif not os.path.isfile(filename):
         raise ConfigurationError("%s is not a file." % (filename))
     env = {}
-    for line in codecs.open(filename, 'r', 'utf-8'):
+    with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj:
-        line = line.strip()
+        for line in fileobj:
-        if line and not line.startswith('#'):
+            line = line.strip()
-            k, v = split_env(line)
+            if line and not line.startswith('#'):
-            env[k] = v
+                k, v = split_env(line)
+                env[k] = v
     return env
 
 

compose/config/serialize.py

@@ -102,4 +102,7 @@ def denormalize_service_dict(service_dict, version):
                 service_dict['healthcheck']['timeout']
             )
 
+    if 'secrets' in service_dict:
+        service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets'])
+
     return service_dict

compose/config/types.py

@@ -253,3 +253,8 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')):
     @property
     def merge_field(self):
         return self.source
+
+    def repr(self):
+        return dict(
+            [(k, v) for k, v in self._asdict().items() if v is not None]
+        )

compose/project.py

@@ -365,7 +365,7 @@ class Project(object):
 
             # TODO: get labels from the API v1.22 , see github issue 2618
             try:
-                # this can fail if the conatiner has been removed
+                # this can fail if the container has been removed
                 container = Container.from_id(self.client, event['id'])
             except APIError:
                 continue

docker-compose.spec

@@ -37,6 +37,11 @@ exe = EXE(pyz,
                 'compose/config/config_schema_v3.0.json',
                 'DATA'
             ),
+            (
+                'compose/config/config_schema_v3.1.json',
+                'compose/config/config_schema_v3.1.json',
+                'DATA'
+            ),
             (
                 'compose/GITSHA',
                 'compose/GITSHA',

requirements-build.txt

@@ -1 +1 @@
-pyinstaller==3.1.1
+pyinstaller==3.2.1

script/release/utils.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 #
-# Util functions for release scritps
+# Util functions for release scripts
 #
 
 set -e

script/run/run.sh

@@ -15,7 +15,7 @@
 
 set -e
 
-VERSION="1.8.0"
+VERSION="1.12.0dev"
 IMAGE="docker/compose:$VERSION"
 
 

setup.py

@@ -1,10 +1,10 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import
+from __future__ import print_function
 from __future__ import unicode_literals
 
 import codecs
-import logging
 import os
 import re
 import sys
@@ -64,11 +64,9 @@ try:
         for key, value in extras_require.items():
             if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]):
                 install_requires.extend(value)
-except Exception:
+except Exception as e:
-    logging.getLogger(__name__).exception(
+    print("Failed to compute platform dependencies: {}. ".format(e) +
-        'Failed to compute platform dependencies. All dependencies will be '
+          "All dependencies will be installed as a result.", file=sys.stderr)
-        'installed as a result.'
-    )
     for key, value in extras_require.items():
         if key.startswith(':'):
             install_requires.extend(value)

tests/acceptance/cli_test.py

@@ -1234,7 +1234,7 @@ class CLITestCase(DockerClientTestCase):
         container = service.containers(stopped=True, one_off=OneOffFilter.only)[0]
         self.assertEqual(user, container.get('Config.User'))
 
-    def test_run_service_with_environement_overridden(self):
+    def test_run_service_with_environment_overridden(self):
         name = 'service'
         self.base_dir = 'tests/fixtures/environment-composefile'
         self.dispatch([
@@ -1246,9 +1246,9 @@ class CLITestCase(DockerClientTestCase):
         ])
         service = self.project.get_service(name)
         container = service.containers(stopped=True, one_off=OneOffFilter.only)[0]
-        # env overriden
+        # env overridden
         self.assertEqual('notbar', container.environment['foo'])
-        # keep environement from yaml
+        # keep environment from yaml
         self.assertEqual('world', container.environment['hello'])
         # added option from command line
         self.assertEqual('beta', container.environment['alpha'])
@@ -1293,7 +1293,7 @@ class CLITestCase(DockerClientTestCase):
         self.assertEqual(port_range[0], "0.0.0.0:49153")
         self.assertEqual(port_range[1], "0.0.0.0:49154")
 
-    def test_run_service_with_explicitly_maped_ports(self):
+    def test_run_service_with_explicitly_mapped_ports(self):
         # create one off container
         self.base_dir = 'tests/fixtures/ports-composefile'
         self.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple'])
@@ -1310,7 +1310,7 @@ class CLITestCase(DockerClientTestCase):
         self.assertEqual(port_short, "0.0.0.0:30000")
         self.assertEqual(port_full, "0.0.0.0:30001")
 
-    def test_run_service_with_explicitly_maped_ip_ports(self):
+    def test_run_service_with_explicitly_mapped_ip_ports(self):
         # create one off container
         self.base_dir = 'tests/fixtures/ports-composefile'
         self.dispatch([

tests/unit/cli_test.py

@@ -184,7 +184,7 @@ class CLITestCase(unittest.TestCase):
             mock_client.create_host_config.call_args[1].get('restart_policy')
         )
 
-    def test_command_manula_and_service_ports_together(self):
+    def test_command_manual_and_service_ports_together(self):
         project = Project.from_config(
             name='composetest',
             client=None,

tests/unit/config/config_test.py

@@ -13,12 +13,14 @@ import pytest
 
 from ...helpers import build_config_details
 from compose.config import config
+from compose.config import types
 from compose.config.config import resolve_build_args
 from compose.config.config import resolve_environment
 from compose.config.config import V1
 from compose.config.config import V2_0
 from compose.config.config import V2_1
 from compose.config.config import V3_0
+from compose.config.config import V3_1
 from compose.config.environment import Environment
 from compose.config.errors import ConfigurationError
 from compose.config.errors import VERSION_EXPLANATION
@@ -52,6 +54,10 @@ def service_sort(services):
     return sorted(services, key=itemgetter('name'))
 
 
+def secret_sort(secrets):
+    return sorted(secrets, key=itemgetter('source'))
+
+
 class ConfigTest(unittest.TestCase):
     def test_load(self):
         service_dicts = config.load(
@@ -168,6 +174,9 @@ class ConfigTest(unittest.TestCase):
             cfg = config.load(build_config_details({'version': version}))
             assert cfg.version == V3_0
 
+        cfg = config.load(build_config_details({'version': '3.1'}))
+        assert cfg.version == V3_1
+
     def test_v1_file_version(self):
         cfg = config.load(build_config_details({'web': {'image': 'busybox'}}))
         assert cfg.version == V1
@@ -1766,6 +1775,38 @@ class ConfigTest(unittest.TestCase):
             'labels': {'com.docker.compose.test': 'yes'}
         }
 
+    def test_merge_different_secrets(self):
+        base = {
+            'image': 'busybox',
+            'secrets': [
+                {'source': 'src.txt'}
+            ]
+        }
+        override = {'secrets': ['other-src.txt']}
+
+        actual = config.merge_service_dicts(base, override, V3_1)
+        assert secret_sort(actual['secrets']) == secret_sort([
+            {'source': 'src.txt'},
+            {'source': 'other-src.txt'}
+        ])
+
+    def test_merge_secrets_override(self):
+        base = {
+            'image': 'busybox',
+            'secrets': ['src.txt'],
+        }
+        override = {
+            'secrets': [
+                {
+                    'source': 'src.txt',
+                    'target': 'data.txt',
+                    'mode': 0o400
+                }
+            ]
+        }
+        actual = config.merge_service_dicts(base, override, V3_1)
+        assert actual['secrets'] == override['secrets']
+
     def test_external_volume_config(self):
         config_details = build_config_details({
             'version': '2',
@@ -1845,6 +1886,91 @@ class ConfigTest(unittest.TestCase):
             config.load(config_details)
         assert 'has neither an image nor a build context' in exc.exconly()
 
+    def test_load_secrets(self):
+        base_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '3.1',
+                'services': {
+                    'web': {
+                        'image': 'example/web',
+                        'secrets': [
+                            'one',
+                            {
+                                'source': 'source',
+                                'target': 'target',
+                                'uid': '100',
+                                'gid': '200',
+                                'mode': 0o777,
+                            },
+                        ],
+                    },
+                },
+                'secrets': {
+                    'one': {'file': 'secret.txt'},
+                },
+            })
+        details = config.ConfigDetails('.', [base_file])
+        service_dicts = config.load(details).services
+        expected = [
+            {
+                'name': 'web',
+                'image': 'example/web',
+                'secrets': [
+                    types.ServiceSecret('one', None, None, None, None),
+                    types.ServiceSecret('source', 'target', '100', '200', 0o777),
+                ],
+            },
+        ]
+        assert service_sort(service_dicts) == service_sort(expected)
+
+    def test_load_secrets_multi_file(self):
+        base_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '3.1',
+                'services': {
+                    'web': {
+                        'image': 'example/web',
+                        'secrets': ['one'],
+                    },
+                },
+                'secrets': {
+                    'one': {'file': 'secret.txt'},
+                },
+            })
+        override_file = config.ConfigFile(
+            'base.yaml',
+            {
+                'version': '3.1',
+                'services': {
+                    'web': {
+                        'secrets': [
+                            {
+                                'source': 'source',
+                                'target': 'target',
+                                'uid': '100',
+                                'gid': '200',
+                                'mode': 0o777,
+                            },
+                        ],
+                    },
+                },
+            })
+        details = config.ConfigDetails('.', [base_file, override_file])
+        service_dicts = config.load(details).services
+        expected = [
+            {
+                'name': 'web',
+                'image': 'example/web',
+                'secrets': [
+                    types.ServiceSecret('one', None, None, None, None),
+                    types.ServiceSecret('source', 'target', '100', '200', 0o777),
+                ],
+            },
+        ]
+        assert service_sort(service_dicts) == service_sort(expected)
+
 
 class NetworkModeTest(unittest.TestCase):
     def test_network_mode_standard(self):
@@ -3401,3 +3527,24 @@ class SerializeTest(unittest.TestCase):
         denormalized_service = denormalize_service_dict(processed_service, V2_1)
         assert denormalized_service['healthcheck']['interval'] == '100s'
         assert denormalized_service['healthcheck']['timeout'] == '30s'
+
+    def test_denormalize_secrets(self):
+        service_dict = {
+            'name': 'web',
+            'image': 'example/web',
+            'secrets': [
+                types.ServiceSecret('one', None, None, None, None),
+                types.ServiceSecret('source', 'target', '100', '200', 0o777),
+            ],
+        }
+        denormalized_service = denormalize_service_dict(service_dict, V3_1)
+        assert secret_sort(denormalized_service['secrets']) == secret_sort([
+            {'source': 'one'},
+            {
+                'source': 'source',
+                'target': 'target',
+                'uid': '100',
+                'gid': '200',
+                'mode': 0o777,
+            },
+        ])