Bump 1.9.0

Signed-off-by: Joffrey F <joffrey@docker.com>

CHANGELOG.md

@@ -1,6 +1,106 @@
 Change log
 ==========
 
+1.9.0 (2016-11-16)
+-----------------
+
+**Breaking changes**
+
+- When using Compose with Docker Toolbox/Machine on Windows, volume paths are
+  no longer converted from `C:\Users` to `/c/Users`-style by default. To
+  re-enable this conversion so that your volumes keep working, set the
+  environment variable `COMPOSE_CONVERT_WINDOWS_PATHS=1`. Users of
+  Docker for Windows are not affected and do not need to set the variable.
+
+New Features
+
+- Interactive mode for `docker-compose run` and `docker-compose exec` is
+  now supported on Windows platforms. Please note that the `docker` binary
+  is required to be present on the system for this feature to work.
+
+- Introduced version 2.1 of the `docker-compose.yml` specification. This
+  version requires to be used with Docker Engine 1.12 or above.
+    - Added support for setting volume labels and network labels in
+  `docker-compose.yml`.
+    - Added support for the `isolation` parameter in service definitions.
+    - Added support for link-local IPs in the service networks definitions.
+    - Added support for shell-style inline defaults in variable interpolation.
+      The supported forms are `${FOO-default}` (fall back if FOO is unset) and
+      `${FOO:-default}` (fall back if FOO is unset or empty).
+
+- Added support for the `group_add` and `oom_score_adj` parameters in
+  service definitions.
+
+- Added support for the `internal` and `enable_ipv6` parameters in network
+  definitions.
+
+- Compose now defaults to using the `npipe` protocol on Windows.
+
+- Overriding a `logging` configuration will now properly merge the `options`
+  mappings if the `driver` values do not conflict.
+
+Bug Fixes
+
+- Fixed several bugs related to `npipe` protocol support on Windows.
+
+- Fixed an issue with Windows paths being incorrectly converted when
+  using Docker on Windows Server.
+
+- Fixed a bug where an empty `restart` value would sometimes result in an
+  exception being raised.
+
+- Fixed an issue where service logs containing unicode characters would
+  sometimes cause an error to occur.
+
+- Fixed a bug where unicode values in environment variables would sometimes
+  raise a unicode exception when retrieved.
+
+- Fixed an issue where Compose would incorrectly detect a configuration
+  mismatch for overlay networks.
+
+
+1.8.1 (2016-09-22)
+-----------------
+
+Bug Fixes
+
+- Fixed a bug where users using a credentials store were not able
+  to access their private images.
+
+- Fixed a bug where users using identity tokens to authenticate
+  were not able to access their private images.
+
+- Fixed a bug where an `HttpHeaders` entry in the docker configuration
+  file would cause Compose to crash when trying to build an image.
+
+- Fixed a few bugs related to the handling of Windows paths in volume
+  binding declarations.
+
+- Fixed a bug where Compose would sometimes crash while trying to
+  read a streaming response from the engine.
+
+- Fixed an issue where Compose would crash when encountering an API error
+  while streaming container logs.
+
+- Fixed an issue where Compose would erroneously try to output logs from
+  drivers not handled by the Engine's API.
+
+- Fixed a bug where options from the `docker-machine config` command would
+  not be properly interpreted by Compose.
+
+- Fixed a bug where the connection to the Docker Engine would
+  sometimes fail when running a large number of services simultaneously.
+
+- Fixed an issue where Compose would sometimes print a misleading
+  suggestion message when running the `bundle` command.
+
+- Fixed a bug where connection errors would not be handled properly by
+  Compose during the project initialization phase.
+
+- Fixed a bug where a misleading error would appear when encountering
+  a connection timeout.
+
+
 1.8.0 (2016-06-14)
 -----------------
 

Jenkinsfile

@@ -1,8 +1,80 @@
-// Only run on Linux atm
+#!groovy
-wrappedNode(label: 'docker') {
-  deleteDir()
-  stage "checkout"
-  checkout scm
 
+def image
+
+def checkDocs = { ->
+  wrappedNode(label: 'linux') {
+    deleteDir(); checkout(scm)
     documentationChecker("docs")
   }
+}
+
+def buildImage = { ->
+  wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) {
+    stage("build image") {
+      deleteDir(); checkout(scm)
+      def imageName = "dockerbuildbot/compose:${gitCommit()}"
+      image = docker.image(imageName)
+      try {
+        image.pull()
+      } catch (Exception exc) {
+        image = docker.build(imageName, ".")
+        image.push()
+      }
+    }
+  }
+}
+
+def runTests = { Map settings ->
+  def dockerVersions = settings.get("dockerVersions", null)
+  def pythonVersions = settings.get("pythonVersions", null)
+
+  if (!pythonVersions) {
+    throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py34')`")
+  }
+  if (!dockerVersions) {
+    throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`")
+  }
+
+  { ->
+    wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) {
+      stage("test python=${pythonVersions} / docker=${dockerVersions}") {
+        deleteDir(); checkout(scm)
+        def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim()
+        echo "Using local system's storage driver: ${storageDriver}"
+        sh """docker run \\
+          -t \\
+          --rm \\
+          --privileged \\
+          --volume="\$(pwd)/.git:/code/.git" \\
+          --volume="/var/run/docker.sock:/var/run/docker.sock" \\
+          -e "TAG=${image.id}" \\
+          -e "STORAGE_DRIVER=${storageDriver}" \\
+          -e "DOCKER_VERSIONS=${dockerVersions}" \\
+          -e "BUILD_NUMBER=\$BUILD_TAG" \\
+          -e "PY_TEST_VERSIONS=${pythonVersions}" \\
+          --entrypoint="script/ci" \\
+          ${image.id} \\
+          --verbose
+        """
+      }
+    }
+  }
+}
+
+def buildAndTest = { ->
+  buildImage()
+  // TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all
+  parallel(
+    failFast: true,
+    all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"),
+    all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"),
+  )
+}
+
+
+parallel(
+  failFast: false,
+  docs: checkDocs,
+  test: buildAndTest
+)

compose/__init__.py

@@ -1,4 +1,4 @@
 from __future__ import absolute_import
 from __future__ import unicode_literals
 
-__version__ = '1.9.0dev'
+__version__ = '1.9.0'

compose/cli/errors.py

@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import contextlib
 import logging
 import socket
+from distutils.spawn import find_executable
 from textwrap import dedent
 
 from docker.errors import APIError
@@ -13,7 +14,6 @@ from requests.exceptions import SSLError
 from requests.packages.urllib3.exceptions import ReadTimeoutError
 
 from ..const import API_VERSION_TO_ENGINE_VERSION
-from .utils import call_silently
 from .utils import is_docker_for_mac_installed
 from .utils import is_mac
 from .utils import is_ubuntu
@@ -90,11 +90,11 @@ def exit_with_error(msg):
 
 
 def get_conn_error_message(url):
-    if call_silently(['which', 'docker']) != 0:
+    if find_executable('docker') is None:
         return docker_not_found_msg("Couldn't connect to Docker daemon.")
     if is_docker_for_mac_installed():
         return conn_error_docker_for_mac
-    if call_silently(['which', 'docker-machine']) == 0:
+    if find_executable('docker-machine') is not None:
         return conn_error_docker_machine
     return conn_error_generic.format(url=url)
 

compose/cli/main.py

@@ -10,6 +10,7 @@ import pipes
 import re
 import subprocess
 import sys
+from distutils.spawn import find_executable
 from inspect import getdoc
 from operator import attrgetter
 
@@ -1063,9 +1064,8 @@ def exit_if(condition, message, exit_code):
 
 
 def call_docker(args):
-    try:
+    executable_path = find_executable('docker')
-        executable_path = subprocess.check_output(["which", "docker"]).strip()
+    if not executable_path:
-    except subprocess.CalledProcessError:
         raise UserError(errors.docker_not_found_msg("Couldn't find `docker` binary."))
 
     args = [executable_path] + args

compose/config/config.py

@@ -771,7 +771,7 @@ def merge_service_dicts(base, override, version):
     for field in ['dns', 'dns_search', 'env_file', 'tmpfs']:
         md.merge_field(field, merge_list_or_string)
 
-    md.merge_field('logging', merge_logging)
+    md.merge_field('logging', merge_logging, default={})
 
     for field in set(ALLOWED_KEYS) - set(md):
         md.merge_scalar(field)

compose/config/config_schema_v2.1.json

@@ -140,6 +140,7 @@
         "mac_address": {"type": "string"},
         "mem_limit": {"type": ["number", "string"]},
         "memswap_limit": {"type": ["number", "string"]},
+        "mem_swappiness": {"type": "integer"},
         "network_mode": {"type": "string"},
 
         "networks": {
@@ -168,6 +169,14 @@
             }
           ]
         },
+        "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000},
+        "group_add": {
+            "type": "array",
+            "items": {
+                "type": ["string", "number"]
+            },
+            "uniqueItems": true
+        },
         "pid": {"type": ["string", "null"]},
 
         "ports": {
@@ -248,6 +257,7 @@
           },
           "additionalProperties": false
         },
+        "internal": {"type": "boolean"},
         "enable_ipv6": {"type": "boolean"},
         "labels": {"$ref": "#/definitions/list_or_dict"}
       },

compose/network.py

@@ -12,6 +12,10 @@ from .config import ConfigurationError
 
 log = logging.getLogger(__name__)
 
+OPTS_EXCEPTIONS = [
+    'com.docker.network.driver.overlay.vxlanid_list',
+]
+
 
 class Network(object):
     def __init__(self, client, project, name, driver=None, driver_opts=None,
@@ -49,14 +53,7 @@ class Network(object):
 
         try:
             data = self.inspect()
-            if self.driver and data['Driver'] != self.driver:
+            check_remote_network_config(data, self)
-                raise ConfigurationError(
-                    'Network "{}" needs to be recreated - driver has changed'
-                    .format(self.full_name))
-            if data['Options'] != (self.driver_opts or {}):
-                raise ConfigurationError(
-                    'Network "{}" needs to be recreated - options have changed'
-                    .format(self.full_name))
         except NotFound:
             driver_name = 'the default driver'
             if self.driver:
@@ -113,6 +110,24 @@ def create_ipam_config_from_dict(ipam_dict):
     )
 
 
+def check_remote_network_config(remote, local):
+    if local.driver and remote.get('Driver') != local.driver:
+        raise ConfigurationError(
+            'Network "{}" needs to be recreated - driver has changed'
+            .format(local.full_name)
+        )
+    local_opts = local.driver_opts or {}
+    remote_opts = remote.get('Options') or {}
+    for k in set.union(set(remote_opts.keys()), set(local_opts.keys())):
+        if k in OPTS_EXCEPTIONS:
+            continue
+        if remote_opts.get(k) != local_opts.get(k):
+            raise ConfigurationError(
+                'Network "{}" needs to be recreated - options have changed'
+                .format(local.full_name)
+            )
+
+
 def build_networks(name, config_data, client):
     network_config = config_data.networks or {}
     networks = {

compose/project.py

@@ -538,6 +538,10 @@ def get_volumes_from(project, service_dict):
 def warn_for_swarm_mode(client):
     info = client.info()
     if info.get('Swarm', {}).get('LocalNodeState') == 'active':
+        if info.get('ServerVersion', '').startswith('ucp'):
+            # UCP does multi-node scheduling with traditional Compose files.
+            return
+
         log.warn(
             "The Docker Engine you're using is running in swarm mode.\n\n"
             "Compose does not use swarm mode to deploy services to multiple nodes in a swarm. "

compose/service.py

@@ -10,6 +10,7 @@ from operator import attrgetter
 import enum
 import six
 from docker.errors import APIError
+from docker.errors import NotFound
 from docker.utils import LogConfig
 from docker.utils.ports import build_port_bindings
 from docker.utils.ports import split_port
@@ -829,12 +830,11 @@ class Service(object):
         repo, tag, separator = parse_repository_tag(self.options['image'])
         tag = tag or 'latest'
         log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag))
-        output = self.client.pull(repo, tag=tag, stream=True)
-
         try:
+            output = self.client.pull(repo, tag=tag, stream=True)
             return progress_stream.get_digest_from_pull(
                 stream_output(output, sys.stdout))
-        except StreamOutputError as e:
+        except (StreamOutputError, NotFound) as e:
             if not ignore_pull_failures:
                 raise
             else:

docker-compose.spec

@@ -27,6 +27,11 @@ exe = EXE(pyz,
                 'compose/config/config_schema_v2.0.json',
                 'DATA'
             ),
+            (
+                'compose/config/config_schema_v2.1.json',
+                'compose/config/config_schema_v2.1.json',
+                'DATA'
+            ),
             (
                 'compose/GITSHA',
                 'compose/GITSHA',

requirements.txt

@@ -1,7 +1,7 @@
 PyYAML==3.11
 backports.ssl-match-hostname==3.5.0.1; python_version < '3'
 cached-property==1.2.0
-docker-py==1.10.4
+docker-py==1.10.6
 dockerpty==0.4.1
 docopt==0.6.1
 enum34==1.0.4; python_version < '3.4'

script/release/make-branch

@@ -65,8 +65,8 @@ git config "branch.${BRANCH}.release" $VERSION
 
 editor=${EDITOR:-vim}
 
-echo "Update versions in docs/install.md, compose/__init__.py, script/run/run.sh"
+echo "Update versions in compose/__init__.py, script/run/run.sh"
-$editor docs/install.md
+# $editor docs/install.md
 $editor compose/__init__.py
 $editor script/run/run.sh
 

script/run/run.sh

@@ -15,7 +15,7 @@
 
 set -e
 
-VERSION="1.8.0"
+VERSION="1.9.0"
 IMAGE="docker/compose:$VERSION"
 
 

script/test/all

@@ -24,6 +24,7 @@ fi
 
 
 BUILD_NUMBER=${BUILD_NUMBER-$USER}
+PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py34}
 
 for version in $DOCKER_VERSIONS; do
   >&2 echo "Running tests against Docker $version"
@@ -58,6 +59,6 @@ for version in $DOCKER_VERSIONS; do
     --env="DOCKER_VERSION=$version" \
     --entrypoint="tox" \
     "$TAG" \
-    -e py27,py34 -- "$@"
+    -e "$PY_TEST_VERSIONS" -- "$@"
 
 done

setup.py

@@ -34,7 +34,7 @@ install_requires = [
     'requests >= 2.6.1, != 2.11.0, < 2.12',
     'texttable >= 0.8.1, < 0.9',
     'websocket-client >= 0.32.0, < 1.0',
-    'docker-py >= 1.10.4, < 2.0',
+    'docker-py >= 1.10.6, < 2.0',
     'dockerpty >= 0.4.1, < 0.5',
     'six >= 1.3.0, < 2',
     'jsonschema >= 2.5.1, < 3',

tests/acceptance/cli_test.py

@@ -330,12 +330,13 @@ class CLITestCase(DockerClientTestCase):
     def test_pull_with_ignore_pull_failures(self):
         result = self.dispatch([
             '-f', 'ignore-pull-failures.yml',
-            'pull', '--ignore-pull-failures'])
+            'pull', '--ignore-pull-failures']
+        )
 
         assert 'Pulling simple (busybox:latest)...' in result.stderr
         assert 'Pulling another (nonexisting-image:latest)...' in result.stderr
-        assert 'Error: image library/nonexisting-image' in result.stderr
+        assert ('repository nonexisting-image not found' in result.stderr or
-        assert 'not found' in result.stderr
+                'image library/nonexisting-image:latest not found' in result.stderr)
 
     def test_build_plain(self):
         self.base_dir = 'tests/fixtures/simple-dockerfile'

tests/integration/project_test.py

@@ -826,9 +826,9 @@ class ProjectTest(DockerClientTestCase):
             name='composetest',
             config_data=config_data
         )
-        project.up()
+        project.up(detached=True)
 
-        service_container = project.get_service('web').containers()[0]
+        service_container = project.get_service('web').containers(stopped=True)[0]
         ipam_config = service_container.inspect().get(
             'NetworkSettings', {}
         ).get(
@@ -857,8 +857,8 @@ class ProjectTest(DockerClientTestCase):
             name='composetest',
             config_data=config_data
         )
-        project.up()
+        project.up(detached=True)
-        service_container = project.get_service('web').containers()[0]
+        service_container = project.get_service('web').containers(stopped=True)[0]
         assert service_container.inspect()['HostConfig']['Isolation'] == 'default'
 
     @v2_1_only()

tests/unit/cli/errors_test.py

@@ -16,9 +16,9 @@ def mock_logging():
         yield mock_log
 
 
-def patch_call_silently(side_effect):
+def patch_find_executable(side_effect):
     return mock.patch(
-        'compose.cli.errors.call_silently',
+        'compose.cli.errors.find_executable',
         autospec=True,
         side_effect=side_effect)
 
@@ -27,7 +27,7 @@ class TestHandleConnectionErrors(object):
 
     def test_generic_connection_error(self, mock_logging):
         with pytest.raises(errors.ConnectionError):
-            with patch_call_silently([0, 1]):
+            with patch_find_executable(['/bin/docker', None]):
                 with handle_connection_errors(mock.Mock()):
                     raise ConnectionError()
 

tests/unit/config/config_test.py

@@ -1657,6 +1657,51 @@ class ConfigTest(unittest.TestCase):
             }
         }
 
+    def test_merge_logging_v2_no_base(self):
+        base = {
+            'image': 'alpine:edge'
+        }
+        override = {
+            'logging': {
+                'driver': 'json-file',
+                'options': {
+                    'frequency': '2000'
+                }
+            }
+        }
+        actual = config.merge_service_dicts(base, override, V2_0)
+        assert actual == {
+            'image': 'alpine:edge',
+            'logging': {
+                'driver': 'json-file',
+                'options': {
+                    'frequency': '2000'
+                }
+            }
+        }
+
+    def test_merge_logging_v2_no_override(self):
+        base = {
+            'image': 'alpine:edge',
+            'logging': {
+                'driver': 'syslog',
+                'options': {
+                    'frequency': '2000'
+                }
+            }
+        }
+        override = {}
+        actual = config.merge_service_dicts(base, override, V2_0)
+        assert actual == {
+            'image': 'alpine:edge',
+            'logging': {
+                'driver': 'syslog',
+                'options': {
+                    'frequency': '2000'
+                }
+            }
+        }
+
     def test_external_volume_config(self):
         config_details = build_config_details({
             'version': '2',
@@ -2448,6 +2493,15 @@ class EnvTest(unittest.TestCase):
             {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'bar'},
         )
 
+    def test_environment_overrides_env_file(self):
+        self.assertEqual(
+            resolve_environment({
+                'environment': {'FOO': 'baz'},
+                'env_file': ['tests/fixtures/env/one.env'],
+            }),
+            {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'baz'},
+        )
+
     def test_resolve_environment_with_multiple_env_files(self):
         service_dict = {
             'env_file': [

tests/unit/network_test.py

@@ -0,0 +1,55 @@
+from __future__ import absolute_import
+from __future__ import unicode_literals
+
+import pytest
+
+from .. import unittest
+from compose.config import ConfigurationError
+from compose.network import check_remote_network_config
+from compose.network import Network
+
+
+class NetworkTest(unittest.TestCase):
+    def test_check_remote_network_config_success(self):
+        options = {'com.docker.network.driver.foo': 'bar'}
+        net = Network(
+            None, 'compose_test', 'net1', 'bridge',
+            options
+        )
+        check_remote_network_config(
+            {'Driver': 'bridge', 'Options': options}, net
+        )
+
+    def test_check_remote_network_config_whitelist(self):
+        options = {'com.docker.network.driver.foo': 'bar'}
+        remote_options = {
+            'com.docker.network.driver.overlay.vxlanid_list': '257',
+            'com.docker.network.driver.foo': 'bar'
+        }
+        net = Network(
+            None, 'compose_test', 'net1', 'overlay',
+            options
+        )
+        check_remote_network_config(
+            {'Driver': 'overlay', 'Options': remote_options}, net
+        )
+
+    def test_check_remote_network_config_driver_mismatch(self):
+        net = Network(None, 'compose_test', 'net1', 'overlay')
+        with pytest.raises(ConfigurationError):
+            check_remote_network_config(
+                {'Driver': 'bridge', 'Options': {}}, net
+            )
+
+    def test_check_remote_network_config_options_mismatch(self):
+        net = Network(None, 'compose_test', 'net1', 'overlay')
+        with pytest.raises(ConfigurationError):
+            check_remote_network_config({'Driver': 'overlay', 'Options': {
+                'com.docker.network.driver.foo': 'baz'
+            }}, net)
+
+    def test_check_remote_network_config_null_remote(self):
+        net = Network(None, 'compose_test', 'net1', 'overlay')
+        check_remote_network_config(
+            {'Driver': 'overlay', 'Options': None}, net
+        )