diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec7d5b5..17487890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,106 @@ Change log ========== +1.9.0 (2016-11-16) +----------------- + +**Breaking changes** + +- When using Compose with Docker Toolbox/Machine on Windows, volume paths are + no longer converted from `C:\Users` to `/c/Users`-style by default. To + re-enable this conversion so that your volumes keep working, set the + environment variable `COMPOSE_CONVERT_WINDOWS_PATHS=1`. Users of + Docker for Windows are not affected and do not need to set the variable. + +New Features + +- Interactive mode for `docker-compose run` and `docker-compose exec` is + now supported on Windows platforms. Please note that the `docker` binary + is required to be present on the system for this feature to work. + +- Introduced version 2.1 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.12 or above. + - Added support for setting volume labels and network labels in + `docker-compose.yml`. + - Added support for the `isolation` parameter in service definitions. + - Added support for link-local IPs in the service networks definitions. + - Added support for shell-style inline defaults in variable interpolation. + The supported forms are `${FOO-default}` (fall back if FOO is unset) and + `${FOO:-default}` (fall back if FOO is unset or empty). + +- Added support for the `group_add` and `oom_score_adj` parameters in + service definitions. + +- Added support for the `internal` and `enable_ipv6` parameters in network + definitions. + +- Compose now defaults to using the `npipe` protocol on Windows. + +- Overriding a `logging` configuration will now properly merge the `options` + mappings if the `driver` values do not conflict. + +Bug Fixes + +- Fixed several bugs related to `npipe` protocol support on Windows. + +- Fixed an issue with Windows paths being incorrectly converted when + using Docker on Windows Server. + +- Fixed a bug where an empty `restart` value would sometimes result in an + exception being raised. + +- Fixed an issue where service logs containing unicode characters would + sometimes cause an error to occur. + +- Fixed a bug where unicode values in environment variables would sometimes + raise a unicode exception when retrieved. + +- Fixed an issue where Compose would incorrectly detect a configuration + mismatch for overlay networks. + + +1.8.1 (2016-09-22) +----------------- + +Bug Fixes + +- Fixed a bug where users using a credentials store were not able + to access their private images. + +- Fixed a bug where users using identity tokens to authenticate + were not able to access their private images. + +- Fixed a bug where an `HttpHeaders` entry in the docker configuration + file would cause Compose to crash when trying to build an image. + +- Fixed a few bugs related to the handling of Windows paths in volume + binding declarations. + +- Fixed a bug where Compose would sometimes crash while trying to + read a streaming response from the engine. + +- Fixed an issue where Compose would crash when encountering an API error + while streaming container logs. + +- Fixed an issue where Compose would erroneously try to output logs from + drivers not handled by the Engine's API. + +- Fixed a bug where options from the `docker-machine config` command would + not be properly interpreted by Compose. + +- Fixed a bug where the connection to the Docker Engine would + sometimes fail when running a large number of services simultaneously. + +- Fixed an issue where Compose would sometimes print a misleading + suggestion message when running the `bundle` command. + +- Fixed a bug where connection errors would not be handled properly by + Compose during the project initialization phase. + +- Fixed a bug where a misleading error would appear when encountering + a connection timeout. + + 1.8.0 (2016-06-14) ----------------- diff --git a/Jenkinsfile b/Jenkinsfile index fa29520b..19ccb4bb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,80 @@ -// Only run on Linux atm -wrappedNode(label: 'docker') { - deleteDir() - stage "checkout" - checkout scm +#!groovy - documentationChecker("docs") +def image + +def checkDocs = { -> + wrappedNode(label: 'linux') { + deleteDir(); checkout(scm) + documentationChecker("docs") + } } + +def buildImage = { -> + wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { + stage("build image") { + deleteDir(); checkout(scm) + def imageName = "dockerbuildbot/compose:${gitCommit()}" + image = docker.image(imageName) + try { + image.pull() + } catch (Exception exc) { + image = docker.build(imageName, ".") + image.push() + } + } + } +} + +def runTests = { Map settings -> + def dockerVersions = settings.get("dockerVersions", null) + def pythonVersions = settings.get("pythonVersions", null) + + if (!pythonVersions) { + throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py34')`") + } + if (!dockerVersions) { + throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`") + } + + { -> + wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { + stage("test python=${pythonVersions} / docker=${dockerVersions}") { + deleteDir(); checkout(scm) + def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() + echo "Using local system's storage driver: ${storageDriver}" + sh """docker run \\ + -t \\ + --rm \\ + --privileged \\ + --volume="\$(pwd)/.git:/code/.git" \\ + --volume="/var/run/docker.sock:/var/run/docker.sock" \\ + -e "TAG=${image.id}" \\ + -e "STORAGE_DRIVER=${storageDriver}" \\ + -e "DOCKER_VERSIONS=${dockerVersions}" \\ + -e "BUILD_NUMBER=\$BUILD_TAG" \\ + -e "PY_TEST_VERSIONS=${pythonVersions}" \\ + --entrypoint="script/ci" \\ + ${image.id} \\ + --verbose + """ + } + } + } +} + +def buildAndTest = { -> + buildImage() + // TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all + parallel( + failFast: true, + all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), + all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"), + ) +} + + +parallel( + failFast: false, + docs: checkDocs, + test: buildAndTest +) diff --git a/compose/__init__.py b/compose/__init__.py index 6e610652..072a6a0b 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.9.0dev' +__version__ = '1.9.0' diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 4fdec08a..5b977095 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import contextlib import logging import socket +from distutils.spawn import find_executable from textwrap import dedent from docker.errors import APIError @@ -13,7 +14,6 @@ from requests.exceptions import SSLError from requests.packages.urllib3.exceptions import ReadTimeoutError from ..const import API_VERSION_TO_ENGINE_VERSION -from .utils import call_silently from .utils import is_docker_for_mac_installed from .utils import is_mac from .utils import is_ubuntu @@ -90,11 +90,11 @@ def exit_with_error(msg): def get_conn_error_message(url): - if call_silently(['which', 'docker']) != 0: + if find_executable('docker') is None: return docker_not_found_msg("Couldn't connect to Docker daemon.") if is_docker_for_mac_installed(): return conn_error_docker_for_mac - if call_silently(['which', 'docker-machine']) == 0: + if find_executable('docker-machine') is not None: return conn_error_docker_machine return conn_error_generic.format(url=url) diff --git a/compose/cli/main.py b/compose/cli/main.py index 58b95c2f..08e58e37 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -10,6 +10,7 @@ import pipes import re import subprocess import sys +from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter @@ -1063,9 +1064,8 @@ def exit_if(condition, message, exit_code): def call_docker(args): - try: - executable_path = subprocess.check_output(["which", "docker"]).strip() - except subprocess.CalledProcessError: + executable_path = find_executable('docker') + if not executable_path: raise UserError(errors.docker_not_found_msg("Couldn't find `docker` binary.")) args = [executable_path] + args diff --git a/compose/config/config.py b/compose/config/config.py index 437ed389..9d23b34d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -771,7 +771,7 @@ def merge_service_dicts(base, override, version): for field in ['dns', 'dns_search', 'env_file', 'tmpfs']: md.merge_field(field, merge_list_or_string) - md.merge_field('logging', merge_logging) + md.merge_field('logging', merge_logging, default={}) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index f30b9054..3561a8cf 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -140,6 +140,7 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, + "mem_swappiness": {"type": "integer"}, "network_mode": {"type": "string"}, "networks": { @@ -168,6 +169,14 @@ } ] }, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true + }, "pid": {"type": ["string", "null"]}, "ports": { @@ -248,6 +257,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, diff --git a/compose/network.py b/compose/network.py index e581a4fe..3b57cb94 100644 --- a/compose/network.py +++ b/compose/network.py @@ -12,6 +12,10 @@ from .config import ConfigurationError log = logging.getLogger(__name__) +OPTS_EXCEPTIONS = [ + 'com.docker.network.driver.overlay.vxlanid_list', +] + class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, @@ -49,14 +53,7 @@ class Network(object): try: data = self.inspect() - if self.driver and data['Driver'] != self.driver: - raise ConfigurationError( - 'Network "{}" needs to be recreated - driver has changed' - .format(self.full_name)) - if data['Options'] != (self.driver_opts or {}): - raise ConfigurationError( - 'Network "{}" needs to be recreated - options have changed' - .format(self.full_name)) + check_remote_network_config(data, self) except NotFound: driver_name = 'the default driver' if self.driver: @@ -113,6 +110,24 @@ def create_ipam_config_from_dict(ipam_dict): ) +def check_remote_network_config(remote, local): + if local.driver and remote.get('Driver') != local.driver: + raise ConfigurationError( + 'Network "{}" needs to be recreated - driver has changed' + .format(local.full_name) + ) + local_opts = local.driver_opts or {} + remote_opts = remote.get('Options') or {} + for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): + if k in OPTS_EXCEPTIONS: + continue + if remote_opts.get(k) != local_opts.get(k): + raise ConfigurationError( + 'Network "{}" needs to be recreated - options have changed' + .format(local.full_name) + ) + + def build_networks(name, config_data, client): network_config = config_data.networks or {} networks = { diff --git a/compose/project.py b/compose/project.py index f85e285f..60647fe9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -538,6 +538,10 @@ def get_volumes_from(project, service_dict): def warn_for_swarm_mode(client): info = client.info() if info.get('Swarm', {}).get('LocalNodeState') == 'active': + if info.get('ServerVersion', '').startswith('ucp'): + # UCP does multi-node scheduling with traditional Compose files. + return + log.warn( "The Docker Engine you're using is running in swarm mode.\n\n" "Compose does not use swarm mode to deploy services to multiple nodes in a swarm. " diff --git a/compose/service.py b/compose/service.py index 760d29a7..ad426706 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ from operator import attrgetter import enum import six from docker.errors import APIError +from docker.errors import NotFound from docker.utils import LogConfig from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port @@ -829,12 +830,11 @@ class Service(object): repo, tag, separator = parse_repository_tag(self.options['image']) tag = tag or 'latest' log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) - output = self.client.pull(repo, tag=tag, stream=True) - try: + output = self.client.pull(repo, tag=tag, stream=True) return progress_stream.get_digest_from_pull( stream_output(output, sys.stdout)) - except StreamOutputError as e: + except (StreamOutputError, NotFound) as e: if not ignore_pull_failures: raise else: diff --git a/docker-compose.spec b/docker-compose.spec index 3a165dd6..e57d6b7a 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -27,6 +27,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v2.0.json', 'DATA' ), + ( + 'compose/config/config_schema_v2.1.json', + 'compose/config/config_schema_v2.1.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/requirements.txt b/requirements.txt index 474efbdf..933146c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 -docker-py==1.10.4 +docker-py==1.10.6 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/script/release/make-branch b/script/release/make-branch index 7ccf3f05..bf81219d 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -65,8 +65,8 @@ git config "branch.${BRANCH}.release" $VERSION editor=${EDITOR:-vim} -echo "Update versions in docs/install.md, compose/__init__.py, script/run/run.sh" -$editor docs/install.md +echo "Update versions in compose/__init__.py, script/run/run.sh" +# $editor docs/install.md $editor compose/__init__.py $editor script/run/run.sh diff --git a/script/run/run.sh b/script/run/run.sh index 6205747a..85c7e720 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.8.0" +VERSION="1.9.0" IMAGE="docker/compose:$VERSION" diff --git a/script/test/all b/script/test/all index 08bf1618..7151a75e 100755 --- a/script/test/all +++ b/script/test/all @@ -24,6 +24,7 @@ fi BUILD_NUMBER=${BUILD_NUMBER-$USER} +PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py34} for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" @@ -58,6 +59,6 @@ for version in $DOCKER_VERSIONS; do --env="DOCKER_VERSION=$version" \ --entrypoint="tox" \ "$TAG" \ - -e py27,py34 -- "$@" + -e "$PY_TEST_VERSIONS" -- "$@" done diff --git a/setup.py b/setup.py index 442bc80c..672ea80e 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.10.4, < 2.0', + 'docker-py >= 1.10.6, < 2.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a7cd78f1..f153bd95 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -330,12 +330,13 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_ignore_pull_failures(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', - 'pull', '--ignore-pull-failures']) + 'pull', '--ignore-pull-failures'] + ) assert 'Pulling simple (busybox:latest)...' in result.stderr assert 'Pulling another (nonexisting-image:latest)...' in result.stderr - assert 'Error: image library/nonexisting-image' in result.stderr - assert 'not found' in result.stderr + assert ('repository nonexisting-image not found' in result.stderr or + 'image library/nonexisting-image:latest not found' in result.stderr) def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index e3c34af9..3afefb5a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -826,9 +826,9 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data ) - project.up() + project.up(detached=True) - service_container = project.get_service('web').containers()[0] + service_container = project.get_service('web').containers(stopped=True)[0] ipam_config = service_container.inspect().get( 'NetworkSettings', {} ).get( @@ -857,8 +857,8 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data ) - project.up() - service_container = project.get_service('web').containers()[0] + project.up(detached=True) + service_container = project.get_service('web').containers(stopped=True)[0] assert service_container.inspect()['HostConfig']['Isolation'] == 'default' @v2_1_only() diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index 1d454a08..a7b57562 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -16,9 +16,9 @@ def mock_logging(): yield mock_log -def patch_call_silently(side_effect): +def patch_find_executable(side_effect): return mock.patch( - 'compose.cli.errors.call_silently', + 'compose.cli.errors.find_executable', autospec=True, side_effect=side_effect) @@ -27,7 +27,7 @@ class TestHandleConnectionErrors(object): def test_generic_connection_error(self, mock_logging): with pytest.raises(errors.ConnectionError): - with patch_call_silently([0, 1]): + with patch_find_executable(['/bin/docker', None]): with handle_connection_errors(mock.Mock()): raise ConnectionError() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d205e282..51c5e226 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1657,6 +1657,51 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_logging_v2_no_base(self): + base = { + 'image': 'alpine:edge' + } + override = { + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000' + } + } + } + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000' + } + } + } + + def test_merge_logging_v2_no_override(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'frequency': '2000' + } + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'frequency': '2000' + } + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', @@ -2448,6 +2493,15 @@ class EnvTest(unittest.TestCase): {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'bar'}, ) + def test_environment_overrides_env_file(self): + self.assertEqual( + resolve_environment({ + 'environment': {'FOO': 'baz'}, + 'env_file': ['tests/fixtures/env/one.env'], + }), + {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'baz'}, + ) + def test_resolve_environment_with_multiple_env_files(self): service_dict = { 'env_file': [ diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py new file mode 100644 index 00000000..12d06f41 --- /dev/null +++ b/tests/unit/network_test.py @@ -0,0 +1,55 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import pytest + +from .. import unittest +from compose.config import ConfigurationError +from compose.network import check_remote_network_config +from compose.network import Network + + +class NetworkTest(unittest.TestCase): + def test_check_remote_network_config_success(self): + options = {'com.docker.network.driver.foo': 'bar'} + net = Network( + None, 'compose_test', 'net1', 'bridge', + options + ) + check_remote_network_config( + {'Driver': 'bridge', 'Options': options}, net + ) + + def test_check_remote_network_config_whitelist(self): + options = {'com.docker.network.driver.foo': 'bar'} + remote_options = { + 'com.docker.network.driver.overlay.vxlanid_list': '257', + 'com.docker.network.driver.foo': 'bar' + } + net = Network( + None, 'compose_test', 'net1', 'overlay', + options + ) + check_remote_network_config( + {'Driver': 'overlay', 'Options': remote_options}, net + ) + + def test_check_remote_network_config_driver_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + with pytest.raises(ConfigurationError): + check_remote_network_config( + {'Driver': 'bridge', 'Options': {}}, net + ) + + def test_check_remote_network_config_options_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + with pytest.raises(ConfigurationError): + check_remote_network_config({'Driver': 'overlay', 'Options': { + 'com.docker.network.driver.foo': 'baz' + }}, net) + + def test_check_remote_network_config_null_remote(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + check_remote_network_config( + {'Driver': 'overlay', 'Options': None}, net + )