From cb3bf869f46e2aa2ead695b942a9d1d7b07b23f3 Mon Sep 17 00:00:00 2001 From: Andreas Kohn Date: Tue, 20 Sep 2016 13:57:34 +0200 Subject: [PATCH 01/98] Fix typo Signed-off-by: Andreas Kohn --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec7d5b5..cd4a2370 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -43,7 +43,7 @@ Bug Fixes - Fixed a bug in Windows environment where volume mappings of the host's root directory would be parsed incorrectly. -- Fixed a bug where `docker-compose config` would ouput an invalid +- Fixed a bug where `docker-compose config` would output an invalid Compose file if external networks were specified. - Fixed an issue where unset buildargs would be assigned a string From 90356b7040be49e402b1e176f4db7461659fbbd5 Mon Sep 17 00:00:00 2001 From: Matthew Bray Date: Wed, 28 Sep 2016 12:04:13 +0100 Subject: [PATCH 02/98] Zsh completion: permit multiple --file arguments Before this change: ``` $ docker-compose --file docker-compose.yml - -- option -- --help -h -- Get help --host -H -- Daemon socket to connect to --project-name -p -- Specify an alternate project name (default: directory name) --skip-hostname-check -- Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address) --tls -- Use TLS; implied by --tlsverify --tlscacert -- Trust certs signed only by this CA --tlscert -- Path to TLS certificate file --tlskey -- Path to TLS key file --tlsverify -- Use TLS and verify the remote --verbose -- Show more output --version -v -- Print version and exit ``` (Note the `--file` argument is no longer available to complete.) After this change: ``` docker-compose --file docker-compose.yml - -- option -- --file -f -- Specify an alternate docker-compose file (default: docker-compose.yml) --help -h -- Get help --host -H -- Daemon socket to connect to --project-name -p -- Specify an alternate project name (default: directory name) --skip-hostname-check -- Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address) --tls -- Use TLS; implied by --tlsverify --tlscacert -- Trust certs signed only by this CA --tlscert -- Path to TLS certificate file --tlskey -- Path to TLS key file --tlsverify -- Use TLS and verify the remote --verbose -- Show more output --version -v -- Print version and exit ``` Signed-off-by: Matt Bray --- contrib/completion/zsh/_docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 928e28de..fae75842 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -388,7 +388,7 @@ _docker-compose() { _arguments -C \ '(- :)'{-h,--help}'[Get help]' \ - '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ + '*'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ '--verbose[Show more output]' \ '(- :)'{-v,--version}'[Print version and exit]' \ From a37d99f20114efced6381336990252f2e8238850 Mon Sep 17 00:00:00 2001 From: Matt Bray Date: Fri, 30 Sep 2016 00:38:48 +0100 Subject: [PATCH 03/98] Zsh completion: change --file description text Signed-off-by: Matt Bray --- contrib/completion/zsh/_docker-compose | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index fae75842..ceb7d0f5 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -386,9 +386,17 @@ _docker-compose() { integer ret=1 typeset -A opt_args + local file_description + + if [[ -n ${words[(r)-f]} || -n ${words[(r)--file]} ]] ; then + file_description="Specify an override docker-compose file (default: docker-compose.override.yml)" + else + file_description="Specify an alternate docker-compose file (default: docker-compose.yml)" + fi + _arguments -C \ '(- :)'{-h,--help}'[Get help]' \ - '*'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ + '*'{-f,--file}"[${file_description}]:file:_files -g '*.yml'" \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ '--verbose[Show more output]' \ '(- :)'{-v,--version}'[Print version and exit]' \ From f039c8b43cae5fab4b8a22004ae6ab56d4d698b8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 20 Oct 2016 17:39:55 -0700 Subject: [PATCH 04/98] Update release process document to account for recent changes. Signed-off-by: Joffrey F --- project/RELEASE-PROCESS.md | 62 ++++++++++++++++++++------------------ 1 file changed, 33 insertions(+), 29 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 930af15a..c1834f2f 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -20,18 +20,30 @@ release. As part of this script you'll be asked to: -1. Update the version in `docs/install.md` and `compose/__init__.py`. +1. Update the version in `compose/__init__.py` and `script/run/run.sh`. - If the next release will be an RC, append `rcN`, e.g. `1.4.0rc1`. + If the next release will be an RC, append `-rcN`, e.g. `1.4.0-rc1`. 2. Write release notes in `CHANGES.md`. - Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context where appropriate. + Almost every feature enhancement should be mentioned, with the most + visible/exciting ones first. Use descriptive sentences and give context + where appropriate. - Bug fixes are worth mentioning if it's likely that they've affected lots of people, or if they were regressions in the previous version. + Bug fixes are worth mentioning if it's likely that they've affected lots + of people, or if they were regressions in the previous version. Improvements to the code are not worth mentioning. +3. Create a new repository on [bintray](https://bintray.com/docker-compose). + The name has to match the name of the branch (e.g. `bump-1.9.0`) and the + type should be "Generic". Other fields can be left blank. + +4. Check that the `vnext-compose` branch on + [the docs repo](https://github.com/docker/docker.github.io/) has + documentation for all the new additions in the upcoming release, and create + a PR there for what needs to be amended. + ## When a PR is merged into master that we want in the release @@ -55,8 +67,8 @@ Check out the bump branch and run the `build-binaries` script When prompted build the non-linux binaries and test them. -1. Download the osx binary from Bintray. Make sure that the latest build has - finished, otherwise you'll be downloading an old binary. +1. Download the osx binary from Bintray. Make sure that the latest Travis + build has finished, otherwise you'll be downloading an old binary. https://dl.bintray.com/docker-compose/$BRANCH_NAME/ @@ -67,22 +79,24 @@ When prompted build the non-linux binaries and test them. 3. Draft a release from the tag on GitHub (the script will open the window for you) - In the "Tag version" dropdown, select the tag you just pushed. + The tag will only be present on Github when you run the `push-release` + script in step 7, but you can pre-fill it at that point. -4. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: +4. Paste in installation instructions and release notes. Here's an example - + change the Compose version and Docker version as appropriate: - Firstly, note that Compose 1.5.0 requires Docker 1.8.0 or later. + If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. - Secondly, if you're a Mac user, the **[Docker Toolbox](https://www.docker.com/toolbox)** will install Compose 1.5.0 for you, alongside the latest versions of the Docker Engine, Machine and Kitematic. + Note that Compose 1.9.0 requires Docker Engine 1.10.0 or later for version 2 of the Compose File format, and Docker Engine 1.9.1 or later for version 1. Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. - Otherwise, you can use the usual commands to install/upgrade. Either download the binary: + Alternatively, you can use the usual commands to install or upgrade Compose: - curl -L https://github.com/docker/compose/releases/download/1.5.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + ``` + curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + ``` - Or install the PyPi package: - - pip install -U docker-compose==1.5.0 + See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. Here's what's new: @@ -99,6 +113,8 @@ When prompted build the non-linux binaries and test them. ./script/release/push-release +8. Merge the bump PR. + 8. Publish the release on GitHub. 9. Check that all the binaries download (following the install instructions) and run. @@ -107,19 +123,7 @@ When prompted build the non-linux binaries and test them. ## If it’s a stable release (not an RC) -1. Merge the bump PR. - -2. Make sure `origin/release` is updated locally: - - git fetch origin - -3. Update the `docs` branch on the upstream repo: - - git push git@github.com:docker/compose.git origin/release:docs - -4. Let the docs team know that it’s been updated so they can publish it. - -5. Close the release’s milestone. +1. Close the release’s milestone. ## If it’s a minor release (1.x.0), rather than a patch release (1.x.y) From 3d8dc6f47a1484b0b6f8e3bd5cbf6115524d3892 Mon Sep 17 00:00:00 2001 From: Michal Zdrojewski Date: Mon, 24 Oct 2016 15:05:01 +0100 Subject: [PATCH 05/98] fix(docs): updated documentation links Signed-off-by: Michal Zdrojewski --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 93550f5a..5cf69b05 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ Compose is a tool for defining and running multi-container Docker applications. With Compose, you use a Compose file to configure your application's services. Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](https://github.com/docker/compose/blob/release/docs/overview.md#features). +see [the list of features](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in -[Common Use Cases](https://github.com/docker/compose/blob/release/docs/overview.md#common-use-cases). +[Common Use Cases](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#common-use-cases). Using Compose is basically a three-step process. @@ -35,7 +35,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](https://github.com/docker/compose/blob/release/docs/compose-file.md) +[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file.md) Compose has commands for managing the whole lifecycle of your application: From 2c24bc3a083e138898cfe619a9cbf414f6df12d8 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Oct 2016 10:55:04 -0700 Subject: [PATCH 06/98] Add missing config schema to docker-compose.spec Signed-off-by: Joffrey F --- docker-compose.spec | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.spec b/docker-compose.spec index 3a165dd6..e57d6b7a 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -27,6 +27,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v2.0.json', 'DATA' ), + ( + 'compose/config/config_schema_v2.1.json', + 'compose/config/config_schema_v2.1.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', From ea68be3441ef2c0c100b8bac9499fa4180106eb5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Oct 2016 11:36:44 -0700 Subject: [PATCH 07/98] Do not print Swarm mode warning when connecting to a UCP server Signed-off-by: Joffrey F --- compose/project.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/project.py b/compose/project.py index f85e285f..60647fe9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -538,6 +538,10 @@ def get_volumes_from(project, service_dict): def warn_for_swarm_mode(client): info = client.info() if info.get('Swarm', {}).get('LocalNodeState') == 'active': + if info.get('ServerVersion', '').startswith('ucp'): + # UCP does multi-node scheduling with traditional Compose files. + return + log.warn( "The Docker Engine you're using is running in swarm mode.\n\n" "Compose does not use swarm mode to deploy services to multiple nodes in a swarm. " From 43e29b41c04dfa00fee294f57ccd2e5d1a80f99a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Oct 2016 11:51:45 -0700 Subject: [PATCH 08/98] Fix schema divergence - add missing fields to compose 2.1 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v2.1.json | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index f30b9054..3561a8cf 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -140,6 +140,7 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, + "mem_swappiness": {"type": "integer"}, "network_mode": {"type": "string"}, "networks": { @@ -168,6 +169,14 @@ } ] }, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true + }, "pid": {"type": ["string", "null"]}, "ports": { @@ -248,6 +257,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "enable_ipv6": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, From d2fb146913a1204b805ceb05e3d2b1b1a1006675 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Oct 2016 12:12:19 -0700 Subject: [PATCH 09/98] Replace "which" calls with the portable find_executable function Signed-off-by: Joffrey F --- compose/cli/errors.py | 6 +++--- compose/cli/main.py | 6 +++--- tests/unit/cli/errors_test.py | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 4fdec08a..5b977095 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import contextlib import logging import socket +from distutils.spawn import find_executable from textwrap import dedent from docker.errors import APIError @@ -13,7 +14,6 @@ from requests.exceptions import SSLError from requests.packages.urllib3.exceptions import ReadTimeoutError from ..const import API_VERSION_TO_ENGINE_VERSION -from .utils import call_silently from .utils import is_docker_for_mac_installed from .utils import is_mac from .utils import is_ubuntu @@ -90,11 +90,11 @@ def exit_with_error(msg): def get_conn_error_message(url): - if call_silently(['which', 'docker']) != 0: + if find_executable('docker') is None: return docker_not_found_msg("Couldn't connect to Docker daemon.") if is_docker_for_mac_installed(): return conn_error_docker_for_mac - if call_silently(['which', 'docker-machine']) == 0: + if find_executable('docker-machine') is not None: return conn_error_docker_machine return conn_error_generic.format(url=url) diff --git a/compose/cli/main.py b/compose/cli/main.py index 58b95c2f..08e58e37 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -10,6 +10,7 @@ import pipes import re import subprocess import sys +from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter @@ -1063,9 +1064,8 @@ def exit_if(condition, message, exit_code): def call_docker(args): - try: - executable_path = subprocess.check_output(["which", "docker"]).strip() - except subprocess.CalledProcessError: + executable_path = find_executable('docker') + if not executable_path: raise UserError(errors.docker_not_found_msg("Couldn't find `docker` binary.")) args = [executable_path] + args diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py index 1d454a08..a7b57562 100644 --- a/tests/unit/cli/errors_test.py +++ b/tests/unit/cli/errors_test.py @@ -16,9 +16,9 @@ def mock_logging(): yield mock_log -def patch_call_silently(side_effect): +def patch_find_executable(side_effect): return mock.patch( - 'compose.cli.errors.call_silently', + 'compose.cli.errors.find_executable', autospec=True, side_effect=side_effect) @@ -27,7 +27,7 @@ class TestHandleConnectionErrors(object): def test_generic_connection_error(self, mock_logging): with pytest.raises(errors.ConnectionError): - with patch_call_silently([0, 1]): + with patch_find_executable(['/bin/docker', None]): with handle_connection_errors(mock.Mock()): raise ConnectionError() From 60d005b055119ce976265eaf34e4daa6483ead58 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Oct 2016 13:58:45 -0700 Subject: [PATCH 10/98] Improve robustness of a couple integration tests with occasional failures Signed-off-by: Joffrey F --- tests/integration/project_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index e3c34af9..3afefb5a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -826,9 +826,9 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data ) - project.up() + project.up(detached=True) - service_container = project.get_service('web').containers()[0] + service_container = project.get_service('web').containers(stopped=True)[0] ipam_config = service_container.inspect().get( 'NetworkSettings', {} ).get( @@ -857,8 +857,8 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data ) - project.up() - service_container = project.get_service('web').containers()[0] + project.up(detached=True) + service_container = project.get_service('web').containers(stopped=True)[0] assert service_container.inspect()['HostConfig']['Isolation'] == 'default' @v2_1_only() From 99343fd76cc632d30ff2132ea715728317e10250 Mon Sep 17 00:00:00 2001 From: Albin Kerouanton Date: Tue, 25 Oct 2016 11:06:39 +0200 Subject: [PATCH 11/98] Fix path of the parent dir of COMPOSE_FILE Signed-off-by: Albin Kerouanton --- script/run/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run/run.sh b/script/run/run.sh index 6205747a..5872b081 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -35,7 +35,7 @@ if [ "$(pwd)" != '/' ]; then VOLUMES="-v $(pwd):$(pwd)" fi if [ -n "$COMPOSE_FILE" ]; then - compose_dir=$(dirname $COMPOSE_FILE) + compose_dir=$(realpath $(dirname $COMPOSE_FILE)) fi # TODO: also check --file argument if [ -n "$compose_dir" ]; then From 4871523d5e113e1f16a4ce50533d9b6f0bb80237 Mon Sep 17 00:00:00 2001 From: Mike Dougherty Date: Fri, 22 Apr 2016 12:56:07 -0700 Subject: [PATCH 12/98] Update Jenkinsfile to perform existing jenkins tasks Signed-off-by: Mike Dougherty --- Jenkinsfile | 84 +++++++++++++++++++++++++++++++++++++++++++++---- script/test/all | 3 +- 2 files changed, 80 insertions(+), 7 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index fa29520b..5de9a3fb 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -1,8 +1,80 @@ -// Only run on Linux atm -wrappedNode(label: 'docker') { - deleteDir() - stage "checkout" - checkout scm +#!groovy - documentationChecker("docs") +def image + +def checkDocs = { -> + wrappedNode(label: 'linux') { + deleteDir(); checkout(scm) + documentationChecker("docs") + } } + +def buildImage = { -> + wrappedNode(label: "linux && !zfs") { + stage("build image") { + deleteDir(); checkout(scm) + def imageName = "dockerbuildbot/compose:${gitCommit()}" + image = docker.image(imageName) + try { + image.pull() + } catch (Exception exc) { + image = docker.build(imageName, ".") + image.push() + } + } + } +} + +def runTests = { Map settings -> + def dockerVersions = settings.get("dockerVersions", null) + def pythonVersions = settings.get("pythonVersions", null) + + if (!pythonVersions) { + throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py34')`") + } + if (!dockerVersions) { + throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`") + } + + { -> + wrappedNode(label: "linux && !zfs") { + stage("test python=${pythonVersions} / docker=${dockerVersions}") { + deleteDir(); checkout(scm) + def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() + echo "Using local system's storage driver: ${storageDriver}" + sh """docker run \\ + -t \\ + --rm \\ + --privileged \\ + --volume="\$(pwd)/.git:/code/.git" \\ + --volume="/var/run/docker.sock:/var/run/docker.sock" \\ + -e "TAG=${image.id}" \\ + -e "STORAGE_DRIVER=${storageDriver}" \\ + -e "DOCKER_VERSIONS=${dockerVersions}" \\ + -e "BUILD_NUMBER=\$BUILD_TAG" \\ + -e "PY_TEST_VERSIONS=${pythonVersions}" \\ + --entrypoint="script/ci" \\ + ${image.id} \\ + --verbose + """ + } + } + } +} + +def buildAndTest = { -> + buildImage() + // TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all + parallel( + failFast: true, + all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), + all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"), + ) +} + + +parallel( + failFast: false, + docs: checkDocs, + test: buildAndTest +) diff --git a/script/test/all b/script/test/all index 08bf1618..7151a75e 100755 --- a/script/test/all +++ b/script/test/all @@ -24,6 +24,7 @@ fi BUILD_NUMBER=${BUILD_NUMBER-$USER} +PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py34} for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" @@ -58,6 +59,6 @@ for version in $DOCKER_VERSIONS; do --env="DOCKER_VERSION=$version" \ --entrypoint="tox" \ "$TAG" \ - -e py27,py34 -- "$@" + -e "$PY_TEST_VERSIONS" -- "$@" done From 046144e8f40de571b0d9c0029329e20712ca4736 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 27 Oct 2016 12:13:32 -0700 Subject: [PATCH 13/98] Bump docker-py version to include latest patch Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 474efbdf..e72a88e7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 -docker-py==1.10.4 +docker-py==1.10.5 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 442bc80c..19ff5c6a 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.10.4, < 2.0', + 'docker-py >= 1.10.5, < 2.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From ba43d08fbdc1e7ec25978cc9068c3a9a7be91aab Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Nov 2016 11:09:29 -0700 Subject: [PATCH 14/98] Add whitelisted driver option added by the overlay driver to avoid breakage Signed-off-by: Joffrey F --- compose/network.py | 21 +++++++++++++++++ tests/unit/network_test.py | 47 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 tests/unit/network_test.py diff --git a/compose/network.py b/compose/network.py index e581a4fe..8e38401c 100644 --- a/compose/network.py +++ b/compose/network.py @@ -12,6 +12,10 @@ from .config import ConfigurationError log = logging.getLogger(__name__) +OPTS_EXCEPTIONS = [ + 'com.docker.network.driver.overlay.vxlanid_list', +] + class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, @@ -113,6 +117,23 @@ def create_ipam_config_from_dict(ipam_dict): ) +def check_remote_network_config(remote, local): + if local.driver and remote['Driver'] != local.driver: + raise ConfigurationError( + 'Network "{}" needs to be recreated - driver has changed' + .format(local.full_name) + ) + local_opts = local.driver_opts or {} + for k in set.union(set(remote['Options'].keys()), set(local_opts.keys())): + if k in OPTS_EXCEPTIONS: + continue + if remote['Options'].get(k) != local_opts.get(k): + raise ConfigurationError( + 'Network "{}" needs to be recreated - options have changed' + .format(local.full_name) + ) + + def build_networks(name, config_data, client): network_config = config_data.networks or {} networks = { diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py new file mode 100644 index 00000000..4720b053 --- /dev/null +++ b/tests/unit/network_test.py @@ -0,0 +1,47 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import pytest + +from .. import unittest +from compose.config import ConfigurationError +from compose.network import check_remote_network_config +from compose.network import Network + + +class NetworkTest(unittest.TestCase): + def test_check_remote_network_config_success(self): + options = {'com.docker.network.driver.foo': 'bar'} + net = Network( + None, 'compose_test', 'net1', 'bridge', + options + ) + check_remote_network_config( + {'Driver': 'bridge', 'Options': options}, net + ) + + def test_check_remote_network_config_whitelist(self): + options = {'com.docker.network.driver.foo': 'bar'} + remote_options = { + 'com.docker.network.driver.overlay.vxlanid_list': '257', + 'com.docker.network.driver.foo': 'bar' + } + net = Network( + None, 'compose_test', 'net1', 'overlay', + options + ) + check_remote_network_config( + {'Driver': 'overlay', 'Options': remote_options}, net + ) + + def test_check_remote_network_config_driver_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + with pytest.raises(ConfigurationError): + check_remote_network_config({'Driver': 'bridge', 'Options': {}}, net) + + def test_check_remote_network_config_options_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + with pytest.raises(ConfigurationError): + check_remote_network_config({'Driver': 'overlay', 'Options': { + 'com.docker.network.driver.foo': 'baz' + }}, net) From 7a430dbe96ad01831e21eab7eaebf26fdaa9249c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 2 Nov 2016 16:43:29 -0700 Subject: [PATCH 15/98] Updated docker-py dependency to latest version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index e72a88e7..933146c7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,7 +1,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 -docker-py==1.10.5 +docker-py==1.10.6 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 19ff5c6a..672ea80e 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.10.5, < 2.0', + 'docker-py >= 1.10.6, < 2.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From da1508051dcc7f106f891078b688bc31db7e43c5 Mon Sep 17 00:00:00 2001 From: Mike Dougherty Date: Thu, 3 Nov 2016 13:59:56 -0700 Subject: [PATCH 16/98] Remove docs checker from Jenkinsfile and use cleanWorkspace option on wrappedNode Signed-off-by: Mike Dougherty --- Jenkinsfile | 34 +++++++++------------------------- 1 file changed, 9 insertions(+), 25 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5de9a3fb..e2f86daa 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,17 +2,10 @@ def image -def checkDocs = { -> - wrappedNode(label: 'linux') { - deleteDir(); checkout(scm) - documentationChecker("docs") - } -} - def buildImage = { -> - wrappedNode(label: "linux && !zfs") { + wrappedNode(label: "linux && !zfs", cleanWorkspace: true) { stage("build image") { - deleteDir(); checkout(scm) + checkout(scm) def imageName = "dockerbuildbot/compose:${gitCommit()}" image = docker.image(imageName) try { @@ -37,9 +30,9 @@ def runTests = { Map settings -> } { -> - wrappedNode(label: "linux && !zfs") { + wrappedNode(label: "linux && !zfs", cleanWorkspace: true) { stage("test python=${pythonVersions} / docker=${dockerVersions}") { - deleteDir(); checkout(scm) + checkout(scm) def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() echo "Using local system's storage driver: ${storageDriver}" sh """docker run \\ @@ -62,19 +55,10 @@ def runTests = { Map settings -> } } -def buildAndTest = { -> - buildImage() - // TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all - parallel( - failFast: true, - all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), - all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"), - ) -} - - +buildImage() +// TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all parallel( - failFast: false, - docs: checkDocs, - test: buildAndTest + failFast: true, + all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), + all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"), ) From 10417eebd70f028f57e56ef9a04d8ed51abdad99 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 3 Nov 2016 16:28:44 -0700 Subject: [PATCH 17/98] Fix logging dict merging Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 45 ++++++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 437ed389..9d23b34d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -771,7 +771,7 @@ def merge_service_dicts(base, override, version): for field in ['dns', 'dns_search', 'env_file', 'tmpfs']: md.merge_field(field, merge_list_or_string) - md.merge_field('logging', merge_logging) + md.merge_field('logging', merge_logging, default={}) for field in set(ALLOWED_KEYS) - set(md): md.merge_scalar(field) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d205e282..66ae0147 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1657,6 +1657,51 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_logging_v2_no_base(self): + base = { + 'image': 'alpine:edge' + } + override = { + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000' + } + } + } + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000' + } + } + } + + def test_merge_logging_v2_no_override(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'frequency': '2000' + } + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'frequency': '2000' + } + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 4aa7d15d9771fadd22e8c1ff952526fdd6a67d77 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 4 Nov 2016 10:51:14 -0700 Subject: [PATCH 18/98] Call check_remote_network_config from Network.ensure Signed-off-by: Joffrey F --- compose/network.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/compose/network.py b/compose/network.py index 8e38401c..06935a46 100644 --- a/compose/network.py +++ b/compose/network.py @@ -53,14 +53,7 @@ class Network(object): try: data = self.inspect() - if self.driver and data['Driver'] != self.driver: - raise ConfigurationError( - 'Network "{}" needs to be recreated - driver has changed' - .format(self.full_name)) - if data['Options'] != (self.driver_opts or {}): - raise ConfigurationError( - 'Network "{}" needs to be recreated - options have changed' - .format(self.full_name)) + check_remote_network_config(data, self) except NotFound: driver_name = 'the default driver' if self.driver: From ba249e51796e6b35ed13e5f03716a9520a2dacfc Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 9 Nov 2016 15:10:02 +0000 Subject: [PATCH 19/98] Test that values in 'environment' override env files Signed-off-by: Aanand Prasad --- tests/unit/config/config_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 66ae0147..51c5e226 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2493,6 +2493,15 @@ class EnvTest(unittest.TestCase): {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'bar'}, ) + def test_environment_overrides_env_file(self): + self.assertEqual( + resolve_environment({ + 'environment': {'FOO': 'baz'}, + 'env_file': ['tests/fixtures/env/one.env'], + }), + {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'baz'}, + ) + def test_resolve_environment_with_multiple_env_files(self): service_dict = { 'env_file': [ From 91620ae97bbffd247e2f78132255c0fe97910c4f Mon Sep 17 00:00:00 2001 From: Jari Takkala Date: Fri, 29 Jul 2016 06:46:42 -0400 Subject: [PATCH 20/98] Add sysctl option support when creating service Closes #3765 Signed-off-by: Jari Takkala --- compose/config/config.py | 16 +++++++++++----- compose/config/config_schema_v2.1.json | 1 + compose/service.py | 2 ++ 3 files changed, 14 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 9d23b34d..57039e69 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -83,6 +83,7 @@ DOCKER_CONFIG_KEYS = [ 'shm_size', 'stdin_open', 'stop_signal', + 'sysctls', 'tty', 'user', 'volume_driver', @@ -629,6 +630,9 @@ def process_service(service_config): if 'extra_hosts' in service_dict: service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) + if 'sysctls' in service_dict: + service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) + for field in ['dns', 'dns_search', 'tmpfs']: if field in service_dict: service_dict[field] = to_list(service_dict[field]) @@ -757,6 +761,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('labels', parse_labels) md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) + md.merge_mapping('sysctls', parse_sysctls) md.merge_sequence('links', ServiceLink.parse) for field in ['volumes', 'devices']: @@ -831,11 +836,11 @@ def merge_environment(base, override): return env -def split_label(label): - if '=' in label: - return label.split('=', 1) +def split_kv(kvpair): + if '=' in kvpair: + return kvpair.split('=', 1) else: - return label, '' + return kvpair, '' def parse_dict_or_list(split_func, type_name, arguments): @@ -856,8 +861,9 @@ def parse_dict_or_list(split_func, type_name, arguments): parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments') parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment') -parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels') +parse_labels = functools.partial(parse_dict_or_list, split_kv, 'labels') parse_networks = functools.partial(parse_dict_or_list, lambda k: (k, None), 'networks') +parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') def parse_ulimits(ulimits): diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 3561a8cf..fc95f2cb 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -193,6 +193,7 @@ "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, diff --git a/compose/service.py b/compose/service.py index 760d29a7..c917aac4 100644 --- a/compose/service.py +++ b/compose/service.py @@ -62,6 +62,7 @@ DOCKER_START_KEYS = [ 'restart', 'security_opt', 'shm_size', + 'sysctls', 'volumes_from', ] @@ -707,6 +708,7 @@ class Service(object): cgroup_parent=options.get('cgroup_parent'), cpu_quota=options.get('cpu_quota'), shm_size=options.get('shm_size'), + sysctls=options.get('sysctls'), tmpfs=options.get('tmpfs'), oom_score_adj=options.get('oom_score_adj'), mem_swappiness=options.get('mem_swappiness'), From 0291d9ade5574ff31aecc1a9f263f1ec21a901cf Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 14 Nov 2016 17:23:25 -0800 Subject: [PATCH 21/98] Limit testing pool to Ubuntu hosts to avoid errors with dind not starting properly. Signed-off-by: Joffrey F --- Jenkinsfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index e2f86daa..51136b1f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -3,7 +3,7 @@ def image def buildImage = { -> - wrappedNode(label: "linux && !zfs", cleanWorkspace: true) { + wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { stage("build image") { checkout(scm) def imageName = "dockerbuildbot/compose:${gitCommit()}" @@ -30,7 +30,7 @@ def runTests = { Map settings -> } { -> - wrappedNode(label: "linux && !zfs", cleanWorkspace: true) { + wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { stage("test python=${pythonVersions} / docker=${dockerVersions}") { checkout(scm) def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() From efb4ed1b9e130ca5ac54f6e0fb23ce68c3689c1f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 14 Nov 2016 18:03:26 -0800 Subject: [PATCH 22/98] Handle new pull failures behavior in Engine 1.13 Signed-off-by: Joffrey F --- compose/service.py | 6 +++--- tests/acceptance/cli_test.py | 7 ++++--- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/compose/service.py b/compose/service.py index 760d29a7..ad426706 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ from operator import attrgetter import enum import six from docker.errors import APIError +from docker.errors import NotFound from docker.utils import LogConfig from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port @@ -829,12 +830,11 @@ class Service(object): repo, tag, separator = parse_repository_tag(self.options['image']) tag = tag or 'latest' log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) - output = self.client.pull(repo, tag=tag, stream=True) - try: + output = self.client.pull(repo, tag=tag, stream=True) return progress_stream.get_digest_from_pull( stream_output(output, sys.stdout)) - except StreamOutputError as e: + except (StreamOutputError, NotFound) as e: if not ignore_pull_failures: raise else: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a7cd78f1..f153bd95 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -330,12 +330,13 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_ignore_pull_failures(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', - 'pull', '--ignore-pull-failures']) + 'pull', '--ignore-pull-failures'] + ) assert 'Pulling simple (busybox:latest)...' in result.stderr assert 'Pulling another (nonexisting-image:latest)...' in result.stderr - assert 'Error: image library/nonexisting-image' in result.stderr - assert 'not found' in result.stderr + assert ('repository nonexisting-image not found' in result.stderr or + 'image library/nonexisting-image:latest not found' in result.stderr) def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' From 7f60ff5ae6b320dc0b38f8f2bcc850eca95f49ae Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Nov 2016 15:38:09 -0800 Subject: [PATCH 23/98] Avoid breaking when remote driver options are null. Signed-off-by: Joffrey F --- compose/network.py | 7 ++++--- tests/unit/network_test.py | 10 +++++++++- 2 files changed, 13 insertions(+), 4 deletions(-) diff --git a/compose/network.py b/compose/network.py index 06935a46..3b57cb94 100644 --- a/compose/network.py +++ b/compose/network.py @@ -111,16 +111,17 @@ def create_ipam_config_from_dict(ipam_dict): def check_remote_network_config(remote, local): - if local.driver and remote['Driver'] != local.driver: + if local.driver and remote.get('Driver') != local.driver: raise ConfigurationError( 'Network "{}" needs to be recreated - driver has changed' .format(local.full_name) ) local_opts = local.driver_opts or {} - for k in set.union(set(remote['Options'].keys()), set(local_opts.keys())): + remote_opts = remote.get('Options') or {} + for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): if k in OPTS_EXCEPTIONS: continue - if remote['Options'].get(k) != local_opts.get(k): + if remote_opts.get(k) != local_opts.get(k): raise ConfigurationError( 'Network "{}" needs to be recreated - options have changed' .format(local.full_name) diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py index 4720b053..12d06f41 100644 --- a/tests/unit/network_test.py +++ b/tests/unit/network_test.py @@ -37,7 +37,9 @@ class NetworkTest(unittest.TestCase): def test_check_remote_network_config_driver_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay') with pytest.raises(ConfigurationError): - check_remote_network_config({'Driver': 'bridge', 'Options': {}}, net) + check_remote_network_config( + {'Driver': 'bridge', 'Options': {}}, net + ) def test_check_remote_network_config_options_mismatch(self): net = Network(None, 'compose_test', 'net1', 'overlay') @@ -45,3 +47,9 @@ class NetworkTest(unittest.TestCase): check_remote_network_config({'Driver': 'overlay', 'Options': { 'com.docker.network.driver.foo': 'baz' }}, net) + + def test_check_remote_network_config_null_remote(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + check_remote_network_config( + {'Driver': 'overlay', 'Options': None}, net + ) From d717c88b6e81f5eb0769bc0670a6b78de842b2ce Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 9 Nov 2016 17:16:36 +0000 Subject: [PATCH 24/98] Support version 3.0 of the Compose file format Signed-off-by: Aanand Prasad --- compose/config/config.py | 13 +- compose/config/config_schema_v3.0.json | 378 ++++++++++++++++++++++ compose/config/errors.py | 4 +- compose/config/serialize.py | 3 +- compose/const.py | 3 + tests/acceptance/cli_test.py | 55 ++++ tests/fixtures/v3-full/docker-compose.yml | 37 +++ tests/unit/config/config_test.py | 6 + 8 files changed, 491 insertions(+), 8 deletions(-) create mode 100644 compose/config/config_schema_v3.0.json create mode 100644 tests/fixtures/v3-full/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 9d23b34d..fb77436d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -15,6 +15,7 @@ from cached_property import cached_property from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 from ..const import COMPOSEFILE_V2_1 as V2_1 +from ..const import COMPOSEFILE_V3_0 as V3_0 from ..utils import build_string_dict from ..utils import splitdrive from .environment import env_vars_from_file @@ -175,7 +176,10 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): if version == '2': version = V2_0 - if version not in (V2_0, V2_1): + if version == '3': + version = V3_0 + + if version not in (V2_0, V2_1, V3_0): raise ConfigurationError( 'Version in "{}" is unsupported. {}' .format(self.filename, VERSION_EXPLANATION)) @@ -433,7 +437,7 @@ def process_config_file(config_file, environment, service_name=None): 'service', environment) - if config_file.version in (V2_0, V2_1): + if config_file.version in (V2_0, V2_1, V3_0): processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( @@ -446,9 +450,10 @@ def process_config_file(config_file, environment, service_name=None): config_file.get_networks(), 'network', environment) - - if config_file.version == V1: + elif config_file.version == V1: processed_config = services + else: + raise Exception("Unsupported version: {}".format(repr(config_file.version))) config_file = config_file._replace(config=processed_config) validate_against_config_schema(config_file) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json new file mode 100644 index 00000000..9ac31b1f --- /dev/null +++ b/compose/config/config_schema_v3.0.json @@ -0,0 +1,378 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.0.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "stdin_open": {"type": "boolean"}, + "stop_signal": {"type": "string"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": ["object", "null"], + "properties": { + "interval": {"type":"string"}, + "timeout": {"type":"string"}, + "retries": {"type": "number"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + } + }, + "additionalProperties": false + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionaProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/errors.py b/compose/config/errors.py index d14cbbdd..16ed01b8 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -3,8 +3,8 @@ from __future__ import unicode_literals VERSION_EXPLANATION = ( - 'You might be seeing this error because you\'re using the wrong Compose ' - 'file version. Either specify a version of "2" (or "2.0") and place your ' + 'You might be seeing this error because you\'re using the wrong Compose file version. ' + 'Either specify a supported version ("2.0", "2.1", "3.0") and place your ' 'service definitions under the `services` key, or omit the `version` key ' 'and place your service definitions at the root of the file to use ' 'version 1.\nFor more on the Compose file format versions, see ' diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 95b1387f..768f3d47 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -6,7 +6,6 @@ import yaml from compose.config import types from compose.config.config import V1 -from compose.config.config import V2_0 from compose.config.config import V2_1 @@ -34,7 +33,7 @@ def denormalize_config(config): del net_conf['external_name'] version = config.version - if version not in (V2_0, V2_1): + if version == V1: version = V2_1 return { diff --git a/compose/const.py b/compose/const.py index e7b1ae97..3da39855 100644 --- a/compose/const.py +++ b/compose/const.py @@ -17,15 +17,18 @@ LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' +COMPOSEFILE_V3_0 = '3.0' API_VERSIONS = { COMPOSEFILE_V1: '1.21', COMPOSEFILE_V2_0: '1.22', COMPOSEFILE_V2_1: '1.24', + COMPOSEFILE_V3_0: '1.24', } API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V1]: '1.9.0', API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', + API_VERSIONS[COMPOSEFILE_V3_0]: '1.12.0', } diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f153bd95..e9a41691 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -285,6 +285,61 @@ class CLITestCase(DockerClientTestCase): 'volumes': {}, } + def test_config_v3(self): + self.base_dir = 'tests/fixtures/v3-full' + result = self.dispatch(['config']) + + assert yaml.load(result.stdout) == { + 'version': '3.0', + 'networks': {}, + 'volumes': {}, + 'services': { + 'web': { + 'image': 'busybox', + 'deploy': { + 'mode': 'replicated', + 'replicas': 6, + 'labels': ['FOO=BAR'], + 'update_config': { + 'parallelism': 3, + 'delay': '10s', + 'failure_action': 'continue', + 'monitor': '60s', + 'max_failure_ratio': 0.3, + }, + 'resources': { + 'limits': { + 'cpus': '0.001', + 'memory': '50M', + }, + 'reservations': { + 'cpus': '0.0001', + 'memory': '20M', + }, + }, + 'restart_policy': { + 'condition': 'on_failure', + 'delay': '5s', + 'max_attempts': 3, + 'window': '120s', + }, + 'placement': { + 'constraints': ['node=foo'], + }, + }, + + 'healthcheck': { + 'command': 'cat /etc/passwd', + 'interval': '10s', + 'timeout': '1s', + 'retries': 5, + }, + + 'stop_grace_period': '20s', + }, + }, + } + def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml new file mode 100644 index 00000000..1187dd7b --- /dev/null +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -0,0 +1,37 @@ +version: "3" +services: + web: + image: busybox + + deploy: + mode: replicated + replicas: 6 + labels: [FOO=BAR] + update_config: + parallelism: 3 + delay: 10s + failure_action: continue + monitor: 60s + max_failure_ratio: 0.3 + resources: + limits: + cpus: '0.001' + memory: 50M + reservations: + cpus: '0.0001' + memory: 20M + restart_policy: + condition: on_failure + delay: 5s + max_attempts: 3 + window: 120s + placement: + constraints: [node=foo] + + healthcheck: + command: cat /etc/passwd + interval: 10s + timeout: 1s + retries: 5 + + stop_grace_period: 20s diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 51c5e226..114145e1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -18,6 +18,7 @@ from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 +from compose.config.config import V3_0 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION @@ -156,9 +157,14 @@ class ConfigTest(unittest.TestCase): for version in ['2', '2.0']: cfg = config.load(build_config_details({'version': version})) assert cfg.version == V2_0 + cfg = config.load(build_config_details({'version': '2.1'})) assert cfg.version == V2_1 + for version in ['3', '3.0']: + cfg = config.load(build_config_details({'version': version})) + assert cfg.version == V3_0 + def test_v1_file_version(self): cfg = config.load(build_config_details({'web': {'image': 'busybox'}})) assert cfg.version == V1 From f75ef6862fe20f378c9702cda652deb34a49f946 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 10 Nov 2016 17:30:46 +0000 Subject: [PATCH 25/98] Warn if any services use 'deploy' Signed-off-by: Aanand Prasad --- compose/config/config.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index fb77436d..940b9eb0 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -330,6 +330,14 @@ def load(config_details): for service_dict in service_dicts: match_named_volumes(service_dict, volumes) + services_using_deploy = [s for s in service_dicts if s.get('deploy')] + if services_using_deploy: + log.warn( + "Some services ({}) use the 'deploy' key, which will be ignored. " + "Compose does not support deploy configuration - use the experimental " + "`docker deploy` command to deploy to a swarm." + .format(", ".join(sorted(s['name'] for s in services_using_deploy)))) + return Config(main_file.version, service_dicts, volumes, networks) From 6cac48c0564f7f6b4e4d91055f50ec8c86505dd9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 10 Nov 2016 17:38:19 -0500 Subject: [PATCH 26/98] Add a vendored and modified pytimeparse Signed-off-by: Daniel Nephin --- compose/timeparse.py | 96 ++++++++++++++++++++++++++++++++++++ tests/unit/timeparse_test.py | 52 +++++++++++++++++++ 2 files changed, 148 insertions(+) create mode 100644 compose/timeparse.py create mode 100644 tests/unit/timeparse_test.py diff --git a/compose/timeparse.py b/compose/timeparse.py new file mode 100644 index 00000000..16ef8a6d --- /dev/null +++ b/compose/timeparse.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +timeparse.py +(c) Will Roberts 1 February, 2014 + +This is a vendored and modified copy of: +github.com/wroberts/pytimeparse @ cc0550d + +It has been modified to mimic the behaviour of +https://golang.org/pkg/time/#ParseDuration +''' +# MIT LICENSE +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +from __future__ import absolute_import +from __future__ import unicode_literals + +import re + +HOURS = r'(?P[\d.]+)h' +MINS = r'(?P[\d.]+)m' +SECS = r'(?P[\d.]+)s' +MILLI = r'(?P[\d.]+)ms' +MICRO = r'(?P[\d.]+)(?:us|µs)' +NANO = r'(?P[\d.]+)ns' + + +def opt(x): + return r'(?:{x})?'.format(x=x) + + +TIMEFORMAT = r'{HOURS}{MINS}{SECS}{MILLI}{MICRO}{NANO}'.format( + HOURS=opt(HOURS), + MINS=opt(MINS), + SECS=opt(SECS), + MILLI=opt(MILLI), + MICRO=opt(MICRO), + NANO=opt(NANO), +) + +MULTIPLIERS = dict([ + ('hours', 60 * 60), + ('mins', 60), + ('secs', 1), + ('milli', 1.0 / 1000), + ('micro', 1.0 / 1000.0 / 1000), + ('nano', 1.0 / 1000.0 / 1000.0 / 1000.0), +]) + + +def timeparse(sval): + """Parse a time expression, returning it as a number of seconds. If + possible, the return value will be an `int`; if this is not + possible, the return will be a `float`. Returns `None` if a time + expression cannot be parsed from the given string. + + Arguments: + - `sval`: the string value to parse + + >>> timeparse('1m24s') + 84 + >>> timeparse('1.2 minutes') + 72 + >>> timeparse('1.2 seconds') + 1.2 + """ + match = re.match(r'\s*' + TIMEFORMAT + r'\s*$', sval, re.I) + if not match or not match.group(0).strip(): + return + + mdict = match.groupdict() + return sum( + MULTIPLIERS[k] * cast(v) for (k, v) in mdict.items() if v is not None) + + +def cast(value): + return int(value, 10) if value.isdigit() else float(value) diff --git a/tests/unit/timeparse_test.py b/tests/unit/timeparse_test.py new file mode 100644 index 00000000..e9fe6c24 --- /dev/null +++ b/tests/unit/timeparse_test.py @@ -0,0 +1,52 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from compose import timeparse + + +def test_milli(): + assert timeparse.timeparse('5ms') == 0.005 + + +def test_milli_float(): + assert timeparse.timeparse('50.5ms') == 0.0505 + + +def test_second_milli(): + assert timeparse.timeparse('200s5ms') == 200.005 + + +def test_second_milli_micro(): + assert timeparse.timeparse('200s5ms10us') == 200.00501 + + +def test_second(): + assert timeparse.timeparse('200s') == 200 + + +def test_second_as_float(): + assert timeparse.timeparse('20.5s') == 20.5 + + +def test_minute(): + assert timeparse.timeparse('32m') == 1920 + + +def test_hour_minute(): + assert timeparse.timeparse('2h32m') == 9120 + + +def test_minute_as_float(): + assert timeparse.timeparse('1.5m') == 90 + + +def test_hour_minute_second(): + assert timeparse.timeparse('5h34m56s') == 20096 + + +def test_invalid_with_space(): + assert timeparse.timeparse('5h 34m 56s') is None + + +def test_invalid_with_comma(): + assert timeparse.timeparse('5h,34m,56s') is None From 079c95c3401adb0f837e6b4e54132cdd41eada68 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 10 Nov 2016 17:47:05 -0500 Subject: [PATCH 27/98] Use stop grace period for container stop. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 14 +++++++++----- compose/parallel.py | 4 ---- compose/project.py | 20 ++++++++++++++++---- compose/service.py | 23 ++++++++++++++++------- tests/unit/timeparse_test.py | 4 ++++ 5 files changed, 45 insertions(+), 20 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 08e58e37..cf53f6aa 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -24,7 +24,6 @@ from ..config import ConfigurationError from ..config import parse_environment from ..config.environment import Environment from ..config.serialize import serialize_config -from ..const import DEFAULT_TIMEOUT from ..const import IS_WINDOWS_PLATFORM from ..errors import StreamParseError from ..progress_stream import StreamOutputError @@ -726,7 +725,7 @@ class TopLevelCommand(object): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = timeout_from_opts(options) for s in options['SERVICE=NUM']: if '=' not in s: @@ -760,7 +759,7 @@ class TopLevelCommand(object): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = timeout_from_opts(options) self.project.stop(service_names=options['SERVICE'], timeout=timeout) def restart(self, options): @@ -773,7 +772,7 @@ class TopLevelCommand(object): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = timeout_from_opts(options) containers = self.project.restart(service_names=options['SERVICE'], timeout=timeout) exit_if(not containers, 'No containers to restart', 1) @@ -831,7 +830,7 @@ class TopLevelCommand(object): start_deps = not options['--no-deps'] cascade_stop = options['--abort-on-container-exit'] service_names = options['SERVICE'] - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = timeout_from_opts(options) remove_orphans = options['--remove-orphans'] detached = options.get('-d') @@ -896,6 +895,11 @@ def convergence_strategy_from_opts(options): return ConvergenceStrategy.changed +def timeout_from_opts(options): + timeout = options.get('--timeout') + return None if timeout is None else int(timeout) + + def image_type_from_opt(flag, value): if not value: return ImageType.none diff --git a/compose/parallel.py b/compose/parallel.py index 7ac66b37..26718872 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -248,7 +248,3 @@ def parallel_unpause(containers, options): def parallel_kill(containers, options): parallel_operation(containers, 'kill', options, 'Killing') - - -def parallel_restart(containers, options): - parallel_operation(containers, 'restart', options, 'Restarting') diff --git a/compose/project.py b/compose/project.py index 60647fe9..eef2f3b8 100644 --- a/compose/project.py +++ b/compose/project.py @@ -14,7 +14,6 @@ from .config import ConfigurationError from .config.config import V1 from .config.sort_services import get_container_name_from_network_mode from .config.sort_services import get_service_name_from_network_mode -from .const import DEFAULT_TIMEOUT from .const import IMAGE_EVENTS from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT @@ -250,7 +249,7 @@ class Project(object): parallel.parallel_execute( containers, - operator.methodcaller('stop', **options), + self.build_container_operation_with_timeout_func('stop', options), operator.attrgetter('name'), 'Stopping', get_deps) @@ -291,7 +290,12 @@ class Project(object): def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) - parallel.parallel_restart(containers, options) + + parallel.parallel_execute( + containers, + self.build_container_operation_with_timeout_func('restart', options), + operator.attrgetter('name'), + 'Restarting') return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): @@ -365,7 +369,7 @@ class Project(object): start_deps=True, strategy=ConvergenceStrategy.changed, do_build=BuildAction.none, - timeout=DEFAULT_TIMEOUT, + timeout=None, detached=False, remove_orphans=False): @@ -506,6 +510,14 @@ class Project(object): dep_services.append(service) return acc + dep_services + def build_container_operation_with_timeout_func(self, operation, options): + def container_operation_with_timeout(container): + if options.get('timeout') is None: + service = self.get_service(container.service) + options['timeout'] = service.stop_timeout(None) + return getattr(container, operation)(**options) + return container_operation_with_timeout + def get_volumes_from(project, service_dict): volumes_from = service_dict.pop('volumes_from', None) diff --git a/compose/service.py b/compose/service.py index ad426706..39737694 100644 --- a/compose/service.py +++ b/compose/service.py @@ -17,6 +17,7 @@ from docker.utils.ports import split_port from . import __version__ from . import progress_stream +from . import timeparse from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.types import VolumeSpec @@ -169,7 +170,7 @@ class Service(object): self.start_container_if_stopped(c, **options) return containers - def scale(self, desired_num, timeout=DEFAULT_TIMEOUT): + def scale(self, desired_num, timeout=None): """ Adjusts the number of containers to the specified number and ensures they are running. @@ -196,7 +197,7 @@ class Service(object): return container def stop_and_remove(container): - container.stop(timeout=timeout) + container.stop(timeout=self.stop_timeout(timeout)) container.remove() running_containers = self.containers(stopped=False) @@ -374,7 +375,7 @@ class Service(object): def execute_convergence_plan(self, plan, - timeout=DEFAULT_TIMEOUT, + timeout=None, detached=False, start=True): (action, containers) = plan @@ -421,7 +422,7 @@ class Service(object): def recreate_container( self, container, - timeout=DEFAULT_TIMEOUT, + timeout=None, attach_logs=False, start_new_container=True): """Recreate a container. @@ -432,7 +433,7 @@ class Service(object): """ log.info("Recreating %s" % container.name) - container.stop(timeout=timeout) + container.stop(timeout=self.stop_timeout(timeout)) container.rename_to_tmp_name() new_container = self.create_container( previous_container=container, @@ -446,6 +447,14 @@ class Service(object): container.remove() return new_container + def stop_timeout(self, timeout): + if timeout is not None: + return timeout + timeout = timeparse.timeparse(self.options.get('stop_grace_period') or '') + if timeout is not None: + return timeout + return DEFAULT_TIMEOUT + def start_container_if_stopped(self, container, attach_logs=False, quiet=False): if not container.is_running: if not quiet: @@ -483,10 +492,10 @@ class Service(object): link_local_ips=netdefs.get('link_local_ips', None), ) - def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): + def remove_duplicate_containers(self, timeout=None): for c in self.duplicate_containers(): log.info('Removing %s' % c.name) - c.stop(timeout=timeout) + c.stop(timeout=self.stop_timeout(timeout)) c.remove() def duplicate_containers(self): diff --git a/tests/unit/timeparse_test.py b/tests/unit/timeparse_test.py index e9fe6c24..9915932c 100644 --- a/tests/unit/timeparse_test.py +++ b/tests/unit/timeparse_test.py @@ -50,3 +50,7 @@ def test_invalid_with_space(): def test_invalid_with_comma(): assert timeparse.timeparse('5h,34m,56s') is None + + +def test_invalid_with_empty_string(): + assert timeparse.timeparse('') is None From b93211881b913091501a76d062f033754a653740 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 16 Nov 2016 13:35:29 -0800 Subject: [PATCH 28/98] Changelog update Signed-off-by: Joffrey F --- CHANGELOG.md | 100 +++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8ec7d5b5..17487890 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,106 @@ Change log ========== +1.9.0 (2016-11-16) +----------------- + +**Breaking changes** + +- When using Compose with Docker Toolbox/Machine on Windows, volume paths are + no longer converted from `C:\Users` to `/c/Users`-style by default. To + re-enable this conversion so that your volumes keep working, set the + environment variable `COMPOSE_CONVERT_WINDOWS_PATHS=1`. Users of + Docker for Windows are not affected and do not need to set the variable. + +New Features + +- Interactive mode for `docker-compose run` and `docker-compose exec` is + now supported on Windows platforms. Please note that the `docker` binary + is required to be present on the system for this feature to work. + +- Introduced version 2.1 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.12 or above. + - Added support for setting volume labels and network labels in + `docker-compose.yml`. + - Added support for the `isolation` parameter in service definitions. + - Added support for link-local IPs in the service networks definitions. + - Added support for shell-style inline defaults in variable interpolation. + The supported forms are `${FOO-default}` (fall back if FOO is unset) and + `${FOO:-default}` (fall back if FOO is unset or empty). + +- Added support for the `group_add` and `oom_score_adj` parameters in + service definitions. + +- Added support for the `internal` and `enable_ipv6` parameters in network + definitions. + +- Compose now defaults to using the `npipe` protocol on Windows. + +- Overriding a `logging` configuration will now properly merge the `options` + mappings if the `driver` values do not conflict. + +Bug Fixes + +- Fixed several bugs related to `npipe` protocol support on Windows. + +- Fixed an issue with Windows paths being incorrectly converted when + using Docker on Windows Server. + +- Fixed a bug where an empty `restart` value would sometimes result in an + exception being raised. + +- Fixed an issue where service logs containing unicode characters would + sometimes cause an error to occur. + +- Fixed a bug where unicode values in environment variables would sometimes + raise a unicode exception when retrieved. + +- Fixed an issue where Compose would incorrectly detect a configuration + mismatch for overlay networks. + + +1.8.1 (2016-09-22) +----------------- + +Bug Fixes + +- Fixed a bug where users using a credentials store were not able + to access their private images. + +- Fixed a bug where users using identity tokens to authenticate + were not able to access their private images. + +- Fixed a bug where an `HttpHeaders` entry in the docker configuration + file would cause Compose to crash when trying to build an image. + +- Fixed a few bugs related to the handling of Windows paths in volume + binding declarations. + +- Fixed a bug where Compose would sometimes crash while trying to + read a streaming response from the engine. + +- Fixed an issue where Compose would crash when encountering an API error + while streaming container logs. + +- Fixed an issue where Compose would erroneously try to output logs from + drivers not handled by the Engine's API. + +- Fixed a bug where options from the `docker-machine config` command would + not be properly interpreted by Compose. + +- Fixed a bug where the connection to the Docker Engine would + sometimes fail when running a large number of services simultaneously. + +- Fixed an issue where Compose would sometimes print a misleading + suggestion message when running the `bundle` command. + +- Fixed a bug where connection errors would not be handled properly by + Compose during the project initialization phase. + +- Fixed a bug where a misleading error would appear when encountering + a connection timeout. + + 1.8.0 (2016-06-14) ----------------- From 716a6baa59c62266af0bb7628ec88586f470140c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 14 Nov 2016 18:31:38 +0000 Subject: [PATCH 29/98] Implement 'healthcheck' option Signed-off-by: Aanand Prasad --- compose/config/config.py | 29 +++++++++++ compose/config/config_schema_v3.0.json | 5 +- compose/service.py | 4 +- compose/utils.py | 16 ++++++ requirements.txt | 2 +- tests/acceptance/cli_test.py | 50 +++++++++++++++++-- tests/fixtures/healthcheck/docker-compose.yml | 24 +++++++++ tests/fixtures/v3-full/docker-compose.yml | 2 +- tests/unit/config/config_test.py | 49 ++++++++++++++++++ 9 files changed, 172 insertions(+), 9 deletions(-) create mode 100644 tests/fixtures/healthcheck/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 940b9eb0..e5a37e84 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -17,6 +17,7 @@ from ..const import COMPOSEFILE_V2_0 as V2_0 from ..const import COMPOSEFILE_V2_1 as V2_1 from ..const import COMPOSEFILE_V3_0 as V3_0 from ..utils import build_string_dict +from ..utils import parse_nanoseconds_int from ..utils import splitdrive from .environment import env_vars_from_file from .environment import Environment @@ -65,6 +66,7 @@ DOCKER_CONFIG_KEYS = [ 'extra_hosts', 'group_add', 'hostname', + 'healthcheck', 'image', 'ipc', 'labels', @@ -642,6 +644,10 @@ def process_service(service_config): if 'extra_hosts' in service_dict: service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) + if 'healthcheck' in service_dict: + service_dict['healthcheck'] = process_healthcheck( + service_dict['healthcheck'], service_config.name) + for field in ['dns', 'dns_search', 'tmpfs']: if field in service_dict: service_dict[field] = to_list(service_dict[field]) @@ -649,6 +655,29 @@ def process_service(service_config): return service_dict +def process_healthcheck(raw, service_name): + hc = {} + + if raw.get('disable'): + if len(raw) > 1: + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + '"disable: true" cannot be combined with other options' + .format(service_name)) + hc['test'] = ['NONE'] + elif 'test' in raw: + hc['test'] = raw['test'] + + if 'interval' in raw: + hc['interval'] = parse_nanoseconds_int(raw['interval']) + if 'timeout' in raw: + hc['timeout'] = parse_nanoseconds_int(raw['timeout']) + if 'retries' in raw: + hc['retries'] = raw['retries'] + + return hc + + def finalize_service(service_config, service_names, version, environment): service_dict = dict(service_config.config) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 9ac31b1f..4edd8dd4 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -205,12 +205,13 @@ "interval": {"type":"string"}, "timeout": {"type":"string"}, "retries": {"type": "number"}, - "command": { + "test": { "oneOf": [ {"type": "string"}, {"type": "array", "items": {"type": "string"}} ] - } + }, + "disable": {"type": "boolean"} }, "additionalProperties": false }, diff --git a/compose/service.py b/compose/service.py index 39737694..cf52d489 100644 --- a/compose/service.py +++ b/compose/service.py @@ -17,7 +17,6 @@ from docker.utils.ports import split_port from . import __version__ from . import progress_stream -from . import timeparse from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.types import VolumeSpec @@ -35,6 +34,7 @@ from .parallel import parallel_start from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash +from .utils import parse_seconds_float log = logging.getLogger(__name__) @@ -450,7 +450,7 @@ class Service(object): def stop_timeout(self, timeout): if timeout is not None: return timeout - timeout = timeparse.timeparse(self.options.get('stop_grace_period') or '') + timeout = parse_seconds_float(self.options.get('stop_grace_period')) if timeout is not None: return timeout return DEFAULT_TIMEOUT diff --git a/compose/utils.py b/compose/utils.py index 8f05e308..b8bdf732 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -11,6 +11,7 @@ import ntpath import six from .errors import StreamParseError +from .timeparse import timeparse json_decoder = json.JSONDecoder() @@ -107,6 +108,21 @@ def microseconds_from_time_nano(time_nano): return int(time_nano % 1000000000 / 1000) +def nanoseconds_from_time_seconds(time_seconds): + return time_seconds * 1000000000 + + +def parse_seconds_float(value): + return timeparse(value or '') + + +def parse_nanoseconds_int(value): + parsed = timeparse(value or '') + if parsed is None: + return None + return int(parsed * 1000000000) + + def build_string_dict(source_dict): return dict((k, str(v if v is not None else '')) for k, v in source_dict.items()) diff --git a/requirements.txt b/requirements.txt index 933146c7..63469799 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 -docker-py==1.10.6 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' +git+https://github.com/docker/docker-py.git@2ff7371ae7703033f981e1b137a3be0caf7a4f9c#egg=docker-py ipaddress==1.0.16 jsonschema==2.5.1 pypiwin32==219; sys_platform == 'win32' diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e9a41691..97518f4f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -21,6 +21,7 @@ from .. import mock from compose.cli.command import get_project from compose.container import Container from compose.project import OneOffFilter +from compose.utils import nanoseconds_from_time_seconds from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox @@ -329,9 +330,9 @@ class CLITestCase(DockerClientTestCase): }, 'healthcheck': { - 'command': 'cat /etc/passwd', - 'interval': '10s', - 'timeout': '1s', + 'test': 'cat /etc/passwd', + 'interval': 10000000000, + 'timeout': 1000000000, 'retries': 5, }, @@ -925,6 +926,49 @@ class CLITestCase(DockerClientTestCase): assert foo_container.get('HostConfig.NetworkMode') == \ 'container:{}'.format(bar_container.id) + def test_up_with_healthcheck(self): + def wait_on_health_status(container, status): + def condition(): + container.inspect() + return container.get('State.Health.Status') == status + + return wait_on_condition(condition, delay=0.5) + + self.base_dir = 'tests/fixtures/healthcheck' + self.dispatch(['up', '-d'], None) + + passes = self.project.get_service('passes') + passes_container = passes.containers()[0] + + assert passes_container.get('Config.Healthcheck') == { + "Test": ["CMD-SHELL", "/bin/true"], + "Interval": nanoseconds_from_time_seconds(1), + "Timeout": nanoseconds_from_time_seconds(30*60), + "Retries": 1, + } + + wait_on_health_status(passes_container, 'healthy') + + fails = self.project.get_service('fails') + fails_container = fails.containers()[0] + + assert fails_container.get('Config.Healthcheck') == { + "Test": ["CMD", "/bin/false"], + "Interval": nanoseconds_from_time_seconds(2.5), + "Retries": 2, + } + + wait_on_health_status(fails_container, 'unhealthy') + + disabled = self.project.get_service('disabled') + disabled_container = disabled.containers()[0] + + assert disabled_container.get('Config.Healthcheck') == { + "Test": ["NONE"], + } + + assert 'Health' not in disabled_container.get('State') + def test_up_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', '--no-deps', 'web'], None) diff --git a/tests/fixtures/healthcheck/docker-compose.yml b/tests/fixtures/healthcheck/docker-compose.yml new file mode 100644 index 00000000..2c45b8d8 --- /dev/null +++ b/tests/fixtures/healthcheck/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3" +services: + passes: + image: busybox + command: top + healthcheck: + test: "/bin/true" + interval: 1s + timeout: 30m + retries: 1 + + fails: + image: busybox + command: top + healthcheck: + test: ["CMD", "/bin/false"] + interval: 2.5s + retries: 2 + + disabled: + image: busybox + command: top + healthcheck: + disable: true diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index 1187dd7b..b4d1b642 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -29,7 +29,7 @@ services: constraints: [node=foo] healthcheck: - command: cat /etc/passwd + test: cat /etc/passwd interval: 10s timeout: 1s retries: 5 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 114145e1..f7df3aee 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -24,6 +24,7 @@ from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM +from compose.utils import nanoseconds_from_time_seconds from tests import mock from tests import unittest @@ -3171,6 +3172,54 @@ class BuildPathTest(unittest.TestCase): assert 'build path' in exc.exconly() +class HealthcheckTest(unittest.TestCase): + def test_healthcheck(self): + service_dict = make_service_dict( + 'test', + {'healthcheck': { + 'test': ['CMD', 'true'], + 'interval': '1s', + 'timeout': '1m', + 'retries': 3, + }}, + '.', + ) + + assert service_dict['healthcheck'] == { + 'test': ['CMD', 'true'], + 'interval': nanoseconds_from_time_seconds(1), + 'timeout': nanoseconds_from_time_seconds(60), + 'retries': 3, + } + + def test_disable(self): + service_dict = make_service_dict( + 'test', + {'healthcheck': { + 'disable': True, + }}, + '.', + ) + + assert service_dict['healthcheck'] == { + 'test': ['NONE'], + } + + def test_disable_with_other_config_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + make_service_dict( + 'invalid-healthcheck', + {'healthcheck': { + 'disable': True, + 'interval': '1s', + }}, + '.', + ) + + assert 'invalid-healthcheck' in excinfo.exconly() + assert 'disable' in excinfo.exconly() + + class GetDefaultConfigFilesTestCase(unittest.TestCase): files = [ From c26a2afaf36443fbc4ea813e34027bdb05ded0e1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 21 Nov 2016 16:34:49 -0500 Subject: [PATCH 30/98] Update messages about docker stack deploy. Signed-off-by: Daniel Nephin --- compose/config/config.py | 4 ++-- compose/project.py | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 940b9eb0..5215b361 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -334,8 +334,8 @@ def load(config_details): if services_using_deploy: log.warn( "Some services ({}) use the 'deploy' key, which will be ignored. " - "Compose does not support deploy configuration - use the experimental " - "`docker deploy` command to deploy to a swarm." + "Compose does not support deploy configuration - use " + "`docker stack deploy` to deploy to a swarm." .format(", ".join(sorted(s['name'] for s in services_using_deploy)))) return Config(main_file.version, service_dicts, volumes, networks) diff --git a/compose/project.py b/compose/project.py index eef2f3b8..0178bbab 100644 --- a/compose/project.py +++ b/compose/project.py @@ -559,9 +559,7 @@ def warn_for_swarm_mode(client): "Compose does not use swarm mode to deploy services to multiple nodes in a swarm. " "All containers will be scheduled on the current node.\n\n" "To deploy your application across the swarm, " - "use the bundle feature of the Docker experimental build.\n\n" - "More info:\n" - "https://docs.docker.com/compose/bundles\n" + "use `docker stack deploy`.\n" ) From 024b5dd6da4c96822ea84e31cedcb88a8949a245 Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 22 Nov 2016 11:14:56 +0000 Subject: [PATCH 31/98] case PyPI correctly Signed-off-by: Thomas Grainger --- CHANGELOG.md | 2 +- script/release/push-release | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 17487890..9780df98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -814,7 +814,7 @@ Fig has been renamed to Docker Compose, or just Compose for short. This has seve - The command you type is now `docker-compose`, not `fig`. - You should rename your fig.yml to docker-compose.yml. -- If you’re installing via PyPi, the package is now `docker-compose`, so install it with `pip install docker-compose`. +- If you’re installing via PyPI, the package is now `docker-compose`, so install it with `pip install docker-compose`. Besides that, there’s a lot of new stuff in this release: diff --git a/script/release/push-release b/script/release/push-release index 33d0d777..d5ae3de9 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -54,7 +54,7 @@ git push $GITHUB_REPO $VERSION echo "Uploading the docker image" docker push docker/compose:$VERSION -echo "Uploading sdist to pypi" +echo "Uploading sdist to PyPI" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst ./script/build/write-git-sha From dc9184a90fd22b13265162c4bc8e04c9867a768d Mon Sep 17 00:00:00 2001 From: Ian Campbell Date: Fri, 25 Nov 2016 10:12:22 +0000 Subject: [PATCH 32/98] progress_stream: Avoid undefined ANSI escape codes The ANSI escape codes \e[0A (cursor up 0 lines) and \e[0B (cursor down 0 lines) are not well defined and are treated differently by different terminals. In particular xterm treats 0 as a missing parameter and therefore defaults to 1, whereas rxvt-unicode treats these escapes as a request to move 0 lines. However the use of these codes is unnecessary and were really just hiding the fact that we were not correctly computing diff when adding a new line. Having added the new line to the ids map and output the corresponding \n the correct diff would be 1 and not 0 (which xterm interprets as 1) as currently. Rather than changing the hardcoded 0 to a 1 pull the diff calculation out and always do it since it produces the correct answer in both cases. This fixes similar corruption when compose is pulling an image to that seen with `docker pull` and rxvt-unicode (and likely other terminals in that family) seen in docker/docker#28111. This is the same as the fix made to Docker's pkg/jsonmessage in https://github.com/docker/docker/pull/28238 (and I have shamelessly ripped off most of this commit message from there). Signed-off-by: Ian Campbell --- compose/progress_stream.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index a0f5601f..5314f89f 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -32,12 +32,11 @@ def stream_output(output, stream): if not image_id: continue - if image_id in lines: - diff = len(lines) - lines[image_id] - else: + if image_id not in lines: lines[image_id] = len(lines) stream.write("\n") - diff = 0 + + diff = len(lines) - lines[image_id] # move cursor up `diff` rows stream.write("%c[%dA" % (27, diff)) From 6aacf5142750654a9b1f165223478fab3a424eeb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 2 Dec 2016 16:10:15 -0800 Subject: [PATCH 33/98] Win32 interactive run - Connect container to networks before starting Signed-off-by: Joffrey F --- compose/cli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index cf53f6aa..c25ccbfa 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -988,6 +988,7 @@ def run_one_off_container(container_options, project, service, options): try: try: if IS_WINDOWS_PLATFORM: + service.connect_container_to_networks(container) exit_code = call_docker(["start", "--attach", "--interactive", container.id]) else: operation = RunOperation( From 62f8b1402e1bbe726cf83ea204077754385fe53a Mon Sep 17 00:00:00 2001 From: Yong Wen Chua Date: Mon, 5 Dec 2016 14:25:56 +0800 Subject: [PATCH 34/98] Implement `userns_mode` HostConfig for services Fixes #3349 This allows the key `userns_mode` to be used in service definitions. Since `userns_mode` requires API version > 1.23, this is only available in 2.1 and 3.0 versions of compose file Signed-off-by: Yong Wen Chua --- compose/config/config.py | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v3.0.json | 1 + compose/service.py | 4 +++- tests/integration/service_test.py | 13 +++++++++++++ 5 files changed, 19 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5215b361..b83b12bf 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -86,6 +86,7 @@ DOCKER_CONFIG_KEYS = [ 'stop_signal', 'tty', 'user', + 'userns_mode', 'volume_driver', 'volumes', 'volumes_from', diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 3561a8cf..b5d614d6 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -217,6 +217,7 @@ } }, "user": {"type": "string"}, + "userns_mode": {"type": "string"}, "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "volume_driver": {"type": "string"}, "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 9ac31b1f..297c60be 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -192,6 +192,7 @@ } }, "user": {"type": "string"}, + "userns_mode": {"type": "string"}, "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "working_dir": {"type": "string"} }, diff --git a/compose/service.py b/compose/service.py index 39737694..6a62067b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -64,6 +64,7 @@ DOCKER_START_KEYS = [ 'restart', 'security_opt', 'shm_size', + 'userns_mode', 'volumes_from', ] @@ -720,7 +721,8 @@ class Service(object): tmpfs=options.get('tmpfs'), oom_score_adj=options.get('oom_score_adj'), mem_swappiness=options.get('mem_swappiness'), - group_add=options.get('group_add') + group_add=options.get('group_add'), + userns_mode=options.get('userns_mode') ) # TODO: Add as an argument to create_host_config once it's supported diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index a5ca81ee..09758eee 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -30,6 +30,7 @@ from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy from compose.service import NetworkMode from compose.service import Service +from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only @@ -842,6 +843,18 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.PidMode'), 'host') + @v2_1_only() + def test_userns_mode_none_defined(self): + service = self.create_service('web', userns_mode=None) + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.UsernsMode'), '') + + @v2_1_only() + def test_userns_mode_host(self): + service = self.create_service('web', userns_mode='host') + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.UsernsMode'), 'host') + def test_dns_no_value(self): service = self.create_service('web') container = create_and_start_container(service) From 4d0575355c1a0201ed7fc11063a4efbca6971a96 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Dec 2016 17:14:05 -0800 Subject: [PATCH 35/98] Bump master version to 1.10.0dev Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/__init__.py b/compose/__init__.py index 6e610652..6f05b282 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.9.0dev' +__version__ = '1.10.0dev' From e04a12b5ca9fb30d152ab6a83ae8c0ec15ac85b0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 6 Dec 2016 17:01:35 -0500 Subject: [PATCH 36/98] Increase minimum version for v3. Signed-off-by: Daniel Nephin --- compose/const.py | 4 ++-- tests/acceptance/cli_test.py | 2 ++ tests/integration/testcases.py | 23 +++++++++++------------ 3 files changed, 15 insertions(+), 14 deletions(-) diff --git a/compose/const.py b/compose/const.py index 3da39855..ca8d7fe5 100644 --- a/compose/const.py +++ b/compose/const.py @@ -23,12 +23,12 @@ API_VERSIONS = { COMPOSEFILE_V1: '1.21', COMPOSEFILE_V2_0: '1.22', COMPOSEFILE_V2_1: '1.24', - COMPOSEFILE_V3_0: '1.24', + COMPOSEFILE_V3_0: '1.25', } API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V1]: '1.9.0', API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', - API_VERSIONS[COMPOSEFILE_V3_0]: '1.12.0', + API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', } diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e9a41691..0d5d5058 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -26,6 +26,7 @@ from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only +from tests.integration.testcases import v3_only ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -285,6 +286,7 @@ class CLITestCase(DockerClientTestCase): 'volumes': {}, } + @v3_only() def test_config_v3(self): self.base_dir = 'tests/fixtures/v3-full' result = self.dispatch(['config']) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index c7743fb8..f6bc402b 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -45,11 +45,11 @@ def engine_max_version(): return V2_1 -def v2_only(): +def build_version_required_decorator(ignored_versions): def decorator(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): - if engine_max_version() == V1: + if engine_max_version() in ignored_versions: skip("Engine version is too low") return return f(self, *args, **kwargs) @@ -58,17 +58,16 @@ def v2_only(): return decorator -def v2_1_only(): - def decorator(f): - @functools.wraps(f) - def wrapper(self, *args, **kwargs): - if engine_max_version() in (V1, V2_0): - skip('Engine version is too low') - return - return f(self, *args, **kwargs) - return wrapper +def v2_only(): + return build_version_required_decorator((V1,)) - return decorator + +def v2_1_only(): + return build_version_required_decorator((V1, V2_0)) + + +def v3_only(): + return build_version_required_decorator((V1, V2_0, V2_1)) class DockerClientTestCase(unittest.TestCase): From 04e5925a230d07a6ffd855b4c3812f5a6b54a523 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Dec 2016 17:42:07 -0800 Subject: [PATCH 37/98] Use docker SDK 2.0 Signed-off-by: Joffrey F --- compose/cli/docker_client.py | 4 ++-- compose/network.py | 8 ++++---- compose/service.py | 2 +- requirements.txt | 2 +- setup.py | 2 +- tests/unit/bundle_test.py | 2 +- tests/unit/cli_test.py | 4 ++-- tests/unit/container_test.py | 2 +- tests/unit/project_test.py | 2 +- tests/unit/service_test.py | 10 +++++----- tests/unit/volume_test.py | 2 +- 11 files changed, 20 insertions(+), 20 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index b196d303..018d2451 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -3,7 +3,7 @@ from __future__ import unicode_literals import logging -from docker import Client +from docker import APIClient from docker.errors import TLSParameterError from docker.tls import TLSConfig from docker.utils import kwargs_from_env @@ -71,4 +71,4 @@ def docker_client(environment, version=None, tls_config=None, host=None, kwargs['user_agent'] = generate_user_agent() - return Client(**kwargs) + return APIClient(**kwargs) diff --git a/compose/network.py b/compose/network.py index 3b57cb94..8a29b73e 100644 --- a/compose/network.py +++ b/compose/network.py @@ -4,8 +4,8 @@ from __future__ import unicode_literals import logging from docker.errors import NotFound -from docker.utils import create_ipam_config -from docker.utils import create_ipam_pool +from docker.types import IPAMConfig +from docker.types import IPAMPool from .config import ConfigurationError @@ -96,10 +96,10 @@ def create_ipam_config_from_dict(ipam_dict): if not ipam_dict: return None - return create_ipam_config( + return IPAMConfig( driver=ipam_dict.get('driver'), pool_configs=[ - create_ipam_pool( + IPAMPool( subnet=config.get('subnet'), iprange=config.get('ip_range'), gateway=config.get('gateway'), diff --git a/compose/service.py b/compose/service.py index cf52d489..c32788fe 100644 --- a/compose/service.py +++ b/compose/service.py @@ -11,7 +11,7 @@ import enum import six from docker.errors import APIError from docker.errors import NotFound -from docker.utils import LogConfig +from docker.types import LogConfig from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port diff --git a/requirements.txt b/requirements.txt index 63469799..8f6fe169 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,11 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 +docker==2.0.0 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' functools32==3.2.3.post2; python_version < '3.2' -git+https://github.com/docker/docker-py.git@2ff7371ae7703033f981e1b137a3be0caf7a4f9c#egg=docker-py ipaddress==1.0.16 jsonschema==2.5.1 pypiwin32==219; sys_platform == 'win32' diff --git a/setup.py b/setup.py index 672ea80e..9dba2167 100644 --- a/setup.py +++ b/setup.py @@ -34,7 +34,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.10.6, < 2.0', + 'docker >= 2.0.0, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index 223b3b07..a279cab0 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -15,7 +15,7 @@ from compose.config.config import Config def mock_service(): return mock.create_autospec( service.Service, - client=mock.create_autospec(docker.Client), + client=mock.create_autospec(docker.APIClient), options={}) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 2c90b29b..f9b60bff 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -97,7 +97,7 @@ class CLITestCase(unittest.TestCase): @mock.patch('compose.cli.main.RunOperation', autospec=True) @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal, mock_run_operation): - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) project = Project.from_config( name='composetest', client=mock_client, @@ -128,7 +128,7 @@ class CLITestCase(unittest.TestCase): assert call_kwargs['logs'] is False def test_run_service_with_restart_always(self): - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) project = Project.from_config( name='composetest', diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 62e3aa2c..04f43016 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -98,7 +98,7 @@ class ContainerTest(unittest.TestCase): self.assertEqual(container.name_without_project, "custom_name_of_container") def test_inspect_if_not_inspected(self): - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) container = Container(mock_client, dict(Id="the_id")) container.inspect_if_not_inspected() diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 9569adc9..9a12438f 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -19,7 +19,7 @@ from compose.service import Service class ProjectTest(unittest.TestCase): def setUp(self): - self.mock_client = mock.create_autospec(docker.Client) + self.mock_client = mock.create_autospec(docker.APIClient) def test_from_config(self): config = Config( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 1d5aa10f..2d5b1761 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -34,7 +34,7 @@ from compose.service import warn_on_masked_volume class ServiceTest(unittest.TestCase): def setUp(self): - self.mock_client = mock.create_autospec(docker.Client) + self.mock_client = mock.create_autospec(docker.APIClient) def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') @@ -666,7 +666,7 @@ class ServiceTest(unittest.TestCase): class TestServiceNetwork(object): def test_connect_container_to_networks_short_aliase_exists(self): - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) service = Service( 'db', mock_client, @@ -751,7 +751,7 @@ class NetTestCase(unittest.TestCase): def test_network_mode_service(self): container_id = 'bbbb' service_name = 'web' - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) mock_client.containers.return_value = [ {'Id': container_id, 'Name': container_id, 'Image': 'abcd'}, ] @@ -765,7 +765,7 @@ class NetTestCase(unittest.TestCase): def test_network_mode_service_no_containers(self): service_name = 'web' - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) mock_client.containers.return_value = [] service = Service(name=service_name, client=mock_client) @@ -783,7 +783,7 @@ def build_mount(destination, source, mode='rw'): class ServiceVolumesTest(unittest.TestCase): def setUp(self): - self.mock_client = mock.create_autospec(docker.Client) + self.mock_client = mock.create_autospec(docker.APIClient) def test_build_volume_binding(self): binding = build_volume_binding(VolumeSpec.parse('/outside:/inside', True)) diff --git a/tests/unit/volume_test.py b/tests/unit/volume_test.py index d7ad0792..24829192 100644 --- a/tests/unit/volume_test.py +++ b/tests/unit/volume_test.py @@ -10,7 +10,7 @@ from tests import mock @pytest.fixture def mock_client(): - return mock.create_autospec(docker.Client) + return mock.create_autospec(docker.APIClient) class TestVolume(object): From fb165d9c1505e2505f637cb2cede77e4d0731884 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 16 Dec 2016 12:21:59 -0800 Subject: [PATCH 38/98] Add v3_only marker to healthcheck test Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 856b8f93..2d0ce715 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -316,8 +316,8 @@ class CLITestCase(DockerClientTestCase): 'memory': '50M', }, 'reservations': { - 'cpus': '0.0001', - 'memory': '20M', + 'cpus': '0.0001', + 'memory': '20M', }, }, 'restart_policy': { @@ -928,6 +928,7 @@ class CLITestCase(DockerClientTestCase): assert foo_container.get('HostConfig.NetworkMode') == \ 'container:{}'.format(bar_container.id) + @v3_only() def test_up_with_healthcheck(self): def wait_on_health_status(container, status): def condition(): @@ -945,7 +946,7 @@ class CLITestCase(DockerClientTestCase): assert passes_container.get('Config.Healthcheck') == { "Test": ["CMD-SHELL", "/bin/true"], "Interval": nanoseconds_from_time_seconds(1), - "Timeout": nanoseconds_from_time_seconds(30*60), + "Timeout": nanoseconds_from_time_seconds(30 * 60), "Retries": 1, } From e736151ee497a65df4e7e271ebd059731e250760 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Dec 2016 16:11:21 -0800 Subject: [PATCH 39/98] Make created networks attachable for file format >=2.1 Signed-off-by: Joffrey F --- compose/network.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/network.py b/compose/network.py index 8a29b73e..eb76e292 100644 --- a/compose/network.py +++ b/compose/network.py @@ -6,6 +6,7 @@ import logging from docker.errors import NotFound from docker.types import IPAMConfig from docker.types import IPAMPool +from docker.utils import version_gte from .config import ConfigurationError @@ -72,6 +73,7 @@ class Network(object): internal=self.internal, enable_ipv6=self.enable_ipv6, labels=self.labels, + attachable=version_gte(self.client._version, '1.24') or None ) def remove(self): From eb6441c8e337a1e797adee26b77c0c03465375a2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Dec 2016 20:35:09 -0800 Subject: [PATCH 40/98] Add sysctls option to 3.0 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v3.0.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 8b8ceac0..1f93347f 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -167,6 +167,7 @@ "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, "stop_signal": {"type": "string"}, "stop_grace_period": {"type": "string", "format": "duration"}, From 346802715dd3869c2a33c5d361ec8a56fa3352c5 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 24 Oct 2016 16:27:47 -0700 Subject: [PATCH 41/98] Use colorama to enable colored output on Windows Signed-off-by: Joffrey F --- compose/cli/colors.py | 4 ++++ requirements.txt | 1 + setup.py | 1 + 3 files changed, 6 insertions(+) diff --git a/compose/cli/colors.py b/compose/cli/colors.py index 3c18886f..6677a376 100644 --- a/compose/cli/colors.py +++ b/compose/cli/colors.py @@ -1,5 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals + +import colorama + NAMES = [ 'grey', 'red', @@ -30,6 +33,7 @@ def make_color_fn(code): return lambda s: ansi_color(code, s) +colorama.init() for (name, code) in get_pairs(): globals()[name] = make_color_fn(code) diff --git a/requirements.txt b/requirements.txt index 8f6fe169..bae5d9ea 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 +colorama==0.3.7 docker==2.0.0 dockerpty==0.4.1 docopt==0.6.1 diff --git a/setup.py b/setup.py index 9dba2167..8b4cf709 100644 --- a/setup.py +++ b/setup.py @@ -29,6 +29,7 @@ def find_version(*file_paths): install_requires = [ 'cached-property >= 1.2.0, < 2', + 'colorama >= 0.3.7, < 0.4', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', 'requests >= 2.6.1, != 2.11.0, < 2.12', From ba47fb99ba7670fc8435af0a958a3ed01188711a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 5 Dec 2016 18:21:10 -0800 Subject: [PATCH 42/98] Add default labels to networks and volumes created by Compose Signed-off-by: Joffrey F --- compose/const.py | 2 ++ compose/network.py | 18 ++++++++++++++++-- compose/volume.py | 16 +++++++++++++++- tests/acceptance/cli_test.py | 8 ++++---- tests/integration/network_test.py | 17 +++++++++++++++++ tests/integration/project_test.py | 7 ++++--- tests/integration/volume_test.py | 10 ++++++++++ 7 files changed, 68 insertions(+), 10 deletions(-) create mode 100644 tests/integration/network_test.py diff --git a/compose/const.py b/compose/const.py index ca8d7fe5..1b1be5c7 100644 --- a/compose/const.py +++ b/compose/const.py @@ -11,7 +11,9 @@ LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' +LABEL_NETWORK = 'com.docker.compose.network' LABEL_VERSION = 'com.docker.compose.version' +LABEL_VOLUME = 'com.docker.compose.volume' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' COMPOSEFILE_V1 = '1' diff --git a/compose/network.py b/compose/network.py index eb76e292..d98f68d2 100644 --- a/compose/network.py +++ b/compose/network.py @@ -7,8 +7,11 @@ from docker.errors import NotFound from docker.types import IPAMConfig from docker.types import IPAMPool from docker.utils import version_gte +from docker.utils import version_lt from .config import ConfigurationError +from .const import LABEL_NETWORK +from .const import LABEL_PROJECT log = logging.getLogger(__name__) @@ -72,8 +75,8 @@ class Network(object): ipam=self.ipam, internal=self.internal, enable_ipv6=self.enable_ipv6, - labels=self.labels, - attachable=version_gte(self.client._version, '1.24') or None + labels=self._labels, + attachable=version_gte(self.client._version, '1.24') or None, ) def remove(self): @@ -93,6 +96,17 @@ class Network(object): return self.external_name return '{0}_{1}'.format(self.project, self.name) + @property + def _labels(self): + if version_lt(self.client._version, '1.23'): + return None + labels = self.labels.copy() if self.labels else {} + labels.update({ + LABEL_PROJECT: self.project, + LABEL_NETWORK: self.name, + }) + return labels + def create_ipam_config_from_dict(ipam_dict): if not ipam_dict: diff --git a/compose/volume.py b/compose/volume.py index 1fd1d51c..ab6a88fa 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -4,8 +4,11 @@ from __future__ import unicode_literals import logging from docker.errors import NotFound +from docker.utils import version_lt from .config import ConfigurationError +from .const import LABEL_PROJECT +from .const import LABEL_VOLUME log = logging.getLogger(__name__) @@ -23,7 +26,7 @@ class Volume(object): def create(self): return self.client.create_volume( - self.full_name, self.driver, self.driver_opts, labels=self.labels + self.full_name, self.driver, self.driver_opts, labels=self._labels ) def remove(self): @@ -53,6 +56,17 @@ class Volume(object): return self.external_name return '{0}_{1}'.format(self.project, self.name) + @property + def _labels(self): + if version_lt(self.client._version, '1.23'): + return None + labels = self.labels.copy() if self.labels else {} + labels.update({ + LABEL_PROJECT: self.project, + LABEL_VOLUME: self.name, + }) + return labels + class ProjectVolumes(object): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 2d0ce715..b9766226 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -850,8 +850,8 @@ class CLITestCase(DockerClientTestCase): ] assert [n['Name'] for n in networks] == [network_with_label] - - assert networks[0]['Labels'] == {'label_key': 'label_val'} + assert 'label_key' in networks[0]['Labels'] + assert networks[0]['Labels']['label_key'] == 'label_val' @v2_1_only() def test_up_with_volume_labels(self): @@ -870,8 +870,8 @@ class CLITestCase(DockerClientTestCase): ] assert [v['Name'] for v in volumes] == [volume_with_label] - - assert volumes[0]['Labels'] == {'label_key': 'label_val'} + assert 'label_key' in volumes[0]['Labels'] + assert volumes[0]['Labels']['label_key'] == 'label_val' @v2_only() def test_up_no_services(self): diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py new file mode 100644 index 00000000..2ff610fb --- /dev/null +++ b/tests/integration/network_test.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from .testcases import DockerClientTestCase +from compose.const import LABEL_NETWORK +from compose.const import LABEL_PROJECT +from compose.network import Network + + +class NetworkTest(DockerClientTestCase): + def test_network_default_labels(self): + net = Network(self.client, 'composetest', 'foonet') + net.ensure() + net_data = net.inspect() + labels = net_data['Labels'] + assert labels[LABEL_NETWORK] == net.name + assert labels[LABEL_PROJECT] == net.project diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 3afefb5a..de073239 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -942,8 +942,8 @@ class ProjectTest(DockerClientTestCase): ] assert [n['Name'] for n in networks] == ['composetest_{}'.format(network_name)] - - assert networks[0]['Labels'] == {'label_key': 'label_val'} + assert 'label_key' in networks[0]['Labels'] + assert networks[0]['Labels']['label_key'] == 'label_val' @v2_only() def test_project_up_volumes(self): @@ -1009,7 +1009,8 @@ class ProjectTest(DockerClientTestCase): assert [v['Name'] for v in volumes] == ['composetest_{}'.format(volume_name)] - assert volumes[0]['Labels'] == {'label_key': 'label_val'} + assert 'label_key' in volumes[0]['Labels'] + assert volumes[0]['Labels']['label_key'] == 'label_val' @v2_only() def test_project_up_logging_with_multiple_files(self): diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index a75250ac..add16962 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals from docker.errors import DockerException from .testcases import DockerClientTestCase +from compose.const import LABEL_PROJECT +from compose.const import LABEL_VOLUME from compose.volume import Volume @@ -94,3 +96,11 @@ class VolumeTest(DockerClientTestCase): assert vol.exists() is False vol.create() assert vol.exists() is True + + def test_volume_default_labels(self): + vol = self.create_volume('volume01') + vol.create() + vol_data = vol.inspect() + labels = vol_data['Labels'] + assert labels[LABEL_VOLUME] == vol.name + assert labels[LABEL_PROJECT] == vol.project From a74b2f2f70a82bc56fb463d46480ea1baad9d4ab Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Dec 2016 15:16:53 -0500 Subject: [PATCH 43/98] Fix schema typo. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.0.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 1f93347f..6212058c 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -270,7 +270,7 @@ "cpus": {"type": "string"}, "memory": {"type": "string"} }, - "additionaProperties": false + "additionalProperties": false }, "network": { From c73fc26824f2bffa791472a27039da0d2dd1ccbe Mon Sep 17 00:00:00 2001 From: Jun Guo Date: Wed, 4 Jan 2017 15:31:12 +0800 Subject: [PATCH 44/98] Fix 404 issue, change APIError to more accureate ImageNotFound Signed-off-by: Jun Guo --- compose/service.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index e3862b6e..ee8c88a9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,6 +10,7 @@ from operator import attrgetter import enum import six from docker.errors import APIError +from docker.errors import ImageNotFound from docker.errors import NotFound from docker.types import LogConfig from docker.utils.ports import build_port_bindings @@ -318,11 +319,8 @@ class Service(object): def image(self): try: return self.client.inspect_image(self.image_name) - except APIError as e: - if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - raise NoSuchImageError("Image '{}' not found".format(self.image_name)) - else: - raise + except ImageNotFound: + raise NoSuchImageError("Image '{}' not found".format(self.image_name)) @property def image_name(self): From 2648af6807f83f0dd85b236e89e4bc3ee5db15fc Mon Sep 17 00:00:00 2001 From: Thomas Grainger Date: Tue, 22 Nov 2016 11:09:56 +0000 Subject: [PATCH 45/98] enable universal wheels Signed-off-by: Thomas Grainger --- Dockerfile.run | 2 +- script/build/image | 4 ++-- script/release/push-release | 6 +++--- setup.cfg | 2 ++ setup.py | 23 ++++++++++++++++++++++- 5 files changed, 30 insertions(+), 7 deletions(-) create mode 100644 setup.cfg diff --git a/Dockerfile.run b/Dockerfile.run index 4e76d64f..c6852af1 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -7,7 +7,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release.tar.gz /code/docker-compose +ADD dist/docker-compose-release-py2.py3-none-any.whl /code/docker-compose RUN pip install --no-deps /code/docker-compose/docker-compose-* ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/script/build/image b/script/build/image index bdd98f03..28aa2047 100755 --- a/script/build/image +++ b/script/build/image @@ -11,6 +11,6 @@ TAG=$1 VERSION="$(python setup.py --version)" ./script/build/write-git-sha -python setup.py sdist -cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz +python setup.py sdist bdist_wheel +cp dist/docker-compose-$VERSION-py2.py3-none-any.whl dist/docker-compose-release-py2.py3-none-any.whl docker build -t docker/compose:$TAG -f Dockerfile.run . diff --git a/script/release/push-release b/script/release/push-release index d5ae3de9..d1a9e3f6 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -54,13 +54,13 @@ git push $GITHUB_REPO $VERSION echo "Uploading the docker image" docker push docker/compose:$VERSION -echo "Uploading sdist to PyPI" +echo "Uploading package to PyPI" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst ./script/build/write-git-sha -python setup.py sdist +python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker-compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..3c6e79cf --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index 8b4cf709..00ca9f4c 100644 --- a/setup.py +++ b/setup.py @@ -4,10 +4,12 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import logging import os import re import sys +import pkg_resources from setuptools import find_packages from setuptools import setup @@ -49,7 +51,25 @@ tests_require = [ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') - install_requires.append('enum34 >= 1.0.4, < 2') + +extras_require = { + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'] +} + + +try: + if 'bdist_wheel' not in sys.argv: + for key, value in extras_require.items(): + if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): + install_requires.extend(value) +except Exception: + logging.getLogger(__name__).exception( + 'Something went wrong calculating platform specific dependencies, so ' + "you're getting them all!" + ) + for key, value in extras_require.items(): + if key.startswith(':'): + install_requires.extend(value) setup( @@ -63,6 +83,7 @@ setup( include_package_data=True, test_suite='nose.collector', install_requires=install_requires, + extras_require=extras_require, tests_require=tests_require, entry_points=""" [console_scripts] From 0edfe08bf017841a1a2624b98e4a05bb11fc6d4f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 13 Dec 2016 16:54:41 -0800 Subject: [PATCH 46/98] Add healthchecks to 2.1 schema. Update depends_on Signed-off-by: Joffrey F --- compose/config/config_schema_v2.1.json | 42 +++++++++++++++++++++++++- 1 file changed, 41 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 3057f2de..5ab9f71f 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -77,7 +77,28 @@ "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, - "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "depends_on": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "condition": { + "type": "string", + "enum": ["service_started", "service_healthy"] + } + }, + "required": ["condition"] + } + } + } + ] + }, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, @@ -120,6 +141,7 @@ "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, "hostname": {"type": "string"}, "image": {"type": "string"}, "ipc": {"type": "string"}, @@ -231,6 +253,24 @@ "additionalProperties": false }, + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disabled": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "network": { "id": "#/definitions/network", "type": "object", From f6edd610f36ec9f6ffdd16df970f3d6cefb1169d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 14 Dec 2016 15:39:42 -0800 Subject: [PATCH 47/98] Add 3.0 schema to docker-compose.spec Signed-off-by: Joffrey F --- docker-compose.spec | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docker-compose.spec b/docker-compose.spec index e57d6b7a..ec5a2039 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -32,6 +32,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v2.1.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.0.json', + 'compose/config/config_schema_v3.0.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', From 04394b1d0a82f4507edc8863c4ab9cf64944f6d9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Dec 2016 20:20:03 -0800 Subject: [PATCH 48/98] Expand depends_on to allow different conditions (service_start, service_healthy) Rework "up" and "start" to wait on conditional state of dependent services Add integration tests Signed-off-by: Joffrey F --- compose/config/config.py | 12 +++- compose/config/validation.py | 8 ++- compose/errors.py | 21 ++++++ compose/parallel.py | 7 +- compose/project.py | 12 +++- compose/service.py | 59 ++++++++++++++-- tests/acceptance/cli_test.py | 2 +- tests/integration/project_test.py | 114 ++++++++++++++++++++++++++++++ tests/unit/config/config_test.py | 9 ++- tests/unit/parallel_test.py | 2 +- 10 files changed, 228 insertions(+), 18 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index dccd11e0..73a34017 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -649,6 +649,8 @@ def process_service(service_config): if 'sysctls' in service_dict: service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) + service_dict = process_depends_on(service_dict) + for field in ['dns', 'dns_search', 'tmpfs']: if field in service_dict: service_dict[field] = to_list(service_dict[field]) @@ -658,6 +660,14 @@ def process_service(service_config): return service_dict +def process_depends_on(service_dict): + if 'depends_on' in service_dict and not isinstance(service_dict['depends_on'], dict): + service_dict['depends_on'] = dict([ + (svc, {'condition': 'service_started'}) for svc in service_dict['depends_on'] + ]) + return service_dict + + def process_healthcheck(service_dict, service_name): if 'healthcheck' not in service_dict: return service_dict @@ -665,7 +675,7 @@ def process_healthcheck(service_dict, service_name): hc = {} raw = service_dict['healthcheck'] - if raw.get('disable'): + if raw.get('disable') or raw.get('disabled'): if len(raw) > 1: raise ConfigurationError( 'Service "{}" defines an invalid healthcheck: ' diff --git a/compose/config/validation.py b/compose/config/validation.py index 7452e984..3f23f0a7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -180,11 +180,13 @@ def validate_links(service_config, service_names): def validate_depends_on(service_config, service_names): - for dependency in service_config.config.get('depends_on', []): + deps = service_config.config.get('depends_on', {}) + for dependency in deps.keys(): if dependency not in service_names: raise ConfigurationError( "Service '{s.name}' depends on service '{dep}' which is " - "undefined.".format(s=service_config, dep=dependency)) + "undefined.".format(s=service_config, dep=dependency) + ) def get_unsupported_config_msg(path, error_key): @@ -201,7 +203,7 @@ def anglicize_json_type(json_type): def is_service_dict_schema(schema_id): - return schema_id in ('config_schema_v1.json', '#/properties/services') + return schema_id in ('config_schema_v1.json', '#/properties/services') def handle_error_for_schema_with_id(error, path): diff --git a/compose/errors.py b/compose/errors.py index 376cc555..415b41e7 100644 --- a/compose/errors.py +++ b/compose/errors.py @@ -10,3 +10,24 @@ class OperationFailedError(Exception): class StreamParseError(RuntimeError): def __init__(self, reason): self.msg = reason + + +class HealthCheckException(Exception): + def __init__(self, reason): + self.msg = reason + + +class HealthCheckFailed(HealthCheckException): + def __init__(self, container_id): + super(HealthCheckFailed, self).__init__( + 'Container "{}" is unhealthy.'.format(container_id) + ) + + +class NoHealthCheckConfigured(HealthCheckException): + def __init__(self, service_name): + super(NoHealthCheckConfigured, self).__init__( + 'Service "{}" is missing a healthcheck configuration'.format( + service_name + ) + ) diff --git a/compose/parallel.py b/compose/parallel.py index 26718872..b2654dcf 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -165,13 +165,14 @@ def feed_queue(objects, func, get_deps, results, state): for obj in pending: deps = get_deps(obj) - if any(dep in state.failed for dep in deps): + if any(dep[0] in state.failed for dep in deps): log.debug('{} has upstream errors - not processing'.format(obj)) results.put((obj, None, UpstreamError())) state.failed.add(obj) elif all( - dep not in objects or dep in state.finished - for dep in deps + dep not in objects or ( + dep in state.finished and (not ready_check or ready_check(dep)) + ) for dep, ready_check in deps ): log.debug('Starting producer thread for {}'.format(obj)) t = Thread(target=producer, args=(obj, func, results)) diff --git a/compose/project.py b/compose/project.py index 0178bbab..d99ef7c9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -227,7 +227,10 @@ class Project(object): services = self.get_services(service_names) def get_deps(service): - return {self.get_service(dep) for dep in service.get_dependency_names()} + return { + (self.get_service(dep), config) + for dep, config in service.get_dependency_configs().items() + } parallel.parallel_execute( services, @@ -243,7 +246,7 @@ class Project(object): def get_deps(container): # actually returning inversed dependencies - return {other for other in containers + return {(other, None) for other in containers if container.service in self.get_service(other.service).get_dependency_names()} @@ -394,7 +397,10 @@ class Project(object): ) def get_deps(service): - return {self.get_service(dep) for dep in service.get_dependency_names()} + return { + (self.get_service(dep), config) + for dep, config in service.get_dependency_configs().items() + } results, errors = parallel.parallel_execute( services, diff --git a/compose/service.py b/compose/service.py index e3862b6e..1338f154 100644 --- a/compose/service.py +++ b/compose/service.py @@ -28,6 +28,8 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION from .container import Container +from .errors import HealthCheckFailed +from .errors import NoHealthCheckConfigured from .errors import OperationFailedError from .parallel import parallel_execute from .parallel import parallel_start @@ -69,6 +71,9 @@ DOCKER_START_KEYS = [ 'volumes_from', ] +CONDITION_STARTED = 'service_started' +CONDITION_HEALTHY = 'service_healthy' + class BuildError(Exception): def __init__(self, service, reason): @@ -533,10 +538,38 @@ class Service(object): def get_dependency_names(self): net_name = self.network_mode.service_name - return (self.get_linked_service_names() + - self.get_volumes_from_names() + - ([net_name] if net_name else []) + - self.options.get('depends_on', [])) + return ( + self.get_linked_service_names() + + self.get_volumes_from_names() + + ([net_name] if net_name else []) + + list(self.options.get('depends_on', {}).keys()) + ) + + def get_dependency_configs(self): + net_name = self.network_mode.service_name + configs = dict( + [(name, None) for name in self.get_linked_service_names()] + ) + configs.update(dict( + [(name, None) for name in self.get_volumes_from_names()] + )) + configs.update({net_name: None} if net_name else {}) + configs.update(self.options.get('depends_on', {})) + for svc, config in self.options.get('depends_on', {}).items(): + if config['condition'] == CONDITION_STARTED: + configs[svc] = lambda s: True + elif config['condition'] == CONDITION_HEALTHY: + configs[svc] = lambda s: s.is_healthy() + else: + # The config schema already prevents this, but it might be + # bypassed if Compose is called programmatically. + raise ValueError( + 'depends_on condition "{}" is invalid.'.format( + config['condition'] + ) + ) + + return configs def get_linked_service_names(self): return [service.name for (service, _) in self.links] @@ -871,6 +904,24 @@ class Service(object): else: log.error(six.text_type(e)) + def is_healthy(self): + """ Check that all containers for this service report healthy. + Returns false if at least one healthcheck is pending. + If an unhealthy container is detected, raise a HealthCheckFailed + exception. + """ + result = True + for ctnr in self.containers(): + ctnr.inspect() + status = ctnr.get('State.Health.Status') + if status is None: + raise NoHealthCheckConfigured(self.name) + elif status == 'starting': + result = False + elif status == 'unhealthy': + raise HealthCheckFailed(ctnr.short_id) + return result + def short_id_alias_exists(container, network): aliases = container.get( diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226..77d57840 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -928,7 +928,7 @@ class CLITestCase(DockerClientTestCase): assert foo_container.get('HostConfig.NetworkMode') == \ 'container:{}'.format(bar_container.id) - @v3_only() + @v2_1_only() def test_up_with_healthcheck(self): def wait_on_health_status(container, status): def condition(): diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index de073239..855974de 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -19,6 +19,8 @@ from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.container import Container +from compose.errors import HealthCheckFailed +from compose.errors import NoHealthCheckConfigured from compose.project import Project from compose.project import ProjectError from compose.service import ConvergenceStrategy @@ -1375,3 +1377,115 @@ class ProjectTest(DockerClientTestCase): ctnr for ctnr in project._labeled_containers() if ctnr.labels.get(LABEL_SERVICE) == 'service1' ]) == 0 + + @v2_1_only() + def test_project_up_healthy_dependency(self): + config_dict = { + 'version': '2.1', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'healthcheck': { + 'test': 'exit 0', + 'retries': 1, + 'timeout': '10s', + 'interval': '0.1s' + }, + }, + 'svc2': { + 'image': 'busybox:latest', + 'command': 'top', + 'depends_on': { + 'svc1': {'condition': 'service_healthy'}, + } + } + } + } + config_data = build_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + containers = project.containers() + assert len(containers) == 2 + + svc1 = project.get_service('svc1') + svc2 = project.get_service('svc2') + assert 'svc1' in svc2.get_dependency_names() + assert svc1.is_healthy() + + @v2_1_only() + def test_project_up_unhealthy_dependency(self): + config_dict = { + 'version': '2.1', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'healthcheck': { + 'test': 'exit 1', + 'retries': 1, + 'timeout': '10s', + 'interval': '0.1s' + }, + }, + 'svc2': { + 'image': 'busybox:latest', + 'command': 'top', + 'depends_on': { + 'svc1': {'condition': 'service_healthy'}, + } + } + } + } + config_data = build_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + with pytest.raises(HealthCheckFailed): + project.up() + containers = project.containers() + assert len(containers) == 1 + + svc1 = project.get_service('svc1') + svc2 = project.get_service('svc2') + assert 'svc1' in svc2.get_dependency_names() + with pytest.raises(HealthCheckFailed): + svc1.is_healthy() + + @v2_1_only() + def test_project_up_no_healthcheck_dependency(self): + config_dict = { + 'version': '2.1', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'healthcheck': { + 'disabled': True + }, + }, + 'svc2': { + 'image': 'busybox:latest', + 'command': 'top', + 'depends_on': { + 'svc1': {'condition': 'service_healthy'}, + } + } + } + } + config_data = build_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + with pytest.raises(NoHealthCheckConfigured): + project.up() + containers = project.containers() + assert len(containers) == 1 + + svc1 = project.get_service('svc1') + svc2 = project.get_service('svc2') + assert 'svc1' in svc2.get_dependency_names() + with pytest.raises(NoHealthCheckConfigured): + svc1.is_healthy() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f7df3aee..7a68333f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -920,7 +920,10 @@ class ConfigTest(unittest.TestCase): 'build': {'context': os.path.abspath('/')}, 'image': 'example/web', 'volumes': [VolumeSpec.parse('/home/user/project:/code')], - 'depends_on': ['db', 'other'], + 'depends_on': { + 'db': {'condition': 'container_start'}, + 'other': {'condition': 'container_start'}, + }, }, { 'name': 'db', @@ -3055,7 +3058,9 @@ class ExtendsTest(unittest.TestCase): image: example """) services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) - assert service_sort(services)[2]['depends_on'] == ['other'] + assert service_sort(services)[2]['depends_on'] == { + 'other': {'condition': 'container_start'} + } @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py index 479c0f1d..2a50b718 100644 --- a/tests/unit/parallel_test.py +++ b/tests/unit/parallel_test.py @@ -25,7 +25,7 @@ deps = { def get_deps(obj): - return deps[obj] + return [(dep, None) for dep in deps[obj]] def test_parallel_execute(): From bef230853012d78e7ab58de65b653839401be974 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 19 Dec 2016 20:39:16 -0800 Subject: [PATCH 49/98] Fix condition name in config tests Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 2 +- tests/unit/config/config_test.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 77d57840..b9766226 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -928,7 +928,7 @@ class CLITestCase(DockerClientTestCase): assert foo_container.get('HostConfig.NetworkMode') == \ 'container:{}'.format(bar_container.id) - @v2_1_only() + @v3_only() def test_up_with_healthcheck(self): def wait_on_health_status(container, status): def condition(): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 7a68333f..31a888ed 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -921,8 +921,8 @@ class ConfigTest(unittest.TestCase): 'image': 'example/web', 'volumes': [VolumeSpec.parse('/home/user/project:/code')], 'depends_on': { - 'db': {'condition': 'container_start'}, - 'other': {'condition': 'container_start'}, + 'db': {'condition': 'service_started'}, + 'other': {'condition': 'service_started'}, }, }, { @@ -3059,7 +3059,7 @@ class ExtendsTest(unittest.TestCase): """) services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) assert service_sort(services)[2]['depends_on'] == { - 'other': {'condition': 'container_start'} + 'other': {'condition': 'service_started'} } From 8145429399346a8d800369aad17f5fe69237c2ad Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 13:14:23 -0800 Subject: [PATCH 50/98] Unify healthcheck spec definition in v2 and v3 Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- compose/config/config_schema_v2.1.json | 2 +- compose/config/config_schema_v3.0.json | 12 ++++++------ tests/integration/project_test.py | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 73a34017..fd935591 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -675,7 +675,7 @@ def process_healthcheck(service_dict, service_name): hc = {} raw = service_dict['healthcheck'] - if raw.get('disable') or raw.get('disabled'): + if raw.get('disable'): if len(raw) > 1: raise ConfigurationError( 'Service "{}" defines an invalid healthcheck: ' diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 5ab9f71f..d0d5233a 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -258,7 +258,7 @@ "type": "object", "additionalProperties": false, "properties": { - "disabled": {"type": "boolean"}, + "disable": {"type": "boolean"}, "interval": {"type": "string"}, "retries": {"type": "number"}, "test": { diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 1f93347f..8d075d47 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -202,10 +202,11 @@ "healthcheck": { "id": "#/definitions/healthcheck", - "type": ["object", "null"], + "type": "object", + "additionalProperties": false, "properties": { - "interval": {"type":"string"}, - "timeout": {"type":"string"}, + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -213,9 +214,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "disable": {"type": "boolean"} - }, - "additionalProperties": false + "timeout": {"type": "string"} + } }, "deployment": { "id": "#/definitions/deployment", diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 855974de..c5e3cf50 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1463,7 +1463,7 @@ class ProjectTest(DockerClientTestCase): 'image': 'busybox:latest', 'command': 'top', 'healthcheck': { - 'disabled': True + 'disable': True }, }, 'svc2': { From 1be41f59c9119c72b3c39045e4e4031608fa18df Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 14:30:20 -0800 Subject: [PATCH 51/98] Add support for stop_grace_period in v2 Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 1 + compose/config/config_schema_v2.1.json | 1 + compose/config/config_schema_v3.0.json | 2 +- 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 76688916..77494715 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -192,6 +192,7 @@ "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "shm_size": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index d0d5233a..97ec5fa1 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -217,6 +217,7 @@ "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 8d075d47..2b410446 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -169,8 +169,8 @@ "shm_size": {"type": ["number", "string"]}, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, - "stop_signal": {"type": "string"}, "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { From 534b4ed820ec2b2e2f7b296e0065c42ebf3489cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 4 Jan 2017 15:26:11 -0800 Subject: [PATCH 52/98] Falsy values in COMPOSE_CONVERT_WINDOWS_PATHS are properly recognized Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- compose/config/environment.py | 11 ++++++++ tests/unit/config/environment_test.py | 40 +++++++++++++++++++++++++++ 3 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 tests/unit/config/environment_test.py diff --git a/compose/config/config.py b/compose/config/config.py index fd935591..c11460fa 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -712,7 +712,7 @@ def finalize_service(service_config, service_names, version, environment): if 'volumes' in service_dict: service_dict['volumes'] = [ VolumeSpec.parse( - v, environment.get('COMPOSE_CONVERT_WINDOWS_PATHS') + v, environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') ) for v in service_dict['volumes'] ] diff --git a/compose/config/environment.py b/compose/config/environment.py index 5d6b5af6..7b926930 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -105,3 +105,14 @@ class Environment(dict): super(Environment, self).get(key.upper(), *args, **kwargs) ) return super(Environment, self).get(key, *args, **kwargs) + + def get_boolean(self, key): + # Convert a value to a boolean using "common sense" rules. + # Unset, empty, "0" and "false" (i-case) yield False. + # All other values yield True. + value = self.get(key) + if not value: + return False + if value.lower() in ['0', 'false']: + return False + return True diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py new file mode 100644 index 00000000..20446d2b --- /dev/null +++ b/tests/unit/config/environment_test.py @@ -0,0 +1,40 @@ +# encoding: utf-8 +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from compose.config.environment import Environment +from tests import unittest + + +class EnvironmentTest(unittest.TestCase): + def test_get_simple(self): + env = Environment({ + 'FOO': 'bar', + 'BAR': '1', + 'BAZ': '' + }) + + assert env.get('FOO') == 'bar' + assert env.get('BAR') == '1' + assert env.get('BAZ') == '' + + def test_get_undefined(self): + env = Environment({ + 'FOO': 'bar' + }) + assert env.get('FOOBAR') is None + + def test_get_boolean(self): + env = Environment({ + 'FOO': '', + 'BAR': '0', + 'BAZ': 'FALSE', + 'FOOBAR': 'true', + }) + + assert env.get_boolean('FOO') is False + assert env.get_boolean('BAR') is False + assert env.get_boolean('BAZ') is False + assert env.get_boolean('FOOBAR') is True + assert env.get_boolean('UNDEFINED') is False From e063c5739fedeb56450075920451e3fd8b57a826 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 5 Jan 2017 11:15:24 -0800 Subject: [PATCH 53/98] Fix config schemas (misplaced "additionalProperties") Signed-off-by: Joffrey F --- compose/config/config_schema_v2.0.json | 6 +++--- compose/config/config_schema_v2.1.json | 6 +++--- compose/config/config_schema_v3.0.json | 3 ++- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/compose/config/config_schema_v2.0.json b/compose/config/config_schema_v2.0.json index 77494715..59c7b30c 100644 --- a/compose/config/config_schema_v2.0.json +++ b/compose/config/config_schema_v2.0.json @@ -276,9 +276,9 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } - }, - "additionalProperties": false + }, + "additionalProperties": false + } }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json index 97ec5fa1..d1ffff89 100644 --- a/compose/config/config_schema_v2.1.json +++ b/compose/config/config_schema_v2.1.json @@ -322,10 +322,10 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } + }, + "additionalProperties": false }, - "labels": {"$ref": "#/definitions/list_or_dict"}, - "additionalProperties": false + "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false }, diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 2b410446..194fd8e6 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -328,7 +328,8 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } + }, + "additionalProperties": false } }, "labels": {"$ref": "#/definitions/list_or_dict"}, From 2c157e8fa9a94d71637643a6ec807db8b21a9d29 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 6 Jan 2017 17:45:57 -0800 Subject: [PATCH 54/98] Use docker SDK 2.0.1 Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index bae5d9ea..4b7c7b76 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.0 +docker==2.0.1 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 8b4cf709..7954d92b 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.0, < 3.0', + 'docker >= 2.0.1, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From 52792b7a963af9c593e61c78c7f0c7f62550a85b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jan 2017 14:57:32 -0800 Subject: [PATCH 55/98] Update setup.py extra_requires Signed-off-by: Joffrey F --- setup.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/setup.py b/setup.py index 0ceb2a22..2f2ba742 100644 --- a/setup.py +++ b/setup.py @@ -53,7 +53,9 @@ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') extras_require = { - ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'] + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], + ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], + ':python_version < "3.3"': ['ipaddress >= 1.0.16'], } @@ -64,8 +66,8 @@ try: install_requires.extend(value) except Exception: logging.getLogger(__name__).exception( - 'Something went wrong calculating platform specific dependencies, so ' - "you're getting them all!" + 'Failed to compute platform dependencies. All dependencies will be ' + 'installed as a result.' ) for key, value in extras_require.items(): if key.startswith(':'): From 19190ea0df43978d1a9c9f0fefd644ca5b08aee3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Jan 2017 16:43:26 -0800 Subject: [PATCH 56/98] Fix docker image build script when using universal wheels Signed-off-by: Joffrey F --- Dockerfile.run | 5 +++-- script/build/image | 3 +-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/Dockerfile.run b/Dockerfile.run index c6852af1..de46e35e 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,5 +1,6 @@ FROM alpine:3.4 +ARG version RUN apk -U add \ python \ py-pip @@ -7,7 +8,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release-py2.py3-none-any.whl /code/docker-compose -RUN pip install --no-deps /code/docker-compose/docker-compose-* +COPY dist/docker_compose-${version}-py2.py3-none-any.whl /code/ +RUN pip install --no-deps /code/docker_compose-${version}-py2.py3-none-any.whl ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/script/build/image b/script/build/image index 28aa2047..3590ce14 100755 --- a/script/build/image +++ b/script/build/image @@ -12,5 +12,4 @@ VERSION="$(python setup.py --version)" ./script/build/write-git-sha python setup.py sdist bdist_wheel -cp dist/docker-compose-$VERSION-py2.py3-none-any.whl dist/docker-compose-release-py2.py3-none-any.whl -docker build -t docker/compose:$TAG -f Dockerfile.run . +docker build --build-arg version=$VERSION -t docker/compose:$TAG -f Dockerfile.run . From 29b46d5b26055ade86a2e0e608342dd298e5a8c3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 15:39:48 -0800 Subject: [PATCH 57/98] Use correct wheel file name in twine upload command Signed-off-by: Joffrey F --- script/release/push-release | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/script/release/push-release b/script/release/push-release index d1a9e3f6..9db6f689 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -60,12 +60,13 @@ sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/maste ./script/build/write-git-sha python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker-compose-${VERSION/-/}-py2.py3-none-any.whl + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker_compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi echo "Testing pip package" +deactivate || true virtualenv venv-test source venv-test/bin/activate pip install docker-compose==$VERSION From 2df31bb13c9a6820aba1d9b5a827329eded2b9cd Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Jan 2017 16:25:40 -0800 Subject: [PATCH 58/98] Provide valid serialization of depends_on when format is not 2.1 Signed-off-by: Joffrey F --- compose/config/serialize.py | 9 ++++++++- tests/unit/config/config_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 768f3d47..05ac0d60 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -56,9 +56,16 @@ def denormalize_service_dict(service_dict, version): service_dict = service_dict.copy() if 'restart' in service_dict: - service_dict['restart'] = types.serialize_restart_spec(service_dict['restart']) + service_dict['restart'] = types.serialize_restart_spec( + service_dict['restart'] + ) if version == V1 and 'network_mode' not in service_dict: service_dict['network_mode'] = 'bridge' + if 'depends_on' in service_dict and version != V2_1: + service_dict['depends_on'] = sorted([ + svc for svc in service_dict['depends_on'].keys() + ]) + return service_dict diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 31a888ed..ca7c6168 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -22,6 +22,7 @@ from compose.config.config import V3_0 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION +from compose.config.serialize import denormalize_service_dict from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds @@ -3269,3 +3270,33 @@ def get_config_filename_for_files(filenames, subdir=None): return os.path.basename(filename) finally: shutil.rmtree(project_dir) + + +class SerializeTest(unittest.TestCase): + def test_denormalize_depends_on_v3(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V3_0) == { + 'image': 'busybox', + 'command': 'true', + 'depends_on': ['service2', 'service3'] + } + + def test_denormalize_depends_on_v2_1(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V2_1) == service_dict From 931027c59828f242391de41cbeadfd6d1664588a Mon Sep 17 00:00:00 2001 From: muicoder Date: Mon, 16 Jan 2017 10:43:29 +0800 Subject: [PATCH 59/98] add IMAGE_EVENTS: load/save Signed-off-by: muicoder --- compose/const.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/const.py b/compose/const.py index 1b1be5c7..354c6d76 100644 --- a/compose/const.py +++ b/compose/const.py @@ -5,7 +5,7 @@ import sys DEFAULT_TIMEOUT = 10 HTTP_TIMEOUT = 60 -IMAGE_EVENTS = ['delete', 'import', 'pull', 'push', 'tag', 'untag'] +IMAGE_EVENTS = ['delete', 'import', 'load', 'pull', 'push', 'save', 'tag', 'untag'] IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' From 56a1b02aac33d09ec7761729a8d6ddcb0fbdea0e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 17 Jan 2017 13:22:16 -0800 Subject: [PATCH 60/98] Catch healthcheck exceptions in parallel_execute Signed-off-by: Joffrey F --- compose/parallel.py | 40 ++++++++++++++++++------------- tests/integration/project_test.py | 4 ++-- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/compose/parallel.py b/compose/parallel.py index b2654dcf..e495410c 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -12,6 +12,8 @@ from six.moves.queue import Empty from six.moves.queue import Queue from compose.cli.signals import ShutdownException +from compose.errors import HealthCheckFailed +from compose.errors import NoHealthCheckConfigured from compose.errors import OperationFailedError from compose.utils import get_output_stream @@ -48,7 +50,7 @@ def parallel_execute(objects, func, get_name, msg, get_deps=None): elif isinstance(exception, APIError): errors[get_name(obj)] = exception.explanation writer.write(get_name(obj), 'error') - elif isinstance(exception, OperationFailedError): + elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): errors[get_name(obj)] = exception.msg writer.write(get_name(obj), 'error') elif isinstance(exception, UpstreamError): @@ -164,21 +166,27 @@ def feed_queue(objects, func, get_deps, results, state): for obj in pending: deps = get_deps(obj) - - if any(dep[0] in state.failed for dep in deps): - log.debug('{} has upstream errors - not processing'.format(obj)) - results.put((obj, None, UpstreamError())) - state.failed.add(obj) - elif all( - dep not in objects or ( - dep in state.finished and (not ready_check or ready_check(dep)) - ) for dep, ready_check in deps - ): - log.debug('Starting producer thread for {}'.format(obj)) - t = Thread(target=producer, args=(obj, func, results)) - t.daemon = True - t.start() - state.started.add(obj) + try: + if any(dep[0] in state.failed for dep in deps): + log.debug('{} has upstream errors - not processing'.format(obj)) + results.put((obj, None, UpstreamError())) + state.failed.add(obj) + elif all( + dep not in objects or ( + dep in state.finished and (not ready_check or ready_check(dep)) + ) for dep, ready_check in deps + ): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=producer, args=(obj, func, results)) + t.daemon = True + t.start() + state.started.add(obj) + except (HealthCheckFailed, NoHealthCheckConfigured) as e: + log.debug( + 'Healthcheck for service(s) upstream of {} failed - ' + 'not processing'.format(obj) + ) + results.put((obj, None, e)) if state.is_done(): results.put(STOP) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index c5e3cf50..ee2b7817 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1443,7 +1443,7 @@ class ProjectTest(DockerClientTestCase): project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - with pytest.raises(HealthCheckFailed): + with pytest.raises(ProjectError): project.up() containers = project.containers() assert len(containers) == 1 @@ -1479,7 +1479,7 @@ class ProjectTest(DockerClientTestCase): project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - with pytest.raises(NoHealthCheckConfigured): + with pytest.raises(ProjectError): project.up() containers = project.containers() assert len(containers) == 1 From 1a02121ab55876f92bcceb62d1e81b7f114f0c79 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Jan 2017 17:52:03 -0800 Subject: [PATCH 61/98] depends_on merge now retains condition information when present Signed-off-by: Joffrey F --- compose/config/config.py | 6 +++++- tests/unit/config/config_test.py | 34 ++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index c11460fa..7e77421e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -818,6 +818,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('ulimits', parse_ulimits) md.merge_mapping('networks', parse_networks) md.merge_mapping('sysctls', parse_sysctls) + md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) for field in ['volumes', 'devices']: @@ -825,7 +826,7 @@ def merge_service_dicts(base, override, version): for field in [ 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', - 'security_opt', 'volumes_from', 'depends_on', + 'security_opt', 'volumes_from', ]: md.merge_field(field, merge_unique_items_lists, default=[]) @@ -920,6 +921,9 @@ parse_environment = functools.partial(parse_dict_or_list, split_env, 'environmen parse_labels = functools.partial(parse_dict_or_list, split_kv, 'labels') parse_networks = functools.partial(parse_dict_or_list, lambda k: (k, None), 'networks') parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') +parse_depends_on = functools.partial( + parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on' +) def parse_ulimits(ulimits): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ca7c6168..ab8bfcfc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1713,6 +1713,40 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_depends_on_no_override(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == base + + def test_merge_depends_on_mixed_syntax(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = { + 'depends_on': ['app3'] + } + + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'}, + 'app3': {'condition': 'service_started'} + } + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From 169289c8b66dcab760cdc7e1534fc0bece326d44 Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Fri, 20 Jan 2017 00:52:13 +0800 Subject: [PATCH 62/98] find a fishbone Signed-off-by: Aaron.L.Xu --- script/test/versions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test/versions.py b/script/test/versions.py index 45ead143..0c3b8162 100755 --- a/script/test/versions.py +++ b/script/test/versions.py @@ -5,7 +5,7 @@ version tags for recent releases, or the default release. The default release is the most recent non-RC version. -Recent is a list of unqiue major.minor versions, where each is the most +Recent is a list of unique major.minor versions, where each is the most recent version in the series. For example, if the list of versions is: From 1c46525c2baf8532434c320bf0443a520381431d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 14:47:31 -0800 Subject: [PATCH 63/98] 1.11.0dev Signed-off-by: Joffrey F --- CHANGELOG.md | 48 +++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- 2 files changed, 49 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c645ca2..6699f880 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,54 @@ Change log ========== +1.10.0 (2017-01-18) +------------------- + +### New Features + +#### Compose file version 3.0 + +- Introduced version 3.0 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.13 or above and is + specifically designed to work with the `docker stack` commands. + +#### Compose file version 2.1 and up + +- Healthcheck configuration can now be done in the service definition using + the `healthcheck` parameter + +- Containers dependencies can now be set up to wait on positive healthchecks + when declared using `depends_on`. See the documentation for the updated + syntax. + **Note:** This feature will not be ported to version 3 Compose files. + +- Added support for the `sysctls` parameter in service definitions + +- Added support for the `userns_mode` parameter in service definitions + +- Compose now adds identifying labels to networks and volumes it creates + +#### Compose file version 2.0 and up + +- Added support for the `stop_grace_period` option in service definitions. + +### Bugfixes + +- Colored output now works properly on Windows. + +- Fixed a bug where docker-compose run would fail to set up link aliases + in interactive mode on Windows. + +- Networks created by Compose are now always made attachable + (Compose files v2.1 and up). + +- Fixed a bug where falsy values of `COMPOSE_CONVERT_WINDOWS_PATHS` + (`0`, `false`, empty value) were being interpreted as true. + +- Fixed a bug where forward slashes in some .dockerignore patterns weren't + being parsed correctly on Windows + + 1.9.0 (2016-11-16) ----------------- diff --git a/compose/__init__.py b/compose/__init__.py index 6f05b282..38417836 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.10.0dev' +__version__ = '1.11.0dev' From 5c2165eaafbb625eb2058b199b571a228e86df03 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 15:41:31 -0800 Subject: [PATCH 64/98] Fix volume definition in v3 schema Signed-off-by: Joffrey F --- compose/config/config_schema_v3.0.json | 4 ++-- tests/acceptance/cli_test.py | 8 +++++++- tests/fixtures/v3-full/docker-compose.yml | 4 ++++ tests/integration/testcases.py | 7 +++++-- 4 files changed, 18 insertions(+), 5 deletions(-) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index ae4c0530..584b6ef5 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -330,9 +330,9 @@ "name": {"type": "string"} }, "additionalProperties": false - } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226..ce31dd18 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -295,7 +295,13 @@ class CLITestCase(DockerClientTestCase): assert yaml.load(result.stdout) == { 'version': '3.0', 'networks': {}, - 'volumes': {}, + 'volumes': { + 'foobar': { + 'labels': { + 'com.docker.compose.test': 'true', + }, + }, + }, 'services': { 'web': { 'image': 'busybox', diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml index b4d1b642..a1661ab9 100644 --- a/tests/fixtures/v3-full/docker-compose.yml +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -35,3 +35,7 @@ services: retries: 5 stop_grace_period: 20s +volumes: + foobar: + labels: + com.docker.compose.test: 'true' diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index f6bc402b..230bd2d9 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -13,6 +13,7 @@ from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 +from compose.config.config import V3_0 from compose.config.environment import Environment from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT @@ -36,13 +37,15 @@ def get_links(container): def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return V2_1 + return V3_0 version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 elif version_lt(version, '1.12'): return V2_0 - return V2_1 + elif version_lt(version, '1.13'): + return V2_1 + return V3_0 def build_version_required_decorator(ignored_versions): From d83d31889ea937524db798aa8260638036503764 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 19 Jan 2017 16:05:13 -0800 Subject: [PATCH 65/98] Remove external_name from volume def in config output Signed-off-by: Joffrey F --- compose/config/serialize.py | 7 ++++++- tests/acceptance/cli_test.py | 14 ++++++++++++++ tests/fixtures/volumes/docker-compose.yml | 2 ++ tests/fixtures/volumes/external-volumes.yml | 16 ++++++++++++++++ 4 files changed, 38 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/volumes/docker-compose.yml create mode 100644 tests/fixtures/volumes/external-volumes.yml diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 05ac0d60..9ea287a4 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -32,6 +32,11 @@ def denormalize_config(config): if 'external_name' in net_conf: del net_conf['external_name'] + volumes = config.volumes.copy() + for vol_name, vol_conf in volumes.items(): + if 'external_name' in vol_conf: + del vol_conf['external_name'] + version = config.version if version == V1: version = V2_1 @@ -40,7 +45,7 @@ def denormalize_config(config): 'version': version, 'services': services, 'networks': networks, - 'volumes': config.volumes, + 'volumes': volumes, } diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b9766226..287c043c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -262,6 +262,20 @@ class CLITestCase(DockerClientTestCase): } } + def test_config_external_volume(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True + }, + 'bar': { + 'external': {'name': 'some_bar'} + } + } + def test_config_v1(self): self.base_dir = 'tests/fixtures/v1-config' result = self.dispatch(['config']) diff --git a/tests/fixtures/volumes/docker-compose.yml b/tests/fixtures/volumes/docker-compose.yml new file mode 100644 index 00000000..da711ac4 --- /dev/null +++ b/tests/fixtures/volumes/docker-compose.yml @@ -0,0 +1,2 @@ +version: '2.1' +services: {} diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes.yml new file mode 100644 index 00000000..05c6c484 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes.yml @@ -0,0 +1,16 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar From 644e1716c33e0b3a3dcb4e5227b7dac0a289cffd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 20 Jan 2017 12:55:59 -0500 Subject: [PATCH 66/98] Add missing network.internal to v3 schema. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.0.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json index 584b6ef5..fbcd8bb8 100644 --- a/compose/config/config_schema_v3.0.json +++ b/compose/config/config_schema_v3.0.json @@ -308,6 +308,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false From 20d6f450b5e37e5c634fdf517df32d7484a2b3ff Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 20 Jan 2017 15:05:53 -0800 Subject: [PATCH 67/98] Don't encode build context path on Windows Signed-off-by: Joffrey F --- compose/service.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 20a40c68..724e0565 100644 --- a/compose/service.py +++ b/compose/service.py @@ -22,6 +22,7 @@ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT +from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF @@ -769,9 +770,9 @@ class Service(object): build_opts = self.options.get('build', {}) path = build_opts.get('context') - # python2 os.path() doesn't support unicode, so we need to encode it to - # a byte string - if not six.PY3: + # python2 os.stat() doesn't support unicode on some UNIX, so we + # encode it to a bytestring to be safe + if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') build_output = self.client.build( From e10d1140b95d33a589b1a971e0edda703bfb1e9b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 25 Jan 2017 18:00:09 -0800 Subject: [PATCH 68/98] Convert time data back to string values when serializing config Signed-off-by: Joffrey F --- compose/config/serialize.py | 29 +++++++++++++++++++++++++ tests/acceptance/cli_test.py | 4 ++-- tests/unit/config/config_test.py | 36 ++++++++++++++++++++++++++++++++ 3 files changed, 67 insertions(+), 2 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 9ea287a4..3745de82 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -57,6 +57,25 @@ def serialize_config(config): width=80) +def serialize_ns_time_value(value): + result = (value, 'ns') + table = [ + (1000., 'us'), + (1000., 'ms'), + (1000., 's'), + (60., 'm'), + (60., 'h') + ] + for stage in table: + tmp = value / stage[0] + if tmp == int(value / stage[0]): + value = tmp + result = (int(value), stage[1]) + else: + break + return '{0}{1}'.format(*result) + + def denormalize_service_dict(service_dict, version): service_dict = service_dict.copy() @@ -73,4 +92,14 @@ def denormalize_service_dict(service_dict, version): svc for svc in service_dict['depends_on'].keys() ]) + if 'healthcheck' in service_dict: + if 'interval' in service_dict['healthcheck']: + service_dict['healthcheck']['interval'] = serialize_ns_time_value( + service_dict['healthcheck']['interval'] + ) + if 'timeout' in service_dict['healthcheck']: + service_dict['healthcheck']['timeout'] = serialize_ns_time_value( + service_dict['healthcheck']['timeout'] + ) + return service_dict diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 41a06c95..58160c80 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -353,8 +353,8 @@ class CLITestCase(DockerClientTestCase): 'healthcheck': { 'test': 'cat /etc/passwd', - 'interval': 10000000000, - 'timeout': 1000000000, + 'interval': '10s', + 'timeout': '1s', 'retries': 5, }, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ab8bfcfc..d7947a4e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -23,6 +23,7 @@ from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION from compose.config.serialize import denormalize_service_dict +from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from compose.utils import nanoseconds_from_time_seconds @@ -3334,3 +3335,38 @@ class SerializeTest(unittest.TestCase): } assert denormalize_service_dict(service_dict, V2_1) == service_dict + + def test_serialize_time(self): + data = { + 9: '9ns', + 9000: '9us', + 9000000: '9ms', + 90000000: '90ms', + 900000000: '900ms', + 999999999: '999999999ns', + 1000000000: '1s', + 60000000000: '1m', + 60000000001: '60000000001ns', + 9000000000000: '150m', + 90000000000000: '25h', + } + + for k, v in data.items(): + assert serialize_ns_time_value(k) == v + + def test_denormalize_healthcheck(self): + service_dict = { + 'image': 'test', + 'healthcheck': { + 'test': 'exit 1', + 'interval': '1m40s', + 'timeout': '30s', + 'retries': 5 + } + } + processed_service = config.process_service(config.ServiceConfig( + '.', 'test', 'test', service_dict + )) + denormalized_service = denormalize_service_dict(processed_service, V2_1) + assert denormalized_service['healthcheck']['interval'] == '100s' + assert denormalized_service['healthcheck']['timeout'] == '30s' From 5895d8bbc9939524b449e296ac93d8e98aa70eb0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 24 Jan 2017 15:02:27 -0800 Subject: [PATCH 69/98] Detect conflicting version of the docker python SDK and prevent execution until issue is fixed Signed-off-by: Joffrey F --- compose/cli/main.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index c25ccbfa..db068272 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,6 +14,30 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # A regular import statement causes PyInstaller to freak out while + # trying to load pip. This way it is simply ignored. + pip = __import__('pip') + pip_packages = pip.get_installed_distributions() + if 'docker-py' in [pkg.project_name for pkg in pip_packages]: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) +except ImportError: + # pip is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass + + from . import errors from . import signals from .. import __version__ From 22249add84831a02976f6c98020f05eb7418b287 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 6 Jun 2016 15:48:47 -0700 Subject: [PATCH 70/98] Use newer version of PyInstaller to fix prelinking issues Signed-off-by: Joffrey F --- requirements-build.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-build.txt b/requirements-build.txt index 3f1dbd75..27f610ca 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.1.1 +pyinstaller==3.2.1 From 2593366a3ef1fb0673049687f0ca6733a28cf03f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 16:26:35 -0800 Subject: [PATCH 71/98] Bump docker SDK version Signed-off-by: Joffrey F --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 4b7c7b76..3b06bff4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,7 +2,7 @@ PyYAML==3.11 backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 colorama==0.3.7 -docker==2.0.1 +docker==2.0.2 dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4; python_version < '3.4' diff --git a/setup.py b/setup.py index 2f2ba742..0b1d4e08 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ install_requires = [ 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker >= 2.0.1, < 3.0', + 'docker >= 2.0.2, < 3.0', 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', From a82de8863ebdc586a45f54aef348cd17340089e3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Dec 2016 15:44:33 -0500 Subject: [PATCH 72/98] Add v3.1 with secrets. Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.1.json | 426 +++++++++++++++++++++++++ 1 file changed, 426 insertions(+) create mode 100644 compose/config/config_schema_v3.1.json diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json new file mode 100644 index 00000000..16616498 --- /dev/null +++ b/compose/config/config_schema_v3.1.json @@ -0,0 +1,426 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.1.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secrets" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "number"}, + "gid": {"type": "number"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_signal": {"type": "string"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": ["object", "null"], + "properties": { + "interval": {"type":"string"}, + "timeout": {"type":"string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "disable": {"type": "boolean"} + }, + "additionalProperties": false + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} From add56ce8182328fefd7fbe4500360a929ea511df Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Jan 2017 15:58:14 -0500 Subject: [PATCH 73/98] Read service secrets as a type. Signed-off-by: Daniel Nephin --- compose/config/config.py | 13 +++++++++++-- compose/config/config_schema_v3.1.json | 6 +++--- compose/config/types.py | 23 +++++++++++++++++++++-- compose/const.py | 3 +++ 4 files changed, 38 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7e77421e..3ca994a7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -12,10 +12,12 @@ import six import yaml from cached_property import cached_property +from . import types from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 from ..const import COMPOSEFILE_V2_1 as V2_1 from ..const import COMPOSEFILE_V3_0 as V3_0 +from ..const import COMPOSEFILE_V3_1 as V3_1 from ..utils import build_string_dict from ..utils import parse_nanoseconds_int from ..utils import splitdrive @@ -82,6 +84,7 @@ DOCKER_CONFIG_KEYS = [ 'privileged', 'read_only', 'restart', + 'secrets', 'security_opt', 'shm_size', 'stdin_open', @@ -202,8 +205,11 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def get_networks(self): return {} if self.version == V1 else self.config.get('networks', {}) + def get_secrets(self): + return {} if self.version < V3_1 else self.config.get('secrets', {}) -class Config(namedtuple('_Config', 'version services volumes networks')): + +class Config(namedtuple('_Config', 'version services volumes networks secrets')): """ :param version: configuration version :type version: int @@ -328,6 +334,8 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) + secrets = load_mapping( + config_details.config_files, 'get_secrets', 'Secrets') service_dicts = load_services(config_details, main_file) if main_file.version != V1: @@ -342,7 +350,7 @@ def load(config_details): "`docker stack deploy` to deploy to a swarm." .format(", ".join(sorted(s['name'] for s in services_using_deploy)))) - return Config(main_file.version, service_dicts, volumes, networks) + return Config(main_file.version, service_dicts, volumes, networks, secrets) def load_mapping(config_files, get_func, entity_type): @@ -820,6 +828,7 @@ def merge_service_dicts(base, override, version): md.merge_mapping('sysctls', parse_sysctls) md.merge_mapping('depends_on', parse_depends_on) md.merge_sequence('links', ServiceLink.parse) + md.merge_sequence('secrets', types.ServiceSecret.parse) for field in ['volumes', 'devices']: md.merge_field(field, merge_path_mappings) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index 16616498..c43f296b 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -46,7 +46,7 @@ "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "#/definitions/secrets" + "$ref": "#/definitions/secret" } }, "additionalProperties": false @@ -188,8 +188,8 @@ "properties": { "source": {"type": "string"}, "target": {"type": "string"}, - "uid": {"type": "number"}, - "gid": {"type": "number"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, "mode": {"type": "number"} } } diff --git a/compose/config/types.py b/compose/config/types.py index 4c106747..17d5c8b3 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -10,8 +10,8 @@ from collections import namedtuple import six -from compose.config.config import V1 -from compose.config.errors import ConfigurationError +from ..const import COMPOSEFILE_V1 as V1 +from .errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM from compose.utils import splitdrive @@ -234,3 +234,22 @@ class ServiceLink(namedtuple('_ServiceLink', 'target alias')): @property def merge_field(self): return self.alias + + +class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): + + @classmethod + def parse(cls, spec): + if isinstance(spec, six.string_types): + return cls(spec, None, None, None, None) + return cls( + spec.get('source'), + spec.get('target'), + spec.get('uid'), + spec.get('gid'), + spec.get('mode'), + ) + + @property + def merge_field(self): + return self.source diff --git a/compose/const.py b/compose/const.py index 1b1be5c7..0f2b00c4 100644 --- a/compose/const.py +++ b/compose/const.py @@ -20,12 +20,14 @@ COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' COMPOSEFILE_V3_0 = '3.0' +COMPOSEFILE_V3_1 = '3.1' API_VERSIONS = { COMPOSEFILE_V1: '1.21', COMPOSEFILE_V2_0: '1.22', COMPOSEFILE_V2_1: '1.24', COMPOSEFILE_V3_0: '1.25', + COMPOSEFILE_V3_1: '1.25', } API_VERSION_TO_ENGINE_VERSION = { @@ -33,4 +35,5 @@ API_VERSION_TO_ENGINE_VERSION = { API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', + API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', } From e0c6397999464dfe94f7e738dc36b2225f88972f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Jan 2017 17:18:04 -0500 Subject: [PATCH 74/98] Implement secrets using bind mounts Signed-off-by: Daniel Nephin --- compose/config/config.py | 48 +++++++++++++++++++++++++++----------- compose/const.py | 2 ++ compose/project.py | 27 +++++++++++++++++++++ compose/service.py | 23 +++++++++++++++--- tests/unit/bundle_test.py | 3 ++- tests/unit/project_test.py | 12 ++++++++++ 6 files changed, 98 insertions(+), 17 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3ca994a7..0e8b52e7 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -334,8 +334,7 @@ def load(config_details): networks = load_mapping( config_details.config_files, 'get_networks', 'Network' ) - secrets = load_mapping( - config_details.config_files, 'get_secrets', 'Secrets') + secrets = load_secrets(config_details.config_files, config_details.working_dir) service_dicts = load_services(config_details, main_file) if main_file.version != V1: @@ -364,22 +363,12 @@ def load_mapping(config_files, get_func, entity_type): external = config.get('external') if external: - if len(config.keys()) > 1: - raise ConfigurationError( - '{} {} declared as external but specifies' - ' additional attributes ({}). '.format( - entity_type, - name, - ', '.join([k for k in config.keys() if k != 'external']) - ) - ) + validate_external(entity_type, name, config) if isinstance(external, dict): config['external_name'] = external.get('name') else: config['external_name'] = name - mapping[name] = config - if 'driver_opts' in config: config['driver_opts'] = build_string_dict( config['driver_opts'] @@ -391,6 +380,39 @@ def load_mapping(config_files, get_func, entity_type): return mapping +def validate_external(entity_type, name, config): + if len(config.keys()) <= 1: + return + + raise ConfigurationError( + "{} {} declared as external but specifies additional attributes " + "({}).".format( + entity_type, name, ', '.join(k for k in config if k != 'external'))) + + +def load_secrets(config_files, working_dir): + mapping = {} + + for config_file in config_files: + for name, config in config_file.get_secrets().items(): + mapping[name] = config or {} + if not config: + continue + + external = config.get('external') + if external: + validate_external('Secret', name, config) + if isinstance(external, dict): + config['external_name'] = external.get('name') + else: + config['external_name'] = name + + if 'file' in config: + config['file'] = expand_path(working_dir, config['file']) + + return mapping + + def load_services(config_details, config_file): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( diff --git a/compose/const.py b/compose/const.py index 0f2b00c4..3f8f90ab 100644 --- a/compose/const.py +++ b/compose/const.py @@ -16,6 +16,8 @@ LABEL_VERSION = 'com.docker.compose.version' LABEL_VOLUME = 'com.docker.compose.volume' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +SECRETS_PATH = '/run/secrets' + COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' COMPOSEFILE_V2_1 = '2.1' diff --git a/compose/project.py b/compose/project.py index d99ef7c9..22576e86 100644 --- a/compose/project.py +++ b/compose/project.py @@ -104,6 +104,11 @@ class Project(object): for volume_spec in service_dict.get('volumes', []) ] + secrets = get_secrets( + service_dict['name'], + service_dict.get('secrets') or [], + config_data.secrets) + project.services.append( Service( service_dict.pop('name'), @@ -114,6 +119,7 @@ class Project(object): links=links, network_mode=network_mode, volumes_from=volumes_from, + secrets=secrets, **service_dict) ) @@ -553,6 +559,27 @@ def get_volumes_from(project, service_dict): return [build_volume_from(vf) for vf in volumes_from] +def get_secrets(service, service_secrets, secret_defs): + secrets = [] + + for secret in service_secrets: + secret_def = secret_defs.get(secret.source) + if not secret_def: + raise ConfigurationError( + "Service \"{service}\" uses an undefined secret \"{secret}\" " + .format(service=service, secret=secret.source)) + + if secret_def.get('external_name'): + log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " + "External secrets are not available to containers created by " + "docker-compose.".format(service=service, secret=secret.source)) + continue + + secrets.append({'secret': secret, 'file': secret_def.get('file')}) + + return secrets + + def warn_for_swarm_mode(client): info = client.info() if info.get('Swarm', {}).get('LocalNodeState') == 'active': diff --git a/compose/service.py b/compose/service.py index 724e0565..9f2fc68b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -17,6 +17,7 @@ from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from . import __version__ +from . import const from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment @@ -139,6 +140,7 @@ class Service(object): volumes_from=None, network_mode=None, networks=None, + secrets=None, **options ): self.name = name @@ -149,6 +151,7 @@ class Service(object): self.volumes_from = volumes_from or [] self.network_mode = network_mode or NetworkMode(None) self.networks = networks or {} + self.secrets = secrets or [] self.options = options def __repr__(self): @@ -692,9 +695,14 @@ class Service(object): override_options['binds'] = binds container_options['environment'].update(affinity) - if 'volumes' in container_options: - container_options['volumes'] = dict( - (v.internal, {}) for v in container_options['volumes']) + container_options['volumes'] = dict( + (v.internal, {}) for v in container_options.get('volumes') or {}) + + secret_volumes = self.get_secret_volumes() + if secret_volumes: + override_options['binds'].extend(v.repr() for v in secret_volumes) + container_options['volumes'].update( + (v.internal, {}) for v in secret_volumes) container_options['image'] = self.image_name @@ -765,6 +773,15 @@ class Service(object): return host_config + def get_secret_volumes(self): + def build_spec(secret): + target = '{}/{}'.format( + const.SECRETS_PATH, + secret['secret'].target or secret['secret'].source) + return VolumeSpec(secret['file'], target, 'ro') + + return [build_spec(secret) for secret in self.secrets] + def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py index a279cab0..21bdb31b 100644 --- a/tests/unit/bundle_test.py +++ b/tests/unit/bundle_test.py @@ -77,7 +77,8 @@ def test_to_bundle(): version=2, services=services, volumes={'special': {}}, - networks={'extra': {}}) + networks={'extra': {}}, + secrets={}) with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: output = bundle.to_bundle(config, image_digests) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 9a12438f..32d0adfa 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -36,6 +36,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ) project = Project.from_config( name='composetest', @@ -64,6 +65,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) @@ -170,6 +172,7 @@ class ProjectTest(unittest.TestCase): }], networks=None, volumes=None, + secrets=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] @@ -202,6 +205,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] @@ -227,6 +231,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) with mock.patch.object(Service, 'containers') as mock_return: @@ -360,6 +365,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') @@ -384,6 +390,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') @@ -417,6 +424,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) @@ -437,6 +445,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) @@ -457,6 +466,7 @@ class ProjectTest(unittest.TestCase): ], networks={'custom': {}}, volumes=None, + secrets=None, ), ) @@ -487,6 +497,7 @@ class ProjectTest(unittest.TestCase): }], networks=None, volumes=None, + secrets=None, ), ) self.assertEqual([c.id for c in project.containers()], ['1']) @@ -503,6 +514,7 @@ class ProjectTest(unittest.TestCase): }], networks={'default': {}}, volumes={'data': {}}, + secrets=None, ), ) self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops') From 4053adc7d356270143f8389d41f857a128d9febb Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Jan 2017 14:54:35 -0500 Subject: [PATCH 75/98] Add an integration test for secrets using bind mounts. Signed-off-by: Daniel Nephin --- compose/project.py | 2 +- tests/fixtures/secrets/default | 1 + tests/integration/project_test.py | 132 ++++++++++++++++++------------ tests/integration/testcases.py | 9 +- 4 files changed, 86 insertions(+), 58 deletions(-) create mode 100644 tests/fixtures/secrets/default diff --git a/compose/project.py b/compose/project.py index 22576e86..e522e2ec 100644 --- a/compose/project.py +++ b/compose/project.py @@ -106,7 +106,7 @@ class Project(object): secrets = get_secrets( service_dict['name'], - service_dict.get('secrets') or [], + service_dict.pop('secrets', None) or [], config_data.secrets) project.services.append( diff --git a/tests/fixtures/secrets/default b/tests/fixtures/secrets/default new file mode 100644 index 00000000..f9dc2014 --- /dev/null +++ b/tests/fixtures/secrets/default @@ -0,0 +1 @@ +This is the secret diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ee2b7817..30b107e8 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import os.path import random import py @@ -8,12 +9,14 @@ import pytest from docker.errors import NotFound from .. import mock -from ..helpers import build_config +from ..helpers import build_config as load_config from .testcases import DockerClientTestCase from compose.config import config from compose.config import ConfigurationError +from compose.config import types from compose.config.config import V2_0 from compose.config.config import V2_1 +from compose.config.config import V3_1 from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT @@ -26,6 +29,16 @@ from compose.project import ProjectError from compose.service import ConvergenceStrategy from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only +from tests.integration.testcases import v3_only + + +def build_config(**kwargs): + return config.Config( + version=kwargs.get('version'), + services=kwargs.get('services'), + volumes=kwargs.get('volumes'), + networks=kwargs.get('networks'), + secrets=kwargs.get('secrets')) class ProjectTest(DockerClientTestCase): @@ -70,7 +83,7 @@ class ProjectTest(DockerClientTestCase): def test_volumes_from_service(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'data': { 'image': 'busybox:latest', 'volumes': ['/var/data'], @@ -96,7 +109,7 @@ class ProjectTest(DockerClientTestCase): ) project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'db': { 'image': 'busybox:latest', 'volumes_from': ['composetest_data_container'], @@ -112,7 +125,7 @@ class ProjectTest(DockerClientTestCase): project = Project.from_config( name='composetest', client=self.client, - config_data=build_config({ + config_data=load_config({ 'version': V2_0, 'services': { 'net': { @@ -139,7 +152,7 @@ class ProjectTest(DockerClientTestCase): def get_project(): return Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'version': V2_0, 'services': { 'web': { @@ -174,7 +187,7 @@ class ProjectTest(DockerClientTestCase): def test_net_from_service_v1(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'net': { 'image': 'busybox:latest', 'command': ["top"] @@ -198,7 +211,7 @@ class ProjectTest(DockerClientTestCase): def get_project(): return Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'web': { 'image': 'busybox:latest', 'net': 'container:composetest_net_container' @@ -469,7 +482,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_starts_depends(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -504,7 +517,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_with_no_deps(self): project = Project.from_config( name='composetest', - config_data=build_config({ + config_data=load_config({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -564,7 +577,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_project_up_networks(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -576,7 +589,6 @@ class ProjectTest(DockerClientTestCase): 'baz': {'aliases': ['extra']}, }, }], - volumes={}, networks={ 'foo': {'driver': 'bridge'}, 'bar': {'driver': None}, @@ -610,14 +622,13 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_ipam_config(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'networks': {'front': None}, }], - volumes={}, networks={ 'front': { 'driver': 'bridge', @@ -671,7 +682,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_network_static_addresses(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -684,7 +695,6 @@ class ProjectTest(DockerClientTestCase): } }, }], - volumes={}, networks={ 'static_test': { 'driver': 'bridge', @@ -726,7 +736,7 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_enable_ipv6(self): self.require_api_version('1.23') - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -738,7 +748,6 @@ class ProjectTest(DockerClientTestCase): } }, }], - volumes={}, networks={ 'static_test': { 'driver': 'bridge', @@ -770,7 +779,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_network_static_addresses_missing_subnet(self): - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -782,7 +791,6 @@ class ProjectTest(DockerClientTestCase): } }, }], - volumes={}, networks={ 'static_test': { 'driver': 'bridge', @@ -807,7 +815,7 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_network_link_local_ips(self): - config_data = config.Config( + config_data = build_config( version=V2_1, services=[{ 'name': 'web', @@ -818,7 +826,6 @@ class ProjectTest(DockerClientTestCase): } } }], - volumes={}, networks={ 'linklocaltest': {'driver': 'bridge'} } @@ -844,15 +851,13 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_isolation(self): self.require_api_version('1.24') - config_data = config.Config( + config_data = build_config( version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', 'isolation': 'default' }], - volumes={}, - networks={} ) project = Project.from_config( client=self.client, @@ -866,15 +871,13 @@ class ProjectTest(DockerClientTestCase): @v2_1_only() def test_up_with_invalid_isolation(self): self.require_api_version('1.24') - config_data = config.Config( + config_data = build_config( version=V2_1, services=[{ 'name': 'web', 'image': 'busybox:latest', 'isolation': 'foobar' }], - volumes={}, - networks={} ) project = Project.from_config( client=self.client, @@ -887,14 +890,13 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_project_up_with_network_internal(self): self.require_api_version('1.23') - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'networks': {'internal': None}, }], - volumes={}, networks={ 'internal': {'driver': 'bridge', 'internal': True}, }, @@ -917,14 +919,13 @@ class ProjectTest(DockerClientTestCase): network_name = 'network_with_label' - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'networks': {network_name: None} }], - volumes={}, networks={ network_name: {'labels': {'label_key': 'label_val'}} } @@ -951,7 +952,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -959,7 +960,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( @@ -979,7 +979,7 @@ class ProjectTest(DockerClientTestCase): volume_name = 'volume_with_label' - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -993,7 +993,6 @@ class ProjectTest(DockerClientTestCase): } } }, - networks={}, ) project = Project.from_config( @@ -1106,7 +1105,7 @@ class ProjectTest(DockerClientTestCase): def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1114,7 +1113,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {}}, - networks={}, ) project = Project.from_config( @@ -1124,14 +1122,14 @@ class ProjectTest(DockerClientTestCase): project.volumes.initialize() volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) - self.assertEqual(volume_data['Driver'], 'local') + assert volume_data['Name'] == full_vol_name + assert volume_data['Driver'] == 'local' @v2_only() def test_project_up_implicit_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1139,7 +1137,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {}}, - networks={}, ) project = Project.from_config( @@ -1152,11 +1149,45 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v3_only() + def test_project_up_with_secrets(self): + config_data = build_config( + version=V3_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'cat /run/secrets/special', + 'secrets': [ + types.ServiceSecret.parse({'source': 'super', 'target': 'special'}), + ], + }], + secrets={ + 'super': { + 'file': os.path.abspath('tests/fixtures/secrets/default'), + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + project.stop() + + containers = project.containers(stopped=True) + assert len(containers) == 1 + container, = containers + + output = container.logs() + assert output == "This is the secret\n" + @v2_only() def test_initialize_volumes_invalid_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1164,7 +1195,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'foobar'}}, - networks={}, ) project = Project.from_config( @@ -1179,7 +1209,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1187,7 +1217,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( name='composetest', @@ -1218,7 +1247,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1226,7 +1255,6 @@ class ProjectTest(DockerClientTestCase): 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( name='composetest', @@ -1257,7 +1285,7 @@ class ProjectTest(DockerClientTestCase): vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) self.client.create_volume(vol_name) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1267,7 +1295,6 @@ class ProjectTest(DockerClientTestCase): volumes={ vol_name: {'external': True, 'external_name': vol_name} }, - networks=None, ) project = Project.from_config( name='composetest', @@ -1282,7 +1309,7 @@ class ProjectTest(DockerClientTestCase): def test_initialize_volumes_inexistent_external_volume(self): vol_name = '{0:x}'.format(random.getrandbits(32)) - config_data = config.Config( + config_data = build_config( version=V2_0, services=[{ 'name': 'web', @@ -1292,7 +1319,6 @@ class ProjectTest(DockerClientTestCase): volumes={ vol_name: {'external': True, 'external_name': vol_name} }, - networks=None, ) project = Project.from_config( name='composetest', @@ -1349,7 +1375,7 @@ class ProjectTest(DockerClientTestCase): } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1357,7 +1383,7 @@ class ProjectTest(DockerClientTestCase): config_dict['service2'] = config_dict['service1'] del config_dict['service1'] - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 230bd2d9..efc1551b 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -41,9 +41,9 @@ def engine_max_version(): version = os.environ['DOCKER_VERSION'].partition('-')[0] if version_lt(version, '1.10'): return V1 - elif version_lt(version, '1.12'): + if version_lt(version, '1.12'): return V2_0 - elif version_lt(version, '1.13'): + if version_lt(version, '1.13'): return V2_1 return V3_0 @@ -52,8 +52,9 @@ def build_version_required_decorator(ignored_versions): def decorator(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): - if engine_max_version() in ignored_versions: - skip("Engine version is too low") + max_version = engine_max_version() + if max_version in ignored_versions: + skip("Engine version %s is too low" % max_version) return return f(self, *args, **kwargs) return wrapper From 0d609b68acd12948f181200b3dac85b24c9e1441 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Jan 2017 15:00:33 -0500 Subject: [PATCH 76/98] Add a warning for unsupported secret fields. Signed-off-by: Daniel Nephin --- compose/project.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/project.py b/compose/project.py index e522e2ec..0330ab80 100644 --- a/compose/project.py +++ b/compose/project.py @@ -575,6 +575,12 @@ def get_secrets(service, service_secrets, secret_defs): "docker-compose.".format(service=service, secret=secret.source)) continue + if secret.uid or secret.gid or secret.mode: + log.warn("Service \"{service}\" uses secret \"{secret}\" with uid, " + "gid, or mode. These fields are not supported by this " + "implementation of the Compose file".format( + service=service, secret=secret.source)) + secrets.append({'secret': secret, 'file': secret_def.get('file')}) return secrets From 3a2735abb933fc8f067e888e6009eac9e2be3132 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 20 Jan 2017 17:10:23 -0500 Subject: [PATCH 77/98] Rebase compose v3.1 on the latest v3 Signed-off-by: Daniel Nephin --- compose/config/config_schema_v3.1.json | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json index c43f296b..b7037485 100644 --- a/compose/config/config_schema_v3.1.json +++ b/compose/config/config_schema_v3.1.json @@ -198,8 +198,8 @@ }, "sysctls": {"$ref": "#/definitions/list_or_dict"}, "stdin_open": {"type": "boolean"}, - "stop_signal": {"type": "string"}, "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { @@ -231,10 +231,11 @@ "healthcheck": { "id": "#/definitions/healthcheck", - "type": ["object", "null"], + "type": "object", + "additionalProperties": false, "properties": { - "interval": {"type":"string"}, - "timeout": {"type":"string"}, + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, "retries": {"type": "number"}, "test": { "oneOf": [ @@ -242,9 +243,8 @@ {"type": "array", "items": {"type": "string"}} ] }, - "disable": {"type": "boolean"} - }, - "additionalProperties": false + "timeout": {"type": "string"} + } }, "deployment": { "id": "#/definitions/deployment", @@ -337,6 +337,7 @@ }, "additionalProperties": false }, + "internal": {"type": "boolean"}, "labels": {"$ref": "#/definitions/list_or_dict"} }, "additionalProperties": false @@ -357,10 +358,11 @@ "type": ["boolean", "object"], "properties": { "name": {"type": "string"} - } - } + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, @@ -374,9 +376,9 @@ "properties": { "name": {"type": "string"} } - } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} }, - "labels": {"$ref": "#/definitions/list_or_dict"}, "additionalProperties": false }, From 59d1847d9bc88f9b4248267e93fe0435ce973da9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Jan 2017 12:51:05 -0500 Subject: [PATCH 78/98] Fix some test failures. Signed-off-by: Daniel Nephin --- tests/integration/project_test.py | 37 +++++++++++++++++++++++++++---- 1 file changed, 33 insertions(+), 4 deletions(-) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 30b107e8..28762cd2 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1151,6 +1151,8 @@ class ProjectTest(DockerClientTestCase): @v3_only() def test_project_up_with_secrets(self): + create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) + config_data = build_config( version=V3_1, services=[{ @@ -1181,7 +1183,7 @@ class ProjectTest(DockerClientTestCase): container, = containers output = container.logs() - assert output == "This is the secret\n" + assert output == b"This is the secret\n" @v2_only() def test_initialize_volumes_invalid_volume_driver(self): @@ -1428,7 +1430,7 @@ class ProjectTest(DockerClientTestCase): } } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1465,7 +1467,7 @@ class ProjectTest(DockerClientTestCase): } } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1501,7 +1503,7 @@ class ProjectTest(DockerClientTestCase): } } } - config_data = build_config(config_dict) + config_data = load_config(config_dict) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) @@ -1515,3 +1517,30 @@ class ProjectTest(DockerClientTestCase): assert 'svc1' in svc2.get_dependency_names() with pytest.raises(NoHealthCheckConfigured): svc1.is_healthy() + + +def create_host_file(client, filename): + dirname = os.path.dirname(filename) + + with open(filename, 'r') as fh: + content = fh.read() + + container = client.create_container( + 'busybox:latest', + ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], + volumes={dirname: {}}, + host_config=client.create_host_config( + binds={dirname: {'bind': dirname, 'ro': False}}, + network_mode='none', + ), + ) + try: + client.start(container) + exitcode = client.wait(container) + + if exitcode != 0: + output = client.logs(container) + raise Exception( + "Container exited with code {}:\n{}".format(exitcode, output)) + finally: + client.remove_container(container, force=True) From 8efb7e6e8bbd1542db768fc1b90c6c7282f0944b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 31 Jan 2017 12:51:46 -0800 Subject: [PATCH 79/98] Don't strip ANSI color codes when output is not a TTY Signed-off-by: Joffrey F --- compose/cli/colors.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/cli/colors.py b/compose/cli/colors.py index 6677a376..f1251e43 100644 --- a/compose/cli/colors.py +++ b/compose/cli/colors.py @@ -33,7 +33,7 @@ def make_color_fn(code): return lambda s: ansi_color(code, s) -colorama.init() +colorama.init(strip=False) for (name, code) in get_pairs(): globals()[name] = make_color_fn(code) From 84774cacd210bb176c2daf73106c7dc849a6a0d7 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 15:16:09 -0800 Subject: [PATCH 80/98] Upgrade python and pip versions in Dockerfile Add libbz2 dependency Signed-off-by: Joffrey F --- Dockerfile | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/Dockerfile b/Dockerfile index 63fac3eb..a03e1510 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN set -ex; \ ca-certificates \ curl \ libsqlite3-dev \ + libbz2-dev \ ; \ rm -rf /var/lib/apt/lists/* @@ -20,40 +21,32 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker -# Build Python 2.7.9 from source +# Build Python 2.7.13 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \ - cd Python-2.7.9; \ + curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + cd Python-2.7.13; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.9 + rm -rf /Python-2.7.13 # Build python 3.4 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \ - cd Python-3.4.3; \ + curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + cd Python-3.4.6; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.3 + rm -rf /Python-3.4.6 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib -# Install setuptools -RUN set -ex; \ - curl -L https://bootstrap.pypa.io/ez_setup.py | python - # Install pip RUN set -ex; \ - curl -L https://pypi.python.org/packages/source/p/pip/pip-8.1.1.tar.gz | tar -xz; \ - cd pip-8.1.1; \ - python setup.py install; \ - cd ..; \ - rm -rf pip-8.1.1 + curl -L https://bootstrap.pypa.io/get-pip.py | python # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen From a67500ee5728032aa902a640014bc35b6ab4d715 Mon Sep 17 00:00:00 2001 From: Peter Urda Date: Fri, 14 Oct 2016 00:44:52 -0700 Subject: [PATCH 81/98] Added `top` to `docker-compose` to display running processes. This commit allows `docker-compose` to access `top` for containers much like running `docker top` directly on a given container. This commit includes: * `docker-compose` CLI changes to expose `top` * Completions for `bash` and `zsh` * Required testing for the new `top` command Signed-off-by: Peter Urda --- compose/cli/main.py | 28 ++++++++++++++++++++++++++ contrib/completion/bash/docker-compose | 13 ++++++++++++ contrib/completion/zsh/_docker-compose | 5 +++++ tests/acceptance/cli_test.py | 20 ++++++++++++++++++ tests/fixtures/top/docker-compose.yml | 6 ++++++ 5 files changed, 72 insertions(+) create mode 100644 tests/fixtures/top/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index db068272..e2ebce48 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -215,6 +215,7 @@ class TopLevelCommand(object): scale Set number of containers for a service start Start services stop Stop services + top Display the running processes unpause Unpause services up Create and start containers version Show the Docker-Compose version information @@ -800,6 +801,33 @@ class TopLevelCommand(object): containers = self.project.restart(service_names=options['SERVICE'], timeout=timeout) exit_if(not containers, 'No containers to restart', 1) + def top(self, options): + """ + Display the running processes + + Usage: top [SERVICE...] + + """ + containers = sorted( + self.project.containers(service_names=options['SERVICE'], stopped=False) + + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), + key=attrgetter('name') + ) + + for idx, container in enumerate(containers): + if idx > 0: + print() + + top_data = self.project.client.top(container.name) + headers = top_data.get("Titles") + rows = [] + + for process in top_data.get("Processes", []): + rows.append(process) + + print(container.name) + print(Formatter().table(headers, rows)) + def unpause(self, options): """ Unpause services. diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 991f6572..77d02b42 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -434,6 +434,18 @@ _docker_compose_stop() { } +_docker_compose_top() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + __docker_compose_services_running + ;; + esac +} + + _docker_compose_unpause() { case "$cur" in -*) @@ -499,6 +511,7 @@ _docker_compose() { scale start stop + top unpause up version diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index ceb7d0f5..66d924f7 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -341,6 +341,11 @@ __docker-compose_subcommand() { $opts_timeout \ '*:running services:__docker-compose_runningservices' && ret=0 ;; + (top) + _arguments \ + $opts_help \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; (unpause) _arguments \ $opts_help \ diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 58160c80..160e1913 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1907,3 +1907,23 @@ class CLITestCase(DockerClientTestCase): "BAZ=2", ]) self.assertTrue(expected_env <= set(web.get('Config.Env'))) + + def test_top_services_not_running(self): + self.base_dir = 'tests/fixtures/top' + result = self.dispatch(['top']) + assert len(result.stdout) == 0 + + def test_top_services_running(self): + self.base_dir = 'tests/fixtures/top' + self.dispatch(['up', '-d']) + result = self.dispatch(['top']) + + self.assertIn('top_service_a', result.stdout) + self.assertIn('top_service_b', result.stdout) + self.assertNotIn('top_not_a_service', result.stdout) + + def test_top_processes_running(self): + self.base_dir = 'tests/fixtures/top' + self.dispatch(['up', '-d']) + result = self.dispatch(['top']) + assert result.stdout.count("top") == 4 diff --git a/tests/fixtures/top/docker-compose.yml b/tests/fixtures/top/docker-compose.yml new file mode 100644 index 00000000..d632a836 --- /dev/null +++ b/tests/fixtures/top/docker-compose.yml @@ -0,0 +1,6 @@ +service_a: + image: busybox:latest + command: top +service_b: + image: busybox:latest + command: top From 7e8958e6cab8edbfabd732e29a1c01b375a8bc02 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 1 Feb 2017 15:43:20 -0800 Subject: [PATCH 82/98] Add missing comma in DOCKER_CONFIG_KEYS list Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 0e8b52e7..746f63d5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -78,7 +78,7 @@ DOCKER_CONFIG_KEYS = [ 'memswap_limit', 'mem_swappiness', 'net', - 'oom_score_adj' + 'oom_score_adj', 'pid', 'ports', 'privileged', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d7947a4e..860e5835 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1748,6 +1748,24 @@ class ConfigTest(unittest.TestCase): } } + def test_merge_pid(self): + # Regression: https://github.com/docker/compose/issues/4184 + base = { + 'image': 'busybox', + 'pid': 'host' + } + + override = { + 'labels': {'com.docker.compose.test': 'yes'} + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'busybox', + 'pid': 'host', + 'labels': {'com.docker.compose.test': 'yes'} + } + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', From cf43e6edf7c734c3a98306bf9b4a01eb7f516005 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 2 Feb 2017 14:22:50 -0800 Subject: [PATCH 83/98] Don't re-parse healthcheck values coming from extended services Signed-off-by: Joffrey F --- compose/config/config.py | 10 ++++++++-- tests/fixtures/extends/healthcheck-1.yml | 9 +++++++++ tests/fixtures/extends/healthcheck-2.yml | 6 ++++++ tests/unit/config/config_test.py | 13 +++++++++++++ 4 files changed, 36 insertions(+), 2 deletions(-) create mode 100644 tests/fixtures/extends/healthcheck-1.yml create mode 100644 tests/fixtures/extends/healthcheck-2.yml diff --git a/compose/config/config.py b/compose/config/config.py index 0e8b52e7..63ee25ab 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -716,9 +716,15 @@ def process_healthcheck(service_dict, service_name): hc['test'] = raw['test'] if 'interval' in raw: - hc['interval'] = parse_nanoseconds_int(raw['interval']) + if not isinstance(raw['interval'], six.integer_types): + hc['interval'] = parse_nanoseconds_int(raw['interval']) + else: # Conversion has been done previously + hc['interval'] = raw['interval'] if 'timeout' in raw: - hc['timeout'] = parse_nanoseconds_int(raw['timeout']) + if not isinstance(raw['timeout'], six.integer_types): + hc['timeout'] = parse_nanoseconds_int(raw['timeout']) + else: # Conversion has been done previously + hc['timeout'] = raw['timeout'] if 'retries' in raw: hc['retries'] = raw['retries'] diff --git a/tests/fixtures/extends/healthcheck-1.yml b/tests/fixtures/extends/healthcheck-1.yml new file mode 100644 index 00000000..4c311e62 --- /dev/null +++ b/tests/fixtures/extends/healthcheck-1.yml @@ -0,0 +1,9 @@ +version: '2.1' +services: + demo: + image: foobar:latest + healthcheck: + test: ["CMD", "/health.sh"] + interval: 10s + timeout: 5s + retries: 36 diff --git a/tests/fixtures/extends/healthcheck-2.yml b/tests/fixtures/extends/healthcheck-2.yml new file mode 100644 index 00000000..11bc9f09 --- /dev/null +++ b/tests/fixtures/extends/healthcheck-2.yml @@ -0,0 +1,6 @@ +version: '2.1' +services: + demo: + extends: + file: healthcheck-1.yml + service: demo diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d7947a4e..a3be6df8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -3098,6 +3098,19 @@ class ExtendsTest(unittest.TestCase): 'other': {'condition': 'service_started'} } + def test_extends_with_healthcheck(self): + service_dicts = load_from_filename('tests/fixtures/extends/healthcheck-2.yml') + assert service_sort(service_dicts) == [{ + 'name': 'demo', + 'image': 'foobar:latest', + 'healthcheck': { + 'test': ['CMD', '/health.sh'], + 'interval': 10000000000, + 'timeout': 5000000000, + 'retries': 36, + } + }] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From a3a9d8944a413897ae1f0305d16d6d1071487ad6 Mon Sep 17 00:00:00 2001 From: Kevin Jing Qiu Date: Thu, 26 Jan 2017 14:23:12 -0500 Subject: [PATCH 84/98] Close the open file handle using context manager Signed-off-by: Kevin Jing Qiu --- compose/config/environment.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/compose/config/environment.py b/compose/config/environment.py index 7b926930..4ba228c8 100644 --- a/compose/config/environment.py +++ b/compose/config/environment.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import contextlib import logging import os @@ -31,11 +32,12 @@ def env_vars_from_file(filename): elif not os.path.isfile(filename): raise ConfigurationError("%s is not a file." % (filename)) env = {} - for line in codecs.open(filename, 'r', 'utf-8'): - line = line.strip() - if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v + with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + for line in fileobj: + line = line.strip() + if line and not line.startswith('#'): + k, v = split_env(line) + env[k] = v return env From b392b6e12ed724ce39e0e65fd5582b70c47803af Mon Sep 17 00:00:00 2001 From: fate-grand-order Date: Tue, 7 Feb 2017 15:59:34 +0800 Subject: [PATCH 85/98] fix typo in CHANGELOG.md Signed-off-by: fate-grand-order --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7c645ca2..e969b453 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -525,7 +525,7 @@ Bug Fixes: if at least one container is using the network. - When printings logs during `up` or `logs`, flush the output buffer after - each line to prevent buffering issues from hideing logs. + each line to prevent buffering issues from hiding logs. - Recreate a container if one of its dependencies is being created. Previously a container was only recreated if it's dependencies already From f0835268296111cac54faa8701b7aba751c0a239 Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Wed, 8 Feb 2017 18:50:14 +0800 Subject: [PATCH 86/98] referencing right segment of code Signed-off-by: Aaron.L.Xu --- compose/bundle.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/bundle.py b/compose/bundle.py index 854cc799..505ce91f 100644 --- a/compose/bundle.py +++ b/compose/bundle.py @@ -202,7 +202,7 @@ def convert_service_to_bundle(name, service_dict, image_digest): return container_config -# See https://github.com/docker/swarmkit/blob//agent/exec/container/container.go#L95 +# See https://github.com/docker/swarmkit/blob/agent/exec/container/container.go#L95 def set_command_and_args(config, entrypoint, command): if isinstance(entrypoint, six.string_types): entrypoint = split_command(entrypoint) From 979a0d53f7e989f13dc77865c7d1f4775f97319e Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 3 Feb 2017 15:26:34 -0800 Subject: [PATCH 87/98] Bump 1.11.0-rc1 Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index 38417836..ae8d759d 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0dev' +__version__ = '1.11.0-rc1' diff --git a/script/run/run.sh b/script/run/run.sh index 5872b081..9de11d5f 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.8.0" +VERSION="1.11.0-rc1" IMAGE="docker/compose:$VERSION" From 01d1895a350c445eab9deb37d1cd8b6fe3328b47 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 11:30:29 -0800 Subject: [PATCH 88/98] Bump 1.11.0 Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index ae8d759d..d7468af2 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0-rc1' +__version__ = '1.11.0' diff --git a/script/run/run.sh b/script/run/run.sh index 9de11d5f..b45630f0 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.11.0-rc1" +VERSION="1.11.0" IMAGE="docker/compose:$VERSION" From 2cd6cb9a47b2d00cfad7d49a26641020f7f8a66a Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 30 Jan 2017 16:20:59 -0800 Subject: [PATCH 89/98] Bump 1.10.1 Signed-off-by: Joffrey F --- CHANGELOG.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6699f880..d0681e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,53 @@ Change log ========== +1.11.0 (2017-02-08) +------------------- + +### New Features + +#### Compose file version 3.1 + +- Introduced version 3.1 of the `docker-compose.yml` specification. This + version requires Docker Engine 1.13.0 or above. It introduces support + for secrets. See the documentation for more information + +#### Compose file version 2.0 and up + +- Introduced the `docker-compose top` command that displays processes running + for the different services managed by Compose. + +### Bugfixes + +- Fixed a bug where extending a service defining a healthcheck dictionary + would cause `docker-compose` to error out. + +- Fixed an issue where the `pid` entry in a service definition was being + ignored when using multiple Compose files. + +1.10.1 (2017-02-01) +------------------ + +### Bugfixes + +- Fixed an issue where presence of older versions of the docker-py + package would cause unexpected crashes while running Compose + +- Fixed an issue where healthcheck dependencies would be lost when + using multiple compose files for a project + +- Fixed a few issues that made the output of the `config` command + invalid + +- Fixed an issue where adding volume labels to v3 Compose files would + result in an error + +- Fixed an issue on Windows where build context paths containing unicode + characters were being improperly encoded + +- Fixed a bug where Compose would occasionally crash while streaming logs + when containers would stop or restart + 1.10.0 (2017-01-18) ------------------- From fc7b74d7f900d6e94ebd46a05cdcadcfd1f7d407 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 8 Feb 2017 13:47:08 -0800 Subject: [PATCH 90/98] Bump to next dev version Signed-off-by: Joffrey F --- compose/__init__.py | 2 +- script/run/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index d7468af2..b2ca86f8 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.11.0' +__version__ = '1.12.0dev' diff --git a/script/run/run.sh b/script/run/run.sh index b45630f0..4e173894 100755 --- a/script/run/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.11.0" +VERSION="1.12.0dev" IMAGE="docker/compose:$VERSION" From 47e4442722373ce43f7878ee98fe9aceb1b9f177 Mon Sep 17 00:00:00 2001 From: kevinetc123 Date: Thu, 9 Feb 2017 19:10:26 +0800 Subject: [PATCH 91/98] fix typo in project.py Signed-off-by: kevinetc123 --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 0330ab80..133071e7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -365,7 +365,7 @@ class Project(object): # TODO: get labels from the API v1.22 , see github issue 2618 try: - # this can fail if the conatiner has been removed + # this can fail if the container has been removed container = Container.from_id(self.client, event['id']) except APIError: continue From c092fa37de820e7d6dd20b30d2c4dec28f214dd3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 9 Feb 2017 10:27:06 -0500 Subject: [PATCH 92/98] Fix version 3.1 Signed-off-by: Daniel Nephin --- compose/config/config.py | 11 ++++------- docker-compose.spec | 5 +++++ tests/unit/config/config_test.py | 4 ++++ 3 files changed, 13 insertions(+), 7 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ae85674b..09a717be 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -186,11 +186,6 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): if version == '3': version = V3_0 - if version not in (V2_0, V2_1, V3_0): - raise ConfigurationError( - 'Version in "{}" is unsupported. {}' - .format(self.filename, VERSION_EXPLANATION)) - return version def get_service(self, name): @@ -479,7 +474,7 @@ def process_config_file(config_file, environment, service_name=None): 'service', environment) - if config_file.version in (V2_0, V2_1, V3_0): + if config_file.version in (V2_0, V2_1, V3_0, V3_1): processed_config = dict(config_file.config) processed_config['services'] = services processed_config['volumes'] = interpolate_config_section( @@ -495,7 +490,9 @@ def process_config_file(config_file, environment, service_name=None): elif config_file.version == V1: processed_config = services else: - raise Exception("Unsupported version: {}".format(repr(config_file.version))) + raise ConfigurationError( + 'Version in "{}" is unsupported. {}' + .format(config_file.filename, VERSION_EXPLANATION)) config_file = config_file._replace(config=processed_config) validate_against_config_schema(config_file) diff --git a/docker-compose.spec b/docker-compose.spec index ec5a2039..ef0e2593 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -37,6 +37,11 @@ exe = EXE(pyz, 'compose/config/config_schema_v3.0.json', 'DATA' ), + ( + 'compose/config/config_schema_v3.1.json', + 'compose/config/config_schema_v3.1.json', + 'DATA' + ), ( 'compose/GITSHA', 'compose/GITSHA', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 666b21f2..ef57bb57 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -19,6 +19,7 @@ from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.config import V2_1 from compose.config.config import V3_0 +from compose.config.config import V3_1 from compose.config.environment import Environment from compose.config.errors import ConfigurationError from compose.config.errors import VERSION_EXPLANATION @@ -168,6 +169,9 @@ class ConfigTest(unittest.TestCase): cfg = config.load(build_config_details({'version': version})) assert cfg.version == V3_0 + cfg = config.load(build_config_details({'version': '3.1'})) + assert cfg.version == V3_1 + def test_v1_file_version(self): cfg = config.load(build_config_details({'web': {'image': 'busybox'}})) assert cfg.version == V1 From dc5b3f3b3eb53ce747003089c8ae5ff21f4b1f70 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 10 Feb 2017 17:05:33 -0500 Subject: [PATCH 93/98] Fix secrets config. Signed-off-by: Daniel Nephin --- compose/config/config.py | 5 ++ compose/config/types.py | 8 +++ setup.py | 10 ++-- tests/unit/config/config_test.py | 86 ++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 09a717be..4c9cf423 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -763,6 +763,11 @@ def finalize_service(service_config, service_names, version, environment): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + if 'secrets' in service_dict: + service_dict['secrets'] = [ + types.ServiceSecret.parse(s) for s in service_dict['secrets'] + ] + normalize_build(service_dict, service_config.working_dir, environment) service_dict['name'] = service_config.name diff --git a/compose/config/types.py b/compose/config/types.py index 17d5c8b3..f86c0319 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -253,3 +253,11 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): @property def merge_field(self): return self.source + + def repr(self): + return dict( + source=self.source, + target=self.target, + uid=self.uid, + gid=self.gid, + mode=self.mode) diff --git a/setup.py b/setup.py index 0b1d4e08..eafbc356 100644 --- a/setup.py +++ b/setup.py @@ -1,10 +1,10 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import +from __future__ import print_function from __future__ import unicode_literals import codecs -import logging import os import re import sys @@ -64,11 +64,9 @@ try: for key, value in extras_require.items(): if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): install_requires.extend(value) -except Exception: - logging.getLogger(__name__).exception( - 'Failed to compute platform dependencies. All dependencies will be ' - 'installed as a result.' - ) +except Exception as e: + print("Failed to compute platform dependencies: {}. ".format(e) + + "All dependencies will be installed as a result.", file=sys.stderr) for key, value in extras_require.items(): if key.startswith(':'): install_requires.extend(value) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ef57bb57..d4d1ad2c 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -13,6 +13,7 @@ import pytest from ...helpers import build_config_details from compose.config import config +from compose.config import types from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.config import V1 @@ -1849,6 +1850,91 @@ class ConfigTest(unittest.TestCase): config.load(config_details) assert 'has neither an image nor a build context' in exc.exconly() + def test_load_secrets(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': [ + 'one', + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + details = config.ConfigDetails('.', [base_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + + def test_load_secrets_multi_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': ['one'], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + override_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'secrets': [ + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + class NetworkModeTest(unittest.TestCase): def test_network_mode_standard(self): From 252699c1d124ba365e4b9078f4a44f3acbcb08be Mon Sep 17 00:00:00 2001 From: Petr Karmashev Date: Sun, 12 Feb 2017 02:03:04 +0300 Subject: [PATCH 94/98] Compose file reference link fix in README.md Signed-off-by: Petr Karmashev --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5cf69b05..35a10b90 100644 --- a/README.md +++ b/README.md @@ -35,7 +35,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file.md) +[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file/compose-versioning.md) Compose has commands for managing the whole lifecycle of your application: From abce83ef25528fb36979208888ba0033c89f47a3 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 13 Feb 2017 16:04:06 -0800 Subject: [PATCH 95/98] Fix `config` command output with service.secrets section Signed-off-by: Joffrey F --- compose/config/serialize.py | 3 ++ compose/config/types.py | 7 ++-- tests/unit/config/config_test.py | 57 ++++++++++++++++++++++++++++++++ 3 files changed, 62 insertions(+), 5 deletions(-) diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 3745de82..46d283f0 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -102,4 +102,7 @@ def denormalize_service_dict(service_dict, version): service_dict['healthcheck']['timeout'] ) + if 'secrets' in service_dict: + service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets']) + return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index f86c0319..811e6c1f 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -256,8 +256,5 @@ class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): def repr(self): return dict( - source=self.source, - target=self.target, - uid=self.uid, - gid=self.gid, - mode=self.mode) + [(k, v) for k, v in self._asdict().items() if v is not None] + ) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d4d1ad2c..c26272d9 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -54,6 +54,10 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) +def secret_sort(secrets): + return sorted(secrets, key=itemgetter('source')) + + class ConfigTest(unittest.TestCase): def test_load(self): service_dicts = config.load( @@ -1771,6 +1775,38 @@ class ConfigTest(unittest.TestCase): 'labels': {'com.docker.compose.test': 'yes'} } + def test_merge_different_secrets(self): + base = { + 'image': 'busybox', + 'secrets': [ + {'source': 'src.txt'} + ] + } + override = {'secrets': ['other-src.txt']} + + actual = config.merge_service_dicts(base, override, V3_1) + assert secret_sort(actual['secrets']) == secret_sort([ + {'source': 'src.txt'}, + {'source': 'other-src.txt'} + ]) + + def test_merge_secrets_override(self): + base = { + 'image': 'busybox', + 'secrets': ['src.txt'], + } + override = { + 'secrets': [ + { + 'source': 'src.txt', + 'target': 'data.txt', + 'mode': 0o400 + } + ] + } + actual = config.merge_service_dicts(base, override, V3_1) + assert actual['secrets'] == override['secrets'] + def test_external_volume_config(self): config_details = build_config_details({ 'version': '2', @@ -3491,3 +3527,24 @@ class SerializeTest(unittest.TestCase): denormalized_service = denormalize_service_dict(processed_service, V2_1) assert denormalized_service['healthcheck']['interval'] == '100s' assert denormalized_service['healthcheck']['timeout'] == '30s' + + def test_denormalize_secrets(self): + service_dict = { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + } + denormalized_service = denormalize_service_dict(service_dict, V3_1) + assert secret_sort(denormalized_service['secrets']) == secret_sort([ + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ]) From 66f4a795a2ded1a26b6cf8474edb423727dd585d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 15 Feb 2017 16:07:08 -0800 Subject: [PATCH 96/98] Don't import pip inside Compose Signed-off-by: Joffrey F --- compose/cli/__init__.py | 37 +++++++++++++++++++++++++++++++++++++ compose/cli/main.py | 24 ------------------------ 2 files changed, 37 insertions(+), 24 deletions(-) diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index e69de29b..c5db4455 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import subprocess +import sys + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # We don't try importing pip because it messes with package imports + # on some Linux distros (Ubuntu, Fedora) + # https://github.com/docker/compose/issues/4425 + # https://github.com/docker/compose/issues/4481 + # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py + s_cmd = subprocess.Popen( + ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + packages = s_cmd.communicate()[0].splitlines() + dockerpy_installed = len( + list(filter(lambda p: p.startswith(b'docker-py=='), packages)) + ) > 0 + if dockerpy_installed: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) + +except OSError: + # pip command is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass diff --git a/compose/cli/main.py b/compose/cli/main.py index e2ebce48..51ba36a0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -14,30 +14,6 @@ from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter - -# Attempt to detect https://github.com/docker/compose/issues/4344 -try: - # A regular import statement causes PyInstaller to freak out while - # trying to load pip. This way it is simply ignored. - pip = __import__('pip') - pip_packages = pip.get_installed_distributions() - if 'docker-py' in [pkg.project_name for pkg in pip_packages]: - from .colors import red - print( - red('ERROR:'), - "Dependency conflict: an older version of the 'docker-py' package " - "is polluting the namespace. " - "Run the following command to remedy the issue:\n" - "pip uninstall docker docker-py; pip install docker", - file=sys.stderr - ) - sys.exit(1) -except ImportError: - # pip is not available, which indicates it's probably the binary - # distribution of Compose which is not affected - pass - - from . import errors from . import signals from .. import __version__ From 27297fd1af64aae4e48fce254bace122014977fd Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Thu, 16 Feb 2017 11:14:25 +0800 Subject: [PATCH 97/98] fix a typo in script/release/utils.sh Signed-off-by: Aaron.L.Xu --- script/release/utils.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/utils.sh b/script/release/utils.sh index b4e5a2e6..321c1fb7 100644 --- a/script/release/utils.sh +++ b/script/release/utils.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Util functions for release scritps +# Util functions for release scripts # set -e From d20e3f334215a55cbebc27d8dde82108a58a0bae Mon Sep 17 00:00:00 2001 From: "Aaron.L.Xu" Date: Thu, 16 Feb 2017 15:25:04 +0800 Subject: [PATCH 98/98] function-name-modification for tests/* Signed-off-by: Aaron.L.Xu --- tests/acceptance/cli_test.py | 10 +++++----- tests/unit/cli_test.py | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 160e1913..8366ca75 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1234,7 +1234,7 @@ class CLITestCase(DockerClientTestCase): container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] self.assertEqual(user, container.get('Config.User')) - def test_run_service_with_environement_overridden(self): + def test_run_service_with_environment_overridden(self): name = 'service' self.base_dir = 'tests/fixtures/environment-composefile' self.dispatch([ @@ -1246,9 +1246,9 @@ class CLITestCase(DockerClientTestCase): ]) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] - # env overriden + # env overridden self.assertEqual('notbar', container.environment['foo']) - # keep environement from yaml + # keep environment from yaml self.assertEqual('world', container.environment['hello']) # added option from command line self.assertEqual('beta', container.environment['alpha']) @@ -1293,7 +1293,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[0], "0.0.0.0:49153") self.assertEqual(port_range[1], "0.0.0.0:49154") - def test_run_service_with_explicitly_maped_ports(self): + def test_run_service_with_explicitly_mapped_ports(self): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple']) @@ -1310,7 +1310,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "0.0.0.0:30000") self.assertEqual(port_full, "0.0.0.0:30001") - def test_run_service_with_explicitly_maped_ip_ports(self): + def test_run_service_with_explicitly_mapped_ip_ports(self): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch([ diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index f9b60bff..317650cb 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -184,7 +184,7 @@ class CLITestCase(unittest.TestCase): mock_client.create_host_config.call_args[1].get('restart_policy') ) - def test_command_manula_and_service_ports_together(self): + def test_command_manual_and_service_ports_together(self): project = Project.from_config( name='composetest', client=None,