From 338f2f4507919bda988be076f1654b4eb9dea497 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 15:14:04 -0400 Subject: [PATCH 001/359] Add a script to generate contributor list. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 11 +++++++---- script/release/contributors | 27 +++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 4 deletions(-) create mode 100755 script/release/contributors diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 85bbaf29..a7fea69e 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -82,17 +82,20 @@ When prompted build the non-linux binaries and test them. 5. Attach the binaries and `script/run.sh` -6. If everything looks good, it's time to push the release. +6. Add "Thanks" with a list of contributors. The contributor list can be generated + by running `./script/release/contributors`. + +7. If everything looks good, it's time to push the release. ./script/release/push-release -7. Publish the release on GitHub. +8. Publish the release on GitHub. -8. Check that both binaries download (following the install instructions) and run. +9. Check that both binaries download (following the install instructions) and run. -9. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. ## If it’s a stable release (not an RC) diff --git a/script/release/contributors b/script/release/contributors new file mode 100755 index 00000000..bb9fe871 --- /dev/null +++ b/script/release/contributors @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + + +function usage() { + >&2 cat << EOM +Print the list of github contributors for the release + +Usage: + + $0 +EOM + exit 1 +} + +[[ -n "$1" ]] || usage +PREV_RELEASE=$1 +VERSION=HEAD +URL="https://api.github.com/repos/docker/compose/compare" + +curl -sf "$URL/$PREV_RELEASE...$VERSION" | \ + jq -r '.commits[].author.login' | \ + sort | \ + uniq -c | \ + sort -nr | \ + awk '{print "@"$2","}' | \ + xargs echo From 58e6d4487abca4f0da7260fac6e2e6ed503d8d3b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 15:16:58 -0400 Subject: [PATCH 002/359] Fix some release docs. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index a7fea69e..ffa18077 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -54,7 +54,7 @@ When prompted build the non-linux binaries and test them. 2. Download the windows binary from AppVeyor - https://ci.appveyor.com/project/docker/compose/build//artifacts + https://ci.appveyor.com/project/docker/compose 3. Draft a release from the tag on GitHub (the script will open the window for you) @@ -93,7 +93,7 @@ When prompted build the non-linux binaries and test them. 8. Publish the release on GitHub. -9. Check that both binaries download (following the install instructions) and run. +9. Check that all the binaries download (following the install instructions) and run. 10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. From fbe8e769028cca49401101b0bfed88d6f838d186 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 14 Oct 2015 15:20:57 -0400 Subject: [PATCH 003/359] Add missing merge for release branch. Signed-off-by: Daniel Nephin --- script/release/make-branch | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/release/make-branch b/script/release/make-branch index dde1fb65..e2eae4d5 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -39,7 +39,7 @@ fi DEFAULT_REMOTE=release REMOTE="$(find_remote "$GITHUB_REPO")" -# If we don't have a docker origin add one +# If we don't have a docker remote add one if [ -z "$REMOTE" ]; then echo "Creating $DEFAULT_REMOTE remote" git remote add ${DEFAULT_REMOTE} ${GITHUB_REPO} @@ -55,6 +55,8 @@ read -n1 -r -p "Continue? (ctrl+c to cancel)" git fetch $REMOTE -p git checkout -b $BRANCH $BASE_VERSION +echo "Merging remote release branch into new release branch" +git merge --strategy=ours --no-edit $REMOTE/release # Store the release version for this branch in git, so that other release # scripts can use it From 7e59a7ea32e2eb8eace45d69c4f01689fbc81fab Mon Sep 17 00:00:00 2001 From: Tim Butler Date: Thu, 15 Oct 2015 17:09:57 +1000 Subject: [PATCH 004/359] Fix link to Release Process doc in README.md Signed-off-by: Tim Butler --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 3c776a71..d779d607 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ Want to help build Compose? Check out our [contributing documentation](https://g Releasing --------- -Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/RELEASE_PROCESS.md). +Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/project/RELEASE-PROCESS.md). From c6ff81fe302a6472037b9e8796544c3ce923314f Mon Sep 17 00:00:00 2001 From: Per Persson Date: Thu, 15 Oct 2015 15:13:27 +0200 Subject: [PATCH 005/359] Remove incorrectly placed comment I'm not sure if it should be there at all, but at least it should hardly be where it currently is located. Signed-off-by: Per Persson --- docs/compose-file.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 67322335..90730fec 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -48,8 +48,6 @@ See `man 7 capabilities` for a full list. - NET_ADMIN - SYS_ADMIN -Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. - ### command Override the default command. @@ -106,6 +104,8 @@ Compose will use an alternate file to build with. dockerfile: Dockerfile-alternate +Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. + ### env_file Add environment variables from a file. Can be a single value or a list. From ccfb6e6fa863614190b8290aab8fe6bfed5f0d06 Mon Sep 17 00:00:00 2001 From: Karol Duleba Date: Wed, 14 Oct 2015 19:06:05 +0100 Subject: [PATCH 006/359] Docs for shorthand notation of extends. Issue #1989 Signed-off-by: Karol Duleba --- docs/extends.md | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/docs/extends.md b/docs/extends.md index d88ce61c..f0b9e9ea 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -59,6 +59,10 @@ You can go further and define (or re-define) configuration locally in - DEBUG=1 cpu_shares: 5 + important_web: + extends: web + cpu_shares: 10 + You can also write other services and link your `web` service to them: web: @@ -233,7 +237,8 @@ manually keep both environments in sync. ### Reference You can use `extends` on any service together with other configuration keys. It -always expects a dictionary that should always contain the key: `service` and optionally the `file` key. +expects a dictionary that contains a `service` key and optionally a `file` key. +The `extends` key can also take a string, whose value is the name of a `service` defined in the same file. The `file` key specifies the location of a Compose configuration file defining the extension. The `file` value can be an absolute or relative path. If you From 53fc6d06f69fbe78b4b175487bd6371d323588c9 Mon Sep 17 00:00:00 2001 From: Cameron Eagans Date: Thu, 15 Oct 2015 16:58:27 -0600 Subject: [PATCH 007/359] docker-compose pull SERVICE should not pull SERVICE's dependencies Signed-off-by: Cameron Eagans --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 0e20a4ce..fdd70caf 100644 --- a/compose/project.py +++ b/compose/project.py @@ -335,7 +335,7 @@ class Project(object): return plans def pull(self, service_names=None, ignore_pull_failures=False): - for service in self.get_services(service_names, include_deps=True): + for service in self.get_services(service_names, include_deps=False): service.pull(ignore_pull_failures) def containers(self, service_names=None, stopped=False, one_off=False): From 1ed23fb2de1b3e44658ab7439763b879f439acc7 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 16 Oct 2015 11:58:27 +0530 Subject: [PATCH 008/359] Revert networking-related changes to getting started guides Signed-off-by: Aanand Prasad --- docs/django.md | 7 +++++-- docs/rails.md | 4 +++- docs/wordpress.md | 2 ++ 3 files changed, 10 insertions(+), 3 deletions(-) diff --git a/docs/django.md b/docs/django.md index 2ebf4b4b..c7ebf58b 100644 --- a/docs/django.md +++ b/docs/django.md @@ -64,8 +64,9 @@ and a `docker-compose.yml` file. The `docker-compose.yml` file describes the services that make your app. In this example those services are a web server and database. The compose file - also describes which Docker images these services use, any volumes they might - need mounted inside the containers, and any ports they might + also describes which Docker images these services use, how they link + together, any volumes they might need mounted inside the containers. + Finally, the `docker-compose.yml` file describes which ports these services expose. See the [`docker-compose.yml` reference](yml.md) for more information on how this file works. @@ -80,6 +81,8 @@ and a `docker-compose.yml` file. - .:/code ports: - "8000:8000" + links: + - db This file defines two services: The `db` service and the `web` service. diff --git a/docs/rails.md b/docs/rails.md index 9801ef74..a33cac26 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,7 +37,7 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' -Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to expose the web app's port. +Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. db: image: postgres @@ -48,6 +48,8 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th - .:/myapp ports: - "3000:3000" + links: + - db ### Build the project diff --git a/docs/wordpress.md b/docs/wordpress.md index 5c9bcdbd..8c1f5b0a 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -46,6 +46,8 @@ and a separate MySQL instance: command: php -S 0.0.0.0:8000 -t /code ports: - "8000:8000" + links: + - db volumes: - .:/code db: From 7b109bc02617222fe8b9ae64435b77920200a00e Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Fri, 16 Oct 2015 12:57:54 +0100 Subject: [PATCH 009/359] Attempt to document escaping env vars People are likely to run into their env vars being set to empty strings, if they're not aware that they need to escape them for Compose to not interpolate them. Signed-off-by: Mazz Mosley --- docs/compose-file.md | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 90730fec..b72a7cc4 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -428,9 +428,18 @@ Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not supported. -If you need to put a literal dollar sign in a configuration value, use a double -dollar sign (`$$`). +You can use a `$$` (double-dollar sign) when your configuration needs a literal +dollar sign. This also prevents Compose from interpolating a value, so a `$$` +allows you to refer to environment variables that you don't want processed by +Compose. + web: + build: . + command: "$$VAR_NOT_INTERPOLATED_BY_COMPOSE" + +If you forget and use a single dollar sign (`$`), Compose interprets the value as an environment variable and will warn you: + + The VAR_NOT_INTERPOLATED_BY_COMPOSE is not set. Substituting an empty string. ## Compose documentation From 26dc0b785b064a60d8438e386358a5c051a89907 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 14:47:04 -0400 Subject: [PATCH 010/359] Give the user a better error message (without a stack trace) when there is a yaml error. Signed-off-by: Daniel Nephin --- compose/config/config.py | 5 +++-- tests/unit/config/config_test.py | 13 +++++++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 3bcd769a..59b98f60 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -610,5 +610,6 @@ def load_yaml(filename): try: with open(filename, 'r') as fh: return yaml.safe_load(fh) - except IOError as e: - raise ConfigurationError(six.text_type(e)) + except (IOError, yaml.YAMLError) as e: + error_name = getattr(e, '__module__', '') + '.' + e.__class__.__name__ + raise ConfigurationError(u"{}: {}".format(error_name, e)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 20ae7fa3..b4bd9c71 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -5,6 +5,7 @@ import shutil import tempfile from operator import itemgetter +import py import pytest from compose.config import config @@ -349,6 +350,18 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_yaml_with_yaml_error(self): + tmpdir = py.test.ensuretemp('invalid_yaml_test') + invalid_yaml_file = tmpdir.join('docker-compose.yml') + invalid_yaml_file.write(""" + web: + this is bogus: ok: what + """) + with pytest.raises(ConfigurationError) as exc: + config.load_yaml(str(invalid_yaml_file)) + + assert 'line 3, column 32' in exc.exconly() + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) From 43a89e1702b15a3322b084bb9bcc390dd62ada7d Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sat, 17 Oct 2015 09:26:32 -0700 Subject: [PATCH 011/359] bash completion for networking options Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e64b24a0..0eed1f18 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -105,11 +105,15 @@ _docker_compose_docker_compose() { --project-name|-p) return ;; + --x-network-driver) + COMPREPLY=( $( compgen -W "bridge host none overlay" -- "$cur" ) ) + return + ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -h --verbose --version -v --file -f --project-name -p" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v --x-networking --x-network-driver" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -410,6 +414,9 @@ _docker_compose() { (( counter++ )) compose_project="${words[$counter]}" ;; + --x-network-driver) + (( counter++ )) + ;; -*) ;; *) From fa44a5fac23e5b5685d355c033e12d07d2e7f0ef Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Sat, 17 Oct 2015 09:51:37 -0700 Subject: [PATCH 012/359] fix problem with bash completion in old bash Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e64b24a0..7184ec00 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -17,6 +17,12 @@ # . ~/.docker-compose-completion.sh +# suppress trailing whitespace +__docker_compose_nospace() { + # compopt is not available in ancient bash versions + type compopt &>/dev/null && compopt -o nospace +} + # For compatibility reasons, Compose and therefore its completion supports several # stack compositon files as listed here, in descending priority. # Support for these filenames might be dropped in some future version. @@ -255,7 +261,7 @@ _docker_compose_run() { case "$prev" in -e) COMPREPLY=( $( compgen -e -- "$cur" ) ) - compopt -o nospace + __docker_compose_nospace return ;; --entrypoint|--name|--user|-u) @@ -291,7 +297,7 @@ _docker_compose_scale() { ;; *) COMPREPLY=( $(compgen -S "=" -W "$(___docker_compose_all_services_in_compose_file)" -- "$cur") ) - compopt -o nospace + __docker_compose_nospace ;; esac } From 258c8bc54d30622a8b731d1b8f50fdd5544d2da0 Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Sun, 18 Oct 2015 00:40:51 +0530 Subject: [PATCH 013/359] Fix specifies_host_port() to handle port binding with host IP but no host port Signed-off-by: Viranch Mehta --- compose/service.py | 24 +++++++++++++-- tests/unit/service_test.py | 62 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 83 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5f1d5946..aa2e4f78 100644 --- a/compose/service.py +++ b/compose/service.py @@ -770,10 +770,28 @@ class Service(object): return self.options.get('container_name') def specifies_host_port(self): - for port in self.options.get('ports', []): - if ':' in str(port): + def has_host_port(binding): + _, external_bindings = split_port(binding) + + # there are no external bindings + if external_bindings is None: + return False + + # we only need to check the first binding from the range + external_binding = external_bindings[0] + + # non-tuple binding means there is a host port specified + if not isinstance(external_binding, tuple): return True - return False + + # extract actual host port from tuple of (host_ip, host_port) + _, host_port = external_binding + if host_port is not None: + return True + + return False + + return any(has_host_port(binding) for binding in self.options.get('ports', [])) def pull(self, ignore_pull_failures=False): if 'image' not in self.options: diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c5e1a9fb..2c46ce40 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -444,6 +444,68 @@ class ServiceTest(unittest.TestCase): } self.assertEqual(config_dict, expected) + def test_specifies_host_port_with_no_ports(self): + service = Service( + 'foo', + image='foo') + self.assertEqual(service.specifies_host_port(), False) + + def test_specifies_host_port_with_container_port(self): + service = Service( + 'foo', + image='foo', + ports=["2000"]) + self.assertEqual(service.specifies_host_port(), False) + + def test_specifies_host_port_with_host_port(self): + service = Service( + 'foo', + image='foo', + ports=["1000:2000"]) + self.assertEqual(service.specifies_host_port(), True) + + def test_specifies_host_port_with_host_ip_no_port(self): + service = Service( + 'foo', + image='foo', + ports=["127.0.0.1::2000"]) + self.assertEqual(service.specifies_host_port(), False) + + def test_specifies_host_port_with_host_ip_and_port(self): + service = Service( + 'foo', + image='foo', + ports=["127.0.0.1:1000:2000"]) + self.assertEqual(service.specifies_host_port(), True) + + def test_specifies_host_port_with_container_port_range(self): + service = Service( + 'foo', + image='foo', + ports=["2000-3000"]) + self.assertEqual(service.specifies_host_port(), False) + + def test_specifies_host_port_with_host_port_range(self): + service = Service( + 'foo', + image='foo', + ports=["1000-2000:2000-3000"]) + self.assertEqual(service.specifies_host_port(), True) + + def test_specifies_host_port_with_host_ip_no_port_range(self): + service = Service( + 'foo', + image='foo', + ports=["127.0.0.1::2000-3000"]) + self.assertEqual(service.specifies_host_port(), False) + + def test_specifies_host_port_with_host_ip_and_port_range(self): + service = Service( + 'foo', + image='foo', + ports=["127.0.0.1:1000-2000:2000-3000"]) + self.assertEqual(service.specifies_host_port(), True) + class NetTestCase(unittest.TestCase): From 07e9f6500ce2f695920d0a2786c308fba750cc2e Mon Sep 17 00:00:00 2001 From: Viranch Mehta Date: Sun, 18 Oct 2015 01:06:50 +0530 Subject: [PATCH 014/359] Pipe curl's download directly to extract/execute program to reduce number of commands Signed-off-by: Viranch Mehta --- Dockerfile | 22 +++++++--------------- 1 file changed, 7 insertions(+), 15 deletions(-) diff --git a/Dockerfile b/Dockerfile index c6dbdefd..b28a438d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,46 +22,38 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest \ # Build Python 2.7.9 from source RUN set -ex; \ - curl -LO https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz; \ - tar -xzf Python-2.7.9.tgz; \ + curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \ cd Python-2.7.9; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.9; \ - rm Python-2.7.9.tgz + rm -rf /Python-2.7.9 # Build python 3.4 from source RUN set -ex; \ - curl -LO https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz; \ - tar -xzf Python-3.4.3.tgz; \ + curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \ cd Python-3.4.3; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.3; \ - rm Python-3.4.3.tgz + rm -rf /Python-3.4.3 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib # Install setuptools RUN set -ex; \ - curl -LO https://bootstrap.pypa.io/ez_setup.py; \ - python ez_setup.py; \ - rm ez_setup.py + curl -L https://bootstrap.pypa.io/ez_setup.py | python # Install pip RUN set -ex; \ - curl -LO https://pypi.python.org/packages/source/p/pip/pip-7.0.1.tar.gz; \ - tar -xzf pip-7.0.1.tar.gz; \ + curl -L https://pypi.python.org/packages/source/p/pip/pip-7.0.1.tar.gz | tar -xz; \ cd pip-7.0.1; \ python setup.py install; \ cd ..; \ - rm -rf pip-7.0.1; \ - rm pip-7.0.1.tar.gz + rm -rf pip-7.0.1 # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen From 37921b40dd9ca76d585af6613388fe9fdc6e147c Mon Sep 17 00:00:00 2001 From: Nicolas Delaby Date: Mon, 19 Oct 2015 10:43:30 +0200 Subject: [PATCH 015/359] Add trove classifier to declare supported python versions. Signed-off-by: Nicolas Delaby --- setup.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/setup.py b/setup.py index 4020122b..bf2ee07f 100644 --- a/setup.py +++ b/setup.py @@ -66,4 +66,8 @@ setup( [console_scripts] docker-compose=compose.cli.main:main """, + classifiers=[ + 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.4', + ], ) From ac06366ef9081fa83c8d1fef4c67e99f28bee8ea Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Thu, 15 Oct 2015 23:02:40 +0200 Subject: [PATCH 016/359] Add zsh completion for 'docker-compose --x-networking --x-network-driver' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index cefcb109..d79b25d1 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -330,6 +330,8 @@ _docker-compose() { '(- :)'{-v,--version}'[Print version and exit]' \ '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ + '--x-networking[(EXPERIMENTAL) Use new Docker networking functionality. Requires Docker 1.9 or later.]' \ + '--x-network-driver[(EXPERIMENTAL) Specify a network driver (default: "bridge"). Requires Docker 1.9 or later.]:Network Driver:(bridge host none overlay)' \ '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 From 4d8e667c3e475e811b8072647833067740c810c2 Mon Sep 17 00:00:00 2001 From: Corin Lawson Date: Mon, 19 Oct 2015 21:47:01 +1100 Subject: [PATCH 017/359] Powershell script to run compose in a container. This script assumes the typical environment that Windows users operate, namely, VirtualBox running the boot2docker ISO, managed by docker-machine. I wrote this script for my Windows using colleagues and first placed it in the public domain as a gist: https://gist.github.com/au-phiware/25213e72c80040f398ba In short, that script works for me. I have adapted that script to use the (yet to be) official image (docker/compose:latest) and also added an extra environment variable to provide additional options to docker run. Signed-off-by: Corin Lawson --- script/run.ps1 | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 script/run.ps1 diff --git a/script/run.ps1 b/script/run.ps1 new file mode 100644 index 00000000..47ec5469 --- /dev/null +++ b/script/run.ps1 @@ -0,0 +1,22 @@ +# Run docker-compose in a container via boot2docker. +# +# The current directory will be mirrored as a volume and additional +# volumes (or any other options) can be mounted by using +# $Env:DOCKER_COMPOSE_OPTIONS. + +if ($Env:DOCKER_COMPOSE_VERSION -eq $null -or $Env:DOCKER_COMPOSE_VERSION.Length -eq 0) { + $Env:DOCKER_COMPOSE_VERSION = "latest" +} + +if ($Env:DOCKER_COMPOSE_OPTIONS -eq $null) { + $Env:DOCKER_COMPOSE_OPTIONS = "" +} + +if (-not $Env:DOCKER_HOST) { + docker-machine env --shell=powershell default | Invoke-Expression + if (-not $?) { exit $LastExitCode } +} + +$local="/$($PWD -replace '^(.):(.*)$', '"$1".ToLower()+"$2".Replace("\","/")' | Invoke-Expression)" +docker run --rm -ti -v /var/run/docker.sock:/var/run/docker.sock -v "${local}:$local" -w "$local" $Env:DOCKER_COMPOSE_OPTIONS "docker/compose:$Env:DOCKER_COMPOSE_VERSION" $args +exit $LastExitCode From ff83c459d04491faedb69db9ef74f65e00c2d116 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 19 Oct 2015 14:36:56 +0100 Subject: [PATCH 018/359] Improve error message for type constraints It was missing a space between the different types, when there were 3 possible type values. Signed-off-by: Mazz Mosley --- compose/config/validation.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 46b891b7..19faa0bc 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -143,7 +143,7 @@ def process_errors(errors, service_name=None): if len(validator) >= 2: first_type = anglicize_validator(validator[0]) last_type = anglicize_validator(validator[-1]) - types_from_validator = "{}{}".format(first_type, ", ".join(validator[1:-1])) + types_from_validator = "{}".format(", ".join([first_type] + validator[1:-1])) msg = "{} or {}".format( types_from_validator, @@ -163,7 +163,6 @@ def process_errors(errors, service_name=None): Inspecting the context value of a ValidationError gives us information about which sub schema failed and which kind of error it is. """ - required = [context for context in error.context if context.validator == 'required'] if required: return required[0].message From 08add665e98044279ead90e093d40eb2161efb27 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 19 Oct 2015 15:15:24 +0100 Subject: [PATCH 019/359] Environment keys can contain empty values Environment keys that contain no value, get populated with values taken from the environment not from the build phase but from running the command `up`. Signed-off-by: Mazz Mosley --- compose/config/fields_schema.json | 2 +- compose/config/validation.py | 2 +- tests/unit/config/config_test.py | 17 +++++++++++++++++ 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index cc37f444..e254e353 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -41,7 +41,7 @@ "type": "object", "patternProperties": { "^[^-]+$": { - "type": ["string", "number", "boolean"], + "type": ["string", "number", "boolean", "null"], "format": "environment" } }, diff --git a/compose/config/validation.py b/compose/config/validation.py index 19faa0bc..427f21ad 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -143,7 +143,7 @@ def process_errors(errors, service_name=None): if len(validator) >= 2: first_type = anglicize_validator(validator[0]) last_type = anglicize_validator(validator[-1]) - types_from_validator = "{}".format(", ".join([first_type] + validator[1:-1])) + types_from_validator = ", ".join([first_type] + validator[1:-1]) msg = "{} or {}".format( types_from_validator, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 20ae7fa3..935e2f9d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -414,6 +414,23 @@ class InterpolationTest(unittest.TestCase): self.assertIn('in service "web"', cm.exception.msg) self.assertIn('"${"', cm.exception.msg) + def test_empty_environment_key_allowed(self): + service_dict = config.load( + build_config_details( + { + 'web': { + 'build': '.', + 'environment': { + 'POSTGRES_PASSWORD': '' + }, + }, + }, + '.', + None, + ) + )[0] + self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') + class VolumeConfigTest(unittest.TestCase): def test_no_binding(self): From 129e2f94826ad01b559c5b0e7f1ebc70ec7c97d8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 12:54:31 -0400 Subject: [PATCH 020/359] Fix check for tags. If there is a branch with the same name as a tag it fails without the --tags. This was only a problem when we're branching from a git tag. Signed-off-by: Daniel Nephin --- script/release/make-branch | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/make-branch b/script/release/make-branch index e2eae4d5..48fa771b 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -46,7 +46,7 @@ if [ -z "$REMOTE" ]; then fi # handle the difference between a branch and a tag -if [ -z "$(git name-rev $BASE_VERSION | grep tags)" ]; then +if [ -z "$(git name-rev --tags $BASE_VERSION | grep tags)" ]; then BASE_VERSION=$REMOTE/$BASE_VERSION fi From 937e087c6cf78d3a12ce940ad96344fbb296b005 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 15:45:56 -0400 Subject: [PATCH 021/359] Fixes #2203 - properly validate files when multiple files are used. Remove the single-use decorators so the functionality can be used directly as a function. Signed-off-by: Daniel Nephin --- compose/config/config.py | 22 +++++++------------- compose/config/validation.py | 35 +++++++++++++------------------- tests/unit/config/config_test.py | 22 ++++++++++++++++++++ 3 files changed, 43 insertions(+), 36 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 59b98f60..05e57258 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -14,7 +14,6 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extended_service_exists from .validation import validate_extends_file_path -from .validation import validate_service_names from .validation import validate_top_level_object @@ -165,16 +164,6 @@ def find_candidates_in_parent_dirs(filenames, path): return (candidates, path) -@validate_top_level_object -@validate_service_names -def pre_process_config(config): - """ - Pre validation checks and processing of the config file to interpolate env - vars returning a config dict ready to be tested against the schema. - """ - return interpolate_environment_variables(config) - - def load(config_details): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top @@ -194,7 +183,7 @@ def load(config_details): return service_dict def load_file(filename, config): - processed_config = pre_process_config(config) + processed_config = interpolate_environment_variables(config) validate_against_fields_schema(processed_config) return [ build_service(filename, name, service_config) @@ -209,7 +198,10 @@ def load(config_details): } config_file = config_details.config_files[0] + validate_top_level_object(config_file.config) for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file.config) + config_file = ConfigFile( config_file.filename, merge_services(config_file.config, next_file.config)) @@ -283,9 +275,9 @@ class ServiceLoader(object): ) self.extended_service_name = extends['service'] - full_extended_config = pre_process_config( - load_yaml(self.extended_config_path) - ) + config = load_yaml(self.extended_config_path) + validate_top_level_object(config) + full_extended_config = interpolate_environment_variables(config) validate_extended_service_exists( self.extended_service_name, diff --git a/compose/config/validation.py b/compose/config/validation.py index 46b891b7..aea72286 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -2,7 +2,6 @@ import json import logging import os import sys -from functools import wraps import six from docker.utils.ports import split_port @@ -65,27 +64,21 @@ def format_boolean_in_environment(instance): return True -def validate_service_names(func): - @wraps(func) - def func_wrapper(config): - for service_name in config.keys(): - if type(service_name) is int: - raise ConfigurationError( - "Service name: {} needs to be a string, eg '{}'".format(service_name, service_name) - ) - return func(config) - return func_wrapper - - -def validate_top_level_object(func): - @wraps(func) - def func_wrapper(config): - if not isinstance(config, dict): +def validate_service_names(config): + for service_name in config.keys(): + if not isinstance(service_name, six.string_types): raise ConfigurationError( - "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." - ) - return func(config) - return func_wrapper + "Service name: {} needs to be a string, eg '{}'".format( + service_name, + service_name)) + + +def validate_top_level_object(config): + if not isinstance(config, dict): + raise ConfigurationError( + "Top level object needs to be a dictionary. Check your .yml file " + "that you have defined a service at the top level.") + validate_service_names(config) def validate_extends_file_path(service_name, extends_options, filename): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index b4bd9c71..e8caea81 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -134,6 +134,28 @@ class ConfigTest(unittest.TestCase): ] self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_load_with_multiple_files_and_empty_override(self): + base_file = config.ConfigFile( + 'base.yaml', + {'web': {'image': 'example/web'}}) + override_file = config.ConfigFile('override.yaml', None) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Top level object needs to be a dictionary' in exc.exconly() + + def test_load_with_multiple_files_and_empty_base(self): + base_file = config.ConfigFile('base.yaml', None) + override_file = config.ConfigFile( + 'override.yaml', + {'web': {'image': 'example/web'}}) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Top level object needs to be a dictionary' in exc.exconly() + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From 938d49cbdc81665de26c7148befc833b50e867e9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 16 Oct 2015 11:18:07 -0400 Subject: [PATCH 022/359] Fixes #2205 - extends must be copied from override file. Signed-off-by: Daniel Nephin --- compose/config/config.py | 19 ++++++++++++---- tests/unit/config/config_test.py | 37 ++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 05e57258..ff8861b5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -193,7 +193,9 @@ def load(config_details): def merge_services(base, override): all_service_names = set(base) | set(override) return { - name: merge_service_dicts(base.get(name, {}), override.get(name, {})) + name: merge_service_dicts_from_files( + base.get(name, {}), + override.get(name, {})) for name in all_service_names } @@ -270,9 +272,7 @@ class ServiceLoader(object): extends, self.filename ) - self.extended_config_path = self.get_extended_config_path( - extends - ) + self.extended_config_path = self.get_extended_config_path(extends) self.extended_service_name = extends['service'] config = load_yaml(self.extended_config_path) @@ -355,6 +355,17 @@ def process_container_options(service_dict, working_dir=None): return service_dict +def merge_service_dicts_from_files(base, override): + """When merging services from multiple files we need to merge the `extends` + field. This is not handled by `merge_service_dicts()` which is used to + perform the `extends`. + """ + new_service = merge_service_dicts(base, override) + if 'extends' in override: + new_service['extends'] = override['extends'] + return new_service + + def merge_service_dicts(base, override): d = base.copy() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e8caea81..eea3451f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -156,6 +156,43 @@ class ConfigTest(unittest.TestCase): config.load(details) assert 'Top level object needs to be a dictionary' in exc.exconly() + def test_load_with_multiple_files_and_extends_in_override_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': {'image': 'example/web'}, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'web': { + 'extends': { + 'file': 'common.yml', + 'service': 'base', + }, + 'volumes': ['/home/user/project:/code'], + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + tmpdir = py.test.ensuretemp('config_test') + tmpdir.join('common.yml').write(""" + base: + labels: ['label=one'] + """) + with tmpdir.as_cwd(): + service_dicts = config.load(details) + + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'volumes': ['/home/user/project:/code'], + 'labels': {'label': 'one'}, + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From 3f0e0835850462b750167f708d17793b45dc9ef6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Oct 2015 11:39:06 -0400 Subject: [PATCH 023/359] Force windows drives to be lowercase. Signed-off-by: Daniel Nephin --- compose/service.py | 2 +- tests/unit/config/config_test.py | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5f1d5946..7daf7f2f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -952,7 +952,7 @@ def normalize_paths_for_engine(external_path, internal_path): drive, tail = os.path.splitdrive(external_path) if drive: - reformatted_drive = "/{}".format(drive.replace(":", "")) + reformatted_drive = "/{}".format(drive.lower().replace(":", "")) external_path = reformatted_drive + tail external_path = "/".join(external_path.split("\\")) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c8b76f36..d15cd9a6 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -537,8 +537,8 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], ['/var/lib/data:/data']) def test_absolute_windows_path_does_not_expand(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['C:\\data:/data']}, working_dir='.') - self.assertEqual(d['volumes'], ['C:\\data:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['c:\\data:/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['c:\\data:/data']) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') def test_relative_path_does_expand_posix(self): @@ -553,14 +553,14 @@ class VolumeConfigTest(unittest.TestCase): @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') def test_relative_path_does_expand_windows(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject\\data:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject\\data:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\otherproject:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\otherproject:/data']) @mock.patch.dict(os.environ) def test_home_directory_with_driver_does_not_expand(self): From 5523c3d7457f69732e269f7dc45eec9808ba2170 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Oct 2015 11:49:10 -0400 Subject: [PATCH 024/359] Minor refactor to use guard and replace instead of split+join Signed-off-by: Daniel Nephin --- compose/service.py | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/compose/service.py b/compose/service.py index 7daf7f2f..f18afa48 100644 --- a/compose/service.py +++ b/compose/service.py @@ -943,24 +943,22 @@ def build_volume_binding(volume_spec): def normalize_paths_for_engine(external_path, internal_path): - """ - Windows paths, c:\my\path\shiny, need to be changed to be compatible with + """Windows paths, c:\my\path\shiny, need to be changed to be compatible with the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ """ - if IS_WINDOWS_PLATFORM: - if external_path: - drive, tail = os.path.splitdrive(external_path) - - if drive: - reformatted_drive = "/{}".format(drive.lower().replace(":", "")) - external_path = reformatted_drive + tail - - external_path = "/".join(external_path.split("\\")) - - return external_path, "/".join(internal_path.split("\\")) - else: + if not IS_WINDOWS_PLATFORM: return external_path, internal_path + if external_path: + drive, tail = os.path.splitdrive(external_path) + + if drive: + external_path = '/' + drive.lower().rstrip(':') + tail + + external_path = external_path.replace('\\', '/') + + return external_path, internal_path.replace('\\', '/') + def parse_volume_spec(volume_config): """ From b1f8ed84a303d5300d3b27a387799d582c55ae2d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Oct 2015 13:10:32 -0400 Subject: [PATCH 025/359] Cleanup some unit tests and whitespace. Remove some unnecessary newlines. Remove a unittest that was attempting to test behaviour that was removed a while ago, so isn't testing anything. Updated some unit tests to use mocks instead of a custom fake. Signed-off-by: Daniel Nephin --- compose/service.py | 12 +++------ tests/unit/service_test.py | 50 ++++++++++++++------------------------ 2 files changed, 21 insertions(+), 41 deletions(-) diff --git a/compose/service.py b/compose/service.py index 5f1d5946..7862867f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -300,9 +300,7 @@ class Service(object): Create a container for this service. If the image doesn't exist, attempt to pull it. """ - self.ensure_image_exists( - do_build=do_build, - ) + self.ensure_image_exists(do_build=do_build) container_options = self._get_container_create_options( override_options, @@ -316,9 +314,7 @@ class Service(object): return Container.create(self.client, **container_options) - def ensure_image_exists(self, - do_build=True): - + def ensure_image_exists(self, do_build=True): try: self.image() return @@ -403,9 +399,7 @@ class Service(object): (action, containers) = plan if action == 'create': - container = self.create_container( - do_build=do_build, - ) + container = self.create_container(do_build=do_build) self.start_container(container) return [container] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c5e1a9fb..03575f93 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -351,44 +351,37 @@ class ServiceTest(unittest.TestCase): self.assertEqual(parse_repository_tag("user/repo@sha256:digest"), ("user/repo", "sha256:digest", "@")) self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@")) - @mock.patch('compose.service.Container', autospec=True) - def test_create_container_latest_is_used_when_no_tag_specified(self, mock_container): - service = Service('foo', client=self.mock_client, image='someimage') - images = [] - - def pull(repo, tag=None, **kwargs): - self.assertEqual('someimage', repo) - self.assertEqual('latest', tag) - images.append({'Id': 'abc123'}) - return [] - - service.image = lambda *args, **kwargs: mock_get_image(images) - self.mock_client.pull = pull - - service.create_container() - self.assertEqual(1, len(images)) - def test_create_container_with_build(self): service = Service('foo', client=self.mock_client, build='.') - - images = [] - service.image = lambda *args, **kwargs: mock_get_image(images) - service.build = lambda: images.append({'Id': 'abc123'}) + self.mock_client.inspect_image.side_effect = [ + NoSuchImageError, + {'Id': 'abc123'}, + ] + self.mock_client.build.return_value = [ + '{"stream": "Successfully built abcd"}', + ] service.create_container(do_build=True) - self.assertEqual(1, len(images)) + self.mock_client.build.assert_called_once_with( + tag='default_foo', + dockerfile=None, + stream=True, + path='.', + pull=False, + nocache=False, + rm=True, + ) def test_create_container_no_build(self): service = Service('foo', client=self.mock_client, build='.') - service.image = lambda: {'Id': 'abc123'} + self.mock_client.inspect_image.return_value = {'Id': 'abc123'} service.create_container(do_build=False) self.assertFalse(self.mock_client.build.called) def test_create_container_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build='.') - service.image = lambda *args, **kwargs: mock_get_image([]) - + self.mock_client.inspect_image.side_effect = NoSuchImageError with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) @@ -488,13 +481,6 @@ class NetTestCase(unittest.TestCase): self.assertEqual(net.service_name, service_name) -def mock_get_image(images): - if images: - return images[0] - else: - raise NoSuchImageError() - - class ServiceVolumesTest(unittest.TestCase): def setUp(self): From 45056322748514cf66632409d7fe4ad12aeef533 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 19 Oct 2015 12:52:38 -0400 Subject: [PATCH 026/359] Some minor style cleanup - fixed a docstring to make it PEP257 compliant - wrapped some long lines - used a more specific error Signed-off-by: Daniel Nephin --- compose/config/config.py | 29 ++++++++++++++++++----------- 1 file changed, 18 insertions(+), 11 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ff8861b5..440f3920 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -212,9 +212,16 @@ def load(config_details): class ServiceLoader(object): - def __init__(self, working_dir, filename, service_name, service_dict, already_seen=None): + def __init__( + self, + working_dir, + filename, + service_name, + service_dict, + already_seen=None + ): if working_dir is None: - raise Exception("No working_dir passed to ServiceLoader()") + raise ValueError("No working_dir passed to ServiceLoader()") self.working_dir = os.path.abspath(working_dir) @@ -311,33 +318,33 @@ class ServiceLoader(object): return merge_service_dicts(other_service_dict, self.service_dict) def get_extended_config_path(self, extends_options): - """ - Service we are extending either has a value for 'file' set, which we + """Service we are extending either has a value for 'file' set, which we need to obtain a full path too or we are extending from a service defined in our own file. """ if 'file' in extends_options: - extends_from_filename = extends_options['file'] - return expand_path(self.working_dir, extends_from_filename) - + return expand_path(self.working_dir, extends_options['file']) return self.filename def signature(self, name): - return (self.filename, name) + return self.filename, name def validate_extended_service_dict(service_dict, filename, service): error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) if 'links' in service_dict: - raise ConfigurationError("%s services with 'links' cannot be extended" % error_prefix) + raise ConfigurationError( + "%s services with 'links' cannot be extended" % error_prefix) if 'volumes_from' in service_dict: - raise ConfigurationError("%s services with 'volumes_from' cannot be extended" % error_prefix) + raise ConfigurationError( + "%s services with 'volumes_from' cannot be extended" % error_prefix) if 'net' in service_dict: if get_service_name_from_net(service_dict['net']) is not None: - raise ConfigurationError("%s services with 'net: container' cannot be extended" % error_prefix) + raise ConfigurationError( + "%s services with 'net: container' cannot be extended" % error_prefix) def process_container_options(service_dict, working_dir=None): From b500fa235128d56ea41a8c018f75d81e8722f74d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 19 Oct 2015 13:40:13 -0400 Subject: [PATCH 027/359] Refactor ServiceLoader to be immutable. Mutable objects are harder to debug and harder to reason about. ServiceLoader was almost immutable. There was just a single function which set fields for a second function. Instead of mutating the object, we can pass those values as parameters to the next function. Signed-off-by: Daniel Nephin --- compose/config/config.py | 89 +++++++++++++++++++--------------------- 1 file changed, 43 insertions(+), 46 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 440f3920..1a3b30ac 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -239,80 +239,61 @@ class ServiceLoader(object): raise CircularReference(self.already_seen + [self.signature(name)]) def make_service_dict(self): - self.resolve_environment() - if 'extends' in self.service_dict: - self.validate_and_construct_extends() - self.service_dict = self.resolve_extends() + service_dict = dict(self.service_dict) + env = resolve_environment(self.working_dir, self.service_dict) + if env: + service_dict['environment'] = env + service_dict.pop('env_file', None) + + if 'extends' in service_dict: + service_dict = self.resolve_extends(*self.validate_and_construct_extends()) if not self.already_seen: - validate_against_service_schema(self.service_dict, self.service_name) + validate_against_service_schema(service_dict, self.service_name) - return process_container_options(self.service_dict, working_dir=self.working_dir) - - def resolve_environment(self): - """ - Unpack any environment variables from an env_file, if set. - Interpolate environment values if set. - """ - if 'environment' not in self.service_dict and 'env_file' not in self.service_dict: - return - - env = {} - - if 'env_file' in self.service_dict: - for f in get_env_files(self.service_dict, working_dir=self.working_dir): - env.update(env_vars_from_file(f)) - del self.service_dict['env_file'] - - env.update(parse_environment(self.service_dict.get('environment'))) - env = dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) - - self.service_dict['environment'] = env + return process_container_options(service_dict, working_dir=self.working_dir) def validate_and_construct_extends(self): extends = self.service_dict['extends'] if not isinstance(extends, dict): extends = {'service': extends} - validate_extends_file_path( - self.service_name, - extends, - self.filename - ) - self.extended_config_path = self.get_extended_config_path(extends) - self.extended_service_name = extends['service'] + validate_extends_file_path(self.service_name, extends, self.filename) + config_path = self.get_extended_config_path(extends) + service_name = extends['service'] - config = load_yaml(self.extended_config_path) + config = load_yaml(config_path) validate_top_level_object(config) full_extended_config = interpolate_environment_variables(config) validate_extended_service_exists( - self.extended_service_name, + service_name, full_extended_config, - self.extended_config_path + config_path ) validate_against_fields_schema(full_extended_config) - self.extended_config = full_extended_config[self.extended_service_name] + service_config = full_extended_config[service_name] + return config_path, service_config, service_name - def resolve_extends(self): - other_working_dir = os.path.dirname(self.extended_config_path) + def resolve_extends(self, extended_config_path, service_config, service_name): + other_working_dir = os.path.dirname(extended_config_path) other_already_seen = self.already_seen + [self.signature(self.service_name)] other_loader = ServiceLoader( - working_dir=other_working_dir, - filename=self.extended_config_path, - service_name=self.service_name, - service_dict=self.extended_config, + other_working_dir, + extended_config_path, + self.service_name, + service_config, already_seen=other_already_seen, ) - other_loader.detect_cycle(self.extended_service_name) + other_loader.detect_cycle(service_name) other_service_dict = other_loader.make_service_dict() validate_extended_service_dict( other_service_dict, - filename=self.extended_config_path, - service=self.extended_service_name, + extended_config_path, + service_name, ) return merge_service_dicts(other_service_dict, self.service_dict) @@ -330,6 +311,22 @@ class ServiceLoader(object): return self.filename, name +def resolve_environment(working_dir, service_dict): + """Unpack any environment variables from an env_file, if set. + Interpolate environment values if set. + """ + if 'environment' not in service_dict and 'env_file' not in service_dict: + return {} + + env = {} + if 'env_file' in service_dict: + for env_file in get_env_files(service_dict, working_dir=working_dir): + env.update(env_vars_from_file(env_file)) + + env.update(parse_environment(service_dict.get('environment'))) + return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) + + def validate_extended_service_dict(service_dict, filename, service): error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) From 0fed5e686438aefd9968733fc3c480e7c1339568 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 13:05:14 -0400 Subject: [PATCH 028/359] Use inspect network to query for an existing network. And more tests for get_network() Signed-off-by: Daniel Nephin --- compose/project.py | 9 +++++---- tests/integration/project_test.py | 17 +++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) diff --git a/compose/project.py b/compose/project.py index fdd70caf..d4934c26 100644 --- a/compose/project.py +++ b/compose/project.py @@ -5,6 +5,7 @@ import logging from functools import reduce from docker.errors import APIError +from docker.errors import NotFound from .config import ConfigurationError from .config import get_service_name_from_net @@ -363,10 +364,10 @@ class Project(object): return [c for c in containers if matches_service_names(c)] def get_network(self): - networks = self.client.networks(names=[self.name]) - if networks: - return networks[0] - return None + try: + return self.client.inspect_network(self.name) + except NotFound: + return None def ensure_network_exists(self): # TODO: recreate network if driver has changed? diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ff50c80b..ac0f121c 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from .testcases import DockerClientTestCase +from compose.cli.docker_client import docker_client from compose.config import config from compose.const import LABEL_PROJECT from compose.container import Container @@ -96,6 +97,22 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) + def test_get_network_does_not_exist(self): + self.require_engine_version("1.9") + client = docker_client(version='1.21') + + project = Project('composetest', [], client) + assert project.get_network() is None + + def test_get_network(self): + self.require_engine_version("1.9") + client = docker_client(version='1.21') + + network_name = 'network_does_exist' + project = Project(network_name, [], client) + client.create_network(network_name) + assert project.get_network()['name'] == network_name + def test_net_from_service(self): project = Project.from_dicts( name='composetest', From 1bc3c97f2a770d8268b817f0d8f17160e7b322b2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 15:26:44 -0400 Subject: [PATCH 029/359] Make storage driver configurable in CI Signed-off-by: Daniel Nephin --- script/ci | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/script/ci b/script/ci index 12dc3c47..f30265c0 100755 --- a/script/ci +++ b/script/ci @@ -11,7 +11,9 @@ set -ex docker version export DOCKER_VERSIONS=all -export DOCKER_DAEMON_ARGS="--storage-driver=overlay" +STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} +export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" + GIT_VOLUME="--volumes-from=$(hostname)" . script/test-versions From f7100b2ef3cf7eb275548bf4c437653b266cf66d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 15:40:50 -0400 Subject: [PATCH 030/359] Change version check from engine version to api version. Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 4 ++-- tests/integration/project_test.py | 4 ++-- tests/integration/testcases.py | 12 ++++-------- 3 files changed, 8 insertions(+), 12 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 78519d14..19cc822e 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -187,7 +187,7 @@ class CLITestCase(DockerClientTestCase): ) def test_up_without_networking(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d'], None) @@ -205,7 +205,7 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(web_container.get('HostConfig.Links')) def test_up_with_networking(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['--x-networking', 'up', '-d'], None) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ac0f121c..fd45b939 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -98,14 +98,14 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) def test_get_network_does_not_exist(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') client = docker_client(version='1.21') project = Project('composetest', [], client) assert project.get_network() is None def test_get_network(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') client = docker_client(version='1.21') network_name = 'network_does_exist' diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index a412fb04..686a2b69 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -76,11 +76,7 @@ class DockerClientTestCase(unittest.TestCase): build_output = self.client.build(*args, **kwargs) stream_output(build_output, open('/dev/null', 'w')) - def require_engine_version(self, minimum): - # Drop '-dev' or '-rcN' suffix - engine = self.client.version()['Version'].split('-', 1)[0] - if version_lt(engine, minimum): - skip( - "Engine version is too low ({} < {})" - .format(engine, minimum) - ) + def require_api_version(self, minimum): + api_version = self.client.version()['ApiVersion'] + if version_lt(api_version, minimum): + skip("API version is too low ({} < {})".format(api_version, minimum)) From ae47435425e3922fac0cf4faaa89269ea0efc6e3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 22 Oct 2015 12:12:43 -0400 Subject: [PATCH 031/359] Fix unicode in environment variables for python2. Signed-off-by: Daniel Nephin --- compose/config/config.py | 5 ++++- tests/fixtures/env/resolve.env | 2 +- tests/unit/cli_test.py | 5 +++-- tests/unit/config/config_test.py | 8 +++++++- 4 files changed, 15 insertions(+), 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 1a3b30ac..40b4ffa4 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,3 +1,4 @@ +import codecs import logging import os import sys @@ -455,6 +456,8 @@ def parse_environment(environment): def split_env(env): + if isinstance(env, six.binary_type): + env = env.decode('utf-8') if '=' in env: return env.split('=', 1) else: @@ -477,7 +480,7 @@ def env_vars_from_file(filename): if not os.path.exists(filename): raise ConfigurationError("Couldn't find env file: %s" % filename) env = {} - for line in open(filename, 'r'): + for line in codecs.open(filename, 'r', 'utf-8'): line = line.strip() if line and not line.startswith('#'): k, v = split_env(line) diff --git a/tests/fixtures/env/resolve.env b/tests/fixtures/env/resolve.env index 720520d2..b4f76b29 100644 --- a/tests/fixtures/env/resolve.env +++ b/tests/fixtures/env/resolve.env @@ -1,4 +1,4 @@ -FILE_DEF=F1 +FILE_DEF=bär FILE_DEF_EMPTY= ENV_DEF NO_DEF diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 0c78e6bb..5b63d2e8 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import absolute_import from __future__ import unicode_literals @@ -98,7 +99,7 @@ class CLITestCase(unittest.TestCase): command.run(mock_project, { 'SERVICE': 'service', 'COMMAND': None, - '-e': ['BAR=NEW', 'OTHER=THREE'], + '-e': ['BAR=NEW', 'OTHER=bär'.encode('utf-8')], '--user': None, '--no-deps': None, '--allow-insecure-ssl': None, @@ -114,7 +115,7 @@ class CLITestCase(unittest.TestCase): _, _, call_kwargs = mock_client.create_container.mock_calls[0] self.assertEqual( call_kwargs['environment'], - {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': 'THREE'}) + {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': u'bär'}) def test_run_service_with_restart_always(self): command = TopLevelCommand() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index d15cd9a6..a54b006f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import print_function import os @@ -894,7 +895,12 @@ class EnvTest(unittest.TestCase): ) self.assertEqual( service_dict['environment'], - {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, + { + 'FILE_DEF': u'bär', + 'FILE_DEF_EMPTY': '', + 'ENV_DEF': 'E3', + 'NO_DEF': '' + }, ) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') From cf197253cdc10c2860c558412d4646d1795b7160 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 22 Oct 2015 17:17:11 +1000 Subject: [PATCH 032/359] Possible link fixes Signed-off-by: Sven Dowideit --- docs/django.md | 2 +- docs/env.md | 2 +- docs/index.md | 2 +- docs/networking.md | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/django.md b/docs/django.md index c7ebf58b..fd18784e 100644 --- a/docs/django.md +++ b/docs/django.md @@ -67,7 +67,7 @@ and a `docker-compose.yml` file. also describes which Docker images these services use, how they link together, any volumes they might need mounted inside the containers. Finally, the `docker-compose.yml` file describes which ports these services - expose. See the [`docker-compose.yml` reference](yml.md) for more + expose. See the [`docker-compose.yml` reference](compose-file.md) for more information on how this file works. 9. Add the following configuration to the file. diff --git a/docs/env.md b/docs/env.md index 8f3cc3cc..d7b51ba2 100644 --- a/docs/env.md +++ b/docs/env.md @@ -11,7 +11,7 @@ weight=3 # Compose environment variables reference -**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](yml.md#links) for details. +**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. Compose uses [Docker links] to expose services' containers to one another. Each linked container injects a set of environment variables, each of which begins with the uppercase name of the container. diff --git a/docs/index.md b/docs/index.md index e19e7d7f..62c78d68 100644 --- a/docs/index.md +++ b/docs/index.md @@ -154,7 +154,7 @@ Now, when you run `docker-compose up`, Compose will pull a Redis image, build an If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 doesn't resolve, you can also try http://localhost:5000. +If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. You should get a message in your browser saying: diff --git a/docs/networking.md b/docs/networking.md index f4227917..9a6d792d 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -12,11 +12,11 @@ weight=6 # Networking in Compose -> **Note:** Compose’s networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. +> **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. -Compose sets up a single default [network](http://TODO/docker-networking-docs) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. +Compose sets up a single default [network](/engine/reference/commandline/network_create.md) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. -> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [CLI docs](cli.md#p-project-name-name) for how to override it. +> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: @@ -65,7 +65,7 @@ Docker links are a one-way, single-host communication system. They should now be ## Specifying the network driver -By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](http://TODO/custom-driver-docs). +By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](/engine/extend/plugins_network.md). You can specify which one to use with the `--x-network-driver` flag: From 0340361f56e7bf80fe0961d387b7d58fb7098e06 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 22 Sep 2015 16:42:43 -0400 Subject: [PATCH 033/359] Upgrade pyinstaller to 3.0 Signed-off-by: Daniel Nephin --- requirements-build.txt | 2 +- script/build-windows.ps1 | 8 ++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/requirements-build.txt b/requirements-build.txt index 5da6fa49..20aad420 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller +pyinstaller==3.0 diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 6e8a7c5a..42a4a501 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -42,11 +42,6 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } virtualenv .\venv # Install dependencies -# TODO: pip warns when installing from a git sha, so we need to set ErrorAction to -# 'Continue'. See -# https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77 -# This can be removed once pyinstaller 3.x is released and we upgrade -$ErrorActionPreference = "Continue" .\venv\Scripts\pip install pypiwin32==219 .\venv\Scripts\pip install -r requirements.txt .\venv\Scripts\pip install --no-deps . @@ -54,8 +49,9 @@ $ErrorActionPreference = "Continue" # Build binary # pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue +$ErrorActionPreference = "Continue" .\venv\Scripts\pyinstaller .\docker-compose.spec $ErrorActionPreference = "Stop" -Move-Item -Force .\dist\docker-compose .\dist\docker-compose-Windows-x86_64.exe +Move-Item -Force .\dist\docker-compose.exe .\dist\docker-compose-Windows-x86_64.exe .\dist\docker-compose-Windows-x86_64.exe --version From fe760a7b62719f61fccf30e4b806ffae39b163ae Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 26 Oct 2015 17:08:45 +0000 Subject: [PATCH 034/359] Include additional classifiers I've included Python 2/3 as they are not parent classifiers but sibling classifiers. They denote that this project will work with *some* versions of python and by having them, they'll show up for people searching for python 2 or 3 projects. According to the internet :) Signed-off-by: Mazz Mosley --- setup.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/setup.py b/setup.py index bf2ee07f..bd6f201d 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,13 @@ setup( docker-compose=compose.cli.main:main """, classifiers=[ + 'Development Status :: 5 - Production/Stable', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: Apache Software License', + 'Programming Language :: Python :: 2', 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3', 'Programming Language :: Python :: 3.4', ], ) From 7878d38deeac394e220cab294c3783eac63f17b5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 26 Oct 2015 13:29:59 -0400 Subject: [PATCH 035/359] Fix running one-off containers with --x-networking by disabling linking to self. docker create fails if networking and links are used together. Signed-off-by: Daniel Nephin --- compose/service.py | 3 +++ tests/unit/service_test.py | 8 ++++++++ 2 files changed, 11 insertions(+) diff --git a/compose/service.py b/compose/service.py index 3bb47432..370ab1eb 100644 --- a/compose/service.py +++ b/compose/service.py @@ -539,6 +539,9 @@ class Service(object): return 1 if not numbers else max(numbers) + 1 def _get_links(self, link_to_self): + if self.use_networking: + return [] + links = [] for service, link_name in self.links: for container in service.containers(): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index b5bac291..c7a5a355 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -499,6 +499,14 @@ class ServiceTest(unittest.TestCase): ports=["127.0.0.1:1000-2000:2000-3000"]) self.assertEqual(service.specifies_host_port(), True) + def test_get_links_with_networking(self): + service = Service( + 'foo', + image='foo', + links=[(Service('one'), 'one')], + use_networking=True) + self.assertEqual(service._get_links(link_to_self=True), []) + class NetTestCase(unittest.TestCase): From 7603ebea9b2f484f3cf8f193f64748fb91b67bf6 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 21 Oct 2015 17:28:16 +0100 Subject: [PATCH 036/359] Attach to a container's log_stream before they're started So we're not displaying output of all previous logs for a container, we attach, if possible, to a container before the container is started. LogPrinter checks if a container has a log_stream already attached and print from that rather than always attempting to attach one itself. Signed-off-by: Mazz Mosley --- compose/cli/log_printer.py | 10 ++++-- compose/cli/main.py | 11 +++++-- compose/container.py | 8 +++++ compose/project.py | 6 ++-- compose/service.py | 54 ++++++++++++++++++++++----------- tests/integration/state_test.py | 4 ++- 6 files changed, 66 insertions(+), 27 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 6e1499e1..66920726 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -73,9 +73,13 @@ def build_no_log_generator(container, prefix, color_func): def build_log_generator(container, prefix, color_func): - # Attach to container before log printer starts running - stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) - line_generator = split_buffer(stream) + # if the container doesn't have a log_stream we need to attach to container + # before log printer starts running + if container.log_stream is None: + stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) + line_generator = split_buffer(stream) + else: + line_generator = split_buffer(container.log_stream) for line in line_generator: yield prefix + line diff --git a/compose/cli/main.py b/compose/cli/main.py index c800d95f..5505b89f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -565,16 +565,18 @@ class TopLevelCommand(DocoptCommand): start_deps = not options['--no-deps'] service_names = options['SERVICE'] timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + detached = options.get('-d') to_attach = project.up( service_names=service_names, start_deps=start_deps, strategy=convergence_strategy_from_opts(options), do_build=not options['--no-build'], - timeout=timeout + timeout=timeout, + detached=detached ) - if not options['-d']: + if not detached: log_printer = build_log_printer(to_attach, service_names, monochrome) attach_to_logs(project, log_printer, service_names, timeout) @@ -636,7 +638,10 @@ def convergence_strategy_from_opts(options): def build_log_printer(containers, service_names, monochrome): if service_names: - containers = [c for c in containers if c.service in service_names] + containers = [ + container + for container in containers if container.service in service_names + ] return LogPrinter(containers, monochrome=monochrome) diff --git a/compose/container.py b/compose/container.py index a03acf56..64773b9e 100644 --- a/compose/container.py +++ b/compose/container.py @@ -19,6 +19,7 @@ class Container(object): self.client = client self.dictionary = dictionary self.has_been_inspected = has_been_inspected + self.log_stream = None @classmethod def from_ps(cls, client, dictionary, **kwargs): @@ -146,6 +147,13 @@ class Container(object): log_type = self.log_driver return not log_type or log_type == 'json-file' + def attach_log_stream(self): + """A log stream can only be attached if the container uses a json-file + log driver. + """ + if self.has_api_logs: + self.log_stream = self.attach(stdout=True, stderr=True, stream=True) + def get(self, key): """Return a value from the container or None if the value is not set. diff --git a/compose/project.py b/compose/project.py index d4934c26..68edaddc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -290,7 +290,8 @@ class Project(object): start_deps=True, strategy=ConvergenceStrategy.changed, do_build=True, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + detached=False): services = self.get_services(service_names, include_deps=start_deps) @@ -308,7 +309,8 @@ class Project(object): for container in service.execute_convergence_plan( plans[service.name], do_build=do_build, - timeout=timeout + timeout=timeout, + detached=detached ) ] diff --git a/compose/service.py b/compose/service.py index 3bb47432..aefeda31 100644 --- a/compose/service.py +++ b/compose/service.py @@ -395,11 +395,17 @@ class Service(object): def execute_convergence_plan(self, plan, do_build=True, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + detached=False): (action, containers) = plan + should_attach_logs = not detached if action == 'create': container = self.create_container(do_build=do_build) + + if should_attach_logs: + container.attach_log_stream() + self.start_container(container) return [container] @@ -407,15 +413,16 @@ class Service(object): elif action == 'recreate': return [ self.recreate_container( - c, - timeout=timeout + container, + timeout=timeout, + attach_logs=should_attach_logs ) - for c in containers + for container in containers ] elif action == 'start': - for c in containers: - self.start_container_if_stopped(c) + for container in containers: + self.start_container_if_stopped(container, attach_logs=should_attach_logs) return containers @@ -428,16 +435,7 @@ class Service(object): else: raise Exception("Invalid action: {}".format(action)) - def recreate_container(self, - container, - timeout=DEFAULT_TIMEOUT): - """Recreate a container. - - The original container is renamed to a temporary name so that data - volumes can be copied to the new container, before the original - container is removed. - """ - log.info("Recreating %s" % container.name) + def _recreate_stop_container(self, container, timeout): try: container.stop(timeout=timeout) except APIError as e: @@ -448,26 +446,46 @@ class Service(object): else: raise + def _recreate_rename_container(self, container): # Use a hopefully unique container name by prepending the short id self.client.rename( container.id, - '%s_%s' % (container.short_id, container.name)) + '%s_%s' % (container.short_id, container.name) + ) + def recreate_container(self, + container, + timeout=DEFAULT_TIMEOUT, + attach_logs=False): + """Recreate a container. + + The original container is renamed to a temporary name so that data + volumes can be copied to the new container, before the original + container is removed. + """ + log.info("Recreating %s" % container.name) + + self._recreate_stop_container(container, timeout) + self._recreate_rename_container(container) new_container = self.create_container( do_build=False, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), quiet=True, ) + if attach_logs: + new_container.attach_log_stream() self.start_container(new_container) container.remove() return new_container - def start_container_if_stopped(self, container): + def start_container_if_stopped(self, container, attach_logs=False): if container.is_running: return container else: log.info("Starting %s" % container.name) + if attach_logs: + container.attach_log_stream() return self.start_container(container) def start_container(self, container): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index ef7276bd..02e9d315 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -18,6 +18,7 @@ from compose.service import ConvergenceStrategy class ProjectTestCase(DockerClientTestCase): def run_up(self, cfg, **kwargs): kwargs.setdefault('timeout', 1) + kwargs.setdefault('detached', True) project = self.make_project(cfg) project.up(**kwargs) @@ -184,7 +185,8 @@ def converge(service, do_build=True): """Create a converge plan from a strategy and execute the plan.""" plan = service.convergence_plan(strategy) - return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) + containers, logging_threads = zip(*service.execute_convergence_plan(plan, do_build=do_build, timeout=1)) + return containers class ServiceStateTest(DockerClientTestCase): From bee063c07dd8ca8c7f0295a9a319c14b815c4422 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Mon, 26 Oct 2015 10:27:57 +0000 Subject: [PATCH 037/359] Fix tests Signed-off-by: Mazz Mosley --- tests/integration/service_test.py | 1 + tests/integration/state_test.py | 3 +-- tests/unit/cli/log_printer_test.py | 1 + tests/unit/service_test.py | 4 +--- 4 files changed, 4 insertions(+), 5 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8a8e4d54..38d7d5b5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -362,6 +362,7 @@ class ServiceTest(DockerClientTestCase): new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) + self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 02e9d315..3230aefc 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -185,8 +185,7 @@ def converge(service, do_build=True): """Create a converge plan from a strategy and execute the plan.""" plan = service.convergence_plan(strategy) - containers, logging_threads = zip(*service.execute_convergence_plan(plan, do_build=do_build, timeout=1)) - return containers + return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) class ServiceStateTest(DockerClientTestCase): diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 2c916898..575fcaf7 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -16,6 +16,7 @@ def build_mock_container(reader): name='myapp_web_1', name_without_project='web_1', has_api_logs=True, + log_stream=None, attach=reader, wait=mock.Mock(return_value=0), ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index b5bac291..494c2cde 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -323,9 +323,7 @@ class ServiceTest(unittest.TestCase): new_container = service.recreate_container(mock_container) mock_container.stop.assert_called_once_with(timeout=10) - self.mock_client.rename.assert_called_once_with( - mock_container.id, - '%s_%s' % (mock_container.short_id, mock_container.name)) + mock_container.rename_to_tmp_name.assert_called_once_with() new_container.start.assert_called_once_with() mock_container.remove.assert_called_once_with() From 30a84f1be6c68d1cbeaad865242f891f5cff82df Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 27 Oct 2015 11:55:35 +0000 Subject: [PATCH 038/359] Move rename functionality into Container Signed-off-by: Mazz Mosley --- compose/container.py | 9 +++++++++ compose/service.py | 9 +-------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/compose/container.py b/compose/container.py index 64773b9e..dd69e8dd 100644 --- a/compose/container.py +++ b/compose/container.py @@ -192,6 +192,15 @@ class Container(object): def remove(self, **options): return self.client.remove_container(self.id, **options) + def rename_to_tmp_name(self): + """Rename the container to a hopefully unique temporary container name + by prepending the short id. + """ + self.client.rename( + self.id, + '%s_%s' % (self.short_id, self.name) + ) + def inspect_if_not_inspected(self): if not self.has_been_inspected: self.inspect() diff --git a/compose/service.py b/compose/service.py index aefeda31..518a0d27 100644 --- a/compose/service.py +++ b/compose/service.py @@ -446,13 +446,6 @@ class Service(object): else: raise - def _recreate_rename_container(self, container): - # Use a hopefully unique container name by prepending the short id - self.client.rename( - container.id, - '%s_%s' % (container.short_id, container.name) - ) - def recreate_container(self, container, timeout=DEFAULT_TIMEOUT, @@ -466,7 +459,7 @@ class Service(object): log.info("Recreating %s" % container.name) self._recreate_stop_container(container, timeout) - self._recreate_rename_container(container) + container.rename_to_tmp_name() new_container = self.create_container( do_build=False, previous_container=container, From 76d52b1c5f02e526010468c22d5a438883ab4777 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 27 Oct 2015 11:59:09 +0000 Subject: [PATCH 039/359] Remove redundant try/except Code cleanup. We no longer need this as the api returns a 304 for any stopped containers, which doesn't raise an error. Signed-off-by: Mazz Mosley --- compose/service.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/compose/service.py b/compose/service.py index 518a0d27..2ca004cf 100644 --- a/compose/service.py +++ b/compose/service.py @@ -435,17 +435,6 @@ class Service(object): else: raise Exception("Invalid action: {}".format(action)) - def _recreate_stop_container(self, container, timeout): - try: - container.stop(timeout=timeout) - except APIError as e: - if (e.response.status_code == 500 - and e.explanation - and 'no such process' in str(e.explanation)): - pass - else: - raise - def recreate_container(self, container, timeout=DEFAULT_TIMEOUT, @@ -458,7 +447,7 @@ class Service(object): """ log.info("Recreating %s" % container.name) - self._recreate_stop_container(container, timeout) + container.stop(timeout=timeout) container.rename_to_tmp_name() new_container = self.create_container( do_build=False, From 379af594dab93e608d9589bb41c755429190a71d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 28 Oct 2015 14:59:43 +0000 Subject: [PATCH 040/359] Include link to github for code&issues Signed-off-by: Mazz Mosley --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index d779d607..ed176550 100644 --- a/README.md +++ b/README.md @@ -45,6 +45,8 @@ Installation and documentation - Full documentation is available on [Docker's website](http://docs.docker.com/compose/). - If you have any questions, you can talk in real-time with other developers in the #docker-compose IRC channel on Freenode. [Click here to join using IRCCloud.](https://www.irccloud.com/invite?hostname=irc.freenode.net&channel=%23docker-compose) +- Code repository for Compose is on [Github](https://github.com/docker/compose) +- If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new) Contributing ------------ From c4f0f24c57365a2bada4be295778bf93f506ece4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 12:01:34 -0400 Subject: [PATCH 041/359] Fix release script notes about software and typos. Signed-off-by: Daniel Nephin --- project/RELEASE-PROCESS.md | 12 ++++++++++-- script/release/cherry-pick-pr | 2 +- script/release/push-release | 5 ++++- 3 files changed, 15 insertions(+), 4 deletions(-) diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index ffa18077..040a2602 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -1,6 +1,14 @@ Building a Compose release ========================== +## Prerequisites + +The release scripts require the following tools installed on the host: + +* https://hub.github.com/ +* https://stedolan.github.io/jq/ +* http://pandoc.org/ + ## To get started with a new release Create a branch, update version, and add release notes by running `make-branch` @@ -40,10 +48,10 @@ As part of this script you'll be asked to: ## To release a version (whether RC or stable) -Check out the bump branch and run the `build-binary` script +Check out the bump branch and run the `build-binaries` script git checkout bump-$VERSION - ./script/release/build-binary + ./script/release/build-binaries When prompted build the non-linux binaries and test them. diff --git a/script/release/cherry-pick-pr b/script/release/cherry-pick-pr index 60460087..f4a5a740 100755 --- a/script/release/cherry-pick-pr +++ b/script/release/cherry-pick-pr @@ -22,7 +22,7 @@ EOM if [ -z "$(command -v hub 2> /dev/null)" ]; then >&2 echo "$0 requires https://hub.github.com/." - >&2 echo "Please install it and ake sure it is available on your \$PATH." + >&2 echo "Please install it and make sure it is available on your \$PATH." exit 2 fi diff --git a/script/release/push-release b/script/release/push-release index 039436da..9229f093 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -34,7 +34,9 @@ GITHUB_REPO=git@github.com:$REPO sha=$(git rev-parse HEAD) url=$API/$REPO/statuses/$sha build_status=$(curl -s $url | jq -r '.[0].state') -if [[ "$build_status" != "success" ]]; then +if [ -n "$SKIP_BUILD_CHECK" ]; then + echo "Skipping build status check..." +elif [[ "$build_status" != "success" ]]; then >&2 echo "Build status is $build_status, but it should be success." exit -1 fi @@ -61,6 +63,7 @@ source venv-test/bin/activate pip install docker-compose==$VERSION docker-compose version deactivate +rm -rf venv-test echo "Now publish the github release, and test the downloads." echo "Email maintainers@dockerproject.org and engineering@docker.com about the new release." From bbc76e6034d426e75dba9f2c2517e7f82f7ea1f8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 12:11:29 -0400 Subject: [PATCH 042/359] Convert the README to rst and fix the logo url before packaging it up for pypi. Signed-off-by: Daniel Nephin --- .gitignore | 1 + MANIFEST.in | 2 ++ script/release/push-release | 15 +++++++++++---- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 1b0c5011..83a08a0e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /dist /docs/_site /venv +README.rst diff --git a/MANIFEST.in b/MANIFEST.in index 43ae06d3..0342e35b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,8 @@ include requirements.txt include requirements-dev.txt include tox.ini include *.md +exclude README.md +include README.rst include compose/config/*.json recursive-include contrib/completion * recursive-include tests * diff --git a/script/release/push-release b/script/release/push-release index 9229f093..ccdf2496 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -21,11 +21,17 @@ VERSION="$(git config "branch.${BRANCH}.release")" || usage if [ -z "$(command -v jq 2> /dev/null)" ]; then >&2 echo "$0 requires https://stedolan.github.io/jq/" - >&2 echo "Please install it and ake sure it is available on your \$PATH." + >&2 echo "Please install it and make sure it is available on your \$PATH." exit 2 fi +if [ -z "$(command -v pandoc 2> /dev/null)" ]; then + >&2 echo "$0 requires http://pandoc.org/" + >&2 echo "Please install it and make sure it is available on your \$PATH." + exit 2 +fi + API=https://api.github.com/repos REPO=docker/compose GITHUB_REPO=git@github.com:$REPO @@ -45,12 +51,13 @@ echo "Tagging the release as $VERSION" git tag $VERSION git push $GITHUB_REPO $VERSION -echo "Uploading sdist to pypi" -python setup.py sdist - echo "Uploading the docker image" docker push docker/compose:$VERSION +echo "Uploading sdist to pypi" +pandoc -f markdown -t rst README.md -o README.rst +sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst +python setup.py sdist if [ "$(command -v twine 2> /dev/null)" ]; then twine upload ./dist/docker-compose-${VERSION}.tar.gz else From 830640534053df40d06e63e04b63c15d8929ae9c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 12:40:59 -0400 Subject: [PATCH 043/359] On error print daemon logs Signed-off-by: Daniel Nephin --- script/test-versions | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/script/test-versions b/script/test-versions index 89793359..43326ccb 100755 --- a/script/test-versions +++ b/script/test-versions @@ -28,10 +28,15 @@ for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" daemon_container="compose-dind-$version-$BUILD_NUMBER" - trap "docker rm -vf $daemon_container" EXIT - # TODO: remove when we stop testing against 1.7.x - daemon=$([[ "$version" == "1.7"* ]] && echo "-d" || echo "daemon") + function on_exit() { + if [[ "$?" != "0" ]]; then + docker logs "$daemon_container" + fi + docker rm -vf "$daemon_container" + } + + trap "on_exit" EXIT docker run \ -d \ @@ -39,7 +44,7 @@ for version in $DOCKER_VERSIONS; do --privileged \ --volume="/var/lib/docker" \ dockerswarm/dind:$version \ - docker $daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ docker run \ --rm \ From c341860d113e4ec042fa4d9bb7340fed9d9db66d Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 28 Oct 2015 16:48:35 +0000 Subject: [PATCH 044/359] Clarify `dockerfile` requires `build` key Credit to @funkyfuture for the first PR addressing the clarification. https://github.com/docker/compose/pull/1767 Signed-off-by: Mazz Mosley --- docs/compose-file.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index b72a7cc4..d4591608 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -100,8 +100,10 @@ Custom DNS search domains. Can be a single value or a list. Alternate Dockerfile. -Compose will use an alternate file to build with. +Compose will use an alternate file to build with. A build path must also be +specified using the `build` key. + build: /path/to/build/dir dockerfile: Dockerfile-alternate Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. From e13b8949b0546a4c2028b3457f5934c2ae4c1f0b Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Wed, 28 Oct 2015 16:27:14 +0000 Subject: [PATCH 045/359] Add cross references for env/cli Signed-off-by: Mazz Mosley --- docs/reference/docker-compose.md | 15 +++++++++------ docs/reference/overview.md | 9 +++++++-- 2 files changed, 16 insertions(+), 8 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index 32fcbe70..8712072e 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -87,15 +87,18 @@ relative to the current working directory. The `-f` flag is optional. If you don't provide this flag on the command line, Compose traverses the working directory and its subdirectories looking for a -`docker-compose.yml` and a `docker-compose.override.yml` file. You must supply -at least the `docker-compose.yml` file. If both files are present, Compose -combines the two files into a single configuration. The configuration in the -`docker-compose.override.yml` file is applied over and in addition to the values -in the `docker-compose.yml` file. +`docker-compose.yml` and a `docker-compose.override.yml` file. You must +supply at least the `docker-compose.yml` file. If both files are present, +Compose combines the two files into a single configuration. The configuration +in the `docker-compose.override.yml` file is applied over and in addition to +the values in the `docker-compose.yml` file. + +See also the `COMPOSE_FILE` [environment variable](overview.md#compose-file). Each configuration has a project name. If you supply a `-p` flag, you can specify a project name. If you don't specify the flag, Compose uses the current -directory name. +directory name. See also the `COMPOSE_PROJECT_NAME` [environment variable]( +overview.md#compose-project-name) ## Where to go next diff --git a/docs/reference/overview.md b/docs/reference/overview.md index 3f589a9d..8e3967b2 100644 --- a/docs/reference/overview.md +++ b/docs/reference/overview.md @@ -32,11 +32,16 @@ Docker command-line client. If you're using `docker-machine`, then the `eval "$( Sets the project name. This value is prepended along with the service name to the container container on start up. For example, if you project name is `myapp` and it includes two services `db` and `web` then compose starts containers named `myapp_db_1` and `myapp_web_1` respectively. -Setting this is optional. If you do not set this, the `COMPOSE_PROJECT_NAME` defaults to the `basename` of the current working directory. +Setting this is optional. If you do not set this, the `COMPOSE_PROJECT_NAME` +defaults to the `basename` of the project directory. See also the `-p` +[command-line option](docker-compose.md). ### COMPOSE\_FILE -Specify the file containing the compose configuration. If not provided, Compose looks for a file named `docker-compose.yml` in the current directory and then each parent directory in succession until a file by that name is found. +Specify the file containing the compose configuration. If not provided, +Compose looks for a file named `docker-compose.yml` in the current directory +and then each parent directory in succession until a file by that name is +found. See also the `-f` [command-line option](docker-compose.md). ### COMPOSE\_API\_VERSION From 0ef3b47f74afe2a835ae0b9ad60c92b8c27612f2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 15:17:11 -0400 Subject: [PATCH 046/359] Update docs about networking for current release. Signed-off-by: Daniel Nephin --- docs/networking.md | 36 ++++++++++++++++++++++-------------- project/ISSUE-TRIAGE.md | 25 +++++++++++++------------ 2 files changed, 35 insertions(+), 26 deletions(-) diff --git a/docs/networking.md b/docs/networking.md index 9a6d792d..718d56c7 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -14,7 +14,11 @@ weight=6 > **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. -Compose sets up a single default [network](/engine/reference/commandline/network_create.md) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. +Compose sets up a single default +[network](/engine/reference/commandline/network_create.md) for your app. Each +container for a service joins the default network and is both *reachable* by +other containers on that network, and *discoverable* by them at a hostname +identical to the container name. > **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. @@ -30,13 +34,23 @@ For example, suppose your app is in a directory called `myapp`, and your `docker When you run `docker-compose --x-networking up`, the following happens: 1. A network called `myapp` is created. -2. A container is created using `web`'s configuration. It joins the network `myapp` under the name `web`. -3. A container is created using `db`'s configuration. It joins the network `myapp` under the name `db`. +2. A container is created using `web`'s configuration. It joins the network +`myapp` under the name `myapp_web_1`. +3. A container is created using `db`'s configuration. It joins the network +`myapp` under the name `myapp_db_1`. -Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s application code could connect to the URL `postgres://db:5432` and start using the Postgres database. +Each container can now look up the hostname `myapp_web_1` or `myapp_db_1` and +get back the appropriate container's IP address. For example, `web`'s +application code could connect to the URL `postgres://myapp_db_1:5432` and start +using the Postgres database. Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. +> **Note:** in the next release there will be additional aliases for the +> container, including a short name without the project name and container +> index. The full container name will remain as one of the alias for backwards +> compatibility. + ## Updating containers If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. @@ -45,19 +59,13 @@ If any containers have connections open to the old container, they will be close ## Configure how services are published -By default, containers for each service are published on the network with the same name as the service. If you want to change the name, or stop containers from being discoverable at all, you can use the `hostname` option: +By default, containers for each service are published on the network with the +container name. If you want to change the name, or stop containers from being +discoverable at all, you can use the `container_name` option: web: build: . - hostname: "my-web-application" - -This will also change the hostname inside the container, so that the `hostname` command will return `my-web-application`. - -## Scaling services - -If you create multiple containers for a service with `docker-compose scale`, each container will join the network with the same name. For example, if you run `docker-compose scale web=3`, then 3 containers will join the network under the name `web`. Inside any container on the network, looking up the name `web` will return the IP address of one of them, but Docker and Compose do not provide any guarantees about which one. - -This limitation will be addressed in a future version of Compose, where a load balancer will join under the service name and balance traffic between the service's containers in a configurable manner. + container_name: "my-web-application" ## Links diff --git a/project/ISSUE-TRIAGE.md b/project/ISSUE-TRIAGE.md index 58312a60..b89cdc24 100644 --- a/project/ISSUE-TRIAGE.md +++ b/project/ISSUE-TRIAGE.md @@ -20,15 +20,16 @@ The following labels are provided in additional to the standard labels: Most issues should fit into one of the following functional areas: -| Area | -|----------------| -| area/build | -| area/cli | -| area/config | -| area/logs | -| area/packaging | -| area/run | -| area/scale | -| area/tests | -| area/up | -| area/volumes | +| Area | +|-----------------| +| area/build | +| area/cli | +| area/config | +| area/logs | +| area/networking | +| area/packaging | +| area/run | +| area/scale | +| area/tests | +| area/up | +| area/volumes | From a71d9af522611947a74ffd7e0d45b720e9b69f98 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 17:54:38 -0400 Subject: [PATCH 047/359] Disable a test against docker 1.8.3 because it fails due to a bug in docker engine. Signed-off-by: Daniel Nephin --- tests/integration/service_test.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 38d7d5b5..4ac04545 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -693,10 +693,11 @@ class ServiceTest(DockerClientTestCase): @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): - """ - Test that calling scale on a service that has a custom container name + """Test that calling scale on a service that has a custom container name results in warning output. """ + # Disable this test against earlier versions because it is flaky + self.require_api_version('1.21') service = self.create_service('app', container_name='custom-container') self.assertEqual(service.custom_container_name(), 'custom-container') From 5dc14f39254a25cf87b104c9f2577f37e48c6de4 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Tue, 27 Oct 2015 17:37:48 +0000 Subject: [PATCH 048/359] Handle non-ascii chars in volume directories Signed-off-by: Mazz Mosley --- compose/config/config.py | 2 +- tests/unit/config/config_test.py | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 40b4ffa4..5bc534fe 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -505,7 +505,7 @@ def resolve_volume_path(volume, working_dir, service_name): if host_path.startswith('.'): host_path = expand_path(working_dir, host_path) host_path = os.path.expanduser(host_path) - return "{}:{}".format(host_path, container_path) + return u"{}:{}".format(host_path, container_path) else: return container_path diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a54b006f..5ad7e1c0 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -573,6 +573,11 @@ class VolumeConfigTest(unittest.TestCase): }, working_dir='.') self.assertEqual(d['volumes'], ['~:/data']) + def test_volume_path_with_non_ascii_directory(self): + volume = u'/Füü/data:/data' + container_path = config.resolve_volume_path(volume, ".", "test") + self.assertEqual(container_path, volume) + class MergePathMappingTest(object): def config_name(self): From eab9d86a3d016e7f846514ae81d7fd5b16c23257 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 11:28:17 -0400 Subject: [PATCH 049/359] Logs are available for all log drivers except for none. Signed-off-by: Daniel Nephin --- compose/container.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/container.py b/compose/container.py index dd69e8dd..1ca48380 100644 --- a/compose/container.py +++ b/compose/container.py @@ -145,7 +145,7 @@ class Container(object): @property def has_api_logs(self): log_type = self.log_driver - return not log_type or log_type == 'json-file' + return not log_type or log_type != 'none' def attach_log_stream(self): """A log stream can only be attached if the container uses a json-file From 983dc12160915f1148c35296712290702cb32338 Mon Sep 17 00:00:00 2001 From: Mazz Mosley Date: Thu, 29 Oct 2015 16:52:00 +0000 Subject: [PATCH 050/359] Clarify the command is an example Signed-off-by: Mazz Mosley --- docs/install.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/install.md b/docs/install.md index 2d4d6cad..944ce349 100644 --- a/docs/install.md +++ b/docs/install.md @@ -30,13 +30,14 @@ To install Compose, do the following: 3. Go to the Compose repository release page on GitHub. -4. Follow the instructions from the release page and run the `curl` command in your terminal. +4. Follow the instructions from the release page and run the `curl` command, +which the release page specifies, in your terminal. > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands below, then `exit`. - The command has the following format: + The following is an example command illustrating the format: curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose From 596261e75985ffa7ed07c2dedfdc9d763e988db0 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 29 Oct 2015 17:56:01 +0100 Subject: [PATCH 051/359] Ensure network exists when calling run before up Otherwise the daemon will error out because the network doesn't exist yet. Signed-off-by: Joffrey F --- compose/cli/main.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 5505b89f..4369aa70 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -380,6 +380,8 @@ class TopLevelCommand(DocoptCommand): start_deps=True, strategy=ConvergenceStrategy.never, ) + elif project.use_networking: + project.ensure_network_exists() tty = True if detach or options['-T'] or not sys.stdin.isatty(): From d836973a04fbba01e693e74c4124f2dbceb4b57b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 14:06:50 -0400 Subject: [PATCH 052/359] Use colors when logging warnings or errors, so they are more obvious. Signed-off-by: Daniel Nephin --- compose/cli/formatter.py | 23 +++++++++++++++++++++++ compose/cli/main.py | 22 ++++++++++++++-------- compose/config/interpolation.py | 2 +- tests/unit/cli/main_test.py | 29 +++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 9 deletions(-) diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index 9ed52c4a..d0ed0f87 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -1,10 +1,13 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging import os import texttable +from compose.cli import colors + def get_tty_width(): tty_size = os.popen('stty size', 'r').read().split() @@ -15,6 +18,7 @@ def get_tty_width(): class Formatter(object): + """Format tabular data for printing.""" def table(self, headers, rows): table = texttable.Texttable(max_width=get_tty_width()) table.set_cols_dtype(['t' for h in headers]) @@ -23,3 +27,22 @@ class Formatter(object): table.set_chars(['-', '|', '+', '-']) return table.draw() + + +class ConsoleWarningFormatter(logging.Formatter): + """A logging.Formatter which prints WARNING and ERROR messages with + a prefix of the log level colored appropriate for the log level. + """ + + def get_level_message(self, record): + separator = ': ' + if record.levelno == logging.WARNING: + return colors.yellow(record.levelname) + separator + if record.levelno == logging.ERROR: + return colors.red(record.levelname) + separator + + return '' + + def format(self, record): + message = super(ConsoleWarningFormatter, self).format(record) + return self.get_level_message(record) + message diff --git a/compose/cli/main.py b/compose/cli/main.py index 5505b89f..1542f52f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -28,6 +28,7 @@ from .command import project_from_options from .docopt_command import DocoptCommand from .docopt_command import NoSuchCommand from .errors import UserError +from .formatter import ConsoleWarningFormatter from .formatter import Formatter from .log_printer import LogPrinter from .utils import get_version_info @@ -41,7 +42,7 @@ log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) INSECURE_SSL_WARNING = """ -Warning: --allow-insecure-ssl is deprecated and has no effect. +--allow-insecure-ssl is deprecated and has no effect. It will be removed in a future version of Compose. """ @@ -91,13 +92,18 @@ def setup_logging(): logging.getLogger("requests").propagate = False -def setup_console_handler(verbose): - if verbose: - console_handler.setFormatter(logging.Formatter('%(name)s.%(funcName)s: %(message)s')) - console_handler.setLevel(logging.DEBUG) +def setup_console_handler(handler, verbose): + if handler.stream.isatty(): + format_class = ConsoleWarningFormatter else: - console_handler.setFormatter(logging.Formatter()) - console_handler.setLevel(logging.INFO) + format_class = logging.Formatter + + if verbose: + handler.setFormatter(format_class('%(name)s.%(funcName)s: %(message)s')) + handler.setLevel(logging.DEBUG) + else: + handler.setFormatter(format_class()) + handler.setLevel(logging.INFO) # stolen from docopt master @@ -153,7 +159,7 @@ class TopLevelCommand(DocoptCommand): return options def perform_command(self, options, handler, command_options): - setup_console_handler(options.get('--verbose')) + setup_console_handler(console_handler, options.get('--verbose')) if options['COMMAND'] in ('help', 'version'): # Skip looking up the compose file. diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index f870ab4b..f8e1da61 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -78,7 +78,7 @@ class BlankDefaultDict(dict): except KeyError: if key not in self.missing_keys: log.warn( - "The {} variable is not set. Substituting a blank string." + "The {} variable is not set. Defaulting to a blank string." .format(key) ) self.missing_keys.append(key) diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index a5b36980..ee837fcd 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -1,11 +1,15 @@ from __future__ import absolute_import +import logging + from compose import container from compose.cli.errors import UserError +from compose.cli.formatter import ConsoleWarningFormatter from compose.cli.log_printer import LogPrinter from compose.cli.main import attach_to_logs from compose.cli.main import build_log_printer from compose.cli.main import convergence_strategy_from_opts +from compose.cli.main import setup_console_handler from compose.project import Project from compose.service import ConvergenceStrategy from tests import mock @@ -60,6 +64,31 @@ class CLIMainTestCase(unittest.TestCase): timeout=timeout) +class SetupConsoleHandlerTestCase(unittest.TestCase): + + def setUp(self): + self.stream = mock.Mock() + self.stream.isatty.return_value = True + self.handler = logging.StreamHandler(stream=self.stream) + + def test_with_tty_verbose(self): + setup_console_handler(self.handler, True) + assert type(self.handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' in self.handler.formatter._fmt + assert '%(funcName)s' in self.handler.formatter._fmt + + def test_with_tty_not_verbose(self): + setup_console_handler(self.handler, False) + assert type(self.handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' not in self.handler.formatter._fmt + assert '%(funcName)s' not in self.handler.formatter._fmt + + def test_with_not_a_tty(self): + self.stream.isatty.return_value = False + setup_console_handler(self.handler, False) + assert type(self.handler.formatter) == logging.Formatter + + class ConvergeStrategyFromOptsTestCase(unittest.TestCase): def test_invalid_opts(self): From 841ed4ed218e9d281a10a098d5f2ca81a5d4c46c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 14:15:08 -0400 Subject: [PATCH 053/359] Remove the duplicate 'Warning' prefix now that the logger adds the prefix. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 5 ++--- compose/config/validation.py | 8 +++++--- compose/service.py | 2 +- tests/unit/cli/formatter_test.py | 35 ++++++++++++++++++++++++++++++++ tests/unit/config/config_test.py | 2 +- 5 files changed, 44 insertions(+), 8 deletions(-) create mode 100644 tests/unit/cli/formatter_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 1542f52f..34c63e77 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -59,9 +59,8 @@ def main(): log.error(e.msg) sys.exit(1) except NoSuchCommand as e: - log.error("No such command: %s", e.command) - log.error("") - log.error("\n".join(parse_doc_section("commands:", getdoc(e.supercommand)))) + commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) + log.error("No such command: %s\n\n%s", e.command, commands) sys.exit(1) except APIError as e: log.error(e.explanation) diff --git a/compose/config/validation.py b/compose/config/validation.py index 8cfc405f..542081d5 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -57,9 +57,11 @@ def format_boolean_in_environment(instance): """ if isinstance(instance, bool): log.warn( - "Warning: There is a boolean value in the 'environment' key.\n" - "Environment variables can only be strings.\nPlease add quotes to any boolean values to make them string " - "(eg, 'True', 'yes', 'N').\nThis warning will become an error in a future release. \r\n" + "There is a boolean value in the 'environment' key.\n" + "Environment variables can only be strings.\n" + "Please add quotes to any boolean values to make them string " + "(eg, 'True', 'yes', 'N').\n" + "This warning will become an error in a future release. \r\n" ) return True diff --git a/compose/service.py b/compose/service.py index ad29f87f..8d716c0b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -862,7 +862,7 @@ class ServiceNet(object): if containers: return 'container:' + containers[0].id - log.warn("Warning: Service %s is trying to use reuse the network stack " + log.warn("Service %s is trying to use reuse the network stack " "of another service that is not running." % (self.id)) return None diff --git a/tests/unit/cli/formatter_test.py b/tests/unit/cli/formatter_test.py new file mode 100644 index 00000000..1c3b6a68 --- /dev/null +++ b/tests/unit/cli/formatter_test.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import logging + +from compose.cli import colors +from compose.cli.formatter import ConsoleWarningFormatter +from tests import unittest + + +MESSAGE = 'this is the message' + + +def makeLogRecord(level): + return logging.LogRecord('name', level, 'pathame', 0, MESSAGE, (), None) + + +class ConsoleWarningFormatterTestCase(unittest.TestCase): + + def setUp(self): + self.formatter = ConsoleWarningFormatter() + + def test_format_warn(self): + output = self.formatter.format(makeLogRecord(logging.WARN)) + expected = colors.yellow('WARNING') + ': ' + assert output == expected + MESSAGE + + def test_format_error(self): + output = self.formatter.format(makeLogRecord(logging.ERROR)) + expected = colors.red('ERROR') + ': ' + assert output == expected + MESSAGE + + def test_format_info(self): + output = self.formatter.format(makeLogRecord(logging.INFO)) + assert output == MESSAGE diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 5ad7e1c0..7246b661 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -380,7 +380,7 @@ class ConfigTest(unittest.TestCase): @mock.patch('compose.config.validation.log') def test_logs_warning_for_boolean_in_environment(self, mock_logging): - expected_warning_msg = "Warning: There is a boolean value in the 'environment' key." + expected_warning_msg = "There is a boolean value in the 'environment' key." config.load( build_config_details( {'web': { From 072e7687ae1e710ee8b82b44f5baba8f20caa4cb Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 30 Oct 2015 14:15:47 +0100 Subject: [PATCH 054/359] Integration test for run command with networking enabled Signed-off-by: Joffrey F --- tests/integration/cli_test.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 19cc822e..45f45645 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -508,6 +508,20 @@ class CLITestCase(DockerClientTestCase): container, = service.containers(stopped=True, one_off=True) self.assertEqual(container.name, name) + @mock.patch('dockerpty.start') + def test_run_with_networking(self, _): + self.require_api_version('1.21') + client = docker_client(version='1.21') + self.command.base_dir = 'tests/fixtures/simple-dockerfile' + self.command.dispatch(['--x-networking', 'run', 'simple', 'true'], None) + service = self.project.get_service('simple') + container, = service.containers(stopped=True, one_off=True) + networks = client.networks(names=[self.project.name]) + for n in networks: + self.addCleanup(client.remove_network, n['id']) + self.assertEqual(len(networks), 1) + self.assertEqual(container.human_readable_command, u'true') + def test_rm(self): service = self.project.get_service('simple') service.create_container() From 491d052088c3c5d66dba0bde175047748c44c73f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 11:53:36 -0400 Subject: [PATCH 055/359] Don't set a default network driver, let the server decide. Signed-off-by: Daniel Nephin --- compose/project.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 68edaddc..1e01eaf6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -83,7 +83,7 @@ class Project(object): self.services = services self.client = client self.use_networking = use_networking - self.network_driver = network_driver or 'bridge' + self.network_driver = network_driver def labels(self, one_off=False): return [ From c7d164d01c82e1349a1d9ef2beae1754dbc3500a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Oct 2015 15:04:35 -0400 Subject: [PATCH 056/359] Fixes #1843, #1936 - chown files back to host user in django example. Also add a missing 'touch Gemfile.lock' to fix the rails tutorial. Signed-off-by: Daniel Nephin --- docs/django.md | 16 ++++++++++++++-- docs/rails.md | 14 ++++++++++++-- 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/docs/django.md b/docs/django.md index fd18784e..2bb67399 100644 --- a/docs/django.md +++ b/docs/django.md @@ -110,8 +110,20 @@ In this step, you create a Django started project by building the image from the 3. After the `docker-compose` command completes, list the contents of your project. - $ ls - Dockerfile docker-compose.yml composeexample manage.py requirements.txt + $ ls -l + drwxr-xr-x 2 root root composeexample + -rw-rw-r-- 1 user user docker-compose.yml + -rw-rw-r-- 1 user user Dockerfile + -rwxr-xr-x 1 root root manage.py + -rw-rw-r-- 1 user user requirements.txt + + The files `django-admin` created are owned by root. This happens because + the container runs as the `root` user. + +4. Change the ownership of the new files. + + sudo chown -R $USER:$USER . + ## Connect the database diff --git a/docs/rails.md b/docs/rails.md index a33cac26..e81675c5 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,6 +37,10 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' +You'll need an empty `Gemfile.lock` in order to build our `Dockerfile`. + + $ touch Gemfile.lock + Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. db: @@ -69,6 +73,12 @@ image. Once it's done, you should have generated a fresh app: README.rdoc config.ru public Rakefile db test + +The files `rails new` created are owned by root. This happens because the +container runs as the `root` user. Change the ownership of the new files. + + sudo chown -R $USER:$USER . + Uncomment the line in your new `Gemfile` which loads `therubyracer`, so you've got a Javascript runtime: @@ -80,6 +90,7 @@ rebuild.) $ docker-compose build + ### Connect the database The app is now bootable, but you're not quite there yet. By default, Rails @@ -87,8 +98,7 @@ expects a database to be running on `localhost` - so you need to point it at the `db` container instead. You also need to change the database and username to align with the defaults set by the `postgres` image. -Open up your newly-generated `database.yml` file. Replace its contents with the -following: +Replace the contents of `config/database.yml` with the following: development: &default adapter: postgresql From 1b5b40761943bf9e358f7f70859d62097cdb4f1b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 15:26:56 -0400 Subject: [PATCH 057/359] Fix networking tests to work with new API in engine rc4 (https://github.com/docker/docker/pull/17536) Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 12 ++++++------ tests/integration/project_test.py | 2 +- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 45f45645..d621f2d1 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -215,17 +215,17 @@ class CLITestCase(DockerClientTestCase): networks = client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(client.remove_network, n['id']) + self.addCleanup(client.remove_network, n['Id']) self.assertEqual(len(networks), 1) - self.assertEqual(networks[0]['driver'], 'bridge') + self.assertEqual(networks[0]['Driver'], 'bridge') - network = client.inspect_network(networks[0]['id']) - self.assertEqual(len(network['containers']), len(services)) + network = client.inspect_network(networks[0]['Id']) + self.assertEqual(len(network['Containers']), len(services)) for service in services: containers = service.containers() self.assertEqual(len(containers), 1) - self.assertIn(containers[0].id, network['containers']) + self.assertIn(containers[0].id, network['Containers']) self.assertEqual(containers[0].get('Config.Hostname'), service.name) web_container = self.project.get_service('web').containers()[0] @@ -518,7 +518,7 @@ class CLITestCase(DockerClientTestCase): container, = service.containers(stopped=True, one_off=True) networks = client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(client.remove_network, n['id']) + self.addCleanup(client.remove_network, n['Id']) self.assertEqual(len(networks), 1) self.assertEqual(container.human_readable_command, u'true') diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index fd45b939..95052387 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -111,7 +111,7 @@ class ProjectTest(DockerClientTestCase): network_name = 'network_does_exist' project = Project(network_name, [], client) client.create_network(network_name) - assert project.get_network()['name'] == network_name + assert project.get_network()['Name'] == network_name def test_net_from_service(self): project = Project.from_dicts( From abde64d610135e957aeb76958fdf384882fcc717 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 13:43:29 -0500 Subject: [PATCH 058/359] On a test failure only show the last 100 lines of daemon output. Signed-off-by: Daniel Nephin --- script/build-linux-inner | 2 +- script/test-versions | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/script/build-linux-inner b/script/build-linux-inner index 1d0f7905..01137ff2 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -8,7 +8,7 @@ VENV=/code/.tox/py27 mkdir -p `pwd`/dist chmod 777 `pwd`/dist -$VENV/bin/pip install -r requirements-build.txt +$VENV/bin/pip install -q -r requirements-build.txt su -c "$VENV/bin/pyinstaller docker-compose.spec" user mv dist/docker-compose $TARGET $TARGET version diff --git a/script/test-versions b/script/test-versions index 43326ccb..623b107b 100755 --- a/script/test-versions +++ b/script/test-versions @@ -31,7 +31,7 @@ for version in $DOCKER_VERSIONS; do function on_exit() { if [[ "$?" != "0" ]]; then - docker logs "$daemon_container" + docker logs "$daemon_container" 2>&1 | tail -n 100 fi docker rm -vf "$daemon_container" } @@ -45,6 +45,7 @@ for version in $DOCKER_VERSIONS; do --volume="/var/lib/docker" \ dockerswarm/dind:$version \ docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + 2>&1 | tail -n 10 docker run \ --rm \ From 53a0de7cf2231da88c7501bb367eb4e7f1c4d425 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Oct 2015 16:29:36 -0400 Subject: [PATCH 059/359] Add missing title to compose file reference. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/compose-file.md b/docs/compose-file.md index d4591608..7723a784 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -24,6 +24,11 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +## Service configuration reference + +This section contains a list of all configuration options supported by a service +definition. + ### build Path to a directory containing a Dockerfile. When the value supplied is a From 186d43c59f82e4210b379885368eb73a4162003d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 10:49:41 -0400 Subject: [PATCH 060/359] Extract the getting started guide from the index page. Signed-off-by: Daniel Nephin --- docs/completion.md | 2 +- docs/django.md | 1 + docs/extends.md | 1 + docs/gettingstarted.md | 163 +++++++++++++++++++++++++++++++++++++++++ docs/index.md | 139 +---------------------------------- docs/install.md | 1 + docs/production.md | 3 - docs/rails.md | 2 +- docs/wordpress.md | 2 +- 9 files changed, 170 insertions(+), 144 deletions(-) create mode 100644 docs/gettingstarted.md diff --git a/docs/completion.md b/docs/completion.md index bc8bedc9..3c2022d8 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -5,7 +5,7 @@ description = "Compose CLI reference" keywords = ["fig, composition, compose, docker, orchestration, cli, reference"] [menu.main] parent="smn_workw_compose" -weight=3 +weight=10 +++ diff --git a/docs/django.md b/docs/django.md index 2bb67399..d4d2bd1e 100644 --- a/docs/django.md +++ b/docs/django.md @@ -173,6 +173,7 @@ In this section, you set up the database connection for Django. - [User guide](../index.md) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) diff --git a/docs/extends.md b/docs/extends.md index f0b9e9ea..e63cf466 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -360,6 +360,7 @@ locally-defined bindings taking precedence: - [User guide](/) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md new file mode 100644 index 00000000..f2024b39 --- /dev/null +++ b/docs/gettingstarted.md @@ -0,0 +1,163 @@ + + + +## Getting Started + +Let's get started with a walkthrough of getting a simple Python web app running +on Compose. It assumes a little knowledge of Python, but the concepts +demonstrated here should be understandable even if you're not familiar with +Python. + +### Installation and set-up + +First, [install Docker and Compose](install.md). + +Next, you'll want to make a directory for the project: + + $ mkdir composetest + $ cd composetest + +Inside this directory, create `app.py`, a simple Python web app that uses the Flask +framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): + + from flask import Flask + from redis import Redis + + app = Flask(__name__) + redis = Redis(host='redis', port=6379) + + @app.route('/') + def hello(): + redis.incr('hits') + return 'Hello World! I have been seen %s times.' % redis.get('hits') + + if __name__ == "__main__": + app.run(host="0.0.0.0", debug=True) + +Next, define the Python dependencies in a file called `requirements.txt`: + + flask + redis + +### Create a Docker image + +Now, create a Docker image containing all of your app's dependencies. You +specify how to build the image using a file called +[`Dockerfile`](http://docs.docker.com/reference/builder/): + + FROM python:2.7 + ADD . /code + WORKDIR /code + RUN pip install -r requirements.txt + CMD python app.py + +This tells Docker to: + +* Build an image starting with the Python 2.7 image. +* Add the current directory `.` into the path `/code` in the image. +* Set the working directory to `/code`. +* Install the Python dependencies. +* Set the default command for the container to `python app.py` + +For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + +You can build the image by running `docker build -t web .`. + +### Define services + +Next, define a set of services using `docker-compose.yml`: + + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + links: + - redis + redis: + image: redis + +This template defines two services, `web` and `redis`. The `web` service: + +* Builds from the `Dockerfile` in the current directory. +* Forwards the exposed port 5000 on the container to port 5000 on the host machine. +* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. +* Links the web container to the Redis service. + +The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. + +### Build and run your app with Compose + +Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: + + $ docker-compose up + Pulling image redis... + Building web... + Starting composetest_redis_1... + Starting composetest_web_1... + redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 + web_1 | * Running on http://0.0.0.0:5000/ + web_1 | * Restarting with stat + +If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. + +If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. + +You should get a message in your browser saying: + +`Hello World! I have been seen 1 times.` + +Refreshing the page will increment the number. + +If you want to run your services in the background, you can pass the `-d` flag +(for "detached" mode) to `docker-compose up` and use `docker-compose ps` to +see what is currently running: + + $ docker-compose up -d + Starting composetest_redis_1... + Starting composetest_web_1... + $ docker-compose ps + Name Command State Ports + ------------------------------------------------------------------- + composetest_redis_1 /usr/local/bin/run Up + composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp + +The `docker-compose run` command allows you to run one-off commands for your +services. For example, to see what environment variables are available to the +`web` service: + + $ docker-compose run web env + +See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. + +If you started Compose with `docker-compose up -d`, you'll probably want to stop +your services once you've finished with them: + + $ docker-compose stop + +At this point, you have seen the basics of how Compose works. + +- Next, try the quick start guide for [Django](django.md), + [Rails](rails.md), or [WordPress](wordpress.md). +- See the reference guides for complete details on the [commands](./reference/index.md), the + [configuration file](compose-file.md) and [environment variables](env.md). + +## More Compose documentation + +- [User guide](/) +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with WordPress](wordpress.md) +- [Command line reference](./reference/index.md) +- [Compose file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index 62c78d68..19a6c801 100644 --- a/docs/index.md +++ b/docs/index.md @@ -50,150 +50,13 @@ Compose has commands for managing the whole lifecycle of your application: ## Compose documentation - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) -## Quick start - -Let's get started with a walkthrough of getting a simple Python web app running -on Compose. It assumes a little knowledge of Python, but the concepts -demonstrated here should be understandable even if you're not familiar with -Python. - -### Installation and set-up - -First, [install Docker and Compose](install.md). - -Next, you'll want to make a directory for the project: - - $ mkdir composetest - $ cd composetest - -Inside this directory, create `app.py`, a simple Python web app that uses the Flask -framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): - - from flask import Flask - from redis import Redis - - app = Flask(__name__) - redis = Redis(host='redis', port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - -Next, define the Python dependencies in a file called `requirements.txt`: - - flask - redis - -### Create a Docker image - -Now, create a Docker image containing all of your app's dependencies. You -specify how to build the image using a file called -[`Dockerfile`](http://docs.docker.com/reference/builder/): - - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py - -This tells Docker to: - -* Build an image starting with the Python 2.7 image. -* Add the current directory `.` into the path `/code` in the image. -* Set the working directory to `/code`. -* Install the Python dependencies. -* Set the default command for the container to `python app.py` - -For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). - -You can build the image by running `docker build -t web .`. - -### Define services - -Next, define a set of services using `docker-compose.yml`: - - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - redis: - image: redis - -This template defines two services, `web` and `redis`. The `web` service: - -* Builds from the `Dockerfile` in the current directory. -* Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. - -The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. - -### Build and run your app with Compose - -Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: - - $ docker-compose up - Pulling image redis... - Building web... - Starting composetest_redis_1... - Starting composetest_web_1... - redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 - web_1 | * Running on http://0.0.0.0:5000/ - web_1 | * Restarting with stat - -If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. - -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. - -You should get a message in your browser saying: - -`Hello World! I have been seen 1 times.` - -Refreshing the page will increment the number. - -If you want to run your services in the background, you can pass the `-d` flag -(for "detached" mode) to `docker-compose up` and use `docker-compose ps` to -see what is currently running: - - $ docker-compose up -d - Starting composetest_redis_1... - Starting composetest_web_1... - $ docker-compose ps - Name Command State Ports - ------------------------------------------------------------------- - composetest_redis_1 /usr/local/bin/run Up - composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp - -The `docker-compose run` command allows you to run one-off commands for your -services. For example, to see what environment variables are available to the -`web` service: - - $ docker-compose run web env - -See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. - -If you started Compose with `docker-compose up -d`, you'll probably want to stop -your services once you've finished with them: - - $ docker-compose stop - -At this point, you have seen the basics of how Compose works. - -- Next, try the quick start guide for [Django](django.md), - [Rails](rails.md), or [WordPress](wordpress.md). -- See the reference guides for complete details on the [commands](./reference/index.md), the - [configuration file](compose-file.md) and [environment variables](env.md). ## Release Notes diff --git a/docs/install.md b/docs/install.md index 944ce349..e19bda0f 100644 --- a/docs/install.md +++ b/docs/install.md @@ -127,6 +127,7 @@ To uninstall Docker Compose if you installed using `pip`: ## Where to go next - [User guide](/) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) diff --git a/docs/production.md b/docs/production.md index 8793f927..0b0e46c3 100644 --- a/docs/production.md +++ b/docs/production.md @@ -86,8 +86,5 @@ guide. ## Compose documentation - [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) diff --git a/docs/rails.md b/docs/rails.md index e81675c5..8e16af64 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -135,8 +135,8 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If - [User guide](/) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) -- [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 8c1f5b0a..373ef4d0 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -95,8 +95,8 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma - [User guide](/) - [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) From 86d845fde3ec63d762a2dbd0dbcfc21eb8011f52 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 11:08:23 -0400 Subject: [PATCH 061/359] Flush out features and use cases. Signed-off-by: Daniel Nephin --- README.md | 17 ++++++---- docs/index.md | 94 ++++++++++++++++++++++++++++++++++++++++++++++----- 2 files changed, 97 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index ed176550..55346f24 100644 --- a/README.md +++ b/README.md @@ -4,13 +4,15 @@ Docker Compose *(Previously known as Fig)* -Compose is a tool for defining and running multi-container applications with -Docker. With Compose, you define a multi-container application in a single -file, then spin your application up in a single command which does everything -that needs to be done to get it running. +Compose is a tool for defining and running multi-container Docker applications. +With Compose, you define a multi-container application in a compose +file then, using a single command, you create and start all the containers +from your configuration. To learn more about all the features of Compose +see [the list of features](#features) -Compose is great for development environments, staging servers, and CI. We don't -recommend that you use it in production yet. +Compose is great for development, testing, and staging environments, as well as +CI workflows. You can learn more about each case in +[Common Use Cases](#common-use-cases). Using Compose is basically a three-step process. @@ -33,6 +35,9 @@ A `docker-compose.yml` looks like this: redis: image: redis +For more information about the Compose file, see the +[Compose file reference](docs/yml.md) + Compose has commands for managing the whole lifecycle of your application: * Start, stop and rebuild services diff --git a/docs/index.md b/docs/index.md index 19a6c801..ac7e07f9 100644 --- a/docs/index.md +++ b/docs/index.md @@ -11,20 +11,22 @@ parent="smn_workw_compose" # Overview of Docker Compose -Compose is a tool for defining and running multi-container applications with -Docker. With Compose, you define a multi-container application in a single -file, then spin your application up in a single command which does everything -that needs to be done to get it running. +Compose is a tool for defining and running multi-container Docker applications. +With Compose, you define a multi-container application in a compose +file then, using a single command, you create and start all the containers +from your configuration. To learn more about all the features of Compose +see [the list of features](#features) -Compose is great for development environments, staging servers, and CI. We don't -recommend that you use it in production yet. +Compose is great for development, testing, and staging environments, as well as +CI workflows. You can learn more about each case in +[Common Use Cases](#common-use-cases). Using Compose is basically a three-step process. 1. Define your app's environment with a `Dockerfile` so it can be reproduced anywhere. 2. Define the services that make up your app in `docker-compose.yml` so -they can be run together in an isolated environment: +they can be run together in an isolated environment. 3. Lastly, run `docker-compose up` and Compose will start and run your entire app. A `docker-compose.yml` looks like this: @@ -40,6 +42,9 @@ A `docker-compose.yml` looks like this: redis: image: redis +For more information about the Compose file, see the +[Compose file reference](yml.md) + Compose has commands for managing the whole lifecycle of your application: * Start, stop and rebuild services @@ -57,11 +62,84 @@ Compose has commands for managing the whole lifecycle of your application: - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) +## Features + +#### Preserve volume data + +Compose preserves all volumes used by your services. When `docker-compose up` +runs, if it finds any containers from previous runs, it copies the volumes from +the old container to the new container. This process ensures that any data +you've created in volumes isn't lost. + + +#### Only recreate containers that have changed + +Compose caches the configuration used to create a container. When you +restart a service that has not changed, Compose re-uses the existing +containers. Re-using containers means that you can make changes to your +environment very quickly. + + +#### Variables and moving a composition to different environments + +> New in `docker-compose` 1.5 + +Compose supports variables in the Compose file. You can use these variables +to customize your composition for different environments, or different users. +See [Variable substitution](compose-file.md#variable-substitution) for more +details. + +Compose files can also be extended from other files using the `extends` +field in a compose file, or by using multiple files. See [extends](extends.md) +for more details. + + +## Common Use Cases + +Compose can be used in many different ways. Some common use cases are outlined +below. + +### Development environments + +When you're developing software it is often helpful to be able to run the +application and interact with it. If the application has any service dependencies +(databases, queues, caches, web services, etc) you need a way to document the +dependencies, configuration and operation of each. Compose provides a convenient +format for definition these dependencies (the [Compose file](yml.md)) and a CLI +tool for starting an isolated environment. Compose can replace a multi-page +"developer getting started guide" with a single machine readable configuration +file and a single command `docker-compose up`. + +### Automated testing environments + +An important part of any Continuous Deployment or Continuous Integration process +is the automated test suite. Automated end-to-end testing requires an +environment in which to run tests. Compose provides a convenient way to create +and destroy isolated testing environments for your test suite. By defining the full +environment in a [Compose file](yml.md) you can create and destroy these +environments in just a few commands: + + $ docker-compose up -d + $ ./run_tests + $ docker-compose stop + $ docker-compose rm -f + +### Single host deployments + +Compose has traditionally been focused on development and testing workflows, +but with each release we're making progress on more production-oriented features. +Compose can be used to deploy to a remote docker engine, for example a cloud +instance provisioned with [Docker Machine](https://docs.docker.com/machine/) or +a [Docker Swarm](https://docs.docker.com/swarm/) cluster. + +See [compose in production](production.md) for more details. + ## Release Notes To see a detailed list of changes for past and current releases of Docker -Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). +Compose, please refer to the +[CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). ## Getting help From db3b90b84efa1ddd39b282e67f775f75609d14a3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 16:28:47 -0400 Subject: [PATCH 062/359] Updates to gettingstarted guide from PR feedback. Signed-off-by: Daniel Nephin --- docs/gettingstarted.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index f2024b39..9cc478d7 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -21,13 +21,13 @@ Python. First, [install Docker and Compose](install.md). -Next, you'll want to make a directory for the project: +Create a directory for the project: $ mkdir composetest $ cd composetest Inside this directory, create `app.py`, a simple Python web app that uses the Flask -framework and increments a value in Redis. Don't worry if you don't have Redis installed, docker is going to take care of that for you when we [define services](#define-services): +framework and increments a value in Redis. from flask import Flask from redis import Redis @@ -74,7 +74,7 @@ You can build the image by running `docker build -t web .`. ### Define services -Next, define a set of services using `docker-compose.yml`: +Define a set of services using `docker-compose.yml`: web: build: . @@ -91,8 +91,8 @@ This template defines two services, `web` and `redis`. The `web` service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Mounts the current directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. -* Links the web container to the Redis service. +* Mounts the project directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. +* Links the web service to the Redis service. The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. @@ -113,7 +113,7 @@ If you're using [Docker Machine](https://docs.docker.com/machine), then `docker- If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. -You should get a message in your browser saying: +You will see a message in your browser saying: `Hello World! I have been seen 1 times.` From d9bc91b7cc0a8cc867caaac1bf36c1eab2b6cb4b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 23 Oct 2015 16:51:03 -0400 Subject: [PATCH 063/359] Update intro docs based on feedback. Signed-off-by: Daniel Nephin --- README.md | 4 +- docs/gettingstarted.md | 207 +++++++++++++++++++++++------------------ docs/index.md | 50 ++++++---- 3 files changed, 147 insertions(+), 114 deletions(-) diff --git a/README.md b/README.md index 55346f24..bd110307 100644 --- a/README.md +++ b/README.md @@ -2,8 +2,6 @@ Docker Compose ============== ![Docker Compose](logo.png?raw=true "Docker Compose Logo") -*(Previously known as Fig)* - Compose is a tool for defining and running multi-container Docker applications. With Compose, you define a multi-container application in a compose file then, using a single command, you create and start all the containers @@ -36,7 +34,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](docs/yml.md) +[Compose file reference](docs/compose-file.md) Compose has commands for managing the whole lifecycle of your application: diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index 9cc478d7..f685bf38 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -10,84 +10,103 @@ weight=3 -## Getting Started +# Getting Started -Let's get started with a walkthrough of getting a simple Python web app running -on Compose. It assumes a little knowledge of Python, but the concepts -demonstrated here should be understandable even if you're not familiar with -Python. +On this page you build a simple Python web application running on Compose. The +application uses the Flask framework and increments a value in Redis. While the +sample uses Python, the concepts demonstrated here should be understandable even +if you're not familiar with it. -### Installation and set-up +## Prerequisites -First, [install Docker and Compose](install.md). +Make sure you have already +[installed both Docker Engine and Docker Compose](install.md). You +don't need to install Python, it is provided by a Docker image. -Create a directory for the project: +## Step 1: Setup - $ mkdir composetest - $ cd composetest +1. Create a directory for the project: -Inside this directory, create `app.py`, a simple Python web app that uses the Flask -framework and increments a value in Redis. + $ mkdir composetest + $ cd composetest - from flask import Flask - from redis import Redis +2. With your favorite text editor create a file called `app.py` in your project + directory. - app = Flask(__name__) - redis = Redis(host='redis', port=6379) + from flask import Flask + from redis import Redis - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.' % redis.get('hits') + app = Flask(__name__) + redis = Redis(host='redis', port=6379) - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) + @app.route('/') + def hello(): + redis.incr('hits') + return 'Hello World! I have been seen %s times.' % redis.get('hits') -Next, define the Python dependencies in a file called `requirements.txt`: + if __name__ == "__main__": + app.run(host="0.0.0.0", debug=True) - flask - redis +3. Create another file called `requirements.txt` in your project directory and + add the following: -### Create a Docker image + flask + redis -Now, create a Docker image containing all of your app's dependencies. You -specify how to build the image using a file called -[`Dockerfile`](http://docs.docker.com/reference/builder/): + These define the applications dependencies. - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py +## Step 2: Create a Docker image -This tells Docker to: +In this step, you build a new Docker image. The image contains all the +dependencies the Python application requires, including Python itself. -* Build an image starting with the Python 2.7 image. -* Add the current directory `.` into the path `/code` in the image. -* Set the working directory to `/code`. -* Install the Python dependencies. -* Set the default command for the container to `python app.py` +1. In your project directory create a file named `Dockerfile` and add the + following: -For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + FROM python:2.7 + ADD . /code + WORKDIR /code + RUN pip install -r requirements.txt + CMD python app.py -You can build the image by running `docker build -t web .`. + This tells Docker to: -### Define services + * Build an image starting with the Python 2.7 image. + * Add the current directory `.` into the path `/code` in the image. + * Set the working directory to `/code`. + * Install the Python dependencies. + * Set the default command for the container to `python app.py` + + For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + +2. Build the image. + + $ docker build -t web . + + This command builds an image named `web` from the contents of the current + directory. The command automatically locates the `Dockerfile`, `app.py`, and + `requirements.txt` files. + + +## Step 3: Define services Define a set of services using `docker-compose.yml`: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis +1. Create a file called docker-compose.yml in your project directory and add + the following: -This template defines two services, `web` and `redis`. The `web` service: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + links: + - redis + redis: + image: redis + +This Compose file defines two services, `web` and `redis`. The web service: * Builds from the `Dockerfile` in the current directory. * Forwards the exposed port 5000 on the container to port 5000 on the host machine. @@ -96,68 +115,74 @@ This template defines two services, `web` and `redis`. The `web` service: The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. -### Build and run your app with Compose +## Step 4: Build and run your app with Compose -Now, when you run `docker-compose up`, Compose will pull a Redis image, build an image for your code, and start everything up: +1. From your project directory, start up your application. - $ docker-compose up - Pulling image redis... - Building web... - Starting composetest_redis_1... - Starting composetest_web_1... - redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 - web_1 | * Running on http://0.0.0.0:5000/ - web_1 | * Restarting with stat + $ docker-compose up + Pulling image redis... + Building web... + Starting composetest_redis_1... + Starting composetest_web_1... + redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 + web_1 | * Running on http://0.0.0.0:5000/ + web_1 | * Restarting with stat -If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. + Compose pulls a Redis image, builds an image for your code, and start the + services you defined. -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. +2. Enter `http://0.0.0.0:5000/` in a browser to see the application running. -You will see a message in your browser saying: + If you're using Docker on Linux natively, then the web app should now be + listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 + doesn't resolve, you can also try http://localhost:5000. -`Hello World! I have been seen 1 times.` + If you're using Docker Machine on a Mac, use `docker-machine ip MACHINE_VM` to get + the IP address of your Docker host. Then, `open http://MACHINE_VM_IP:5000` in a + browser. -Refreshing the page will increment the number. + You should see a message in your browser saying: + + `Hello World! I have been seen 1 times.` + +3. Refresh the page. + + The number should increment. + +## Step 5: Experiment with some other commands If you want to run your services in the background, you can pass the `-d` flag (for "detached" mode) to `docker-compose up` and use `docker-compose ps` to see what is currently running: - $ docker-compose up -d - Starting composetest_redis_1... - Starting composetest_web_1... - $ docker-compose ps - Name Command State Ports - ------------------------------------------------------------------- - composetest_redis_1 /usr/local/bin/run Up - composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp + $ docker-compose up -d + Starting composetest_redis_1... + Starting composetest_web_1... + $ docker-compose ps + Name Command State Ports + ------------------------------------------------------------------- + composetest_redis_1 /usr/local/bin/run Up + composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp The `docker-compose run` command allows you to run one-off commands for your services. For example, to see what environment variables are available to the `web` service: - $ docker-compose run web env + $ docker-compose run web env See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. If you started Compose with `docker-compose up -d`, you'll probably want to stop your services once you've finished with them: - $ docker-compose stop + $ docker-compose stop At this point, you have seen the basics of how Compose works. + +## Where to go next + - Next, try the quick start guide for [Django](django.md), [Rails](rails.md), or [WordPress](wordpress.md). -- See the reference guides for complete details on the [commands](./reference/index.md), the - [configuration file](compose-file.md) and [environment variables](env.md). - -## More Compose documentation - -- [User guide](/) -- [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) +- [Explore the full list of Compose commands](./reference/index.md) +- [Compose configuration file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index ac7e07f9..6ea0e99a 100644 --- a/docs/index.md +++ b/docs/index.md @@ -43,7 +43,7 @@ A `docker-compose.yml` looks like this: image: redis For more information about the Compose file, see the -[Compose file reference](yml.md) +[Compose file reference](compose-file.md) Compose has commands for managing the whole lifecycle of your application: @@ -64,6 +64,12 @@ Compose has commands for managing the whole lifecycle of your application: ## Features +The features of Compose that make it effective are: + +* [Preserve volume data](#preserve-volume-data) +* [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) +* [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) + #### Preserve volume data Compose preserves all volumes used by your services. When `docker-compose up` @@ -80,18 +86,15 @@ containers. Re-using containers means that you can make changes to your environment very quickly. -#### Variables and moving a composition to different environments - -> New in `docker-compose` 1.5 +#### Variables and moving a composition between environments Compose supports variables in the Compose file. You can use these variables to customize your composition for different environments, or different users. See [Variable substitution](compose-file.md#variable-substitution) for more details. -Compose files can also be extended from other files using the `extends` -field in a compose file, or by using multiple files. See [extends](extends.md) -for more details. +You can extend a Compose file using the `extends` field or by creating multiple +Compose files. See [extends](extends.md) for more details. ## Common Use Cases @@ -101,14 +104,19 @@ below. ### Development environments -When you're developing software it is often helpful to be able to run the -application and interact with it. If the application has any service dependencies -(databases, queues, caches, web services, etc) you need a way to document the -dependencies, configuration and operation of each. Compose provides a convenient -format for definition these dependencies (the [Compose file](yml.md)) and a CLI -tool for starting an isolated environment. Compose can replace a multi-page -"developer getting started guide" with a single machine readable configuration -file and a single command `docker-compose up`. +When you're developing software, the ability to run an application in an +isolated environment and interact with it is crucial. The Compose command +line tool can be used to create the environment and interact with it. + +The [Compose file](compose-file.md) provides a way to document and configure +all of the application's service dependencies (databases, queues, caches, +web service APIs, etc). Using the Compose command line tool you can create +and start one or more containers for each dependency with a single command +(`docker-compose up`). + +Together, these features provide a convenient way for developers to get +started on a project. Compose can reduce a multi-page "developer getting +started guide" to a single machine readable Compose file and a few commands. ### Automated testing environments @@ -116,7 +124,7 @@ An important part of any Continuous Deployment or Continuous Integration process is the automated test suite. Automated end-to-end testing requires an environment in which to run tests. Compose provides a convenient way to create and destroy isolated testing environments for your test suite. By defining the full -environment in a [Compose file](yml.md) you can create and destroy these +environment in a [Compose file](compose-file.md) you can create and destroy these environments in just a few commands: $ docker-compose up -d @@ -128,11 +136,13 @@ environments in just a few commands: Compose has traditionally been focused on development and testing workflows, but with each release we're making progress on more production-oriented features. -Compose can be used to deploy to a remote docker engine, for example a cloud -instance provisioned with [Docker Machine](https://docs.docker.com/machine/) or -a [Docker Swarm](https://docs.docker.com/swarm/) cluster. +You can use Compose to deploy to a remote Docker Engine. The Docker Engine may +be a single instance provisioned with +[Docker Machine](https://docs.docker.com/machine/) or an entire +[Docker Swarm](https://docs.docker.com/swarm/) cluster. -See [compose in production](production.md) for more details. +For details on using production-oriented features, see +[compose in production](production.md) in this documentation. ## Release Notes From a3fb13e14195835d08006f23a9f26f6907f77262 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 27 Oct 2015 16:51:49 -0400 Subject: [PATCH 064/359] Add another feature to the docs - multiple environments per host. Signed-off-by: Daniel Nephin --- docs/index.md | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/docs/index.md b/docs/index.md index 6ea0e99a..ebc1320e 100644 --- a/docs/index.md +++ b/docs/index.md @@ -66,11 +66,29 @@ Compose has commands for managing the whole lifecycle of your application: The features of Compose that make it effective are: -* [Preserve volume data](#preserve-volume-data) +* [Multiple isolated environments on a single host](#Multiple-isolated-environments-on-a-single-host) +* [Preserve volume data when containers are created](#preserve-volume-data-when-containers-are-created) * [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) * [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) -#### Preserve volume data +#### Multiple isolated environments on a single host + +Compose uses a project name to isolate environments from each other. You can use +this project name to: + +* on a dev host, to create multiple copies of a single environment (ex: you want + to run a stable copy for each feature branch of a project) +* on a CI server, to keep builds from interfering with each other, you can set + the project name to a unique build number +* on a shared host or dev host, to prevent different projects which may use the + same service names, from interfering with each other + +The default project name is the basename of the project directory. You can set +a custom project name by using the +[`-p` command line option](./reference/docker-compose.md) or the +[`COMPOSE_PROJECT_NAME` environment variable](./reference/overview.md#compose-project-name). + +#### Preserve volume data when containers are created Compose preserves all volumes used by your services. When `docker-compose up` runs, if it finds any containers from previous runs, it copies the volumes from From c58cf036e34b34a7c435f0c4c80b13f058788d70 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 13:27:06 -0400 Subject: [PATCH 065/359] Touch up intro paragraph with feedback from @moxiegirl. Signed-off-by: Daniel Nephin --- README.md | 6 +++--- docs/index.md | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index bd110307..5052db39 100644 --- a/README.md +++ b/README.md @@ -3,10 +3,10 @@ Docker Compose ![Docker Compose](logo.png?raw=true "Docker Compose Logo") Compose is a tool for defining and running multi-container Docker applications. -With Compose, you define a multi-container application in a compose -file then, using a single command, you create and start all the containers +With Compose, you use a Compose file to configure your application's services. +Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](#features) +see [the list of features](docs/index.md#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in diff --git a/docs/index.md b/docs/index.md index ebc1320e..279154ee 100644 --- a/docs/index.md +++ b/docs/index.md @@ -12,10 +12,10 @@ parent="smn_workw_compose" # Overview of Docker Compose Compose is a tool for defining and running multi-container Docker applications. -With Compose, you define a multi-container application in a compose -file then, using a single command, you create and start all the containers +With Compose, you use a Compose file to configure your application's services. +Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](#features) +see [the list of features](#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in From 8057bb3fcce2c99126db365671753bc0af003d8c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 16:46:49 -0500 Subject: [PATCH 066/359] Update cli tests to use subprocess. Signed-off-by: Daniel Nephin --- tests/integration/cli_test.py | 447 ++++++++++++++++------------------ 1 file changed, 209 insertions(+), 238 deletions(-) diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index d621f2d1..7ae187b2 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -2,30 +2,32 @@ from __future__ import absolute_import import os import shlex -import sys +import subprocess +from collections import namedtuple from operator import attrgetter -from six import StringIO +import pytest from .. import mock from .testcases import DockerClientTestCase from compose.cli.command import get_project from compose.cli.docker_client import docker_client -from compose.cli.errors import UserError -from compose.cli.main import TopLevelCommand -from compose.project import NoSuchService + + +ProcessResult = namedtuple('ProcessResult', 'stdout stderr') + + +BUILD_CACHE_TEXT = 'Using cache' +BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:latest' class CLITestCase(DockerClientTestCase): + def setUp(self): super(CLITestCase, self).setUp() - self.old_sys_exit = sys.exit - sys.exit = lambda code=0: None - self.command = TopLevelCommand() - self.command.base_dir = 'tests/fixtures/simple-composefile' + self.base_dir = 'tests/fixtures/simple-composefile' def tearDown(self): - sys.exit = self.old_sys_exit self.project.kill() self.project.remove_stopped() for container in self.project.containers(stopped=True, one_off=True): @@ -34,129 +36,121 @@ class CLITestCase(DockerClientTestCase): @property def project(self): - # Hack: allow project to be overridden. This needs refactoring so that - # the project object is built exactly once, by the command object, and - # accessed by the test case object. - if hasattr(self, '_project'): - return self._project + # Hack: allow project to be overridden + if not hasattr(self, '_project'): + self._project = get_project(self.base_dir) + return self._project - return get_project(self.command.base_dir) + def dispatch(self, options, project_options=None, returncode=0): + project_options = project_options or [] + proc = subprocess.Popen( + ['docker-compose'] + project_options + options, + # Note: this might actually be a patched sys.stdout, so we have + # to specify it here, even though it's the default + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=self.base_dir) + print("Running process: %s" % proc.pid) + stdout, stderr = proc.communicate() + if proc.returncode != returncode: + print(stderr) + assert proc.returncode == returncode + return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) def test_help(self): - old_base_dir = self.command.base_dir - self.command.base_dir = 'tests/fixtures/no-composefile' - with self.assertRaises(SystemExit) as exc_context: - self.command.dispatch(['help', 'up'], None) - self.assertIn('Usage: up [options] [SERVICE...]', str(exc_context.exception)) + old_base_dir = self.base_dir + self.base_dir = 'tests/fixtures/no-composefile' + result = self.dispatch(['help', 'up'], returncode=1) + assert 'Usage: up [options] [SERVICE...]' in result.stderr # self.project.kill() fails during teardown # unless there is a composefile. - self.command.base_dir = old_base_dir + self.base_dir = old_base_dir - # TODO: address the "Inappropriate ioctl for device" warnings in test output def test_ps(self): self.project.get_service('simple').create_container() - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['ps'], None) - self.assertIn('simplecomposefile_simple_1', mock_stdout.getvalue()) + result = self.dispatch(['ps']) + assert 'simplecomposefile_simple_1' in result.stdout def test_ps_default_composefile(self): - self.command.base_dir = 'tests/fixtures/multiple-composefiles' - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['up', '-d'], None) - self.command.dispatch(['ps'], None) + self.base_dir = 'tests/fixtures/multiple-composefiles' + self.dispatch(['up', '-d']) + result = self.dispatch(['ps']) - output = mock_stdout.getvalue() - self.assertIn('multiplecomposefiles_simple_1', output) - self.assertIn('multiplecomposefiles_another_1', output) - self.assertNotIn('multiplecomposefiles_yetanother_1', output) + self.assertIn('multiplecomposefiles_simple_1', result.stdout) + self.assertIn('multiplecomposefiles_another_1', result.stdout) + self.assertNotIn('multiplecomposefiles_yetanother_1', result.stdout) def test_ps_alternate_composefile(self): config_path = os.path.abspath( 'tests/fixtures/multiple-composefiles/compose2.yml') - self._project = get_project(self.command.base_dir, [config_path]) + self._project = get_project(self.base_dir, [config_path]) - self.command.base_dir = 'tests/fixtures/multiple-composefiles' - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['-f', 'compose2.yml', 'up', '-d'], None) - self.command.dispatch(['-f', 'compose2.yml', 'ps'], None) + self.base_dir = 'tests/fixtures/multiple-composefiles' + self.dispatch(['-f', 'compose2.yml', 'up', '-d']) + result = self.dispatch(['-f', 'compose2.yml', 'ps']) - output = mock_stdout.getvalue() - self.assertNotIn('multiplecomposefiles_simple_1', output) - self.assertNotIn('multiplecomposefiles_another_1', output) - self.assertIn('multiplecomposefiles_yetanother_1', output) + self.assertNotIn('multiplecomposefiles_simple_1', result.stdout) + self.assertNotIn('multiplecomposefiles_another_1', result.stdout) + self.assertIn('multiplecomposefiles_yetanother_1', result.stdout) - @mock.patch('compose.service.log') - def test_pull(self, mock_logging): - self.command.dispatch(['pull'], None) - mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') - mock_logging.info.assert_any_call('Pulling another (busybox:latest)...') + def test_pull(self): + result = self.dispatch(['pull']) + assert sorted(result.stderr.split('\n'))[1:] == [ + 'Pulling another (busybox:latest)...', + 'Pulling simple (busybox:latest)...', + ] - @mock.patch('compose.service.log') - def test_pull_with_digest(self, mock_logging): - self.command.dispatch(['-f', 'digest.yml', 'pull'], None) - mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') - mock_logging.info.assert_any_call( - 'Pulling digest (busybox@' - 'sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d)...') + def test_pull_with_digest(self): + result = self.dispatch(['-f', 'digest.yml', 'pull']) - @mock.patch('compose.service.log') - def test_pull_with_ignore_pull_failures(self, mock_logging): - self.command.dispatch(['-f', 'ignore-pull-failures.yml', 'pull', '--ignore-pull-failures'], None) - mock_logging.info.assert_any_call('Pulling simple (busybox:latest)...') - mock_logging.info.assert_any_call('Pulling another (nonexisting-image:latest)...') - mock_logging.error.assert_any_call('Error: image library/nonexisting-image:latest not found') + assert 'Pulling simple (busybox:latest)...' in result.stderr + assert ('Pulling digest (busybox@' + 'sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b520' + '04ee8502d)...') in result.stderr + + def test_pull_with_ignore_pull_failures(self): + result = self.dispatch([ + '-f', 'ignore-pull-failures.yml', + 'pull', '--ignore-pull-failures']) + + assert 'Pulling simple (busybox:latest)...' in result.stderr + assert 'Pulling another (nonexisting-image:latest)...' in result.stderr + assert 'Error: image library/nonexisting-image:latest not found' in result.stderr def test_build_plain(self): - self.command.base_dir = 'tests/fixtures/simple-dockerfile' - self.command.dispatch(['build', 'simple'], None) + self.base_dir = 'tests/fixtures/simple-dockerfile' + self.dispatch(['build', 'simple']) - cache_indicator = 'Using cache' - pull_indicator = 'Status: Image is up to date for busybox:latest' - - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['build', 'simple'], None) - output = mock_stdout.getvalue() - self.assertIn(cache_indicator, output) - self.assertNotIn(pull_indicator, output) + result = self.dispatch(['build', 'simple']) + assert BUILD_CACHE_TEXT in result.stdout + assert BUILD_PULL_TEXT not in result.stdout def test_build_no_cache(self): - self.command.base_dir = 'tests/fixtures/simple-dockerfile' - self.command.dispatch(['build', 'simple'], None) + self.base_dir = 'tests/fixtures/simple-dockerfile' + self.dispatch(['build', 'simple']) - cache_indicator = 'Using cache' - pull_indicator = 'Status: Image is up to date for busybox:latest' - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['build', '--no-cache', 'simple'], None) - output = mock_stdout.getvalue() - self.assertNotIn(cache_indicator, output) - self.assertNotIn(pull_indicator, output) + result = self.dispatch(['build', '--no-cache', 'simple']) + assert BUILD_CACHE_TEXT not in result.stdout + assert BUILD_PULL_TEXT not in result.stdout def test_build_pull(self): - self.command.base_dir = 'tests/fixtures/simple-dockerfile' - self.command.dispatch(['build', 'simple'], None) + self.base_dir = 'tests/fixtures/simple-dockerfile' + self.dispatch(['build', 'simple'], None) - cache_indicator = 'Using cache' - pull_indicator = 'Status: Image is up to date for busybox:latest' - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['build', '--pull', 'simple'], None) - output = mock_stdout.getvalue() - self.assertIn(cache_indicator, output) - self.assertIn(pull_indicator, output) + result = self.dispatch(['build', '--pull', 'simple']) + assert BUILD_CACHE_TEXT in result.stdout + assert BUILD_PULL_TEXT in result.stdout def test_build_no_cache_pull(self): - self.command.base_dir = 'tests/fixtures/simple-dockerfile' - self.command.dispatch(['build', 'simple'], None) + self.base_dir = 'tests/fixtures/simple-dockerfile' + self.dispatch(['build', 'simple']) - cache_indicator = 'Using cache' - pull_indicator = 'Status: Image is up to date for busybox:latest' - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: - self.command.dispatch(['build', '--no-cache', '--pull', 'simple'], None) - output = mock_stdout.getvalue() - self.assertNotIn(cache_indicator, output) - self.assertIn(pull_indicator, output) + result = self.dispatch(['build', '--no-cache', '--pull', 'simple']) + assert BUILD_CACHE_TEXT not in result.stdout + assert BUILD_PULL_TEXT in result.stdout def test_up_detached(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') another = self.project.get_service('another') self.assertEqual(len(service.containers()), 1) @@ -168,12 +162,14 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(container.get('Config.AttachStdout')) self.assertFalse(container.get('Config.AttachStdin')) + # TODO: needs rework + @pytest.mark.skipif(True, reason="runs top") def test_up_attached(self): with mock.patch( 'compose.cli.main.attach_to_logs', autospec=True ) as mock_attach: - self.command.dispatch(['up'], None) + self.dispatch(['up'], None) _, args, kwargs = mock_attach.mock_calls[0] _project, log_printer, _names, _timeout = args @@ -189,8 +185,8 @@ class CLITestCase(DockerClientTestCase): def test_up_without_networking(self): self.require_api_version('1.21') - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['up', '-d'], None) + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '-d'], None) client = docker_client(version='1.21') networks = client.networks(names=[self.project.name]) @@ -207,8 +203,8 @@ class CLITestCase(DockerClientTestCase): def test_up_with_networking(self): self.require_api_version('1.21') - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['--x-networking', 'up', '-d'], None) + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['--x-networking', 'up', '-d'], None) client = docker_client(version='1.21') services = self.project.get_services() @@ -232,8 +228,8 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(web_container.get('HostConfig.Links')) def test_up_with_links(self): - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['up', '-d', 'web'], None) + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '-d', 'web'], None) web = self.project.get_service('web') db = self.project.get_service('db') console = self.project.get_service('console') @@ -242,8 +238,8 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(console.containers()), 0) def test_up_with_no_deps(self): - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['up', '-d', '--no-deps', 'web'], None) + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '-d', '--no-deps', 'web'], None) web = self.project.get_service('web') db = self.project.get_service('db') console = self.project.get_service('console') @@ -252,13 +248,13 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(console.containers()), 0) def test_up_with_force_recreate(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) old_ids = [c.id for c in service.containers()] - self.command.dispatch(['up', '-d', '--force-recreate'], None) + self.dispatch(['up', '-d', '--force-recreate'], None) self.assertEqual(len(service.containers()), 1) new_ids = [c.id for c in service.containers()] @@ -266,13 +262,13 @@ class CLITestCase(DockerClientTestCase): self.assertNotEqual(old_ids, new_ids) def test_up_with_no_recreate(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) old_ids = [c.id for c in service.containers()] - self.command.dispatch(['up', '-d', '--no-recreate'], None) + self.dispatch(['up', '-d', '--no-recreate'], None) self.assertEqual(len(service.containers()), 1) new_ids = [c.id for c in service.containers()] @@ -280,11 +276,12 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(old_ids, new_ids) def test_up_with_force_recreate_and_no_recreate(self): - with self.assertRaises(UserError): - self.command.dispatch(['up', '-d', '--force-recreate', '--no-recreate'], None) + self.dispatch( + ['up', '-d', '--force-recreate', '--no-recreate'], + returncode=1) def test_up_with_timeout(self): - self.command.dispatch(['up', '-d', '-t', '1'], None) + self.dispatch(['up', '-d', '-t', '1'], None) service = self.project.get_service('simple') another = self.project.get_service('another') self.assertEqual(len(service.containers()), 1) @@ -296,10 +293,9 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(config['AttachStdout']) self.assertFalse(config['AttachStdin']) - @mock.patch('dockerpty.start') - def test_run_service_without_links(self, mock_stdout): - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['run', 'console', '/bin/true'], None) + def test_run_service_without_links(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['run', 'console', '/bin/true']) self.assertEqual(len(self.project.containers()), 0) # Ensure stdin/out was open @@ -309,44 +305,40 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(config['AttachStdout']) self.assertTrue(config['AttachStdin']) - @mock.patch('dockerpty.start') - def test_run_service_with_links(self, _): - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['run', 'web', '/bin/true'], None) + def test_run_service_with_links(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['run', 'web', '/bin/true'], None) db = self.project.get_service('db') console = self.project.get_service('console') self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) - @mock.patch('dockerpty.start') - def test_run_with_no_deps(self, _): - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['run', '--no-deps', 'web', '/bin/true'], None) + def test_run_with_no_deps(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['run', '--no-deps', 'web', '/bin/true']) db = self.project.get_service('db') self.assertEqual(len(db.containers()), 0) - @mock.patch('dockerpty.start') - def test_run_does_not_recreate_linked_containers(self, _): - self.command.base_dir = 'tests/fixtures/links-composefile' - self.command.dispatch(['up', '-d', 'db'], None) + def test_run_does_not_recreate_linked_containers(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '-d', 'db']) db = self.project.get_service('db') self.assertEqual(len(db.containers()), 1) old_ids = [c.id for c in db.containers()] - self.command.dispatch(['run', 'web', '/bin/true'], None) + self.dispatch(['run', 'web', '/bin/true'], None) self.assertEqual(len(db.containers()), 1) new_ids = [c.id for c in db.containers()] self.assertEqual(old_ids, new_ids) - @mock.patch('dockerpty.start') - def test_run_without_command(self, _): - self.command.base_dir = 'tests/fixtures/commands-composefile' + def test_run_without_command(self): + self.base_dir = 'tests/fixtures/commands-composefile' self.check_build('tests/fixtures/simple-dockerfile', tag='composetest_test') - self.command.dispatch(['run', 'implicit'], None) + self.dispatch(['run', 'implicit']) service = self.project.get_service('implicit') containers = service.containers(stopped=True, one_off=True) self.assertEqual( @@ -354,7 +346,7 @@ class CLITestCase(DockerClientTestCase): [u'/bin/sh -c echo "success"'], ) - self.command.dispatch(['run', 'explicit'], None) + self.dispatch(['run', 'explicit']) service = self.project.get_service('explicit') containers = service.containers(stopped=True, one_off=True) self.assertEqual( @@ -362,14 +354,10 @@ class CLITestCase(DockerClientTestCase): [u'/bin/true'], ) - @mock.patch('dockerpty.start') - def test_run_service_with_entrypoint_overridden(self, _): - self.command.base_dir = 'tests/fixtures/dockerfile_with_entrypoint' + def test_run_service_with_entrypoint_overridden(self): + self.base_dir = 'tests/fixtures/dockerfile_with_entrypoint' name = 'service' - self.command.dispatch( - ['run', '--entrypoint', '/bin/echo', name, 'helloworld'], - None - ) + self.dispatch(['run', '--entrypoint', '/bin/echo', name, 'helloworld']) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] self.assertEqual( @@ -377,37 +365,34 @@ class CLITestCase(DockerClientTestCase): [u'/bin/echo', u'helloworld'], ) - @mock.patch('dockerpty.start') - def test_run_service_with_user_overridden(self, _): - self.command.base_dir = 'tests/fixtures/user-composefile' + def test_run_service_with_user_overridden(self): + self.base_dir = 'tests/fixtures/user-composefile' name = 'service' user = 'sshd' - args = ['run', '--user={user}'.format(user=user), name] - self.command.dispatch(args, None) + self.dispatch(['run', '--user={user}'.format(user=user), name], returncode=1) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] self.assertEqual(user, container.get('Config.User')) - @mock.patch('dockerpty.start') - def test_run_service_with_user_overridden_short_form(self, _): - self.command.base_dir = 'tests/fixtures/user-composefile' + def test_run_service_with_user_overridden_short_form(self): + self.base_dir = 'tests/fixtures/user-composefile' name = 'service' user = 'sshd' - args = ['run', '-u', user, name] - self.command.dispatch(args, None) + self.dispatch(['run', '-u', user, name], returncode=1) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] self.assertEqual(user, container.get('Config.User')) - @mock.patch('dockerpty.start') - def test_run_service_with_environement_overridden(self, _): + def test_run_service_with_environement_overridden(self): name = 'service' - self.command.base_dir = 'tests/fixtures/environment-composefile' - self.command.dispatch( - ['run', '-e', 'foo=notbar', '-e', 'allo=moto=bobo', - '-e', 'alpha=beta', name], - None - ) + self.base_dir = 'tests/fixtures/environment-composefile' + self.dispatch([ + 'run', '-e', 'foo=notbar', + '-e', 'allo=moto=bobo', + '-e', 'alpha=beta', + name, + '/bin/true', + ]) service = self.project.get_service(name) container = service.containers(stopped=True, one_off=True)[0] # env overriden @@ -419,11 +404,10 @@ class CLITestCase(DockerClientTestCase): # make sure a value with a = don't crash out self.assertEqual('moto=bobo', container.environment['allo']) - @mock.patch('dockerpty.start') - def test_run_service_without_map_ports(self, _): + def test_run_service_without_map_ports(self): # create one off container - self.command.base_dir = 'tests/fixtures/ports-composefile' - self.command.dispatch(['run', '-d', 'simple'], None) + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['run', '-d', 'simple']) container = self.project.get_service('simple').containers(one_off=True)[0] # get port information @@ -437,12 +421,10 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_random, None) self.assertEqual(port_assigned, None) - @mock.patch('dockerpty.start') - def test_run_service_with_map_ports(self, _): - + def test_run_service_with_map_ports(self): # create one off container - self.command.base_dir = 'tests/fixtures/ports-composefile' - self.command.dispatch(['run', '-d', '--service-ports', 'simple'], None) + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['run', '-d', '--service-ports', 'simple']) container = self.project.get_service('simple').containers(one_off=True)[0] # get port information @@ -460,12 +442,10 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[0], "0.0.0.0:49153") self.assertEqual(port_range[1], "0.0.0.0:49154") - @mock.patch('dockerpty.start') - def test_run_service_with_explicitly_maped_ports(self, _): - + def test_run_service_with_explicitly_maped_ports(self): # create one off container - self.command.base_dir = 'tests/fixtures/ports-composefile' - self.command.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple'], None) + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple']) container = self.project.get_service('simple').containers(one_off=True)[0] # get port information @@ -479,12 +459,10 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "0.0.0.0:30000") self.assertEqual(port_full, "0.0.0.0:30001") - @mock.patch('dockerpty.start') - def test_run_service_with_explicitly_maped_ip_ports(self, _): - + def test_run_service_with_explicitly_maped_ip_ports(self): # create one off container - self.command.base_dir = 'tests/fixtures/ports-composefile' - self.command.dispatch(['run', '-d', '-p', '127.0.0.1:30000:3000', '--publish', '127.0.0.1:30001:3001', 'simple'], None) + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['run', '-d', '-p', '127.0.0.1:30000:3000', '--publish', '127.0.0.1:30001:3001', 'simple'], None) container = self.project.get_service('simple').containers(one_off=True)[0] # get port information @@ -498,22 +476,20 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "127.0.0.1:30000") self.assertEqual(port_full, "127.0.0.1:30001") - @mock.patch('dockerpty.start') - def test_run_with_custom_name(self, _): - self.command.base_dir = 'tests/fixtures/environment-composefile' + def test_run_with_custom_name(self): + self.base_dir = 'tests/fixtures/environment-composefile' name = 'the-container-name' - self.command.dispatch(['run', '--name', name, 'service'], None) + self.dispatch(['run', '--name', name, 'service', '/bin/true']) service = self.project.get_service('service') container, = service.containers(stopped=True, one_off=True) self.assertEqual(container.name, name) - @mock.patch('dockerpty.start') - def test_run_with_networking(self, _): + def test_run_with_networking(self): self.require_api_version('1.21') client = docker_client(version='1.21') - self.command.base_dir = 'tests/fixtures/simple-dockerfile' - self.command.dispatch(['--x-networking', 'run', 'simple', 'true'], None) + self.base_dir = 'tests/fixtures/simple-dockerfile' + self.dispatch(['--x-networking', 'run', 'simple', 'true'], None) service = self.project.get_service('simple') container, = service.containers(stopped=True, one_off=True) networks = client.networks(names=[self.project.name]) @@ -527,71 +503,70 @@ class CLITestCase(DockerClientTestCase): service.create_container() service.kill() self.assertEqual(len(service.containers(stopped=True)), 1) - self.command.dispatch(['rm', '--force'], None) + self.dispatch(['rm', '--force'], None) self.assertEqual(len(service.containers(stopped=True)), 0) service = self.project.get_service('simple') service.create_container() service.kill() self.assertEqual(len(service.containers(stopped=True)), 1) - self.command.dispatch(['rm', '-f'], None) + self.dispatch(['rm', '-f'], None) self.assertEqual(len(service.containers(stopped=True)), 0) def test_stop(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.command.dispatch(['stop', '-t', '1'], None) + self.dispatch(['stop', '-t', '1'], None) self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) def test_pause_unpause(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertFalse(service.containers()[0].is_paused) - self.command.dispatch(['pause'], None) + self.dispatch(['pause'], None) self.assertTrue(service.containers()[0].is_paused) - self.command.dispatch(['unpause'], None) + self.dispatch(['unpause'], None) self.assertFalse(service.containers()[0].is_paused) def test_logs_invalid_service_name(self): - with self.assertRaises(NoSuchService): - self.command.dispatch(['logs', 'madeupname'], None) + self.dispatch(['logs', 'madeupname'], returncode=1) def test_kill(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.command.dispatch(['kill'], None) + self.dispatch(['kill'], None) self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) def test_kill_signal_sigstop(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.command.dispatch(['kill', '-s', 'SIGSTOP'], None) + self.dispatch(['kill', '-s', 'SIGSTOP'], None) self.assertEqual(len(service.containers()), 1) # The container is still running. It has only been paused self.assertTrue(service.containers()[0].is_running) def test_kill_stopped_service(self): - self.command.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') - self.command.dispatch(['kill', '-s', 'SIGSTOP'], None) + self.dispatch(['kill', '-s', 'SIGSTOP'], None) self.assertTrue(service.containers()[0].is_running) - self.command.dispatch(['kill', '-s', 'SIGKILL'], None) + self.dispatch(['kill', '-s', 'SIGKILL'], None) self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) @@ -601,7 +576,7 @@ class CLITestCase(DockerClientTestCase): container = service.create_container() service.start_container(container) started_at = container.dictionary['State']['StartedAt'] - self.command.dispatch(['restart', '-t', '1'], None) + self.dispatch(['restart', '-t', '1'], None) container.inspect() self.assertNotEqual( container.dictionary['State']['FinishedAt'], @@ -615,53 +590,51 @@ class CLITestCase(DockerClientTestCase): def test_scale(self): project = self.project - self.command.scale(project, {'SERVICE=NUM': ['simple=1']}) + self.dispatch(['scale', 'simple=1']) self.assertEqual(len(project.get_service('simple').containers()), 1) - self.command.scale(project, {'SERVICE=NUM': ['simple=3', 'another=2']}) + self.dispatch(['scale', 'simple=3', 'another=2']) self.assertEqual(len(project.get_service('simple').containers()), 3) self.assertEqual(len(project.get_service('another').containers()), 2) - self.command.scale(project, {'SERVICE=NUM': ['simple=1', 'another=1']}) + self.dispatch(['scale', 'simple=1', 'another=1']) self.assertEqual(len(project.get_service('simple').containers()), 1) self.assertEqual(len(project.get_service('another').containers()), 1) - self.command.scale(project, {'SERVICE=NUM': ['simple=1', 'another=1']}) + self.dispatch(['scale', 'simple=1', 'another=1']) self.assertEqual(len(project.get_service('simple').containers()), 1) self.assertEqual(len(project.get_service('another').containers()), 1) - self.command.scale(project, {'SERVICE=NUM': ['simple=0', 'another=0']}) + self.dispatch(['scale', 'simple=0', 'another=0']) self.assertEqual(len(project.get_service('simple').containers()), 0) self.assertEqual(len(project.get_service('another').containers()), 0) def test_port(self): - self.command.base_dir = 'tests/fixtures/ports-composefile' - self.command.dispatch(['up', '-d'], None) + self.base_dir = 'tests/fixtures/ports-composefile' + self.dispatch(['up', '-d'], None) container = self.project.get_service('simple').get_container() - @mock.patch('sys.stdout', new_callable=StringIO) - def get_port(number, mock_stdout): - self.command.dispatch(['port', 'simple', str(number)], None) - return mock_stdout.getvalue().rstrip() + def get_port(number): + result = self.dispatch(['port', 'simple', str(number)]) + return result.stdout.rstrip() self.assertEqual(get_port(3000), container.get_local_port(3000)) self.assertEqual(get_port(3001), "0.0.0.0:49152") self.assertEqual(get_port(3002), "0.0.0.0:49153") def test_port_with_scale(self): - self.command.base_dir = 'tests/fixtures/ports-composefile-scale' - self.command.dispatch(['scale', 'simple=2'], None) + self.base_dir = 'tests/fixtures/ports-composefile-scale' + self.dispatch(['scale', 'simple=2'], None) containers = sorted( self.project.containers(service_names=['simple']), key=attrgetter('name')) - @mock.patch('sys.stdout', new_callable=StringIO) - def get_port(number, mock_stdout, index=None): + def get_port(number, index=None): if index is None: - self.command.dispatch(['port', 'simple', str(number)], None) + result = self.dispatch(['port', 'simple', str(number)]) else: - self.command.dispatch(['port', '--index=' + str(index), 'simple', str(number)], None) - return mock_stdout.getvalue().rstrip() + result = self.dispatch(['port', '--index=' + str(index), 'simple', str(number)]) + return result.stdout.rstrip() self.assertEqual(get_port(3000), containers[0].get_local_port(3000)) self.assertEqual(get_port(3000, index=1), containers[0].get_local_port(3000)) @@ -670,8 +643,8 @@ class CLITestCase(DockerClientTestCase): def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') - self.command.dispatch(['-f', config_path, 'up', '-d'], None) - self._project = get_project(self.command.base_dir, [config_path]) + self.dispatch(['-f', config_path, 'up', '-d'], None) + self._project = get_project(self.base_dir, [config_path]) containers = self.project.containers(stopped=True) self.assertEqual(len(containers), 1) @@ -681,20 +654,18 @@ class CLITestCase(DockerClientTestCase): def test_home_and_env_var_in_volume_path(self): os.environ['VOLUME_NAME'] = 'my-volume' os.environ['HOME'] = '/tmp/home-dir' - expected_host_path = os.path.join(os.environ['HOME'], os.environ['VOLUME_NAME']) - self.command.base_dir = 'tests/fixtures/volume-path-interpolation' - self.command.dispatch(['up', '-d'], None) + self.base_dir = 'tests/fixtures/volume-path-interpolation' + self.dispatch(['up', '-d'], None) container = self.project.containers(stopped=True)[0] actual_host_path = container.get('Volumes')['/container-path'] components = actual_host_path.split('/') - self.assertTrue(components[-2:] == ['home-dir', 'my-volume'], - msg="Last two components differ: %s, %s" % (actual_host_path, expected_host_path)) + assert components[-2:] == ['home-dir', 'my-volume'] def test_up_with_default_override_file(self): - self.command.base_dir = 'tests/fixtures/override-files' - self.command.dispatch(['up', '-d'], None) + self.base_dir = 'tests/fixtures/override-files' + self.dispatch(['up', '-d'], None) containers = self.project.containers() self.assertEqual(len(containers), 2) @@ -704,15 +675,15 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(db.human_readable_command, 'top') def test_up_with_multiple_files(self): - self.command.base_dir = 'tests/fixtures/override-files' + self.base_dir = 'tests/fixtures/override-files' config_paths = [ 'docker-compose.yml', 'docker-compose.override.yml', 'extra.yml', ] - self._project = get_project(self.command.base_dir, config_paths) - self.command.dispatch( + self._project = get_project(self.base_dir, config_paths) + self.dispatch( [ '-f', config_paths[0], '-f', config_paths[1], @@ -731,8 +702,8 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(other.human_readable_command, 'top') def test_up_with_extends(self): - self.command.base_dir = 'tests/fixtures/extends' - self.command.dispatch(['up', '-d'], None) + self.base_dir = 'tests/fixtures/extends' + self.dispatch(['up', '-d'], None) self.assertEqual( set([s.name for s in self.project.services]), From 45635f709781f4fccec15af19fb60e8c96cf6639 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 17:52:37 -0500 Subject: [PATCH 067/359] Move cli tests to a new testing package. These cli tests are now a different kind of that that run the compose binary. They are not the same as integration tests that test some internal interface. Signed-off-by: Daniel Nephin --- tests/acceptance/__init__.py | 0 tests/{integration => acceptance}/cli_test.py | 29 ++++--------------- .../fixtures/echo-services/docker-compose.yml | 6 ++++ 3 files changed, 12 insertions(+), 23 deletions(-) create mode 100644 tests/acceptance/__init__.py rename tests/{integration => acceptance}/cli_test.py (96%) create mode 100644 tests/fixtures/echo-services/docker-compose.yml diff --git a/tests/acceptance/__init__.py b/tests/acceptance/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/integration/cli_test.py b/tests/acceptance/cli_test.py similarity index 96% rename from tests/integration/cli_test.py rename to tests/acceptance/cli_test.py index 7ae187b2..0add049e 100644 --- a/tests/integration/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -6,12 +6,10 @@ import subprocess from collections import namedtuple from operator import attrgetter -import pytest - from .. import mock -from .testcases import DockerClientTestCase from compose.cli.command import get_project from compose.cli.docker_client import docker_client +from tests.integration.testcases import DockerClientTestCase ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -45,8 +43,6 @@ class CLITestCase(DockerClientTestCase): project_options = project_options or [] proc = subprocess.Popen( ['docker-compose'] + project_options + options, - # Note: this might actually be a patched sys.stdout, so we have - # to specify it here, even though it's the default stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=self.base_dir) @@ -150,7 +146,7 @@ class CLITestCase(DockerClientTestCase): assert BUILD_PULL_TEXT in result.stdout def test_up_detached(self): - self.dispatch(['up', '-d'], None) + self.dispatch(['up', '-d']) service = self.project.get_service('simple') another = self.project.get_service('another') self.assertEqual(len(service.containers()), 1) @@ -162,25 +158,12 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(container.get('Config.AttachStdout')) self.assertFalse(container.get('Config.AttachStdin')) - # TODO: needs rework - @pytest.mark.skipif(True, reason="runs top") def test_up_attached(self): - with mock.patch( - 'compose.cli.main.attach_to_logs', - autospec=True - ) as mock_attach: - self.dispatch(['up'], None) - _, args, kwargs = mock_attach.mock_calls[0] - _project, log_printer, _names, _timeout = args + self.base_dir = 'tests/fixtures/echo-services' + result = self.dispatch(['up', '--no-color']) - service = self.project.get_service('simple') - another = self.project.get_service('another') - self.assertEqual(len(service.containers()), 1) - self.assertEqual(len(another.containers()), 1) - self.assertEqual( - set(log_printer.containers), - set(self.project.containers()) - ) + assert 'simple_1 | simple' in result.stdout + assert 'another_1 | another' in result.stdout def test_up_without_networking(self): self.require_api_version('1.21') diff --git a/tests/fixtures/echo-services/docker-compose.yml b/tests/fixtures/echo-services/docker-compose.yml new file mode 100644 index 00000000..8014f3d9 --- /dev/null +++ b/tests/fixtures/echo-services/docker-compose.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: echo simple +another: + image: busybox:latest + command: echo another From 3d9e3d08779c38e52f9892eea20f05afb115ffa1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 13:33:13 -0500 Subject: [PATCH 068/359] Remove service.start_container() It has been an unnecessary wrapper around container.start() for a little while now, so we can call it directly. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- compose/service.py | 13 ++++--------- tests/acceptance/cli_test.py | 2 +- tests/integration/resilience_test.py | 4 ++-- tests/integration/service_test.py | 27 ++++++++++++++------------- 5 files changed, 22 insertions(+), 26 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index b54b307e..11aeac38 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -448,7 +448,7 @@ class TopLevelCommand(DocoptCommand): raise e if detach: - service.start_container(container) + container.start() print(container.name) else: dockerpty.start(project.client, container.id, interactive=not options['-T']) diff --git a/compose/service.py b/compose/service.py index 8d716c0b..e121ee95 100644 --- a/compose/service.py +++ b/compose/service.py @@ -406,7 +406,7 @@ class Service(object): if should_attach_logs: container.attach_log_stream() - self.start_container(container) + container.start() return [container] @@ -457,21 +457,16 @@ class Service(object): ) if attach_logs: new_container.attach_log_stream() - self.start_container(new_container) + new_container.start() container.remove() return new_container def start_container_if_stopped(self, container, attach_logs=False): - if container.is_running: - return container - else: + if not container.is_running: log.info("Starting %s" % container.name) if attach_logs: container.attach_log_stream() - return self.start_container(container) - - def start_container(self, container): - container.start() + container.start() return container def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0add049e..41e9718b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -557,7 +557,7 @@ class CLITestCase(DockerClientTestCase): def test_restart(self): service = self.project.get_service('simple') container = service.create_container() - service.start_container(container) + container.start() started_at = container.dictionary['State']['StartedAt'] self.dispatch(['restart', '-t', '1'], None) container.inspect() diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index befd72c7..53aedfec 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -13,7 +13,7 @@ class ResilienceTest(DockerClientTestCase): self.project = Project('composetest', [self.db], self.client) container = self.db.create_container() - self.db.start_container(container) + container.start() self.host_path = container.get('Volumes')['/var/db'] def test_successful_recreate(self): @@ -31,7 +31,7 @@ class ResilienceTest(DockerClientTestCase): self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) def test_start_failure(self): - with mock.patch('compose.service.Service.start_container', crash): + with mock.patch('compose.container.Container.start', crash): with self.assertRaises(Crash): self.project.up(strategy=ConvergenceStrategy.always) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4ac04545..f083908b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -30,7 +30,8 @@ from compose.service import VolumeFromSpec def create_and_start_container(service, **override_options): container = service.create_container(**override_options) - return service.start_container(container) + container.start() + return container class ServiceTest(DockerClientTestCase): @@ -115,19 +116,19 @@ class ServiceTest(DockerClientTestCase): def test_create_container_with_unspecified_volume(self): service = self.create_service('db', volumes=['/var/db']) container = service.create_container() - service.start_container(container) + container.start() self.assertIn('/var/db', container.get('Volumes')) def test_create_container_with_volume_driver(self): service = self.create_service('db', volume_driver='foodriver') container = service.create_container() - service.start_container(container) + container.start() self.assertEqual('foodriver', container.get('Config.VolumeDriver')) def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(container.get('HostConfig.CpuShares'), 73) def test_build_extra_hosts(self): @@ -165,7 +166,7 @@ class ServiceTest(DockerClientTestCase): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts)) def test_create_container_with_extra_hosts_dicts(self): @@ -173,33 +174,33 @@ class ServiceTest(DockerClientTestCase): extra_hosts_list = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts_list)) def test_create_container_with_cpu_set(self): service = self.create_service('db', cpuset='0') container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(container.get('HostConfig.CpusetCpus'), '0') def test_create_container_with_read_only_root_fs(self): read_only = True service = self.create_service('db', read_only=read_only) container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(container.get('HostConfig.ReadonlyRootfs'), read_only, container.get('HostConfig')) def test_create_container_with_security_opt(self): security_opt = ['label:disable'] service = self.create_service('db', security_opt=security_opt) container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) def test_create_container_with_mac_address(self): service = self.create_service('db', mac_address='02:42:ac:11:65:43') container = service.create_container() - service.start_container(container) + container.start() self.assertEqual(container.inspect()['Config']['MacAddress'], '02:42:ac:11:65:43') def test_create_container_with_specified_volume(self): @@ -208,7 +209,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', volumes=['%s:%s' % (host_path, container_path)]) container = service.create_container() - service.start_container(container) + container.start() volumes = container.inspect()['Volumes'] self.assertIn(container_path, volumes) @@ -281,7 +282,7 @@ class ServiceTest(DockerClientTestCase): ] ) host_container = host_service.create_container() - host_service.start_container(host_container) + host_container.start() self.assertIn(volume_container_1.id + ':rw', host_container.get('HostConfig.VolumesFrom')) self.assertIn(volume_container_2.id + ':rw', @@ -300,7 +301,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=1', old_container.get('Config.Env')) self.assertEqual(old_container.name, 'composetest_db_1') - service.start_container(old_container) + old_container.start() old_container.inspect() # reload volume data volume_path = old_container.get('Volumes')['/etc'] From 7014cabb04fed3e31fe65c034604f1224195a2fa Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 12:34:55 -0400 Subject: [PATCH 069/359] Remove duplication from extends docs. Start restructuring extends docs in preparation for adding documentation about using multiple compose files. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 51 ++++----- docs/extends.md | 240 +++++++++++-------------------------------- 2 files changed, 80 insertions(+), 211 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 7723a784..00b04d58 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -168,44 +168,29 @@ accessible to linked services. Only the internal port can be specified. Extend another service, in the current file or another, optionally overriding configuration. -Here's a simple example. Suppose we have 2 files - **common.yml** and -**development.yml**. We can use `extends` to define a service in -**development.yml** which uses configuration defined in **common.yml**: +You can use `extends` on any service together with other configuration keys. +The value must be a dictionary with the key: `service` and may optionally have +the `file` key. -**common.yml** + extends: + file: common.yml + service: webapp - webapp: - build: ./webapp - environment: - - DEBUG=false - - SEND_EMAILS=false +The `file` key specifies the location of a Compose configuration file defining +the service which is being extended. The `file` value can be an absolute or +relative path. If you specify a relative path, Docker Compose treats it as +relative to the location of the current file. If you don't specify a `file`, +Compose looks in the current configuration file. -**development.yml** +The `service` key specifies the name of the service to extend, for example `web` +or `database`. - web: - extends: - file: common.yml - service: webapp - ports: - - "8000:8000" - links: - - db - environment: - - DEBUG=true - db: - image: postgres +You can extend a service that itself extends another. You can extend +indefinitely. Compose does not support circular references and `docker-compose` +returns an error if it encounters one. -Here, the `web` service in **development.yml** inherits the configuration of -the `webapp` service in **common.yml** - the `build` and `environment` keys - -and adds `ports` and `links` configuration. It overrides one of the defined -environment variables (DEBUG) with a new value, and the other one -(SEND_EMAILS) is left untouched. - -The `file` key is optional, if it is not set then Compose will look for the -service within the current file. - -For more on `extends`, see the [tutorial](extends.md#example) and -[reference](extends.md#reference). +For more on `extends`, see the +[the extends documentation](extends.md#extending-services). ### external_links diff --git a/docs/extends.md b/docs/extends.md index e63cf466..c97b2b4f 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -10,20 +10,29 @@ weight=2 -## Extending services in Compose +## Extending services and Compose files + +Compose supports two ways to sharing common configuration and +extend a service with that shared configuration. + +1. Extending individual services with [the `extends` field](#extending-services) +2. Extending entire compositions by + [exnteding compose files](#extending-compose-files) + +### Extending services Docker Compose's `extends` keyword enables sharing of common configurations among different files, or even different projects entirely. Extending services -is useful if you have several applications that reuse commonly-defined services. -Using `extends` you can define a service in one place and refer to it from -anywhere. +is useful if you have several services that reuse a common set of configuration +options. Using `extends` you can define a common set of service options in one +place and refer to it from anywhere. -Alternatively, you can deploy the same application to multiple environments with -a slightly different set of services in each case (or with changes to the -configuration of some services). Moreover, you can do so without copy-pasting -the configuration around. +> **Note:** `links` and `volumes_from` are never shared between services using +> `extends`. See +> [Adding and overriding configuration](#adding-and-overriding-configuration) +> for more information. -### Understand the extends configuration +#### Understand the extends configuration When defining any service in `docker-compose.yml`, you can declare that you are extending another service like this: @@ -77,183 +86,46 @@ You can also write other services and link your `web` service to them: db: image: postgres -For full details on how to use `extends`, refer to the [reference](#reference). +#### Example use case -### Example use case +Extending an individual service is useful when you have multiple services that +have a common configuration. In this example we have a composition that with +a web application and a queue worker. Both services use the same codebase and +share many configuration options. -In this example, you’ll repurpose the example app from the [quick start -guide](/). (If you're not familiar with Compose, it's recommended that -you go through the quick start first.) This example assumes you want to use -Compose both to develop an application locally and then deploy it to a -production environment. +In a **common.yml** we'll define the common configuration: -The local and production environments are similar, but there are some -differences. In development, you mount the application code as a volume so that -it can pick up changes; in production, the code should be immutable from the -outside. This ensures it’s not accidentally changed. The development environment -uses a local Redis container, but in production another team manages the Redis -service, which is listening at `redis-production.example.com`. + app: + build: . + environment: + CONFIG_FILE_PATH: /code/config + API_KEY: xxxyyy + cpu_shares: 5 -To configure with `extends` for this sample, you must: - -1. Define the web application as a Docker image in `Dockerfile` and a Compose - service in `common.yml`. - -2. Define the development environment in the standard Compose file, - `docker-compose.yml`. - - - Use `extends` to pull in the web service. - - Configure a volume to enable code reloading. - - Create an additional Redis service for the application to use locally. - -3. Define the production environment in a third Compose file, `production.yml`. - - - Use `extends` to pull in the web service. - - Configure the web service to talk to the external, production Redis service. - -#### Define the web app - -Defining the web application requires the following: - -1. Create an `app.py` file. - - This file contains a simple Python application that uses Flask to serve HTTP - and increments a counter in Redis: - - from flask import Flask - from redis import Redis - import os - - app = Flask(__name__) - redis = Redis(host=os.environ['REDIS_HOST'], port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.\n' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - - This code uses a `REDIS_HOST` environment variable to determine where to - find Redis. - -2. Define the Python dependencies in a `requirements.txt` file: - - flask - redis - -3. Create a `Dockerfile` to build an image containing the app: - - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py - -4. Create a Compose configuration file called `common.yml`: - - This configuration defines how to run the app. - - web: - build: . - ports: - - "5000:5000" - - Typically, you would have dropped this configuration into - `docker-compose.yml` file, but in order to pull it into multiple files with - `extends`, it needs to be in a separate file. - -#### Define the development environment - -1. Create a `docker-compose.yml` file. - - The `extends` option pulls in the `web` service from the `common.yml` file - you created in the previous section. - - web: - extends: - file: common.yml - service: web - volumes: - - .:/code - links: - - redis - environment: - - REDIS_HOST=redis - redis: - image: redis - - The new addition defines a `web` service that: - - - Fetches the base configuration for `web` out of `common.yml`. - - Adds `volumes` and `links` configuration to the base (`common.yml`) - configuration. - - Sets the `REDIS_HOST` environment variable to point to the linked redis - container. This environment uses a stock `redis` image from the Docker Hub. - -2. Run `docker-compose up`. - - Compose creates, links, and starts a web and redis container linked together. - It mounts your application code inside the web container. - -3. Verify that the code is mounted by changing the message in - `app.py`—say, from `Hello world!` to `Hello from Compose!`. - - Don't forget to refresh your browser to see the change! - -#### Define the production environment - -You are almost done. Now, define your production environment: - -1. Create a `production.yml` file. - - As with `docker-compose.yml`, the `extends` option pulls in the `web` service - from `common.yml`. - - web: - extends: - file: common.yml - service: web - environment: - - REDIS_HOST=redis-production.example.com - -2. Run `docker-compose -f production.yml up`. - - Compose creates *just* a web container and configures the Redis connection via - the `REDIS_HOST` environment variable. This variable points to the production - Redis instance. - - > **Note**: If you try to load up the webapp in your browser you'll get an - > error—`redis-production.example.com` isn't actually a Redis server. - -You've now done a basic `extends` configuration. As your application develops, -you can make any necessary changes to the web service in `common.yml`. Compose -picks up both the development and production environments when you next run -`docker-compose`. You don't have to do any copy-and-paste, and you don't have to -manually keep both environments in sync. +In a **docker-compose.yml** we'll define the concrete services which use the +common configuration: -### Reference + webapp: + extends: + file: common.yml + service: app + command: /code/run_web_app + ports: + - 8080:8080 + links: + - queue + - db -You can use `extends` on any service together with other configuration keys. It -expects a dictionary that contains a `service` key and optionally a `file` key. -The `extends` key can also take a string, whose value is the name of a `service` defined in the same file. + queue_worker: + extends: + file: common.yml + service: app + command: /code/run_worker + links: + - queue -The `file` key specifies the location of a Compose configuration file defining -the extension. The `file` value can be an absolute or relative path. If you -specify a relative path, Docker Compose treats it as relative to the location -of the current file. If you don't specify a `file`, Compose looks in the -current configuration file. - -The `service` key specifies the name of the service to extend, for example `web` -or `database`. - -You can extend a service that itself extends another. You can extend -indefinitely. Compose does not support circular references and `docker-compose` -returns an error if it encounters them. - -#### Adding and overriding configuration +#### Adding and overriding configuration Compose copies configurations from the original service over to the local one, **except** for `links` and `volumes_from`. These exceptions exist to avoid @@ -282,6 +154,8 @@ listed below.** In the case of `build` and `image`, using one in the local service causes Compose to discard the other, if it was defined in the original service. +Example of image replacing build: + # original service build: . @@ -291,6 +165,9 @@ Compose to discard the other, if it was defined in the original service. # result image: redis + +Example of build replacing image: + # original service image: redis @@ -356,6 +233,13 @@ locally-defined bindings taking precedence: - /local-dir/bar:/bar - /local-dir/baz/:baz + +### Extending Compose files + +> **Note:** This feature is new in `docker-compose` 1.5 + + + ## Compose documentation - [User guide](/) From 1c4c7ccface1e71e462b1ff3d840c0d0804d230d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 29 Oct 2015 15:21:06 -0400 Subject: [PATCH 070/359] Support a volume to the docs directory and add --watch, so docs can be refreshed. Signed-off-by: Daniel Nephin --- docs/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/Makefile b/docs/Makefile index 021e8f6e..b9ef0548 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,8 +13,8 @@ DOCKER_ENVS := \ -e TIMEOUT # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds -# to allow `make DOCSDIR=docs docs-shell` (to create a bind mount in docs) -DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR)/$(DOCSDIR):/$(DOCSDIR)) +# to allow `make DOCSDIR=1 docs-shell` (to create a bind mount in docs) +DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR):/docs/content/compose) # to allow `make DOCSPORT=9000 docs` DOCSPORT := 8000 @@ -37,7 +37,7 @@ GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) default: docs docs: docs-build - $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) --watch docs-draft: docs-build $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) From c5cf5cfad45afb66f6b47fd6b7a0937e1e82f7f3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 17:17:38 -0400 Subject: [PATCH 071/359] Changes to production.md for working with multiple Compose files. Signed-off-by: Daniel Nephin --- docs/production.md | 30 +++++++++++------------------- 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/docs/production.md b/docs/production.md index 0b0e46c3..39f0e1fe 100644 --- a/docs/production.md +++ b/docs/production.md @@ -12,11 +12,9 @@ weight=1 ## Using Compose in production -While **Compose is not yet considered production-ready**, if you'd like to experiment and learn more about using it in production deployments, this guide -can help. -The project is actively working towards becoming -production-ready; to learn more about the progress being made, check out the roadmap for details -on how it's coming along and what still needs to be done. +> Compose is still primarily aimed at development and testing environments. +> Compose may be used for smaller production deployments, but is probably +> not yet suitable for larger deployments. When deploying to production, you'll almost certainly want to make changes to your app configuration that are more appropriate to a live environment. These @@ -30,22 +28,16 @@ changes may include: - Specifying a restart policy (e.g., `restart: always`) to avoid downtime - Adding extra services (e.g., a log aggregator) -For this reason, you'll probably want to define a separate Compose file, say -`production.yml`, which specifies production-appropriate configuration. +For this reason, you'll probably want to define an additional Compose file, say +`production.yml`, which specifies production-appropriate +configuration. This configuration file only needs to include the changes you'd +like to make from the original Compose file. The additional Compose file +can be applied over the original `docker-compose.yml` to create a new configuration. -> **Note:** The [extends](extends.md) keyword is useful for maintaining multiple -> Compose files which re-use common services without having to manually copy and -> paste. +Once you've got a second configuration file, tell Compose to use it with the +`-f` option: -Once you've got an alternate configuration file, make Compose use it -by setting the `COMPOSE_FILE` environment variable: - - $ export COMPOSE_FILE=production.yml - $ docker-compose up -d - -> **Note:** You can also use the file for a one-off command without setting -> an environment variable. You do this by passing the `-f` flag, e.g., -> `docker-compose -f production.yml up -d`. + $ docker-compose -f docker-compose.yml -f production.yml up -d ### Deploying changes From eab265befaa6c48d44ad2b9314ccc97ad6630f7b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 12:56:59 -0400 Subject: [PATCH 072/359] Document using multiple Compose files use cases. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 18 ++-- docs/extends.md | 208 ++++++++++++++++++++++++++++++++++--------- docs/production.md | 3 + 3 files changed, 176 insertions(+), 53 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 00b04d58..4f8fc9e0 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -169,21 +169,21 @@ Extend another service, in the current file or another, optionally overriding configuration. You can use `extends` on any service together with other configuration keys. -The value must be a dictionary with the key: `service` and may optionally have -the `file` key. +The `extends` value must be a dictionary defined with a required `service` +and an optional `file` key. extends: file: common.yml service: webapp -The `file` key specifies the location of a Compose configuration file defining -the service which is being extended. The `file` value can be an absolute or -relative path. If you specify a relative path, Docker Compose treats it as -relative to the location of the current file. If you don't specify a `file`, -Compose looks in the current configuration file. +The `service` the name of the service being extended, for example +`web` or `database`. The `file` is the location of a Compose configuration +file defining that service. -The `service` key specifies the name of the service to extend, for example `web` -or `database`. +If you omit the `file` Compose looks for the service configuration in the +current file. The `file` value can be an absolute or relative path. If you +specify a relative path, Compose treats it as relative to the location of the +current file. You can extend a service that itself extends another. You can extend indefinitely. Compose does not support circular references and `docker-compose` diff --git a/docs/extends.md b/docs/extends.md index c97b2b4f..58def22d 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -10,16 +10,15 @@ weight=2 -## Extending services and Compose files +# Extending services and Compose files -Compose supports two ways to sharing common configuration and -extend a service with that shared configuration. +Compose supports two methods of sharing common configuration: 1. Extending individual services with [the `extends` field](#extending-services) 2. Extending entire compositions by - [exnteding compose files](#extending-compose-files) + [using multiple compose files](#multiple-compose-files) -### Extending services +## Extending services Docker Compose's `extends` keyword enables sharing of common configurations among different files, or even different projects entirely. Extending services @@ -30,9 +29,9 @@ place and refer to it from anywhere. > **Note:** `links` and `volumes_from` are never shared between services using > `extends`. See > [Adding and overriding configuration](#adding-and-overriding-configuration) -> for more information. + > for more information. -#### Understand the extends configuration +### Understand the extends configuration When defining any service in `docker-compose.yml`, you can declare that you are extending another service like this: @@ -54,8 +53,8 @@ looks like this: - "/data" In this case, you'll get exactly the same result as if you wrote -`docker-compose.yml` with that `build`, `ports` and `volumes` configuration -defined directly under `web`. +`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration +values defined directly under `web`. You can go further and define (or re-define) configuration locally in `docker-compose.yml`: @@ -86,14 +85,14 @@ You can also write other services and link your `web` service to them: db: image: postgres -#### Example use case +### Example use case Extending an individual service is useful when you have multiple services that -have a common configuration. In this example we have a composition that with -a web application and a queue worker. Both services use the same codebase and -share many configuration options. +have a common configuration. The example below is a composition with +two services: a web application and a queue worker. Both services use the same +codebase and share many configuration options. -In a **common.yml** we'll define the common configuration: +In a **common.yml** we define the common configuration: app: build: . @@ -102,10 +101,9 @@ In a **common.yml** we'll define the common configuration: API_KEY: xxxyyy cpu_shares: 5 -In a **docker-compose.yml** we'll define the concrete services which use the +In a **docker-compose.yml** we define the concrete services which use the common configuration: - webapp: extends: file: common.yml @@ -121,11 +119,11 @@ common configuration: extends: file: common.yml service: app - command: /code/run_worker - links: - - queue + command: /code/run_worker + links: + - queue -#### Adding and overriding configuration +### Adding and overriding configuration Compose copies configurations from the original service over to the local one, **except** for `links` and `volumes_from`. These exceptions exist to avoid @@ -134,13 +132,11 @@ locally. This ensures dependencies between services are clearly visible when reading the current file. Defining these locally also ensures changes to the referenced file don't result in breakage. -If a configuration option is defined in both the original service and the local -service, the local value either *override*s or *extend*s the definition of the -original service. This works differently for other configuration options. +If a configuration option is defined in both the original service the local +service, the local value *replaces* or *extends* the original value. For single-value options like `image`, `command` or `mem_limit`, the new value -replaces the old value. **This is the default behaviour - all exceptions are -listed below.** +replaces the old value. # original service command: python app.py @@ -195,8 +191,8 @@ For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and - "4000" - "5000" -In the case of `environment` and `labels`, Compose "merges" entries together -with locally-defined values taking precedence: +In the case of `environment`, `labels`, `volumes` and `devices`, Compose +"merges" entries together with locally-defined values taking precedence: # original service environment: @@ -214,30 +210,154 @@ with locally-defined values taking precedence: - BAR=local - BAZ=local -Finally, for `volumes` and `devices`, Compose "merges" entries together with -locally-defined bindings taking precedence: - # original service - volumes: - - /original-dir/foo:/foo - - /original-dir/bar:/bar +## Multiple Compose files - # local service - volumes: - - /local-dir/bar:/bar - - /local-dir/baz/:baz +Using multiple Compose files enables you to customize a composition for +different environments or different workflows. - # result - volumes: - - /original-dir/foo:/foo - - /local-dir/bar:/bar - - /local-dir/baz/:baz +### Understanding multiple Compose files + +By default, Compose reads two files, a `docker-compose.yml` and an optional +`docker-compose.override.yml` file. By convention, the `docker-compose.yml` +contains your base configuration. The override file, as its name implies, can +contain configuration overrides for existing services or entirely new +services. + +If a service is defined in both files, Compose merges the configurations using +the same rules as the `extends` field (see [Adding and overriding +configuration](#adding-and-overriding-configuration)), with one exception. If a +service contains `links` or `volumes_from` those fields are copied over and +replace any values in the original service, in the same way single-valued fields +are copied. + +To use multiple override files, or an override file with a different name, you +can use the `-f` option to specify the list of files. Compose merges files in +the order they're specified on the command line. See the [`docker-compose` +command reference](./reference/docker-compose.md) for more information about +using `-f`. + +When you use multiple configuration files, you must make sure all paths in the +files are relative to the base Compose file (the first Compose file specified +with `-f`). This is required because override files need not be valid +Compose files. Override files can contain small fragments of configuration. +Tracking which fragment of a service is relative to which path is difficult and +confusing, so to keep paths easier to understand, all paths must be defined +relative to the base file. + +### Example use case + +In this section are two common use cases for multiple compose files: changing a +composition for different environments, and running administrative tasks +against a composition. + +#### Different environments + +A common use case for multiple files is changing a development composition +for a production-like environment (which may be production, staging or CI). +To support these differences, you can split your Compose configuration into +a few different files: + +Start with a base file that defines the canonical configuration for the +services. + +**docker-compose.yml** + + web: + image: example/my_web_app:latest + links: + - db + - cache + + db: + image: postgres:latest + + cache: + image: redis:latest + +In this example the development configuration exposes some ports to the +host, mounts our code as a volume, and builds the web image. + +**docker-compose.override.yml** -### Extending Compose files + web: + build: . + volumes: + - '.:/code' + ports: + - 8883:80 + environment: + DEBUG: 'true' -> **Note:** This feature is new in `docker-compose` 1.5 + db: + command: '-d' + ports: + - 5432:5432 + cache: + ports: + - 6379:6379 + +When you run `docker-compose up` it reads the overrides automatically. + +Now, it would be nice to use this composition in a production environment. So, +create another override file (which might be stored in a different git +repo or managed by a different team). + +**docker-compose.prod.yml** + + web: + ports: + - 80:80 + environment: + PRODUCTION: 'true' + + cache: + environment: + TTL: '500' + +To deploy with this production Compose file you can run + + docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d + +This deploys all three services using the configuration in +`docker-compose.yml` and `docker-compose.prod.yml` (but not the +dev configuration in `docker-compose.override.yml`). + + +See [production](production.md) for more information about Compose in +production. + +#### Administrative tasks + +Another common use case is running adhoc or administrative tasks against one +or more services in a composition. This example demonstrates running a +database backup. + +Start with a **docker-compose.yml**. + + web: + image: example/my_web_app:latest + links: + - db + + db: + image: postgres:latest + +In a **docker-compose.admin.yml** add a new service to run the database +export or backup. + + dbadmin: + build: database_admin/ + links: + - db + +To start a normal environment run `docker-compose up -d`. To run a database +backup, include the `docker-compose.admin.yml` as well. + + docker-compose -f docker-compose.yml -f docker-compose.admin.yml \ + run dbadmin db-backup ## Compose documentation diff --git a/docs/production.md b/docs/production.md index 39f0e1fe..0a5e77b5 100644 --- a/docs/production.md +++ b/docs/production.md @@ -39,6 +39,9 @@ Once you've got a second configuration file, tell Compose to use it with the $ docker-compose -f docker-compose.yml -f production.yml up -d +See [Using multiple compose files](extends.md#different-environments) for a more +complete example. + ### Deploying changes When you make changes to your app code, you'll need to rebuild your image and From 8bdde9a7313fbaf5221dff20b314aa0fed0dd66a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 2 Nov 2015 15:14:42 -0500 Subject: [PATCH 073/359] Replace composition with Compose app. Signed-off-by: Daniel Nephin --- docs/extends.md | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index 58def22d..e4d09af9 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -15,8 +15,8 @@ weight=2 Compose supports two methods of sharing common configuration: 1. Extending individual services with [the `extends` field](#extending-services) -2. Extending entire compositions by - [using multiple compose files](#multiple-compose-files) +2. Extending entire Compose file by + [using multiple Compose files](#multiple-compose-files) ## Extending services @@ -88,7 +88,7 @@ You can also write other services and link your `web` service to them: ### Example use case Extending an individual service is useful when you have multiple services that -have a common configuration. The example below is a composition with +have a common configuration. The example below is a Compose app with two services: a web application and a queue worker. Both services use the same codebase and share many configuration options. @@ -213,8 +213,8 @@ In the case of `environment`, `labels`, `volumes` and `devices`, Compose ## Multiple Compose files -Using multiple Compose files enables you to customize a composition for -different environments or different workflows. +Using multiple Compose files enables you to customize a Compose application +for different environments or different workflows. ### Understanding multiple Compose files @@ -248,12 +248,12 @@ relative to the base file. ### Example use case In this section are two common use cases for multiple compose files: changing a -composition for different environments, and running administrative tasks -against a composition. +Compose app for different environments, and running administrative tasks +against a Compose app. #### Different environments -A common use case for multiple files is changing a development composition +A common use case for multiple files is changing a development Compose app for a production-like environment (which may be production, staging or CI). To support these differences, you can split your Compose configuration into a few different files: @@ -301,7 +301,7 @@ host, mounts our code as a volume, and builds the web image. When you run `docker-compose up` it reads the overrides automatically. -Now, it would be nice to use this composition in a production environment. So, +Now, it would be nice to use this Compose app in a production environment. So, create another override file (which might be stored in a different git repo or managed by a different team). @@ -332,7 +332,7 @@ production. #### Administrative tasks Another common use case is running adhoc or administrative tasks against one -or more services in a composition. This example demonstrates running a +or more services in a Compose app. This example demonstrates running a database backup. Start with a **docker-compose.yml**. From e503e085ac98e6e744020f5068be40412f443f67 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Nov 2015 11:52:44 -0500 Subject: [PATCH 074/359] Re-order extends docs. Signed-off-by: Daniel Nephin --- docs/extends.md | 391 ++++++++++++++++++++++++------------------------ 1 file changed, 197 insertions(+), 194 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index e4d09af9..b21b6d76 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -14,201 +14,9 @@ weight=2 Compose supports two methods of sharing common configuration: -1. Extending individual services with [the `extends` field](#extending-services) -2. Extending entire Compose file by +1. Extending an entire Compose file by [using multiple Compose files](#multiple-compose-files) - -## Extending services - -Docker Compose's `extends` keyword enables sharing of common configurations -among different files, or even different projects entirely. Extending services -is useful if you have several services that reuse a common set of configuration -options. Using `extends` you can define a common set of service options in one -place and refer to it from anywhere. - -> **Note:** `links` and `volumes_from` are never shared between services using -> `extends`. See -> [Adding and overriding configuration](#adding-and-overriding-configuration) - > for more information. - -### Understand the extends configuration - -When defining any service in `docker-compose.yml`, you can declare that you are -extending another service like this: - - web: - extends: - file: common-services.yml - service: webapp - -This instructs Compose to re-use the configuration for the `webapp` service -defined in the `common-services.yml` file. Suppose that `common-services.yml` -looks like this: - - webapp: - build: . - ports: - - "8000:8000" - volumes: - - "/data" - -In this case, you'll get exactly the same result as if you wrote -`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration -values defined directly under `web`. - -You can go further and define (or re-define) configuration locally in -`docker-compose.yml`: - - web: - extends: - file: common-services.yml - service: webapp - environment: - - DEBUG=1 - cpu_shares: 5 - - important_web: - extends: web - cpu_shares: 10 - -You can also write other services and link your `web` service to them: - - web: - extends: - file: common-services.yml - service: webapp - environment: - - DEBUG=1 - cpu_shares: 5 - links: - - db - db: - image: postgres - -### Example use case - -Extending an individual service is useful when you have multiple services that -have a common configuration. The example below is a Compose app with -two services: a web application and a queue worker. Both services use the same -codebase and share many configuration options. - -In a **common.yml** we define the common configuration: - - app: - build: . - environment: - CONFIG_FILE_PATH: /code/config - API_KEY: xxxyyy - cpu_shares: 5 - -In a **docker-compose.yml** we define the concrete services which use the -common configuration: - - webapp: - extends: - file: common.yml - service: app - command: /code/run_web_app - ports: - - 8080:8080 - links: - - queue - - db - - queue_worker: - extends: - file: common.yml - service: app - command: /code/run_worker - links: - - queue - -### Adding and overriding configuration - -Compose copies configurations from the original service over to the local one, -**except** for `links` and `volumes_from`. These exceptions exist to avoid -implicit dependencies—you always define `links` and `volumes_from` -locally. This ensures dependencies between services are clearly visible when -reading the current file. Defining these locally also ensures changes to the -referenced file don't result in breakage. - -If a configuration option is defined in both the original service the local -service, the local value *replaces* or *extends* the original value. - -For single-value options like `image`, `command` or `mem_limit`, the new value -replaces the old value. - - # original service - command: python app.py - - # local service - command: python otherapp.py - - # result - command: python otherapp.py - -In the case of `build` and `image`, using one in the local service causes -Compose to discard the other, if it was defined in the original service. - -Example of image replacing build: - - # original service - build: . - - # local service - image: redis - - # result - image: redis - - -Example of build replacing image: - - # original service - image: redis - - # local service - build: . - - # result - build: . - -For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and -`dns_search`, Compose concatenates both sets of values: - - # original service - expose: - - "3000" - - # local service - expose: - - "4000" - - "5000" - - # result - expose: - - "3000" - - "4000" - - "5000" - -In the case of `environment`, `labels`, `volumes` and `devices`, Compose -"merges" entries together with locally-defined values taking precedence: - - # original service - environment: - - FOO=original - - BAR=original - - # local service - environment: - - BAR=local - - BAZ=local - - # result - environment: - - FOO=original - - BAR=local - - BAZ=local +2. Extending individual services with [the `extends` field](#extending-services) ## Multiple Compose files @@ -360,6 +168,201 @@ backup, include the `docker-compose.admin.yml` as well. run dbadmin db-backup +## Extending services + +Docker Compose's `extends` keyword enables sharing of common configurations +among different files, or even different projects entirely. Extending services +is useful if you have several services that reuse a common set of configuration +options. Using `extends` you can define a common set of service options in one +place and refer to it from anywhere. + +> **Note:** `links` and `volumes_from` are never shared between services using +> `extends`. See +> [Adding and overriding configuration](#adding-and-overriding-configuration) + > for more information. + +### Understand the extends configuration + +When defining any service in `docker-compose.yml`, you can declare that you are +extending another service like this: + + web: + extends: + file: common-services.yml + service: webapp + +This instructs Compose to re-use the configuration for the `webapp` service +defined in the `common-services.yml` file. Suppose that `common-services.yml` +looks like this: + + webapp: + build: . + ports: + - "8000:8000" + volumes: + - "/data" + +In this case, you'll get exactly the same result as if you wrote +`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration +values defined directly under `web`. + +You can go further and define (or re-define) configuration locally in +`docker-compose.yml`: + + web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 + + important_web: + extends: web + cpu_shares: 10 + +You can also write other services and link your `web` service to them: + + web: + extends: + file: common-services.yml + service: webapp + environment: + - DEBUG=1 + cpu_shares: 5 + links: + - db + db: + image: postgres + +### Example use case + +Extending an individual service is useful when you have multiple services that +have a common configuration. The example below is a Compose app with +two services: a web application and a queue worker. Both services use the same +codebase and share many configuration options. + +In a **common.yml** we define the common configuration: + + app: + build: . + environment: + CONFIG_FILE_PATH: /code/config + API_KEY: xxxyyy + cpu_shares: 5 + +In a **docker-compose.yml** we define the concrete services which use the +common configuration: + + webapp: + extends: + file: common.yml + service: app + command: /code/run_web_app + ports: + - 8080:8080 + links: + - queue + - db + + queue_worker: + extends: + file: common.yml + service: app + command: /code/run_worker + links: + - queue + +## Adding and overriding configuration + +Compose copies configurations from the original service over to the local one, +**except** for `links` and `volumes_from`. These exceptions exist to avoid +implicit dependencies—you always define `links` and `volumes_from` +locally. This ensures dependencies between services are clearly visible when +reading the current file. Defining these locally also ensures changes to the +referenced file don't result in breakage. + +If a configuration option is defined in both the original service the local +service, the local value *replaces* or *extends* the original value. + +For single-value options like `image`, `command` or `mem_limit`, the new value +replaces the old value. + + # original service + command: python app.py + + # local service + command: python otherapp.py + + # result + command: python otherapp.py + +In the case of `build` and `image`, using one in the local service causes +Compose to discard the other, if it was defined in the original service. + +Example of image replacing build: + + # original service + build: . + + # local service + image: redis + + # result + image: redis + + +Example of build replacing image: + + # original service + image: redis + + # local service + build: . + + # result + build: . + +For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and +`dns_search`, Compose concatenates both sets of values: + + # original service + expose: + - "3000" + + # local service + expose: + - "4000" + - "5000" + + # result + expose: + - "3000" + - "4000" + - "5000" + +In the case of `environment`, `labels`, `volumes` and `devices`, Compose +"merges" entries together with locally-defined values taking precedence: + + # original service + environment: + - FOO=original + - BAR=original + + # local service + environment: + - BAR=local + - BAZ=local + + # result + environment: + - FOO=original + - BAR=local + - BAZ=local + + + + ## Compose documentation - [User guide](/) From 385b4280a1f6d2b36dccb43a0f99858693ff673f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Nov 2015 15:33:37 -0500 Subject: [PATCH 075/359] Fix jenkins CI by using an older docker version to match the host. Signed-off-by: Daniel Nephin --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index b28a438d..acf9b6ae 100644 --- a/Dockerfile +++ b/Dockerfile @@ -16,7 +16,7 @@ RUN set -ex; \ ; \ rm -rf /var/lib/apt/lists/* -RUN curl https://get.docker.com/builds/Linux/x86_64/docker-latest \ +RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker From 6b002fb9225907f1c9b48523879b1263b607bdeb Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Nov 2015 11:30:34 -0500 Subject: [PATCH 076/359] Cherry-pick release notes froim 1.5.0 And bump version to 1.6.0dev Signed-off-by: Daniel Nephin --- CHANGELOG.md | 101 ++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 2 +- 3 files changed, 103 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 598f5e57..dde42542 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,107 @@ Change log ========== +1.5.0 (2015-11-03) +------------------ + +**Breaking changes:** + +With the introduction of variable substitution support in the Compose file, any +Compose file that uses an environment variable (`$VAR` or `${VAR}`) in the `command:` +or `entrypoint:` field will break. + +Previously these values were interpolated inside the container, with a value +from the container environment. In Compose 1.5.0, the values will be +interpolated on the host, with a value from the host environment. + +To migrate a Compose file to 1.5.0, escape the variables with an extra `$` +(ex: `$$VAR` or `$${VAR}`). See +https://github.com/docker/compose/blob/8cc8e61/docs/compose-file.md#variable-substitution + +Major features: + +- Compose is now available for Windows. + +- Environment variables can be used in the Compose file. See + https://github.com/docker/compose/blob/8cc8e61/docs/compose-file.md#variable-substitution + +- Multiple compose files can be specified, allowing you to override + settings in the default Compose file. See + https://github.com/docker/compose/blob/8cc8e61/docs/reference/docker-compose.md + for more details. + +- Compose now produces better error messages when a file contains + invalid configuration. + +- `up` now waits for all services to exit before shutting down, + rather than shutting down as soon as one container exits. + +- Experimental support for the new docker networking system can be + enabled with the `--x-networking` flag. Read more here: + https://github.com/docker/docker/blob/8fee1c20/docs/userguide/dockernetworks.md + +New features: + +- You can now optionally pass a mode to `volumes_from`, e.g. + `volumes_from: ["servicename:ro"]`. + +- Since Docker now lets you create volumes with names, you can refer to those + volumes by name in `docker-compose.yml`. For example, + `volumes: ["mydatavolume:/data"]` will mount the volume named + `mydatavolume` at the path `/data` inside the container. + + If the first component of an entry in `volumes` starts with a `.`, `/` or + `~`, it is treated as a path and expansion of relative paths is performed as + necessary. Otherwise, it is treated as a volume name and passed straight + through to Docker. + + Read more on named volumes and volume drivers here: + https://github.com/docker/docker/blob/244d9c33/docs/userguide/dockervolumes.md + +- `docker-compose build --pull` instructs Compose to pull the base image for + each Dockerfile before building. + +- `docker-compose pull --ignore-pull-failures` instructs Compose to continue + if it fails to pull a single service's image, rather than aborting. + +- You can now specify an IPC namespace in `docker-compose.yml` with the `ipc` + option. + +- Containers created by `docker-compose run` can now be named with the + `--name` flag. + +- If you install Compose with pip or use it as a library, it now works with + Python 3. + +- `image` now supports image digests (in addition to ids and tags), e.g. + `image: "busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d"` + +- `ports` now supports ranges of ports, e.g. + + ports: + - "3000-3005" + - "9000-9001:8000-8001" + +- `docker-compose run` now supports a `-p|--publish` parameter, much like + `docker run -p`, for publishing specific ports to the host. + +- `docker-compose pause` and `docker-compose unpause` have been implemented, + analogous to `docker pause` and `docker unpause`. + +- When using `extends` to copy configuration from another service in the same + Compose file, you can omit the `file` option. + +- Compose can be installed and run as a Docker image. This is an experimental + feature. + +Bug fixes: + +- All values for the `log_driver` option which are supported by the Docker + daemon are now supported by Compose. + +- `docker-compose build` can now be run successfully against a Swarm cluster. + + 1.4.2 (2015-09-22) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index e3ace983..7c16c97b 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.5.0dev' +__version__ = '1.6.0dev' diff --git a/docs/install.md b/docs/install.md index e19bda0f..c5304409 100644 --- a/docs/install.md +++ b/docs/install.md @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.4.2 + docker-compose version: 1.5.0 ## Alternative install options From d18ad4c81221e13346b8840a35a3defb71d86378 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Nov 2015 16:58:30 -0500 Subject: [PATCH 077/359] Fix rebase-bump-commit script when used with a final release. Previously it would find commits for RC releases, which broke the rebase. Signed-off-by: Daniel Nephin --- script/release/rebase-bump-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/rebase-bump-commit b/script/release/rebase-bump-commit index 14ad22a9..23877bb5 100755 --- a/script/release/rebase-bump-commit +++ b/script/release/rebase-bump-commit @@ -22,7 +22,7 @@ VERSION="$(git config "branch.${BRANCH}.release")" || usage COMMIT_MSG="Bump $VERSION" -sha="$(git log --grep "$COMMIT_MSG" --format="%H")" +sha="$(git log --grep "$COMMIT_MSG\$" --format="%H")" if [ -z "$sha" ]; then >&2 echo "No commit with message \"$COMMIT_MSG\"" exit 2 From 0227b3adbdec8595d013ce70364dff08a691ef4e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Nov 2015 10:26:54 -0500 Subject: [PATCH 078/359] Upgrade pyyaml to 3.11 Signed-off-by: Daniel Nephin --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index daaaa950..60327d72 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -PyYAML==3.10 +PyYAML==3.11 docker-py==1.5.0 dockerpty==0.3.4 docopt==0.6.1 From ce322047a052d96cf9a6f1dd65df66385b215e50 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 28 Oct 2015 13:16:19 -0400 Subject: [PATCH 079/359] Move config hash tests to service_test.py Signed-off-by: Daniel Nephin --- tests/integration/service_test.py | 37 +++++++++++++++++++++++++++++++ tests/integration/state_test.py | 36 ------------------------------ 2 files changed, 37 insertions(+), 36 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index f083908b..804f5219 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -14,6 +14,7 @@ from .. import mock from .testcases import DockerClientTestCase from .testcases import pull_busybox from compose import __version__ +from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT @@ -23,6 +24,7 @@ from compose.container import Container from compose.service import build_extra_hosts from compose.service import ConfigError from compose.service import ConvergencePlan +from compose.service import ConvergenceStrategy from compose.service import Net from compose.service import Service from compose.service import VolumeFromSpec @@ -930,3 +932,38 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(set(service.containers(stopped=True)), set([original, duplicate])) self.assertEqual(set(service.duplicate_containers()), set([duplicate])) + + +def converge(service, + strategy=ConvergenceStrategy.changed, + do_build=True): + """Create a converge plan from a strategy and execute the plan.""" + plan = service.convergence_plan(strategy) + return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) + + +class ConfigHashTest(DockerClientTestCase): + def test_no_config_hash_when_one_off(self): + web = self.create_service('web') + container = web.create_container(one_off=True) + self.assertNotIn(LABEL_CONFIG_HASH, container.labels) + + def test_no_config_hash_when_overriding_options(self): + web = self.create_service('web') + container = web.create_container(environment={'FOO': '1'}) + self.assertNotIn(LABEL_CONFIG_HASH, container.labels) + + def test_config_hash_with_custom_labels(self): + web = self.create_service('web', labels={'foo': '1'}) + container = converge(web)[0] + self.assertIn(LABEL_CONFIG_HASH, container.labels) + self.assertIn('foo', container.labels) + + def test_config_hash_sticks_around(self): + web = self.create_service('web', command=["top"]) + container = converge(web)[0] + self.assertIn(LABEL_CONFIG_HASH, container.labels) + + web = self.create_service('web', command=["top", "-d", "1"]) + container = converge(web)[0] + self.assertIn(LABEL_CONFIG_HASH, container.labels) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 3230aefc..cb904572 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -10,7 +10,6 @@ import tempfile from .testcases import DockerClientTestCase from compose.config import config -from compose.const import LABEL_CONFIG_HASH from compose.project import Project from compose.service import ConvergenceStrategy @@ -180,14 +179,6 @@ class ProjectWithDependenciesTest(ProjectTestCase): self.assertEqual(len(containers), 2) -def converge(service, - strategy=ConvergenceStrategy.changed, - do_build=True): - """Create a converge plan from a strategy and execute the plan.""" - plan = service.convergence_plan(strategy) - return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) - - class ServiceStateTest(DockerClientTestCase): """Test cases for Service.convergence_plan.""" @@ -278,30 +269,3 @@ class ServiceStateTest(DockerClientTestCase): self.assertEqual(('recreate', [container]), web.convergence_plan()) finally: shutil.rmtree(context) - - -class ConfigHashTest(DockerClientTestCase): - def test_no_config_hash_when_one_off(self): - web = self.create_service('web') - container = web.create_container(one_off=True) - self.assertNotIn(LABEL_CONFIG_HASH, container.labels) - - def test_no_config_hash_when_overriding_options(self): - web = self.create_service('web') - container = web.create_container(environment={'FOO': '1'}) - self.assertNotIn(LABEL_CONFIG_HASH, container.labels) - - def test_config_hash_with_custom_labels(self): - web = self.create_service('web', labels={'foo': '1'}) - container = converge(web)[0] - self.assertIn(LABEL_CONFIG_HASH, container.labels) - self.assertIn('foo', container.labels) - - def test_config_hash_sticks_around(self): - web = self.create_service('web', command=["top"]) - container = converge(web)[0] - self.assertIn(LABEL_CONFIG_HASH, container.labels) - - web = self.create_service('web', command=["top", "-d", "1"]) - container = converge(web)[0] - self.assertIn(LABEL_CONFIG_HASH, container.labels) From 8ff960afd13b71bbe84fb7bef2120cc8f958de93 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 3 Nov 2015 20:00:54 -0500 Subject: [PATCH 080/359] Fix service recreate when image changes to build. Signed-off-by: Daniel Nephin --- compose/service.py | 13 ++++--- tests/integration/state_test.py | 65 ++++++++++++++++++-------------- tests/unit/config/config_test.py | 2 + 3 files changed, 46 insertions(+), 34 deletions(-) diff --git a/compose/service.py b/compose/service.py index e121ee95..e5a4cc4a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -414,6 +414,7 @@ class Service(object): return [ self.recreate_container( container, + do_build=do_build, timeout=timeout, attach_logs=should_attach_logs ) @@ -435,10 +436,12 @@ class Service(object): else: raise Exception("Invalid action: {}".format(action)) - def recreate_container(self, - container, - timeout=DEFAULT_TIMEOUT, - attach_logs=False): + def recreate_container( + self, + container, + do_build=False, + timeout=DEFAULT_TIMEOUT, + attach_logs=False): """Recreate a container. The original container is renamed to a temporary name so that data @@ -450,7 +453,7 @@ class Service(object): container.stop(timeout=timeout) container.rename_to_tmp_name() new_container = self.create_container( - do_build=False, + do_build=do_build, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), quiet=True, diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index cb904572..7830ba32 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -4,9 +4,7 @@ by `docker-compose up`. """ from __future__ import unicode_literals -import os -import shutil -import tempfile +import py from .testcases import DockerClientTestCase from compose.config import config @@ -232,40 +230,49 @@ class ServiceStateTest(DockerClientTestCase): image_id = self.client.images(name='busybox')[0]['Id'] self.client.tag(image_id, repository=repo, tag=tag) + self.addCleanup(self.client.remove_image, image) - try: - web = self.create_service('web', image=image) - container = web.create_container() + web = self.create_service('web', image=image) + container = web.create_container() - # update the image - c = self.client.create_container(image, ['touch', '/hello.txt']) - self.client.commit(c, repository=repo, tag=tag) - self.client.remove_container(c) + # update the image + c = self.client.create_container(image, ['touch', '/hello.txt']) + self.client.commit(c, repository=repo, tag=tag) + self.client.remove_container(c) - web = self.create_service('web', image=image) - self.assertEqual(('recreate', [container]), web.convergence_plan()) - - finally: - self.client.remove_image(image) + web = self.create_service('web', image=image) + self.assertEqual(('recreate', [container]), web.convergence_plan()) def test_trigger_recreate_with_build(self): - context = tempfile.mkdtemp() + context = py.test.ensuretemp('test_trigger_recreate_with_build') + self.addCleanup(context.remove) + base_image = "FROM busybox\nLABEL com.docker.compose.test_image=true\n" + dockerfile = context.join('Dockerfile') + dockerfile.write(base_image) - try: - dockerfile = os.path.join(context, 'Dockerfile') + web = self.create_service('web', build=str(context)) + container = web.create_container() - with open(dockerfile, 'w') as f: - f.write(base_image) + dockerfile.write(base_image + 'CMD echo hello world\n') + web.build() - web = self.create_service('web', build=context) - container = web.create_container() + web = self.create_service('web', build=str(context)) + self.assertEqual(('recreate', [container]), web.convergence_plan()) - with open(dockerfile, 'w') as f: - f.write(base_image + 'CMD echo hello world\n') - web.build() + def test_image_changed_to_build(self): + context = py.test.ensuretemp('test_image_changed_to_build') + self.addCleanup(context.remove) + context.join('Dockerfile').write(""" + FROM busybox + LABEL com.docker.compose.test_image=true + """) - web = self.create_service('web', build=context) - self.assertEqual(('recreate', [container]), web.convergence_plan()) - finally: - shutil.rmtree(context) + web = self.create_service('web', image='busybox') + container = web.create_container() + + web = self.create_service('web', build=str(context)) + plan = web.convergence_plan() + self.assertEqual(('recreate', [container]), plan) + containers = web.execute_convergence_plan(plan) + self.assertEqual(len(containers), 1) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 7246b661..69b23585 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -177,6 +177,7 @@ class ConfigTest(unittest.TestCase): details = config.ConfigDetails('.', [base_file, override_file]) tmpdir = py.test.ensuretemp('config_test') + self.addCleanup(tmpdir.remove) tmpdir.join('common.yml').write(""" base: labels: ['label=one'] @@ -412,6 +413,7 @@ class ConfigTest(unittest.TestCase): def test_load_yaml_with_yaml_error(self): tmpdir = py.test.ensuretemp('invalid_yaml_test') + self.addCleanup(tmpdir.remove) invalid_yaml_file = tmpdir.join('docker-compose.yml') invalid_yaml_file.write(""" web: From 26c7dd37126cca09b149d3de7f134d5c8a766fdd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Nov 2015 15:54:59 -0500 Subject: [PATCH 081/359] Handle non-utf8 unicode without raising an error. Signed-off-by: Daniel Nephin --- compose/config/config.py | 2 +- compose/utils.py | 2 +- tests/unit/utils_test.py | 16 ++++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5bc534fe..ff3ae780 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -457,7 +457,7 @@ def parse_environment(environment): def split_env(env): if isinstance(env, six.binary_type): - env = env.decode('utf-8') + env = env.decode('utf-8', 'replace') if '=' in env: return env.split('=', 1) else: diff --git a/compose/utils.py b/compose/utils.py index c8fddc5f..08f6034f 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -95,7 +95,7 @@ def stream_as_text(stream): """ for data in stream: if not isinstance(data, six.text_type): - data = data.decode('utf-8') + data = data.decode('utf-8', 'replace') yield data diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index b272c734..e3d0bc00 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -1,3 +1,6 @@ +# encoding: utf-8 +from __future__ import unicode_literals + from .. import unittest from compose import utils @@ -14,3 +17,16 @@ class JsonSplitterTestCase(unittest.TestCase): utils.json_splitter(data), ({'foo': 'bar'}, '{"next": "obj"}') ) + + +class StreamAsTextTestCase(unittest.TestCase): + + def test_stream_with_non_utf_unicode_character(self): + stream = [b'\xed\xf3\xf3'] + output, = utils.stream_as_text(stream) + assert output == '���' + + def test_stream_with_utf_character(self): + stream = ['ěĝ'.encode('utf-8')] + output, = utils.stream_as_text(stream) + assert output == 'ěĝ' From d32bb8efeea1e265d54f7cd7415db202d854f1f2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Nov 2015 15:33:42 -0500 Subject: [PATCH 082/359] Fix #1549 - flush after each line of logs. Includes some refactoring of log_printer_test to support checking for flush(), and so that each test calls the unit-under-test directly, instead of through a helper function. Signed-off-by: Daniel Nephin --- compose/cli/log_printer.py | 1 + tests/unit/cli/log_printer_test.py | 82 +++++++++++++++--------------- 2 files changed, 42 insertions(+), 41 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 66920726..864657a4 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -26,6 +26,7 @@ class LogPrinter(object): generators = list(self._make_log_generators(self.monochrome, prefix_width)) for line in Multiplexer(generators).loop(): self.output.write(line) + self.output.flush() def _make_log_generators(self, monochrome, prefix_width): def no_color(text): diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 575fcaf7..5b04226c 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -1,13 +1,13 @@ from __future__ import absolute_import from __future__ import unicode_literals -import mock +import pytest import six from compose.cli.log_printer import LogPrinter from compose.cli.log_printer import wait_on_exit from compose.container import Container -from tests import unittest +from tests import mock def build_mock_container(reader): @@ -22,40 +22,52 @@ def build_mock_container(reader): ) -class LogPrinterTest(unittest.TestCase): - def get_default_output(self, monochrome=False): - def reader(*args, **kwargs): - yield b"hello\nworld" - container = build_mock_container(reader) - output = run_log_printer([container], monochrome=monochrome) - return output +@pytest.fixture +def output_stream(): + output = six.StringIO() + output.flush = mock.Mock() + return output - def test_single_container(self): - output = self.get_default_output() - self.assertIn('hello', output) - self.assertIn('world', output) +@pytest.fixture +def mock_container(): + def reader(*args, **kwargs): + yield b"hello\nworld" + return build_mock_container(reader) - def test_monochrome(self): - output = self.get_default_output(monochrome=True) - self.assertNotIn('\033[', output) - def test_polychrome(self): - output = self.get_default_output() - self.assertIn('\033[', output) +class TestLogPrinter(object): - def test_unicode(self): + def test_single_container(self, output_stream, mock_container): + LogPrinter([mock_container], output=output_stream).run() + + output = output_stream.getvalue() + assert 'hello' in output + assert 'world' in output + # Call count is 2 lines + "container exited line" + assert output_stream.flush.call_count == 3 + + def test_monochrome(self, output_stream, mock_container): + LogPrinter([mock_container], output=output_stream, monochrome=True).run() + assert '\033[' not in output_stream.getvalue() + + def test_polychrome(self, output_stream, mock_container): + LogPrinter([mock_container], output=output_stream).run() + assert '\033[' in output_stream.getvalue() + + def test_unicode(self, output_stream): glyph = u'\u2022' def reader(*args, **kwargs): yield glyph.encode('utf-8') + b'\n' container = build_mock_container(reader) - output = run_log_printer([container]) + LogPrinter([container], output=output_stream).run() + output = output_stream.getvalue() if six.PY2: output = output.decode('utf-8') - self.assertIn(glyph, output) + assert glyph in output def test_wait_on_exit(self): exit_status = 3 @@ -65,24 +77,12 @@ class LogPrinterTest(unittest.TestCase): wait=mock.Mock(return_value=exit_status)) expected = '{} exited with code {}\n'.format(mock_container.name, exit_status) - self.assertEqual(expected, wait_on_exit(mock_container)) + assert expected == wait_on_exit(mock_container) - def test_generator_with_no_logs(self): - mock_container = mock.Mock( - spec=Container, - has_api_logs=False, - log_driver='none', - name_without_project='web_1', - wait=mock.Mock(return_value=0)) + def test_generator_with_no_logs(self, mock_container, output_stream): + mock_container.has_api_logs = False + mock_container.log_driver = 'none' + LogPrinter([mock_container], output=output_stream).run() - output = run_log_printer([mock_container]) - self.assertIn( - "WARNING: no logs are available with the 'none' log driver\n", - output - ) - - -def run_log_printer(containers, monochrome=False): - output = six.StringIO() - LogPrinter(containers, output=output, monochrome=monochrome).run() - return output.getvalue() + output = output_stream.getvalue() + assert "WARNING: no logs are available with the 'none' log driver\n" in output From 3456002aef26fd025819349f29e46d0062f9040a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 5 Nov 2015 17:50:32 -0500 Subject: [PATCH 083/359] Don't set the hostname to the service name with networking. Signed-off-by: Daniel Nephin --- compose/service.py | 3 --- tests/acceptance/cli_test.py | 1 - tests/unit/service_test.py | 10 ---------- 3 files changed, 14 deletions(-) diff --git a/compose/service.py b/compose/service.py index e5a4cc4a..c17d4f91 100644 --- a/compose/service.py +++ b/compose/service.py @@ -599,9 +599,6 @@ class Service(object): container_options['hostname'] = parts[0] container_options['domainname'] = parts[2] - if 'hostname' not in container_options and self.use_networking: - container_options['hostname'] = self.name - if 'ports' in container_options or 'expose' in self.options: ports = [] all_ports = container_options.get('ports', []) + self.options.get('expose', []) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 41e9718b..fc68f9d8 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -205,7 +205,6 @@ class CLITestCase(DockerClientTestCase): containers = service.containers() self.assertEqual(len(containers), 1) self.assertIn(containers[0].id, network['Containers']) - self.assertEqual(containers[0].get('Config.Hostname'), service.name) web_container = self.project.get_service('web').containers()[0] self.assertFalse(web_container.get('HostConfig.Links')) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8c5c888f..bc0db6fb 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -213,16 +213,6 @@ class ServiceTest(unittest.TestCase): opts = service._get_container_create_options({'image': 'foo'}, 1) self.assertIsNone(opts.get('hostname')) - def test_hostname_defaults_to_service_name_when_using_networking(self): - service = Service( - 'foo', - image='foo', - use_networking=True, - client=self.mock_client, - ) - opts = service._get_container_create_options({'image': 'foo'}, 1) - self.assertEqual(opts['hostname'], 'foo') - def test_get_container_create_options_with_name_option(self): service = Service( 'foo', From 0e19c92e82c75f821c231367b5cda88eefdf1427 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Nov 2015 16:37:44 -0500 Subject: [PATCH 084/359] Make working_dir consistent in the config package. - make it a positional arg, since it's required - make it the first argument for all functions that require it - remove an unnecessary one-line function that was only called in one place Signed-off-by: Daniel Nephin --- compose/config/config.py | 33 ++++++++++++-------------------- tests/unit/config/config_test.py | 2 +- 2 files changed, 13 insertions(+), 22 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 5bc534fe..141fa89d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -252,7 +252,7 @@ class ServiceLoader(object): if not self.already_seen: validate_against_service_schema(service_dict, self.service_name) - return process_container_options(service_dict, working_dir=self.working_dir) + return process_container_options(self.working_dir, service_dict) def validate_and_construct_extends(self): extends = self.service_dict['extends'] @@ -321,7 +321,7 @@ def resolve_environment(working_dir, service_dict): env = {} if 'env_file' in service_dict: - for env_file in get_env_files(service_dict, working_dir=working_dir): + for env_file in get_env_files(working_dir, service_dict): env.update(env_vars_from_file(env_file)) env.update(parse_environment(service_dict.get('environment'))) @@ -345,14 +345,14 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'net: container' cannot be extended" % error_prefix) -def process_container_options(service_dict, working_dir=None): - service_dict = service_dict.copy() +def process_container_options(working_dir, service_dict): + service_dict = dict(service_dict) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: - service_dict['volumes'] = resolve_volume_paths(service_dict, working_dir=working_dir) + service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) if 'build' in service_dict: - service_dict['build'] = resolve_build_path(service_dict['build'], working_dir=working_dir) + service_dict['build'] = expand_path(working_dir, service_dict['build']) if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) @@ -428,7 +428,7 @@ def merge_environment(base, override): return env -def get_env_files(options, working_dir=None): +def get_env_files(working_dir, options): if 'env_file' not in options: return {} @@ -488,17 +488,14 @@ def env_vars_from_file(filename): return env -def resolve_volume_paths(service_dict, working_dir=None): - if working_dir is None: - raise Exception("No working_dir passed to resolve_volume_paths()") - +def resolve_volume_paths(working_dir, service_dict): return [ - resolve_volume_path(v, working_dir, service_dict['name']) - for v in service_dict['volumes'] + resolve_volume_path(working_dir, volume, service_dict['name']) + for volume in service_dict['volumes'] ] -def resolve_volume_path(volume, working_dir, service_name): +def resolve_volume_path(working_dir, volume, service_name): container_path, host_path = split_path_mapping(volume) if host_path is not None: @@ -510,12 +507,6 @@ def resolve_volume_path(volume, working_dir, service_name): return container_path -def resolve_build_path(build_path, working_dir=None): - if working_dir is None: - raise Exception("No working_dir passed to resolve_build_path") - return expand_path(working_dir, build_path) - - def validate_paths(service_dict): if 'build' in service_dict: build_path = service_dict['build'] @@ -582,7 +573,7 @@ def parse_labels(labels): return dict(split_label(e) for e in labels) if isinstance(labels, dict): - return labels + return dict(labels) def split_label(label): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 69b23585..e0d2e870 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -577,7 +577,7 @@ class VolumeConfigTest(unittest.TestCase): def test_volume_path_with_non_ascii_directory(self): volume = u'/Füü/data:/data' - container_path = config.resolve_volume_path(volume, ".", "test") + container_path = config.resolve_volume_path(".", volume, "test") self.assertEqual(container_path, volume) From ec22d98377eb5c12ba3bf5fac0eb4bff379e3242 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Nov 2015 16:46:19 -0500 Subject: [PATCH 085/359] Use VolumeSpec instead of re-parsing the volume string. Signed-off-by: Daniel Nephin --- compose/service.py | 19 +++++++++++-------- tests/unit/service_test.py | 4 ++-- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/compose/service.py b/compose/service.py index e5a4cc4a..16e3fd00 100644 --- a/compose/service.py +++ b/compose/service.py @@ -911,14 +911,15 @@ def merge_volume_bindings(volumes_option, previous_container): """Return a list of volume bindings for a container. Container data volumes are replaced by those from the previous container. """ + volumes = [parse_volume_spec(volume) for volume in volumes_option or []] volume_bindings = dict( - build_volume_binding(parse_volume_spec(volume)) - for volume in volumes_option or [] - if ':' in volume) + build_volume_binding(volume) + for volume in volumes + if volume.external) if previous_container: volume_bindings.update( - get_container_data_volumes(previous_container, volumes_option)) + get_container_data_volumes(previous_container, volumes)) return list(volume_bindings.values()) @@ -929,12 +930,14 @@ def get_container_data_volumes(container, volumes_option): """ volumes = [] - volumes_option = volumes_option or [] container_volumes = container.get('Volumes') or {} - image_volumes = container.image_config['ContainerConfig'].get('Volumes') or {} + image_volumes = [ + parse_volume_spec(volume) + for volume in + container.image_config['ContainerConfig'].get('Volumes') or {} + ] - for volume in set(volumes_option + list(image_volumes)): - volume = parse_volume_spec(volume) + for volume in set(volumes_option + image_volumes): # No need to preserve host volumes if volume.external: continue diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8c5c888f..ed02bb4c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -593,11 +593,11 @@ class ServiceVolumesTest(unittest.TestCase): self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) def test_get_container_data_volumes(self): - options = [ + options = [parse_volume_spec(v) for v in [ '/host/volume:/host/volume:ro', '/new/volume', '/existing/volume', - ] + ]] self.mock_client.inspect_image.return_value = { 'ContainerConfig': { From 7c2a16234f333102dfc21c0597f58c030b4a222e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Nov 2015 12:29:52 -0500 Subject: [PATCH 086/359] Recreate dependents when a dependency is created (not just when it's recreated). Signed-off-by: Daniel Nephin --- compose/project.py | 2 +- tests/integration/state_test.py | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/compose/project.py b/compose/project.py index 1e01eaf6..f0478203 100644 --- a/compose/project.py +++ b/compose/project.py @@ -322,7 +322,7 @@ class Project(object): name for name in service.get_dependency_names() if name in plans - and plans[name].action == 'recreate' + and plans[name].action in ('recreate', 'create') ] if updated_dependencies and strategy.allows_recreate: diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 7830ba32..1fecce87 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -176,6 +176,19 @@ class ProjectWithDependenciesTest(ProjectTestCase): containers = self.run_up(next_cfg) self.assertEqual(len(containers), 2) + def test_service_recreated_when_dependency_created(self): + containers = self.run_up(self.cfg, service_names=['web'], start_deps=False) + self.assertEqual(len(containers), 1) + + containers = self.run_up(self.cfg) + self.assertEqual(len(containers), 3) + + web, = [c for c in containers if c.service == 'web'] + nginx, = [c for c in containers if c.service == 'nginx'] + + self.assertEqual(web.links(), ['composetest_db_1', 'db', 'db_1']) + self.assertEqual(nginx.links(), ['composetest_web_1', 'web', 'web_1']) + class ServiceStateTest(DockerClientTestCase): """Test cases for Service.convergence_plan.""" From 1bfb71032650b9a8b4184316af906df382807d9d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 9 Nov 2015 17:24:21 +0000 Subject: [PATCH 087/359] Fix parallel output We were outputting an extra line, which in *some* cases, on *some* terminals, was causing the output of parallel actions to get messed up. In particular, it would happen when the terminal had just been cleared or hadn't yet filled up with a screen's worth of text. Signed-off-by: Aanand Prasad --- compose/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/utils.py b/compose/utils.py index c8fddc5f..14cca61b 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -164,7 +164,7 @@ def write_out_msg(stream, lines, msg_index, msg, status="done"): stream.write("%c[%dA" % (27, diff)) # erase stream.write("%c[2K\r" % 27) - stream.write("{} {} ... {}\n".format(msg, obj_index, status)) + stream.write("{} {} ... {}\r".format(msg, obj_index, status)) # move back down stream.write("%c[%dB" % (27, diff)) else: From a1e140f5a3376812d3d13b011ac93760dfd4f1c2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 9 Nov 2015 13:07:26 -0800 Subject: [PATCH 088/359] Update service config_dict computation to include volumes_from mode Ensure config_hash is updated when volumes_from mode is changed, and service is recreated on next up as a result. Signed-off-by: Joffrey F --- compose/service.py | 4 +++- tests/unit/service_test.py | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/compose/service.py b/compose/service.py index e5a4cc4a..8d40ab10 100644 --- a/compose/service.py +++ b/compose/service.py @@ -502,7 +502,9 @@ class Service(object): 'image_id': self.image()['Id'], 'links': self.get_link_names(), 'net': self.net.id, - 'volumes_from': self.get_volumes_from_names(), + 'volumes_from': [ + (v.source.name, v.mode) for v in self.volumes_from if isinstance(v.source, Service) + ], } def get_dependency_names(self): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 8c5c888f..bd771225 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -410,7 +410,7 @@ class ServiceTest(unittest.TestCase): 'options': {'image': 'example.com/foo'}, 'links': [('one', 'one')], 'net': 'other', - 'volumes_from': ['two'], + 'volumes_from': [('two', 'rw')], } self.assertEqual(config_dict, expected) From 3474bb6cf5c1961deb20cd186cd2a2dbf3224f0c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Nov 2015 14:30:27 -0500 Subject: [PATCH 089/359] Cleanup workaround in testcase.py Signed-off-by: Daniel Nephin --- tests/integration/testcases.py | 35 ++++++++++++---------------------- 1 file changed, 12 insertions(+), 23 deletions(-) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 686a2b69..60e67b5b 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -42,34 +42,23 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - links = kwargs.get('links', None) - volumes_from = kwargs.get('volumes_from', None) - net = kwargs.get('net', None) + workaround_options = {} + for option in ['links', 'volumes_from', 'net']: + if option in kwargs: + workaround_options[option] = kwargs.pop(option, None) - workaround_options = ['links', 'volumes_from', 'net'] - for key in workaround_options: - try: - del kwargs[key] - except KeyError: - pass - - options = ServiceLoader(working_dir='.', filename=None, service_name=name, service_dict=kwargs).make_service_dict() + options = ServiceLoader( + working_dir='.', + filename=None, + service_name=name, + service_dict=kwargs + ).make_service_dict() + options.update(workaround_options) labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() - if links: - options['links'] = links - if volumes_from: - options['volumes_from'] = volumes_from - if net: - options['net'] = net - - return Service( - project='composetest', - client=self.client, - **options - ) + return Service(project='composetest', client=self.client, **options) def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) From 45724fc667ff54f2ae26e00c27f76d43556b9239 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Nov 2015 13:56:25 -0500 Subject: [PATCH 090/359] Only create the default network if at least one service needs it. Signed-off-by: Daniel Nephin --- compose/project.py | 7 +++++-- tests/integration/project_test.py | 16 ++++++++++++++++ tests/unit/project_test.py | 28 ++++++++++++++++++++++++++++ 3 files changed, 49 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index f0478203..1f10934c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -300,7 +300,7 @@ class Project(object): plans = self._get_convergence_plans(services, strategy) - if self.use_networking: + if self.use_networking and self.uses_default_network(): self.ensure_network_exists() return [ @@ -383,7 +383,10 @@ class Project(object): def remove_network(self): network = self.get_network() if network: - self.client.remove_network(network['id']) + self.client.remove_network(network['Id']) + + def uses_default_network(self): + return any(service.net.mode == self.name for service in self.services) def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 95052387..2ce31900 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -7,6 +7,7 @@ from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project from compose.service import ConvergenceStrategy +from compose.service import Net from compose.service import VolumeFromSpec @@ -111,6 +112,7 @@ class ProjectTest(DockerClientTestCase): network_name = 'network_does_exist' project = Project(network_name, [], client) client.create_network(network_name) + self.addCleanup(client.remove_network, network_name) assert project.get_network()['Name'] == network_name def test_net_from_service(self): @@ -398,6 +400,20 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('data').containers(stopped=True)), 1) self.assertEqual(len(project.get_service('console').containers()), 0) + def test_project_up_with_custom_network(self): + self.require_api_version('1.21') + client = docker_client(version='1.21') + network_name = 'composetest-custom' + + client.create_network(network_name) + self.addCleanup(client.remove_network, network_name) + + web = self.create_service('web', net=Net(network_name)) + project = Project('composetest', [web], client, use_networking=True) + project.up() + + assert project.get_network() is None + def test_unscale_after_restart(self): web = self.create_service('web') project = Project('composetest', [web], self.client) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index fc189fbb..b38f5c78 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -7,6 +7,8 @@ from .. import unittest from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project +from compose.service import ContainerNet +from compose.service import Net from compose.service import Service @@ -263,6 +265,32 @@ class ProjectTest(unittest.TestCase): service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_name) + def test_uses_default_network_true(self): + web = Service('web', project='test', image="alpine", net=Net('test')) + db = Service('web', project='test', image="alpine", net=Net('other')) + project = Project('test', [web, db], None) + assert project.uses_default_network() + + def test_uses_default_network_custom_name(self): + web = Service('web', project='test', image="alpine", net=Net('other')) + project = Project('test', [web], None) + assert not project.uses_default_network() + + def test_uses_default_network_host(self): + web = Service('web', project='test', image="alpine", net=Net('host')) + project = Project('test', [web], None) + assert not project.uses_default_network() + + def test_uses_default_network_container(self): + container = mock.Mock(id='test') + web = Service( + 'web', + project='test', + image="alpine", + net=ContainerNet(container)) + project = Project('test', [web], None) + assert not project.uses_default_network() + def test_container_without_name(self): self.mock_client.containers.return_value = [ {'Image': 'busybox:latest', 'Id': '1', 'Name': '1'}, From c5c36d8b006d9694c34b06e434e08bb17b025250 Mon Sep 17 00:00:00 2001 From: Adrian Budau Date: Tue, 27 Oct 2015 02:00:51 -0700 Subject: [PATCH 091/359] Added --force-rm to compose build. It's a flag passed to docker build that removes the intermediate containers left behind on fail builds. Signed-off-by: Adrian Budau --- compose/cli/main.py | 4 ++- compose/project.py | 4 +-- compose/service.py | 3 +- contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 1 + docs/reference/build.md | 1 + tests/acceptance/cli_test.py | 28 +++++++++++++++++++ .../simple-failing-dockerfile/Dockerfile | 7 +++++ .../docker-compose.yml | 2 ++ tests/unit/service_test.py | 1 + 10 files changed, 48 insertions(+), 5 deletions(-) create mode 100644 tests/fixtures/simple-failing-dockerfile/Dockerfile create mode 100644 tests/fixtures/simple-failing-dockerfile/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 11aeac38..b1aa9951 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -180,12 +180,14 @@ class TopLevelCommand(DocoptCommand): Usage: build [options] [SERVICE...] Options: + --force-rm Always remove intermediate containers. --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. """ + force_rm = bool(options.get('--force-rm', False)) no_cache = bool(options.get('--no-cache', False)) pull = bool(options.get('--pull', False)) - project.build(service_names=options['SERVICE'], no_cache=no_cache, pull=pull) + project.build(service_names=options['SERVICE'], no_cache=no_cache, pull=pull, force_rm=force_rm) def help(self, project, options): """ diff --git a/compose/project.py b/compose/project.py index f0478203..d2ba86b1 100644 --- a/compose/project.py +++ b/compose/project.py @@ -278,10 +278,10 @@ class Project(object): for service in self.get_services(service_names): service.restart(**options) - def build(self, service_names=None, no_cache=False, pull=False): + def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): for service in self.get_services(service_names): if service.can_be_built(): - service.build(no_cache, pull) + service.build(no_cache, pull, force_rm) else: log.info('%s uses an image, skipping' % service.name) diff --git a/compose/service.py b/compose/service.py index f2861954..2055a6fe 100644 --- a/compose/service.py +++ b/compose/service.py @@ -701,7 +701,7 @@ class Service(object): cgroup_parent=cgroup_parent ) - def build(self, no_cache=False, pull=False): + def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) path = self.options['build'] @@ -715,6 +715,7 @@ class Service(object): tag=self.image_name, stream=True, rm=True, + forcerm=force_rm, pull=pull, nocache=no_cache, dockerfile=self.options.get('dockerfile', None), diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e25a43a1..f6f7ad40 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -93,7 +93,7 @@ __docker_compose_services_stopped() { _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --force-rm --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index d79b25d1..08d5150d 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -192,6 +192,7 @@ __docker-compose_subcommand() { (build) _arguments \ $opts_help \ + '--force-rm[Always remove intermediate containers.]' \ '--no-cache[Do not use cache when building the image]' \ '--pull[Always attempt to pull a newer version of the image.]' \ '*:services:__docker-compose_services_from_build' && ret=0 diff --git a/docs/reference/build.md b/docs/reference/build.md index c427199f..84aefc25 100644 --- a/docs/reference/build.md +++ b/docs/reference/build.md @@ -15,6 +15,7 @@ parent = "smn_compose_cli" Usage: build [options] [SERVICE...] Options: +--force-rm Always remove intermediate containers. --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. ``` diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index fc68f9d8..88a43d7f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -9,6 +9,7 @@ from operator import attrgetter from .. import mock from compose.cli.command import get_project from compose.cli.docker_client import docker_client +from compose.container import Container from tests.integration.testcases import DockerClientTestCase @@ -145,6 +146,33 @@ class CLITestCase(DockerClientTestCase): assert BUILD_CACHE_TEXT not in result.stdout assert BUILD_PULL_TEXT in result.stdout + def test_build_failed(self): + self.base_dir = 'tests/fixtures/simple-failing-dockerfile' + self.dispatch(['build', 'simple'], returncode=1) + + labels = ["com.docker.compose.test_failing_image=true"] + containers = [ + Container.from_ps(self.project.client, c) + for c in self.project.client.containers( + all=True, + filters={"label": labels}) + ] + assert len(containers) == 1 + + def test_build_failed_forcerm(self): + self.base_dir = 'tests/fixtures/simple-failing-dockerfile' + self.dispatch(['build', '--force-rm', 'simple'], returncode=1) + + labels = ["com.docker.compose.test_failing_image=true"] + + containers = [ + Container.from_ps(self.project.client, c) + for c in self.project.client.containers( + all=True, + filters={"label": labels}) + ] + assert not containers + def test_up_detached(self): self.dispatch(['up', '-d']) service = self.project.get_service('simple') diff --git a/tests/fixtures/simple-failing-dockerfile/Dockerfile b/tests/fixtures/simple-failing-dockerfile/Dockerfile new file mode 100644 index 00000000..c2d06b16 --- /dev/null +++ b/tests/fixtures/simple-failing-dockerfile/Dockerfile @@ -0,0 +1,7 @@ +FROM busybox:latest +LABEL com.docker.compose.test_image=true +LABEL com.docker.compose.test_failing_image=true +# With the following label the container wil be cleaned up automatically +# Must be kept in sync with LABEL_PROJECT from compose/const.py +LABEL com.docker.compose.project=composetest +RUN exit 1 diff --git a/tests/fixtures/simple-failing-dockerfile/docker-compose.yml b/tests/fixtures/simple-failing-dockerfile/docker-compose.yml new file mode 100644 index 00000000..b0357541 --- /dev/null +++ b/tests/fixtures/simple-failing-dockerfile/docker-compose.yml @@ -0,0 +1,2 @@ +simple: + build: . diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 50ce5da5..cacdcc77 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -356,6 +356,7 @@ class ServiceTest(unittest.TestCase): stream=True, path='.', pull=False, + forcerm=False, nocache=False, rm=True, ) From 133d213e78c060ad0e1448fb52086c120ffdd15d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 10 Nov 2015 10:11:20 -0800 Subject: [PATCH 092/359] Use exit code 1 when encountering a ReadTimeout Signed-off-by: Joffrey F --- compose/cli/main.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index 11aeac38..95db45ce 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -80,6 +80,7 @@ def main(): "If you encounter this issue regularly because of slow network conditions, consider setting " "COMPOSE_HTTP_TIMEOUT to a higher value (current value: %s)." % HTTP_TIMEOUT ) + sys.exit(1) def setup_logging(): From 338bbb5063d882bb751cae3a25651bcf6e61b679 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 10 Nov 2015 13:11:59 -0500 Subject: [PATCH 093/359] Re-order flags in bash completion and remove unnecessary variables from build command. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 9 +++++---- contrib/completion/bash/docker-compose | 2 +- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index b1aa9951..a4b774a9 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -184,10 +184,11 @@ class TopLevelCommand(DocoptCommand): --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. """ - force_rm = bool(options.get('--force-rm', False)) - no_cache = bool(options.get('--no-cache', False)) - pull = bool(options.get('--pull', False)) - project.build(service_names=options['SERVICE'], no_cache=no_cache, pull=pull, force_rm=force_rm) + project.build( + service_names=options['SERVICE'], + no_cache=bool(options.get('--no-cache', False)), + pull=bool(options.get('--pull', False)), + force_rm=bool(options.get('--force-rm', False))) def help(self, project, options): """ diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index f6f7ad40..b4f4387f 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -93,7 +93,7 @@ __docker_compose_services_stopped() { _docker_compose_build() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --force-rm --no-cache --pull" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--force-rm --help --no-cache --pull" -- "$cur" ) ) ;; *) __docker_compose_services_from_build From a8ac6e6f93be89ba82f60e7136a2ccd8fdec4798 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Nov 2015 18:00:24 -0500 Subject: [PATCH 094/359] Add a warning when the host volume config is being ignored. Signed-off-by: Daniel Nephin --- compose/service.py | 27 +++++++++++++++++++++++---- tests/integration/service_test.py | 27 +++++++++++++++++++++++++++ tests/unit/service_test.py | 12 ++++++------ 3 files changed, 56 insertions(+), 10 deletions(-) diff --git a/compose/service.py b/compose/service.py index 2055a6fe..32f19323 100644 --- a/compose/service.py +++ b/compose/service.py @@ -918,8 +918,10 @@ def merge_volume_bindings(volumes_option, previous_container): if volume.external) if previous_container: + data_volumes = get_container_data_volumes(previous_container, volumes) + warn_on_masked_volume(volumes, data_volumes, previous_container.service) volume_bindings.update( - get_container_data_volumes(previous_container, volumes)) + build_volume_binding(volume) for volume in data_volumes) return list(volume_bindings.values()) @@ -929,7 +931,6 @@ def get_container_data_volumes(container, volumes_option): a mapping of volume bindings for those volumes. """ volumes = [] - container_volumes = container.get('Volumes') or {} image_volumes = [ parse_volume_spec(volume) @@ -949,9 +950,27 @@ def get_container_data_volumes(container, volumes_option): # Copy existing volume from old container volume = volume._replace(external=volume_path) - volumes.append(build_volume_binding(volume)) + volumes.append(volume) - return dict(volumes) + return volumes + + +def warn_on_masked_volume(volumes_option, container_volumes, service): + container_volumes = dict( + (volume.internal, volume.external) + for volume in container_volumes) + + for volume in volumes_option: + if container_volumes.get(volume.internal) != volume.external: + log.warn(( + "Service \"{service}\" is using volume \"{volume}\" from the " + "previous container. Host mapping \"{host_path}\" has no effect. " + "Remove the existing containers (with `docker-compose rm {service}`) " + "to use the host volume mapping." + ).format( + service=service, + volume=volume.internal, + host_path=volume.external)) def build_volume_binding(volume_spec): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 804f5219..88214e83 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -369,6 +369,33 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + def test_execute_convergence_plan_when_image_volume_masks_config(self): + service = Service( + project='composetest', + name='db', + client=self.client, + build='tests/fixtures/dockerfile-with-volume', + ) + + old_container = create_and_start_container(service) + self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) + volume_path = old_container.get('Volumes')['/data'] + + service.options['volumes'] = ['/tmp:/data'] + + with mock.patch('compose.service.log') as mock_log: + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [old_container])) + + mock_log.warn.assert_called_once_with(mock.ANY) + _, args, kwargs = mock_log.warn.mock_calls[0] + self.assertIn( + "Service \"db\" is using volume \"/data\" from the previous container", + args[0]) + + self.assertEqual(list(new_container.get('Volumes')), ['/data']) + self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + def test_start_container_passes_through_options(self): db = self.create_service('db') create_and_start_container(db, environment={'FOO': 'BAR'}) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index cacdcc77..b69e0996 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -607,13 +607,13 @@ class ServiceVolumesTest(unittest.TestCase): }, }, has_been_inspected=True) - expected = { - '/existing/volume': '/var/lib/docker/aaaaaaaa:/existing/volume:rw', - '/mnt/image/data': '/var/lib/docker/cccccccc:/mnt/image/data:rw', - } + expected = [ + parse_volume_spec('/var/lib/docker/aaaaaaaa:/existing/volume:rw'), + parse_volume_spec('/var/lib/docker/cccccccc:/mnt/image/data:rw'), + ] - binds = get_container_data_volumes(container, options) - self.assertEqual(binds, expected) + volumes = get_container_data_volumes(container, options) + self.assertEqual(sorted(volumes), sorted(expected)) def test_merge_volume_bindings(self): options = [ From 22d90d21800bd5bf5c695f09cd3c98928781db9e Mon Sep 17 00:00:00 2001 From: Kevin Greene Date: Mon, 26 Oct 2015 17:39:50 -0400 Subject: [PATCH 095/359] Added ulimits functionality to docker compose Signed-off-by: Kevin Greene --- compose/config/config.py | 12 +++++++ compose/config/fields_schema.json | 19 ++++++++++ compose/service.py | 19 ++++++++++ docs/compose-file.md | 11 ++++++ tests/integration/service_test.py | 31 ++++++++++++++++ tests/unit/config/config_test.py | 60 +++++++++++++++++++++++++++++++ 6 files changed, 152 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 434589d3..7931608d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -345,6 +345,15 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'net: container' cannot be extended" % error_prefix) +def validate_ulimits(ulimit_config): + for limit_name, soft_hard_values in six.iteritems(ulimit_config): + if isinstance(soft_hard_values, dict): + if not soft_hard_values['soft'] <= soft_hard_values['hard']: + raise ConfigurationError( + "ulimit_config \"{}\" cannot contain a 'soft' value higher " + "than 'hard' value".format(ulimit_config)) + + def process_container_options(working_dir, service_dict): service_dict = dict(service_dict) @@ -357,6 +366,9 @@ def process_container_options(working_dir, service_dict): if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) + if 'ulimits' in service_dict: + validate_ulimits(service_dict['ulimits']) + return service_dict diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index e254e353..f22b513a 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -116,6 +116,25 @@ "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "stdin_open": {"type": "boolean"}, "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, "user": {"type": "string"}, "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "volume_driver": {"type": "string"}, diff --git a/compose/service.py b/compose/service.py index 2055a6fe..9e0066b7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -676,6 +676,7 @@ class Service(object): devices = options.get('devices', None) cgroup_parent = options.get('cgroup_parent', None) + ulimits = build_ulimits(options.get('ulimits', None)) return self.client.create_host_config( links=self._get_links(link_to_self=one_off), @@ -692,6 +693,7 @@ class Service(object): cap_drop=cap_drop, mem_limit=options.get('mem_limit'), memswap_limit=options.get('memswap_limit'), + ulimits=ulimits, log_config=log_config, extra_hosts=extra_hosts, read_only=read_only, @@ -1073,6 +1075,23 @@ def parse_restart_spec(restart_config): return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} +# Ulimits + + +def build_ulimits(ulimit_config): + if not ulimit_config: + return None + ulimits = [] + for limit_name, soft_hard_values in six.iteritems(ulimit_config): + if isinstance(soft_hard_values, six.integer_types): + ulimits.append({'name': limit_name, 'soft': soft_hard_values, 'hard': soft_hard_values}) + elif isinstance(soft_hard_values, dict): + ulimit_dict = {'name': limit_name} + ulimit_dict.update(soft_hard_values) + ulimits.append(ulimit_dict) + + return ulimits + # Extra hosts diff --git a/docs/compose-file.md b/docs/compose-file.md index 4f8fc9e0..3b36fa2b 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -333,6 +333,17 @@ Override the default labeling scheme for each container. - label:user:USER - label:role:ROLE +### ulimits + +Override the default ulimits for a container. You can either use a number +to set the hard and soft limits, or specify them in a dictionary. + + ulimits: + nproc: 65535 + nofile: + soft: 20000 + hard: 40000 + ### volumes, volume\_driver Mount paths as volumes, optionally specifying a path on the host machine diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 804f5219..2f3be89a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -22,6 +22,7 @@ from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container from compose.service import build_extra_hosts +from compose.service import build_ulimits from compose.service import ConfigError from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy @@ -164,6 +165,36 @@ class ServiceTest(DockerClientTestCase): {'www.example.com': '192.168.0.17', 'api.example.com': '192.168.0.18'}) + def sort_dicts_by_name(self, dictionary_list): + return sorted(dictionary_list, key=lambda k: k['name']) + + def test_build_ulimits_with_invalid_options(self): + self.assertRaises(ConfigError, lambda: build_ulimits({'nofile': {'soft': 10000, 'hard': 10}})) + + def test_build_ulimits_with_integers(self): + self.assertEqual(build_ulimits( + {'nofile': {'soft': 10000, 'hard': 20000}}), + [{'name': 'nofile', 'soft': 10000, 'hard': 20000}]) + self.assertEqual(self.sort_dicts_by_name(build_ulimits( + {'nofile': {'soft': 10000, 'hard': 20000}, 'nproc': {'soft': 65535, 'hard': 65535}})), + self.sort_dicts_by_name([{'name': 'nofile', 'soft': 10000, 'hard': 20000}, + {'name': 'nproc', 'soft': 65535, 'hard': 65535}])) + + def test_build_ulimits_with_dicts(self): + self.assertEqual(build_ulimits( + {'nofile': 20000}), + [{'name': 'nofile', 'soft': 20000, 'hard': 20000}]) + self.assertEqual(self.sort_dicts_by_name(build_ulimits( + {'nofile': 20000, 'nproc': 65535})), + self.sort_dicts_by_name([{'name': 'nofile', 'soft': 20000, 'hard': 20000}, + {'name': 'nproc', 'soft': 65535, 'hard': 65535}])) + + def test_build_ulimits_with_integers_and_dicts(self): + self.assertEqual(self.sort_dicts_by_name(build_ulimits( + {'nproc': 65535, 'nofile': {'soft': 10000, 'hard': 20000}})), + self.sort_dicts_by_name([{'name': 'nofile', 'soft': 10000, 'hard': 20000}, + {'name': 'nproc', 'soft': 65535, 'hard': 65535}])) + def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e0d2e870..f27329ba 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -349,6 +349,66 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_ulimits_invalid_keys_validation_error(self): + expected_error_msg = "Service 'web' configuration key 'ulimits' contains unsupported option: 'not_soft_or_hard'" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + build_config_details( + {'web': { + 'image': 'busybox', + 'ulimits': { + 'nofile': { + "not_soft_or_hard": 100, + "soft": 10000, + "hard": 20000, + } + } + }}, + 'working_dir', + 'filename.yml' + ) + ) + + def test_config_ulimits_required_keys_validation_error(self): + expected_error_msg = "Service 'web' configuration key 'ulimits' u?'hard' is a required property" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + build_config_details( + {'web': { + 'image': 'busybox', + 'ulimits': { + 'nofile': { + "soft": 10000, + } + } + }}, + 'working_dir', + 'filename.yml' + ) + ) + + def test_config_ulimits_soft_greater_than_hard_error(self): + expected_error_msg = "cannot contain a 'soft' value higher than 'hard' value" + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + build_config_details( + {'web': { + 'image': 'busybox', + 'ulimits': { + 'nofile': { + "soft": 10000, + "hard": 1000 + } + } + }}, + 'working_dir', + 'filename.yml' + ) + ) + def test_valid_config_which_allows_two_type_definitions(self): expose_values = [["8000"], [8000]] for expose in expose_values: From 7365a398b327ecee9de01da5deab83275a87d779 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 10 Nov 2015 13:37:07 -0500 Subject: [PATCH 096/359] Update doc wording for ulimits. and move tests to the correct module Signed-off-by: Daniel Nephin --- docs/compose-file.md | 5 ++-- tests/integration/service_test.py | 31 ----------------------- tests/unit/service_test.py | 42 +++++++++++++++++++++++++++++++ 3 files changed, 45 insertions(+), 33 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 3b36fa2b..51d1f5e1 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -335,8 +335,9 @@ Override the default labeling scheme for each container. ### ulimits -Override the default ulimits for a container. You can either use a number -to set the hard and soft limits, or specify them in a dictionary. +Override the default ulimits for a container. You can either specify a single +limit as an integer or soft/hard limits as a mapping. + ulimits: nproc: 65535 diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 2f3be89a..804f5219 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -22,7 +22,6 @@ from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container from compose.service import build_extra_hosts -from compose.service import build_ulimits from compose.service import ConfigError from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy @@ -165,36 +164,6 @@ class ServiceTest(DockerClientTestCase): {'www.example.com': '192.168.0.17', 'api.example.com': '192.168.0.18'}) - def sort_dicts_by_name(self, dictionary_list): - return sorted(dictionary_list, key=lambda k: k['name']) - - def test_build_ulimits_with_invalid_options(self): - self.assertRaises(ConfigError, lambda: build_ulimits({'nofile': {'soft': 10000, 'hard': 10}})) - - def test_build_ulimits_with_integers(self): - self.assertEqual(build_ulimits( - {'nofile': {'soft': 10000, 'hard': 20000}}), - [{'name': 'nofile', 'soft': 10000, 'hard': 20000}]) - self.assertEqual(self.sort_dicts_by_name(build_ulimits( - {'nofile': {'soft': 10000, 'hard': 20000}, 'nproc': {'soft': 65535, 'hard': 65535}})), - self.sort_dicts_by_name([{'name': 'nofile', 'soft': 10000, 'hard': 20000}, - {'name': 'nproc', 'soft': 65535, 'hard': 65535}])) - - def test_build_ulimits_with_dicts(self): - self.assertEqual(build_ulimits( - {'nofile': 20000}), - [{'name': 'nofile', 'soft': 20000, 'hard': 20000}]) - self.assertEqual(self.sort_dicts_by_name(build_ulimits( - {'nofile': 20000, 'nproc': 65535})), - self.sort_dicts_by_name([{'name': 'nofile', 'soft': 20000, 'hard': 20000}, - {'name': 'nproc', 'soft': 65535, 'hard': 65535}])) - - def test_build_ulimits_with_integers_and_dicts(self): - self.assertEqual(self.sort_dicts_by_name(build_ulimits( - {'nproc': 65535, 'nofile': {'soft': 10000, 'hard': 20000}})), - self.sort_dicts_by_name([{'name': 'nofile', 'soft': 10000, 'hard': 20000}, - {'name': 'nproc', 'soft': 65535, 'hard': 65535}])) - def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index cacdcc77..52128d46 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -12,6 +12,7 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.container import Container +from compose.service import build_ulimits from compose.service import build_volume_binding from compose.service import ConfigError from compose.service import ContainerNet @@ -497,6 +498,47 @@ class ServiceTest(unittest.TestCase): self.assertEqual(service._get_links(link_to_self=True), []) +def sort_by_name(dictionary_list): + return sorted(dictionary_list, key=lambda k: k['name']) + + +class BuildUlimitsTestCase(unittest.TestCase): + + def test_build_ulimits_with_dict(self): + ulimits = build_ulimits( + { + 'nofile': {'soft': 10000, 'hard': 20000}, + 'nproc': {'soft': 65535, 'hard': 65535} + } + ) + expected = [ + {'name': 'nofile', 'soft': 10000, 'hard': 20000}, + {'name': 'nproc', 'soft': 65535, 'hard': 65535} + ] + assert sort_by_name(ulimits) == sort_by_name(expected) + + def test_build_ulimits_with_ints(self): + ulimits = build_ulimits({'nofile': 20000, 'nproc': 65535}) + expected = [ + {'name': 'nofile', 'soft': 20000, 'hard': 20000}, + {'name': 'nproc', 'soft': 65535, 'hard': 65535} + ] + assert sort_by_name(ulimits) == sort_by_name(expected) + + def test_build_ulimits_with_integers_and_dicts(self): + ulimits = build_ulimits( + { + 'nproc': 65535, + 'nofile': {'soft': 10000, 'hard': 20000} + } + ) + expected = [ + {'name': 'nofile', 'soft': 10000, 'hard': 20000}, + {'name': 'nproc', 'soft': 65535, 'hard': 65535} + ] + assert sort_by_name(ulimits) == sort_by_name(expected) + + class NetTestCase(unittest.TestCase): def test_net(self): From 98ad5a05e4fb342ba4deed92754da51ca98973b3 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Nov 2015 16:38:38 -0500 Subject: [PATCH 097/359] Validate additional files before merging them. Consolidates all the top level config handling into `process_config_file` which is now used for both files and merge sources. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- compose/config/__init__.py | 1 - compose/config/config.py | 56 +++++++++++++++++--------------- compose/config/validation.py | 10 +----- tests/unit/config/config_test.py | 13 ++++++++ 5 files changed, 45 insertions(+), 37 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 08c1aee0..806926d8 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -13,12 +13,12 @@ from requests.exceptions import ReadTimeout from .. import __version__ from .. import legacy +from ..config import ConfigurationError from ..config import parse_environment from ..const import DEFAULT_TIMEOUT from ..const import HTTP_TIMEOUT from ..const import IS_WINDOWS_PLATFORM from ..progress_stream import StreamOutputError -from ..project import ConfigurationError from ..project import NoSuchService from ..service import BuildError from ..service import ConvergenceStrategy diff --git a/compose/config/__init__.py b/compose/config/__init__.py index de6f10c9..ec607e08 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -1,5 +1,4 @@ # flake8: noqa -from .config import ConfigDetails from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find diff --git a/compose/config/config.py b/compose/config/config.py index 7931608d..feef0387 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -13,7 +13,6 @@ from .errors import ConfigurationError from .interpolation import interpolate_environment_variables from .validation import validate_against_fields_schema from .validation import validate_against_service_schema -from .validation import validate_extended_service_exists from .validation import validate_extends_file_path from .validation import validate_top_level_object @@ -99,6 +98,10 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): :type config: :class:`dict` """ + @classmethod + def from_filename(cls, filename): + return cls(filename, load_yaml(filename)) + def find(base_dir, filenames): if filenames == ['-']: @@ -114,7 +117,7 @@ def find(base_dir, filenames): log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( os.path.dirname(filenames[0]), - [ConfigFile(f, load_yaml(f)) for f in filenames]) + [ConfigFile.from_filename(f) for f in filenames]) def get_default_config_files(base_dir): @@ -183,12 +186,10 @@ def load(config_details): validate_paths(service_dict) return service_dict - def load_file(filename, config): - processed_config = interpolate_environment_variables(config) - validate_against_fields_schema(processed_config) + def build_services(filename, config): return [ build_service(filename, name, service_config) - for name, service_config in processed_config.items() + for name, service_config in config.items() ] def merge_services(base, override): @@ -200,16 +201,27 @@ def load(config_details): for name in all_service_names } - config_file = config_details.config_files[0] - validate_top_level_object(config_file.config) + config_file = process_config_file(config_details.config_files[0]) for next_file in config_details.config_files[1:]: - validate_top_level_object(next_file.config) + next_file = process_config_file(next_file) - config_file = ConfigFile( - config_file.filename, - merge_services(config_file.config, next_file.config)) + config = merge_services(config_file.config, next_file.config) + config_file = config_file._replace(config=config) - return load_file(config_file.filename, config_file.config) + return build_services(config_file.filename, config_file.config) + + +def process_config_file(config_file, service_name=None): + validate_top_level_object(config_file.config) + processed_config = interpolate_environment_variables(config_file.config) + validate_against_fields_schema(processed_config) + + if service_name and service_name not in processed_config: + raise ConfigurationError( + "Cannot extend service '{}' in {}: Service not found".format( + service_name, config_file.filename)) + + return config_file._replace(config=processed_config) class ServiceLoader(object): @@ -259,22 +271,13 @@ class ServiceLoader(object): if not isinstance(extends, dict): extends = {'service': extends} - validate_extends_file_path(self.service_name, extends, self.filename) config_path = self.get_extended_config_path(extends) service_name = extends['service'] - config = load_yaml(config_path) - validate_top_level_object(config) - full_extended_config = interpolate_environment_variables(config) - - validate_extended_service_exists( - service_name, - full_extended_config, - config_path - ) - validate_against_fields_schema(full_extended_config) - - service_config = full_extended_config[service_name] + extended_file = process_config_file( + ConfigFile.from_filename(config_path), + service_name=service_name) + service_config = extended_file.config[service_name] return config_path, service_config, service_name def resolve_extends(self, extended_config_path, service_config, service_name): @@ -304,6 +307,7 @@ class ServiceLoader(object): need to obtain a full path too or we are extending from a service defined in our own file. """ + validate_extends_file_path(self.service_name, extends_options, self.filename) if 'file' in extends_options: return expand_path(self.working_dir, extends_options['file']) return self.filename diff --git a/compose/config/validation.py b/compose/config/validation.py index 542081d5..3bd40410 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -96,14 +96,6 @@ def validate_extends_file_path(service_name, extends_options, filename): ) -def validate_extended_service_exists(extended_service_name, full_extended_config, extended_config_path): - if extended_service_name not in full_extended_config: - msg = ( - "Cannot extend service '%s' in %s: Service not found" - ) % (extended_service_name, extended_config_path) - raise ConfigurationError(msg) - - def get_unsupported_config_msg(service_name, error_key): msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) if error_key in DOCKER_CONFIG_HINTS: @@ -264,7 +256,7 @@ def process_errors(errors, service_name=None): msg)) else: root_msgs.append( - "Service '{}' doesn\'t have any configuration options. " + "Service \"{}\" doesn't have any configuration options. " "All top level keys in your docker-compose.yml must map " "to a dictionary of configuration options.'".format(service_name)) elif error.validator == 'required': diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f27329ba..ab34f4dc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -195,6 +195,19 @@ class ConfigTest(unittest.TestCase): ] self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_load_with_multiple_files_and_invalid_override(self): + base_file = config.ConfigFile( + 'base.yaml', + {'web': {'image': 'example/web'}}) + override_file = config.ConfigFile( + 'override.yaml', + {'bogus': 'thing'}) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Service "bogus" doesn\'t have any configuration' in exc.exconly() + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( From a92d86308f7bc3e571f06df4990e609ec370bfc5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Nov 2015 17:18:47 -0500 Subject: [PATCH 098/359] Rename ServiceLoader to ServiceExtendsResolver ServiceLoader has evolved to be not really all that related to "loading" a service. It's responsibility is more to do with handling the `extends` field, which is only part of loading. The class and its primary method (make_service_dict()) were renamed to better reflect their responsibility. As part of that change process_container_options() was removed from make_service_dict() and renamed to process_service(). It contains logic for handling the non-extends options. This change allows us to remove the hacks from testcase.py and only call the functions we need to format a service dict correctly for integration tests. Signed-off-by: Daniel Nephin --- compose/config/config.py | 27 ++++++++++++--------------- tests/integration/service_test.py | 25 ++++++++++++++++++++++--- tests/integration/testcases.py | 21 +++++++-------------- tests/unit/config/config_test.py | 7 ++++--- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index feef0387..7846ea7b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -177,12 +177,12 @@ def load(config_details): """ def build_service(filename, service_name, service_dict): - loader = ServiceLoader( + resolver = ServiceExtendsResolver( config_details.working_dir, filename, service_name, service_dict) - service_dict = loader.make_service_dict() + service_dict = process_service(config_details.working_dir, resolver.run()) validate_paths(service_dict) return service_dict @@ -224,7 +224,7 @@ def process_config_file(config_file, service_name=None): return config_file._replace(config=processed_config) -class ServiceLoader(object): +class ServiceExtendsResolver(object): def __init__( self, working_dir, @@ -234,7 +234,7 @@ class ServiceLoader(object): already_seen=None ): if working_dir is None: - raise ValueError("No working_dir passed to ServiceLoader()") + raise ValueError("No working_dir passed to ServiceExtendsResolver()") self.working_dir = os.path.abspath(working_dir) @@ -251,7 +251,7 @@ class ServiceLoader(object): if self.signature(name) in self.already_seen: raise CircularReference(self.already_seen + [self.signature(name)]) - def make_service_dict(self): + def run(self): service_dict = dict(self.service_dict) env = resolve_environment(self.working_dir, self.service_dict) if env: @@ -264,7 +264,7 @@ class ServiceLoader(object): if not self.already_seen: validate_against_service_schema(service_dict, self.service_name) - return process_container_options(self.working_dir, service_dict) + return service_dict def validate_and_construct_extends(self): extends = self.service_dict['extends'] @@ -281,19 +281,16 @@ class ServiceLoader(object): return config_path, service_config, service_name def resolve_extends(self, extended_config_path, service_config, service_name): - other_working_dir = os.path.dirname(extended_config_path) - other_already_seen = self.already_seen + [self.signature(self.service_name)] - - other_loader = ServiceLoader( - other_working_dir, + resolver = ServiceExtendsResolver( + os.path.dirname(extended_config_path), extended_config_path, self.service_name, service_config, - already_seen=other_already_seen, + already_seen=self.already_seen + [self.signature(self.service_name)], ) - other_loader.detect_cycle(service_name) - other_service_dict = other_loader.make_service_dict() + resolver.detect_cycle(service_name) + other_service_dict = process_service(resolver.working_dir, resolver.run()) validate_extended_service_dict( other_service_dict, extended_config_path, @@ -358,7 +355,7 @@ def validate_ulimits(ulimit_config): "than 'hard' value".format(ulimit_config)) -def process_container_options(working_dir, service_dict): +def process_service(working_dir, service_dict): service_dict = dict(service_dict) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 804f5219..d4474dcc 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -815,7 +815,13 @@ class ServiceTest(DockerClientTestCase): environment=['ONE=1', 'TWO=2', 'THREE=3'], env_file=['tests/fixtures/env/one.env', 'tests/fixtures/env/two.env']) env = create_and_start_container(service).environment - for k, v in {'ONE': '1', 'TWO': '2', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}.items(): + for k, v in { + 'ONE': '1', + 'TWO': '2', + 'THREE': '3', + 'FOO': 'baz', + 'DOO': 'dah' + }.items(): self.assertEqual(env[k], v) @mock.patch.dict(os.environ) @@ -823,9 +829,22 @@ class ServiceTest(DockerClientTestCase): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' os.environ['ENV_DEF'] = 'E3' - service = self.create_service('web', environment={'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': None, 'NO_DEF': None}) + service = self.create_service( + 'web', + environment={ + 'FILE_DEF': 'F1', + 'FILE_DEF_EMPTY': '', + 'ENV_DEF': None, + 'NO_DEF': None + } + ) env = create_and_start_container(service).environment - for k, v in {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}.items(): + for k, v in { + 'FILE_DEF': 'F1', + 'FILE_DEF_EMPTY': '', + 'ENV_DEF': 'E3', + 'NO_DEF': '' + }.items(): self.assertEqual(env[k], v) def test_with_high_enough_api_version_we_get_default_network_mode(self): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 60e67b5b..5ee6a421 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -7,7 +7,8 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client -from compose.config.config import ServiceLoader +from compose.config.config import process_service +from compose.config.config import resolve_environment from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -42,23 +43,15 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - workaround_options = {} - for option in ['links', 'volumes_from', 'net']: - if option in kwargs: - workaround_options[option] = kwargs.pop(option, None) - - options = ServiceLoader( - working_dir='.', - filename=None, - service_name=name, - service_dict=kwargs - ).make_service_dict() - options.update(workaround_options) + # TODO: remove this once #2299 is fixed + kwargs['name'] = name + options = process_service('.', kwargs) + options['environment'] = resolve_environment('.', kwargs) labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() - return Service(project='composetest', client=self.client, **options) + return Service(client=self.client, project='composetest', **options) def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ab34f4dc..2835a431 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -18,13 +18,14 @@ from tests import unittest def make_service_dict(name, service_dict, working_dir, filename=None): """ - Test helper function to construct a ServiceLoader + Test helper function to construct a ServiceExtendsResolver """ - return config.ServiceLoader( + resolver = config.ServiceExtendsResolver( working_dir=working_dir, filename=filename, service_name=name, - service_dict=service_dict).make_service_dict() + service_dict=service_dict) + return config.process_service(working_dir, resolver.run()) def service_sort(services): From 5e97b806d51e72f282046231af417b4d647cf64f Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 9 Nov 2015 14:24:44 -0500 Subject: [PATCH 099/359] Fix a bug in ExtendsResolver where the service name of the extended service was wrong. This bug can be seen by the change to the test case. When the extended service uses a different name, the error was reported incorrectly. By fixing this bug we can simplify self.signature and self.detect_cycles to always use self.service_name. Signed-off-by: Daniel Nephin --- compose/config/config.py | 20 +++++++++++--------- tests/fixtures/extends/circle-1.yml | 2 +- tests/fixtures/extends/circle-2.yml | 2 +- tests/unit/config/config_test.py | 23 ++++++++++++----------- 4 files changed, 25 insertions(+), 22 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7846ea7b..1ddb2abe 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -247,11 +247,17 @@ class ServiceExtendsResolver(object): self.service_name = service_name self.service_dict['name'] = service_name - def detect_cycle(self, name): - if self.signature(name) in self.already_seen: - raise CircularReference(self.already_seen + [self.signature(name)]) + @property + def signature(self): + return self.filename, self.service_name + + def detect_cycle(self): + if self.signature in self.already_seen: + raise CircularReference(self.already_seen + [self.signature]) def run(self): + self.detect_cycle() + service_dict = dict(self.service_dict) env = resolve_environment(self.working_dir, self.service_dict) if env: @@ -284,12 +290,11 @@ class ServiceExtendsResolver(object): resolver = ServiceExtendsResolver( os.path.dirname(extended_config_path), extended_config_path, - self.service_name, + service_name, service_config, - already_seen=self.already_seen + [self.signature(self.service_name)], + already_seen=self.already_seen + [self.signature], ) - resolver.detect_cycle(service_name) other_service_dict = process_service(resolver.working_dir, resolver.run()) validate_extended_service_dict( other_service_dict, @@ -309,9 +314,6 @@ class ServiceExtendsResolver(object): return expand_path(self.working_dir, extends_options['file']) return self.filename - def signature(self, name): - return self.filename, name - def resolve_environment(working_dir, service_dict): """Unpack any environment variables from an env_file, if set. diff --git a/tests/fixtures/extends/circle-1.yml b/tests/fixtures/extends/circle-1.yml index a034e961..d88ea61d 100644 --- a/tests/fixtures/extends/circle-1.yml +++ b/tests/fixtures/extends/circle-1.yml @@ -5,7 +5,7 @@ bar: web: extends: file: circle-2.yml - service: web + service: other baz: image: busybox quux: diff --git a/tests/fixtures/extends/circle-2.yml b/tests/fixtures/extends/circle-2.yml index fa6ddefc..de05bc8d 100644 --- a/tests/fixtures/extends/circle-2.yml +++ b/tests/fixtures/extends/circle-2.yml @@ -2,7 +2,7 @@ foo: image: busybox bar: image: busybox -web: +other: extends: file: circle-1.yml service: web diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2835a431..71783168 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1080,18 +1080,19 @@ class ExtendsTest(unittest.TestCase): ])) def test_circular(self): - try: + with pytest.raises(config.CircularReference) as exc: load_from_filename('tests/fixtures/extends/circle-1.yml') - raise Exception("Expected config.CircularReference to be raised") - except config.CircularReference as e: - self.assertEqual( - [(os.path.basename(filename), service_name) for (filename, service_name) in e.trail], - [ - ('circle-1.yml', 'web'), - ('circle-2.yml', 'web'), - ('circle-1.yml', 'web'), - ], - ) + + path = [ + (os.path.basename(filename), service_name) + for (filename, service_name) in exc.value.trail + ] + expected = [ + ('circle-1.yml', 'web'), + ('circle-2.yml', 'other'), + ('circle-1.yml', 'web'), + ] + self.assertEqual(path, expected) def test_extends_validation_empty_dictionary(self): with self.assertRaisesRegexp(ConfigurationError, 'service'): From 1f7faadc7712f5bf734e6254d2a8d6f1427f5029 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 6 Nov 2015 11:57:27 -0500 Subject: [PATCH 100/359] Remove name from config schema. Refactors config validation of a service to use a ServiceConfig data object. Instead of passing around a bunch of related scalars, we can use the ServiceConfig object as a parameter to most of the service validation functions. This allows for a fix to the config schema, where the name is a field in the schema, but not actually in the configuration. My passing the name around as part of the ServiceConfig object, we don't need to add it to the config options. Fixes #2299 validate_against_service_schema() is moved from a conditional branch in ServiceExtendsResolver() to happen as one of the last steps after all configuration is merged. This schema only contains constraints which only need to be true at the very end of merging. Signed-off-by: Daniel Nephin --- compose/config/config.py | 101 +++++++++++++++-------------- compose/config/fields_schema.json | 1 - compose/config/service_schema.json | 6 -- compose/config/validation.py | 2 +- tests/integration/testcases.py | 9 ++- tests/unit/config/config_test.py | 10 +-- 6 files changed, 62 insertions(+), 67 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 1ddb2abe..21788551 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -103,6 +103,20 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): return cls(filename, load_yaml(filename)) +class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')): + + @classmethod + def with_abs_paths(cls, working_dir, filename, name, config): + if not working_dir: + raise ValueError("No working_dir for ServiceConfig.") + + return cls( + os.path.abspath(working_dir), + os.path.abspath(filename) if filename else filename, + name, + config) + + def find(base_dir, filenames): if filenames == ['-']: return ConfigDetails( @@ -177,19 +191,22 @@ def load(config_details): """ def build_service(filename, service_name, service_dict): - resolver = ServiceExtendsResolver( + service_config = ServiceConfig.with_abs_paths( config_details.working_dir, filename, service_name, service_dict) - service_dict = process_service(config_details.working_dir, resolver.run()) + resolver = ServiceExtendsResolver(service_config) + service_dict = process_service(resolver.run()) + validate_against_service_schema(service_dict, service_config.name) validate_paths(service_dict) + service_dict['name'] = service_config.name return service_dict - def build_services(filename, config): + def build_services(config_file): return [ - build_service(filename, name, service_config) - for name, service_config in config.items() + build_service(config_file.filename, name, service_dict) + for name, service_dict in config_file.config.items() ] def merge_services(base, override): @@ -208,7 +225,7 @@ def load(config_details): config = merge_services(config_file.config, next_file.config) config_file = config_file._replace(config=config) - return build_services(config_file.filename, config_file.config) + return build_services(config_file) def process_config_file(config_file, service_name=None): @@ -225,31 +242,14 @@ def process_config_file(config_file, service_name=None): class ServiceExtendsResolver(object): - def __init__( - self, - working_dir, - filename, - service_name, - service_dict, - already_seen=None - ): - if working_dir is None: - raise ValueError("No working_dir passed to ServiceExtendsResolver()") - - self.working_dir = os.path.abspath(working_dir) - - if filename: - self.filename = os.path.abspath(filename) - else: - self.filename = filename + def __init__(self, service_config, already_seen=None): + self.service_config = service_config + self.working_dir = service_config.working_dir self.already_seen = already_seen or [] - self.service_dict = service_dict.copy() - self.service_name = service_name - self.service_dict['name'] = service_name @property def signature(self): - return self.filename, self.service_name + return self.service_config.filename, self.service_config.name def detect_cycle(self): if self.signature in self.already_seen: @@ -258,8 +258,8 @@ class ServiceExtendsResolver(object): def run(self): self.detect_cycle() - service_dict = dict(self.service_dict) - env = resolve_environment(self.working_dir, self.service_dict) + service_dict = dict(self.service_config.config) + env = resolve_environment(self.working_dir, self.service_config.config) if env: service_dict['environment'] = env service_dict.pop('env_file', None) @@ -267,13 +267,10 @@ class ServiceExtendsResolver(object): if 'extends' in service_dict: service_dict = self.resolve_extends(*self.validate_and_construct_extends()) - if not self.already_seen: - validate_against_service_schema(service_dict, self.service_name) - - return service_dict + return self.service_config._replace(config=service_dict) def validate_and_construct_extends(self): - extends = self.service_dict['extends'] + extends = self.service_config.config['extends'] if not isinstance(extends, dict): extends = {'service': extends} @@ -286,33 +283,38 @@ class ServiceExtendsResolver(object): service_config = extended_file.config[service_name] return config_path, service_config, service_name - def resolve_extends(self, extended_config_path, service_config, service_name): + def resolve_extends(self, extended_config_path, service_dict, service_name): resolver = ServiceExtendsResolver( - os.path.dirname(extended_config_path), - extended_config_path, - service_name, - service_config, - already_seen=self.already_seen + [self.signature], - ) + ServiceConfig.with_abs_paths( + os.path.dirname(extended_config_path), + extended_config_path, + service_name, + service_dict), + already_seen=self.already_seen + [self.signature]) - other_service_dict = process_service(resolver.working_dir, resolver.run()) + service_config = resolver.run() + other_service_dict = process_service(service_config) validate_extended_service_dict( other_service_dict, extended_config_path, service_name, ) - return merge_service_dicts(other_service_dict, self.service_dict) + return merge_service_dicts(other_service_dict, self.service_config.config) def get_extended_config_path(self, extends_options): """Service we are extending either has a value for 'file' set, which we need to obtain a full path too or we are extending from a service defined in our own file. """ - validate_extends_file_path(self.service_name, extends_options, self.filename) + filename = self.service_config.filename + validate_extends_file_path( + self.service_config.name, + extends_options, + filename) if 'file' in extends_options: return expand_path(self.working_dir, extends_options['file']) - return self.filename + return filename def resolve_environment(working_dir, service_dict): @@ -357,8 +359,9 @@ def validate_ulimits(ulimit_config): "than 'hard' value".format(ulimit_config)) -def process_service(working_dir, service_dict): - service_dict = dict(service_dict) +def process_service(service_config): + working_dir = service_config.working_dir + service_dict = dict(service_config.config) if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) @@ -505,12 +508,12 @@ def env_vars_from_file(filename): def resolve_volume_paths(working_dir, service_dict): return [ - resolve_volume_path(working_dir, volume, service_dict['name']) + resolve_volume_path(working_dir, volume) for volume in service_dict['volumes'] ] -def resolve_volume_path(working_dir, volume, service_name): +def resolve_volume_path(working_dir, volume): container_path, host_path = split_path_mapping(volume) if host_path is not None: diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index f22b513a..7723f2fb 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -89,7 +89,6 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, - "name": {"type": "string"}, "net": {"type": "string"}, "pid": {"type": ["string", "null"]}, diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json index 5cb5d6d0..221c5d8d 100644 --- a/compose/config/service_schema.json +++ b/compose/config/service_schema.json @@ -3,12 +3,6 @@ "type": "object", - "properties": { - "name": {"type": "string"} - }, - - "required": ["name"], - "allOf": [ {"$ref": "fields_schema.json#/definitions/service"}, {"$ref": "#/definitions/service_constraints"} diff --git a/compose/config/validation.py b/compose/config/validation.py index 3bd40410..d3bcb35c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -195,7 +195,7 @@ def process_errors(errors, service_name=None): for error in errors: # handle root level errors - if len(error.path) == 0 and not error.instance.get('name'): + if len(error.path) == 0 and not service_name: if error.validator == 'type': msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." root_msgs.append(msg) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 5ee6a421..d63f0591 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -9,6 +9,7 @@ from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import process_service from compose.config.config import resolve_environment +from compose.config.config import ServiceConfig from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -43,15 +44,13 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - # TODO: remove this once #2299 is fixed - kwargs['name'] = name - - options = process_service('.', kwargs) + service_config = ServiceConfig('.', None, name, kwargs) + options = process_service(service_config) options['environment'] = resolve_environment('.', kwargs) labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() - return Service(client=self.client, project='composetest', **options) + return Service(name, client=self.client, project='composetest', **options) def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 71783168..022ec7c7 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -20,12 +20,12 @@ def make_service_dict(name, service_dict, working_dir, filename=None): """ Test helper function to construct a ServiceExtendsResolver """ - resolver = config.ServiceExtendsResolver( + resolver = config.ServiceExtendsResolver(config.ServiceConfig( working_dir=working_dir, filename=filename, - service_name=name, - service_dict=service_dict) - return config.process_service(working_dir, resolver.run()) + name=name, + config=service_dict)) + return config.process_service(resolver.run()) def service_sort(services): @@ -651,7 +651,7 @@ class VolumeConfigTest(unittest.TestCase): def test_volume_path_with_non_ascii_directory(self): volume = u'/Füü/data:/data' - container_path = config.resolve_volume_path(".", volume, "test") + container_path = config.resolve_volume_path(".", volume) self.assertEqual(container_path, volume) From 19b2c41c7ee7887468d04ceb1fc90f05b232432a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 9 Nov 2015 15:39:55 -0500 Subject: [PATCH 101/359] Add a test for invalid field 'name', and fix an existing test for invalid service names. Signed-off-by: Daniel Nephin --- tests/unit/config/config_test.py | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 022ec7c7..add7a5a4 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -77,8 +77,8 @@ class ConfigTest(unittest.TestCase): ) def test_config_invalid_service_names(self): - with self.assertRaises(ConfigurationError): - for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: + for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: + with pytest.raises(ConfigurationError): config.load( build_config_details( {invalid_name: {'image': 'busybox'}}, @@ -87,6 +87,16 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_with_invalid_field_name(self): + config_details = build_config_details( + {'web': {'image': 'busybox', 'name': 'bogus'}}, + 'working_dir', + 'filename.yml') + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + error_msg = "Unsupported config option for 'web' service: 'name'" + assert error_msg in exc.exconly() + def test_config_integer_service_name_raise_validation_error(self): expected_error_msg = "Service name: 1 needs to be a string, eg '1'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): From b8b4c84573ec7d20524b7d2715d956fcc9f897a8 Mon Sep 17 00:00:00 2001 From: Yves Peter Date: Wed, 4 Nov 2015 23:40:57 +0100 Subject: [PATCH 102/359] Fixes #1490 progress_stream would print a lot of new lines on "docker-compose pull" if there's no tty. Signed-off-by: Yves Peter --- compose/progress_stream.py | 23 +++++++++++++--------- tests/unit/progress_stream_test.py | 31 ++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 9 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index ac8e4b41..c729a6df 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -14,8 +14,14 @@ def stream_output(output, stream): for event in utils.json_stream(output): all_events.append(event) + is_progress_event = 'progress' in event or 'progressDetail' in event - if 'progress' in event or 'progressDetail' in event: + if not is_progress_event: + print_output_event(event, stream, is_terminal) + stream.flush() + + # if it's a progress event and we have a terminal, then display the progress bars + elif is_terminal: image_id = event.get('id') if not image_id: continue @@ -27,17 +33,16 @@ def stream_output(output, stream): stream.write("\n") diff = 0 - if is_terminal: - # move cursor up `diff` rows - stream.write("%c[%dA" % (27, diff)) + # move cursor up `diff` rows + stream.write("%c[%dA" % (27, diff)) - print_output_event(event, stream, is_terminal) + print_output_event(event, stream, is_terminal) - if 'id' in event and is_terminal: - # move cursor back down - stream.write("%c[%dB" % (27, diff)) + if 'id' in event: + # move cursor back down + stream.write("%c[%dB" % (27, diff)) - stream.flush() + stream.flush() return all_events diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index d8f7ec83..b01be11a 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -34,3 +34,34 @@ class ProgressStreamTestCase(unittest.TestCase): ] events = progress_stream.stream_output(output, StringIO()) self.assertEqual(len(events), 1) + + def test_stream_output_progress_event_tty(self): + events = [ + b'{"status": "Already exists", "progressDetail": {}, "id": "8d05e3af52b0"}' + ] + + class TTYStringIO(StringIO): + def isatty(self): + return True + + output = TTYStringIO() + events = progress_stream.stream_output(events, output) + self.assertTrue(len(output.getvalue()) > 0) + + def test_stream_output_progress_event_no_tty(self): + events = [ + b'{"status": "Already exists", "progressDetail": {}, "id": "8d05e3af52b0"}' + ] + output = StringIO() + + events = progress_stream.stream_output(events, output) + self.assertEqual(len(output.getvalue()), 0) + + def test_stream_output_no_progress_event_no_tty(self): + events = [ + b'{"status": "Pulling from library/xy", "id": "latest"}' + ] + output = StringIO() + + events = progress_stream.stream_output(events, output) + self.assertTrue(len(output.getvalue()) > 0) From c573fcc70a297dbc8d971ad13a81bbaa88262dec Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Nov 2015 11:21:24 -0800 Subject: [PATCH 103/359] Reorganize conditional branches to improve readability Signed-off-by: Joffrey F --- compose/progress_stream.py | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/compose/progress_stream.py b/compose/progress_stream.py index c729a6df..a6c8e0a2 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -19,30 +19,33 @@ def stream_output(output, stream): if not is_progress_event: print_output_event(event, stream, is_terminal) stream.flush() + continue + + if not is_terminal: + continue # if it's a progress event and we have a terminal, then display the progress bars - elif is_terminal: - image_id = event.get('id') - if not image_id: - continue + image_id = event.get('id') + if not image_id: + continue - if image_id in lines: - diff = len(lines) - lines[image_id] - else: - lines[image_id] = len(lines) - stream.write("\n") - diff = 0 + if image_id in lines: + diff = len(lines) - lines[image_id] + else: + lines[image_id] = len(lines) + stream.write("\n") + diff = 0 - # move cursor up `diff` rows - stream.write("%c[%dA" % (27, diff)) + # move cursor up `diff` rows + stream.write("%c[%dA" % (27, diff)) - print_output_event(event, stream, is_terminal) + print_output_event(event, stream, is_terminal) - if 'id' in event: - # move cursor back down - stream.write("%c[%dB" % (27, diff)) + if 'id' in event: + # move cursor back down + stream.write("%c[%dB" % (27, diff)) - stream.flush() + stream.flush() return all_events From 513dfda218a96f89de69788db9fb39f007656356 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Nov 2015 12:45:02 -0800 Subject: [PATCH 104/359] Allow dashes in environment variable names See http://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html Environment variable names used by the utilities in the Shell and Utilities volume of POSIX.1-2008 consist solely of uppercase letters, digits, and the ( '_' ) from the characters defined in Portable Character Set and do not begin with a digit. Other characters may be permitted by an implementation; applications shall tolerate the presence of such names. Signed-off-by: Joffrey F --- compose/config/fields_schema.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index f22b513a..454020a8 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -40,7 +40,7 @@ { "type": "object", "patternProperties": { - "^[^-]+$": { + ".+": { "type": ["string", "number", "boolean", "null"], "format": "environment" } From d6b44905f25d8d03a5057c942bc9a955da40be23 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 11 Nov 2015 12:52:30 -0800 Subject: [PATCH 105/359] Add test for environment variable dashes support Signed-off-by: Joffrey F --- tests/unit/config/config_test.py | 24 +++++++++++------------- 1 file changed, 11 insertions(+), 13 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 71783168..3adc02c8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -470,20 +470,18 @@ class ConfigTest(unittest.TestCase): self.assertTrue(mock_logging.warn.called) self.assertTrue(expected_warning_msg in mock_logging.warn.call_args[0][0]) - def test_config_invalid_environment_dict_key_raises_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'environment' contains unsupported option: '---'" - - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - config.load( - build_config_details( - {'web': { - 'image': 'busybox', - 'environment': {'---': 'nope'} - }}, - 'working_dir', - 'filename.yml' - ) + def test_config_valid_environment_dict_key_contains_dashes(self): + services = config.load( + build_config_details( + {'web': { + 'image': 'busybox', + 'environment': {'SPRING_JPA_HIBERNATE_DDL-AUTO': 'none'} + }}, + 'working_dir', + 'filename.yml' ) + ) + self.assertEqual(services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'], 'none') def test_load_yaml_with_yaml_error(self): tmpdir = py.test.ensuretemp('invalid_yaml_test') From f7d808769405e52e4c84acdcf15875614cb71993 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 Nov 2015 12:40:36 -0500 Subject: [PATCH 106/359] Add ids to config schemas Also enforce a max complexity for functions and add some new tests for config. Signed-off-by: Daniel Nephin --- compose/config/fields_schema.json | 6 ++++-- compose/config/service_schema.json | 11 ++++------- compose/config/validation.py | 5 ++++- tests/unit/config/config_test.py | 24 ++++++++++++++++-------- tox.ini | 2 ++ 5 files changed, 30 insertions(+), 18 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 7723f2fb..a174ba87 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -2,15 +2,18 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", + "id": "fields_schema.json", "patternProperties": { "^[a-zA-Z0-9._-]+$": { "$ref": "#/definitions/service" } }, + "additionalProperties": false, "definitions": { "service": { + "id": "#/definitions/service", "type": "object", "properties": { @@ -167,6 +170,5 @@ ] } - }, - "additionalProperties": false + } } diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json index 221c5d8d..05774efd 100644 --- a/compose/config/service_schema.json +++ b/compose/config/service_schema.json @@ -1,15 +1,17 @@ { "$schema": "http://json-schema.org/draft-04/schema#", + "id": "service_schema.json", "type": "object", "allOf": [ {"$ref": "fields_schema.json#/definitions/service"}, - {"$ref": "#/definitions/service_constraints"} + {"$ref": "#/definitions/constraints"} ], "definitions": { - "service_constraints": { + "constraints": { + "id": "#/definitions/constraints", "anyOf": [ { "required": ["build"], @@ -21,13 +23,8 @@ {"required": ["build"]}, {"required": ["dockerfile"]} ]} - }, - { - "required": ["extends"], - "not": {"required": ["build", "image"]} } ] } } - } diff --git a/compose/config/validation.py b/compose/config/validation.py index d3bcb35c..962d41e2 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -307,7 +307,10 @@ def _validate_against_schema(config, schema_filename, format_checker=[], service schema = json.load(schema_fh) resolver = RefResolver(resolver_full_path, schema) - validation_output = Draft4Validator(schema, resolver=resolver, format_checker=FormatChecker(format_checker)) + validation_output = Draft4Validator( + schema, + resolver=resolver, + format_checker=FormatChecker(format_checker)) errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] if errors: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index add7a5a4..03e338cb 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -78,14 +78,12 @@ class ConfigTest(unittest.TestCase): def test_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: - with pytest.raises(ConfigurationError): - config.load( - build_config_details( - {invalid_name: {'image': 'busybox'}}, - 'working_dir', - 'filename.yml' - ) - ) + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + {invalid_name: {'image': 'busybox'}}, + 'working_dir', + 'filename.yml')) + assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() def test_load_with_invalid_field_name(self): config_details = build_config_details( @@ -97,6 +95,16 @@ class ConfigTest(unittest.TestCase): error_msg = "Unsupported config option for 'web' service: 'name'" assert error_msg in exc.exconly() + def test_load_invalid_service_definition(self): + config_details = build_config_details( + {'web': 'wrong'}, + 'working_dir', + 'filename.yml') + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + error_msg = "Service \"web\" doesn\'t have any configuration options" + assert error_msg in exc.exconly() + def test_config_integer_service_name_raise_validation_error(self): expected_error_msg = "Service name: 1 needs to be a string, eg '1'" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): diff --git a/tox.ini b/tox.ini index f05c5ed2..d1098a55 100644 --- a/tox.ini +++ b/tox.ini @@ -43,4 +43,6 @@ directory = coverage-html [flake8] # Allow really long lines for now max-line-length = 140 +# Set this high for now +max-complexity = 20 exclude = compose/packages From fa96484d2835b8711e560d0c22626c67b99b2407 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 Nov 2015 12:43:29 -0500 Subject: [PATCH 107/359] Refactor process_errors into smaller functions So that it passed new max-complexity requirement Signed-off-by: Daniel Nephin --- compose/config/validation.py | 316 +++++++++++++++---------------- tests/unit/config/config_test.py | 2 +- 2 files changed, 149 insertions(+), 169 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 962d41e2..2928238c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -109,189 +109,169 @@ def anglicize_validator(validator): return 'a ' + validator -def process_errors(errors, service_name=None): +def handle_error_for_schema_with_id(error, service_name): + schema_id = error.schema['id'] + + if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties': + return "Invalid service name '{}' - only {} characters are allowed".format( + # The service_name is the key to the json object + list(error.instance)[0], + VALID_NAME_CHARS) + + if schema_id == '#/definitions/constraints': + if 'image' in error.instance and 'build' in error.instance: + return ( + "Service '{}' has both an image and build path specified. " + "A service can either be built to image or use an existing " + "image, not both.".format(service_name)) + if 'image' not in error.instance and 'build' not in error.instance: + return ( + "Service '{}' has neither an image nor a build path " + "specified. Exactly one must be provided.".format(service_name)) + if 'image' in error.instance and 'dockerfile' in error.instance: + return ( + "Service '{}' has both an image and alternate Dockerfile. " + "A service can either be built to image or use an existing " + "image, not both.".format(service_name)) + + if schema_id == '#/definitions/service': + if error.validator == 'additionalProperties': + invalid_config_key = parse_key_from_error_msg(error) + return get_unsupported_config_msg(service_name, invalid_config_key) + + +def handle_generic_service_error(error, service_name): + config_key = " ".join("'%s'" % k for k in error.path) + msg_format = None + error_msg = error.message + + if error.validator == 'oneOf': + msg_format = "Service '{}' configuration key {} {}" + error_msg = _parse_oneof_validator(error) + + elif error.validator == 'type': + msg_format = ("Service '{}' configuration key {} contains an invalid " + "type, it should be {}") + error_msg = _parse_valid_types_from_validator(error.validator_value) + + # TODO: no test case for this branch, there are no config options + # which exercise this branch + elif error.validator == 'required': + msg_format = "Service '{}' configuration key '{}' is invalid, {}" + + elif error.validator == 'dependencies': + msg_format = "Service '{}' configuration key '{}' is invalid: {}" + config_key = list(error.validator_value.keys())[0] + required_keys = ",".join(error.validator_value[config_key]) + error_msg = "when defining '{}' you must set '{}' as well".format( + config_key, + required_keys) + + elif error.path: + msg_format = "Service '{}' configuration key {} value {}" + + if msg_format: + return msg_format.format(service_name, config_key, error_msg) + + return error.message + + +def parse_key_from_error_msg(error): + return error.message.split("'")[1] + + +def _parse_valid_types_from_validator(validator): + """A validator value can be either an array of valid types or a string of + a valid type. Parse the valid types and prefix with the correct article. """ - jsonschema gives us an error tree full of information to explain what has + if not isinstance(validator, list): + return anglicize_validator(validator) + + if len(validator) == 1: + return anglicize_validator(validator[0]) + + return "{}, or {}".format( + ", ".join([anglicize_validator(validator[0])] + validator[1:-1]), + anglicize_validator(validator[-1])) + + +def _parse_oneof_validator(error): + """oneOf has multiple schemas, so we need to reason about which schema, sub + schema or constraint the validation is failing on. + Inspecting the context value of a ValidationError gives us information about + which sub schema failed and which kind of error it is. + """ + types = [] + for context in error.context: + + if context.validator == 'required': + return context.message + + if context.validator == 'additionalProperties': + invalid_config_key = parse_key_from_error_msg(context) + return "contains unsupported option: '{}'".format(invalid_config_key) + + if context.path: + invalid_config_key = " ".join( + "'{}' ".format(fragment) for fragment in context.path + if isinstance(fragment, six.string_types) + ) + return "{}contains {}, which is an invalid type, it should be {}".format( + invalid_config_key, + context.instance, + _parse_valid_types_from_validator(context.validator_value)) + + if context.validator == 'uniqueItems': + return "contains non unique items, please remove duplicates from {}".format( + context.instance) + + if context.validator == 'type': + types.append(context.validator_value) + + valid_types = _parse_valid_types_from_validator(types) + return "contains an invalid type, it should be {}".format(valid_types) + + +def process_errors(errors, service_name=None): + """jsonschema gives us an error tree full of information to explain what has gone wrong. Process each error and pull out relevant information and re-write helpful error messages that are relevant. """ - def _parse_key_from_error_msg(error): - return error.message.split("'")[1] + def format_error_message(error, service_name): + if not service_name and error.path: + # field_schema errors will have service name on the path + service_name = error.path.popleft() - def _clean_error_message(message): - return message.replace("u'", "'") + if 'id' in error.schema: + error_msg = handle_error_for_schema_with_id(error, service_name) + if error_msg: + return error_msg - def _parse_valid_types_from_validator(validator): - """ - A validator value can be either an array of valid types or a string of - a valid type. Parse the valid types and prefix with the correct article. - """ - if isinstance(validator, list): - if len(validator) >= 2: - first_type = anglicize_validator(validator[0]) - last_type = anglicize_validator(validator[-1]) - types_from_validator = ", ".join([first_type] + validator[1:-1]) + return handle_generic_service_error(error, service_name) - msg = "{} or {}".format( - types_from_validator, - last_type - ) - else: - msg = "{}".format(anglicize_validator(validator[0])) - else: - msg = "{}".format(anglicize_validator(validator)) - - return msg - - def _parse_oneof_validator(error): - """ - oneOf has multiple schemas, so we need to reason about which schema, sub - schema or constraint the validation is failing on. - Inspecting the context value of a ValidationError gives us information about - which sub schema failed and which kind of error it is. - """ - required = [context for context in error.context if context.validator == 'required'] - if required: - return required[0].message - - additionalProperties = [context for context in error.context if context.validator == 'additionalProperties'] - if additionalProperties: - invalid_config_key = _parse_key_from_error_msg(additionalProperties[0]) - return "contains unsupported option: '{}'".format(invalid_config_key) - - constraint = [context for context in error.context if len(context.path) > 0] - if constraint: - valid_types = _parse_valid_types_from_validator(constraint[0].validator_value) - invalid_config_key = "".join( - "'{}' ".format(fragment) for fragment in constraint[0].path - if isinstance(fragment, six.string_types) - ) - msg = "{}contains {}, which is an invalid type, it should be {}".format( - invalid_config_key, - constraint[0].instance, - valid_types - ) - return msg - - uniqueness = [context for context in error.context if context.validator == 'uniqueItems'] - if uniqueness: - msg = "contains non unique items, please remove duplicates from {}".format( - uniqueness[0].instance - ) - return msg - - types = [context.validator_value for context in error.context if context.validator == 'type'] - valid_types = _parse_valid_types_from_validator(types) - - msg = "contains an invalid type, it should be {}".format(valid_types) - - return msg - - root_msgs = [] - invalid_keys = [] - required = [] - type_errors = [] - other_errors = [] - - for error in errors: - # handle root level errors - if len(error.path) == 0 and not service_name: - if error.validator == 'type': - msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." - root_msgs.append(msg) - elif error.validator == 'additionalProperties': - invalid_service_name = _parse_key_from_error_msg(error) - msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS) - root_msgs.append(msg) - else: - root_msgs.append(_clean_error_message(error.message)) - - else: - if not service_name: - # field_schema errors will have service name on the path - service_name = error.path[0] - error.path.popleft() - else: - # service_schema errors have the service name passed in, as that - # is not available on error.path or necessarily error.instance - service_name = service_name - - if error.validator == 'additionalProperties': - invalid_config_key = _parse_key_from_error_msg(error) - invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key)) - elif error.validator == 'anyOf': - if 'image' in error.instance and 'build' in error.instance: - required.append( - "Service '{}' has both an image and build path specified. " - "A service can either be built to image or use an existing " - "image, not both.".format(service_name)) - elif 'image' not in error.instance and 'build' not in error.instance: - required.append( - "Service '{}' has neither an image nor a build path " - "specified. Exactly one must be provided.".format(service_name)) - elif 'image' in error.instance and 'dockerfile' in error.instance: - required.append( - "Service '{}' has both an image and alternate Dockerfile. " - "A service can either be built to image or use an existing " - "image, not both.".format(service_name)) - else: - required.append(_clean_error_message(error.message)) - elif error.validator == 'oneOf': - config_key = error.path[0] - msg = _parse_oneof_validator(error) - - type_errors.append("Service '{}' configuration key '{}' {}".format( - service_name, config_key, msg) - ) - elif error.validator == 'type': - msg = _parse_valid_types_from_validator(error.validator_value) - - if len(error.path) > 0: - config_key = " ".join(["'%s'" % k for k in error.path]) - type_errors.append( - "Service '{}' configuration key {} contains an invalid " - "type, it should be {}".format( - service_name, - config_key, - msg)) - else: - root_msgs.append( - "Service \"{}\" doesn't have any configuration options. " - "All top level keys in your docker-compose.yml must map " - "to a dictionary of configuration options.'".format(service_name)) - elif error.validator == 'required': - config_key = error.path[0] - required.append( - "Service '{}' option '{}' is invalid, {}".format( - service_name, - config_key, - _clean_error_message(error.message))) - elif error.validator == 'dependencies': - dependency_key = list(error.validator_value.keys())[0] - required_keys = ",".join(error.validator_value[dependency_key]) - required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format( - dependency_key, service_name, dependency_key, required_keys)) - else: - config_key = " ".join(["'%s'" % k for k in error.path]) - err_msg = "Service '{}' configuration key {} value {}".format(service_name, config_key, error.message) - other_errors.append(err_msg) - - return "\n".join(root_msgs + invalid_keys + required + type_errors + other_errors) + return '\n'.join(format_error_message(error, service_name) for error in errors) def validate_against_fields_schema(config): - schema_filename = "fields_schema.json" - format_checkers = ["ports", "environment"] - return _validate_against_schema(config, schema_filename, format_checkers) + return _validate_against_schema( + config, + "fields_schema.json", + ["ports", "environment"]) def validate_against_service_schema(config, service_name): - schema_filename = "service_schema.json" - format_checkers = ["ports"] - return _validate_against_schema(config, schema_filename, format_checkers, service_name) + return _validate_against_schema( + config, + "service_schema.json", + ["ports"], + service_name) -def _validate_against_schema(config, schema_filename, format_checker=[], service_name=None): +def _validate_against_schema( + config, + schema_filename, + format_checker=(), + service_name=None): config_source_dir = os.path.dirname(os.path.abspath(__file__)) if sys.platform == "win32": diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 03e338cb..9abc58e4 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -867,7 +867,7 @@ class MemoryOptionsTest(unittest.TestCase): a mem_limit """ expected_error_msg = ( - "Invalid 'memswap_limit' configuration for 'foo' service: when " + "Service 'foo' configuration key 'memswap_limit' is invalid: when " "defining 'memswap_limit' you must set 'mem_limit' as well" ) with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): From 589755d03448b04dff441895949116d647b64bc1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 9 Nov 2015 20:01:20 -0500 Subject: [PATCH 108/359] Inclide the filename in validation errors. Signed-off-by: Daniel Nephin --- compose/config/config.py | 4 +- compose/config/interpolation.py | 7 --- compose/config/validation.py | 60 ++++++++++++------ tests/unit/config/config_test.py | 104 +++++++++++++++---------------- 4 files changed, 95 insertions(+), 80 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 21788551..2c1fdeb9 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -229,9 +229,9 @@ def load(config_details): def process_config_file(config_file, service_name=None): - validate_top_level_object(config_file.config) + validate_top_level_object(config_file) processed_config = interpolate_environment_variables(config_file.config) - validate_against_fields_schema(processed_config) + validate_against_fields_schema(processed_config, config_file.filename) if service_name and service_name not in processed_config: raise ConfigurationError( diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index f8e1da61..ba7e35c1 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -18,13 +18,6 @@ def interpolate_environment_variables(config): def process_service(service_name, service_dict, mapping): - if not isinstance(service_dict, dict): - raise ConfigurationError( - 'Service "%s" doesn\'t have any configuration options. ' - 'All top level keys in your docker-compose.yml must map ' - 'to a dictionary of configuration options.' % service_name - ) - return dict( (key, interpolate_value(service_name, key, val, mapping)) for (key, val) in service_dict.items() diff --git a/compose/config/validation.py b/compose/config/validation.py index 2928238c..38866b0f 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -66,21 +66,38 @@ def format_boolean_in_environment(instance): return True -def validate_service_names(config): - for service_name in config.keys(): +def validate_top_level_service_objects(config_file): + """Perform some high level validation of the service name and value. + + This validation must happen before interpolation, which must happen + before the rest of validation, which is why it's separate from the + rest of the service validation. + """ + for service_name, service_dict in config_file.config.items(): if not isinstance(service_name, six.string_types): raise ConfigurationError( - "Service name: {} needs to be a string, eg '{}'".format( + "In file '{}' service name: {} needs to be a string, eg '{}'".format( + config_file.filename, service_name, service_name)) + if not isinstance(service_dict, dict): + raise ConfigurationError( + "In file '{}' service '{}' doesn\'t have any configuration options. " + "All top level keys in your docker-compose.yml must map " + "to a dictionary of configuration options.".format( + config_file.filename, + service_name)) -def validate_top_level_object(config): - if not isinstance(config, dict): + +def validate_top_level_object(config_file): + if not isinstance(config_file.config, dict): raise ConfigurationError( - "Top level object needs to be a dictionary. Check your .yml file " - "that you have defined a service at the top level.") - validate_service_names(config) + "Top level object in '{}' needs to be an object not '{}'. Check " + "that you have defined a service at the top level.".format( + config_file.filename, + type(config_file.config))) + validate_top_level_service_objects(config_file) def validate_extends_file_path(service_name, extends_options, filename): @@ -252,26 +269,28 @@ def process_errors(errors, service_name=None): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config): - return _validate_against_schema( +def validate_against_fields_schema(config, filename): + _validate_against_schema( config, "fields_schema.json", - ["ports", "environment"]) + format_checker=["ports", "environment"], + filename=filename) def validate_against_service_schema(config, service_name): - return _validate_against_schema( + _validate_against_schema( config, "service_schema.json", - ["ports"], - service_name) + format_checker=["ports"], + service_name=service_name) def _validate_against_schema( config, schema_filename, format_checker=(), - service_name=None): + service_name=None, + filename=None): config_source_dir = os.path.dirname(os.path.abspath(__file__)) if sys.platform == "win32": @@ -293,6 +312,11 @@ def _validate_against_schema( format_checker=FormatChecker(format_checker)) errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] - if errors: - error_msg = process_errors(errors, service_name) - raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg)) + if not errors: + return + + error_msg = process_errors(errors, service_name) + file_msg = " in file '{}'".format(filename) if filename else '' + raise ConfigurationError("Validation failed{}, reason(s):\n{}".format( + file_msg, + error_msg)) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 9abc58e4..84ed4943 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -94,6 +94,7 @@ class ConfigTest(unittest.TestCase): config.load(config_details) error_msg = "Unsupported config option for 'web' service: 'name'" assert error_msg in exc.exconly() + assert "Validation failed in file 'filename.yml'" in exc.exconly() def test_load_invalid_service_definition(self): config_details = build_config_details( @@ -102,11 +103,12 @@ class ConfigTest(unittest.TestCase): 'filename.yml') with pytest.raises(ConfigurationError) as exc: config.load(config_details) - error_msg = "Service \"web\" doesn\'t have any configuration options" + error_msg = "service 'web' doesn't have any configuration options" assert error_msg in exc.exconly() def test_config_integer_service_name_raise_validation_error(self): - expected_error_msg = "Service name: 1 needs to be a string, eg '1'" + expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " + "be a string, eg '1'") with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( build_config_details( @@ -156,25 +158,26 @@ class ConfigTest(unittest.TestCase): def test_load_with_multiple_files_and_empty_override(self): base_file = config.ConfigFile( - 'base.yaml', + 'base.yml', {'web': {'image': 'example/web'}}) - override_file = config.ConfigFile('override.yaml', None) + override_file = config.ConfigFile('override.yml', None) details = config.ConfigDetails('.', [base_file, override_file]) with pytest.raises(ConfigurationError) as exc: config.load(details) - assert 'Top level object needs to be a dictionary' in exc.exconly() + error_msg = "Top level object in 'override.yml' needs to be an object" + assert error_msg in exc.exconly() def test_load_with_multiple_files_and_empty_base(self): - base_file = config.ConfigFile('base.yaml', None) + base_file = config.ConfigFile('base.yml', None) override_file = config.ConfigFile( - 'override.yaml', + 'override.yml', {'web': {'image': 'example/web'}}) details = config.ConfigDetails('.', [base_file, override_file]) with pytest.raises(ConfigurationError) as exc: config.load(details) - assert 'Top level object needs to be a dictionary' in exc.exconly() + assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() def test_load_with_multiple_files_and_extends_in_override_file(self): base_file = config.ConfigFile( @@ -225,17 +228,17 @@ class ConfigTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: config.load(details) - assert 'Service "bogus" doesn\'t have any configuration' in exc.exconly() + assert "service 'bogus' doesn't have any configuration" in exc.exconly() + assert "In file 'override.yaml'" in exc.exconly() def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: - config.load( + services = config.load( build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', - 'common.yml' - ) - ) + 'common.yml')) + assert services[0]['name'] == valid_name def test_config_invalid_ports_format_validation(self): expected_error_msg = "Service 'web' configuration key 'ports' contains an invalid type" @@ -300,7 +303,8 @@ class ConfigTest(unittest.TestCase): ) def test_invalid_config_not_a_dictionary(self): - expected_error_msg = "Top level object needs to be a dictionary." + expected_error_msg = ("Top level object in 'filename.yml' needs to be " + "an object.") with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( build_config_details( @@ -382,12 +386,13 @@ class ConfigTest(unittest.TestCase): ) def test_config_ulimits_invalid_keys_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'ulimits' contains unsupported option: 'not_soft_or_hard'" + expected = ("Service 'web' configuration key 'ulimits' 'nofile' contains " + "unsupported option: 'not_soft_or_hard'") - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - config.load( - build_config_details( - {'web': { + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + { + 'web': { 'image': 'busybox', 'ulimits': { 'nofile': { @@ -396,50 +401,43 @@ class ConfigTest(unittest.TestCase): "hard": 20000, } } - }}, - 'working_dir', - 'filename.yml' - ) - ) + } + }, + 'working_dir', + 'filename.yml')) + assert expected in exc.exconly() def test_config_ulimits_required_keys_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'ulimits' u?'hard' is a required property" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - config.load( - build_config_details( - {'web': { + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + { + 'web': { 'image': 'busybox', - 'ulimits': { - 'nofile': { - "soft": 10000, - } - } - }}, - 'working_dir', - 'filename.yml' - ) - ) + 'ulimits': {'nofile': {"soft": 10000}} + } + }, + 'working_dir', + 'filename.yml')) + assert "Service 'web' configuration key 'ulimits' 'nofile'" in exc.exconly() + assert "'hard' is a required property" in exc.exconly() def test_config_ulimits_soft_greater_than_hard_error(self): - expected_error_msg = "cannot contain a 'soft' value higher than 'hard' value" + expected = "cannot contain a 'soft' value higher than 'hard' value" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - config.load( - build_config_details( - {'web': { + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + { + 'web': { 'image': 'busybox', 'ulimits': { - 'nofile': { - "soft": 10000, - "hard": 1000 - } + 'nofile': {"soft": 10000, "hard": 1000} } - }}, - 'working_dir', - 'filename.yml' - ) - ) + } + }, + 'working_dir', + 'filename.yml')) + assert expected in exc.exconly() def test_valid_config_which_allows_two_type_definitions(self): expose_values = [["8000"], [8000]] From 9c305ac10f6c6900e432602ccefa5e0bdd83f9e1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 Nov 2015 13:26:13 -0500 Subject: [PATCH 109/359] Remove name field from the list of ALLOWED_KEYS Signed-off-by: Daniel Nephin --- compose/config/config.py | 1 - 1 file changed, 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index 2c1fdeb9..20126620 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -65,7 +65,6 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'dockerfile', 'expose', 'external_links', - 'name', ] From ea4230e7a2f53a116c22dce20632cb5355cf4c07 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 10 Nov 2015 18:52:21 -0500 Subject: [PATCH 110/359] Handle both SIGINT and SIGTERM for docker-compose up. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 21 +++++++---- tests/acceptance/cli_test.py | 70 +++++++++++++++++++++++++++++------- tests/unit/cli/main_test.py | 8 ++--- 3 files changed, 76 insertions(+), 23 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 806926d8..7b1e0aa3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -658,17 +658,24 @@ def build_log_printer(containers, service_names, monochrome): def attach_to_logs(project, log_printer, service_names, timeout): print("Attaching to", list_containers(log_printer.containers)) - try: - log_printer.run() - finally: - def handler(signal, frame): - project.kill(service_names=service_names) - sys.exit(0) - signal.signal(signal.SIGINT, handler) + def force_shutdown(signal, frame): + project.kill(service_names=service_names) + sys.exit(2) + + def shutdown(signal, frame): + set_signal_handler(force_shutdown) print("Gracefully stopping... (press Ctrl+C again to force)") project.stop(service_names=service_names, timeout=timeout) + set_signal_handler(shutdown) + log_printer.run() + + +def set_signal_handler(handler): + signal.signal(signal.SIGINT, handler) + signal.signal(signal.SIGTERM, handler) + def list_containers(containers): return ", ".join(c.name for c in containers) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 88a43d7f..57f2039e 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -2,7 +2,9 @@ from __future__ import absolute_import import os import shlex +import signal import subprocess +import time from collections import namedtuple from operator import attrgetter @@ -20,6 +22,45 @@ BUILD_CACHE_TEXT = 'Using cache' BUILD_PULL_TEXT = 'Status: Image is up to date for busybox:latest' +def start_process(base_dir, options): + proc = subprocess.Popen( + ['docker-compose'] + options, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + cwd=base_dir) + print("Running process: %s" % proc.pid) + return proc + + +def wait_on_process(proc, returncode=0): + stdout, stderr = proc.communicate() + if proc.returncode != returncode: + print(stderr) + assert proc.returncode == returncode + return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) + + +def wait_on_condition(condition, delay=0.1, timeout=5): + start_time = time.time() + while not condition(): + if time.time() - start_time > timeout: + raise AssertionError("Timeout: %s" % condition) + time.sleep(delay) + + +class ContainerCountCondition(object): + + def __init__(self, project, expected): + self.project = project + self.expected = expected + + def __call__(self): + return len(self.project.containers()) == self.expected + + def __str__(self): + return "waiting for counter count == %s" % self.expected + + class CLITestCase(DockerClientTestCase): def setUp(self): @@ -42,17 +83,8 @@ class CLITestCase(DockerClientTestCase): def dispatch(self, options, project_options=None, returncode=0): project_options = project_options or [] - proc = subprocess.Popen( - ['docker-compose'] + project_options + options, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - cwd=self.base_dir) - print("Running process: %s" % proc.pid) - stdout, stderr = proc.communicate() - if proc.returncode != returncode: - print(stderr) - assert proc.returncode == returncode - return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) + proc = start_process(self.base_dir, project_options + options) + return wait_on_process(proc, returncode=returncode) def test_help(self): old_base_dir = self.base_dir @@ -291,7 +323,7 @@ class CLITestCase(DockerClientTestCase): returncode=1) def test_up_with_timeout(self): - self.dispatch(['up', '-d', '-t', '1'], None) + self.dispatch(['up', '-d', '-t', '1']) service = self.project.get_service('simple') another = self.project.get_service('another') self.assertEqual(len(service.containers()), 1) @@ -303,6 +335,20 @@ class CLITestCase(DockerClientTestCase): self.assertFalse(config['AttachStdout']) self.assertFalse(config['AttachStdin']) + def test_up_handles_sigint(self): + proc = start_process(self.base_dir, ['up', '-t', '2']) + wait_on_condition(ContainerCountCondition(self.project, 2)) + + os.kill(proc.pid, signal.SIGINT) + wait_on_condition(ContainerCountCondition(self.project, 0)) + + def test_up_handles_sigterm(self): + proc = start_process(self.base_dir, ['up', '-t', '2']) + wait_on_condition(ContainerCountCondition(self.project, 2)) + + os.kill(proc.pid, signal.SIGTERM) + wait_on_condition(ContainerCountCondition(self.project, 0)) + def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['run', 'console', '/bin/true']) diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index ee837fcd..db37ac1a 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -57,11 +57,11 @@ class CLIMainTestCase(unittest.TestCase): with mock.patch('compose.cli.main.signal', autospec=True) as mock_signal: attach_to_logs(project, log_printer, service_names, timeout) - mock_signal.signal.assert_called_once_with(mock_signal.SIGINT, mock.ANY) + assert mock_signal.signal.mock_calls == [ + mock.call(mock_signal.SIGINT, mock.ANY), + mock.call(mock_signal.SIGTERM, mock.ANY), + ] log_printer.run.assert_called_once_with() - project.stop.assert_called_once_with( - service_names=service_names, - timeout=timeout) class SetupConsoleHandlerTestCase(unittest.TestCase): From 6236bb0019de51fe482e2ba6be8a99de471a6861 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 10 Nov 2015 19:43:05 -0500 Subject: [PATCH 111/359] Handle both SIGINT and SIGTERM for docker-compose run. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 99 ++++++++++++++++++++---------------- tests/acceptance/cli_test.py | 47 +++++++++++++++++ 2 files changed, 102 insertions(+), 44 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 7b1e0aa3..9fef8d04 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -368,7 +368,6 @@ class TopLevelCommand(DocoptCommand): allocates a TTY. """ service = project.get_service(options['SERVICE']) - detach = options['-d'] if IS_WINDOWS_PLATFORM and not detach: @@ -380,22 +379,6 @@ class TopLevelCommand(DocoptCommand): if options['--allow-insecure-ssl']: log.warn(INSECURE_SSL_WARNING) - if not options['--no-deps']: - deps = service.get_linked_service_names() - - if len(deps) > 0: - project.up( - service_names=deps, - start_deps=True, - strategy=ConvergenceStrategy.never, - ) - elif project.use_networking: - project.ensure_network_exists() - - tty = True - if detach or options['-T'] or not sys.stdin.isatty(): - tty = False - if options['COMMAND']: command = [options['COMMAND']] + options['ARGS'] else: @@ -403,7 +386,7 @@ class TopLevelCommand(DocoptCommand): container_options = { 'command': command, - 'tty': tty, + 'tty': not (detach or options['-T'] or not sys.stdin.isatty()), 'stdin_open': not detach, 'detach': detach, } @@ -435,31 +418,7 @@ class TopLevelCommand(DocoptCommand): if options['--name']: container_options['name'] = options['--name'] - try: - container = service.create_container( - quiet=True, - one_off=True, - **container_options - ) - except APIError as e: - legacy.check_for_legacy_containers( - project.client, - project.name, - [service.name], - allow_one_off=False, - ) - - raise e - - if detach: - container.start() - print(container.name) - else: - dockerpty.start(project.client, container.id, interactive=not options['-T']) - exit_code = container.wait() - if options['--rm']: - project.client.remove_container(container.id) - sys.exit(exit_code) + run_one_off_container(container_options, project, service, options) def scale(self, project, options): """ @@ -647,6 +606,58 @@ def convergence_strategy_from_opts(options): return ConvergenceStrategy.changed +def run_one_off_container(container_options, project, service, options): + if not options['--no-deps']: + deps = service.get_linked_service_names() + if deps: + project.up( + service_names=deps, + start_deps=True, + strategy=ConvergenceStrategy.never) + + if project.use_networking: + project.ensure_network_exists() + + try: + container = service.create_container( + quiet=True, + one_off=True, + **container_options) + except APIError: + legacy.check_for_legacy_containers( + project.client, + project.name, + [service.name], + allow_one_off=False) + raise + + if options['-d']: + container.start() + print(container.name) + return + + def remove_container(force=False): + if options['--rm']: + project.client.remove_container(container.id, force=True) + + def force_shutdown(signal, frame): + project.client.kill(container.id) + remove_container(force=True) + sys.exit(2) + + def shutdown(signal, frame): + set_signal_handler(force_shutdown) + project.client.stop(container.id) + remove_container() + sys.exit(1) + + set_signal_handler(shutdown) + dockerpty.start(project.client, container.id, interactive=not options['-T']) + exit_code = container.wait() + remove_container() + sys.exit(exit_code) + + def build_log_printer(containers, service_names, monochrome): if service_names: containers = [ @@ -657,7 +668,6 @@ def build_log_printer(containers, service_names, monochrome): def attach_to_logs(project, log_printer, service_names, timeout): - print("Attaching to", list_containers(log_printer.containers)) def force_shutdown(signal, frame): project.kill(service_names=service_names) @@ -668,6 +678,7 @@ def attach_to_logs(project, log_printer, service_names, timeout): print("Gracefully stopping... (press Ctrl+C again to force)") project.stop(service_names=service_names, timeout=timeout) + print("Attaching to", list_containers(log_printer.containers)) set_signal_handler(shutdown) log_printer.run() diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 57f2039e..b88ed280 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -8,6 +8,8 @@ import time from collections import namedtuple from operator import attrgetter +from docker import errors + from .. import mock from compose.cli.command import get_project from compose.cli.docker_client import docker_client @@ -61,6 +63,25 @@ class ContainerCountCondition(object): return "waiting for counter count == %s" % self.expected +class ContainerStateCondition(object): + + def __init__(self, client, name, running): + self.client = client + self.name = name + self.running = running + + # State.Running == true + def __call__(self): + try: + container = self.client.inspect_container(self.name) + return container['State']['Running'] == self.running + except errors.APIError: + return False + + def __str__(self): + return "waiting for container to have state %s" % self.expected + + class CLITestCase(DockerClientTestCase): def setUp(self): @@ -554,6 +575,32 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(networks), 1) self.assertEqual(container.human_readable_command, u'true') + def test_run_handles_sigint(self): + proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'simplecomposefile_simple_run_1', + running=True)) + + os.kill(proc.pid, signal.SIGINT) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'simplecomposefile_simple_run_1', + running=False)) + + def test_run_handles_sigterm(self): + proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'simplecomposefile_simple_run_1', + running=True)) + + os.kill(proc.pid, signal.SIGTERM) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'simplecomposefile_simple_run_1', + running=False)) + def test_rm(self): service = self.project.get_service('simple') service.create_container() From c99f2f8efd2927b01c1acdb990a83df8206a96f8 Mon Sep 17 00:00:00 2001 From: Stefan Scherer Date: Fri, 13 Nov 2015 08:33:51 +0100 Subject: [PATCH 112/359] Use uname to build target name for different platforms Signed-off-by: Stefan Scherer --- script/build-linux-inner | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/build-linux-inner b/script/build-linux-inner index 01137ff2..47d5eb2e 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -2,7 +2,7 @@ set -ex -TARGET=dist/docker-compose-Linux-x86_64 +TARGET=dist/docker-compose-$(uname -s)-$(uname -m) VENV=/code/.tox/py27 mkdir -p `pwd`/dist From e1308a8329de0ce12e4405677a1911a5db3bd33b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 10:49:17 -0500 Subject: [PATCH 113/359] Fix extra warnings on masked volumes. Signed-off-by: Daniel Nephin --- compose/service.py | 5 ++++- tests/unit/service_test.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 3b05264b..148da4db 100644 --- a/compose/service.py +++ b/compose/service.py @@ -963,7 +963,10 @@ def warn_on_masked_volume(volumes_option, container_volumes, service): for volume in container_volumes) for volume in volumes_option: - if container_volumes.get(volume.internal) != volume.external: + if ( + volume.internal in container_volumes and + container_volumes.get(volume.internal) != volume.external + ): log.warn(( "Service \"{service}\" is using volume \"{volume}\" from the " "previous container. Host mapping \"{host_path}\" has no effect. " diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 0cff9899..808c391c 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -26,6 +26,8 @@ from compose.service import parse_volume_spec from compose.service import Service from compose.service import ServiceNet from compose.service import VolumeFromSpec +from compose.service import VolumeSpec +from compose.service import warn_on_masked_volume class ServiceTest(unittest.TestCase): @@ -750,6 +752,39 @@ class ServiceVolumesTest(unittest.TestCase): ['/mnt/sda1/host/path:/data:rw'], ) + def test_warn_on_masked_volume_no_warning_when_no_container_volumes(self): + volumes_option = [VolumeSpec('/home/user', '/path', 'rw')] + container_volumes = [] + service = 'service_name' + + with mock.patch('compose.service.log') as mock_log: + warn_on_masked_volume(volumes_option, container_volumes, service) + + assert not mock_log.warn.called + + def test_warn_on_masked_volume_when_masked(self): + volumes_option = [VolumeSpec('/home/user', '/path', 'rw')] + container_volumes = [ + VolumeSpec('/var/lib/docker/path', '/path', 'rw'), + VolumeSpec('/var/lib/docker/path', '/other', 'rw'), + ] + service = 'service_name' + + with mock.patch('compose.service.log') as mock_log: + warn_on_masked_volume(volumes_option, container_volumes, service) + + mock_log.warn.called_once_with(mock.ANY) + + def test_warn_on_masked_no_warning_with_same_path(self): + volumes_option = [VolumeSpec('/home/user', '/path', 'rw')] + container_volumes = [VolumeSpec('/home/user', '/path', 'rw')] + service = 'service_name' + + with mock.patch('compose.service.log') as mock_log: + warn_on_masked_volume(volumes_option, container_volumes, service) + + assert not mock_log.warn.called + def test_create_with_special_volume_mode(self): self.mock_client.inspect_image.return_value = {'Id': 'imageid'} From 61f91ebff7c2158c4d2d51abc88c0e35e84cf256 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Seguin?= Date: Sat, 14 Nov 2015 12:19:57 +0100 Subject: [PATCH 114/359] Fix restart with stopped containers. Fixes #1814 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Seguin --- compose/service.py | 2 +- tests/acceptance/cli_test.py | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index 3b05264b..4f449d15 100644 --- a/compose/service.py +++ b/compose/service.py @@ -185,7 +185,7 @@ class Service(object): c.kill(**options) def restart(self, **options): - for c in self.containers(): + for c in self.containers(stopped=True): log.info("Restarting %s" % c.name) c.restart(**options) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 88a43d7f..34a2d166 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -597,6 +597,15 @@ class CLITestCase(DockerClientTestCase): started_at, ) + def test_restart_stopped_container(self): + service = self.project.get_service('simple') + container = service.create_container() + container.start() + container.kill() + self.assertEqual(len(service.containers(stopped=True)), 1) + self.dispatch(['restart', '-t', '1'], None) + self.assertEqual(len(service.containers(stopped=False)), 1) + def test_scale(self): project = self.project From 265828f4ebc383c18b251b153805ea084eaccf4d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 16 Nov 2015 12:55:35 -0500 Subject: [PATCH 115/359] Fix texttable dep. 0.8.2 was removed from pypi. Signed-off-by: Daniel Nephin --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 60327d72..659cb57f 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,5 +6,5 @@ enum34==1.0.4 jsonschema==2.5.1 requests==2.7.0 six==1.7.3 -texttable==0.8.2 +texttable==0.8.4 websocket-client==0.32.0 From d4b98452012d930121231f3b7be9c2c1db8b8208 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 Nov 2015 17:29:58 -0500 Subject: [PATCH 116/359] Add the git sha to version output Signed-off-by: Daniel Nephin --- .gitignore | 1 + Dockerfile.run | 2 +- MANIFEST.in | 1 + compose/cli/command.py | 4 ++-- compose/cli/utils.py | 39 +++++++++++++++++++++++++++---------- docker-compose.spec | 24 ++++++++++++++++++----- script/build-image | 1 + script/build-linux | 1 + script/build-linux-inner | 1 + script/build-osx | 1 + script/build-windows.ps1 | 2 ++ script/release/push-release | 1 + script/write-git-sha | 7 +++++++ 13 files changed, 67 insertions(+), 18 deletions(-) create mode 100755 script/write-git-sha diff --git a/.gitignore b/.gitignore index 83a08a0e..da728279 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,4 @@ /docs/_site /venv README.rst +compose/GITSHA diff --git a/Dockerfile.run b/Dockerfile.run index 9f3745fe..792077ad 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -8,6 +8,6 @@ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt ADD dist/docker-compose-release.tar.gz /code/docker-compose -RUN pip install /code/docker-compose/docker-compose-* +RUN pip install --no-deps /code/docker-compose/docker-compose-* ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/MANIFEST.in b/MANIFEST.in index 0342e35b..8c6f932b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -7,6 +7,7 @@ include *.md exclude README.md include README.rst include compose/config/*.json +include compose/GITSHA recursive-include contrib/completion * recursive-include tests * global-exclude *.pyc diff --git a/compose/cli/command.py b/compose/cli/command.py index 525217ee..6094b530 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -12,12 +12,12 @@ from requests.exceptions import SSLError from . import errors from . import verbose_proxy -from .. import __version__ from .. import config from ..project import Project from ..service import ConfigError from .docker_client import docker_client from .utils import call_silently +from .utils import get_version_info from .utils import is_mac from .utils import is_ubuntu @@ -71,7 +71,7 @@ def get_client(verbose=False, version=None): client = docker_client(version=version) if verbose: version_info = six.iteritems(client.version()) - log.info("Compose version %s", __version__) + log.info(get_version_info('full')) log.info("Docker base_url: %s", client.base_url) log.info("Docker version: %s", ", ".join("%s=%s" % item for item in version_info)) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index 07510e2f..dd859edc 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -7,10 +7,10 @@ import platform import ssl import subprocess -from docker import version as docker_py_version +import docker from six.moves import input -from .. import __version__ +import compose def yesno(prompt, default=None): @@ -57,13 +57,32 @@ def is_ubuntu(): def get_version_info(scope): - versioninfo = 'docker-compose version: %s' % __version__ + versioninfo = 'docker-compose version {}, build {}'.format( + compose.__version__, + get_build_version()) + if scope == 'compose': return versioninfo - elif scope == 'full': - return versioninfo + '\n' \ - + "docker-py version: %s\n" % docker_py_version \ - + "%s version: %s\n" % (platform.python_implementation(), platform.python_version()) \ - + "OpenSSL version: %s" % ssl.OPENSSL_VERSION - else: - raise RuntimeError('passed unallowed value to `cli.utils.get_version_info`') + if scope == 'full': + return ( + "{}\n" + "docker-py version: {}\n" + "{} version: {}\n" + "OpenSSL version: {}" + ).format( + versioninfo, + docker.version, + platform.python_implementation(), + platform.python_version(), + ssl.OPENSSL_VERSION) + + raise ValueError("{} is not a valid version scope".format(scope)) + + +def get_build_version(): + filename = os.path.join(os.path.dirname(compose.__file__), 'GITSHA') + if not os.path.exists(filename): + return 'unknown' + + with open(filename) as fh: + return fh.read().strip() diff --git a/docker-compose.spec b/docker-compose.spec index 678fc132..24d03e05 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -9,18 +9,32 @@ a = Analysis(['bin/docker-compose'], runtime_hooks=None, cipher=block_cipher) -pyz = PYZ(a.pure, - cipher=block_cipher) +pyz = PYZ(a.pure, cipher=block_cipher) exe = EXE(pyz, a.scripts, a.binaries, a.zipfiles, a.datas, - [('compose/config/fields_schema.json', 'compose/config/fields_schema.json', 'DATA')], - [('compose/config/service_schema.json', 'compose/config/service_schema.json', 'DATA')], + [ + ( + 'compose/config/fields_schema.json', + 'compose/config/fields_schema.json', + 'DATA' + ), + ( + 'compose/config/service_schema.json', + 'compose/config/service_schema.json', + 'DATA' + ), + ( + 'compose/GITSHA', + 'compose/GITSHA', + 'DATA' + ) + ], name='docker-compose', debug=False, strip=None, upx=True, - console=True ) + console=True) diff --git a/script/build-image b/script/build-image index 3ac9729b..89733505 100755 --- a/script/build-image +++ b/script/build-image @@ -10,6 +10,7 @@ fi TAG=$1 VERSION="$(python setup.py --version)" +./script/write-git-sha python setup.py sdist cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz docker build -t docker/compose:$TAG -f Dockerfile.run . diff --git a/script/build-linux b/script/build-linux index ade18bc5..47fb45e1 100755 --- a/script/build-linux +++ b/script/build-linux @@ -9,4 +9,5 @@ docker build -t "$TAG" . | tail -n 200 docker run \ --rm --entrypoint="script/build-linux-inner" \ -v $(pwd)/dist:/code/dist \ + -v $(pwd)/.git:/code/.git \ "$TAG" diff --git a/script/build-linux-inner b/script/build-linux-inner index 01137ff2..50b16dd8 100755 --- a/script/build-linux-inner +++ b/script/build-linux-inner @@ -9,6 +9,7 @@ mkdir -p `pwd`/dist chmod 777 `pwd`/dist $VENV/bin/pip install -q -r requirements-build.txt +./script/write-git-sha su -c "$VENV/bin/pyinstaller docker-compose.spec" user mv dist/docker-compose $TARGET $TARGET version diff --git a/script/build-osx b/script/build-osx index 042964e4..168fd430 100755 --- a/script/build-osx +++ b/script/build-osx @@ -9,6 +9,7 @@ virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . +./script/write-git-sha venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 42a4a501..28011b1d 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -47,6 +47,8 @@ virtualenv .\venv .\venv\Scripts\pip install --no-deps . .\venv\Scripts\pip install --allow-external pyinstaller -r requirements-build.txt +git rev-parse --short HEAD | out-file -encoding ASCII compose\GITSHA + # Build binary # pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue $ErrorActionPreference = "Continue" diff --git a/script/release/push-release b/script/release/push-release index ccdf2496..b754d40f 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -57,6 +57,7 @@ docker push docker/compose:$VERSION echo "Uploading sdist to pypi" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst +./script/write-git-sha python setup.py sdist if [ "$(command -v twine 2> /dev/null)" ]; then twine upload ./dist/docker-compose-${VERSION}.tar.gz diff --git a/script/write-git-sha b/script/write-git-sha new file mode 100755 index 00000000..d16743c6 --- /dev/null +++ b/script/write-git-sha @@ -0,0 +1,7 @@ +#!/bin/bash +# +# Write the current commit sha to the file GITSHA. This file is included in +# packaging so that `docker-compose version` can include the git sha. +# +set -e +git rev-parse --short HEAD > compose/GITSHA From efbfa9e38fb4565953d608e504e2bdc79737d408 Mon Sep 17 00:00:00 2001 From: Simon van der Veldt Date: Wed, 18 Nov 2015 21:38:58 +0100 Subject: [PATCH 117/359] run.sh script: Also pass DOCKER_TLS_VERIFY and DOCKER_CERT_PATH env vars to compose container Signed-off-by: Simon van der Veldt --- script/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run.sh b/script/run.sh index cf46c143..342188e8 100755 --- a/script/run.sh +++ b/script/run.sh @@ -26,7 +26,7 @@ fi if [ -S "$DOCKER_HOST" ]; then DOCKER_ADDR="-v $DOCKER_HOST:$DOCKER_HOST -e DOCKER_HOST" else - DOCKER_ADDR="-e DOCKER_HOST" + DOCKER_ADDR="-e DOCKER_HOST -e DOCKER_TLS_VERIFY -e DOCKER_CERT_PATH" fi From c78c32c2e819cdbf83f9e9ed2dc16ff9e62b78dc Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 16 Nov 2015 12:35:26 -0500 Subject: [PATCH 118/359] Fixes #2398 - the build progress stream can contain empty json objects. Previously these empty objects would hit a bug in splitting objects causing it crash. With this fix the empty objects are returned properly. Signed-off-by: Daniel Nephin --- compose/utils.py | 12 ++++++------ tests/unit/utils_test.py | 28 ++++++++++++++++++++-------- 2 files changed, 26 insertions(+), 14 deletions(-) diff --git a/compose/utils.py b/compose/utils.py index 2c6c4584..a013035e 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -102,7 +102,7 @@ def stream_as_text(stream): def line_splitter(buffer, separator=u'\n'): index = buffer.find(six.text_type(separator)) if index == -1: - return None, None + return None return buffer[:index + 1], buffer[index + 1:] @@ -120,11 +120,11 @@ def split_buffer(stream, splitter=None, decoder=lambda a: a): for data in stream_as_text(stream): buffered += data while True: - item, rest = splitter(buffered) - if not item: + buffer_split = splitter(buffered) + if buffer_split is None: break - buffered = rest + item, buffered = buffer_split yield item if buffered: @@ -140,7 +140,7 @@ def json_splitter(buffer): rest = buffer[json.decoder.WHITESPACE.match(buffer, index).end():] return obj, rest except ValueError: - return None, None + return None def json_stream(stream): @@ -148,7 +148,7 @@ def json_stream(stream): This handles streams which are inconsistently buffered (some entries may be newline delimited, and others are not). """ - return split_buffer(stream_as_text(stream), json_splitter, json_decoder.decode) + return split_buffer(stream, json_splitter, json_decoder.decode) def write_out_msg(stream, lines, msg_index, msg, status="done"): diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index e3d0bc00..15999dde 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -1,25 +1,21 @@ # encoding: utf-8 from __future__ import unicode_literals -from .. import unittest from compose import utils -class JsonSplitterTestCase(unittest.TestCase): +class TestJsonSplitter(object): def test_json_splitter_no_object(self): data = '{"foo": "bar' - self.assertEqual(utils.json_splitter(data), (None, None)) + assert utils.json_splitter(data) is None def test_json_splitter_with_object(self): data = '{"foo": "bar"}\n \n{"next": "obj"}' - self.assertEqual( - utils.json_splitter(data), - ({'foo': 'bar'}, '{"next": "obj"}') - ) + assert utils.json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}') -class StreamAsTextTestCase(unittest.TestCase): +class TestStreamAsText(object): def test_stream_with_non_utf_unicode_character(self): stream = [b'\xed\xf3\xf3'] @@ -30,3 +26,19 @@ class StreamAsTextTestCase(unittest.TestCase): stream = ['ěĝ'.encode('utf-8')] output, = utils.stream_as_text(stream) assert output == 'ěĝ' + + +class TestJsonStream(object): + + def test_with_falsy_entries(self): + stream = [ + '{"one": "two"}\n{}\n', + "[1, 2, 3]\n[]\n", + ] + output = list(utils.json_stream(stream)) + assert output == [ + {'one': 'two'}, + {}, + [1, 2, 3], + [], + ] From 1e8f76767f80ecb4b7aa546eeb787102aff311e4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 18 Nov 2015 14:51:01 -0500 Subject: [PATCH 119/359] Fix env_file and environment when used with extends. Signed-off-by: Daniel Nephin --- compose/config/config.py | 22 +++++++-------- tests/integration/testcases.py | 3 +- tests/unit/config/config_test.py | 48 ++++++++++++++++++++++++++++++++ 3 files changed, 60 insertions(+), 13 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 20126620..fa214767 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -257,16 +257,11 @@ class ServiceExtendsResolver(object): def run(self): self.detect_cycle() - service_dict = dict(self.service_config.config) - env = resolve_environment(self.working_dir, self.service_config.config) - if env: - service_dict['environment'] = env - service_dict.pop('env_file', None) - - if 'extends' in service_dict: + if 'extends' in self.service_config.config: service_dict = self.resolve_extends(*self.validate_and_construct_extends()) + return self.service_config._replace(config=service_dict) - return self.service_config._replace(config=service_dict) + return self.service_config def validate_and_construct_extends(self): extends = self.service_config.config['extends'] @@ -316,16 +311,15 @@ class ServiceExtendsResolver(object): return filename -def resolve_environment(working_dir, service_dict): +def resolve_environment(service_config): """Unpack any environment variables from an env_file, if set. Interpolate environment values if set. """ - if 'environment' not in service_dict and 'env_file' not in service_dict: - return {} + service_dict = service_config.config env = {} if 'env_file' in service_dict: - for env_file in get_env_files(working_dir, service_dict): + for env_file in get_env_files(service_config.working_dir, service_dict): env.update(env_vars_from_file(env_file)) env.update(parse_environment(service_dict.get('environment'))) @@ -362,6 +356,10 @@ def process_service(service_config): working_dir = service_config.working_dir service_dict = dict(service_config.config) + if 'environment' in service_dict or 'env_file' in service_dict: + service_dict['environment'] = resolve_environment(service_config) + service_dict.pop('env_file', None) + if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index d63f0591..de2d1a70 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -46,7 +46,8 @@ class DockerClientTestCase(unittest.TestCase): service_config = ServiceConfig('.', None, name, kwargs) options = process_service(service_config) - options['environment'] = resolve_environment('.', kwargs) + options['environment'] = resolve_environment( + service_config._replace(config=options)) labels = options.setdefault('labels', {}) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3038af80..c69e3430 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1317,6 +1317,54 @@ class ExtendsTest(unittest.TestCase): }, ])) + def test_extends_with_environment_and_env_files(self): + tmpdir = py.test.ensuretemp('test_extends_with_environment') + self.addCleanup(tmpdir.remove) + commondir = tmpdir.mkdir('common') + commondir.join('base.yml').write(""" + app: + image: 'example/app' + env_file: + - 'envs' + environment: + - SECRET + """) + tmpdir.join('docker-compose.yml').write(""" + ext: + extends: + file: common/base.yml + service: app + env_file: + - 'envs' + environment: + - THING + """) + commondir.join('envs').write(""" + COMMON_ENV_FILE=1 + """) + tmpdir.join('envs').write(""" + FROM_ENV_FILE=1 + """) + + expected = [ + { + 'name': 'ext', + 'image': 'example/app', + 'environment': { + 'SECRET': 'secret', + 'FROM_ENV_FILE': '1', + 'COMMON_ENV_FILE': '1', + 'THING': 'thing', + }, + }, + ] + with mock.patch.dict(os.environ): + os.environ['SECRET'] = 'secret' + os.environ['THING'] = 'thing' + config = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + + assert config == expected + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From acf31181e8bff0481703c8f17f93c17ebec59506 Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Wed, 18 Nov 2015 20:17:23 +1000 Subject: [PATCH 120/359] Some small changes to clear up docs-validation complaints Signed-off-by: Sven Dowideit --- docs/README.md | 9 +++++++++ docs/django.md | 2 +- docs/extends.md | 2 +- docs/install.md | 2 +- docs/rails.md | 2 +- docs/wordpress.md | 2 +- 6 files changed, 14 insertions(+), 5 deletions(-) diff --git a/docs/README.md b/docs/README.md index 8fbad30c..d8ab7c3e 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,3 +1,12 @@ + + # Contributing to the Docker Compose documentation The documentation in this directory is part of the [https://docs.docker.com](https://docs.docker.com) website. Docker uses [the Hugo static generator](http://gohugo.io/overview/introduction/) to convert project Markdown files to a static HTML site. diff --git a/docs/django.md b/docs/django.md index d4d2bd1e..b503e574 100644 --- a/docs/django.md +++ b/docs/django.md @@ -171,7 +171,7 @@ In this section, you set up the database connection for Django. ## More Compose documentation -- [User guide](../index.md) +- [User guide](index.md) - [Installing Compose](install.md) - [Getting Started](gettingstarted.md) - [Get started with Rails](rails.md) diff --git a/docs/extends.md b/docs/extends.md index b21b6d76..011a7350 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -365,7 +365,7 @@ In the case of `environment`, `labels`, `volumes` and `devices`, Compose ## Compose documentation -- [User guide](/) +- [User guide](index.md) - [Installing Compose](install.md) - [Getting Started](gettingstarted.md) - [Get started with Django](django.md) diff --git a/docs/install.md b/docs/install.md index c5304409..5f956359 100644 --- a/docs/install.md +++ b/docs/install.md @@ -126,7 +126,7 @@ To uninstall Docker Compose if you installed using `pip`: ## Where to go next -- [User guide](/) +- [User guide](index.md) - [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) diff --git a/docs/rails.md b/docs/rails.md index 8e16af64..d3f1707c 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -133,7 +133,7 @@ That's it. Your app should now be running on port 3000 on your Docker daemon. If ## More Compose documentation -- [User guide](/) +- [User guide](index.md) - [Installing Compose](install.md) - [Getting Started](gettingstarted.md) - [Get started with Django](django.md) diff --git a/docs/wordpress.md b/docs/wordpress.md index 373ef4d0..15746a75 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -93,7 +93,7 @@ database containers. If you're using [Docker Machine](https://docs.docker.com/ma ## More Compose documentation -- [User guide](/) +- [User guide](index.md) - [Installing Compose](install.md) - [Getting Started](gettingstarted.md) - [Get started with Django](django.md) From 3fdf0f43bef4a881b65de871b26ec6feffd98059 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 19 Nov 2015 12:55:03 -0500 Subject: [PATCH 121/359] Add note about required pip version. Signed-off-by: Daniel Nephin --- docs/install.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/install.md b/docs/install.md index 5f956359..d394905d 100644 --- a/docs/install.md +++ b/docs/install.md @@ -70,6 +70,7 @@ to get started. $ pip install docker-compose +> **Note:** pip version 6.0 or greater is required ### Install as a container From 6224b6edd9b776fbeaf4526152e323c0538be8f2 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 19 Nov 2015 14:52:50 -0500 Subject: [PATCH 122/359] Fix use case link in readme. Signed-off-by: Daniel Nephin --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5052db39..c9b4729a 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ see [the list of features](docs/index.md#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in -[Common Use Cases](#common-use-cases). +[Common Use Cases](docs/index.md#common-use-cases). Using Compose is basically a three-step process. From d1adbb9b259c4582b77ac90a96fe2d071fb408aa Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 Nov 2015 20:44:05 -0500 Subject: [PATCH 123/359] Refactor parallel_execute. Signed-off-by: Daniel Nephin --- compose/container.py | 39 ++++++++++ compose/project.py | 34 ++------- compose/service.py | 45 ++++-------- compose/utils.py | 117 +++++++++++++++--------------- tests/integration/service_test.py | 27 ++++--- tox.ini | 2 +- 6 files changed, 137 insertions(+), 127 deletions(-) diff --git a/compose/container.py b/compose/container.py index 1ca48380..dde83bd3 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import operator from functools import reduce import six @@ -8,6 +9,7 @@ import six from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_PROJECT from .const import LABEL_SERVICE +from compose.utils import parallel_execute class Container(object): @@ -250,3 +252,40 @@ def get_container_name(container): # ps shortest_name = min(container['Names'], key=lambda n: len(n.split('/'))) return shortest_name.split('/')[-1] + + +def parallel_operation(containers, operation, options, message): + parallel_execute( + containers, + operator.methodcaller(operation, **options), + operator.attrgetter('name'), + message) + + +def parallel_remove(containers, options): + stopped_containers = [c for c in containers if not c.is_running] + parallel_operation(stopped_containers, 'remove', options, 'Removing') + + +def parallel_stop(containers, options): + parallel_operation(containers, 'stop', options, 'Stopping') + + +def parallel_start(containers, options): + parallel_operation(containers, 'start', options, 'Starting') + + +def parallel_pause(containers, options): + parallel_operation(containers, 'pause', options, 'Pausing') + + +def parallel_unpause(containers, options): + parallel_operation(containers, 'unpause', options, 'Unpausing') + + +def parallel_kill(containers, options): + parallel_operation(containers, 'kill', options, 'Killing') + + +def parallel_restart(containers, options): + parallel_operation(containers, 'restart', options, 'Restarting') diff --git a/compose/project.py b/compose/project.py index 41af8626..dc6dd32f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -7,6 +7,7 @@ from functools import reduce from docker.errors import APIError from docker.errors import NotFound +from . import container from .config import ConfigurationError from .config import get_service_name_from_net from .const import DEFAULT_TIMEOUT @@ -22,7 +23,6 @@ from .service import parse_volume_from_spec from .service import Service from .service import ServiceNet from .service import VolumeFromSpec -from .utils import parallel_execute log = logging.getLogger(__name__) @@ -241,42 +241,22 @@ class Project(object): service.start(**options) def stop(self, service_names=None, **options): - parallel_execute( - objects=self.containers(service_names), - obj_callable=lambda c: c.stop(**options), - msg_index=lambda c: c.name, - msg="Stopping" - ) + container.parallel_stop(self.containers(service_names), options) def pause(self, service_names=None, **options): - for service in reversed(self.get_services(service_names)): - service.pause(**options) + container.parallel_pause(reversed(self.containers(service_names)), options) def unpause(self, service_names=None, **options): - for service in self.get_services(service_names): - service.unpause(**options) + container.parallel_unpause(self.containers(service_names), options) def kill(self, service_names=None, **options): - parallel_execute( - objects=self.containers(service_names), - obj_callable=lambda c: c.kill(**options), - msg_index=lambda c: c.name, - msg="Killing" - ) + container.parallel_kill(self.containers(service_names), options) def remove_stopped(self, service_names=None, **options): - all_containers = self.containers(service_names, stopped=True) - stopped_containers = [c for c in all_containers if not c.is_running] - parallel_execute( - objects=stopped_containers, - obj_callable=lambda c: c.remove(**options), - msg_index=lambda c: c.name, - msg="Removing" - ) + container.parallel_remove(self.containers(service_names, stopped=True), options) def restart(self, service_names=None, **options): - for service in self.get_services(service_names): - service.restart(**options) + container.parallel_restart(self.containers(service_names, stopped=True), options) def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): for service in self.get_services(service_names): diff --git a/compose/service.py b/compose/service.py index b79fd900..ab6f6dd6 100644 --- a/compose/service.py +++ b/compose/service.py @@ -28,6 +28,9 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION from .container import Container +from .container import parallel_remove +from .container import parallel_start +from .container import parallel_stop from .legacy import check_for_legacy_containers from .progress_stream import stream_output from .progress_stream import StreamOutputError @@ -241,12 +244,7 @@ class Service(object): else: containers_to_start = stopped_containers - parallel_execute( - objects=containers_to_start, - obj_callable=lambda c: c.start(), - msg_index=lambda c: c.name, - msg="Starting" - ) + parallel_start(containers_to_start, {}) num_running += len(containers_to_start) @@ -259,35 +257,22 @@ class Service(object): ] parallel_execute( - objects=container_numbers, - obj_callable=lambda n: create_and_start(service=self, number=n), - msg_index=lambda n: n, - msg="Creating and starting" + container_numbers, + lambda n: create_and_start(service=self, number=n), + lambda n: n, + "Creating and starting" ) if desired_num < num_running: num_to_stop = num_running - desired_num - sorted_running_containers = sorted(running_containers, key=attrgetter('number')) - containers_to_stop = sorted_running_containers[-num_to_stop:] + sorted_running_containers = sorted( + running_containers, + key=attrgetter('number')) + parallel_stop( + sorted_running_containers[-num_to_stop:], + dict(timeout=timeout)) - parallel_execute( - objects=containers_to_stop, - obj_callable=lambda c: c.stop(timeout=timeout), - msg_index=lambda c: c.name, - msg="Stopping" - ) - - self.remove_stopped() - - def remove_stopped(self, **options): - containers = [c for c in self.containers(stopped=True) if not c.is_running] - - parallel_execute( - objects=containers, - obj_callable=lambda c: c.remove(**options), - msg_index=lambda c: c.name, - msg="Removing" - ) + parallel_remove(self.containers(stopped=True), {}) def create_container(self, one_off=False, diff --git a/compose/utils.py b/compose/utils.py index a013035e..716f6633 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -17,58 +17,51 @@ log = logging.getLogger(__name__) json_decoder = json.JSONDecoder() -def parallel_execute(objects, obj_callable, msg_index, msg): - """ - For a given list of objects, call the callable passing in the first +def perform_operation(func, arg, callback, index): + try: + callback((index, func(arg))) + except Exception as e: + callback((index, e)) + + +def parallel_execute(objects, func, index_func, msg): + """For a given list of objects, call the callable passing in the first object we give it. """ + objects = list(objects) stream = get_output_stream(sys.stdout) - lines = [] + writer = ParallelStreamWriter(stream, msg) for obj in objects: - write_out_msg(stream, lines, msg_index(obj), msg) + writer.initialize(index_func(obj)) q = Queue() - def inner_execute_function(an_callable, parameter, msg_index): - error = None - try: - result = an_callable(parameter) - except APIError as e: - error = e.explanation - result = "error" - except Exception as e: - error = e - result = 'unexpected_exception' - - q.put((msg_index, result, error)) - - for an_object in objects: + # TODO: limit the number of threads #1828 + for obj in objects: t = Thread( - target=inner_execute_function, - args=(obj_callable, an_object, msg_index(an_object)), - ) + target=perform_operation, + args=(func, obj, q.put, index_func(obj))) t.daemon = True t.start() done = 0 errors = {} - total_to_execute = len(objects) - while done < total_to_execute: + while done < len(objects): try: - msg_index, result, error = q.get(timeout=1) - - if result == 'unexpected_exception': - errors[msg_index] = result, error - if result == 'error': - errors[msg_index] = result, error - write_out_msg(stream, lines, msg_index, msg, status='error') - else: - write_out_msg(stream, lines, msg_index, msg) - done += 1 + msg_index, result = q.get(timeout=1) except Empty: - pass + continue + + if isinstance(result, APIError): + errors[msg_index] = "error", result.explanation + writer.write(msg_index, 'error') + elif isinstance(result, Exception): + errors[msg_index] = "unexpected_exception", result + else: + writer.write(msg_index, 'done') + done += 1 if not errors: return @@ -80,6 +73,36 @@ def parallel_execute(objects, obj_callable, msg_index, msg): raise error +class ParallelStreamWriter(object): + """Write out messages for operations happening in parallel. + + Each operation has it's own line, and ANSI code characters are used + to jump to the correct line, and write over the line. + """ + + def __init__(self, stream, msg): + self.stream = stream + self.msg = msg + self.lines = [] + + def initialize(self, obj_index): + self.lines.append(obj_index) + self.stream.write("{} {} ... \r\n".format(self.msg, obj_index)) + self.stream.flush() + + def write(self, obj_index, status): + position = self.lines.index(obj_index) + diff = len(self.lines) - position + # move up + self.stream.write("%c[%dA" % (27, diff)) + # erase + self.stream.write("%c[2K\r" % 27) + self.stream.write("{} {} ... {}\r".format(self.msg, obj_index, status)) + # move back down + self.stream.write("%c[%dB" % (27, diff)) + self.stream.flush() + + def get_output_stream(stream): if six.PY3: return stream @@ -151,30 +174,6 @@ def json_stream(stream): return split_buffer(stream, json_splitter, json_decoder.decode) -def write_out_msg(stream, lines, msg_index, msg, status="done"): - """ - Using special ANSI code characters we can write out the msg over the top of - a previous status message, if it exists. - """ - obj_index = msg_index - if msg_index in lines: - position = lines.index(obj_index) - diff = len(lines) - position - # move up - stream.write("%c[%dA" % (27, diff)) - # erase - stream.write("%c[2K\r" % 27) - stream.write("{} {} ... {}\r".format(msg, obj_index, status)) - # move back down - stream.write("%c[%dB" % (27, diff)) - else: - diff = 0 - lines.append(obj_index) - stream.write("{} {} ... \r\n".format(msg, obj_index)) - - stream.flush() - - def json_hash(obj): dump = json.dumps(obj, sort_keys=True, separators=(',', ':')) h = hashlib.sha256() diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index aaa4f01e..34869ab8 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -36,6 +36,12 @@ def create_and_start_container(service, **override_options): return container +def remove_stopped(service): + containers = [c for c in service.containers(stopped=True) if not c.is_running] + for container in containers: + container.remove() + + class ServiceTest(DockerClientTestCase): def test_containers(self): foo = self.create_service('foo') @@ -94,14 +100,14 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(service) self.assertEqual(len(service.containers()), 1) - service.remove_stopped() + remove_stopped(service) self.assertEqual(len(service.containers()), 1) service.kill() self.assertEqual(len(service.containers()), 0) self.assertEqual(len(service.containers(stopped=True)), 1) - service.remove_stopped() + remove_stopped(service) self.assertEqual(len(service.containers(stopped=True)), 0) def test_create_container_with_one_off(self): @@ -659,9 +665,8 @@ class ServiceTest(DockerClientTestCase): self.assertIn('Creating', captured_output) self.assertIn('Starting', captured_output) - def test_scale_with_api_returns_errors(self): - """ - Test that when scaling if the API returns an error, that error is handled + def test_scale_with_api_error(self): + """Test that when scaling if the API returns an error, that error is handled and the remaining threads continue. """ service = self.create_service('web') @@ -670,7 +675,10 @@ class ServiceTest(DockerClientTestCase): with mock.patch( 'compose.container.Container.create', - side_effect=APIError(message="testing", response={}, explanation="Boom")): + side_effect=APIError( + message="testing", + response={}, + explanation="Boom")): with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: service.scale(3) @@ -679,9 +687,8 @@ class ServiceTest(DockerClientTestCase): self.assertTrue(service.containers()[0].is_running) self.assertIn("ERROR: for 2 Boom", mock_stdout.getvalue()) - def test_scale_with_api_returns_unexpected_exception(self): - """ - Test that when scaling if the API returns an error, that is not of type + def test_scale_with_unexpected_exception(self): + """Test that when scaling if the API returns an error, that is not of type APIError, that error is re-raised. """ service = self.create_service('web') @@ -903,7 +910,7 @@ class ServiceTest(DockerClientTestCase): self.assertIn(pair, labels) service.kill() - service.remove_stopped() + remove_stopped(service) labels_list = ["%s=%s" % pair for pair in labels_dict.items()] diff --git a/tox.ini b/tox.ini index d1098a55..9d45b0c7 100644 --- a/tox.ini +++ b/tox.ini @@ -44,5 +44,5 @@ directory = coverage-html # Allow really long lines for now max-line-length = 140 # Set this high for now -max-complexity = 20 +max-complexity = 12 exclude = compose/packages From 64447879d2f5a2fe5b8b50819b6620b759715a9d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 17 Nov 2015 12:21:47 -0500 Subject: [PATCH 124/359] Reduce complexity of merge_service_dicts Signed-off-by: Daniel Nephin --- compose/config/config.py | 75 +++++++++++++++++++--------------------- 1 file changed, 35 insertions(+), 40 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 20126620..6c565433 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,5 +1,6 @@ import codecs import logging +import operator import os import sys from collections import namedtuple @@ -389,56 +390,46 @@ def merge_service_dicts_from_files(base, override): def merge_service_dicts(base, override): - d = base.copy() + d = {} - if 'environment' in base or 'environment' in override: - d['environment'] = merge_environment( - base.get('environment'), - override.get('environment'), - ) + def merge_field(field, merge_func, default=None): + if field in base or field in override: + d[field] = merge_func( + base.get(field, default), + override.get(field, default)) - path_mapping_keys = ['volumes', 'devices'] + merge_field('environment', merge_environment) + merge_field('labels', merge_labels) + merge_image_or_build(base, override, d) - for key in path_mapping_keys: - if key in base or key in override: - d[key] = merge_path_mappings( - base.get(key), - override.get(key), - ) + for field in ['volumes', 'devices']: + merge_field(field, merge_path_mappings) - if 'labels' in base or 'labels' in override: - d['labels'] = merge_labels( - base.get('labels'), - override.get('labels'), - ) + for field in ['ports', 'expose', 'external_links']: + merge_field(field, operator.add, default=[]) - if 'image' in override and 'build' in d: - del d['build'] + for field in ['dns', 'dns_search']: + merge_field(field, merge_list_or_string) - if 'build' in override and 'image' in d: - del d['image'] - - list_keys = ['ports', 'expose', 'external_links'] - - for key in list_keys: - if key in base or key in override: - d[key] = base.get(key, []) + override.get(key, []) - - list_or_string_keys = ['dns', 'dns_search'] - - for key in list_or_string_keys: - if key in base or key in override: - d[key] = to_list(base.get(key)) + to_list(override.get(key)) - - already_merged_keys = ['environment', 'labels'] + path_mapping_keys + list_keys + list_or_string_keys - - for k in set(ALLOWED_KEYS) - set(already_merged_keys): - if k in override: - d[k] = override[k] + already_merged_keys = set(d) | {'image', 'build'} + for field in set(ALLOWED_KEYS) - already_merged_keys: + if field in base or field in override: + d[field] = override.get(field, base.get(field)) return d +def merge_image_or_build(base, override, output): + if 'image' in override: + output['image'] = override['image'] + elif 'build' in override: + output['build'] = override['build'] + elif 'image' in base: + output['image'] = base['image'] + elif 'build' in base: + output['build'] = base['build'] + + def merge_environment(base, override): env = parse_environment(base) env.update(parse_environment(override)) @@ -604,6 +595,10 @@ def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, os.path.expanduser(path))) +def merge_list_or_string(base, override): + return to_list(base) + to_list(override) + + def to_list(value): if value is None: return [] From 2351e11cc8410cee9472cda38b685204e6252084 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 20 Nov 2015 17:51:36 -0500 Subject: [PATCH 125/359] Make sure we always have the latest busybox image, so that build --pull tests don't flake. Signed-off-by: Daniel Nephin --- tests/acceptance/cli_test.py | 5 +++++ tests/integration/testcases.py | 6 +----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 7ca6e819..88ec4573 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -15,6 +15,7 @@ from compose.cli.command import get_project from compose.cli.docker_client import docker_client from compose.container import Container from tests.integration.testcases import DockerClientTestCase +from tests.integration.testcases import pull_busybox ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -184,6 +185,8 @@ class CLITestCase(DockerClientTestCase): assert BUILD_PULL_TEXT not in result.stdout def test_build_pull(self): + # Make sure we have the latest busybox already + pull_busybox(self.client) self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple'], None) @@ -192,6 +195,8 @@ class CLITestCase(DockerClientTestCase): assert BUILD_PULL_TEXT in result.stdout def test_build_no_cache_pull(self): + # Make sure we have the latest busybox already + pull_busybox(self.client) self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['build', 'simple']) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index d63f0591..9b7b1f82 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,7 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -from docker import errors from docker.utils import version_lt from pytest import skip @@ -16,10 +15,7 @@ from compose.service import Service def pull_busybox(client): - try: - client.inspect_image('busybox:latest') - except errors.APIError: - client.pull('busybox:latest', stream=False) + client.pull('busybox:latest', stream=False) class DockerClientTestCase(unittest.TestCase): From 13081d4516d6d6b8ebfebc045e78e9be6bd96b74 Mon Sep 17 00:00:00 2001 From: Brandon Burton Date: Fri, 20 Nov 2015 16:02:37 -0800 Subject: [PATCH 126/359] Fixing matrix include so `os: linux` goes to trusty Signed-off-by: Brandon Burton --- .travis.yml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3310e2ad..3bb365a1 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,16 +2,14 @@ sudo: required language: python -services: - - docker - matrix: include: - os: linux + services: + - docker - os: osx language: generic - install: ./script/travis/install script: From b4edf0c45481acb8d780de9ff73156fb7c581337 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 23 Nov 2015 11:34:48 -0500 Subject: [PATCH 127/359] Move parallel_execute to a new module. Signed-off-by: Daniel Nephin --- compose/container.py | 39 ------------- compose/parallel.py | 135 +++++++++++++++++++++++++++++++++++++++++++ compose/project.py | 14 ++--- compose/service.py | 8 +-- compose/utils.py | 94 ------------------------------ 5 files changed, 146 insertions(+), 144 deletions(-) create mode 100644 compose/parallel.py diff --git a/compose/container.py b/compose/container.py index dde83bd3..1ca48380 100644 --- a/compose/container.py +++ b/compose/container.py @@ -1,7 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -import operator from functools import reduce import six @@ -9,7 +8,6 @@ import six from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_PROJECT from .const import LABEL_SERVICE -from compose.utils import parallel_execute class Container(object): @@ -252,40 +250,3 @@ def get_container_name(container): # ps shortest_name = min(container['Names'], key=lambda n: len(n.split('/'))) return shortest_name.split('/')[-1] - - -def parallel_operation(containers, operation, options, message): - parallel_execute( - containers, - operator.methodcaller(operation, **options), - operator.attrgetter('name'), - message) - - -def parallel_remove(containers, options): - stopped_containers = [c for c in containers if not c.is_running] - parallel_operation(stopped_containers, 'remove', options, 'Removing') - - -def parallel_stop(containers, options): - parallel_operation(containers, 'stop', options, 'Stopping') - - -def parallel_start(containers, options): - parallel_operation(containers, 'start', options, 'Starting') - - -def parallel_pause(containers, options): - parallel_operation(containers, 'pause', options, 'Pausing') - - -def parallel_unpause(containers, options): - parallel_operation(containers, 'unpause', options, 'Unpausing') - - -def parallel_kill(containers, options): - parallel_operation(containers, 'kill', options, 'Killing') - - -def parallel_restart(containers, options): - parallel_operation(containers, 'restart', options, 'Restarting') diff --git a/compose/parallel.py b/compose/parallel.py new file mode 100644 index 00000000..2735a397 --- /dev/null +++ b/compose/parallel.py @@ -0,0 +1,135 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import operator +import sys +from threading import Thread + +from docker.errors import APIError +from six.moves.queue import Empty +from six.moves.queue import Queue + +from compose.utils import get_output_stream + + +def perform_operation(func, arg, callback, index): + try: + callback((index, func(arg))) + except Exception as e: + callback((index, e)) + + +def parallel_execute(objects, func, index_func, msg): + """For a given list of objects, call the callable passing in the first + object we give it. + """ + objects = list(objects) + stream = get_output_stream(sys.stdout) + writer = ParallelStreamWriter(stream, msg) + + for obj in objects: + writer.initialize(index_func(obj)) + + q = Queue() + + # TODO: limit the number of threads #1828 + for obj in objects: + t = Thread( + target=perform_operation, + args=(func, obj, q.put, index_func(obj))) + t.daemon = True + t.start() + + done = 0 + errors = {} + + while done < len(objects): + try: + msg_index, result = q.get(timeout=1) + except Empty: + continue + + if isinstance(result, APIError): + errors[msg_index] = "error", result.explanation + writer.write(msg_index, 'error') + elif isinstance(result, Exception): + errors[msg_index] = "unexpected_exception", result + else: + writer.write(msg_index, 'done') + done += 1 + + if not errors: + return + + stream.write("\n") + for msg_index, (result, error) in errors.items(): + stream.write("ERROR: for {} {} \n".format(msg_index, error)) + if result == 'unexpected_exception': + raise error + + +class ParallelStreamWriter(object): + """Write out messages for operations happening in parallel. + + Each operation has it's own line, and ANSI code characters are used + to jump to the correct line, and write over the line. + """ + + def __init__(self, stream, msg): + self.stream = stream + self.msg = msg + self.lines = [] + + def initialize(self, obj_index): + self.lines.append(obj_index) + self.stream.write("{} {} ... \r\n".format(self.msg, obj_index)) + self.stream.flush() + + def write(self, obj_index, status): + position = self.lines.index(obj_index) + diff = len(self.lines) - position + # move up + self.stream.write("%c[%dA" % (27, diff)) + # erase + self.stream.write("%c[2K\r" % 27) + self.stream.write("{} {} ... {}\r".format(self.msg, obj_index, status)) + # move back down + self.stream.write("%c[%dB" % (27, diff)) + self.stream.flush() + + +def parallel_operation(containers, operation, options, message): + parallel_execute( + containers, + operator.methodcaller(operation, **options), + operator.attrgetter('name'), + message) + + +def parallel_remove(containers, options): + stopped_containers = [c for c in containers if not c.is_running] + parallel_operation(stopped_containers, 'remove', options, 'Removing') + + +def parallel_stop(containers, options): + parallel_operation(containers, 'stop', options, 'Stopping') + + +def parallel_start(containers, options): + parallel_operation(containers, 'start', options, 'Starting') + + +def parallel_pause(containers, options): + parallel_operation(containers, 'pause', options, 'Pausing') + + +def parallel_unpause(containers, options): + parallel_operation(containers, 'unpause', options, 'Unpausing') + + +def parallel_kill(containers, options): + parallel_operation(containers, 'kill', options, 'Killing') + + +def parallel_restart(containers, options): + parallel_operation(containers, 'restart', options, 'Restarting') diff --git a/compose/project.py b/compose/project.py index dc6dd32f..e29a2eb5 100644 --- a/compose/project.py +++ b/compose/project.py @@ -7,7 +7,7 @@ from functools import reduce from docker.errors import APIError from docker.errors import NotFound -from . import container +from . import parallel from .config import ConfigurationError from .config import get_service_name_from_net from .const import DEFAULT_TIMEOUT @@ -241,22 +241,22 @@ class Project(object): service.start(**options) def stop(self, service_names=None, **options): - container.parallel_stop(self.containers(service_names), options) + parallel.parallel_stop(self.containers(service_names), options) def pause(self, service_names=None, **options): - container.parallel_pause(reversed(self.containers(service_names)), options) + parallel.parallel_pause(reversed(self.containers(service_names)), options) def unpause(self, service_names=None, **options): - container.parallel_unpause(self.containers(service_names), options) + parallel.parallel_unpause(self.containers(service_names), options) def kill(self, service_names=None, **options): - container.parallel_kill(self.containers(service_names), options) + parallel.parallel_kill(self.containers(service_names), options) def remove_stopped(self, service_names=None, **options): - container.parallel_remove(self.containers(service_names, stopped=True), options) + parallel.parallel_remove(self.containers(service_names, stopped=True), options) def restart(self, service_names=None, **options): - container.parallel_restart(self.containers(service_names, stopped=True), options) + parallel.parallel_restart(self.containers(service_names, stopped=True), options) def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): for service in self.get_services(service_names): diff --git a/compose/service.py b/compose/service.py index ab6f6dd6..dd2399ee 100644 --- a/compose/service.py +++ b/compose/service.py @@ -28,14 +28,14 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION from .container import Container -from .container import parallel_remove -from .container import parallel_start -from .container import parallel_stop from .legacy import check_for_legacy_containers +from .parallel import parallel_execute +from .parallel import parallel_remove +from .parallel import parallel_start +from .parallel import parallel_stop from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash -from .utils import parallel_execute log = logging.getLogger(__name__) diff --git a/compose/utils.py b/compose/utils.py index 716f6633..362629bc 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -2,107 +2,13 @@ import codecs import hashlib import json import json.decoder -import logging -import sys -from threading import Thread import six -from docker.errors import APIError -from six.moves.queue import Empty -from six.moves.queue import Queue -log = logging.getLogger(__name__) - json_decoder = json.JSONDecoder() -def perform_operation(func, arg, callback, index): - try: - callback((index, func(arg))) - except Exception as e: - callback((index, e)) - - -def parallel_execute(objects, func, index_func, msg): - """For a given list of objects, call the callable passing in the first - object we give it. - """ - objects = list(objects) - stream = get_output_stream(sys.stdout) - writer = ParallelStreamWriter(stream, msg) - - for obj in objects: - writer.initialize(index_func(obj)) - - q = Queue() - - # TODO: limit the number of threads #1828 - for obj in objects: - t = Thread( - target=perform_operation, - args=(func, obj, q.put, index_func(obj))) - t.daemon = True - t.start() - - done = 0 - errors = {} - - while done < len(objects): - try: - msg_index, result = q.get(timeout=1) - except Empty: - continue - - if isinstance(result, APIError): - errors[msg_index] = "error", result.explanation - writer.write(msg_index, 'error') - elif isinstance(result, Exception): - errors[msg_index] = "unexpected_exception", result - else: - writer.write(msg_index, 'done') - done += 1 - - if not errors: - return - - stream.write("\n") - for msg_index, (result, error) in errors.items(): - stream.write("ERROR: for {} {} \n".format(msg_index, error)) - if result == 'unexpected_exception': - raise error - - -class ParallelStreamWriter(object): - """Write out messages for operations happening in parallel. - - Each operation has it's own line, and ANSI code characters are used - to jump to the correct line, and write over the line. - """ - - def __init__(self, stream, msg): - self.stream = stream - self.msg = msg - self.lines = [] - - def initialize(self, obj_index): - self.lines.append(obj_index) - self.stream.write("{} {} ... \r\n".format(self.msg, obj_index)) - self.stream.flush() - - def write(self, obj_index, status): - position = self.lines.index(obj_index) - diff = len(self.lines) - position - # move up - self.stream.write("%c[%dA" % (27, diff)) - # erase - self.stream.write("%c[2K\r" % 27) - self.stream.write("{} {} ... {}\r".format(self.msg, obj_index, status)) - # move back down - self.stream.write("%c[%dB" % (27, diff)) - self.stream.flush() - - def get_output_stream(stream): if six.PY3: return stream From c9ca5e86b0e8e0e1c8cd2345da5a55739b430242 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 17:39:02 -0500 Subject: [PATCH 128/359] Remove project name validation project name is already normalized to a valid name before creating a service. Signed-off-by: Daniel Nephin --- compose/service.py | 4 ---- tests/unit/service_test.py | 5 ----- 2 files changed, 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index dd2399ee..9004260f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -18,7 +18,6 @@ from docker.utils.ports import split_port from . import __version__ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment -from .config.validation import VALID_NAME_CHARS from .const import DEFAULT_TIMEOUT from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH @@ -122,9 +121,6 @@ class Service(object): net=None, **options ): - if not re.match('^%s+$' % VALID_NAME_CHARS, project): - raise ConfigError('Invalid project name "%s" - only %s are allowed' % (project, VALID_NAME_CHARS)) - self.name = name self.client = client self.project = project diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 808c391c..78edf3bf 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -35,11 +35,6 @@ class ServiceTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.Client) - def test_project_validation(self): - self.assertRaises(ConfigError, lambda: Service(name='foo', project='>', image='foo')) - - Service(name='foo', project='bar.bar__', image='foo') - def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') self.mock_client.containers.return_value = [] From 068edfa31345760d334b952d27e546077b077388 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 18:20:09 -0500 Subject: [PATCH 129/359] Move parsing of volumes_from to the last step of config parsing. Includes creating a new compose.config.types module for all the domain objects. Signed-off-by: Daniel Nephin --- compose/config/config.py | 19 +++++++++++++++++++ compose/config/types.py | 28 ++++++++++++++++++++++++++++ compose/project.py | 18 ++++++------------ compose/service.py | 19 +------------------ tests/integration/project_test.py | 2 +- tests/integration/service_test.py | 2 +- tests/unit/project_test.py | 23 +++++++++++++---------- tests/unit/service_test.py | 1 + tests/unit/sort_service_test.py | 7 ++++--- 9 files changed, 74 insertions(+), 45 deletions(-) create mode 100644 compose/config/types.py diff --git a/compose/config/config.py b/compose/config/config.py index 84b6748c..8ec352ec 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import codecs import logging import operator @@ -12,6 +14,7 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables +from .types import VolumeFromSpec from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extends_file_path @@ -198,8 +201,12 @@ def load(config_details): service_dict) resolver = ServiceExtendsResolver(service_config) service_dict = process_service(resolver.run()) + + # TODO: move to validate_service() validate_against_service_schema(service_dict, service_config.name) validate_paths(service_dict) + + service_dict = finalize_service(service_config._replace(config=service_dict)) service_dict['name'] = service_config.name return service_dict @@ -353,6 +360,7 @@ def validate_ulimits(ulimit_config): "than 'hard' value".format(ulimit_config)) +# TODO: rename to normalize_service def process_service(service_config): working_dir = service_config.working_dir service_dict = dict(service_config.config) @@ -370,12 +378,23 @@ def process_service(service_config): if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) + # TODO: move to a validate_service() if 'ulimits' in service_dict: validate_ulimits(service_dict['ulimits']) return service_dict +def finalize_service(service_config): + service_dict = dict(service_config.config) + + if 'volumes_from' in service_dict: + service_dict['volumes_from'] = [ + VolumeFromSpec.parse(vf) for vf in service_dict['volumes_from']] + + return service_dict + + def merge_service_dicts_from_files(base, override): """When merging services from multiple files we need to merge the `extends` field. This is not handled by `merge_service_dicts()` which is used to diff --git a/compose/config/types.py b/compose/config/types.py new file mode 100644 index 00000000..73bfd418 --- /dev/null +++ b/compose/config/types.py @@ -0,0 +1,28 @@ +""" +Types for objects parsed from the configuration. +""" +from __future__ import absolute_import +from __future__ import unicode_literals + +from collections import namedtuple + +from compose.config.errors import ConfigurationError + + +class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode')): + + @classmethod + def parse(cls, volume_from_config): + parts = volume_from_config.split(':') + if len(parts) > 2: + raise ConfigurationError( + "volume_from {} has incorrect format, should be " + "service[:mode]".format(volume_from_config)) + + if len(parts) == 1: + source = parts[0] + mode = 'rw' + else: + source, mode = parts + + return cls(source, mode) diff --git a/compose/project.py b/compose/project.py index e29a2eb5..5caa1ea3 100644 --- a/compose/project.py +++ b/compose/project.py @@ -19,10 +19,8 @@ from .legacy import check_for_legacy_containers from .service import ContainerNet from .service import ConvergenceStrategy from .service import Net -from .service import parse_volume_from_spec from .service import Service from .service import ServiceNet -from .service import VolumeFromSpec log = logging.getLogger(__name__) @@ -38,10 +36,7 @@ def sort_service_dicts(services): return [link.split(':')[0] for link in links] def get_service_names_from_volumes_from(volumes_from): - return [ - parse_volume_from_spec(volume_from).source - for volume_from in volumes_from - ] + return [volume_from.source for volume_from in volumes_from] def get_service_dependents(service_dict, services): name = service_dict['name'] @@ -192,16 +187,15 @@ class Project(object): def get_volumes_from(self, service_dict): volumes_from = [] if 'volumes_from' in service_dict: - for volume_from_config in service_dict.get('volumes_from', []): - volume_from_spec = parse_volume_from_spec(volume_from_config) + for volume_from_spec in service_dict.get('volumes_from', []): # Get service try: - service_name = self.get_service(volume_from_spec.source) - volume_from_spec = VolumeFromSpec(service_name, volume_from_spec.mode) + service = self.get_service(volume_from_spec.source) + volume_from_spec = volume_from_spec._replace(source=service) except NoSuchService: try: - container_name = Container.from_id(self.client, volume_from_spec.source) - volume_from_spec = VolumeFromSpec(container_name, volume_from_spec.mode) + container = Container.from_id(self.client, volume_from_spec.source) + volume_from_spec = volume_from_spec._replace(source=container) except APIError: raise ConfigurationError( 'Service "%s" mounts volumes from "%s", which is ' diff --git a/compose/service.py b/compose/service.py index 9004260f..be0502c2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -70,6 +70,7 @@ class BuildError(Exception): self.reason = reason +# TODO: remove class ConfigError(ValueError): pass @@ -86,9 +87,6 @@ class NoSuchImageError(Exception): VolumeSpec = namedtuple('VolumeSpec', 'external internal mode') -VolumeFromSpec = namedtuple('VolumeFromSpec', 'source mode') - - ServiceName = namedtuple('ServiceName', 'project service number') @@ -1029,21 +1027,6 @@ def build_volume_from(volume_from_spec): return ["{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode)] -def parse_volume_from_spec(volume_from_config): - parts = volume_from_config.split(':') - if len(parts) > 2: - raise ConfigError("Volume %s has incorrect format, should be " - "external:internal[:mode]" % volume_from_config) - - if len(parts) == 1: - source = parts[0] - mode = 'rw' - else: - source, mode = parts - - return VolumeFromSpec(source, mode) - - # Labels diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 2ce31900..d65d7ef0 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -3,12 +3,12 @@ from __future__ import unicode_literals from .testcases import DockerClientTestCase from compose.cli.docker_client import docker_client from compose.config import config +from compose.config.types import VolumeFromSpec from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project from compose.service import ConvergenceStrategy from compose.service import Net -from compose.service import VolumeFromSpec def build_service_dicts(service_config): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 34869ab8..34bf93fc 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -14,6 +14,7 @@ from .. import mock from .testcases import DockerClientTestCase from .testcases import pull_busybox from compose import __version__ +from compose.config.types import VolumeFromSpec from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF @@ -27,7 +28,6 @@ from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy from compose.service import Net from compose.service import Service -from compose.service import VolumeFromSpec def create_and_start_container(service, **override_options): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index b38f5c78..f8178ed8 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -4,6 +4,7 @@ import docker from .. import mock from .. import unittest +from compose.config.types import VolumeFromSpec from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project @@ -43,7 +44,7 @@ class ProjectTest(unittest.TestCase): { 'name': 'db', 'image': 'busybox:latest', - 'volumes_from': ['volume'] + 'volumes_from': [VolumeFromSpec('volume', 'ro')] }, { 'name': 'volume', @@ -167,7 +168,7 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'volumes_from': ['aaa'] + 'volumes_from': [VolumeFromSpec('aaa', 'rw')] } ], self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"]) @@ -190,17 +191,13 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'volumes_from': ['vol'] + 'volumes_from': [VolumeFromSpec('vol', 'rw')] } ], self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"]) - @mock.patch.object(Service, 'containers') - def test_use_volumes_from_service_container(self, mock_return): + def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] - mock_return.return_value = [ - mock.Mock(id=container_id, spec=Container) - for container_id in container_ids] project = Project.from_dicts('test', [ { @@ -210,10 +207,16 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'volumes_from': ['vol'] + 'volumes_from': [VolumeFromSpec('vol', 'rw')] } ], None) - self.assertEqual(project.get_service('test')._get_volumes_from(), [container_ids[0] + ':rw']) + with mock.patch.object(Service, 'containers') as mock_return: + mock_return.return_value = [ + mock.Mock(id=container_id, spec=Container) + for container_id in container_ids] + self.assertEqual( + project.get_service('test')._get_volumes_from(), + [container_ids[0] + ':rw']) def test_net_unset(self): project = Project.from_dicts('test', [ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 78edf3bf..efcc58e2 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -6,6 +6,7 @@ import pytest from .. import mock from .. import unittest +from compose.config.types import VolumeFromSpec from compose.const import IS_WINDOWS_PLATFORM from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF diff --git a/tests/unit/sort_service_test.py b/tests/unit/sort_service_test.py index a7e522a1..ef088287 100644 --- a/tests/unit/sort_service_test.py +++ b/tests/unit/sort_service_test.py @@ -1,4 +1,5 @@ from .. import unittest +from compose.config.types import VolumeFromSpec from compose.project import DependencyError from compose.project import sort_service_dicts @@ -73,7 +74,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'parent', - 'volumes_from': ['child'] + 'volumes_from': [VolumeFromSpec('child', 'rw')] }, { 'links': ['parent'], @@ -116,7 +117,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'parent', - 'volumes_from': ['child'] + 'volumes_from': [VolumeFromSpec('child', 'ro')] }, { 'name': 'child' @@ -141,7 +142,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'two', - 'volumes_from': ['one'] + 'volumes_from': [VolumeFromSpec('one', 'rw')] }, { 'name': 'one' From 12b82a20ff331f420c040dbb9cf1ea44fd74d7d5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 18:29:25 -0500 Subject: [PATCH 130/359] Move restart spec to the config.types module. Signed-off-by: Daniel Nephin --- compose/config/config.py | 4 ++++ compose/config/types.py | 17 +++++++++++++++++ compose/service.py | 22 +--------------------- tests/integration/service_test.py | 24 ++++++------------------ tests/unit/cli_test.py | 2 +- 5 files changed, 29 insertions(+), 40 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8ec352ec..9b03ea4f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -14,6 +14,7 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables +from .types import parse_restart_spec from .types import VolumeFromSpec from .validation import validate_against_fields_schema from .validation import validate_against_service_schema @@ -392,6 +393,9 @@ def finalize_service(service_config): service_dict['volumes_from'] = [ VolumeFromSpec.parse(vf) for vf in service_dict['volumes_from']] + if 'restart' in service_dict: + service_dict['restart'] = parse_restart_spec(service_dict['restart']) + return service_dict diff --git a/compose/config/types.py b/compose/config/types.py index 73bfd418..0ab53c82 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -26,3 +26,20 @@ class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode')): source, mode = parts return cls(source, mode) + + +def parse_restart_spec(restart_config): + if not restart_config: + return None + parts = restart_config.split(':') + if len(parts) > 2: + raise ConfigurationError( + "Restart %s has incorrect format, should be " + "mode[:max_retry]" % restart_config) + if len(parts) == 2: + name, max_retry_count = parts + else: + name, = parts + max_retry_count = 0 + + return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} diff --git a/compose/service.py b/compose/service.py index be0502c2..33d9a7be 100644 --- a/compose/service.py +++ b/compose/service.py @@ -648,8 +648,6 @@ class Service(object): if isinstance(dns_search, six.string_types): dns_search = [dns_search] - restart = parse_restart_spec(options.get('restart', None)) - extra_hosts = build_extra_hosts(options.get('extra_hosts', None)) read_only = options.get('read_only', None) @@ -667,7 +665,7 @@ class Service(object): devices=devices, dns=dns, dns_search=dns_search, - restart_policy=restart, + restart_policy=options.get('restart'), cap_add=cap_add, cap_drop=cap_drop, mem_limit=options.get('mem_limit'), @@ -1043,24 +1041,6 @@ def build_container_labels(label_options, service_labels, number, config_hash): return labels -# Restart policy - - -def parse_restart_spec(restart_config): - if not restart_config: - return None - parts = restart_config.split(':') - if len(parts) > 2: - raise ConfigError("Restart %s has incorrect format, should be " - "mode[:max_retry]" % restart_config) - if len(parts) == 2: - name, max_retry_count = parts - else: - name, = parts - max_retry_count = 0 - - return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} - # Ulimits diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 34bf93fc..15d8ca07 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -786,23 +786,21 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertIsNone(container.get('HostConfig.Dns')) - def test_dns_single_value(self): - service = self.create_service('web', dns='8.8.8.8') - container = create_and_start_container(service) - self.assertEqual(container.get('HostConfig.Dns'), ['8.8.8.8']) - def test_dns_list(self): service = self.create_service('web', dns=['8.8.8.8', '9.9.9.9']) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.Dns'), ['8.8.8.8', '9.9.9.9']) def test_restart_always_value(self): - service = self.create_service('web', restart='always') + service = self.create_service('web', restart={'Name': 'always'}) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.RestartPolicy.Name'), 'always') def test_restart_on_failure_value(self): - service = self.create_service('web', restart='on-failure:5') + service = self.create_service('web', restart={ + 'Name': 'on-failure', + 'MaximumRetryCount': 5 + }) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.RestartPolicy.Name'), 'on-failure') self.assertEqual(container.get('HostConfig.RestartPolicy.MaximumRetryCount'), 5) @@ -817,17 +815,7 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.CapDrop'), ['SYS_ADMIN', 'NET_ADMIN']) - def test_dns_search_no_value(self): - service = self.create_service('web') - container = create_and_start_container(service) - self.assertIsNone(container.get('HostConfig.DnsSearch')) - - def test_dns_search_single_value(self): - service = self.create_service('web', dns_search='example.com') - container = create_and_start_container(service) - self.assertEqual(container.get('HostConfig.DnsSearch'), ['example.com']) - - def test_dns_search_list(self): + def test_dns_search(self): service = self.create_service('web', dns_search=['dc1.example.com', 'dc2.example.com']) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.DnsSearch'), ['dc1.example.com', 'dc2.example.com']) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 5b63d2e8..23dc4262 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -124,7 +124,7 @@ class CLITestCase(unittest.TestCase): mock_project.get_service.return_value = Service( 'service', client=mock_client, - restart='always', + restart={'Name': 'always', 'MaximumRetryCount': 0}, image='someimage') command.run(mock_project, { 'SERVICE': 'service', From efec2aae6c86b6577f30451d54ac7030dbf39b13 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 18:58:24 -0500 Subject: [PATCH 131/359] Fixes #2008 - re-use list_or_dict schema for all the types At the same time, moves extra_hosts validation to the config module. Signed-off-by: Daniel Nephin --- compose/config/config.py | 4 ++++ compose/config/fields_schema.json | 31 +++++++++++--------------- compose/config/types.py | 16 ++++++++++++++ compose/config/validation.py | 4 ++-- compose/service.py | 36 +++---------------------------- tests/integration/service_test.py | 33 ---------------------------- tests/unit/config/config_test.py | 25 ++++++++++++++++++++- tests/unit/config/types_test.py | 29 +++++++++++++++++++++++++ 8 files changed, 90 insertions(+), 88 deletions(-) create mode 100644 tests/unit/config/types_test.py diff --git a/compose/config/config.py b/compose/config/config.py index 9b03ea4f..55adcaf2 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -14,6 +14,7 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables +from .types import parse_extra_hosts from .types import parse_restart_spec from .types import VolumeFromSpec from .validation import validate_against_fields_schema @@ -379,6 +380,9 @@ def process_service(service_config): if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) + if 'extra_hosts' in service_dict: + service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) + # TODO: move to a validate_service() if 'ulimits' in service_dict: validate_ulimits(service_dict['ulimits']) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index ca3b3a50..9cbcfd1b 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -37,22 +37,7 @@ "domainname": {"type": "string"}, "entrypoint": {"$ref": "#/definitions/string_or_list"}, "env_file": {"$ref": "#/definitions/string_or_list"}, - - "environment": { - "oneOf": [ - { - "type": "object", - "patternProperties": { - ".+": { - "type": ["string", "number", "boolean", "null"], - "format": "environment" - } - }, - "additionalProperties": false - }, - {"type": "array", "items": {"type": "string"}, "uniqueItems": true} - ] - }, + "environment": {"$ref": "#/definitions/list_or_dict"}, "expose": { "type": "array", @@ -165,10 +150,18 @@ "list_or_dict": { "oneOf": [ - {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - {"type": "object"} + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "boolean", "null"], + "format": "bool-value-in-mapping" + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} ] } - } } diff --git a/compose/config/types.py b/compose/config/types.py index 0ab53c82..b6add089 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -43,3 +43,19 @@ def parse_restart_spec(restart_config): max_retry_count = 0 return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} + + +def parse_extra_hosts(extra_hosts_config): + if not extra_hosts_config: + return {} + + if isinstance(extra_hosts_config, dict): + return dict(extra_hosts_config) + + if isinstance(extra_hosts_config, list): + extra_hosts_dict = {} + for extra_hosts_line in extra_hosts_config: + # TODO: validate string contains ':' ? + host, ip = extra_hosts_line.split(':') + extra_hosts_dict[host.strip()] = ip.strip() + return extra_hosts_dict diff --git a/compose/config/validation.py b/compose/config/validation.py index 38866b0f..38020366 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -49,7 +49,7 @@ def format_ports(instance): return True -@FormatChecker.cls_checks(format="environment") +@FormatChecker.cls_checks(format="bool-value-in-mapping") def format_boolean_in_environment(instance): """ Check if there is a boolean in the environment and display a warning. @@ -273,7 +273,7 @@ def validate_against_fields_schema(config, filename): _validate_against_schema( config, "fields_schema.json", - format_checker=["ports", "environment"], + format_checker=["ports", "bool-value-in-mapping"], filename=filename) diff --git a/compose/service.py b/compose/service.py index 33d9a7be..2bb0030f 100644 --- a/compose/service.py +++ b/compose/service.py @@ -640,6 +640,7 @@ class Service(object): pid = options.get('pid', None) security_opt = options.get('security_opt', None) + # TODO: these options are already normalized by config dns = options.get('dns', None) if isinstance(dns, six.string_types): dns = [dns] @@ -648,9 +649,6 @@ class Service(object): if isinstance(dns_search, six.string_types): dns_search = [dns_search] - extra_hosts = build_extra_hosts(options.get('extra_hosts', None)) - read_only = options.get('read_only', None) - devices = options.get('devices', None) cgroup_parent = options.get('cgroup_parent', None) ulimits = build_ulimits(options.get('ulimits', None)) @@ -672,8 +670,8 @@ class Service(object): memswap_limit=options.get('memswap_limit'), ulimits=ulimits, log_config=log_config, - extra_hosts=extra_hosts, - read_only=read_only, + extra_hosts=options.get('extra_hosts'), + read_only=options.get('read_only'), pid_mode=pid, security_opt=security_opt, ipc_mode=options.get('ipc'), @@ -1057,31 +1055,3 @@ def build_ulimits(ulimit_config): ulimits.append(ulimit_dict) return ulimits - - -# Extra hosts - - -def build_extra_hosts(extra_hosts_config): - if not extra_hosts_config: - return {} - - if isinstance(extra_hosts_config, list): - extra_hosts_dict = {} - for extra_hosts_line in extra_hosts_config: - if not isinstance(extra_hosts_line, six.string_types): - raise ConfigError( - "extra_hosts_config \"%s\" must be either a list of strings or a string->string mapping," % - extra_hosts_config - ) - host, ip = extra_hosts_line.split(':') - extra_hosts_dict.update({host.strip(): ip.strip()}) - extra_hosts_config = extra_hosts_dict - - if isinstance(extra_hosts_config, dict): - return extra_hosts_config - - raise ConfigError( - "extra_hosts_config \"%s\" must be either a list of strings or a string->string mapping," % - extra_hosts_config - ) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 15d8ca07..27a29005 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -22,8 +22,6 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container -from compose.service import build_extra_hosts -from compose.service import ConfigError from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy from compose.service import Net @@ -139,37 +137,6 @@ class ServiceTest(DockerClientTestCase): container.start() self.assertEqual(container.get('HostConfig.CpuShares'), 73) - def test_build_extra_hosts(self): - # string - self.assertRaises(ConfigError, lambda: build_extra_hosts("www.example.com: 192.168.0.17")) - - # list of strings - self.assertEqual(build_extra_hosts( - ["www.example.com:192.168.0.17"]), - {'www.example.com': '192.168.0.17'}) - self.assertEqual(build_extra_hosts( - ["www.example.com: 192.168.0.17"]), - {'www.example.com': '192.168.0.17'}) - self.assertEqual(build_extra_hosts( - ["www.example.com: 192.168.0.17", - "static.example.com:192.168.0.19", - "api.example.com: 192.168.0.18"]), - {'www.example.com': '192.168.0.17', - 'static.example.com': '192.168.0.19', - 'api.example.com': '192.168.0.18'}) - - # list of dictionaries - self.assertRaises(ConfigError, lambda: build_extra_hosts( - [{'www.example.com': '192.168.0.17'}, - {'api.example.com': '192.168.0.18'}])) - - # dictionaries - self.assertEqual(build_extra_hosts( - {'www.example.com': '192.168.0.17', - 'api.example.com': '192.168.0.18'}), - {'www.example.com': '192.168.0.17', - 'api.example.com': '192.168.0.18'}) - def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index c69e3430..f923fb37 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -32,7 +32,7 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) -def build_config_details(contents, working_dir, filename): +def build_config_details(contents, working_dir='working_dir', filename='filename.yml'): return config.ConfigDetails( working_dir, [config.ConfigFile(filename, contents)]) @@ -512,6 +512,29 @@ class ConfigTest(unittest.TestCase): assert 'line 3, column 32' in exc.exconly() + def test_validate_extra_hosts_invalid(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details({ + 'web': { + 'image': 'alpine', + 'extra_hosts': "www.example.com: 192.168.0.17", + } + })) + assert "'extra_hosts' contains an invalid type" in exc.exconly() + + def test_validate_extra_hosts_invalid_list(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details({ + 'web': { + 'image': 'alpine', + 'extra_hosts': [ + {'www.example.com': '192.168.0.17'}, + {'api.example.com': '192.168.0.18'} + ], + } + })) + assert "which is an invalid type" in exc.exconly() + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py new file mode 100644 index 00000000..25692ca3 --- /dev/null +++ b/tests/unit/config/types_test.py @@ -0,0 +1,29 @@ +from compose.config.types import parse_extra_hosts + + +def test_parse_extra_hosts_list(): + expected = {'www.example.com': '192.168.0.17'} + assert parse_extra_hosts(["www.example.com:192.168.0.17"]) == expected + + expected = {'www.example.com': '192.168.0.17'} + assert parse_extra_hosts(["www.example.com: 192.168.0.17"]) == expected + + assert parse_extra_hosts([ + "www.example.com: 192.168.0.17", + "static.example.com:192.168.0.19", + "api.example.com: 192.168.0.18" + ]) == { + 'www.example.com': '192.168.0.17', + 'static.example.com': '192.168.0.19', + 'api.example.com': '192.168.0.18' + } + + +def test_parse_extra_hosts_dict(): + assert parse_extra_hosts({ + 'www.example.com': '192.168.0.17', + 'api.example.com': '192.168.0.18' + }) == { + 'www.example.com': '192.168.0.17', + 'api.example.com': '192.168.0.18' + } From dac75b07dc16d9b6f08654301dd9ddd5d0b48393 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 19:40:10 -0500 Subject: [PATCH 132/359] Move volume parsing to config.types module This removes the last of the old service.ConfigError Signed-off-by: Daniel Nephin --- compose/cli/command.py | 17 ++---- compose/config/config.py | 5 ++ compose/config/types.py | 59 +++++++++++++++++++ compose/service.py | 69 ++--------------------- tests/acceptance/cli_test.py | 2 +- tests/integration/project_test.py | 11 ++-- tests/integration/resilience_test.py | 6 +- tests/integration/service_test.py | 43 ++++++-------- tests/integration/testcases.py | 11 ++-- tests/unit/config/config_test.py | 38 +++++++------ tests/unit/config/types_test.py | 37 ++++++++++++ tests/unit/service_test.py | 84 +++++++--------------------- 12 files changed, 186 insertions(+), 196 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 6094b530..157e0016 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -14,7 +14,6 @@ from . import errors from . import verbose_proxy from .. import config from ..project import Project -from ..service import ConfigError from .docker_client import docker_client from .utils import call_silently from .utils import get_version_info @@ -84,16 +83,12 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(base_dir, config_path) api_version = '1.21' if use_networking else None - try: - return Project.from_dicts( - get_project_name(config_details.working_dir, project_name), - config.load(config_details), - get_client(verbose=verbose, version=api_version), - use_networking=use_networking, - network_driver=network_driver, - ) - except ConfigError as e: - raise errors.UserError(six.text_type(e)) + return Project.from_dicts( + get_project_name(config_details.working_dir, project_name), + config.load(config_details), + get_client(verbose=verbose, version=api_version), + use_networking=use_networking, + network_driver=network_driver) def get_project_name(working_dir, project_name=None): diff --git a/compose/config/config.py b/compose/config/config.py index 55adcaf2..5b1de5ef 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -17,6 +17,7 @@ from .interpolation import interpolate_environment_variables from .types import parse_extra_hosts from .types import parse_restart_spec from .types import VolumeFromSpec +from .types import VolumeSpec from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extends_file_path @@ -397,6 +398,10 @@ def finalize_service(service_config): service_dict['volumes_from'] = [ VolumeFromSpec.parse(vf) for vf in service_dict['volumes_from']] + if 'volumes' in service_dict: + service_dict['volumes'] = [ + VolumeSpec.parse(v) for v in service_dict['volumes']] + if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) diff --git a/compose/config/types.py b/compose/config/types.py index b6add089..cec1f6cf 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -4,9 +4,11 @@ Types for objects parsed from the configuration. from __future__ import absolute_import from __future__ import unicode_literals +import os from collections import namedtuple from compose.config.errors import ConfigurationError +from compose.const import IS_WINDOWS_PLATFORM class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode')): @@ -59,3 +61,60 @@ def parse_extra_hosts(extra_hosts_config): host, ip = extra_hosts_line.split(':') extra_hosts_dict[host.strip()] = ip.strip() return extra_hosts_dict + + +def normalize_paths_for_engine(external_path, internal_path): + """Windows paths, c:\my\path\shiny, need to be changed to be compatible with + the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ + """ + if not IS_WINDOWS_PLATFORM: + return external_path, internal_path + + if external_path: + drive, tail = os.path.splitdrive(external_path) + + if drive: + external_path = '/' + drive.lower().rstrip(':') + tail + + external_path = external_path.replace('\\', '/') + + return external_path, internal_path.replace('\\', '/') + + +class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): + + @classmethod + def parse(cls, volume_config): + """Parse a volume_config path and split it into external:internal[:mode] + parts to be returned as a valid VolumeSpec. + """ + if IS_WINDOWS_PLATFORM: + # relative paths in windows expand to include the drive, eg C:\ + # so we join the first 2 parts back together to count as one + drive, tail = os.path.splitdrive(volume_config) + parts = tail.split(":") + + if drive: + parts[0] = drive + parts[0] + else: + parts = volume_config.split(':') + + if len(parts) > 3: + raise ConfigurationError( + "Volume %s has incorrect format, should be " + "external:internal[:mode]" % volume_config) + + if len(parts) == 1: + external, internal = normalize_paths_for_engine( + None, + os.path.normpath(parts[0])) + else: + external, internal = normalize_paths_for_engine( + os.path.normpath(parts[0]), + os.path.normpath(parts[1])) + + mode = 'rw' + if len(parts) == 3: + mode = parts[2] + + return cls(external, internal, mode) diff --git a/compose/service.py b/compose/service.py index 2bb0030f..6340d074 100644 --- a/compose/service.py +++ b/compose/service.py @@ -2,7 +2,6 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -import os import re import sys from collections import namedtuple @@ -18,8 +17,8 @@ from docker.utils.ports import split_port from . import __version__ from .config import DOCKER_CONFIG_KEYS from .config import merge_environment +from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT -from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF @@ -70,11 +69,6 @@ class BuildError(Exception): self.reason = reason -# TODO: remove -class ConfigError(ValueError): - pass - - class NeedsBuildError(Exception): def __init__(self, service): self.service = service @@ -84,9 +78,6 @@ class NoSuchImageError(Exception): pass -VolumeSpec = namedtuple('VolumeSpec', 'external internal mode') - - ServiceName = namedtuple('ServiceName', 'project service number') @@ -598,8 +589,7 @@ class Service(object): if 'volumes' in container_options: container_options['volumes'] = dict( - (parse_volume_spec(v).internal, {}) - for v in container_options['volumes']) + (v.internal, {}) for v in container_options['volumes']) container_options['environment'] = merge_environment( self.options.get('environment'), @@ -884,11 +874,10 @@ def parse_repository_tag(repo_path): # Volumes -def merge_volume_bindings(volumes_option, previous_container): +def merge_volume_bindings(volumes, previous_container): """Return a list of volume bindings for a container. Container data volumes are replaced by those from the previous container. """ - volumes = [parse_volume_spec(volume) for volume in volumes_option or []] volume_bindings = dict( build_volume_binding(volume) for volume in volumes @@ -910,7 +899,7 @@ def get_container_data_volumes(container, volumes_option): volumes = [] container_volumes = container.get('Volumes') or {} image_volumes = [ - parse_volume_spec(volume) + VolumeSpec.parse(volume) for volume in container.image_config['ContainerConfig'].get('Volumes') or {} ] @@ -957,56 +946,6 @@ def build_volume_binding(volume_spec): return volume_spec.internal, "{}:{}:{}".format(*volume_spec) -def normalize_paths_for_engine(external_path, internal_path): - """Windows paths, c:\my\path\shiny, need to be changed to be compatible with - the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ - """ - if not IS_WINDOWS_PLATFORM: - return external_path, internal_path - - if external_path: - drive, tail = os.path.splitdrive(external_path) - - if drive: - external_path = '/' + drive.lower().rstrip(':') + tail - - external_path = external_path.replace('\\', '/') - - return external_path, internal_path.replace('\\', '/') - - -def parse_volume_spec(volume_config): - """ - Parse a volume_config path and split it into external:internal[:mode] - parts to be returned as a valid VolumeSpec. - """ - if IS_WINDOWS_PLATFORM: - # relative paths in windows expand to include the drive, eg C:\ - # so we join the first 2 parts back together to count as one - drive, tail = os.path.splitdrive(volume_config) - parts = tail.split(":") - - if drive: - parts[0] = drive + parts[0] - else: - parts = volume_config.split(':') - - if len(parts) > 3: - raise ConfigError("Volume %s has incorrect format, should be " - "external:internal[:mode]" % volume_config) - - if len(parts) == 1: - external, internal = normalize_paths_for_engine(None, os.path.normpath(parts[0])) - else: - external, internal = normalize_paths_for_engine(os.path.normpath(parts[0]), os.path.normpath(parts[1])) - - mode = 'rw' - if len(parts) == 3: - mode = parts[2] - - return VolumeSpec(external, internal, mode) - - def build_volume_from(volume_from_spec): """ volume_from can be either a service or a container. We want to return the diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 7ca6e819..73a1d66c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -37,7 +37,7 @@ def start_process(base_dir, options): def wait_on_process(proc, returncode=0): stdout, stderr = proc.communicate() if proc.returncode != returncode: - print(stderr) + print(stderr.decode('utf-8')) assert proc.returncode == returncode return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d65d7ef0..443ff978 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -4,6 +4,7 @@ from .testcases import DockerClientTestCase from compose.cli.docker_client import docker_client from compose.config import config from compose.config.types import VolumeFromSpec +from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project @@ -214,7 +215,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up(self): web = self.create_service('web') - db = self.create_service('db', volumes=['/var/db']) + db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) project = Project('composetest', [web, db], self.client) project.start() self.assertEqual(len(project.containers()), 0) @@ -238,7 +239,7 @@ class ProjectTest(DockerClientTestCase): def test_recreate_preserves_volumes(self): web = self.create_service('web') - db = self.create_service('db', volumes=['/etc']) + db = self.create_service('db', volumes=[VolumeSpec.parse('/etc')]) project = Project('composetest', [web, db], self.client) project.start() self.assertEqual(len(project.containers()), 0) @@ -257,7 +258,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_with_no_recreate_running(self): web = self.create_service('web') - db = self.create_service('db', volumes=['/var/db']) + db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) project = Project('composetest', [web, db], self.client) project.start() self.assertEqual(len(project.containers()), 0) @@ -277,7 +278,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_with_no_recreate_stopped(self): web = self.create_service('web') - db = self.create_service('db', volumes=['/var/db']) + db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) project = Project('composetest', [web, db], self.client) project.start() self.assertEqual(len(project.containers()), 0) @@ -316,7 +317,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_starts_links(self): console = self.create_service('console') - db = self.create_service('db', volumes=['/var/db']) + db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) web = self.create_service('web', links=[(db, 'db')]) project = Project('composetest', [web, db, console], self.client) diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 53aedfec..7f75356d 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -3,13 +3,17 @@ from __future__ import unicode_literals from .. import mock from .testcases import DockerClientTestCase +from compose.config.types import VolumeSpec from compose.project import Project from compose.service import ConvergenceStrategy class ResilienceTest(DockerClientTestCase): def setUp(self): - self.db = self.create_service('db', volumes=['/var/db'], command='top') + self.db = self.create_service( + 'db', + volumes=[VolumeSpec.parse('/var/db')], + command='top') self.project = Project('composetest', [self.db], self.client) container = self.db.create_container() diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 27a29005..5dd3d2e6 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -15,6 +15,7 @@ from .testcases import DockerClientTestCase from .testcases import pull_busybox from compose import __version__ from compose.config.types import VolumeFromSpec +from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_CONTAINER_NUMBER from compose.const import LABEL_ONE_OFF @@ -120,7 +121,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(container.name, 'composetest_db_run_1') def test_create_container_with_unspecified_volume(self): - service = self.create_service('db', volumes=['/var/db']) + service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) container = service.create_container() container.start() self.assertIn('/var/db', container.get('Volumes')) @@ -182,7 +183,9 @@ class ServiceTest(DockerClientTestCase): host_path = '/tmp/host-path' container_path = '/container-path' - service = self.create_service('db', volumes=['%s:%s' % (host_path, container_path)]) + service = self.create_service( + 'db', + volumes=[VolumeSpec(host_path, container_path, 'rw')]) container = service.create_container() container.start() @@ -195,11 +198,10 @@ class ServiceTest(DockerClientTestCase): msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) def test_recreate_preserves_volume_with_trailing_slash(self): - """ - When the Compose file specifies a trailing slash in the container path, make + """When the Compose file specifies a trailing slash in the container path, make sure we copy the volume over when recreating. """ - service = self.create_service('data', volumes=['/data/']) + service = self.create_service('data', volumes=[VolumeSpec.parse('/data/')]) old_container = create_and_start_container(service) volume_path = old_container.get('Volumes')['/data'] @@ -213,7 +215,7 @@ class ServiceTest(DockerClientTestCase): """ host_path = '/tmp/data' container_path = '/data' - volumes = ['{}:{}/'.format(host_path, container_path)] + volumes = [VolumeSpec.parse('{}:{}/'.format(host_path, container_path))] tmp_container = self.client.create_container( 'busybox', 'true', @@ -267,7 +269,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service( 'db', environment={'FOO': '1'}, - volumes=['/etc'], + volumes=[VolumeSpec.parse('/etc')], entrypoint=['top'], command=['-d', '1'] ) @@ -305,7 +307,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service( 'db', environment={'FOO': '1'}, - volumes=['/var/db'], + volumes=[VolumeSpec.parse('/var/db')], entrypoint=['top'], command=['-d', '1'] ) @@ -343,10 +345,8 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(new_container.get('Volumes')['/data'], volume_path) def test_execute_convergence_plan_when_image_volume_masks_config(self): - service = Service( - project='composetest', - name='db', - client=self.client, + service = self.create_service( + 'db', build='tests/fixtures/dockerfile-with-volume', ) @@ -354,7 +354,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) volume_path = old_container.get('Volumes')['/data'] - service.options['volumes'] = ['/tmp:/data'] + service.options['volumes'] = [VolumeSpec.parse('/tmp:/data')] with mock.patch('compose.service.log') as mock_log: new_container, = service.execute_convergence_plan( @@ -864,22 +864,11 @@ class ServiceTest(DockerClientTestCase): for pair in expected.items(): self.assertIn(pair, labels) - service.kill() - remove_stopped(service) - - labels_list = ["%s=%s" % pair for pair in labels_dict.items()] - - service = self.create_service('web', labels=labels_list) - labels = create_and_start_container(service).labels.items() - for pair in expected.items(): - self.assertIn(pair, labels) - def test_empty_labels(self): - labels_list = ['foo', 'bar'] - - service = self.create_service('web', labels=labels_list) + labels_dict = {'foo': '', 'bar': ''} + service = self.create_service('web', labels=labels_dict) labels = create_and_start_container(service).labels.items() - for name in labels_list: + for name in labels_dict: self.assertIn((name, ''), labels) def test_custom_container_name(self): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index de2d1a70..f5de50ee 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -7,9 +7,7 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client -from compose.config.config import process_service from compose.config.config import resolve_environment -from compose.config.config import ServiceConfig from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -45,13 +43,12 @@ class DockerClientTestCase(unittest.TestCase): kwargs['command'] = ["top"] service_config = ServiceConfig('.', None, name, kwargs) - options = process_service(service_config) - options['environment'] = resolve_environment( - service_config._replace(config=options)) - labels = options.setdefault('labels', {}) + kwargs['environment'] = resolve_environment(service_config) + + labels = dict(kwargs.setdefault('labels', {})) labels['com.docker.compose.test-name'] = self.id() - return Service(name, client=self.client, project='composetest', **options) + return Service(name, client=self.client, project='composetest', **kwargs) def check_build(self, *args, **kwargs): kwargs.setdefault('rm', True) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f923fb37..b2a4cd68 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -11,6 +11,7 @@ import pytest from compose.config import config from compose.config.errors import ConfigurationError +from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest @@ -147,7 +148,7 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'build': '/', 'links': ['db'], - 'volumes': ['/home/user/project:/code'], + 'volumes': [VolumeSpec.parse('/home/user/project:/code')], }, { 'name': 'db', @@ -211,7 +212,7 @@ class ConfigTest(unittest.TestCase): { 'name': 'web', 'image': 'example/web', - 'volumes': ['/home/user/project:/code'], + 'volumes': [VolumeSpec.parse('/home/user/project:/code')], 'labels': {'label': 'one'}, }, ] @@ -626,14 +627,11 @@ class VolumeConfigTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' - d = config.load( - build_config_details( - {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - '.', - None, - ) - )[0] - self.assertEqual(d['volumes'], ['/host/path:/container/path']) + d = config.load(build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + ))[0] + self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) @@ -1031,19 +1029,21 @@ class EnvTest(unittest.TestCase): build_config_details( {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, "tests/fixtures/env", - None, ) )[0] - self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) + self.assertEqual( + set(service_dict['volumes']), + set([VolumeSpec.parse('/tmp:/host/tmp')])) service_dict = config.load( build_config_details( {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, "tests/fixtures/env", - None, ) )[0] - self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) + self.assertEqual( + set(service_dict['volumes']), + set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')])) def load_from_filename(filename): @@ -1290,8 +1290,14 @@ class ExtendsTest(unittest.TestCase): dicts = load_from_filename('tests/fixtures/volume-path/docker-compose.yml') paths = [ - '%s:/foo' % os.path.abspath('tests/fixtures/volume-path/common/foo'), - '%s:/bar' % os.path.abspath('tests/fixtures/volume-path/bar'), + VolumeSpec( + os.path.abspath('tests/fixtures/volume-path/common/foo'), + '/foo', + 'rw'), + VolumeSpec( + os.path.abspath('tests/fixtures/volume-path/bar'), + '/bar', + 'rw') ] self.assertEqual(set(dicts[0]['volumes']), set(paths)) diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 25692ca3..4df66548 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -1,4 +1,9 @@ +import pytest + +from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts +from compose.config.types import VolumeSpec +from compose.const import IS_WINDOWS_PLATFORM def test_parse_extra_hosts_list(): @@ -27,3 +32,35 @@ def test_parse_extra_hosts_dict(): 'www.example.com': '192.168.0.17', 'api.example.com': '192.168.0.18' } + + +class TestVolumeSpec(object): + + def test_parse_volume_spec_only_one_path(self): + spec = VolumeSpec.parse('/the/volume') + assert spec == (None, '/the/volume', 'rw') + + def test_parse_volume_spec_internal_and_external(self): + spec = VolumeSpec.parse('external:interval') + assert spec == ('external', 'interval', 'rw') + + def test_parse_volume_spec_with_mode(self): + spec = VolumeSpec.parse('external:interval:ro') + assert spec == ('external', 'interval', 'ro') + + spec = VolumeSpec.parse('external:interval:z') + assert spec == ('external', 'interval', 'z') + + def test_parse_volume_spec_too_many_parts(self): + with pytest.raises(ConfigurationError) as exc: + VolumeSpec.parse('one:two:three:four') + assert 'has incorrect format' in exc.exconly() + + @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') + def test_parse_volume_windows_absolute_path(self): + windows_path = "c:\\Users\\me\\Documents\\shiny\\config:\\opt\\shiny\\config:ro" + assert VolumeSpec.parse(windows_path) == ( + "/c/Users/me/Documents/shiny/config", + "/opt/shiny/config", + "ro" + ) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index efcc58e2..a439f0da 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,12 +2,11 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker -import pytest from .. import mock from .. import unittest from compose.config.types import VolumeFromSpec -from compose.const import IS_WINDOWS_PLATFORM +from compose.config.types import VolumeSpec from compose.const import LABEL_CONFIG_HASH from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT @@ -15,7 +14,6 @@ from compose.const import LABEL_SERVICE from compose.container import Container from compose.service import build_ulimits from compose.service import build_volume_binding -from compose.service import ConfigError from compose.service import ContainerNet from compose.service import get_container_data_volumes from compose.service import merge_volume_bindings @@ -23,7 +21,6 @@ from compose.service import NeedsBuildError from compose.service import Net from compose.service import NoSuchImageError from compose.service import parse_repository_tag -from compose.service import parse_volume_spec from compose.service import Service from compose.service import ServiceNet from compose.service import VolumeFromSpec @@ -585,46 +582,12 @@ class ServiceVolumesTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.Client) - def test_parse_volume_spec_only_one_path(self): - spec = parse_volume_spec('/the/volume') - self.assertEqual(spec, (None, '/the/volume', 'rw')) - - def test_parse_volume_spec_internal_and_external(self): - spec = parse_volume_spec('external:interval') - self.assertEqual(spec, ('external', 'interval', 'rw')) - - def test_parse_volume_spec_with_mode(self): - spec = parse_volume_spec('external:interval:ro') - self.assertEqual(spec, ('external', 'interval', 'ro')) - - spec = parse_volume_spec('external:interval:z') - self.assertEqual(spec, ('external', 'interval', 'z')) - - def test_parse_volume_spec_too_many_parts(self): - with self.assertRaises(ConfigError): - parse_volume_spec('one:two:three:four') - - @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') - def test_parse_volume_windows_absolute_path(self): - windows_absolute_path = "c:\\Users\\me\\Documents\\shiny\\config:\\opt\\shiny\\config:ro" - - spec = parse_volume_spec(windows_absolute_path) - - self.assertEqual( - spec, - ( - "/c/Users/me/Documents/shiny/config", - "/opt/shiny/config", - "ro" - ) - ) - def test_build_volume_binding(self): - binding = build_volume_binding(parse_volume_spec('/outside:/inside')) - self.assertEqual(binding, ('/inside', '/outside:/inside:rw')) + binding = build_volume_binding(VolumeSpec.parse('/outside:/inside')) + assert binding == ('/inside', '/outside:/inside:rw') def test_get_container_data_volumes(self): - options = [parse_volume_spec(v) for v in [ + options = [VolumeSpec.parse(v) for v in [ '/host/volume:/host/volume:ro', '/new/volume', '/existing/volume', @@ -648,19 +611,19 @@ class ServiceVolumesTest(unittest.TestCase): }, has_been_inspected=True) expected = [ - parse_volume_spec('/var/lib/docker/aaaaaaaa:/existing/volume:rw'), - parse_volume_spec('/var/lib/docker/cccccccc:/mnt/image/data:rw'), + VolumeSpec.parse('/var/lib/docker/aaaaaaaa:/existing/volume:rw'), + VolumeSpec.parse('/var/lib/docker/cccccccc:/mnt/image/data:rw'), ] volumes = get_container_data_volumes(container, options) - self.assertEqual(sorted(volumes), sorted(expected)) + assert sorted(volumes) == sorted(expected) def test_merge_volume_bindings(self): options = [ - '/host/volume:/host/volume:ro', - '/host/rw/volume:/host/rw/volume', - '/new/volume', - '/existing/volume', + VolumeSpec.parse('/host/volume:/host/volume:ro'), + VolumeSpec.parse('/host/rw/volume:/host/rw/volume'), + VolumeSpec.parse('/new/volume'), + VolumeSpec.parse('/existing/volume'), ] self.mock_client.inspect_image.return_value = { @@ -686,8 +649,8 @@ class ServiceVolumesTest(unittest.TestCase): 'web', image='busybox', volumes=[ - '/host/path:/data1', - '/host/path:/data2', + VolumeSpec.parse('/host/path:/data1'), + VolumeSpec.parse('/host/path:/data2'), ], client=self.mock_client, ) @@ -716,7 +679,7 @@ class ServiceVolumesTest(unittest.TestCase): service = Service( 'web', image='busybox', - volumes=['/host/path:/data'], + volumes=[VolumeSpec.parse('/host/path:/data')], client=self.mock_client, ) @@ -784,22 +747,17 @@ class ServiceVolumesTest(unittest.TestCase): def test_create_with_special_volume_mode(self): self.mock_client.inspect_image.return_value = {'Id': 'imageid'} - create_calls = [] - - def create_container(*args, **kwargs): - create_calls.append((args, kwargs)) - return {'Id': 'containerid'} - - self.mock_client.create_container = create_container - - volumes = ['/tmp:/foo:z'] + self.mock_client.create_container.return_value = {'Id': 'containerid'} + volume = '/tmp:/foo:z' Service( 'web', client=self.mock_client, image='busybox', - volumes=volumes, + volumes=[VolumeSpec.parse(volume)], ).create_container() - self.assertEqual(len(create_calls), 1) - self.assertEqual(self.mock_client.create_host_config.call_args[1]['binds'], volumes) + assert self.mock_client.create_container.call_count == 1 + self.assertEqual( + self.mock_client.create_host_config.call_args[1]['binds'], + [volume]) From effa9834a5722d2f5f5738087f0207c1eca9fd0b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 13 Nov 2015 19:49:14 -0500 Subject: [PATCH 133/359] Remove unnecessary intermediate variables in get_container_host_config. Signed-off-by: Daniel Nephin --- compose/service.py | 44 ++++++++++++-------------------------------- 1 file changed, 12 insertions(+), 32 deletions(-) diff --git a/compose/service.py b/compose/service.py index 6340d074..08d563e9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -496,7 +496,7 @@ class Service(object): # TODO: Implement issue #652 here return build_container_name(self.project, self.name, number, one_off) - # TODO: this would benefit from github.com/docker/docker/pull/11943 + # TODO: this would benefit from github.com/docker/docker/pull/14699 # to remove the need to inspect every container def _next_container_number(self, one_off=False): containers = filter(None, [ @@ -618,54 +618,34 @@ class Service(object): def _get_container_host_config(self, override_options, one_off=False): options = dict(self.options, **override_options) - port_bindings = build_port_bindings(options.get('ports') or []) - privileged = options.get('privileged', False) - cap_add = options.get('cap_add', None) - cap_drop = options.get('cap_drop', None) log_config = LogConfig( type=options.get('log_driver', ""), config=options.get('log_opt', None) ) - pid = options.get('pid', None) - security_opt = options.get('security_opt', None) - - # TODO: these options are already normalized by config - dns = options.get('dns', None) - if isinstance(dns, six.string_types): - dns = [dns] - - dns_search = options.get('dns_search', None) - if isinstance(dns_search, six.string_types): - dns_search = [dns_search] - - devices = options.get('devices', None) - cgroup_parent = options.get('cgroup_parent', None) - ulimits = build_ulimits(options.get('ulimits', None)) - return self.client.create_host_config( links=self._get_links(link_to_self=one_off), - port_bindings=port_bindings, + port_bindings=build_port_bindings(options.get('ports') or []), binds=options.get('binds'), volumes_from=self._get_volumes_from(), - privileged=privileged, + privileged=options.get('privileged', False), network_mode=self.net.mode, - devices=devices, - dns=dns, - dns_search=dns_search, + devices=options.get('devices'), + dns=options.get('dns'), + dns_search=options.get('dns_search'), restart_policy=options.get('restart'), - cap_add=cap_add, - cap_drop=cap_drop, + cap_add=options.get('cap_add'), + cap_drop=options.get('cap_drop'), mem_limit=options.get('mem_limit'), memswap_limit=options.get('memswap_limit'), - ulimits=ulimits, + ulimits=build_ulimits(options.get('ulimits')), log_config=log_config, extra_hosts=options.get('extra_hosts'), read_only=options.get('read_only'), - pid_mode=pid, - security_opt=security_opt, + pid_mode=options.get('pid'), + security_opt=options.get('security_opt'), ipc_mode=options.get('ipc'), - cgroup_parent=cgroup_parent + cgroup_parent=options.get('cgroup_parent'), ) def build(self, no_cache=False, pull=False, force_rm=False): From 533f33271a9be73546673aa9aacd823ca8ea9c38 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 17 Nov 2015 13:35:28 -0500 Subject: [PATCH 134/359] Move service sorting to config package. Signed-off-by: Daniel Nephin --- compose/config/__init__.py | 1 - compose/config/config.py | 17 ++---- compose/config/errors.py | 4 ++ compose/config/sort_services.py | 55 +++++++++++++++++++ compose/project.py | 51 +---------------- tests/integration/testcases.py | 1 + tests/unit/config/config_test.py | 23 +++++++- .../sort_services_test.py} | 6 +- tests/unit/project_test.py | 23 -------- tests/unit/service_test.py | 2 - 10 files changed, 91 insertions(+), 92 deletions(-) create mode 100644 compose/config/sort_services.py rename tests/unit/{sort_service_test.py => config/sort_services_test.py} (98%) diff --git a/compose/config/__init__.py b/compose/config/__init__.py index ec607e08..6fe9ff9f 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -2,7 +2,6 @@ from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find -from .config import get_service_name_from_net from .config import load from .config import merge_environment from .config import parse_environment diff --git a/compose/config/config.py b/compose/config/config.py index 5b1de5ef..9d438ca1 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -14,6 +14,8 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables +from .sort_services import get_service_name_from_net +from .sort_services import sort_service_dicts from .types import parse_extra_hosts from .types import parse_restart_spec from .types import VolumeFromSpec @@ -214,10 +216,10 @@ def load(config_details): return service_dict def build_services(config_file): - return [ + return sort_service_dicts([ build_service(config_file.filename, name, service_dict) for name, service_dict in config_file.config.items() - ] + ]) def merge_services(base, override): all_service_names = set(base) | set(override) @@ -638,17 +640,6 @@ def to_list(value): return value -def get_service_name_from_net(net_config): - if not net_config: - return - - if not net_config.startswith('container:'): - return - - _, net_name = net_config.split(':', 1) - return net_name - - def load_yaml(filename): try: with open(filename, 'r') as fh: diff --git a/compose/config/errors.py b/compose/config/errors.py index 037b7ec8..6d6a69df 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -6,6 +6,10 @@ class ConfigurationError(Exception): return self.msg +class DependencyError(ConfigurationError): + pass + + class CircularReference(ConfigurationError): def __init__(self, trail): self.trail = trail diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py new file mode 100644 index 00000000..5d9adab1 --- /dev/null +++ b/compose/config/sort_services.py @@ -0,0 +1,55 @@ +from compose.config.errors import DependencyError + + +def get_service_name_from_net(net_config): + if not net_config: + return + + if not net_config.startswith('container:'): + return + + _, net_name = net_config.split(':', 1) + return net_name + + +def sort_service_dicts(services): + # Topological sort (Cormen/Tarjan algorithm). + unmarked = services[:] + temporary_marked = set() + sorted_services = [] + + def get_service_names(links): + return [link.split(':')[0] for link in links] + + def get_service_names_from_volumes_from(volumes_from): + return [volume_from.source for volume_from in volumes_from] + + def get_service_dependents(service_dict, services): + name = service_dict['name'] + return [ + service for service in services + if (name in get_service_names(service.get('links', [])) or + name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or + name == get_service_name_from_net(service.get('net'))) + ] + + def visit(n): + if n['name'] in temporary_marked: + if n['name'] in get_service_names(n.get('links', [])): + raise DependencyError('A service can not link to itself: %s' % n['name']) + if n['name'] in n.get('volumes_from', []): + raise DependencyError('A service can not mount itself as volume: %s' % n['name']) + else: + raise DependencyError('Circular import between %s' % ' and '.join(temporary_marked)) + if n in unmarked: + temporary_marked.add(n['name']) + for m in get_service_dependents(n, services): + visit(m) + temporary_marked.remove(n['name']) + unmarked.remove(n) + sorted_services.insert(0, n) + + while unmarked: + visit(unmarked[-1]) + + return sorted_services diff --git a/compose/project.py b/compose/project.py index 5caa1ea3..30e81693 100644 --- a/compose/project.py +++ b/compose/project.py @@ -9,7 +9,7 @@ from docker.errors import NotFound from . import parallel from .config import ConfigurationError -from .config import get_service_name_from_net +from .config.sort_services import get_service_name_from_net from .const import DEFAULT_TIMEOUT from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT @@ -26,49 +26,6 @@ from .service import ServiceNet log = logging.getLogger(__name__) -def sort_service_dicts(services): - # Topological sort (Cormen/Tarjan algorithm). - unmarked = services[:] - temporary_marked = set() - sorted_services = [] - - def get_service_names(links): - return [link.split(':')[0] for link in links] - - def get_service_names_from_volumes_from(volumes_from): - return [volume_from.source for volume_from in volumes_from] - - def get_service_dependents(service_dict, services): - name = service_dict['name'] - return [ - service for service in services - if (name in get_service_names(service.get('links', [])) or - name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or - name == get_service_name_from_net(service.get('net'))) - ] - - def visit(n): - if n['name'] in temporary_marked: - if n['name'] in get_service_names(n.get('links', [])): - raise DependencyError('A service can not link to itself: %s' % n['name']) - if n['name'] in n.get('volumes_from', []): - raise DependencyError('A service can not mount itself as volume: %s' % n['name']) - else: - raise DependencyError('Circular import between %s' % ' and '.join(temporary_marked)) - if n in unmarked: - temporary_marked.add(n['name']) - for m in get_service_dependents(n, services): - visit(m) - temporary_marked.remove(n['name']) - unmarked.remove(n) - sorted_services.insert(0, n) - - while unmarked: - visit(unmarked[-1]) - - return sorted_services - - class Project(object): """ A collection of services. @@ -96,7 +53,7 @@ class Project(object): if use_networking: remove_links(service_dicts) - for service_dict in sort_service_dicts(service_dicts): + for service_dict in service_dicts: links = project.get_links(service_dict) volumes_from = project.get_volumes_from(service_dict) net = project.get_net(service_dict) @@ -404,7 +361,3 @@ class NoSuchService(Exception): def __str__(self): return self.msg - - -class DependencyError(ConfigurationError): - pass diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index f5de50ee..a2218d6b 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -8,6 +8,7 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment +from compose.config.config import ServiceConfig from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index b2a4cd68..a5eeb64f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -77,7 +77,7 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_invalid_service_names(self): + def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( @@ -232,6 +232,27 @@ class ConfigTest(unittest.TestCase): assert "service 'bogus' doesn't have any configuration" in exc.exconly() assert "In file 'override.yaml'" in exc.exconly() + def test_load_sorts_in_dependency_order(self): + config_details = build_config_details({ + 'web': { + 'image': 'busybox:latest', + 'links': ['db'], + }, + 'db': { + 'image': 'busybox:latest', + 'volumes_from': ['volume:ro'] + }, + 'volume': { + 'image': 'busybox:latest', + 'volumes': ['/tmp'], + } + }) + services = config.load(config_details) + + assert services[0]['name'] == 'volume' + assert services[1]['name'] == 'db' + assert services[2]['name'] == 'web' + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( diff --git a/tests/unit/sort_service_test.py b/tests/unit/config/sort_services_test.py similarity index 98% rename from tests/unit/sort_service_test.py rename to tests/unit/config/sort_services_test.py index ef088287..8d0c3ae4 100644 --- a/tests/unit/sort_service_test.py +++ b/tests/unit/config/sort_services_test.py @@ -1,7 +1,7 @@ -from .. import unittest +from compose.config.errors import DependencyError +from compose.config.sort_services import sort_service_dicts from compose.config.types import VolumeFromSpec -from compose.project import DependencyError -from compose.project import sort_service_dicts +from tests import unittest class SortServiceTest(unittest.TestCase): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f8178ed8..f4c6f8ca 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -34,29 +34,6 @@ class ProjectTest(unittest.TestCase): self.assertEqual(project.get_service('db').name, 'db') self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') - def test_from_dict_sorts_in_dependency_order(self): - project = Project.from_dicts('composetest', [ - { - 'name': 'web', - 'image': 'busybox:latest', - 'links': ['db'], - }, - { - 'name': 'db', - 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('volume', 'ro')] - }, - { - 'name': 'volume', - 'image': 'busybox:latest', - 'volumes': ['/tmp'], - } - ], None) - - self.assertEqual(project.services[0].name, 'volume') - self.assertEqual(project.services[1].name, 'db') - self.assertEqual(project.services[2].name, 'web') - def test_from_config(self): dicts = [ { diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index a439f0da..e87ce592 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -23,8 +23,6 @@ from compose.service import NoSuchImageError from compose.service import parse_repository_tag from compose.service import Service from compose.service import ServiceNet -from compose.service import VolumeFromSpec -from compose.service import VolumeSpec from compose.service import warn_on_masked_volume From e40670207f358622e7e2b3f7b73f389dc2bbf63e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 24 Nov 2015 10:40:36 -0500 Subject: [PATCH 135/359] Add missing assert and autospec. Signed-off-by: Daniel Nephin --- tests/unit/service_test.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 808c391c..85d1479d 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -757,7 +757,7 @@ class ServiceVolumesTest(unittest.TestCase): container_volumes = [] service = 'service_name' - with mock.patch('compose.service.log') as mock_log: + with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) assert not mock_log.warn.called @@ -770,17 +770,17 @@ class ServiceVolumesTest(unittest.TestCase): ] service = 'service_name' - with mock.patch('compose.service.log') as mock_log: + with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) - mock_log.warn.called_once_with(mock.ANY) + mock_log.warn.assert_called_once_with(mock.ANY) def test_warn_on_masked_no_warning_with_same_path(self): volumes_option = [VolumeSpec('/home/user', '/path', 'rw')] container_volumes = [VolumeSpec('/home/user', '/path', 'rw')] service = 'service_name' - with mock.patch('compose.service.log') as mock_log: + with mock.patch('compose.service.log', autospec=True) as mock_log: warn_on_masked_volume(volumes_option, container_volumes, service) assert not mock_log.warn.called From 6e89a5708fab417b54d8b5b498abe38e621afde9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 12 Nov 2015 13:23:04 -0500 Subject: [PATCH 136/359] cherry-pick release notes from 1.5.1 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 53 +++++++++++++++++++++++++++++++++++++++++++++++++ docs/install.md | 6 +++--- script/run.sh | 2 +- 3 files changed, 57 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dde42542..428e5a93 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,59 @@ Change log ========== +1.5.1 (2015-11-12) +------------------ + +- Add the `--force-rm` option to `build`. + +- Add the `ulimit` option for services in the Compose file. + +- Fixed a bug where `up` would error with "service needs to be built" if + a service changed from using `image` to using `build`. + +- Fixed a bug that would cause incorrect output of parallel operations + on some terminals. + +- Fixed a bug that prevented a container from being recreated when the + mode of a `volumes_from` was changed. + +- Fixed a regression in 1.5.0 where non-utf-8 unicode characters would cause + `up` or `logs` to crash. + +- Fixed a regression in 1.5.0 where Compose would use a success exit status + code when a command fails due to an HTTP timeout communicating with the + docker daemon. + +- Fixed a regression in 1.5.0 where `name` was being accepted as a valid + service option which would override the actual name of the service. + +- When using `--x-networking` Compose no longer sets the hostname to the + container name. + +- When using `--x-networking` Compose will only create the default network + if at least one container is using the network. + +- When printings logs during `up` or `logs`, flush the output buffer after + each line to prevent buffering issues from hideing logs. + +- Recreate a container if one of its dependencies is being created. + Previously a container was only recreated if it's dependencies already + existed, but were being recreated as well. + +- Add a warning when a `volume` in the Compose file is being ignored + and masked by a container volume from a previous container. + +- Improve the output of `pull` when run without a tty. + +- When using multiple Compose files, validate each before attempting to merge + them together. Previously invalid files would result in not helpful errors. + +- Allow dashes in keys in the `environment` service option. + +- Improve validation error messages by including the filename as part of the + error message. + + 1.5.0 (2015-11-03) ------------------ diff --git a/docs/install.md b/docs/install.md index d394905d..861954b4 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,7 +39,7 @@ which the release page specifies, in your terminal. The following is an example command illustrating the format: - curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.5.1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose If you have problems installing with `curl`, see [Alternative Install Options](#alternative-install-options). @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.5.0 + docker-compose version: 1.5.1 ## Alternative install options @@ -77,7 +77,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.1/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 342188e8..9563b2e9 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.0" +VERSION="1.5.1" IMAGE="docker/compose:$VERSION" From e67bc2569ccc3c811d16cea284eca1f65a227c33 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 23 Nov 2015 13:24:57 -0500 Subject: [PATCH 137/359] Properly resolve environment from all sources. Split env resolving into two phases. The first phase is to expand the paths of env_files, which is done before merging extends. Once all files are merged together, the final phase is to read the env_files and use them as the base for environment variables. Signed-off-by: Daniel Nephin --- compose/config/config.py | 34 +++++------- tests/integration/testcases.py | 5 +- tests/unit/config/config_test.py | 94 +++++++++++++++++--------------- 3 files changed, 64 insertions(+), 69 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 9d438ca1..36914331 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -324,16 +324,13 @@ class ServiceExtendsResolver(object): return filename -def resolve_environment(service_config): +def resolve_environment(service_dict): """Unpack any environment variables from an env_file, if set. Interpolate environment values if set. """ - service_dict = service_config.config - env = {} - if 'env_file' in service_dict: - for env_file in get_env_files(service_config.working_dir, service_dict): - env.update(env_vars_from_file(env_file)) + for env_file in service_dict.get('env_file', []): + env.update(env_vars_from_file(env_file)) env.update(parse_environment(service_dict.get('environment'))) return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) @@ -370,9 +367,11 @@ def process_service(service_config): working_dir = service_config.working_dir service_dict = dict(service_config.config) - if 'environment' in service_dict or 'env_file' in service_dict: - service_dict['environment'] = resolve_environment(service_config) - service_dict.pop('env_file', None) + if 'env_file' in service_dict: + service_dict['env_file'] = [ + expand_path(working_dir, path) + for path in to_list(service_dict['env_file']) + ] if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) @@ -396,6 +395,10 @@ def process_service(service_config): def finalize_service(service_config): service_dict = dict(service_config.config) + if 'environment' in service_dict or 'env_file' in service_dict: + service_dict['environment'] = resolve_environment(service_dict) + service_dict.pop('env_file', None) + if 'volumes_from' in service_dict: service_dict['volumes_from'] = [ VolumeFromSpec.parse(vf) for vf in service_dict['volumes_from']] @@ -440,7 +443,7 @@ def merge_service_dicts(base, override): for field in ['ports', 'expose', 'external_links']: merge_field(field, operator.add, default=[]) - for field in ['dns', 'dns_search']: + for field in ['dns', 'dns_search', 'env_file']: merge_field(field, merge_list_or_string) already_merged_keys = set(d) | {'image', 'build'} @@ -468,17 +471,6 @@ def merge_environment(base, override): return env -def get_env_files(working_dir, options): - if 'env_file' not in options: - return {} - - env_files = options.get('env_file', []) - if not isinstance(env_files, list): - env_files = [env_files] - - return [expand_path(working_dir, path) for path in env_files] - - def parse_environment(environment): if not environment: return {} diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 334693f7..9ea68e39 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -7,7 +7,6 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment -from compose.config.config import ServiceConfig from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -39,9 +38,7 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - service_config = ServiceConfig('.', None, name, kwargs) - kwargs['environment'] = resolve_environment(service_config) - + kwargs['environment'] = resolve_environment(kwargs) labels = dict(kwargs.setdefault('labels', {})) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a5eeb64f..2cd26e8f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -10,6 +10,7 @@ import py import pytest from compose.config import config +from compose.config.config import resolve_environment from compose.config.errors import ConfigurationError from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM @@ -973,65 +974,54 @@ class EnvTest(unittest.TestCase): os.environ['FILE_DEF_EMPTY'] = 'E2' os.environ['ENV_DEF'] = 'E3' - service_dict = make_service_dict( - 'foo', { - 'build': '.', - 'environment': { - 'FILE_DEF': 'F1', - 'FILE_DEF_EMPTY': '', - 'ENV_DEF': None, - 'NO_DEF': None - }, + service_dict = { + 'build': '.', + 'environment': { + 'FILE_DEF': 'F1', + 'FILE_DEF_EMPTY': '', + 'ENV_DEF': None, + 'NO_DEF': None }, - 'tests/' - ) - + } self.assertEqual( - service_dict['environment'], + resolve_environment(service_dict), {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, ) - def test_env_from_file(self): - service_dict = make_service_dict( - 'foo', - {'build': '.', 'env_file': 'one.env'}, - 'tests/fixtures/env', - ) + def test_resolve_environment_from_env_file(self): self.assertEqual( - service_dict['environment'], + resolve_environment({'env_file': ['tests/fixtures/env/one.env']}), {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'bar'}, ) - def test_env_from_multiple_files(self): - service_dict = make_service_dict( - 'foo', - {'build': '.', 'env_file': ['one.env', 'two.env']}, - 'tests/fixtures/env', - ) + def test_resolve_environment_with_multiple_env_files(self): + service_dict = { + 'env_file': [ + 'tests/fixtures/env/one.env', + 'tests/fixtures/env/two.env' + ] + } self.assertEqual( - service_dict['environment'], + resolve_environment(service_dict), {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'baz', 'DOO': 'dah'}, ) - def test_env_nonexistent_file(self): - options = {'env_file': 'nonexistent.env'} - self.assertRaises( - ConfigurationError, - lambda: make_service_dict('foo', options, 'tests/fixtures/env'), - ) + def test_resolve_environment_nonexistent_file(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + {'foo': {'image': 'example', 'env_file': 'nonexistent.env'}}, + working_dir='tests/fixtures/env')) + + assert 'Couldn\'t find env file' in exc.exconly() + assert 'nonexistent.env' in exc.exconly() @mock.patch.dict(os.environ) - def test_resolve_environment_from_file(self): + def test_resolve_environment_from_env_file_with_empty_values(self): os.environ['FILE_DEF'] = 'E1' os.environ['FILE_DEF_EMPTY'] = 'E2' os.environ['ENV_DEF'] = 'E3' - service_dict = make_service_dict( - 'foo', - {'build': '.', 'env_file': 'resolve.env'}, - 'tests/fixtures/env', - ) self.assertEqual( - service_dict['environment'], + resolve_environment({'env_file': ['tests/fixtures/env/resolve.env']}), { 'FILE_DEF': u'bär', 'FILE_DEF_EMPTY': '', @@ -1378,6 +1368,8 @@ class ExtendsTest(unittest.TestCase): - 'envs' environment: - SECRET + - TEST_ONE=common + - TEST_TWO=common """) tmpdir.join('docker-compose.yml').write(""" ext: @@ -1388,12 +1380,20 @@ class ExtendsTest(unittest.TestCase): - 'envs' environment: - THING + - TEST_ONE=top """) commondir.join('envs').write(""" - COMMON_ENV_FILE=1 + COMMON_ENV_FILE + TEST_ONE=common-env-file + TEST_TWO=common-env-file + TEST_THREE=common-env-file + TEST_FOUR=common-env-file """) tmpdir.join('envs').write(""" - FROM_ENV_FILE=1 + TOP_ENV_FILE + TEST_ONE=top-env-file + TEST_TWO=top-env-file + TEST_THREE=top-env-file """) expected = [ @@ -1402,15 +1402,21 @@ class ExtendsTest(unittest.TestCase): 'image': 'example/app', 'environment': { 'SECRET': 'secret', - 'FROM_ENV_FILE': '1', - 'COMMON_ENV_FILE': '1', + 'TOP_ENV_FILE': 'secret', + 'COMMON_ENV_FILE': 'secret', 'THING': 'thing', + 'TEST_ONE': 'top', + 'TEST_TWO': 'common', + 'TEST_THREE': 'top-env-file', + 'TEST_FOUR': 'common-env-file', }, }, ] with mock.patch.dict(os.environ): os.environ['SECRET'] = 'secret' os.environ['THING'] = 'thing' + os.environ['COMMON_ENV_FILE'] = 'secret' + os.environ['TOP_ENV_FILE'] = 'secret' config = load_from_filename(str(tmpdir.join('docker-compose.yml'))) assert config == expected From fd06d699f22fc9d473ac4201feba5847782688ec Mon Sep 17 00:00:00 2001 From: Sven Dowideit Date: Thu, 26 Nov 2015 20:30:12 +1000 Subject: [PATCH 138/359] Use FROM docs/base:latest again Signed-off-by: Sven Dowideit --- docs/Dockerfile | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/Dockerfile b/docs/Dockerfile index 0114f04e..83b65633 100644 --- a/docs/Dockerfile +++ b/docs/Dockerfile @@ -1,4 +1,4 @@ -FROM docs/base:hugo-github-linking +FROM docs/base:latest MAINTAINER Mary Anthony (@moxiegirl) RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/engine @@ -9,7 +9,8 @@ RUN svn checkout https://github.com/kitematic/kitematic/trunk/docs /docs/content RUN svn checkout https://github.com/docker/tutorials/trunk/docs /docs/content/tutorials RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content +ENV PROJECT=compose # To get the git info for this repo COPY . /src -COPY . /docs/content/compose/ +COPY . /docs/content/$PROJECT/ From b85bfce65e26a85150be2073576e3ebe840f3ee1 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 26 Nov 2015 15:06:30 +0000 Subject: [PATCH 139/359] Fix ports validation test We were essentially only testing that *at least one* of the invalid values fails the validation check, rather than that *all* of them fail. Signed-off-by: Aanand Prasad --- tests/unit/config/config_test.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2cd26e8f..7ddec8ab 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -264,9 +264,8 @@ class ConfigTest(unittest.TestCase): assert services[0]['name'] == valid_name def test_config_invalid_ports_format_validation(self): - expected_error_msg = "Service 'web' configuration key 'ports' contains an invalid type" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: + for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: + with pytest.raises(ConfigurationError): config.load( build_config_details( {'web': {'image': 'busybox', 'ports': invalid_ports}}, From f7239f41efe039137da352e249ae0914d683524f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jonas=20Eckerstr=C3=B6m?= Date: Wed, 29 Apr 2015 10:22:24 +0200 Subject: [PATCH 140/359] Added support for url buid paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Jonas Eckerström --- compose/config/config.py | 27 ++++++++++++++++++++++++--- tests/unit/config/config_test.py | 14 ++++++++++++++ 2 files changed, 38 insertions(+), 3 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 36914331..242e7af9 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -76,6 +76,13 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'external_links', ] +DOCKER_VALID_URL_PREFIXES = ( + 'http://', + 'https://', + 'git://', + 'github.com/', + 'git@', +) SUPPORTED_FILENAMES = [ 'docker-compose.yml', @@ -377,7 +384,7 @@ def process_service(service_config): service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) if 'build' in service_dict: - service_dict['build'] = expand_path(working_dir, service_dict['build']) + service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) @@ -539,11 +546,25 @@ def resolve_volume_path(working_dir, volume): return container_path +def resolve_build_path(working_dir, build_path): + if is_url(build_path): + return build_path + return expand_path(working_dir, build_path) + + +def is_url(build_path): + return build_path.startswith(DOCKER_VALID_URL_PREFIXES) + + def validate_paths(service_dict): if 'build' in service_dict: build_path = service_dict['build'] - if not os.path.exists(build_path) or not os.access(build_path, os.R_OK): - raise ConfigurationError("build path %s either does not exist or is not accessible." % build_path) + if ( + not is_url(build_path) and + (not os.path.exists(build_path) or not os.access(build_path, os.R_OK)) + ): + raise ConfigurationError( + "build path %s either does not exist or is not accessible." % build_path) def merge_path_mappings(base, override): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2cd26e8f..6de794ad 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1497,6 +1497,20 @@ class BuildPathTest(unittest.TestCase): service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml') self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) + def test_valid_url_path(self): + valid_urls = [ + 'git://github.com/docker/docker', + 'git@github.com:docker/docker.git', + 'git@bitbucket.org:atlassianlabs/atlassian-docker.git', + 'https://github.com/docker/docker.git', + 'http://github.com/docker/docker.git', + ] + for valid_url in valid_urls: + service_dict = config.load(build_config_details({ + 'validurl': {'build': valid_url}, + }, '.', None)) + assert service_dict[0]['build'] == valid_url + class GetDefaultConfigFilesTestCase(unittest.TestCase): From 2ab3cb212a3c4c0e3b6f3daf6792d3e8cb60782c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 19 Nov 2015 15:46:14 -0500 Subject: [PATCH 141/359] Add integration test and docs for build with a git url. Signed-off-by: Daniel Nephin --- compose/config/config.py | 3 ++- docs/compose-file.md | 11 +++++++---- tests/integration/service_test.py | 7 +++++++ tests/unit/config/config_test.py | 16 +++++++++++++++- 4 files changed, 31 insertions(+), 6 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 242e7af9..c716393d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -564,7 +564,8 @@ def validate_paths(service_dict): (not os.path.exists(build_path) or not os.access(build_path, os.R_OK)) ): raise ConfigurationError( - "build path %s either does not exist or is not accessible." % build_path) + "build path %s either does not exist, is not accessible, " + "or is not a valid URL." % build_path) def merge_path_mappings(base, override): diff --git a/docs/compose-file.md b/docs/compose-file.md index 51d1f5e1..800d2aa9 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -31,15 +31,18 @@ definition. ### build -Path to a directory containing a Dockerfile. When the value supplied is a -relative path, it is interpreted as relative to the location of the yml file -itself. This directory is also the build context that is sent to the Docker daemon. +Either a path to a directory containing a Dockerfile, or a url to a git repository. + +When the value supplied is a relative path, it is interpreted as relative to the +location of the Compose file. This directory is also the build context that is +sent to the Docker daemon. Compose will build and tag it with a generated name, and use that image thereafter. build: /path/to/build/dir -Using `build` together with `image` is not allowed. Attempting to do so results in an error. +Using `build` together with `image` is not allowed. Attempting to do so results in +an error. ### cap_add, cap_drop diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 5dd3d2e6..b03baac5 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -507,6 +507,13 @@ class ServiceTest(DockerClientTestCase): self.create_service('web', build=text_type(base_dir)).build() self.assertEqual(len(self.client.images(name='composetest_web')), 1) + def test_build_with_git_url(self): + build_url = "https://github.com/dnephin/docker-build-from-url.git" + service = self.create_service('buildwithurl', build=build_url) + self.addCleanup(self.client.remove_image, service.image_name) + service.build() + assert service.image() + def test_start_container_stays_unpriviliged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 6de794ad..e15ac350 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1497,13 +1497,14 @@ class BuildPathTest(unittest.TestCase): service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml') self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) - def test_valid_url_path(self): + def test_valid_url_in_build_path(self): valid_urls = [ 'git://github.com/docker/docker', 'git@github.com:docker/docker.git', 'git@bitbucket.org:atlassianlabs/atlassian-docker.git', 'https://github.com/docker/docker.git', 'http://github.com/docker/docker.git', + 'github.com/docker/docker.git', ] for valid_url in valid_urls: service_dict = config.load(build_config_details({ @@ -1511,6 +1512,19 @@ class BuildPathTest(unittest.TestCase): }, '.', None)) assert service_dict[0]['build'] == valid_url + def test_invalid_url_in_build_path(self): + invalid_urls = [ + 'example.com/bogus', + 'ftp://example.com/', + '/path/does/not/exist', + ] + for invalid_url in invalid_urls: + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details({ + 'invalidurl': {'build': invalid_url}, + }, '.', None)) + assert 'build path' in exc.exconly() + class GetDefaultConfigFilesTestCase(unittest.TestCase): From d52508e2b1b8e59900d492cff69169fe4fed7d1f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 26 Nov 2015 18:52:14 +0000 Subject: [PATCH 142/359] Refactor ports section of fields schema Signed-off-by: Aanand Prasad --- compose/config/fields_schema.json | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 9cbcfd1b..3f1f10fa 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -83,16 +83,8 @@ "ports": { "type": "array", "items": { - "oneOf": [ - { - "type": "string", - "format": "ports" - }, - { - "type": "number", - "format": "ports" - } - ] + "type": ["string", "number"], + "format": "ports" }, "uniqueItems": true }, From 374b16843fedca5908d166226e4d1fc9f455acfc Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 26 Nov 2015 18:54:30 +0000 Subject: [PATCH 143/359] Fix ports validation message - The `raises` kwarg to the `cls_check` decorator was being used incorrectly (it should be an exception class, not an object). - We need to check for `error.cause` and get the message out of the exception object. NB: The particular case where validation fails in the case of `ports` is only when ranges don't match in length - no further validation is currently performed client-side. Signed-off-by: Aanand Prasad --- compose/config/validation.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 38020366..24a45e76 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -36,16 +36,12 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' -@FormatChecker.cls_checks( - format="ports", - raises=ValidationError( - "Invalid port formatting, it should be " - "'[[remote_ip:]remote_port:]port[/protocol]'")) +@FormatChecker.cls_checks(format="ports", raises=ValidationError) def format_ports(instance): try: split_port(instance) - except ValueError: - return False + except ValueError as e: + raise ValidationError(six.text_type(e)) return True @@ -184,6 +180,10 @@ def handle_generic_service_error(error, service_name): config_key, required_keys) + elif error.cause: + error_msg = six.text_type(error.cause) + msg_format = "Service '{}' configuration key {} is invalid: {}" + elif error.path: msg_format = "Service '{}' configuration key {} value {}" From 042c7048f26713d18da559941bc25f974d60883c Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 26 Nov 2015 19:17:13 +0000 Subject: [PATCH 144/359] Split out ports validation tests into type, uniqueness, format Signed-off-by: Aanand Prasad --- tests/unit/config/config_test.py | 86 ++++++++++++++++++++++++-------- 1 file changed, 64 insertions(+), 22 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 7ddec8ab..f85c52de 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -263,28 +263,6 @@ class ConfigTest(unittest.TestCase): 'common.yml')) assert services[0]['name'] == valid_name - def test_config_invalid_ports_format_validation(self): - for invalid_ports in [{"1": "8000"}, False, 0, "8000", 8000, ["8000", "8000"]]: - with pytest.raises(ConfigurationError): - config.load( - build_config_details( - {'web': {'image': 'busybox', 'ports': invalid_ports}}, - 'working_dir', - 'filename.yml' - ) - ) - - def test_config_valid_ports_format_validation(self): - valid_ports = [["8000", "9000"], ["8000/8050"], ["8000"], [8000], ["49153-49154:3002-3003"]] - for ports in valid_ports: - config.load( - build_config_details( - {'web': {'image': 'busybox', 'ports': ports}}, - 'working_dir', - 'filename.yml' - ) - ) - def test_config_hint(self): expected_error_msg = "(did you mean 'privileged'?)" with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): @@ -558,6 +536,70 @@ class ConfigTest(unittest.TestCase): assert "which is an invalid type" in exc.exconly() +class PortsTest(unittest.TestCase): + INVALID_PORTS_TYPES = [ + {"1": "8000"}, + False, + "8000", + 8000, + ] + + NON_UNIQUE_SINGLE_PORTS = [ + ["8000", "8000"], + ] + + INVALID_PORT_MAPPINGS = [ + ["8000-8001:8000"], + ] + + VALID_SINGLE_PORTS = [ + ["8000"], + ["8000/tcp"], + ["8000", "9000"], + [8000], + [8000, 9000], + ] + + VALID_PORT_MAPPINGS = [ + ["8000:8050"], + ["49153-49154:3002-3003"], + ] + + def test_config_invalid_ports_type_validation(self): + for invalid_ports in self.INVALID_PORTS_TYPES: + with pytest.raises(ConfigurationError) as exc: + self.check_config({'ports': invalid_ports}) + + assert "contains an invalid type" in exc.value.msg + + def test_config_non_unique_ports_validation(self): + for invalid_ports in self.NON_UNIQUE_SINGLE_PORTS: + with pytest.raises(ConfigurationError) as exc: + self.check_config({'ports': invalid_ports}) + + assert "non-unique" in exc.value.msg + + def test_config_invalid_ports_format_validation(self): + for invalid_ports in self.INVALID_PORT_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config({'ports': invalid_ports}) + + assert "Port ranges don't match in length" in exc.value.msg + + def test_config_valid_ports_format_validation(self): + for valid_ports in self.VALID_SINGLE_PORTS + self.VALID_PORT_MAPPINGS: + self.check_config({'ports': valid_ports}) + + def check_config(self, cfg): + config.load( + build_config_details( + {'web': dict(image='busybox', **cfg)}, + 'working_dir', + 'filename.yml' + ) + ) + + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_config_file_with_environment_variable(self): From ccf548b98c5eca779b753c14439d83832e1f6b54 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 26 Nov 2015 19:17:58 +0000 Subject: [PATCH 145/359] Validate the 'expose' option Signed-off-by: Aanand Prasad --- compose/config/fields_schema.json | 5 ++++- compose/config/validation.py | 14 +++++++++++++- tests/unit/config/config_test.py | 27 +++++++++++++++++++++++++++ 3 files changed, 44 insertions(+), 2 deletions(-) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 3f1f10fa..7d5220e3 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -41,7 +41,10 @@ "expose": { "type": "array", - "items": {"type": ["string", "number"]}, + "items": { + "type": ["string", "number"], + "format": "expose" + }, "uniqueItems": true }, diff --git a/compose/config/validation.py b/compose/config/validation.py index 24a45e76..d16bdb9d 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,6 +1,7 @@ import json import logging import os +import re import sys import six @@ -34,6 +35,7 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' +VALID_EXPOSE_FORMAT = r'^\d+(\/[a-zA-Z]+)?$' @FormatChecker.cls_checks(format="ports", raises=ValidationError) @@ -45,6 +47,16 @@ def format_ports(instance): return True +@FormatChecker.cls_checks(format="expose", raises=ValidationError) +def format_expose(instance): + if isinstance(instance, six.string_types): + if not re.match(VALID_EXPOSE_FORMAT, instance): + raise ValidationError( + "should be of the format 'PORT[/PROTOCOL]'") + + return True + + @FormatChecker.cls_checks(format="bool-value-in-mapping") def format_boolean_in_environment(instance): """ @@ -273,7 +285,7 @@ def validate_against_fields_schema(config, filename): _validate_against_schema( config, "fields_schema.json", - format_checker=["ports", "bool-value-in-mapping"], + format_checker=["ports", "expose", "bool-value-in-mapping"], filename=filename) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f85c52de..6c445432 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -590,6 +590,33 @@ class PortsTest(unittest.TestCase): for valid_ports in self.VALID_SINGLE_PORTS + self.VALID_PORT_MAPPINGS: self.check_config({'ports': valid_ports}) + def test_config_invalid_expose_type_validation(self): + for invalid_expose in self.INVALID_PORTS_TYPES: + with pytest.raises(ConfigurationError) as exc: + self.check_config({'expose': invalid_expose}) + + assert "contains an invalid type" in exc.value.msg + + def test_config_non_unique_expose_validation(self): + for invalid_expose in self.NON_UNIQUE_SINGLE_PORTS: + with pytest.raises(ConfigurationError) as exc: + self.check_config({'expose': invalid_expose}) + + assert "non-unique" in exc.value.msg + + def test_config_invalid_expose_format_validation(self): + # Valid port mappings ARE NOT valid 'expose' entries + for invalid_expose in self.INVALID_PORT_MAPPINGS + self.VALID_PORT_MAPPINGS: + with pytest.raises(ConfigurationError) as exc: + self.check_config({'expose': invalid_expose}) + + assert "should be of the format" in exc.value.msg + + def test_config_valid_expose_format_validation(self): + # Valid single ports ARE valid 'expose' entries + for valid_expose in self.VALID_SINGLE_PORTS: + self.check_config({'expose': valid_expose}) + def check_config(self, cfg): config.load( build_config_details( From 2f568984f73e7bade8d01127dd4e8cf7202eaaec Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Nov 2015 11:52:25 -0500 Subject: [PATCH 146/359] Fixes #2368, removes the deprecated --allow-insecure-ssl flag. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 17 ----------------- tests/unit/cli_test.py | 4 ---- 2 files changed, 21 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 9fef8d04..2eba265b 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -41,11 +41,6 @@ if not IS_WINDOWS_PLATFORM: log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) -INSECURE_SSL_WARNING = """ ---allow-insecure-ssl is deprecated and has no effect. -It will be removed in a future version of Compose. -""" - def main(): setup_logging() @@ -303,11 +298,7 @@ class TopLevelCommand(DocoptCommand): Options: --ignore-pull-failures Pull what it can and ignores images with pull failures. - --allow-insecure-ssl Deprecated - no effect. """ - if options['--allow-insecure-ssl']: - log.warn(INSECURE_SSL_WARNING) - project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures') @@ -352,7 +343,6 @@ class TopLevelCommand(DocoptCommand): Usage: run [options] [-p PORT...] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] Options: - --allow-insecure-ssl Deprecated - no effect. -d Detached mode: Run container in the background, print new container name. --name NAME Assign a name to the container @@ -376,9 +366,6 @@ class TopLevelCommand(DocoptCommand): "Please pass the -d flag when using `docker-compose run`." ) - if options['--allow-insecure-ssl']: - log.warn(INSECURE_SSL_WARNING) - if options['COMMAND']: command = [options['COMMAND']] + options['ARGS'] else: @@ -514,7 +501,6 @@ class TopLevelCommand(DocoptCommand): Usage: up [options] [SERVICE...] Options: - --allow-insecure-ssl Deprecated - no effect. -d Detached mode: Run containers in the background, print new container names. --no-color Produce monochrome output. @@ -528,9 +514,6 @@ class TopLevelCommand(DocoptCommand): when attached or when containers are already running. (default: 10) """ - if options['--allow-insecure-ssl']: - log.warn(INSECURE_SSL_WARNING) - monochrome = options['--no-color'] start_deps = not options['--no-deps'] service_names = options['SERVICE'] diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 23dc4262..c962d007 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -102,7 +102,6 @@ class CLITestCase(unittest.TestCase): '-e': ['BAR=NEW', 'OTHER=bär'.encode('utf-8')], '--user': None, '--no-deps': None, - '--allow-insecure-ssl': None, '-d': True, '-T': None, '--entrypoint': None, @@ -132,7 +131,6 @@ class CLITestCase(unittest.TestCase): '-e': [], '--user': None, '--no-deps': None, - '--allow-insecure-ssl': None, '-d': True, '-T': None, '--entrypoint': None, @@ -161,7 +159,6 @@ class CLITestCase(unittest.TestCase): '-e': [], '--user': None, '--no-deps': None, - '--allow-insecure-ssl': None, '-d': True, '-T': None, '--entrypoint': None, @@ -193,7 +190,6 @@ class CLITestCase(unittest.TestCase): '-e': [], '--user': None, '--no-deps': None, - '--allow-insecure-ssl': None, '-d': True, '-T': None, '--entrypoint': None, From a21f9993b3acf20efafad70b02da81debfd37830 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Nov 2015 12:02:13 -0500 Subject: [PATCH 147/359] Remove migrate-to-labels. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 47 +----- compose/legacy.py | 182 --------------------- compose/project.py | 8 - compose/service.py | 12 +- contrib/completion/bash/docker-compose | 10 -- contrib/completion/zsh/_docker-compose | 5 - docs/install.md | 2 +- docs/reference/docker-compose.md | 1 - tests/integration/legacy_test.py | 218 ------------------------- tests/unit/cli_test.py | 6 - 10 files changed, 7 insertions(+), 484 deletions(-) delete mode 100644 compose/legacy.py delete mode 100644 tests/integration/legacy_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 9fef8d04..90b03017 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -12,7 +12,6 @@ from docker.errors import APIError from requests.exceptions import ReadTimeout from .. import __version__ -from .. import legacy from ..config import ConfigurationError from ..config import parse_environment from ..const import DEFAULT_TIMEOUT @@ -55,7 +54,7 @@ def main(): except KeyboardInterrupt: log.error("\nAborting.") sys.exit(1) - except (UserError, NoSuchService, ConfigurationError, legacy.LegacyError) as e: + except (UserError, NoSuchService, ConfigurationError) as e: log.error(e.msg) sys.exit(1) except NoSuchCommand as e: @@ -147,9 +146,7 @@ class TopLevelCommand(DocoptCommand): stop Stop services unpause Unpause services up Create and start containers - migrate-to-labels Recreate containers to add labels version Show the Docker-Compose version information - """ base_dir = '.' @@ -550,32 +547,6 @@ class TopLevelCommand(DocoptCommand): log_printer = build_log_printer(to_attach, service_names, monochrome) attach_to_logs(project, log_printer, service_names, timeout) - def migrate_to_labels(self, project, _options): - """ - Recreate containers to add labels - - If you're coming from Compose 1.2 or earlier, you'll need to remove or - migrate your existing containers after upgrading Compose. This is - because, as of version 1.3, Compose uses Docker labels to keep track - of containers, and so they need to be recreated with labels added. - - If Compose detects containers that were created without labels, it - will refuse to run so that you don't end up with two sets of them. If - you want to keep using your existing containers (for example, because - they have data volumes you want to preserve) you can migrate them with - the following command: - - docker-compose migrate-to-labels - - Alternatively, if you're not worried about keeping them, you can - remove them - Compose will just create new ones. - - docker rm -f myapp_web_1 myapp_db_1 ... - - Usage: migrate-to-labels - """ - legacy.migrate_project_to_labels(project) - def version(self, project, options): """ Show version informations @@ -618,18 +589,10 @@ def run_one_off_container(container_options, project, service, options): if project.use_networking: project.ensure_network_exists() - try: - container = service.create_container( - quiet=True, - one_off=True, - **container_options) - except APIError: - legacy.check_for_legacy_containers( - project.client, - project.name, - [service.name], - allow_one_off=False) - raise + container = service.create_container( + quiet=True, + one_off=True, + **container_options) if options['-d']: container.start() diff --git a/compose/legacy.py b/compose/legacy.py deleted file mode 100644 index 54162417..00000000 --- a/compose/legacy.py +++ /dev/null @@ -1,182 +0,0 @@ -import logging -import re - -from .const import LABEL_VERSION -from .container import Container -from .container import get_container_name - - -log = logging.getLogger(__name__) - - -# TODO: remove this section when migrate_project_to_labels is removed -NAME_RE = re.compile(r'^([^_]+)_([^_]+)_(run_)?(\d+)$') - -ERROR_MESSAGE_FORMAT = """ -Compose found the following containers without labels: - -{names_list} - -As of Compose 1.3.0, containers are identified with labels instead of naming -convention. If you want to continue using these containers, run: - - $ docker-compose migrate-to-labels - -Alternatively, remove them: - - $ docker rm -f {rm_args} -""" - -ONE_OFF_ADDENDUM_FORMAT = """ -You should also remove your one-off containers: - - $ docker rm -f {rm_args} -""" - -ONE_OFF_ERROR_MESSAGE_FORMAT = """ -Compose found the following containers without labels: - -{names_list} - -As of Compose 1.3.0, containers are identified with labels instead of naming convention. - -Remove them before continuing: - - $ docker rm -f {rm_args} -""" - - -def check_for_legacy_containers( - client, - project, - services, - allow_one_off=True): - """Check if there are containers named using the old naming convention - and warn the user that those containers may need to be migrated to - using labels, so that compose can find them. - """ - containers = get_legacy_containers(client, project, services, one_off=False) - - if containers: - one_off_containers = get_legacy_containers(client, project, services, one_off=True) - - raise LegacyContainersError( - [c.name for c in containers], - [c.name for c in one_off_containers], - ) - - if not allow_one_off: - one_off_containers = get_legacy_containers(client, project, services, one_off=True) - - if one_off_containers: - raise LegacyOneOffContainersError( - [c.name for c in one_off_containers], - ) - - -class LegacyError(Exception): - def __unicode__(self): - return self.msg - - __str__ = __unicode__ - - -class LegacyContainersError(LegacyError): - def __init__(self, names, one_off_names): - self.names = names - self.one_off_names = one_off_names - - self.msg = ERROR_MESSAGE_FORMAT.format( - names_list="\n".join(" {}".format(name) for name in names), - rm_args=" ".join(names), - ) - - if one_off_names: - self.msg += ONE_OFF_ADDENDUM_FORMAT.format(rm_args=" ".join(one_off_names)) - - -class LegacyOneOffContainersError(LegacyError): - def __init__(self, one_off_names): - self.one_off_names = one_off_names - - self.msg = ONE_OFF_ERROR_MESSAGE_FORMAT.format( - names_list="\n".join(" {}".format(name) for name in one_off_names), - rm_args=" ".join(one_off_names), - ) - - -def add_labels(project, container): - project_name, service_name, one_off, number = NAME_RE.match(container.name).groups() - if project_name != project.name or service_name not in project.service_names: - return - service = project.get_service(service_name) - service.recreate_container(container) - - -def migrate_project_to_labels(project): - log.info("Running migration to labels for project %s", project.name) - - containers = get_legacy_containers( - project.client, - project.name, - project.service_names, - one_off=False, - ) - - for container in containers: - add_labels(project, container) - - -def get_legacy_containers( - client, - project, - services, - one_off=False): - - return list(_get_legacy_containers_iter( - client, - project, - services, - one_off=one_off, - )) - - -def _get_legacy_containers_iter( - client, - project, - services, - one_off=False): - - containers = client.containers(all=True) - - for service in services: - for container in containers: - if LABEL_VERSION in (container.get('Labels') or {}): - continue - - name = get_container_name(container) - if has_container(project, service, name, one_off=one_off): - yield Container.from_ps(client, container) - - -def has_container(project, service, name, one_off=False): - if not name or not is_valid_name(name, one_off): - return False - container_project, container_service, _container_number = parse_name(name) - return container_project == project and container_service == service - - -def is_valid_name(name, one_off=False): - match = NAME_RE.match(name) - if match is None: - return False - if one_off: - return match.group(3) == 'run_' - else: - return match.group(3) is None - - -def parse_name(name): - match = NAME_RE.match(name) - (project, service_name, _, suffix) = match.groups() - return (project, service_name, int(suffix)) diff --git a/compose/project.py b/compose/project.py index 30e81693..af40d820 100644 --- a/compose/project.py +++ b/compose/project.py @@ -15,7 +15,6 @@ from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .container import Container -from .legacy import check_for_legacy_containers from .service import ContainerNet from .service import ConvergenceStrategy from .service import Net @@ -287,13 +286,6 @@ class Project(object): def matches_service_names(container): return container.labels.get(LABEL_SERVICE) in service_names - if not containers: - check_for_legacy_containers( - self.client, - self.name, - self.service_names, - ) - return [c for c in containers if matches_service_names(c)] def get_network(self): diff --git a/compose/service.py b/compose/service.py index 08d563e9..0b9a2aa3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -26,7 +26,6 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION from .container import Container -from .legacy import check_for_legacy_containers from .parallel import parallel_execute from .parallel import parallel_remove from .parallel import parallel_start @@ -122,21 +121,12 @@ class Service(object): def containers(self, stopped=False, one_off=False, filters={}): filters.update({'label': self.labels(one_off=one_off)}) - containers = list(filter(None, [ + return list(filter(None, [ Container.from_ps(self.client, container) for container in self.client.containers( all=stopped, filters=filters)])) - if not containers: - check_for_legacy_containers( - self.client, - self.project, - [self.name], - ) - - return containers - def get_container(self, number=1): """Return a :class:`compose.container.Container` for this service. The container must be active, and match `number`. diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index b4f4387f..c22e6abc 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -164,15 +164,6 @@ _docker_compose_logs() { } -_docker_compose_migrate_to_labels() { - case "$cur" in - -*) - COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) - ;; - esac -} - - _docker_compose_pause() { case "$cur" in -*) @@ -385,7 +376,6 @@ _docker_compose() { help kill logs - migrate-to-labels pause port ps diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 08d5150d..0b50b535 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -212,11 +212,6 @@ __docker-compose_subcommand() { '--no-color[Produce monochrome output.]' \ '*:services:__docker-compose_services_all' && ret=0 ;; - (migrate-to-labels) - _arguments -A '-*' \ - $opts_help \ - '(-):Recreate containers to add labels' && ret=0 - ;; (pause) _arguments \ $opts_help \ diff --git a/docs/install.md b/docs/install.md index 861954b4..cc15cec1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -98,7 +98,7 @@ be recreated with labels added. If Compose detects containers that were created without labels, it will refuse to run so that you don't end up with two sets of them. If you want to keep using your existing containers (for example, because they have data volumes you want -to preserve) you can migrate them with the following command: +to preserve) you can use compose 1.5.x to migrate them with the following command: $ docker-compose migrate-to-labels diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index 8712072e..c19e4284 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -40,7 +40,6 @@ Commands: stop Stop services unpause Unpause services up Create and start containers - migrate-to-labels Recreate containers to add labels version Show the Docker-Compose version information ``` diff --git a/tests/integration/legacy_test.py b/tests/integration/legacy_test.py deleted file mode 100644 index 3465d57f..00000000 --- a/tests/integration/legacy_test.py +++ /dev/null @@ -1,218 +0,0 @@ -import unittest - -from docker.errors import APIError - -from .. import mock -from .testcases import DockerClientTestCase -from compose import legacy -from compose.project import Project - - -class UtilitiesTestCase(unittest.TestCase): - def test_has_container(self): - self.assertTrue( - legacy.has_container("composetest", "web", "composetest_web_1", one_off=False), - ) - self.assertFalse( - legacy.has_container("composetest", "web", "composetest_web_run_1", one_off=False), - ) - - def test_has_container_one_off(self): - self.assertFalse( - legacy.has_container("composetest", "web", "composetest_web_1", one_off=True), - ) - self.assertTrue( - legacy.has_container("composetest", "web", "composetest_web_run_1", one_off=True), - ) - - def test_has_container_different_project(self): - self.assertFalse( - legacy.has_container("composetest", "web", "otherapp_web_1", one_off=False), - ) - self.assertFalse( - legacy.has_container("composetest", "web", "otherapp_web_run_1", one_off=True), - ) - - def test_has_container_different_service(self): - self.assertFalse( - legacy.has_container("composetest", "web", "composetest_db_1", one_off=False), - ) - self.assertFalse( - legacy.has_container("composetest", "web", "composetest_db_run_1", one_off=True), - ) - - def test_is_valid_name(self): - self.assertTrue( - legacy.is_valid_name("composetest_web_1", one_off=False), - ) - self.assertFalse( - legacy.is_valid_name("composetest_web_run_1", one_off=False), - ) - - def test_is_valid_name_one_off(self): - self.assertFalse( - legacy.is_valid_name("composetest_web_1", one_off=True), - ) - self.assertTrue( - legacy.is_valid_name("composetest_web_run_1", one_off=True), - ) - - def test_is_valid_name_invalid(self): - self.assertFalse( - legacy.is_valid_name("foo"), - ) - self.assertFalse( - legacy.is_valid_name("composetest_web_lol_1", one_off=True), - ) - - def test_get_legacy_containers(self): - client = mock.Mock() - client.containers.return_value = [ - { - "Id": "abc123", - "Image": "def456", - "Name": "composetest_web_1", - "Labels": None, - }, - { - "Id": "ghi789", - "Image": "def456", - "Name": None, - "Labels": None, - }, - { - "Id": "jkl012", - "Image": "def456", - "Labels": None, - }, - ] - - containers = legacy.get_legacy_containers(client, "composetest", ["web"]) - - self.assertEqual(len(containers), 1) - self.assertEqual(containers[0].id, 'abc123') - - -class LegacyTestCase(DockerClientTestCase): - - def setUp(self): - super(LegacyTestCase, self).setUp() - self.containers = [] - - db = self.create_service('db') - web = self.create_service('web', links=[(db, 'db')]) - nginx = self.create_service('nginx', links=[(web, 'web')]) - - self.services = [db, web, nginx] - self.project = Project('composetest', self.services, self.client) - - # Create a legacy container for each service - for service in self.services: - service.ensure_image_exists() - container = self.client.create_container( - name='{}_{}_1'.format(self.project.name, service.name), - **service.options - ) - self.client.start(container) - self.containers.append(container) - - # Create a single one-off legacy container - self.containers.append(self.client.create_container( - name='{}_{}_run_1'.format(self.project.name, db.name), - **self.services[0].options - )) - - def tearDown(self): - super(LegacyTestCase, self).tearDown() - for container in self.containers: - try: - self.client.kill(container) - except APIError: - pass - try: - self.client.remove_container(container) - except APIError: - pass - - def get_legacy_containers(self, **kwargs): - return legacy.get_legacy_containers( - self.client, - self.project.name, - [s.name for s in self.services], - **kwargs - ) - - def test_get_legacy_container_names(self): - self.assertEqual(len(self.get_legacy_containers()), len(self.services)) - - def test_get_legacy_container_names_one_off(self): - self.assertEqual(len(self.get_legacy_containers(one_off=True)), 1) - - def test_migration_to_labels(self): - # Trying to get the container list raises an exception - - with self.assertRaises(legacy.LegacyContainersError) as cm: - self.project.containers(stopped=True) - - self.assertEqual( - set(cm.exception.names), - set(['composetest_db_1', 'composetest_web_1', 'composetest_nginx_1']), - ) - - self.assertEqual( - set(cm.exception.one_off_names), - set(['composetest_db_run_1']), - ) - - # Migrate the containers - - legacy.migrate_project_to_labels(self.project) - - # Getting the list no longer raises an exception - - containers = self.project.containers(stopped=True) - self.assertEqual(len(containers), len(self.services)) - - def test_migration_one_off(self): - # We've already migrated - - legacy.migrate_project_to_labels(self.project) - - # Trying to create a one-off container results in a Docker API error - - with self.assertRaises(APIError) as cm: - self.project.get_service('db').create_container(one_off=True) - - # Checking for legacy one-off containers raises an exception - - with self.assertRaises(legacy.LegacyOneOffContainersError) as cm: - legacy.check_for_legacy_containers( - self.client, - self.project.name, - ['db'], - allow_one_off=False, - ) - - self.assertEqual( - set(cm.exception.one_off_names), - set(['composetest_db_run_1']), - ) - - # Remove the old one-off container - - c = self.client.inspect_container('composetest_db_run_1') - self.client.remove_container(c) - - # Checking no longer raises an exception - - legacy.check_for_legacy_containers( - self.client, - self.project.name, - ['db'], - allow_one_off=False, - ) - - # Creating a one-off container no longer results in an API error - - self.project.get_service('db').create_container(one_off=True) - self.assertIsInstance(self.client.inspect_container('composetest_db_run_1'), dict) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 23dc4262..2473a340 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -74,12 +74,6 @@ class CLITestCase(unittest.TestCase): self.assertIn('Usage: up', str(ctx.exception)) - def test_command_help_dashes(self): - with self.assertRaises(SystemExit) as ctx: - TopLevelCommand().dispatch(['help', 'migrate-to-labels'], None) - - self.assertIn('Usage: migrate-to-labels', str(ctx.exception)) - def test_command_help_nonexistent(self): with self.assertRaises(NoSuchCommand): TopLevelCommand().dispatch(['help', 'nonexistent'], None) From 377f084dfe799d43798c9015081437f98228171d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Nov 2015 13:54:00 -0500 Subject: [PATCH 148/359] Increase timeout in tests. Signed-off-by: Daniel Nephin --- tests/acceptance/cli_test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 282a5219..99b78d08 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -43,7 +43,7 @@ def wait_on_process(proc, returncode=0): return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) -def wait_on_condition(condition, delay=0.1, timeout=5): +def wait_on_condition(condition, delay=0.1, timeout=20): start_time = time.time() while not condition(): if time.time() - start_time > timeout: From 3f39ffe72e471c3ca06c8ac6330e2858ba66795e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 21 Oct 2015 20:13:43 -0400 Subject: [PATCH 149/359] FAQ document for Compose Signed-off-by: Daniel Nephin --- docs/faq.md | 139 +++++++++++++++++++++++++ docs/index.md | 1 + script/travis/render-bintray-config.py | 4 +- 3 files changed, 143 insertions(+), 1 deletion(-) create mode 100644 docs/faq.md diff --git a/docs/faq.md b/docs/faq.md new file mode 100644 index 00000000..b36eb5ac --- /dev/null +++ b/docs/faq.md @@ -0,0 +1,139 @@ + + +# Frequently asked questions + +If you don’t see your question here, feel free to drop by `#docker-compose` on +freenode IRC and ask the community. + +## Why do my services take 10 seconds to stop? + +Compose stop attempts to stop a container by sending a `SIGTERM`. It then waits +for a [default timeout of 10 seconds](./reference/stop.md). After the timeout, +a `SIGKILL` is sent to the container to forcefully kill it. If you +are waiting for this timeout, it means that your containers aren't shutting down +when they receive the `SIGTERM` signal. + +There has already been a lot written about this problem of +[processes handling signals](https://medium.com/@gchudnov/trapping-signals-in-docker-containers-7a57fdda7d86) +in containers. + +To fix this problem, try the following: + +* Make sure you're using the JSON form of `CMD` and `ENTRYPOINT` +in your Dockerfile. + + For example use `["program", "arg1", "arg2"]` not `"program arg1 arg2"`. + Using the string form causes Docker to run your process using `bash` which + doesn't handle signals properly. Compose always uses the JSON form, so don't + worry if you override the command or entrypoint in your Compose file. + +* If you are able, modify the application that you're running to +add an explicit signal handler for `SIGTERM`. + +* If you can't modify the application, wrap the application in a lightweight init +system (like [s6](http://skarnet.org/software/s6/)) or a signal proxy (like +[dumb-init](https://github.com/Yelp/dumb-init) or +[tini](https://github.com/krallin/tini)). Either of these wrappers take care of +handling `SIGTERM` properly. + +## How do I run multiple copies of a Compose file on the same host? + +Compose uses the project name to create unique identifiers for all of a +project's containers and other resources. To run multiple copies of a project, +set a custom project name using the [`-p` command line +option](./reference/docker-compose.md) or the [`COMPOSE_PROJECT_NAME` +environment variable](./reference/overview.md#compose-project-name). + +## What's the difference between `up`, `run`, and `start`? + +Typically, you want `docker-compose up`. Use `up` to start or restart all the +services defined in a `docker-compose.yml`. In the default "attached" +mode, you'll see all the logs from all the containers. In "detached" mode (`-d`), +Compose exits after starting the containers, but the containers continue to run +in the background. + +The `docker-compose run` command is for running "one-off" or "adhoc" tasks. It +requires the service name you want to run and only starts containers for services +that the running service depends on. Use `run` to run tests or perform +an administrative task such as removing or adding data to a data volume +container. The `run` command acts like `docker run -ti` in that it opens an +interactive terminal to the container and returns an exit status matching the +exit status of the process in the container. + +The `docker-compose start` command is useful only to restart containers +that were previously created, but were stopped. It never creates new +containers. + +## Can I use json instead of yaml for my Compose file? + +Yes. [Yaml is a superset of json](http://stackoverflow.com/a/1729545/444646) so +any JSON file should be valid Yaml. To use a JSON file with Compose, +specify the filename to use, for example: + +```bash +docker-compose -f docker-compose.json up +``` + +## How do I get Compose to wait for my database to be ready before starting my application? + +Unfortunately, Compose won't do that for you but for a good reason. + +The problem of waiting for a database to be ready is really just a subset of a +much larger problem of distributed systems. In production, your database could +become unavailable or move hosts at any time. The application needs to be +resilient to these types of failures. + +To handle this, the application would attempt to re-establish a connection to +the database after a failure. If the application retries the connection, +it should eventually be able to connect to the database. + +To wait for the application to be in a good state, you can implement a +healthcheck. A healthcheck makes a request to the application and checks +the response for a success status code. If it is not successful it waits +for a short period of time, and tries again. After some timeout value, the check +stops trying and report a failure. + +If you need to run tests against your application, you can start by running a +healthcheck. Once the healthcheck gets a successful response, you can start +running your tests. + + +## Should I include my code with `COPY`/`ADD` or a volume? + +You can add your code to the image using `COPY` or `ADD` directive in a +`Dockerfile`. This is useful if you need to relocate your code along with the +Docker image, for example when you're sending code to another environment +(production, CI, etc). + +You should use a `volume` if you want to make changes to your code and see them +reflected immediately, for example when you're developing code and your server +supports hot code reloading or live-reload. + +There may be cases where you'll want to use both. You can have the image +include the code using a `COPY`, and use a `volume` in your Compose file to +include the code from the host during development. The volume overrides +the directory contents of the image. + +## Where can I find example compose files? + +There are [many examples of Compose files on +github](https://github.com/search?q=in%3Apath+docker-compose.yml+extension%3Ayml&type=Code). + + +## Compose documentation + +- [Installing Compose](install.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with WordPress](wordpress.md) +- [Command line reference](./reference/index.md) +- [Compose file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md index 279154ee..8b32a754 100644 --- a/docs/index.md +++ b/docs/index.md @@ -59,6 +59,7 @@ Compose has commands for managing the whole lifecycle of your application: - [Get started with Django](django.md) - [Get started with Rails](rails.md) - [Get started with WordPress](wordpress.md) +- [Frequently asked questions](faq.md) - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) diff --git a/script/travis/render-bintray-config.py b/script/travis/render-bintray-config.py index 6aa468d6..fc5d409a 100755 --- a/script/travis/render-bintray-config.py +++ b/script/travis/render-bintray-config.py @@ -1,4 +1,6 @@ #!/usr/bin/env python +from __future__ import print_function + import datetime import os.path import sys @@ -6,4 +8,4 @@ import sys os.environ['DATE'] = str(datetime.date.today()) for line in sys.stdin: - print os.path.expandvars(line), + print(os.path.expandvars(line), end='') From e760c42ae00b9e1cccf2aeff4a17a3f5a0bd8cbe Mon Sep 17 00:00:00 2001 From: jake-low Date: Wed, 2 Dec 2015 21:45:36 -0800 Subject: [PATCH 150/359] Stop warning about ".yaml" extension ".yaml" is the preferred extension according to http://www.yaml.org/faq.html Signed-off-by: jake-low --- compose/config/config.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index c716393d..853157ee 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -161,11 +161,6 @@ def get_default_config_files(base_dir): log.warn("Found multiple config files with supported names: %s", ", ".join(candidates)) log.warn("Using %s\n", winner) - if winner == 'docker-compose.yaml': - log.warn("Please be aware that .yml is the expected extension " - "in most cases, and using .yaml can cause compatibility " - "issues in future.\n") - if winner.startswith("fig."): log.warn("%s is deprecated and will not be supported in future. " "Please rename your config file to docker-compose.yml\n" % winner) From 7698da57ca9c8c17001a137dd128fca2881957ac Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Fri, 4 Dec 2015 16:50:50 +0100 Subject: [PATCH 151/359] update maintainers file for parsing this updates the MAINTAINERS file to the new format, so that it can be parsed and collected in the docker/opensource repository. Signed-off-by: Sebastiaan van Stijn --- MAINTAINERS | 50 ++++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 46 insertions(+), 4 deletions(-) diff --git a/MAINTAINERS b/MAINTAINERS index 00324232..820b2f82 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1,4 +1,46 @@ -Aanand Prasad (@aanand) -Ben Firshman (@bfirsh) -Daniel Nephin (@dnephin) -Mazz Mosley (@mnowster) +# Compose maintainers file +# +# This file describes who runs the docker/compose project and how. +# This is a living document - if you see something out of date or missing, speak up! +# +# It is structured to be consumable by both humans and programs. +# To extract its contents programmatically, use any TOML-compliant parser. +# +# This file is compiled into the MAINTAINERS file in docker/opensource. +# +[Org] + [Org."Core maintainers"] + people = [ + "aanand", + "bfirsh", + "dnephin", + "mnowster", + ] + +[people] + +# A reference list of all people associated with the project. +# All other sections should refer to people by their canonical key +# in the people section. + + # ADD YOURSELF HERE IN ALPHABETICAL ORDER + + [people.aanand] + Name = "Aanand Prasad" + Email = "aanand.prasad@gmail.com" + GitHub = "aanand" + + [people.bfirsh] + Name = "Ben Firshman" + Email = "ben@firshman.co.uk" + GitHub = "bfirsh" + + [people.dnephin] + Name = "Daniel Nephin" + Email = "dnephin@gmail.com" + GitHub = "dnephin" + + [people.mnowster] + Name = "Mazz Mosley" + Email = "mazz@houseofmnowster.com" + GitHub = "mnowster" From de4a18ea6c1b4addfeb3aad32204a01a90a2e776 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 2 Dec 2015 17:06:17 -0800 Subject: [PATCH 152/359] Cherry-pick release notes for 1.5.2 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 22 ++++++++++++++++++++++ docs/install.md | 6 +++--- script/run.sh | 2 +- 3 files changed, 26 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 428e5a93..7b6e0dd3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,28 @@ Change log ========== +1.5.2 (2015-12-03) +------------------ + +- Fixed a bug which broke the use of `environment` and `env_file` with + `extends`, and caused environment keys without values to have a `None` + value, instead of a value from the host environment. + +- Fixed a regression in 1.5.1 that caused a warning about volumes to be + raised incorrectly when containers were recreated. + +- Fixed a bug which prevented building a `Dockerfile` that used `ADD ` + +- Fixed a bug with `docker-compose restart` which prevented it from + starting stopped containers. + +- Fixed handling of SIGTERM and SIGINT to properly stop containers + +- Add support for using a url as the value of `build` + +- Improved the validation of the `expose` option + + 1.5.1 (2015-11-12) ------------------ diff --git a/docs/install.md b/docs/install.md index 861954b4..980d285c 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,7 +39,7 @@ which the release page specifies, in your terminal. The following is an example command illustrating the format: - curl -L https://github.com/docker/compose/releases/download/1.5.1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.5.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose If you have problems installing with `curl`, see [Alternative Install Options](#alternative-install-options). @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.5.1 + docker-compose version: 1.5.2 ## Alternative install options @@ -77,7 +77,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.1/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.2/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 9563b2e9..c5f7cc86 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.1" +VERSION="1.5.2" IMAGE="docker/compose:$VERSION" From 2525752a05464d964b184a8a70d2b96674a69bbc Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Mon, 7 Dec 2015 09:10:19 +0100 Subject: [PATCH 153/359] Use more robust download URL for completions Signed-off-by: Harald Albers --- docs/completion.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/completion.md b/docs/completion.md index 3c2022d8..cac0d1a6 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -23,7 +23,7 @@ On a Mac, install with `brew install bash-completion` Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. - curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose --version | awk 'NR==1{print $NF}')/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose + curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose version --short)/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose Completion will be available upon next login. @@ -32,7 +32,7 @@ Completion will be available upon next login. Place the completion script in your `/path/to/zsh/completion`, using e.g. `~/.zsh/completion/` mkdir -p ~/.zsh/completion - curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose --version | awk 'NR==1{print $NF}')/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose + curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose version --short)/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose Include the directory in your `$fpath`, e.g. by adding in `~/.zshrc` From f2c232bb1013f6734f9556a67766373d08473c5b Mon Sep 17 00:00:00 2001 From: Nick Jones Date: Tue, 1 Dec 2015 16:20:36 +0000 Subject: [PATCH 154/359] Only allocate a tty if we detect one Signed-off-by: Nick Jones --- script/run.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/script/run.sh b/script/run.sh index 9563b2e9..6990799c 100755 --- a/script/run.sh +++ b/script/run.sh @@ -43,5 +43,11 @@ if [ -n "$HOME" ]; then VOLUMES="$VOLUMES -v $HOME:$HOME" fi +# Only allocate tty if we detect one +if [ -t 1 ]; then + DOCKER_RUN_OPTIONS="-ti" +else + DOCKER_RUN_OPTIONS="-i" +fi -exec docker run --rm -ti $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w $(pwd) $IMAGE $@ +exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w $(pwd) $IMAGE $@ From 437f3f8adbc3012e07b4629e8a668a7bb18e8ddd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Sat, 10 Oct 2015 14:41:17 -0400 Subject: [PATCH 155/359] Add docker-compose config subcommand. Signed-off-by: Daniel Nephin --- compose/cli/command.py | 5 ++- compose/cli/main.py | 38 ++++++++++++++++ tests/acceptance/cli_test.py | 44 +++++++++++++++---- .../fixtures/invalid-composefile/invalid.yml | 5 +++ 4 files changed, 82 insertions(+), 10 deletions(-) create mode 100644 tests/fixtures/invalid-composefile/invalid.yml diff --git a/compose/cli/command.py b/compose/cli/command.py index 157e0016..59f6c4bc 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -46,7 +46,7 @@ def friendly_error_message(): def project_from_options(base_dir, options): return get_project( base_dir, - get_config_path(options.get('--file')), + get_config_path_from_options(options), project_name=options.get('--project-name'), verbose=options.get('--verbose'), use_networking=options.get('--x-networking'), @@ -54,7 +54,8 @@ def project_from_options(base_dir, options): ) -def get_config_path(file_option): +def get_config_path_from_options(options): + file_option = options.get('--file') if file_option: return file_option diff --git a/compose/cli/main.py b/compose/cli/main.py index 62db5183..f30ea334 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -8,10 +8,12 @@ import sys from inspect import getdoc from operator import attrgetter +import yaml from docker.errors import APIError from requests.exceptions import ReadTimeout from .. import __version__ +from ..config import config from ..config import ConfigurationError from ..config import parse_environment from ..const import DEFAULT_TIMEOUT @@ -23,6 +25,7 @@ from ..service import BuildError from ..service import ConvergenceStrategy from ..service import NeedsBuildError from .command import friendly_error_message +from .command import get_config_path_from_options from .command import project_from_options from .docopt_command import DocoptCommand from .docopt_command import NoSuchCommand @@ -126,6 +129,7 @@ class TopLevelCommand(DocoptCommand): Commands: build Build or rebuild services + config Validate and view the compose file help Get help on a command kill Kill containers logs View output from containers @@ -158,6 +162,10 @@ class TopLevelCommand(DocoptCommand): handler(None, command_options) return + if options['COMMAND'] == 'config': + handler(options, command_options) + return + project = project_from_options(self.base_dir, options) with friendly_error_message(): handler(project, command_options) @@ -183,6 +191,36 @@ class TopLevelCommand(DocoptCommand): pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False))) + def config(self, config_options, options): + """ + Validate and view the compose file. + + Usage: config [options] + + Options: + -q, --quiet Only validate the configuration, don't print + anything. + --services Print the service names, one per line. + + """ + config_path = get_config_path_from_options(config_options) + compose_config = config.load(config.find(self.base_dir, config_path)) + + if options['--quiet']: + return + + if options['--services']: + print('\n'.join(service['name'] for service in compose_config)) + return + + compose_config = dict( + (service.pop('name'), service) for service in compose_config) + print(yaml.dump( + compose_config, + default_flow_style=False, + indent=2, + width=80)) + def help(self, project, options): """ Get help on a command. diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 99b78d08..0d26ea1f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -7,6 +7,7 @@ import subprocess import time from collections import namedtuple from operator import attrgetter +from textwrap import dedent from docker import errors @@ -90,10 +91,11 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/simple-composefile' def tearDown(self): - self.project.kill() - self.project.remove_stopped() - for container in self.project.containers(stopped=True, one_off=True): - container.remove(force=True) + if self.base_dir: + self.project.kill() + self.project.remove_stopped() + for container in self.project.containers(stopped=True, one_off=True): + container.remove(force=True) super(CLITestCase, self).tearDown() @property @@ -109,13 +111,39 @@ class CLITestCase(DockerClientTestCase): return wait_on_process(proc, returncode=returncode) def test_help(self): - old_base_dir = self.base_dir self.base_dir = 'tests/fixtures/no-composefile' result = self.dispatch(['help', 'up'], returncode=1) assert 'Usage: up [options] [SERVICE...]' in result.stderr - # self.project.kill() fails during teardown - # unless there is a composefile. - self.base_dir = old_base_dir + # Prevent tearDown from trying to create a project + self.base_dir = None + + def test_config_list_services(self): + result = self.dispatch(['config', '--services']) + assert set(result.stdout.rstrip().split('\n')) == {'simple', 'another'} + + def test_config_quiet_with_error(self): + self.base_dir = None + result = self.dispatch([ + '-f', 'tests/fixtures/invalid-composefile/invalid.yml', + 'config', '-q' + ], returncode=1) + assert "'notaservice' doesn't have any configuration" in result.stderr + + def test_config_quiet(self): + assert self.dispatch(['config', '-q']).stdout == '' + + def test_config_default(self): + result = self.dispatch(['config']) + assert dedent(""" + simple: + command: top + image: busybox:latest + """).lstrip() in result.stdout + assert dedent(""" + another: + command: top + image: busybox:latest + """).lstrip() in result.stdout def test_ps(self): self.project.get_service('simple').create_container() diff --git a/tests/fixtures/invalid-composefile/invalid.yml b/tests/fixtures/invalid-composefile/invalid.yml new file mode 100644 index 00000000..0e74be44 --- /dev/null +++ b/tests/fixtures/invalid-composefile/invalid.yml @@ -0,0 +1,5 @@ + +notaservice: oops + +web: + image: 'alpine:edge' From a5b48a3dc2b7e4545959801e5a669e7b6dbc2b23 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 8 Dec 2015 10:23:55 -0800 Subject: [PATCH 156/359] Add bash completion for config subcommand Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index c22e6abc..497a8184 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -102,6 +102,11 @@ _docker_compose_build() { } +_docker_compose_config() { + COMPREPLY=( $( compgen -W "--help --quiet -q --services" -- "$cur" ) ) +} + + _docker_compose_docker_compose() { case "$prev" in --file|-f) @@ -373,6 +378,7 @@ _docker_compose() { local commands=( build + config help kill logs From 999d15b2256cf279afc310ec8ae843f073a21d26 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Seguin?= Date: Tue, 8 Dec 2015 21:11:05 +0100 Subject: [PATCH 157/359] Remove unused functions in service MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Seguin --- compose/service.py | 28 ------------------------ tests/acceptance/cli_test.py | 9 ++++++-- tests/integration/service_test.py | 36 ------------------------------- 3 files changed, 7 insertions(+), 66 deletions(-) diff --git a/compose/service.py b/compose/service.py index 0b9a2aa3..0387b6e9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -141,34 +141,6 @@ class Service(object): for c in self.containers(stopped=True): self.start_container_if_stopped(c, **options) - # TODO: remove these functions, project takes care of starting/stopping, - def stop(self, **options): - for c in self.containers(): - log.info("Stopping %s" % c.name) - c.stop(**options) - - def pause(self, **options): - for c in self.containers(filters={'status': 'running'}): - log.info("Pausing %s" % c.name) - c.pause(**options) - - def unpause(self, **options): - for c in self.containers(filters={'status': 'paused'}): - log.info("Unpausing %s" % c.name) - c.unpause() - - def kill(self, **options): - for c in self.containers(): - log.info("Killing %s" % c.name) - c.kill(**options) - - def restart(self, **options): - for c in self.containers(stopped=True): - log.info("Restarting %s" % c.name) - c.restart(**options) - - # end TODO - def scale(self, desired_num, timeout=DEFAULT_TIMEOUT): """ Adjusts the number of containers to the specified number and ensures diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 0d26ea1f..66619629 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -52,6 +52,11 @@ def wait_on_condition(condition, delay=0.1, timeout=20): time.sleep(delay) +def kill_service(service): + for container in service.containers(): + container.kill() + + class ContainerCountCondition(object): def __init__(self, project, expected): @@ -637,13 +642,13 @@ class CLITestCase(DockerClientTestCase): def test_rm(self): service = self.project.get_service('simple') service.create_container() - service.kill() + kill_service(service) self.assertEqual(len(service.containers(stopped=True)), 1) self.dispatch(['rm', '--force'], None) self.assertEqual(len(service.containers(stopped=True)), 0) service = self.project.get_service('simple') service.create_container() - service.kill() + kill_service(service) self.assertEqual(len(service.containers(stopped=True)), 1) self.dispatch(['rm', '-f'], None) self.assertEqual(len(service.containers(stopped=True)), 0) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index b03baac5..5a809423 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -73,42 +73,6 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(service) self.assertEqual(service.containers()[0].name, 'composetest_web_1') - def test_start_stop(self): - service = self.create_service('scalingtest') - self.assertEqual(len(service.containers(stopped=True)), 0) - - service.create_container() - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) - - service.start() - self.assertEqual(len(service.containers()), 1) - self.assertEqual(len(service.containers(stopped=True)), 1) - - service.stop(timeout=1) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) - - service.stop(timeout=1) - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) - - def test_kill_remove(self): - service = self.create_service('scalingtest') - - create_and_start_container(service) - self.assertEqual(len(service.containers()), 1) - - remove_stopped(service) - self.assertEqual(len(service.containers()), 1) - - service.kill() - self.assertEqual(len(service.containers()), 0) - self.assertEqual(len(service.containers(stopped=True)), 1) - - remove_stopped(service) - self.assertEqual(len(service.containers(stopped=True)), 0) - def test_create_container_with_one_off(self): db = self.create_service('db') container = db.create_container(one_off=True) From 01bba5ea239fb747b9cd5ca34bf9c5f168f8a6d4 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Wed, 9 Dec 2015 08:55:59 +0100 Subject: [PATCH 158/359] Add zsh completion for config subcommand Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 0b50b535..67ca49bb 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -197,6 +197,12 @@ __docker-compose_subcommand() { '--pull[Always attempt to pull a newer version of the image.]' \ '*:services:__docker-compose_services_from_build' && ret=0 ;; + (config) + _arguments \ + $opts_help \ + '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ + '--services[Print the service names, one per line.]' && ret=0 + ;; (help) _arguments ':subcommand:__docker-compose_commands' && ret=0 ;; From fa3528ea2558cc4b1efca9ba5ea061b57191ae30 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 9 Dec 2015 16:32:39 -0800 Subject: [PATCH 159/359] Fix dns and dns_search when used strings and without extends. Signed-off-by: Daniel Nephin --- compose/config/config.py | 4 ++++ tests/unit/config/config_test.py | 21 +++++++++++++++++++-- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 853157ee..a2ccecc4 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -387,6 +387,10 @@ def process_service(service_config): if 'extra_hosts' in service_dict: service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) + for field in ['dns', 'dns_search']: + if field in service_dict: + service_dict[field] = to_list(service_dict[field]) + # TODO: move to a validate_service() if 'ulimits' in service_dict: validate_ulimits(service_dict['ulimits']) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 20705d55..2185b792 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -535,6 +535,23 @@ class ConfigTest(unittest.TestCase): })) assert "which is an invalid type" in exc.exconly() + def test_normalize_dns_options(self): + actual = config.load(build_config_details({ + 'web': { + 'image': 'alpine', + 'dns': '8.8.8.8', + 'dns_search': 'domain.local', + } + })) + assert actual == [ + { + 'name': 'web', + 'image': 'alpine', + 'dns': ['8.8.8.8'], + 'dns_search': ['domain.local'], + } + ] + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ @@ -1080,8 +1097,8 @@ class EnvTest(unittest.TestCase): {'foo': {'image': 'example', 'env_file': 'nonexistent.env'}}, working_dir='tests/fixtures/env')) - assert 'Couldn\'t find env file' in exc.exconly() - assert 'nonexistent.env' in exc.exconly() + assert 'Couldn\'t find env file' in exc.exconly() + assert 'nonexistent.env' in exc.exconly() @mock.patch.dict(os.environ) def test_resolve_environment_from_env_file_with_empty_values(self): From 1d3aeaaae7085995ab2f2d6b482b3dd64794d8aa Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 9 Dec 2015 16:03:26 -0800 Subject: [PATCH 160/359] Ignore extra coverge files These files are created because we run acceptance tests in a subprocess. They have the process id in their name, so they wont be removed by the normal coverage cleanup on each run. Signed-off-by: Daniel Nephin --- .gitignore | 2 +- script/clean | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index da728279..4b318e23 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ *.egg-info *.pyc -/.coverage +.coverage* /.tox /build /coverage-html diff --git a/script/clean b/script/clean index 08ba551a..35faf4db 100755 --- a/script/clean +++ b/script/clean @@ -2,5 +2,6 @@ set -e find . -type f -name '*.pyc' -delete +find . -name .coverage.* -delete find -name __pycache__ -delete rm -rf docs/_site build dist docker-compose.egg-info From 6d709caaa50d094c4948355bfab24c97d8f2ed94 Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Thu, 10 Dec 2015 22:56:45 +0200 Subject: [PATCH 161/359] Fixes incorrect network name shown in the log when no driver is specified Signed-off-by: Dimitar Bonev --- compose/project.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index af40d820..84413174 100644 --- a/compose/project.py +++ b/compose/project.py @@ -297,9 +297,13 @@ class Project(object): def ensure_network_exists(self): # TODO: recreate network if driver has changed? if self.get_network() is None: + driver_name = 'the default driver' + if self.network_driver: + driver_name = 'driver "{}"'.format(self.network_driver) + log.info( - 'Creating network "{}" with driver "{}"' - .format(self.name, self.network_driver) + 'Creating network "{}" with {}' + .format(self.name, driver_name) ) self.client.create_network(self.name, driver=self.network_driver) From c8f266b637c1ec8c8a017d18d4f2bf4339055ae0 Mon Sep 17 00:00:00 2001 From: Jean Praloran Date: Wed, 16 Dec 2015 08:19:28 +1300 Subject: [PATCH 162/359] add restarting status for human_readable_state Signed-off-by: Jean Praloran --- compose/container.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/compose/container.py b/compose/container.py index 1ca48380..8f96a944 100644 --- a/compose/container.py +++ b/compose/container.py @@ -115,6 +115,8 @@ class Container(object): def human_readable_state(self): if self.is_paused: return 'Paused' + if self.is_restarting: + return 'Restarting' if self.is_running: return 'Ghost' if self.get('State.Ghost') else 'Up' else: @@ -134,6 +136,10 @@ class Container(object): def is_running(self): return self.get('State.Running') + @property + def is_restarting(self): + return self.get('State.Restarting') + @property def is_paused(self): return self.get('State.Paused') From bc843d67588c6305b82cd2eb7c057cfdef02a4eb Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Tue, 15 Dec 2015 20:05:22 +0200 Subject: [PATCH 163/359] Start, restart, pause and unpause exit with non-zero if nothing to do Signed-off-by: Dimitar Bonev --- compose/cli/main.py | 18 ++++++++++++++---- compose/project.py | 17 +++++++++++++---- compose/service.py | 4 +++- tests/acceptance/cli_test.py | 16 ++++++++++++++++ 4 files changed, 46 insertions(+), 9 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index f30ea334..781c5762 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -265,7 +265,8 @@ class TopLevelCommand(DocoptCommand): Usage: pause [SERVICE...] """ - project.pause(service_names=options['SERVICE']) + containers = project.pause(service_names=options['SERVICE']) + exit_if(not containers, 'No containers to pause', 1) def port(self, project, options): """ @@ -476,7 +477,8 @@ class TopLevelCommand(DocoptCommand): Usage: start [SERVICE...] """ - project.start(service_names=options['SERVICE']) + containers = project.start(service_names=options['SERVICE']) + exit_if(not containers, 'No containers to start', 1) def stop(self, project, options): """ @@ -504,7 +506,8 @@ class TopLevelCommand(DocoptCommand): (default: 10) """ timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) - project.restart(service_names=options['SERVICE'], timeout=timeout) + containers = project.restart(service_names=options['SERVICE'], timeout=timeout) + exit_if(not containers, 'No containers to restart', 1) def unpause(self, project, options): """ @@ -512,7 +515,8 @@ class TopLevelCommand(DocoptCommand): Usage: unpause [SERVICE...] """ - project.unpause(service_names=options['SERVICE']) + containers = project.unpause(service_names=options['SERVICE']) + exit_if(not containers, 'No containers to unpause', 1) def up(self, project, options): """ @@ -674,3 +678,9 @@ def set_signal_handler(handler): def list_containers(containers): return ", ".join(c.name for c in containers) + + +def exit_if(condition, message, exit_code): + if condition: + log.error(message) + raise SystemExit(exit_code) diff --git a/compose/project.py b/compose/project.py index 84413174..1cb1daa7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -187,17 +187,24 @@ class Project(object): net_name)) def start(self, service_names=None, **options): + containers = [] for service in self.get_services(service_names): - service.start(**options) + service_containers = service.start(**options) + containers.extend(service_containers) + return containers def stop(self, service_names=None, **options): parallel.parallel_stop(self.containers(service_names), options) def pause(self, service_names=None, **options): - parallel.parallel_pause(reversed(self.containers(service_names)), options) + containers = self.containers(service_names) + parallel.parallel_pause(reversed(containers), options) + return containers def unpause(self, service_names=None, **options): - parallel.parallel_unpause(self.containers(service_names), options) + containers = self.containers(service_names) + parallel.parallel_unpause(containers, options) + return containers def kill(self, service_names=None, **options): parallel.parallel_kill(self.containers(service_names), options) @@ -206,7 +213,9 @@ class Project(object): parallel.parallel_remove(self.containers(service_names, stopped=True), options) def restart(self, service_names=None, **options): - parallel.parallel_restart(self.containers(service_names, stopped=True), options) + containers = self.containers(service_names, stopped=True) + parallel.parallel_restart(containers, options) + return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): for service in self.get_services(service_names): diff --git a/compose/service.py b/compose/service.py index 0387b6e9..e04ef271 100644 --- a/compose/service.py +++ b/compose/service.py @@ -138,8 +138,10 @@ class Service(object): raise ValueError("No container found for %s_%s" % (self.name, number)) def start(self, **options): - for c in self.containers(stopped=True): + containers = self.containers(stopped=True) + for c in containers: self.start_container_if_stopped(c, **options) + return containers def scale(self, desired_num, timeout=DEFAULT_TIMEOUT): """ diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 66619629..17b83fe3 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -664,6 +664,10 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) + def test_start_no_containers(self): + result = self.dispatch(['start'], returncode=1) + assert 'No containers to start' in result.stderr + def test_pause_unpause(self): self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') @@ -675,6 +679,14 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['unpause'], None) self.assertFalse(service.containers()[0].is_paused) + def test_pause_no_containers(self): + result = self.dispatch(['pause'], returncode=1) + assert 'No containers to pause' in result.stderr + + def test_unpause_no_containers(self): + result = self.dispatch(['unpause'], returncode=1) + assert 'No containers to unpause' in result.stderr + def test_logs_invalid_service_name(self): self.dispatch(['logs', 'madeupname'], returncode=1) @@ -737,6 +749,10 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['restart', '-t', '1'], None) self.assertEqual(len(service.containers(stopped=False)), 1) + def test_restart_no_containers(self): + result = self.dispatch(['restart'], returncode=1) + assert 'No containers to restart' in result.stderr + def test_scale(self): project = self.project From a5420412646d5d8912949d2b114f364162fd2cd1 Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Wed, 16 Dec 2015 21:25:30 +0200 Subject: [PATCH 164/359] Added support for cpu_quota flag Signed-off-by: Dimitar Bonev --- compose/config/config.py | 1 + compose/config/fields_schema.json | 1 + compose/service.py | 2 ++ docs/compose-file.md | 3 ++- tests/integration/service_test.py | 6 ++++++ 5 files changed, 12 insertions(+), 1 deletion(-) diff --git a/compose/config/config.py b/compose/config/config.py index a2ccecc4..0c644833 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -31,6 +31,7 @@ DOCKER_CONFIG_KEYS = [ 'cap_drop', 'cgroup_parent', 'command', + 'cpu_quota', 'cpu_shares', 'cpuset', 'detach', diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index 7d5220e3..fdf56fd9 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -29,6 +29,7 @@ }, "container_name": {"type": "string"}, "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, diff --git a/compose/service.py b/compose/service.py index e04ef271..3b54c2a7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -59,6 +59,7 @@ DOCKER_START_KEYS = [ 'restart', 'volumes_from', 'security_opt', + 'cpu_quota', ] @@ -610,6 +611,7 @@ class Service(object): security_opt=options.get('security_opt'), ipc_mode=options.get('ipc'), cgroup_parent=options.get('cgroup_parent'), + cpu_quota=options.get('cpu_quota'), ) def build(self, no_cache=False, pull=False, force_rm=False): diff --git a/docs/compose-file.md b/docs/compose-file.md index 800d2aa9..c0f2efbe 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -385,12 +385,13 @@ specifying read-only access(``ro``) or read-write(``rw``). - container_name - service_name:rw -### cpu\_shares, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir +### cpu\_shares, cpu\_quota, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir Each of these is a single value, analogous to its [docker run](https://docs.docker.com/reference/run/) counterpart. cpu_shares: 73 + cpu_quota: 50000 cpuset: 0,1 entrypoint: /code/entrypoint.sh diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 5a809423..59ba487a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -102,6 +102,12 @@ class ServiceTest(DockerClientTestCase): container.start() self.assertEqual(container.get('HostConfig.CpuShares'), 73) + def test_create_container_with_cpu_quota(self): + service = self.create_service('db', cpu_quota=40000) + container = service.create_container() + container.start() + self.assertEqual(container.get('HostConfig.CpuQuota'), 40000) + def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) From 2e9a49b4eb48d7611543bf5cb34130e8f5448dff Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 17 Dec 2015 17:50:45 +0000 Subject: [PATCH 165/359] Clarify behaviour of 'rm' Signed-off-by: Aanand Prasad --- compose/cli/main.py | 5 +++++ docs/reference/rm.md | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index f30ea334..8799880f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -343,6 +343,11 @@ class TopLevelCommand(DocoptCommand): """ Remove stopped service containers. + By default, volumes attached to containers will not be removed. You can see all + volumes with `docker volume ls`. + + Any data which is not in a volume will be lost. + Usage: rm [options] [SERVICE...] Options: diff --git a/docs/reference/rm.md b/docs/reference/rm.md index 2ed959e4..f8479224 100644 --- a/docs/reference/rm.md +++ b/docs/reference/rm.md @@ -20,3 +20,8 @@ Options: ``` Removes stopped service containers. + +By default, volumes attached to containers will not be removed. You can see all +volumes with `docker volume ls`. + +Any data which is not in a volume will be lost. From 3c76d5a46770b68910223456226a5353ce2f51c3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Seguin?= Date: Mon, 14 Dec 2015 22:46:13 +0100 Subject: [PATCH 166/359] Add docker-compose create command. Closes #1125 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Seguin --- compose/cli/main.py | 22 +++++++++++ compose/project.py | 19 +++++++-- compose/service.py | 20 ++++++---- tests/acceptance/cli_test.py | 46 ++++++++++++++++++++++ tests/integration/project_test.py | 65 +++++++++++++++++++++++++++++++ tests/integration/service_test.py | 18 +++++++++ 6 files changed, 179 insertions(+), 11 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index f30ea334..a98795e3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -130,6 +130,7 @@ class TopLevelCommand(DocoptCommand): Commands: build Build or rebuild services config Validate and view the compose file + create Create services help Get help on a command kill Kill containers logs View output from containers @@ -221,6 +222,27 @@ class TopLevelCommand(DocoptCommand): indent=2, width=80)) + def create(self, project, options): + """ + Creates containers for a service. + + Usage: create [options] [SERVICE...] + + Options: + --force-recreate Recreate containers even if their configuration and + image haven't changed. Incompatible with --no-recreate. + --no-recreate If containers already exist, don't recreate them. + Incompatible with --force-recreate. + --no-build Don't build an image, even if it's missing + """ + service_names = options['SERVICE'] + + project.create( + service_names=service_names, + strategy=convergence_strategy_from_opts(options), + do_build=not options['--no-build'] + ) + def help(self, project, options): """ Get help on a command. diff --git a/compose/project.py b/compose/project.py index 84413174..d4046856 100644 --- a/compose/project.py +++ b/compose/project.py @@ -123,6 +123,12 @@ class Project(object): [uniques.append(s) for s in services if s not in uniques] return uniques + def get_services_without_duplicate(self, service_names=None, include_deps=False): + services = self.get_services(service_names, include_deps) + for service in services: + service.remove_duplicate_containers() + return services + def get_links(self, service_dict): links = [] if 'links' in service_dict: @@ -215,6 +221,14 @@ class Project(object): else: log.info('%s uses an image, skipping' % service.name) + def create(self, service_names=None, strategy=ConvergenceStrategy.changed, do_build=True): + services = self.get_services_without_duplicate(service_names, include_deps=True) + + plans = self._get_convergence_plans(services, strategy) + + for service in services: + service.execute_convergence_plan(plans[service.name], do_build, detached=True, start=False) + def up(self, service_names=None, start_deps=True, @@ -223,10 +237,7 @@ class Project(object): timeout=DEFAULT_TIMEOUT, detached=False): - services = self.get_services(service_names, include_deps=start_deps) - - for service in services: - service.remove_duplicate_containers() + services = self.get_services_without_duplicate(service_names, include_deps=start_deps) plans = self._get_convergence_plans(services, strategy) diff --git a/compose/service.py b/compose/service.py index 0387b6e9..791e57ae 100644 --- a/compose/service.py +++ b/compose/service.py @@ -328,7 +328,8 @@ class Service(object): plan, do_build=True, timeout=DEFAULT_TIMEOUT, - detached=False): + detached=False, + start=True): (action, containers) = plan should_attach_logs = not detached @@ -338,7 +339,8 @@ class Service(object): if should_attach_logs: container.attach_log_stream() - container.start() + if start: + container.start() return [container] @@ -348,14 +350,16 @@ class Service(object): container, do_build=do_build, timeout=timeout, - attach_logs=should_attach_logs + attach_logs=should_attach_logs, + start_new_container=start ) for container in containers ] elif action == 'start': - for container in containers: - self.start_container_if_stopped(container, attach_logs=should_attach_logs) + if start: + for container in containers: + self.start_container_if_stopped(container, attach_logs=should_attach_logs) return containers @@ -373,7 +377,8 @@ class Service(object): container, do_build=False, timeout=DEFAULT_TIMEOUT, - attach_logs=False): + attach_logs=False, + start_new_container=True): """Recreate a container. The original container is renamed to a temporary name so that data @@ -392,7 +397,8 @@ class Service(object): ) if attach_logs: new_container.attach_log_stream() - new_container.start() + if start_new_container: + new_container.start() container.remove() return new_container diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 66619629..032b507d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -264,6 +264,52 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_create(self): + self.dispatch(['create']) + service = self.project.get_service('simple') + another = self.project.get_service('another') + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(another.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + self.assertEqual(len(another.containers(stopped=True)), 1) + + def test_create_with_force_recreate(self): + self.dispatch(['create'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + + old_ids = [c.id for c in service.containers(stopped=True)] + + self.dispatch(['create', '--force-recreate'], None) + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + + new_ids = [c.id for c in service.containers(stopped=True)] + + self.assertNotEqual(old_ids, new_ids) + + def test_create_with_no_recreate(self): + self.dispatch(['create'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + + old_ids = [c.id for c in service.containers(stopped=True)] + + self.dispatch(['create', '--no-recreate'], None) + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + + new_ids = [c.id for c in service.containers(stopped=True)] + + self.assertEqual(old_ids, new_ids) + + def test_create_with_force_recreate_and_no_recreate(self): + self.dispatch( + ['create', '--force-recreate', '--no-recreate'], + returncode=1) + def test_up_detached(self): self.dispatch(['up', '-d']) service = self.project.get_service('simple') diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 443ff978..229f653a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -213,6 +213,71 @@ class ProjectTest(DockerClientTestCase): project.remove_stopped() self.assertEqual(len(project.containers(stopped=True)), 0) + def test_create(self): + web = self.create_service('web') + db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) + project = Project('composetest', [web, db], self.client) + + project.create(['db']) + self.assertEqual(len(project.containers()), 0) + self.assertEqual(len(project.containers(stopped=True)), 1) + self.assertEqual(len(db.containers()), 0) + self.assertEqual(len(db.containers(stopped=True)), 1) + self.assertEqual(len(web.containers(stopped=True)), 0) + + def test_create_twice(self): + web = self.create_service('web') + db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) + project = Project('composetest', [web, db], self.client) + + project.create(['db', 'web']) + project.create(['db', 'web']) + self.assertEqual(len(project.containers()), 0) + self.assertEqual(len(project.containers(stopped=True)), 2) + self.assertEqual(len(db.containers()), 0) + self.assertEqual(len(db.containers(stopped=True)), 1) + self.assertEqual(len(web.containers()), 0) + self.assertEqual(len(web.containers(stopped=True)), 1) + + def test_create_with_links(self): + db = self.create_service('db') + web = self.create_service('web', links=[(db, 'db')]) + project = Project('composetest', [db, web], self.client) + + project.create(['web']) + self.assertEqual(len(project.containers()), 0) + self.assertEqual(len(project.containers(stopped=True)), 2) + self.assertEqual(len(db.containers()), 0) + self.assertEqual(len(db.containers(stopped=True)), 1) + self.assertEqual(len(web.containers()), 0) + self.assertEqual(len(web.containers(stopped=True)), 1) + + def test_create_strategy_always(self): + db = self.create_service('db') + project = Project('composetest', [db], self.client) + project.create(['db']) + old_id = project.containers(stopped=True)[0].id + + project.create(['db'], strategy=ConvergenceStrategy.always) + self.assertEqual(len(project.containers()), 0) + self.assertEqual(len(project.containers(stopped=True)), 1) + + db_container = project.containers(stopped=True)[0] + self.assertNotEqual(db_container.id, old_id) + + def test_create_strategy_never(self): + db = self.create_service('db') + project = Project('composetest', [db], self.client) + project.create(['db']) + old_id = project.containers(stopped=True)[0].id + + project.create(['db'], strategy=ConvergenceStrategy.never) + self.assertEqual(len(project.containers()), 0) + self.assertEqual(len(project.containers(stopped=True)), 1) + + db_container = project.containers(stopped=True)[0] + self.assertEqual(db_container.id, old_id) + def test_project_up(self): web = self.create_service('web') db = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 5a809423..84ef696e 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -333,6 +333,24 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + def test_execute_convergence_plan_without_start(self): + service = self.create_service( + 'db', + build='tests/fixtures/dockerfile-with-volume' + ) + + containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False) + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + + containers = service.execute_convergence_plan(ConvergencePlan('recreate', containers), start=False) + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + + service.execute_convergence_plan(ConvergencePlan('start', containers), start=False) + self.assertEqual(len(service.containers()), 0) + self.assertEqual(len(service.containers(stopped=True)), 1) + def test_start_container_passes_through_options(self): db = self.create_service('db') create_and_start_container(db, environment={'FOO': 'BAR'}) From 5ed559fa0ef89c944734226278155050f994ece0 Mon Sep 17 00:00:00 2001 From: Sebastiaan van Stijn Date: Mon, 21 Dec 2015 01:52:54 +0100 Subject: [PATCH 167/359] Update links Updates some links to their new locations, and replaces some http:// with https:// links. Signed-off-by: Sebastiaan van Stijn --- CHANGELOG.md | 4 ++-- CONTRIBUTING.md | 2 +- README.md | 2 +- compose/cli/errors.py | 4 ++-- docs/compose-file.md | 10 +++++----- docs/django.md | 6 +++--- docs/env.md | 2 +- docs/gettingstarted.md | 2 +- docs/index.md | 2 +- docs/install.md | 6 +++--- docs/production.md | 4 ++-- docs/rails.md | 4 ++-- docs/wordpress.md | 8 ++++---- experimental/compose_swarm_networking.md | 4 ++-- 14 files changed, 30 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7b6e0dd3..79aee75f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -301,8 +301,8 @@ Several new configuration keys have been added to `docker-compose.yml`: - `pid: host`, like `docker run --pid=host`, lets you reuse the same PID namespace as the host machine. - `cpuset`, like `docker run --cpuset-cpus`, lets you specify which CPUs to allow execution in. - `read_only`, like `docker run --read-only`, lets you mount a container's filesystem as read-only. -- `security_opt`, like `docker run --security-opt`, lets you specify [security options](https://docs.docker.com/reference/run/#security-configuration). -- `log_driver`, like `docker run --log-driver`, lets you specify a [log driver](https://docs.docker.com/reference/run/#logging-drivers-log-driver). +- `security_opt`, like `docker run --security-opt`, lets you specify [security options](https://docs.docker.com/engine/reference/run/#security-configuration). +- `log_driver`, like `docker run --log-driver`, lets you specify a [log driver](https://docs.docker.com/engine/reference/run/#logging-drivers-log-driver). Many bugs have been fixed, including the following: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 62bf415c..66224752 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -43,7 +43,7 @@ To run the style checks at any time run `tox -e pre-commit`. ## Submitting a pull request -See Docker's [basic contribution workflow](https://docs.docker.com/project/make-a-contribution/#the-basic-contribution-workflow) for a guide on how to submit a pull request for code or documentation. +See Docker's [basic contribution workflow](https://docs.docker.com/opensource/workflow/make-a-contribution/#the-basic-contribution-workflow) for a guide on how to submit a pull request for code or documentation. ## Running the test suite diff --git a/README.md b/README.md index c9b4729a..b60a7eee 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Compose has commands for managing the whole lifecycle of your application: Installation and documentation ------------------------------ -- Full documentation is available on [Docker's website](http://docs.docker.com/compose/). +- Full documentation is available on [Docker's website](https://docs.docker.com/compose/). - If you have any questions, you can talk in real-time with other developers in the #docker-compose IRC channel on Freenode. [Click here to join using IRCCloud.](https://www.irccloud.com/invite?hostname=irc.freenode.net&channel=%23docker-compose) - Code repository for Compose is on [Github](https://github.com/docker/compose) - If you find any problems please fill out an [issue](https://github.com/docker/compose/issues/new) diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 244897f8..ca4413bd 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -27,7 +27,7 @@ class DockerNotFoundUbuntu(UserError): super(DockerNotFoundUbuntu, self).__init__(""" Couldn't connect to Docker daemon. You might need to install Docker: - http://docs.docker.io/en/latest/installation/ubuntulinux/ + https://docs.docker.com/engine/installation/ubuntulinux/ """) @@ -36,7 +36,7 @@ class DockerNotFoundGeneric(UserError): super(DockerNotFoundGeneric, self).__init__(""" Couldn't connect to Docker daemon. You might need to install Docker: - http://docs.docker.io/en/latest/installation/ + https://docs.docker.com/engine/installation/ """) diff --git a/docs/compose-file.md b/docs/compose-file.md index c0f2efbe..2a6028b8 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -231,7 +231,7 @@ pull if it doesn't exist locally. ### labels -Add metadata to containers using [Docker labels](http://docs.docker.com/userguide/labels-custom-metadata/). You can use either an array or a dictionary. +Add metadata to containers using [Docker labels](https://docs.docker.com/engine/userguide/labels-custom-metadata/). You can use either an array or a dictionary. It's recommended that you use reverse-DNS notation to prevent your labels from conflicting with those used by other software. @@ -269,7 +269,7 @@ reference](env.md) for details. ### log_driver Specify a logging driver for the service's containers, as with the ``--log-driver`` -option for docker run ([documented here](https://docs.docker.com/reference/logging/overview/)). +option for docker run ([documented here](https://docs.docker.com/engine/reference/logging/overview/)). The default value is json-file. @@ -371,8 +371,8 @@ a `volume_driver`. > Note: No path expansion will be done if you have also specified a > `volume_driver`. -See [Docker Volumes](https://docs.docker.com/userguide/dockervolumes/) and -[Volume Plugins](https://docs.docker.com/extend/plugins_volume/) for more +See [Docker Volumes](https://docs.docker.com/engine/userguide/dockervolumes/) and +[Volume Plugins](https://docs.docker.com/engine/extend/plugins_volume/) for more information. ### volumes_from @@ -388,7 +388,7 @@ specifying read-only access(``ro``) or read-write(``rw``). ### cpu\_shares, cpu\_quota, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir Each of these is a single value, analogous to its -[docker run](https://docs.docker.com/reference/run/) counterpart. +[docker run](https://docs.docker.com/engine/reference/run/) counterpart. cpu_shares: 73 cpu_quota: 50000 diff --git a/docs/django.md b/docs/django.md index b503e574..2d4fdaf9 100644 --- a/docs/django.md +++ b/docs/django.md @@ -30,8 +30,8 @@ and a `docker-compose.yml` file. The Dockerfile defines an application's image content via one or more build commands that configure that image. Once built, you can run the image in a container. For more information on `Dockerfiles`, see the [Docker user - guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) - and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) + and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). 3. Add the following content to the `Dockerfile`. @@ -144,7 +144,7 @@ In this section, you set up the database connection for Django. } These settings are determined by the - [postgres](https://registry.hub.docker.com/_/postgres/) Docker image + [postgres](https://hub.docker.com/_/postgres/) Docker image specified in `docker-compose.yml`. 3. Save and close the file. diff --git a/docs/env.md b/docs/env.md index d7b51ba2..c0e03a4e 100644 --- a/docs/env.md +++ b/docs/env.md @@ -35,7 +35,7 @@ Protocol (tcp or udp), e.g. `DB_PORT_5432_TCP_PROTO=tcp` name\_NAME
Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` -[Docker links]: http://docs.docker.com/userguide/dockerlinks/ +[Docker links]: https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/ ## Related Information diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index f685bf38..bb3d51e9 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -77,7 +77,7 @@ dependencies the Python application requires, including Python itself. * Install the Python dependencies. * Set the default command for the container to `python app.py` - For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). + For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). 2. Build the image. diff --git a/docs/index.md b/docs/index.md index 8b32a754..36b93a39 100644 --- a/docs/index.md +++ b/docs/index.md @@ -183,4 +183,4 @@ individuals, we have a number of open channels for communication. * To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). -For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/project/get-help/). +For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/opensource/get-help/). diff --git a/docs/install.md b/docs/install.md index 7c73baaa..417a48c1 100644 --- a/docs/install.md +++ b/docs/install.md @@ -20,11 +20,11 @@ To install Compose, do the following: 1. Install Docker Engine version 1.7.1 or greater: - * Mac OS X installation (Toolbox installation includes both Engine and Compose) + * Mac OS X installation (Toolbox installation includes both Engine and Compose) - * Ubuntu installation + * Ubuntu installation - * other system installations + * other system installations 2. Mac OS X users are done installing. Others should continue to the next step. diff --git a/docs/production.md b/docs/production.md index 0a5e77b5..46e221bb 100644 --- a/docs/production.md +++ b/docs/production.md @@ -60,7 +60,7 @@ recreating any services which `web` depends on. You can use Compose to deploy an app to a remote Docker host by setting the `DOCKER_HOST`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` environment variables appropriately. For tasks like this, -[Docker Machine](https://docs.docker.com/machine) makes managing local and +[Docker Machine](https://docs.docker.com/machine/) makes managing local and remote Docker hosts very easy, and is recommended even if you're not deploying remotely. @@ -69,7 +69,7 @@ commands will work with no further configuration. ### Running Compose on a Swarm cluster -[Docker Swarm](https://docs.docker.com/swarm), a Docker-native clustering +[Docker Swarm](https://docs.docker.com/swarm/), a Docker-native clustering system, exposes the same API as a single Docker host, which means you can use Compose against a Swarm instance and run your apps across multiple hosts. diff --git a/docs/rails.md b/docs/rails.md index d3f1707c..e3daff25 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -30,7 +30,7 @@ Dockerfile consists of: RUN bundle install ADD . /myapp -That'll put your application code inside an image that will build a container with Ruby, Bundler and all your dependencies inside it. For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). +That'll put your application code inside an image that will build a container with Ruby, Bundler and all your dependencies inside it. For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten in a moment by `rails new`. @@ -128,7 +128,7 @@ Finally, you need to create the database. In another terminal, run: $ docker-compose run web rake db:create -That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. +That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](https://docs.docker.com/machine/), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. ## More Compose documentation diff --git a/docs/wordpress.md b/docs/wordpress.md index 15746a75..84010491 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -28,9 +28,9 @@ to the name of your project. Next, inside that directory, create a `Dockerfile`, a file that defines what environment your app is going to run in. For more information on how to write Dockerfiles, see the -[Docker user guide](https://docs.docker.com/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the -[Dockerfile reference](http://docs.docker.com/reference/builder/). In this case, -your Dockerfile should be: +[Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the +[Dockerfile reference](https://docs.docker.com/engine/reference/builder/). In +this case, your Dockerfile should be: FROM orchardup/php5 ADD . /code @@ -89,7 +89,7 @@ configuration at the `db` container: With those four files in place, run `docker-compose up` inside your WordPress directory and it'll pull and build the needed images, and then start the web and -database containers. If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` gives you the machine address and you can open `http://MACHINE_VM_IP:8000` in a browser. +database containers. If you're using [Docker Machine](https://docs.docker.com/machine/), then `docker-machine ip MACHINE_VM` gives you the machine address and you can open `http://MACHINE_VM_IP:8000` in a browser. ## More Compose documentation diff --git a/experimental/compose_swarm_networking.md b/experimental/compose_swarm_networking.md index e3dcf6cc..b1fb25dc 100644 --- a/experimental/compose_swarm_networking.md +++ b/experimental/compose_swarm_networking.md @@ -15,9 +15,9 @@ Before you start, you’ll need to install the experimental build of Docker, and $ curl -L https://experimental.docker.com/builds/Darwin/x86_64/docker-latest > /usr/local/bin/docker $ chmod +x /usr/local/bin/docker -- To install Machine, follow the instructions [here](http://docs.docker.com/machine/). +- To install Machine, follow the instructions [here](https://docs.docker.com/machine/install-machine/). -- To install Compose, follow the instructions [here](http://docs.docker.com/compose/install/). +- To install Compose, follow the instructions [here](https://docs.docker.com/compose/install/). You’ll also need a [Docker Hub](https://hub.docker.com/account/signup/) account and a [Digital Ocean](https://www.digitalocean.com/) account. From adde8058292076a564c6fce36f8fc75c2ac52381 Mon Sep 17 00:00:00 2001 From: Tomas Tomecek Date: Sat, 26 Dec 2015 11:03:58 +0100 Subject: [PATCH 168/359] allow running compose from git with: ``` $ git clone docker/compose && cd compose $ export PYTHONPATH="$PWD:$PYTHONPATH" $ python -m compose --help ``` Signed-off-by: Tomas Tomecek --- compose/__main__.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 compose/__main__.py diff --git a/compose/__main__.py b/compose/__main__.py new file mode 100644 index 00000000..199ba2ae --- /dev/null +++ b/compose/__main__.py @@ -0,0 +1,3 @@ +from compose.cli.main import main + +main() From 778c213dfc9c65ac27d4f527018eb66d841d890e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 28 Dec 2015 16:57:55 -0500 Subject: [PATCH 169/359] Fix signal handlers by moving shutdown logic out of handler. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 45 +++++++++++++++--------------------- compose/cli/signals.py | 18 +++++++++++++++ tests/acceptance/cli_test.py | 3 ++- tests/unit/cli/main_test.py | 2 +- 4 files changed, 40 insertions(+), 28 deletions(-) create mode 100644 compose/cli/signals.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 4a766133..e360dddd 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -3,7 +3,6 @@ from __future__ import unicode_literals import logging import re -import signal import sys from inspect import getdoc from operator import attrgetter @@ -12,6 +11,7 @@ import yaml from docker.errors import APIError from requests.exceptions import ReadTimeout +from . import signals from .. import __version__ from ..config import config from ..config import ConfigurationError @@ -655,20 +655,19 @@ def run_one_off_container(container_options, project, service, options): if options['--rm']: project.client.remove_container(container.id, force=True) - def force_shutdown(signal, frame): + signals.set_signal_handler_to_shutdown() + try: + try: + dockerpty.start(project.client, container.id, interactive=not options['-T']) + exit_code = container.wait() + except signals.ShutdownException: + project.client.stop(container.id) + exit_code = 1 + except signals.ShutdownException: project.client.kill(container.id) remove_container(force=True) sys.exit(2) - def shutdown(signal, frame): - set_signal_handler(force_shutdown) - project.client.stop(container.id) - remove_container() - sys.exit(1) - - set_signal_handler(shutdown) - dockerpty.start(project.client, container.id, interactive=not options['-T']) - exit_code = container.wait() remove_container() sys.exit(exit_code) @@ -683,25 +682,19 @@ def build_log_printer(containers, service_names, monochrome): def attach_to_logs(project, log_printer, service_names, timeout): + print("Attaching to", list_containers(log_printer.containers)) + signals.set_signal_handler_to_shutdown() - def force_shutdown(signal, frame): + try: + try: + log_printer.run() + except signals.ShutdownException: + print("Gracefully stopping... (press Ctrl+C again to force)") + project.stop(service_names=service_names, timeout=timeout) + except signals.ShutdownException: project.kill(service_names=service_names) sys.exit(2) - def shutdown(signal, frame): - set_signal_handler(force_shutdown) - print("Gracefully stopping... (press Ctrl+C again to force)") - project.stop(service_names=service_names, timeout=timeout) - - print("Attaching to", list_containers(log_printer.containers)) - set_signal_handler(shutdown) - log_printer.run() - - -def set_signal_handler(handler): - signal.signal(signal.SIGINT, handler) - signal.signal(signal.SIGTERM, handler) - def list_containers(containers): return ", ".join(c.name for c in containers) diff --git a/compose/cli/signals.py b/compose/cli/signals.py new file mode 100644 index 00000000..38474ba4 --- /dev/null +++ b/compose/cli/signals.py @@ -0,0 +1,18 @@ +import signal + + +class ShutdownException(Exception): + pass + + +def shutdown(signal, frame): + raise ShutdownException() + + +def set_signal_handler(handler): + signal.signal(signal.SIGINT, handler) + signal.signal(signal.SIGTERM, handler) + + +def set_signal_handler_to_shutdown(): + set_signal_handler(shutdown) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e6fa38a8..694f8583 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -86,7 +86,8 @@ class ContainerStateCondition(object): return False def __str__(self): - return "waiting for container to have state %s" % self.expected + state = 'running' if self.running else 'stopped' + return "waiting for container to be %s" % state class CLITestCase(DockerClientTestCase): diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index db37ac1a..b63ac746 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -54,7 +54,7 @@ class CLIMainTestCase(unittest.TestCase): service_names = ['web', 'db'] timeout = 12 - with mock.patch('compose.cli.main.signal', autospec=True) as mock_signal: + with mock.patch('compose.cli.main.signals.signal', autospec=True) as mock_signal: attach_to_logs(project, log_printer, service_names, timeout) assert mock_signal.signal.mock_calls == [ From d4e913e42cf7141421d76407ce2ca22935bb1a25 Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Thu, 31 Dec 2015 19:04:38 -0800 Subject: [PATCH 170/359] Fixing TODO visible in docs Signed-off-by: Mary Anthony --- docs/networking.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/networking.md b/docs/networking.md index 718d56c7..91ac0b73 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -79,9 +79,11 @@ You can specify which one to use with the `--x-network-driver` flag: $ docker-compose --x-networking --x-network-driver=overlay up + ## Custom container network modes From 2acc29cf1c25be7078ebeaf70e9d45f8048198da Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 4 Jan 2016 15:35:57 -0500 Subject: [PATCH 171/359] Remove support for fig.yaml, FIG_FILE, and FIG_PROJECT_NAME. Signed-off-by: Daniel Nephin --- compose/cli/command.py | 15 ++------------- compose/config/config.py | 6 ------ contrib/completion/bash/docker-compose | 2 +- contrib/completion/zsh/_docker-compose | 2 +- tests/unit/cli_test.py | 7 ------- tests/unit/config/config_test.py | 2 -- 6 files changed, 4 insertions(+), 30 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index 59f6c4bc..21d6ff0d 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -59,11 +59,7 @@ def get_config_path_from_options(options): if file_option: return file_option - if 'FIG_FILE' in os.environ: - log.warn('The FIG_FILE environment variable is deprecated.') - log.warn('Please use COMPOSE_FILE instead.') - - config_file = os.environ.get('COMPOSE_FILE') or os.environ.get('FIG_FILE') + config_file = os.environ.get('COMPOSE_FILE') return [config_file] if config_file else None @@ -96,14 +92,7 @@ def get_project_name(working_dir, project_name=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - if 'FIG_PROJECT_NAME' in os.environ: - log.warn('The FIG_PROJECT_NAME environment variable is deprecated.') - log.warn('Please use COMPOSE_PROJECT_NAME instead.') - - project_name = ( - project_name or - os.environ.get('COMPOSE_PROJECT_NAME') or - os.environ.get('FIG_PROJECT_NAME')) + project_name = project_name or os.environ.get('COMPOSE_PROJECT_NAME') if project_name is not None: return normalize_name(project_name) diff --git a/compose/config/config.py b/compose/config/config.py index 0c644833..195665b5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -88,8 +88,6 @@ DOCKER_VALID_URL_PREFIXES = ( SUPPORTED_FILENAMES = [ 'docker-compose.yml', 'docker-compose.yaml', - 'fig.yml', - 'fig.yaml', ] DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' @@ -162,10 +160,6 @@ def get_default_config_files(base_dir): log.warn("Found multiple config files with supported names: %s", ", ".join(candidates)) log.warn("Using %s\n", winner) - if winner.startswith("fig."): - log.warn("%s is deprecated and will not be supported in future. " - "Please rename your config file to docker-compose.yml\n" % winner) - return [os.path.join(path, winner)] + get_default_override_file(path) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 497a8184..c18e4f6d 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -28,7 +28,7 @@ __docker_compose_nospace() { # Support for these filenames might be dropped in some future version. __docker_compose_compose_file() { local file - for file in docker-compose.y{,a}ml fig.y{,a}ml ; do + for file in docker-compose.y{,a}ml ; do [ -e $file ] && { echo $file return diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 67ca49bb..35c2b996 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -24,7 +24,7 @@ # Support for these filenames might be dropped in some future version. __docker-compose_compose_file() { local file - for file in docker-compose.y{,a}ml fig.y{,a}ml ; do + for file in docker-compose.y{,a}ml ; do [ -e $file ] && { echo $file return diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 61cef6f6..a5767097 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -42,13 +42,6 @@ class CLITestCase(unittest.TestCase): project_name = get_project_name(None, project_name=name) self.assertEquals('explicitprojectname', project_name) - def test_project_name_from_environment_old_var(self): - name = 'namefromenv' - with mock.patch.dict(os.environ): - os.environ['FIG_PROJECT_NAME'] = name - project_name = get_project_name(None) - self.assertEquals(project_name, name) - def test_project_name_from_environment_new_var(self): name = 'namefromenv' with mock.patch.dict(os.environ): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 2185b792..e975cb9d 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1616,8 +1616,6 @@ class GetDefaultConfigFilesTestCase(unittest.TestCase): files = [ 'docker-compose.yml', 'docker-compose.yaml', - 'fig.yml', - 'fig.yaml', ] def test_get_config_path_default_file_in_basedir(self): From ad9011ed96dc35ef6880badd1068276a4493da37 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 5 Jan 2016 17:30:27 -0500 Subject: [PATCH 172/359] Don't warn when the container volume is specified as a compose option. Signed-off-by: Daniel Nephin --- compose/service.py | 1 + tests/unit/service_test.py | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/compose/service.py b/compose/service.py index d1509871..5540d734 100644 --- a/compose/service.py +++ b/compose/service.py @@ -880,6 +880,7 @@ def warn_on_masked_volume(volumes_option, container_volumes, service): for volume in volumes_option: if ( + volume.external and volume.internal in container_volumes and container_volumes.get(volume.internal) != volume.external ): diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 1c8b441f..637bf3ba 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -742,6 +742,18 @@ class ServiceVolumesTest(unittest.TestCase): assert not mock_log.warn.called + def test_warn_on_masked_no_warning_with_container_only_option(self): + volumes_option = [VolumeSpec(None, '/path', 'rw')] + container_volumes = [ + VolumeSpec('/var/lib/docker/volume/path', '/path', 'rw') + ] + service = 'service_name' + + with mock.patch('compose.service.log', autospec=True) as mock_log: + warn_on_masked_volume(volumes_option, container_volumes, service) + + assert not mock_log.warn.called + def test_create_with_special_volume_mode(self): self.mock_client.inspect_image.return_value = {'Id': 'imageid'} From 73de81b51c0f613ed543f4446a5d76ed9e788659 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 2 Oct 2015 19:04:39 -0400 Subject: [PATCH 173/359] Upgrade tests to use new Mounts in container inspect. Signed-off-by: Daniel Nephin --- compose/cli/docker_client.py | 2 +- compose/container.py | 6 ++++++ tests/acceptance/cli_test.py | 2 +- tests/integration/project_test.py | 20 +++++++++++++------- tests/integration/resilience_test.py | 8 ++++---- 5 files changed, 25 insertions(+), 13 deletions(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 734f4237..24828b11 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -9,7 +9,7 @@ from ..const import HTTP_TIMEOUT log = logging.getLogger(__name__) -DEFAULT_API_VERSION = '1.19' +DEFAULT_API_VERSION = '1.20' def docker_client(version=None): diff --git a/compose/container.py b/compose/container.py index 8f96a944..68218829 100644 --- a/compose/container.py +++ b/compose/container.py @@ -177,6 +177,12 @@ class Container(object): port = self.ports.get("%s/%s" % (port, protocol)) return "{HostIp}:{HostPort}".format(**port[0]) if port else None + def get_mount(self, mount_dest): + for mount in self.get('Mounts'): + if mount['Destination'] == mount_dest: + return mount + return None + def start(self, **options): return self.client.start(self.id, **options) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e6fa38a8..81d34a1a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -871,7 +871,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d'], None) container = self.project.containers(stopped=True)[0] - actual_host_path = container.get('Volumes')['/container-path'] + actual_host_path = container.get_mount('/container-path')['Source'] components = actual_host_path.split('/') assert components[-2:] == ['home-dir', 'my-volume'] diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 229f653a..bff19d55 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -331,15 +331,19 @@ class ProjectTest(DockerClientTestCase): project.up(['db']) self.assertEqual(len(project.containers()), 1) old_db_id = project.containers()[0].id - db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db'] + + container, = project.containers() + db_volume_path = container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) self.assertEqual(len(project.containers()), 2) db_container = [c for c in project.containers() if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - self.assertEqual(db_container.inspect()['Volumes']['/var/db'], - db_volume_path) + mount, = db_container.get('Mounts') + self.assertEqual( + db_container.get_mount('/var/db')['Source'], + db_volume_path) def test_project_up_with_no_recreate_stopped(self): web = self.create_service('web') @@ -354,8 +358,9 @@ class ProjectTest(DockerClientTestCase): old_containers = project.containers(stopped=True) self.assertEqual(len(old_containers), 1) - old_db_id = old_containers[0].id - db_volume_path = old_containers[0].inspect()['Volumes']['/var/db'] + old_container, = old_containers + old_db_id = old_container.id + db_volume_path = old_container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) @@ -365,8 +370,9 @@ class ProjectTest(DockerClientTestCase): db_container = [c for c in new_containers if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - self.assertEqual(db_container.inspect()['Volumes']['/var/db'], - db_volume_path) + self.assertEqual( + db_container.get_mount('/var/db')['Source'], + db_volume_path) def test_project_up_without_all_services(self): console = self.create_service('console') diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 7f75356d..5df751c7 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -18,12 +18,12 @@ class ResilienceTest(DockerClientTestCase): container = self.db.create_container() container.start() - self.host_path = container.get('Volumes')['/var/db'] + self.host_path = container.get_mount('/var/db')['Source'] def test_successful_recreate(self): self.project.up(strategy=ConvergenceStrategy.always) container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_create_failure(self): with mock.patch('compose.service.Service.create_container', crash): @@ -32,7 +32,7 @@ class ResilienceTest(DockerClientTestCase): self.project.up() container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_start_failure(self): with mock.patch('compose.container.Container.start', crash): @@ -41,7 +41,7 @@ class ResilienceTest(DockerClientTestCase): self.project.up() container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) class Crash(Exception): From 3bdcc9d95459618a1301eb801348fc3805a764a6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 6 Oct 2015 18:37:43 -0700 Subject: [PATCH 174/359] Update service tests to use mounts instead of volumes Signed-off-by: Joffrey F --- tests/integration/service_test.py | 30 ++++++++++++++++-------------- 1 file changed, 16 insertions(+), 14 deletions(-) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index ff571656..272fafde 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -88,13 +88,13 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) container = service.create_container() container.start() - self.assertIn('/var/db', container.get('Volumes')) + self.assertIsNotNone(container.get_mount('/var/db')) def test_create_container_with_volume_driver(self): service = self.create_service('db', volume_driver='foodriver') container = service.create_container() container.start() - self.assertEqual('foodriver', container.get('Config.VolumeDriver')) + self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver')) def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) @@ -158,12 +158,10 @@ class ServiceTest(DockerClientTestCase): volumes=[VolumeSpec(host_path, container_path, 'rw')]) container = service.create_container() container.start() - - volumes = container.inspect()['Volumes'] - self.assertIn(container_path, volumes) + self.assertIsNotNone(container.get_mount(container_path)) # Match the last component ("host-path"), because boot2docker symlinks /tmp - actual_host_path = volumes[container_path] + actual_host_path = container.get_mount(container_path)['Source'] self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) @@ -173,10 +171,10 @@ class ServiceTest(DockerClientTestCase): """ service = self.create_service('data', volumes=[VolumeSpec.parse('/data/')]) old_container = create_and_start_container(service) - volume_path = old_container.get('Volumes')['/data'] + volume_path = old_container.get_mount('/data')['Source'] new_container = service.recreate_container(old_container) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_duplicate_volume_trailing_slash(self): """ @@ -250,7 +248,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.name, 'composetest_db_1') old_container.start() old_container.inspect() # reload volume data - volume_path = old_container.get('Volumes')['/etc'] + volume_path = old_container.get_mount('/etc')['Source'] num_containers_before = len(self.client.containers(all=True)) @@ -262,7 +260,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(new_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') - self.assertEqual(new_container.get('Volumes')['/etc'], volume_path) + self.assertEqual(new_container.get_mount('/etc')['Source'], volume_path) self.assertIn( 'affinity:container==%s' % old_container.id, new_container.get('Config.Env')) @@ -305,14 +303,18 @@ class ServiceTest(DockerClientTestCase): ) old_container = create_and_start_container(service) - self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) - volume_path = old_container.get('Volumes')['/data'] + self.assertEqual( + [mount['Destination'] for mount in old_container.get('Mounts')], ['/data'] + ) + volume_path = old_container.get_mount('/data')['Source'] new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) - self.assertEqual(list(new_container.get('Volumes')), ['/data']) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual( + [mount['Destination'] for mount in new_container.get('Mounts')], ['/data'] + ) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_execute_convergence_plan_when_image_volume_masks_config(self): service = self.create_service( From afab5c76eaf28651181b391cacc4614954134a4b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 8 Oct 2015 06:02:26 -0700 Subject: [PATCH 175/359] Update service volume tests to use mounts key Signed-off-by: Joffrey F --- tests/unit/service_test.py | 54 +++++++++++++++++++++++++++++++------- 1 file changed, 44 insertions(+), 10 deletions(-) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 1c8b441f..b2421a34 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -600,12 +600,33 @@ class ServiceVolumesTest(unittest.TestCase): } container = Container(self.mock_client, { 'Image': 'ababab', - 'Volumes': { - '/host/volume': '/host/volume', - '/existing/volume': '/var/lib/docker/aaaaaaaa', - '/removed/volume': '/var/lib/docker/bbbbbbbb', - '/mnt/image/data': '/var/lib/docker/cccccccc', - }, + 'Mounts': [ + { + 'Source': '/host/volume', + 'Destination': '/host/volume', + 'Mode': '', + 'RW': True, + 'Name': 'hostvolume', + }, { + 'Source': '/var/lib/docker/aaaaaaaa', + 'Destination': '/existing/volume', + 'Mode': '', + 'RW': True, + 'Name': 'existingvolume', + }, { + 'Source': '/var/lib/docker/bbbbbbbb', + 'Destination': '/removed/volume', + 'Mode': '', + 'RW': True, + 'Name': 'removedvolume', + }, { + 'Source': '/var/lib/docker/cccccccc', + 'Destination': '/mnt/image/data', + 'Mode': '', + 'RW': True, + 'Name': 'imagedata', + }, + ] }, has_been_inspected=True) expected = [ @@ -630,7 +651,13 @@ class ServiceVolumesTest(unittest.TestCase): intermediate_container = Container(self.mock_client, { 'Image': 'ababab', - 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'}, + 'Mounts': [{ + 'Source': '/var/lib/docker/aaaaaaaa', + 'Destination': '/existing/volume', + 'Mode': '', + 'RW': True, + 'Name': 'existingvolume', + }], }, has_been_inspected=True) expected = [ @@ -693,9 +720,16 @@ class ServiceVolumesTest(unittest.TestCase): self.mock_client.inspect_container.return_value = { 'Id': '123123123', 'Image': 'ababab', - 'Volumes': { - '/data': '/mnt/sda1/host/path', - }, + 'Mounts': [ + { + 'Destination': '/data', + 'Source': '/mnt/sda1/host/path', + 'Mode': '', + 'RW': True, + 'Driver': 'local', + 'Name': 'abcdefff1234' + }, + ] } service._get_container_create_options( From c64b7cbb10b9fd8f1e15d6f0e22a2de4abda7567 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 9 Oct 2015 15:42:15 -0400 Subject: [PATCH 176/359] Ignore errors from API about not being able to kill a container. Signed-off-by: Daniel Nephin --- tests/integration/testcases.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 9ea68e39..04cbe352 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -25,8 +25,7 @@ class DockerClientTestCase(unittest.TestCase): for c in self.client.containers( all=True, filters={'label': '%s=composetest' % LABEL_PROJECT}): - self.client.kill(c['Id']) - self.client.remove_container(c['Id']) + self.client.remove_container(c['Id'], force=True) for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): self.client.remove_image(i) From 97fe2ee40c7018db3cbf0c40feb556e9e7940689 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 4 Nov 2015 17:09:40 -0500 Subject: [PATCH 177/359] Don't preserve host volumes on container recreate. Fixes a regression after the API changed to use Mounts. Signed-off-by: Daniel Nephin --- compose/service.py | 14 ++++++++++---- tests/integration/service_test.py | 17 ++++++++++++----- tests/unit/service_test.py | 5 +++++ 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index d1509871..251620e9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -849,7 +849,13 @@ def get_container_data_volumes(container, volumes_option): a mapping of volume bindings for those volumes. """ volumes = [] - container_volumes = container.get('Volumes') or {} + volumes_option = volumes_option or [] + + container_mounts = dict( + (mount['Destination'], mount) + for mount in container.get('Mounts') or {} + ) + image_volumes = [ VolumeSpec.parse(volume) for volume in @@ -861,13 +867,13 @@ def get_container_data_volumes(container, volumes_option): if volume.external: continue - volume_path = container_volumes.get(volume.internal) + mount = container_mounts.get(volume.internal) # New volume, doesn't exist in the old container - if not volume_path: + if not mount: continue # Copy existing volume from old container - volume = volume._replace(external=volume_path) + volume = volume._replace(external=mount['Source']) volumes.append(volume) return volumes diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 272fafde..41d0800d 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -312,7 +312,8 @@ class ServiceTest(DockerClientTestCase): ConvergencePlan('recreate', [old_container])) self.assertEqual( - [mount['Destination'] for mount in new_container.get('Mounts')], ['/data'] + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] ) self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) @@ -323,8 +324,11 @@ class ServiceTest(DockerClientTestCase): ) old_container = create_and_start_container(service) - self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) - volume_path = old_container.get('Volumes')['/data'] + self.assertEqual( + [mount['Destination'] for mount in old_container.get('Mounts')], + ['/data'] + ) + volume_path = old_container.get_mount('/data')['Source'] service.options['volumes'] = [VolumeSpec.parse('/tmp:/data')] @@ -338,8 +342,11 @@ class ServiceTest(DockerClientTestCase): "Service \"db\" is using volume \"/data\" from the previous container", args[0]) - self.assertEqual(list(new_container.get('Volumes')), ['/data']) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual( + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] + ) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_execute_convergence_plan_without_start(self): service = self.create_service( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index b2421a34..87d6af59 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -234,6 +234,7 @@ class ServiceTest(unittest.TestCase): prev_container = mock.Mock( id='ababab', image_config={'ContainerConfig': {}}) + prev_container.get.return_value = None opts = service._get_container_create_options( {}, @@ -575,6 +576,10 @@ class NetTestCase(unittest.TestCase): self.assertEqual(net.service_name, service_name) +def build_mount(destination, source, mode='rw'): + return {'Source': source, 'Destination': destination, 'Mode': mode} + + class ServiceVolumesTest(unittest.TestCase): def setUp(self): From 4bf2f8c4f910b5216b2755377e7c394e7c5c7ee6 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 20 Oct 2015 12:49:45 -0400 Subject: [PATCH 178/359] Fix lookup of linked containers for API version 1.20 Signed-off-by: Daniel Nephin --- compose/container.py | 10 ---------- tests/acceptance/cli_test.py | 7 +++++-- tests/integration/project_test.py | 1 - tests/integration/service_test.py | 15 ++++++++------- tests/integration/state_test.py | 5 +++-- tests/integration/testcases.py | 10 ++++++++++ 6 files changed, 26 insertions(+), 22 deletions(-) diff --git a/compose/container.py b/compose/container.py index 68218829..5730f224 100644 --- a/compose/container.py +++ b/compose/container.py @@ -228,16 +228,6 @@ class Container(object): self.has_been_inspected = True return self.dictionary - # TODO: only used by tests, move to test module - def links(self): - links = [] - for container in self.client.containers(): - for name in container['Names']: - bits = name.split('/') - if len(bits) > 2 and bits[1] == self.name: - links.append(bits[2]) - return links - def attach(self, *args, **kwargs): return self.client.attach(self.id, *args, **kwargs) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 81d34a1a..1885727a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -16,6 +16,7 @@ from compose.cli.command import get_project from compose.cli.docker_client import docker_client from compose.container import Container from tests.integration.testcases import DockerClientTestCase +from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox @@ -909,7 +910,7 @@ class CLITestCase(DockerClientTestCase): web, other, db = containers self.assertEqual(web.human_readable_command, 'top') - self.assertTrue({'db', 'other'} <= set(web.links())) + self.assertTrue({'db', 'other'} <= set(get_links(web))) self.assertEqual(db.human_readable_command, 'top') self.assertEqual(other.human_readable_command, 'top') @@ -931,7 +932,9 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 2) web = containers[1] - self.assertEqual(set(web.links()), set(['db', 'mydb_1', 'extends_mydb_1'])) + self.assertEqual( + set(get_links(web)), + set(['db', 'mydb_1', 'extends_mydb_1'])) expected_env = set([ "FOO=1", diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index bff19d55..d33cf535 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -340,7 +340,6 @@ class ProjectTest(DockerClientTestCase): db_container = [c for c in project.containers() if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - mount, = db_container.get('Mounts') self.assertEqual( db_container.get_mount('/var/db')['Source'], db_volume_path) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 41d0800d..c288a8ad 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,6 +12,7 @@ from six import text_type from .. import mock from .testcases import DockerClientTestCase +from .testcases import get_links from .testcases import pull_busybox from compose import __version__ from compose.config.types import VolumeFromSpec @@ -88,7 +89,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) container = service.create_container() container.start() - self.assertIsNotNone(container.get_mount('/var/db')) + assert container.get_mount('/var/db') def test_create_container_with_volume_driver(self): service = self.create_service('db', volume_driver='foodriver') @@ -158,7 +159,7 @@ class ServiceTest(DockerClientTestCase): volumes=[VolumeSpec(host_path, container_path, 'rw')]) container = service.create_container() container.start() - self.assertIsNotNone(container.get_mount(container_path)) + assert container.get_mount(container_path) # Match the last component ("host-path"), because boot2docker symlinks /tmp actual_host_path = container.get_mount(container_path)['Source'] @@ -385,7 +386,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', @@ -401,7 +402,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', @@ -419,7 +420,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'composetest_db_2', @@ -433,7 +434,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) c = create_and_start_container(db) - self.assertEqual(set(c.links()), set([])) + self.assertEqual(set(get_links(c)), set([])) def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') @@ -444,7 +445,7 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db, one_off=True) self.assertEqual( - set(c.links()), + set(get_links(c)), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 1fecce87..a54eefa6 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals import py from .testcases import DockerClientTestCase +from .testcases import get_links from compose.config import config from compose.project import Project from compose.service import ConvergenceStrategy @@ -186,8 +187,8 @@ class ProjectWithDependenciesTest(ProjectTestCase): web, = [c for c in containers if c.service == 'web'] nginx, = [c for c in containers if c.service == 'nginx'] - self.assertEqual(web.links(), ['composetest_db_1', 'db', 'db_1']) - self.assertEqual(nginx.links(), ['composetest_web_1', 'web', 'web_1']) + self.assertEqual(set(get_links(web)), {'composetest_db_1', 'db', 'db_1'}) + self.assertEqual(set(get_links(nginx)), {'composetest_web_1', 'web', 'web_1'}) class ServiceStateTest(DockerClientTestCase): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 04cbe352..469859b9 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -16,6 +16,16 @@ def pull_busybox(client): client.pull('busybox:latest', stream=False) +def get_links(container): + links = container.get('HostConfig.Links') or [] + + def format_link(link): + _, alias = link.split(':') + return alias.split('/')[-1] + + return [format_link(link) for link in links] + + class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): From b4be7b870fb99ed39eb47b5f5cc41d82c5af85e2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 16 Nov 2015 19:21:56 -0800 Subject: [PATCH 179/359] Add support for declaring named volumes in compose files * Bump default API version to 1.21 (required for named volume management) * Introduce new, versioned compose file format while maintaining support for current (legacy) format * Test updates to reflect changes made to the internal API Signed-off-by: Joffrey F --- compose/cli/command.py | 5 +- compose/cli/docker_client.py | 3 +- compose/config/config.py | 79 +++++++++++++++++++++++++--- compose/config/fields_schema_v2.json | 43 +++++++++++++++ compose/config/validation.py | 9 ++-- compose/project.py | 31 +++++++++-- compose/volume.py | 19 +++++++ requirements.txt | 3 +- tests/integration/project_test.py | 25 +++++---- tests/integration/service_test.py | 1 + tests/integration/state_test.py | 4 +- tests/integration/testcases.py | 4 ++ tests/unit/config/config_test.py | 53 ++++++++++--------- tests/unit/project_test.py | 64 +++++++++++++++------- 14 files changed, 262 insertions(+), 81 deletions(-) create mode 100644 compose/config/fields_schema_v2.json create mode 100644 compose/volume.py diff --git a/compose/cli/command.py b/compose/cli/command.py index 21d6ff0d..b278af3a 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -80,12 +80,13 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(base_dir, config_path) api_version = '1.21' if use_networking else None - return Project.from_dicts( + return Project.from_config( get_project_name(config_details.working_dir, project_name), config.load(config_details), get_client(verbose=verbose, version=api_version), use_networking=use_networking, - network_driver=network_driver) + network_driver=network_driver + ) def get_project_name(working_dir, project_name=None): diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 24828b11..177d5d6c 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -8,8 +8,7 @@ from ..const import HTTP_TIMEOUT log = logging.getLogger(__name__) - -DEFAULT_API_VERSION = '1.20' +DEFAULT_API_VERSION = '1.21' def docker_client(version=None): diff --git a/compose/config/config.py b/compose/config/config.py index 195665b5..1e82068f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -117,6 +117,17 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): return cls(filename, load_yaml(filename)) +class Config(namedtuple('_Config', 'version services volumes')): + """ + :param version: configuration version + :type version: int + :param services: List of service description dictionaries + :type services: :class:`list` + :param volumes: List of volume description dictionaries + :type volumes: :class:`list` + """ + + class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')): @classmethod @@ -148,6 +159,24 @@ def find(base_dir, filenames): [ConfigFile.from_filename(f) for f in filenames]) +def get_config_version(config_details): + def get_version(config): + validate_top_level_object(config) + return config.config.get('version') + main_file = config_details.config_files[0] + version = get_version(main_file) + for next_file in config_details.config_files[1:]: + next_file_version = get_version(next_file) + if version != next_file_version: + raise ConfigurationError( + "Version mismatch: main file {0} specifies version {1} but " + "extension file {2} uses version {3}".format( + main_file.filename, version, next_file.filename, next_file_version + ) + ) + return version + + def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) @@ -194,10 +223,46 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ + version = get_config_version(config_details) + processed_files = [] + for config_file in config_details.config_files: + processed_files.append( + process_config_file(config_file, version=version) + ) + config_details = config_details._replace(config_files=processed_files) + if not version or isinstance(version, dict): + service_dicts = load_services( + config_details.working_dir, config_details.config_files + ) + volumes = {} + elif version == 2: + config_files = [ + ConfigFile(f.filename, f.config.get('services', {})) + for f in config_details.config_files + ] + service_dicts = load_services( + config_details.working_dir, config_files + ) + volumes = load_volumes(config_details.config_files) + else: + raise ConfigurationError('Invalid config version provided: {0}'.format(version)) + + return Config(version, service_dicts, volumes) + + +def load_volumes(config_files): + volumes = {} + for config_file in config_files: + for name, volume_config in config_file.config.get('volumes', {}).items(): + volumes.update({name: volume_config}) + return volumes + + +def load_services(working_dir, config_files): def build_service(filename, service_name, service_dict): service_config = ServiceConfig.with_abs_paths( - config_details.working_dir, + working_dir, filename, service_name, service_dict) @@ -227,20 +292,20 @@ def load(config_details): for name in all_service_names } - config_file = process_config_file(config_details.config_files[0]) - for next_file in config_details.config_files[1:]: - next_file = process_config_file(next_file) - + config_file = config_files[0] + for next_file in config_files[1:]: config = merge_services(config_file.config, next_file.config) config_file = config_file._replace(config=config) return build_services(config_file) -def process_config_file(config_file, service_name=None): +def process_config_file(config_file, service_name=None, version=None): validate_top_level_object(config_file) processed_config = interpolate_environment_variables(config_file.config) - validate_against_fields_schema(processed_config, config_file.filename) + validate_against_fields_schema( + processed_config, config_file.filename, version + ) if service_name and service_name not in processed_config: raise ConfigurationError( diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json new file mode 100644 index 00000000..2ca41c47 --- /dev/null +++ b/compose/config/fields_schema_v2.json @@ -0,0 +1,43 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + + "type": "object", + "properties": { + "version": { + "enum": [2] + }, + "services": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "fields_schema.json#/definitions/service" + } + } + }, + "volumes": { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + } + } + }, + + "definitions": { + "volume": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["boolean", "string", "number"]} + }, + "additionalProperties": false + } + } + } + }, + "additionalProperties": false +} diff --git a/compose/config/validation.py b/compose/config/validation.py index d16bdb9d..861cb10f 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -281,11 +281,14 @@ def process_errors(errors, service_name=None): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config, filename): +def validate_against_fields_schema(config, filename, version=None): + schema_filename = "fields_schema.json" + if version: + schema_filename = "fields_schema_v{0}.json".format(version) _validate_against_schema( config, - "fields_schema.json", - format_checker=["ports", "expose", "bool-value-in-mapping"], + schema_filename, + format_checker=["ports", "environment", "bool-value-in-mapping"], filename=filename) diff --git a/compose/project.py b/compose/project.py index 76dccfe2..b7f33e3f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -20,6 +20,7 @@ from .service import ConvergenceStrategy from .service import Net from .service import Service from .service import ServiceNet +from .volume import Volume log = logging.getLogger(__name__) @@ -29,12 +30,13 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, use_networking=False, network_driver=None): + def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None): self.name = name self.services = services self.client = client self.use_networking = use_networking self.network_driver = network_driver + self.volumes = volumes or [] def labels(self, one_off=False): return [ @@ -43,16 +45,16 @@ class Project(object): ] @classmethod - def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None): + def from_config(cls, name, config_data, client, use_networking=False, network_driver=None): """ - Construct a ServiceCollection from a list of dicts representing services. + Construct a Project from a config.Config object. """ project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver) if use_networking: - remove_links(service_dicts) + remove_links(config_data.services) - for service_dict in service_dicts: + for service_dict in config_data.services: links = project.get_links(service_dict) volumes_from = project.get_volumes_from(service_dict) net = project.get_net(service_dict) @@ -66,6 +68,14 @@ class Project(object): net=net, volumes_from=volumes_from, **service_dict)) + if config_data.volumes: + for vol_name, data in config_data.volumes.items(): + project.volumes.append( + Volume( + client=client, project=name, name=vol_name, + driver=data.get('driver'), driver_opts=data.get('driver_opts') + ) + ) return project @property @@ -218,6 +228,15 @@ class Project(object): def remove_stopped(self, service_names=None, **options): parallel.parallel_remove(self.containers(service_names, stopped=True), options) + def initialize_volumes(self): + try: + for volume in self.volumes: + volume.create() + except NotFound: + raise ConfigurationError( + 'Volume %s sepcifies nonexistent driver %s' % (volume.name, volume.driver) + ) + def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -253,6 +272,8 @@ class Project(object): if self.use_networking and self.uses_default_network(): self.ensure_network_exists() + self.initialize_volumes() + return [ container for service in services diff --git a/compose/volume.py b/compose/volume.py new file mode 100644 index 00000000..304633d0 --- /dev/null +++ b/compose/volume.py @@ -0,0 +1,19 @@ +from __future__ import unicode_literals + + +class Volume(object): + def __init__(self, client, project, name, driver=None, driver_opts=None): + self.client = client + self.project = project + self.name = name + self.driver = driver + self.driver_opts = driver_opts + + def create(self): + return self.client.create_volume(self.name, self.driver, self.driver_opts) + + def remove(self): + return self.client.remove_volume(self.name) + + def inspect(self): + return self.client.inspect_volume(self.name) diff --git a/requirements.txt b/requirements.txt index 659cb57f..ac02b8db 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,6 @@ +-e git://github.com/docker/docker-py.git@881e24c231ab9921eb0cbd475e85706137983f89#egg=docker-py PyYAML==3.11 -docker-py==1.5.0 +# docker-py==1.5.1 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d33cf535..4107c6cf 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -69,9 +69,9 @@ class ProjectTest(DockerClientTestCase): 'volumes_from': ['data'], }, }) - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=service_dicts, + config_data=service_dicts, client=self.client, ) db = project.get_service('db') @@ -86,9 +86,9 @@ class ProjectTest(DockerClientTestCase): name='composetest_data_container', labels={LABEL_PROJECT: 'composetest'}, ) - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'db': { 'image': 'busybox:latest', 'volumes_from': ['composetest_data_container'], @@ -117,9 +117,9 @@ class ProjectTest(DockerClientTestCase): assert project.get_network()['Name'] == network_name def test_net_from_service(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'net': { 'image': 'busybox:latest', 'command': ["top"] @@ -149,9 +149,9 @@ class ProjectTest(DockerClientTestCase): ) net_container.start() - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'web': { 'image': 'busybox:latest', 'net': 'container:composetest_net_container' @@ -331,7 +331,6 @@ class ProjectTest(DockerClientTestCase): project.up(['db']) self.assertEqual(len(project.containers()), 1) old_db_id = project.containers()[0].id - container, = project.containers() db_volume_path = container.get_mount('/var/db')['Source'] @@ -401,9 +400,9 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(console.containers()), 0) def test_project_up_starts_depends(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -436,9 +435,9 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('console').containers()), 0) def test_project_up_with_no_deps(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'console': { 'image': 'busybox:latest', 'command': ["top"], diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index c288a8ad..4a0eaacb 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -163,6 +163,7 @@ class ServiceTest(DockerClientTestCase): # Match the last component ("host-path"), because boot2docker symlinks /tmp actual_host_path = container.get_mount(container_path)['Source'] + self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index a54eefa6..d07dfa82 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -26,10 +26,10 @@ class ProjectTestCase(DockerClientTestCase): details = config.ConfigDetails( 'working_dir', [config.ConfigFile(None, cfg)]) - return Project.from_dicts( + return Project.from_config( name='composetest', client=self.client, - service_dicts=config.load(details)) + config_data=config.load(details)) class BasicProjectTest(ProjectTestCase): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 469859b9..a9f5e7bb 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -39,6 +39,10 @@ class DockerClientTestCase(unittest.TestCase): for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): self.client.remove_image(i) + volumes = self.client.volumes().get('Volumes') or [] + for v in volumes: + if 'composetests_' in v['Name']: + self.client.remove_volume(v['Name']) def create_service(self, name, **kwargs): if 'image' not in kwargs and 'build' not in kwargs: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e975cb9d..426146cc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -51,7 +51,7 @@ class ConfigTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual( service_sort(service_dicts), @@ -143,7 +143,7 @@ class ConfigTest(unittest.TestCase): }) details = config.ConfigDetails('.', [base_file, override_file]) - service_dicts = config.load(details) + service_dicts = config.load(details).services expected = [ { 'name': 'web', @@ -207,7 +207,7 @@ class ConfigTest(unittest.TestCase): labels: ['label=one'] """) with tmpdir.as_cwd(): - service_dicts = config.load(details) + service_dicts = config.load(details).services expected = [ { @@ -260,7 +260,7 @@ class ConfigTest(unittest.TestCase): build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', - 'common.yml')) + 'common.yml')).services assert services[0]['name'] == valid_name def test_config_hint(self): @@ -451,7 +451,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(service[0]['expose'], expose) def test_valid_config_oneof_string_or_list(self): @@ -466,7 +466,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(service[0]['entrypoint'], entrypoint) @mock.patch('compose.config.validation.log') @@ -496,7 +496,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'], 'none') def test_load_yaml_with_yaml_error(self): @@ -655,7 +655,7 @@ class InterpolationTest(unittest.TestCase): service_dicts = config.load( config.find('tests/fixtures/environment-interpolation', None), - ) + ).services self.assertEqual(service_dicts, [ { @@ -722,7 +722,7 @@ class InterpolationTest(unittest.TestCase): '.', None, ) - )[0] + ).services[0] self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') @@ -734,11 +734,15 @@ class VolumeConfigTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' - d = config.load(build_config_details( - {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - '.', - ))[0] - self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) + + d = config.load( + build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + None, + ) + ).services[0] + self.assertEqual(d['volumes'], ['/host/path:/container/path']) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) @@ -1012,7 +1016,7 @@ class MemoryOptionsTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual(service_dict[0]['memswap_limit'], 2000000) def test_memswap_can_be_a_string(self): @@ -1022,7 +1026,7 @@ class MemoryOptionsTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual(service_dict[0]['memswap_limit'], "512M") @@ -1126,24 +1130,21 @@ class EnvTest(unittest.TestCase): {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, "tests/fixtures/env", ) - )[0] - self.assertEqual( - set(service_dict['volumes']), - set([VolumeSpec.parse('/tmp:/host/tmp')])) + + ).services[0] + self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) service_dict = config.load( build_config_details( {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, "tests/fixtures/env", ) - )[0] - self.assertEqual( - set(service_dict['volumes']), - set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')])) + ).services[0] + self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) def load_from_filename(filename): - return config.load(config.find('.', [filename])) + return config.load(config.find('.', [filename])).services class ExtendsTest(unittest.TestCase): @@ -1313,7 +1314,7 @@ class ExtendsTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEquals(len(service), 1) self.assertIsInstance(service[0], dict) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f4c6f8ca..c0ed5e33 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -4,6 +4,7 @@ import docker from .. import mock from .. import unittest +from compose.config.config import Config from compose.config.types import VolumeFromSpec from compose.const import LABEL_SERVICE from compose.container import Container @@ -18,7 +19,7 @@ class ProjectTest(unittest.TestCase): self.mock_client = mock.create_autospec(docker.Client) def test_from_dict(self): - project = Project.from_dicts('composetest', [ + project = Project.from_config('composetest', Config(None, [ { 'name': 'web', 'image': 'busybox:latest' @@ -27,15 +28,38 @@ class ProjectTest(unittest.TestCase): 'name': 'db', 'image': 'busybox:latest' }, - ], None) + ], None), None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') self.assertEqual(project.get_service('db').name, 'db') self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') + def test_from_dict_sorts_in_dependency_order(self): + project = Project.from_config('composetest', Config(None, [ + { + 'name': 'web', + 'image': 'busybox:latest', + 'links': ['db'], + }, + { + 'name': 'db', + 'image': 'busybox:latest', + 'volumes_from': ['volume'] + }, + { + 'name': 'volume', + 'image': 'busybox:latest', + 'volumes': ['/tmp'], + } + ], None), None) + + self.assertEqual(project.services[0].name, 'volume') + self.assertEqual(project.services[1].name, 'db') + self.assertEqual(project.services[2].name, 'web') + def test_from_config(self): - dicts = [ + dicts = Config(None, [ { 'name': 'web', 'image': 'busybox:latest', @@ -44,8 +68,8 @@ class ProjectTest(unittest.TestCase): 'name': 'db', 'image': 'busybox:latest', }, - ] - project = Project.from_dicts('composetest', dicts, None) + ], None) + project = Project.from_config('composetest', dicts, None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -141,13 +165,13 @@ class ProjectTest(unittest.TestCase): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('aaa', 'rw')] } - ], self.mock_client) + ], None), self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"]) def test_use_volumes_from_service_no_container(self): @@ -160,7 +184,7 @@ class ProjectTest(unittest.TestCase): "Image": 'busybox:latest' } ] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'vol', 'image': 'busybox:latest' @@ -170,13 +194,13 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('vol', 'rw')] } - ], self.mock_client) + ], None), self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"]) def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'vol', 'image': 'busybox:latest' @@ -186,7 +210,7 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('vol', 'rw')] } - ], None) + ], None), None) with mock.patch.object(Service, 'containers') as mock_return: mock_return.return_value = [ mock.Mock(id=container_id, spec=Container) @@ -196,12 +220,12 @@ class ProjectTest(unittest.TestCase): [container_ids[0] + ':rw']) def test_net_unset(self): - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) @@ -210,13 +234,13 @@ class ProjectTest(unittest.TestCase): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', 'net': 'container:aaa' } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_id) @@ -230,7 +254,7 @@ class ProjectTest(unittest.TestCase): "Image": 'busybox:latest' } ] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'aaa', 'image': 'busybox:latest' @@ -240,7 +264,7 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', 'net': 'container:aaa' } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_name) @@ -285,12 +309,12 @@ class ProjectTest(unittest.TestCase): }, }, } - project = Project.from_dicts( + project = Project.from_config( 'test', - [{ + Config(None, [{ 'name': 'web', 'image': 'busybox:latest', - }], + }], None), self.mock_client, ) self.assertEqual([c.id for c in project.containers()], ['1']) From b253efd8a77447c7ad4f6a6aa5101510704782b1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 18 Nov 2015 17:45:10 -0800 Subject: [PATCH 180/359] Update docs to define and document new compose.yml file format Add volume configuration reference section. Signed-off-by: Joffrey F --- docs/compose-file.md | 86 ++++++++++++++++++++++++++++++++++++++++++++ docs/index.md | 26 ++++++++------ 2 files changed, 102 insertions(+), 10 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 2a6028b8..29e0c647 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -24,6 +24,64 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +## Versioning + +It is possible to use different versions of the `compose.yml` format. +Below are the formats currently supported by compose. + + +### Version 1 + +Compose files that do not declare a version are considered "version 1". In +those files, all the [services](#service-configuration-reference) are declared +at the root of the document. + +Version 1 files do not support the declaration of +named [volumes](#volume-configuration-reference) + +Example: + + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + + +### Version 2 + +Compose files using the version 2 syntax must indicate the version number at +the root of the document. All [services](#service-configuration-reference) +must be declared under the `services` key. +Named [volumes](#volume-configuration-reference) must be declared under the +`volumes` key. + +Example: + + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: + driver: default + + ## Service configuration reference This section contains a list of all configuration options supported by a service @@ -413,6 +471,34 @@ Each of these is a single value, analogous to its stdin_open: true tty: true + +## Volume configuration reference + +While it is possible to declare volumes on the fly as part of the service +declaration, this section allows you to create named volumes that can be +reused across multiple services (without relying on `volumes_from`), and are +easily retrieved and inspected using the docker command line or API. +See the [docker volume](http://docs.docker.com/reference/commandline/volume/) +subcommand documentation for more information. + +### driver + +Specify which volume driver should be used for this volume. Defaults to +`local`. An exception will be raised if the driver is not available. + + driver: foobar + +### driver_opts + +Specify a list of options as key-value pairs to pass to the driver for this +volume. Those options are driver dependent - consult the driver's +documentation for more information. Optional. + + driver_opts: + foo: "bar" + baz: 1 + + ## Variable substitution Your configuration options can contain environment variables. Compose uses the diff --git a/docs/index.md b/docs/index.md index 36b93a39..6e8f2090 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,16 +31,22 @@ they can be run together in an isolated environment. A `docker-compose.yml` looks like this: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: + driver: default For more information about the Compose file, see the [Compose file reference](compose-file.md) From abe145bbe77d09a5b31ee1453a37938e132604e1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Dec 2015 17:26:32 -0800 Subject: [PATCH 181/359] Update config resolution to always use explicit version numbers Also includes several bugfixes for resolution and validation. Signed-off-by: Joffrey F --- compose/config/config.py | 46 +++++++++++++------ ...elds_schema.json => fields_schema_v1.json} | 2 +- compose/config/fields_schema_v2.json | 16 +++++-- compose/config/interpolation.py | 12 +++-- compose/config/service_schema.json | 2 +- compose/config/validation.py | 18 ++++---- compose/const.py | 1 + docker-compose.spec | 10 +++- 8 files changed, 73 insertions(+), 34 deletions(-) rename compose/config/{fields_schema.json => fields_schema_v1.json} (99%) diff --git a/compose/config/config.py b/compose/config/config.py index 1e82068f..295dc494 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -10,6 +10,7 @@ from collections import namedtuple import six import yaml +from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError @@ -24,6 +25,7 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extends_file_path from .validation import validate_top_level_object +from .validation import validate_top_level_service_objects DOCKER_CONFIG_KEYS = [ @@ -161,13 +163,24 @@ def find(base_dir, filenames): def get_config_version(config_details): def get_version(config): - validate_top_level_object(config) - return config.config.get('version') + if config.config is None: + return None + version = config.config.get('version', 1) + if isinstance(version, dict): + version = 1 + return version + main_file = config_details.config_files[0] + validate_top_level_object(main_file) version = get_version(main_file) for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file) next_file_version = get_version(next_file) - if version != next_file_version: + if version is None: + version = next_file_version + continue + + if version != next_file_version and next_file_version is not None: raise ConfigurationError( "Version mismatch: main file {0} specifies version {1} but " "extension file {2} uses version {3}".format( @@ -224,6 +237,9 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ version = get_config_version(config_details) + if version not in COMPOSEFILE_VERSIONS: + raise ConfigurationError('Invalid config version provided: {0}'.format(version)) + processed_files = [] for config_file in config_details.config_files: processed_files.append( @@ -231,9 +247,10 @@ def load(config_details): ) config_details = config_details._replace(config_files=processed_files) - if not version or isinstance(version, dict): + if version == 1: service_dicts = load_services( - config_details.working_dir, config_details.config_files + config_details.working_dir, config_details.config_files, + version ) volumes = {} elif version == 2: @@ -242,11 +259,9 @@ def load(config_details): for f in config_details.config_files ] service_dicts = load_services( - config_details.working_dir, config_files + config_details.working_dir, config_files, version ) volumes = load_volumes(config_details.config_files) - else: - raise ConfigurationError('Invalid config version provided: {0}'.format(version)) return Config(version, service_dicts, volumes) @@ -259,14 +274,14 @@ def load_volumes(config_files): return volumes -def load_services(working_dir, config_files): +def load_services(working_dir, config_files, version): def build_service(filename, service_name, service_dict): service_config = ServiceConfig.with_abs_paths( working_dir, filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config) + resolver = ServiceExtendsResolver(service_config, version) service_dict = process_service(resolver.run()) # TODO: move to validate_service() @@ -301,8 +316,8 @@ def load_services(working_dir, config_files): def process_config_file(config_file, service_name=None, version=None): - validate_top_level_object(config_file) - processed_config = interpolate_environment_variables(config_file.config) + validate_top_level_service_objects(config_file, version) + processed_config = interpolate_environment_variables(config_file.config, version) validate_against_fields_schema( processed_config, config_file.filename, version ) @@ -316,10 +331,11 @@ def process_config_file(config_file, service_name=None, version=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, already_seen=None): + def __init__(self, service_config, version, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] + self.version = version @property def signature(self): @@ -348,7 +364,8 @@ class ServiceExtendsResolver(object): extended_file = process_config_file( ConfigFile.from_filename(config_path), - service_name=service_name) + service_name=service_name, version=self.version + ) service_config = extended_file.config[service_name] return config_path, service_config, service_name @@ -359,6 +376,7 @@ class ServiceExtendsResolver(object): extended_config_path, service_name, service_dict), + self.version, already_seen=self.already_seen + [self.signature]) service_config = resolver.run() diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema_v1.json similarity index 99% rename from compose/config/fields_schema.json rename to compose/config/fields_schema_v1.json index fdf56fd9..6f0a3631 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema_v1.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "id": "fields_schema.json", + "id": "fields_schema_v1.json", "patternProperties": { "^[a-zA-Z0-9._-]+$": { diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 2ca41c47..49cab367 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -1,38 +1,44 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", + "id": "fields_schema_v2.json", + "properties": { "version": { "enum": [2] }, "services": { + "id": "#/properties/services", "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "fields_schema.json#/definitions/service" + "$ref": "fields_schema_v1.json#/definitions/service" } - } + }, + "additionalProperties": false }, "volumes": { + "id": "#/properties/volumes", "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { "$ref": "#/definitions/volume" } - } + }, + "additionalProperties": false } }, "definitions": { "volume": { + "id": "#/definitions/volume", "type": "object", "properties": { "driver": {"type": "string"}, "driver_opts": { "type": "object", "patternProperties": { - "^.+$": {"type": ["boolean", "string", "number"]} + "^.+$": {"type": ["string", "number"]} }, "additionalProperties": false } diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index ba7e35c1..a8ff08d8 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -8,13 +8,19 @@ from .errors import ConfigurationError log = logging.getLogger(__name__) -def interpolate_environment_variables(config): +def interpolate_environment_variables(config, version): mapping = BlankDefaultDict(os.environ) + service_dicts = config if version == 1 else config.get('services', {}) - return dict( + interpolated = dict( (service_name, process_service(service_name, service_dict, mapping)) - for (service_name, service_dict) in config.items() + for (service_name, service_dict) in service_dicts.items() ) + if version == 1: + return interpolated + result = dict(config) + result.update({'services': interpolated}) + return result def process_service(service_name, service_dict, mapping): diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json index 05774efd..91a1e005 100644 --- a/compose/config/service_schema.json +++ b/compose/config/service_schema.json @@ -5,7 +5,7 @@ "type": "object", "allOf": [ - {"$ref": "fields_schema.json#/definitions/service"}, + {"$ref": "fields_schema_v1.json#/definitions/service"}, {"$ref": "#/definitions/constraints"} ], diff --git a/compose/config/validation.py b/compose/config/validation.py index 861cb10f..617c95b6 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -74,14 +74,15 @@ def format_boolean_in_environment(instance): return True -def validate_top_level_service_objects(config_file): +def validate_top_level_service_objects(config_file, version): """Perform some high level validation of the service name and value. This validation must happen before interpolation, which must happen before the rest of validation, which is why it's separate from the rest of the service validation. """ - for service_name, service_dict in config_file.config.items(): + service_dicts = config_file.config if version == 1 else config_file.config.get('services', {}) + for service_name, service_dict in service_dicts.items(): if not isinstance(service_name, six.string_types): raise ConfigurationError( "In file '{}' service name: {} needs to be a string, eg '{}'".format( @@ -105,7 +106,6 @@ def validate_top_level_object(config_file): "that you have defined a service at the top level.".format( config_file.filename, type(config_file.config))) - validate_top_level_service_objects(config_file) def validate_extends_file_path(service_name, extends_options, filename): @@ -134,10 +134,14 @@ def anglicize_validator(validator): return 'a ' + validator +def is_service_dict_schema(schema_id): + return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services' + + def handle_error_for_schema_with_id(error, service_name): schema_id = error.schema['id'] - if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties': + if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': return "Invalid service name '{}' - only {} characters are allowed".format( # The service_name is the key to the json object list(error.instance)[0], @@ -281,10 +285,8 @@ def process_errors(errors, service_name=None): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config, filename, version=None): - schema_filename = "fields_schema.json" - if version: - schema_filename = "fields_schema_v{0}.json".format(version) +def validate_against_fields_schema(config, filename, version): + schema_filename = "fields_schema_v{0}.json".format(version) _validate_against_schema( config, schema_filename, diff --git a/compose/const.py b/compose/const.py index 1b689418..9c607ca2 100644 --- a/compose/const.py +++ b/compose/const.py @@ -10,3 +10,4 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +COMPOSEFILE_VERSIONS = (1, 2) diff --git a/docker-compose.spec b/docker-compose.spec index 24d03e05..c760d7b4 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -18,8 +18,13 @@ exe = EXE(pyz, a.datas, [ ( - 'compose/config/fields_schema.json', - 'compose/config/fields_schema.json', + 'compose/config/fields_schema_v1.json', + 'compose/config/fields_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/fields_schema_v2.json', + 'compose/config/fields_schema_v2.json', 'DATA' ), ( @@ -33,6 +38,7 @@ exe = EXE(pyz, 'DATA' ) ], + name='docker-compose', debug=False, strip=None, From df6877a277e1c4bfce24a1fed0bdaa63bdcc84e2 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Dec 2015 17:28:15 -0800 Subject: [PATCH 182/359] Use newer docker-py version Signed-off-by: Joffrey F --- requirements.txt | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index ac02b8db..8c6d5f3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,5 @@ --e git://github.com/docker/docker-py.git@881e24c231ab9921eb0cbd475e85706137983f89#egg=docker-py PyYAML==3.11 -# docker-py==1.5.1 +docker-py==1.6.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 From ecef5d37a7eae730eed47648cbe4eb600eef3004 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Dec 2015 17:28:42 -0800 Subject: [PATCH 183/359] Add v2 configuration tests Signed-off-by: Joffrey F --- compose/config/validation.py | 2 +- compose/service.py | 1 + tests/integration/project_test.py | 80 ++++++++++++++ tests/unit/config/config_test.py | 176 ++++++++++++++++++++++++++++-- tests/unit/project_test.py | 23 ---- 5 files changed, 251 insertions(+), 31 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 617c95b6..6e943974 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -290,7 +290,7 @@ def validate_against_fields_schema(config, filename, version): _validate_against_schema( config, schema_filename, - format_checker=["ports", "environment", "bool-value-in-mapping"], + format_checker=["ports", "expose", "bool-value-in-mapping"], filename=filename) diff --git a/compose/service.py b/compose/service.py index 251620e9..24fa6394 100644 --- a/compose/service.py +++ b/compose/service.py @@ -868,6 +868,7 @@ def get_container_data_volumes(container, volumes_option): continue mount = container_mounts.get(volume.internal) + # New volume, doesn't exist in the old container if not mount: continue diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 4107c6cf..b4acda40 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import random + from .testcases import DockerClientTestCase from compose.cli.docker_client import docker_client from compose.config import config @@ -508,3 +510,81 @@ class ProjectTest(DockerClientTestCase): project.up() service = project.get_service('web') self.assertEqual(len(service.containers()), 1) + + def test_project_up_volumes(self): + vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + config_data = config.Config( + 2, [{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], {vol_name: {'driver': 'local'}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.up() + self.assertEqual(len(project.containers()), 1) + + volume_data = self.client.inspect_volume(vol_name) + self.assertEqual(volume_data['Name'], vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_initialize_volumes(self): + vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + config_data = config.Config( + 2, [{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], {vol_name: {}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + volume_data = self.client.inspect_volume(vol_name) + self.assertEqual(volume_data['Name'], vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_project_up_implicit_volume_driver(self): + vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + config_data = config.Config( + 2, [{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], {vol_name: {}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.up() + + volume_data = self.client.inspect_volume(vol_name) + self.assertEqual(volume_data['Name'], vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_project_up_invalid_volume_driver(self): + vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + config_data = config.Config( + 2, [{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], {vol_name: {'driver': 'foobar'}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError): + project.initialize_volumes() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 426146cc..dac573ed 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -26,7 +26,7 @@ def make_service_dict(name, service_dict, working_dir, filename=None): working_dir=working_dir, filename=filename, name=name, - config=service_dict)) + config=service_dict), version=1) return config.process_service(resolver.run()) @@ -68,6 +68,85 @@ class ConfigTest(unittest.TestCase): ]) ) + def test_load_v2(self): + config_data = config.load( + build_config_details({ + 'version': 2, + 'services': { + 'foo': {'image': 'busybox'}, + 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, + }, + 'volumes': { + 'hello': { + 'driver': 'default', + 'driver_opts': {'beep': 'boop'} + } + } + }, 'working_dir', 'filename.yml') + ) + service_dicts = config_data.services + volume_dict = config_data.volumes + self.assertEqual( + service_sort(service_dicts), + service_sort([ + { + 'name': 'bar', + 'image': 'busybox', + 'environment': {'FOO': '1'}, + }, + { + 'name': 'foo', + 'image': 'busybox', + } + ]) + ) + self.assertEqual(volume_dict, { + 'hello': { + 'driver': 'default', + 'driver_opts': {'beep': 'boop'} + } + }) + + def test_load_service_with_name_version(self): + config_data = config.load( + build_config_details({ + 'version': { + 'image': 'busybox' + } + }, 'working_dir', 'filename.yml') + ) + service_dicts = config_data.services + self.assertEqual( + service_sort(service_dicts), + service_sort([ + { + 'name': 'version', + 'image': 'busybox', + } + ]) + ) + + def test_load_invalid_version(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 18, + 'services': { + 'foo': {'image': 'busybox'} + } + }, 'working_dir', 'filename.yml') + ) + + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 'two point oh', + 'services': { + 'foo': {'image': 'busybox'} + } + }, 'working_dir', 'filename.yml') + ) + def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( @@ -78,6 +157,16 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_throws_error_when_not_dict_v2(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details( + {'version': 2, 'services': {'web': 'busybox:latest'}}, + 'working_dir', + 'filename.yml' + ) + ) + def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: @@ -87,6 +176,17 @@ class ConfigTest(unittest.TestCase): 'filename.yml')) assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() + def test_config_invalid_service_names_v2(self): + for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: + with pytest.raises(ConfigurationError) as exc: + config.load( + build_config_details({ + 'version': 2, + 'services': {invalid_name: {'image': 'busybox'}} + }, 'working_dir', 'filename.yml') + ) + assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() + def test_load_with_invalid_field_name(self): config_details = build_config_details( {'web': {'image': 'busybox', 'name': 'bogus'}}, @@ -120,6 +220,22 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_integer_service_name_raise_validation_error_v2(self): + expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " + "be a string, eg '1'") + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + build_config_details( + { + 'version': 2, + 'services': {1: {'image': 'busybox'}} + }, + 'working_dir', + 'filename.yml' + ) + ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') def test_load_with_multiple_files(self): base_file = config.ConfigFile( @@ -248,12 +364,55 @@ class ConfigTest(unittest.TestCase): 'volumes': ['/tmp'], } }) - services = config.load(config_details) + services = config.load(config_details).services assert services[0]['name'] == 'volume' assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_load_with_multiple_files_v2(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'image': 'example/web', + 'links': ['db'], + }, + 'db': { + 'image': 'example/db', + } + }, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'build': '/', + 'volumes': ['/home/user/project:/code'], + }, + } + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'build': os.path.abspath('/'), + 'links': ['db'], + 'volumes': [VolumeSpec.parse('/home/user/project:/code')], + }, + { + 'name': 'db', + 'image': 'example/db', + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( @@ -742,7 +901,7 @@ class VolumeConfigTest(unittest.TestCase): None, ) ).services[0] - self.assertEqual(d['volumes'], ['/host/path:/container/path']) + self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @mock.patch.dict(os.environ) @@ -1130,9 +1289,10 @@ class EnvTest(unittest.TestCase): {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, "tests/fixtures/env", ) - ).services[0] - self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp'])) + self.assertEqual( + set(service_dict['volumes']), + set([VolumeSpec.parse('/tmp:/host/tmp')])) service_dict = config.load( build_config_details( @@ -1140,7 +1300,9 @@ class EnvTest(unittest.TestCase): "tests/fixtures/env", ) ).services[0] - self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp'])) + self.assertEqual( + set(service_dict['volumes']), + set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')])) def load_from_filename(filename): @@ -1595,7 +1757,7 @@ class BuildPathTest(unittest.TestCase): for valid_url in valid_urls: service_dict = config.load(build_config_details({ 'validurl': {'build': valid_url}, - }, '.', None)) + }, '.', None)).services assert service_dict[0]['build'] == valid_url def test_invalid_url_in_build_path(self): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index c0ed5e33..4bf5f463 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -35,29 +35,6 @@ class ProjectTest(unittest.TestCase): self.assertEqual(project.get_service('db').name, 'db') self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') - def test_from_dict_sorts_in_dependency_order(self): - project = Project.from_config('composetest', Config(None, [ - { - 'name': 'web', - 'image': 'busybox:latest', - 'links': ['db'], - }, - { - 'name': 'db', - 'image': 'busybox:latest', - 'volumes_from': ['volume'] - }, - { - 'name': 'volume', - 'image': 'busybox:latest', - 'volumes': ['/tmp'], - } - ], None), None) - - self.assertEqual(project.services[0].name, 'volume') - self.assertEqual(project.services[1].name, 'db') - self.assertEqual(project.services[2].name, 'web') - def test_from_config(self): dicts = Config(None, [ { From ec5111f1c29c5dd66d161906042ca39183ebd659 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 8 Dec 2015 17:21:20 -0800 Subject: [PATCH 184/359] Volumes are now prefixed with the project name When created through the compose file, volumes are prefixed with the name of the project they belong to + underscore, similarly to how containers are currently handled. Signed-off-by: Joffrey F --- compose/cli/main.py | 4 +-- compose/volume.py | 12 +++++-- tests/integration/project_test.py | 16 +++++---- tests/integration/testcases.py | 2 +- tests/integration/volume_test.py | 55 +++++++++++++++++++++++++++++++ 5 files changed, 77 insertions(+), 12 deletions(-) create mode 100644 tests/integration/volume_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 4a766133..006d33ec 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -211,11 +211,11 @@ class TopLevelCommand(DocoptCommand): return if options['--services']: - print('\n'.join(service['name'] for service in compose_config)) + print('\n'.join(service['name'] for service in compose_config.services)) return compose_config = dict( - (service.pop('name'), service) for service in compose_config) + (service.pop('name'), service) for service in compose_config.services) print(yaml.dump( compose_config, default_flow_style=False, diff --git a/compose/volume.py b/compose/volume.py index 304633d0..055bd6ab 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -10,10 +10,16 @@ class Volume(object): self.driver_opts = driver_opts def create(self): - return self.client.create_volume(self.name, self.driver, self.driver_opts) + return self.client.create_volume( + self.full_name, self.driver, self.driver_opts + ) def remove(self): - return self.client.remove_volume(self.name) + return self.client.remove_volume(self.full_name) def inspect(self): - return self.client.inspect_volume(self.name) + return self.client.inspect_volume(self.full_name) + + @property + def full_name(self): + return '{0}_{1}'.format(self.project, self.name) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index b4acda40..70def617 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -513,6 +513,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_volumes(self): vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( 2, [{ 'name': 'web', @@ -528,12 +529,13 @@ class ProjectTest(DockerClientTestCase): project.up() self.assertEqual(len(project.containers()), 1) - volume_data = self.client.inspect_volume(vol_name) - self.assertEqual(volume_data['Name'], vol_name) + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') def test_initialize_volumes(self): vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( 2, [{ 'name': 'web', @@ -548,12 +550,13 @@ class ProjectTest(DockerClientTestCase): ) project.initialize_volumes() - volume_data = self.client.inspect_volume(vol_name) - self.assertEqual(volume_data['Name'], vol_name) + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') def test_project_up_implicit_volume_driver(self): vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( 2, [{ 'name': 'web', @@ -568,12 +571,13 @@ class ProjectTest(DockerClientTestCase): ) project.up() - volume_data = self.client.inspect_volume(vol_name) - self.assertEqual(volume_data['Name'], vol_name) + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') def test_project_up_invalid_volume_driver(self): vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + config_data = config.Config( 2, [{ 'name': 'web', diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index a9f5e7bb..8e0525ee 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -41,7 +41,7 @@ class DockerClientTestCase(unittest.TestCase): self.client.remove_image(i) volumes = self.client.volumes().get('Volumes') or [] for v in volumes: - if 'composetests_' in v['Name']: + if 'composetest_' in v['Name']: self.client.remove_volume(v['Name']) def create_service(self, name, **kwargs): diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py new file mode 100644 index 00000000..b6086040 --- /dev/null +++ b/tests/integration/volume_test.py @@ -0,0 +1,55 @@ +from __future__ import unicode_literals + +from docker.errors import DockerException + +from .testcases import DockerClientTestCase +from compose.volume import Volume + + +class VolumeTest(DockerClientTestCase): + def setUp(self): + self.tmp_volumes = [] + + def tearDown(self): + for volume in self.tmp_volumes: + try: + self.client.remove_volume(volume.full_name) + except DockerException: + pass + + def create_volume(self, name, driver=None, opts=None): + vol = Volume( + self.client, 'composetest', name, driver=driver, driver_opts=opts + ) + self.tmp_volumes.append(vol) + return vol + + def test_create_volume(self): + vol = self.create_volume('volume01') + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + def test_recreate_existing_volume(self): + vol = self.create_volume('volume01') + + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + def test_inspect_volume(self): + vol = self.create_volume('volume01') + vol.create() + info = vol.inspect() + assert info['Name'] == vol.full_name + + def test_remove_volume(self): + vol = Volume(self.client, 'composetest', 'volume01') + vol.create() + vol.remove() + volumes = self.client.volumes()['Volumes'] + assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 From 661519ac1c8fa7913cd9d289951c41447eeebff9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Dec 2015 14:47:09 -0800 Subject: [PATCH 185/359] Only test latest version in CI script Signed-off-by: Joffrey F --- script/test-versions | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/test-versions b/script/test-versions index 623b107b..a905cedf 100755 --- a/script/test-versions +++ b/script/test-versions @@ -18,7 +18,7 @@ get_versions="docker run --rm if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="$($get_versions recent -n 2)" + DOCKER_VERSIONS="$($get_versions recent -n 1)" fi From f3a9533dc04cfa43be128d97bea3b30fc003b7c6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 9 Dec 2015 17:06:56 -0800 Subject: [PATCH 186/359] version no longer optional arg for process_config_file Signed-off-by: Joffrey F --- compose/config/config.py | 4 ++-- compose/project.py | 2 +- tests/integration/project_test.py | 20 ++++++++++---------- tests/unit/config/config_test.py | 2 +- 4 files changed, 14 insertions(+), 14 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 295dc494..63ef44c3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -315,7 +315,7 @@ def load_services(working_dir, config_files, version): return build_services(config_file) -def process_config_file(config_file, service_name=None, version=None): +def process_config_file(config_file, version, service_name=None): validate_top_level_service_objects(config_file, version) processed_config = interpolate_environment_variables(config_file.config, version) validate_against_fields_schema( @@ -364,7 +364,7 @@ class ServiceExtendsResolver(object): extended_file = process_config_file( ConfigFile.from_filename(config_path), - service_name=service_name, version=self.version + version=self.version, service_name=service_name ) service_config = extended_file.config[service_name] return config_path, service_config, service_name diff --git a/compose/project.py b/compose/project.py index b7f33e3f..36855f16 100644 --- a/compose/project.py +++ b/compose/project.py @@ -234,7 +234,7 @@ class Project(object): volume.create() except NotFound: raise ConfigurationError( - 'Volume %s sepcifies nonexistent driver %s' % (volume.name, volume.driver) + 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) ) def restart(self, service_names=None, **options): diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 70def617..f2c65084 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -512,14 +512,14 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(service.containers()), 1) def test_project_up_volumes(self): - vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - 2, [{ + version=2, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], {vol_name: {'driver': 'local'}} + }], volumes={vol_name: {'driver': 'local'}} ) project = Project.from_config( @@ -534,14 +534,14 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Driver'], 'local') def test_initialize_volumes(self): - vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - 2, [{ + version=2, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], {vol_name: {}} + }], volumes={vol_name: {}} ) project = Project.from_config( @@ -555,14 +555,14 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Driver'], 'local') def test_project_up_implicit_volume_driver(self): - vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - 2, [{ + version=2, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], {vol_name: {}} + }], volumes={vol_name: {}} ) project = Project.from_config( @@ -576,7 +576,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Driver'], 'local') def test_project_up_invalid_volume_driver(self): - vol_name = 'composetests_{0:x}'.format(random.getrandbits(32)) + vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( 2, [{ diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index dac573ed..74978150 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -702,7 +702,7 @@ class ConfigTest(unittest.TestCase): 'dns_search': 'domain.local', } })) - assert actual == [ + assert actual.services == [ { 'name': 'web', 'image': 'alpine', From a7689f3da8a671b2a5dacd58ebc9259c86793f86 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 11 Dec 2015 17:21:04 -0800 Subject: [PATCH 187/359] Handle volume driver change error in config. Assume version=1 if file is empty in get_config_version Empty files are invalid anyway, so this simplifies the algorithm somewhat. https://github.com/docker/compose/pull/2421#discussion_r47223144 Don't leak version considerations in interpolation/service validation Signed-off-by: Joffrey F --- compose/config/config.py | 22 +++++++++++++----- compose/config/interpolation.py | 10 ++------ compose/config/validation.py | 10 ++++---- compose/project.py | 12 ++++++++++ tests/integration/project_test.py | 38 +++++++++++++++++++++++++++++-- tests/unit/config/config_test.py | 23 +++++++++++++++++++ 6 files changed, 94 insertions(+), 21 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 63ef44c3..c77e6100 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -118,6 +118,9 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def from_filename(cls, filename): return cls(filename, load_yaml(filename)) + def get_service_dicts(self, version): + return self.config if version == 1 else self.config.get('services', {}) + class Config(namedtuple('_Config', 'version services volumes')): """ @@ -164,9 +167,11 @@ def find(base_dir, filenames): def get_config_version(config_details): def get_version(config): if config.config is None: - return None + return 1 version = config.config.get('version', 1) if isinstance(version, dict): + # in that case 'version' is probably a service name, so assume + # this is a legacy (version=1) file version = 1 return version @@ -176,9 +181,6 @@ def get_config_version(config_details): for next_file in config_details.config_files[1:]: validate_top_level_object(next_file) next_file_version = get_version(next_file) - if version is None: - version = next_file_version - continue if version != next_file_version and next_file_version is not None: raise ConfigurationError( @@ -316,8 +318,16 @@ def load_services(working_dir, config_files, version): def process_config_file(config_file, version, service_name=None): - validate_top_level_service_objects(config_file, version) - processed_config = interpolate_environment_variables(config_file.config, version) + service_dicts = config_file.get_service_dicts(version) + validate_top_level_service_objects( + config_file.filename, service_dicts + ) + interpolated_config = interpolate_environment_variables(service_dicts) + if version == 2: + processed_config = dict(config_file.config) + processed_config.update({'services': interpolated_config}) + if version == 1: + processed_config = interpolated_config validate_against_fields_schema( processed_config, config_file.filename, version ) diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index a8ff08d8..12eb497b 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -8,19 +8,13 @@ from .errors import ConfigurationError log = logging.getLogger(__name__) -def interpolate_environment_variables(config, version): +def interpolate_environment_variables(service_dicts): mapping = BlankDefaultDict(os.environ) - service_dicts = config if version == 1 else config.get('services', {}) - interpolated = dict( + return dict( (service_name, process_service(service_name, service_dict, mapping)) for (service_name, service_dict) in service_dicts.items() ) - if version == 1: - return interpolated - result = dict(config) - result.update({'services': interpolated}) - return result def process_service(service_name, service_dict, mapping): diff --git a/compose/config/validation.py b/compose/config/validation.py index 6e943974..091014f6 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -74,19 +74,18 @@ def format_boolean_in_environment(instance): return True -def validate_top_level_service_objects(config_file, version): +def validate_top_level_service_objects(filename, service_dicts): """Perform some high level validation of the service name and value. This validation must happen before interpolation, which must happen before the rest of validation, which is why it's separate from the rest of the service validation. """ - service_dicts = config_file.config if version == 1 else config_file.config.get('services', {}) for service_name, service_dict in service_dicts.items(): if not isinstance(service_name, six.string_types): raise ConfigurationError( "In file '{}' service name: {} needs to be a string, eg '{}'".format( - config_file.filename, + filename, service_name, service_name)) @@ -95,8 +94,9 @@ def validate_top_level_service_objects(config_file, version): "In file '{}' service '{}' doesn\'t have any configuration options. " "All top level keys in your docker-compose.yml must map " "to a dictionary of configuration options.".format( - config_file.filename, - service_name)) + filename, service_name + ) + ) def validate_top_level_object(config_file): diff --git a/compose/project.py b/compose/project.py index 36855f16..3801bbb9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -236,6 +236,18 @@ class Project(object): raise ConfigurationError( 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) ) + except APIError as e: + if 'Choose a different volume name' in str(e): + raise ConfigurationError( + 'Configuration for volume {0} specifies driver {1}, but ' + 'a volume with the same name uses a different driver ' + '({3}). If you wish to use the new configuration, please ' + 'remove the existing volume "{2}" first:\n' + '$ docker volume rm {2}'.format( + volume.name, volume.driver, volume.full_name, + volume.inspect()['Driver'] + ) + ) def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index f2c65084..d51830bb 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -579,11 +579,11 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( - 2, [{ + version=2, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], {vol_name: {'driver': 'foobar'}} + }], volumes={vol_name: {'driver': 'foobar'}} ) project = Project.from_config( @@ -592,3 +592,37 @@ class ProjectTest(DockerClientTestCase): ) with self.assertRaises(config.ConfigurationError): project.initialize_volumes() + + def test_project_up_updated_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'local'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + config_data = config_data._replace( + volumes={vol_name: {'driver': 'smb'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError) as e: + project.initialize_volumes() + assert 'Configuration for volume {0} specifies driver smb'.format( + vol_name + ) in str(e.exception) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 74978150..281e81d1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -286,6 +286,18 @@ class ConfigTest(unittest.TestCase): error_msg = "Top level object in 'override.yml' needs to be an object" assert error_msg in exc.exconly() + def test_load_with_multiple_files_and_empty_override_v2(self): + base_file = config.ConfigFile( + 'base.yml', + {'version': 2, 'services': {'web': {'image': 'example/web'}}}) + override_file = config.ConfigFile('override.yml', None) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + error_msg = "Top level object in 'override.yml' needs to be an object" + assert error_msg in exc.exconly() + def test_load_with_multiple_files_and_empty_base(self): base_file = config.ConfigFile('base.yml', None) override_file = config.ConfigFile( @@ -297,6 +309,17 @@ class ConfigTest(unittest.TestCase): config.load(details) assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() + def test_load_with_multiple_files_and_empty_base_v2(self): + base_file = config.ConfigFile('base.yml', None) + override_file = config.ConfigFile( + 'override.tml', + {'version': 2, 'services': {'web': {'image': 'example/web'}}} + ) + details = config.ConfigDetails('.', [base_file, override_file]) + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() + def test_load_with_multiple_files_and_extends_in_override_file(self): base_file = config.ConfigFile( 'base.yaml', From 1dcdd98da4d8f5bce52eddf013875b730e2ed130 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 5 Jan 2016 15:24:58 -0800 Subject: [PATCH 188/359] Add TODO note to restore n-1 version testing after 1.10 release Signed-off-by: Joffrey F --- script/test-versions | 1 + 1 file changed, 1 insertion(+) diff --git a/script/test-versions b/script/test-versions index a905cedf..76e55e11 100755 --- a/script/test-versions +++ b/script/test-versions @@ -18,6 +18,7 @@ get_versions="docker run --rm if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then + # TODO: `-n 2` when engine 1.10 releases DOCKER_VERSIONS="$($get_versions recent -n 1)" fi From c7b71422c0bea8ce19b8f28e426fe4ea597eb83c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 6 Jan 2016 13:30:40 -0500 Subject: [PATCH 189/359] Fix extends with multiple files. Signed-off-by: Daniel Nephin --- compose/config/config.py | 2 ++ tests/unit/config/config_test.py | 31 +++++++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index 195665b5..9e62ef1c 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -422,6 +422,8 @@ def merge_service_dicts_from_files(base, override): new_service = merge_service_dicts(base, override) if 'extends' in override: new_service['extends'] = override['extends'] + elif 'extends' in base: + new_service['extends'] = base['extends'] return new_service diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e975cb9d..97a0838f 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -552,6 +552,37 @@ class ConfigTest(unittest.TestCase): } ] + def test_merge_service_dicts_from_files_with_extends_in_base(self): + base = { + 'volumes': ['.:/app'], + 'extends': {'service': 'app'} + } + override = { + 'image': 'alpine:edge', + } + actual = config.merge_service_dicts_from_files(base, override) + assert actual == { + 'image': 'alpine:edge', + 'volumes': ['.:/app'], + 'extends': {'service': 'app'} + } + + def test_merge_service_dicts_from_files_with_extends_in_override(self): + base = { + 'volumes': ['.:/app'], + 'extends': {'service': 'app'} + } + override = { + 'image': 'alpine:edge', + 'extends': {'service': 'foo'} + } + actual = config.merge_service_dicts_from_files(base, override) + assert actual == { + 'image': 'alpine:edge', + 'volumes': ['.:/app'], + 'extends': {'service': 'foo'} + } + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ From 77d2aae72dbed943e0b7ae58e392a5bca49a4263 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Wed, 23 Dec 2015 23:53:32 +0100 Subject: [PATCH 190/359] Fix typo in unpause reference doc Signed-off-by: Vincent Demeester --- docs/reference/unpause.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/reference/unpause.md b/docs/reference/unpause.md index 6434b09c..846b229e 100644 --- a/docs/reference/unpause.md +++ b/docs/reference/unpause.md @@ -9,7 +9,7 @@ parent = "smn_compose_cli" +++ -# pause +# unpause ``` Usage: unpause [SERVICE...] From 4e75ed42319b372ac79c7b8762c5fec794afa841 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Wed, 23 Dec 2015 23:59:48 +0100 Subject: [PATCH 191/359] Add missing --name flag to run reference doc Signed-off-by: Vincent Demeester --- docs/reference/run.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/reference/run.md b/docs/reference/run.md index c1efb9a7..21890c60 100644 --- a/docs/reference/run.md +++ b/docs/reference/run.md @@ -17,6 +17,7 @@ Usage: run [options] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] Options: -d Detached mode: Run container in the background, print new container name. +--name NAME Assign a name to the container --entrypoint CMD Override the entrypoint of the image. -e KEY=VAL Set an environment variable (can be used multiple times) -u, --user="" Run as specified username or uid From 0bca8d9cb39a01736f2ce043f2ea7b6407ffc281 Mon Sep 17 00:00:00 2001 From: Vincent Demeester Date: Wed, 6 Jan 2016 21:28:47 +0100 Subject: [PATCH 192/359] Add config and create to docs/reference Signed-off-by: Vincent Demeester --- docs/reference/config.md | 23 +++++++++++++++++++++++ docs/reference/create.md | 25 +++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100644 docs/reference/config.md create mode 100644 docs/reference/create.md diff --git a/docs/reference/config.md b/docs/reference/config.md new file mode 100644 index 00000000..1a9706f4 --- /dev/null +++ b/docs/reference/config.md @@ -0,0 +1,23 @@ + + +# config + +```: +Usage: config [options] + +Options: +-q, --quiet Only validate the configuration, don't print + anything. +--services Print the service names, one per line. +``` + +Validate and view the compose file. diff --git a/docs/reference/create.md b/docs/reference/create.md new file mode 100644 index 00000000..a785e2c7 --- /dev/null +++ b/docs/reference/create.md @@ -0,0 +1,25 @@ + + +# create + +``` +Usage: create [options] [SERVICE...] + +Options: +--force-recreate Recreate containers even if their configuration and + image haven't changed. Incompatible with --no-recreate. +--no-recreate If containers already exist, don't recreate them. + Incompatible with --force-recreate. +--no-build Don't build an image, even if it's missing +``` + +Creates containers for a service. From 475a09176850c3f6d9fd51fc6e82e03263a3d733 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 16:22:51 -0400 Subject: [PATCH 193/359] Update pre-commit config to enforace that future imports exist in all files. Signed-off-by: Daniel Nephin --- .pre-commit-config.yaml | 7 ++++++- compose/__init__.py | 1 + compose/cli/colors.py | 1 + compose/cli/docker_client.py | 3 +++ compose/cli/errors.py | 1 + compose/cli/main.py | 1 + compose/cli/multiplexer.py | 1 + compose/cli/verbose_proxy.py | 3 +++ compose/config/__init__.py | 3 +++ compose/config/config.py | 1 + compose/config/errors.py | 4 ++++ compose/config/interpolation.py | 3 +++ compose/config/validation.py | 3 +++ compose/const.py | 3 +++ compose/progress_stream.py | 3 +++ compose/utils.py | 3 +++ script/travis/render-bintray-config.py | 2 ++ script/versions.py | 2 ++ tests/__init__.py | 3 +++ tests/acceptance/cli_test.py | 1 + tests/integration/project_test.py | 1 + tests/integration/state_test.py | 1 + tests/unit/cli/command_test.py | 1 + tests/unit/cli/main_test.py | 1 + tests/unit/config/config_test.py | 2 ++ tests/unit/config/sort_services_test.py | 3 +++ tests/unit/container_test.py | 1 + tests/unit/interpolation_test.py | 3 +++ tests/unit/multiplexer_test.py | 3 +++ tests/unit/project_test.py | 1 + tests/unit/utils_test.py | 1 + 31 files changed, 66 insertions(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3fad8ddc..db2b6506 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -14,7 +14,12 @@ - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports - sha: 3d86483455ab5bd06cc1069fdd5ac57be5463f10 + sha: v0.1.0 hooks: - id: reorder-python-imports language_version: 'python2.7' + args: + - --add-import + - from __future__ import absolute_import + - --add-import + - from __future__ import unicode_literals diff --git a/compose/__init__.py b/compose/__init__.py index 7c16c97b..3ba90fde 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals __version__ = '1.6.0dev' diff --git a/compose/cli/colors.py b/compose/cli/colors.py index af4a32ab..3c18886f 100644 --- a/compose/cli/colors.py +++ b/compose/cli/colors.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals NAMES = [ 'grey', diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 177d5d6c..48ba97bd 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import logging import os diff --git a/compose/cli/errors.py b/compose/cli/errors.py index ca4413bd..03d6a50c 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import unicode_literals from textwrap import dedent diff --git a/compose/cli/main.py b/compose/cli/main.py index 006d33ec..9ea9df71 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index 4c73c6cd..5e8d91a4 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import unicode_literals from threading import Thread diff --git a/compose/cli/verbose_proxy.py b/compose/cli/verbose_proxy.py index 68dfabe5..b1592eab 100644 --- a/compose/cli/verbose_proxy.py +++ b/compose/cli/verbose_proxy.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import functools import logging import pprint diff --git a/compose/config/__init__.py b/compose/config/__init__.py index 6fe9ff9f..dd01f221 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -1,4 +1,7 @@ # flake8: noqa +from __future__ import absolute_import +from __future__ import unicode_literals + from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find diff --git a/compose/config/config.py b/compose/config/config.py index c77e6100..61b40589 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import unicode_literals import codecs import logging diff --git a/compose/config/errors.py b/compose/config/errors.py index 6d6a69df..99129f3d 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -1,3 +1,7 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + + class ConfigurationError(Exception): def __init__(self, msg): self.msg = msg diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 12eb497b..7a757644 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import logging import os from string import Template diff --git a/compose/config/validation.py b/compose/config/validation.py index 091014f6..f2162a87 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import json import logging import os diff --git a/compose/const.py b/compose/const.py index 9c607ca2..f1493cdd 100644 --- a/compose/const.py +++ b/compose/const.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import os import sys diff --git a/compose/progress_stream.py b/compose/progress_stream.py index a6c8e0a2..1f873d1d 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + from compose import utils diff --git a/compose/utils.py b/compose/utils.py index 362629bc..4a7df334 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import codecs import hashlib import json diff --git a/script/travis/render-bintray-config.py b/script/travis/render-bintray-config.py index fc5d409a..c2b11ca3 100755 --- a/script/travis/render-bintray-config.py +++ b/script/travis/render-bintray-config.py @@ -1,5 +1,7 @@ #!/usr/bin/env python from __future__ import print_function +from __future__ import absolute_import +from __future__ import unicode_literals import datetime import os.path diff --git a/script/versions.py b/script/versions.py index 513ca754..98f97ef3 100755 --- a/script/versions.py +++ b/script/versions.py @@ -21,7 +21,9 @@ For example, if the list of versions is: `default` would return `1.7.1` and `recent -n 3` would return `1.8.0-rc2 1.7.1 1.6.2` """ +from __future__ import absolute_import from __future__ import print_function +from __future__ import unicode_literals import argparse import itertools diff --git a/tests/__init__.py b/tests/__init__.py index d3cfb864..1ac1b21c 100644 --- a/tests/__init__.py +++ b/tests/__init__.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import sys if sys.version_info >= (2, 7): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 1885727a..6859c774 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import unicode_literals import os import shlex diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d51830bb..2cf5f556 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals import random diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index d07dfa82..6e656c29 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -2,6 +2,7 @@ Integration tests which cover state convergence (aka smart recreate) performed by `docker-compose up`. """ +from __future__ import absolute_import from __future__ import unicode_literals import py diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 0d4324e3..18044672 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import unicode_literals import pytest from requests.exceptions import ConnectionError diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index db37ac1a..ab236866 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +from __future__ import unicode_literals import logging diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 281e81d1..8cb5d9b2 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1,5 +1,7 @@ # encoding: utf-8 +from __future__ import absolute_import from __future__ import print_function +from __future__ import unicode_literals import os import shutil diff --git a/tests/unit/config/sort_services_test.py b/tests/unit/config/sort_services_test.py index 8d0c3ae4..c2ebbc67 100644 --- a/tests/unit/config/sort_services_test.py +++ b/tests/unit/config/sort_services_test.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + from compose.config.errors import DependencyError from compose.config.sort_services import sort_service_dicts from compose.config.types import VolumeFromSpec diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 5f7bf1ea..88691150 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals import docker diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py index 7444884c..317982a9 100644 --- a/tests/unit/interpolation_test.py +++ b/tests/unit/interpolation_test.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import unittest from compose.config.interpolation import BlankDefaultDict as bddict diff --git a/tests/unit/multiplexer_test.py b/tests/unit/multiplexer_test.py index d565d39d..c56ece1b 100644 --- a/tests/unit/multiplexer_test.py +++ b/tests/unit/multiplexer_test.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import unittest from compose.cli.multiplexer import Multiplexer diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 4bf5f463..a182680b 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals import docker diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 15999dde..8ee37b07 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -1,4 +1,5 @@ # encoding: utf-8 +from __future__ import absolute_import from __future__ import unicode_literals from compose import utils From bf1552da7982b22b874b1938af9bf80094c884e8 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 30 Oct 2015 16:50:31 -0400 Subject: [PATCH 194/359] Use json to encode invalid values in configuration errors so that the user sees a proper repr of the value. Signed-off-by: Daniel Nephin --- compose/__main__.py | 3 +++ compose/config/sort_services.py | 3 +++ compose/config/validation.py | 3 ++- compose/volume.py | 1 + script/travis/render-bintray-config.py | 2 +- tests/integration/volume_test.py | 1 + tests/unit/config/config_test.py | 4 +++- tests/unit/config/types_test.py | 3 +++ 8 files changed, 17 insertions(+), 3 deletions(-) diff --git a/compose/__main__.py b/compose/__main__.py index 199ba2ae..27a7acbb 100644 --- a/compose/__main__.py +++ b/compose/__main__.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + from compose.cli.main import main main() diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py index 5d9adab1..05552122 100644 --- a/compose/config/sort_services.py +++ b/compose/config/sort_services.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + from compose.config.errors import DependencyError diff --git a/compose/config/validation.py b/compose/config/validation.py index f2162a87..74dd461f 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -254,7 +254,8 @@ def _parse_oneof_validator(error): ) return "{}contains {}, which is an invalid type, it should be {}".format( invalid_config_key, - context.instance, + # Always print the json repr of the invalid value + json.dumps(context.instance), _parse_valid_types_from_validator(context.validator_value)) if context.validator == 'uniqueItems': diff --git a/compose/volume.py b/compose/volume.py index 055bd6ab..fb8bd580 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals diff --git a/script/travis/render-bintray-config.py b/script/travis/render-bintray-config.py index c2b11ca3..b5364a0b 100755 --- a/script/travis/render-bintray-config.py +++ b/script/travis/render-bintray-config.py @@ -1,6 +1,6 @@ #!/usr/bin/env python -from __future__ import print_function from __future__ import absolute_import +from __future__ import print_function from __future__ import unicode_literals import datetime diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index b6086040..8ae35378 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -1,3 +1,4 @@ +from __future__ import absolute_import from __future__ import unicode_literals from docker.errors import DockerException diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 8cb5d9b2..abb891a2 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -552,7 +552,9 @@ class ConfigTest(unittest.TestCase): ) def test_config_extra_hosts_list_of_dicts_validation_error(self): - expected_error_msg = "key 'extra_hosts' contains {'somehost': '162.242.195.82'}, which is an invalid type, it should be a string" + expected_error_msg = ( + "key 'extra_hosts' contains {\"somehost\": \"162.242.195.82\"}, " + "which is an invalid type, it should be a string") with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): config.load( diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 4df66548..245b854f 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import pytest from compose.config.errors import ConfigurationError From ed5f7bd3949b289cebace496d9a40e67e05db466 Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Thu, 17 Dec 2015 23:23:00 +0200 Subject: [PATCH 195/359] log_driver and log_opt moved to logging key. Signed-off-by: Dimitar Bonev --- compose/config/config.py | 3 +- compose/config/fields_schema_v1.json | 11 +++++- compose/service.py | 25 +++++++++++-- docs/compose-file.md | 28 +++++++++----- tests/acceptance/cli_test.py | 37 +++++++++++++++++++ .../fixtures/logging-composefile/compose2.yml | 3 ++ .../logging-composefile/docker-compose.yml | 12 ++++++ tests/integration/service_test.py | 4 +- tests/unit/service_test.py | 3 +- 9 files changed, 105 insertions(+), 21 deletions(-) create mode 100644 tests/fixtures/logging-composefile/compose2.yml create mode 100644 tests/fixtures/logging-composefile/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 61b40589..0012aefd 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -51,8 +51,6 @@ DOCKER_CONFIG_KEYS = [ 'ipc', 'labels', 'links', - 'log_driver', - 'log_opt', 'mac_address', 'mem_limit', 'memswap_limit', @@ -78,6 +76,7 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'dockerfile', 'expose', 'external_links', + 'logging', ] DOCKER_VALID_URL_PREFIXES = ( diff --git a/compose/config/fields_schema_v1.json b/compose/config/fields_schema_v1.json index 6f0a3631..790ace34 100644 --- a/compose/config/fields_schema_v1.json +++ b/compose/config/fields_schema_v1.json @@ -75,8 +75,15 @@ "labels": {"$ref": "#/definitions/list_or_dict"}, "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "log_driver": {"type": "string"}, - "log_opt": {"type": "object"}, + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": {"type": "object"} + }, + "additionalProperties": false + }, "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, diff --git a/compose/service.py b/compose/service.py index 24fa6394..366833dd 100644 --- a/compose/service.py +++ b/compose/service.py @@ -510,6 +510,13 @@ class Service(object): return volumes_from + def get_logging_options(self): + logging_dict = self.options.get('logging', {}) + return { + 'log_driver': logging_dict.get('driver', ""), + 'log_opt': logging_dict.get('options', None) + } + def _get_container_create_options( self, override_options, @@ -523,6 +530,8 @@ class Service(object): for k in DOCKER_CONFIG_KEYS if k in self.options) container_options.update(override_options) + container_options.update(self.get_logging_options()) + if self.custom_container_name() and not one_off: container_options['name'] = self.custom_container_name() elif not container_options.get('name'): @@ -590,10 +599,9 @@ class Service(object): def _get_container_host_config(self, override_options, one_off=False): options = dict(self.options, **override_options) - log_config = LogConfig( - type=options.get('log_driver', ""), - config=options.get('log_opt', None) - ) + logging_dict = options.get('logging', None) + log_config = get_log_config(logging_dict) + return self.client.create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=build_port_bindings(options.get('ports') or []), @@ -953,3 +961,12 @@ def build_ulimits(ulimit_config): ulimits.append(ulimit_dict) return ulimits + + +def get_log_config(logging_dict): + log_driver = logging_dict.get('driver', "") if logging_dict else "" + log_options = logging_dict.get('options', None) if logging_dict else None + return LogConfig( + type=log_driver, + config=log_options + ) diff --git a/docs/compose-file.md b/docs/compose-file.md index 29e0c647..40a3cf02 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -324,29 +324,37 @@ for this service, e.g: Environment variables will also be created - see the [environment variable reference](env.md) for details. -### log_driver +### logging -Specify a logging driver for the service's containers, as with the ``--log-driver`` -option for docker run ([documented here](https://docs.docker.com/engine/reference/logging/overview/)). +Logging configuration for the service. This configuration replaces the previous +`log_driver` and `log_opt` keys. + + logging: + driver: log_driver + options: + syslog-address: "tcp://192.168.0.42:123" + +The `driver` name specifies a logging driver for the service's +containers, as with the ``--log-driver`` option for docker run +([documented here](https://docs.docker.com/engine/reference/logging/overview/)). The default value is json-file. - log_driver: "json-file" - log_driver: "syslog" - log_driver: "none" + driver: "json-file" + driver: "syslog" + driver: "none" > **Note:** Only the `json-file` driver makes the logs available directly from > `docker-compose up` and `docker-compose logs`. Using any other driver will not > print any logs. -### log_opt +Specify logging options for the logging driver with the ``options`` key, as with the ``--log-opt`` option for `docker run`. -Specify logging options with `log_opt` for the logging driver, as with the ``--log-opt`` option for `docker run`. Logging options are key value pairs. An example of `syslog` options: - log_driver: "syslog" - log_opt: + driver: "syslog" + options: syslog-address: "tcp://192.168.0.42:123" ### net diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6859c774..c5df3079 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -716,6 +716,43 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['start'], returncode=1) assert 'No containers to start' in result.stderr + def test_up_logging(self): + self.base_dir = 'tests/fixtures/logging-composefile' + self.dispatch(['up', '-d']) + simple = self.project.get_service('simple').containers()[0] + log_config = simple.get('HostConfig.LogConfig') + self.assertTrue(log_config) + self.assertEqual(log_config.get('Type'), 'none') + + another = self.project.get_service('another').containers()[0] + log_config = another.get('HostConfig.LogConfig') + self.assertTrue(log_config) + self.assertEqual(log_config.get('Type'), 'json-file') + self.assertEqual(log_config.get('Config')['max-size'], '10m') + + def test_up_logging_with_multiple_files(self): + self.base_dir = 'tests/fixtures/logging-composefile' + config_paths = [ + 'docker-compose.yml', + 'compose2.yml', + ] + self._project = get_project(self.base_dir, config_paths) + self.dispatch( + [ + '-f', config_paths[0], + '-f', config_paths[1], + 'up', '-d', + ], + None) + + containers = self.project.containers() + self.assertEqual(len(containers), 2) + + another = self.project.get_service('another').containers()[0] + log_config = another.get('HostConfig.LogConfig') + self.assertTrue(log_config) + self.assertEqual(log_config.get('Type'), 'none') + def test_pause_unpause(self): self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') diff --git a/tests/fixtures/logging-composefile/compose2.yml b/tests/fixtures/logging-composefile/compose2.yml new file mode 100644 index 00000000..ba582969 --- /dev/null +++ b/tests/fixtures/logging-composefile/compose2.yml @@ -0,0 +1,3 @@ +another: + logging: + driver: "none" diff --git a/tests/fixtures/logging-composefile/docker-compose.yml b/tests/fixtures/logging-composefile/docker-compose.yml new file mode 100644 index 00000000..877ee5e2 --- /dev/null +++ b/tests/fixtures/logging-composefile/docker-compose.yml @@ -0,0 +1,12 @@ +simple: + image: busybox:latest + command: top + logging: + driver: "none" +another: + image: busybox:latest + command: top + logging: + driver: "json-file" + options: + max-size: "10m" diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4a0eaacb..86bc4d9d 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -888,7 +888,7 @@ class ServiceTest(DockerClientTestCase): self.assertNotEqual(one_off_container.name, 'my-web-container') def test_log_drive_invalid(self): - service = self.create_service('web', log_driver='xxx') + service = self.create_service('web', logging={'driver': 'xxx'}) expected_error_msg = "logger: no log driver named 'xxx' is registered" with self.assertRaisesRegexp(APIError, expected_error_msg): @@ -902,7 +902,7 @@ class ServiceTest(DockerClientTestCase): self.assertFalse(log_config['Config']) def test_log_drive_none(self): - service = self.create_service('web', log_driver='none') + service = self.create_service('web', logging={'driver': 'none'}) log_config = create_and_start_container(service).log_config self.assertEqual('none', log_config['Type']) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 87d6af59..9cc35c7b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -156,7 +156,8 @@ class ServiceTest(unittest.TestCase): self.mock_client.create_host_config.return_value = {} log_opt = {'syslog-address': 'tcp://192.168.0.42:123'} - service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, log_driver='syslog', log_opt=log_opt) + logging = {'driver': 'syslog', 'options': log_opt} + service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, logging=logging) service._get_container_create_options({'some': 'overrides'}, 1) self.assertTrue(self.mock_client.create_host_config.called) From 978e9cf38f057a8dc08fd74b969fd38873a27ba6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 8 Jan 2016 13:10:05 +0000 Subject: [PATCH 196/359] Fix script/clean on systems where `find` requires a path argument Signed-off-by: Aanand Prasad --- script/clean | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/clean b/script/clean index 35faf4db..fb7ba3be 100755 --- a/script/clean +++ b/script/clean @@ -3,5 +3,5 @@ set -e find . -type f -name '*.pyc' -delete find . -name .coverage.* -delete -find -name __pycache__ -delete +find . -name __pycache__ -delete rm -rf docs/_site build dist docker-compose.egg-info From d1d3969661f549311bccde53703a2939402cf769 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 31 Aug 2015 14:31:20 -0400 Subject: [PATCH 197/359] Add docker-compose event Signed-off-by: Daniel Nephin --- compose/cli/main.py | 23 +++++++++++ compose/const.py | 1 + compose/project.py | 38 ++++++++++++++++- compose/utils.py | 4 ++ docs/reference/events.md | 34 ++++++++++++++++ docs/reference/index.md | 11 ++--- tests/acceptance/cli_test.py | 11 +++++ tests/unit/project_test.py | 79 ++++++++++++++++++++++++++++++++++++ 8 files changed, 195 insertions(+), 6 deletions(-) create mode 100644 docs/reference/events.md diff --git a/compose/cli/main.py b/compose/cli/main.py index 9ea9df71..d99816cf 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import json import logging import re import signal @@ -132,6 +133,7 @@ class TopLevelCommand(DocoptCommand): build Build or rebuild services config Validate and view the compose file create Create services + events Receive real time events from containers help Get help on a command kill Kill containers logs View output from containers @@ -244,6 +246,27 @@ class TopLevelCommand(DocoptCommand): do_build=not options['--no-build'] ) + def events(self, project, options): + """ + Receive real time events from containers. + + Usage: events [options] [SERVICE...] + + Options: + --json Output events as a stream of json objects + """ + def format_event(event): + return ("{time}: service={service} event={event} " + "container={container} image={image}").format(**event) + + def json_format_event(event): + event['time'] = event['time'].isoformat() + return json.dumps(event) + + for event in project.events(): + formatter = json_format_event if options['--json'] else format_event + print(formatter(event)) + def help(self, project, options): """ Get help on a command. diff --git a/compose/const.py b/compose/const.py index f1493cdd..84a5057a 100644 --- a/compose/const.py +++ b/compose/const.py @@ -6,6 +6,7 @@ import sys DEFAULT_TIMEOUT = 10 HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) +IMAGE_EVENTS = ['delete', 'import', 'pull', 'push', 'tag', 'untag'] IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' diff --git a/compose/project.py b/compose/project.py index 3801bbb9..b4eed7c8 100644 --- a/compose/project.py +++ b/compose/project.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import datetime import logging from functools import reduce @@ -11,6 +12,7 @@ from . import parallel from .config import ConfigurationError from .config.sort_services import get_service_name_from_net from .const import DEFAULT_TIMEOUT +from .const import IMAGE_EVENTS from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE @@ -20,6 +22,7 @@ from .service import ConvergenceStrategy from .service import Net from .service import Service from .service import ServiceNet +from .utils import microseconds_from_time_nano from .volume import Volume @@ -267,7 +270,40 @@ class Project(object): plans = self._get_convergence_plans(services, strategy) for service in services: - service.execute_convergence_plan(plans[service.name], do_build, detached=True, start=False) + service.execute_convergence_plan( + plans[service.name], + do_build, + detached=True, + start=False) + + def events(self): + def build_container_event(event, container): + time = datetime.datetime.fromtimestamp(event['time']) + time = time.replace( + microsecond=microseconds_from_time_nano(event['timeNano'])) + return { + 'service': container.service, + 'event': event['status'], + 'container': container.id, + 'image': event['from'], + 'time': time, + } + + service_names = set(self.service_names) + for event in self.client.events( + filters={'label': self.labels()}, + decode=True + ): + if event['status'] in IMAGE_EVENTS: + # We don't receive any image events because labels aren't applied + # to images + continue + + # TODO: get labels from the API v1.22 , see github issue 2618 + container = Container.from_id(self.client, event['id']) + if container.service not in service_names: + continue + yield build_container_event(event, container) def up(self, service_names=None, diff --git a/compose/utils.py b/compose/utils.py index 4a7df334..29d8a695 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -88,3 +88,7 @@ def json_hash(obj): h = hashlib.sha256() h.update(dump.encode('utf8')) return h.hexdigest() + + +def microseconds_from_time_nano(time_nano): + return int(time_nano % 1000000000 / 1000) diff --git a/docs/reference/events.md b/docs/reference/events.md new file mode 100644 index 00000000..827258f2 --- /dev/null +++ b/docs/reference/events.md @@ -0,0 +1,34 @@ + + +# events + +``` +Usage: events [options] [SERVICE...] + +Options: + --json Output events as a stream of json objects +``` + +Stream container events for every container in the project. + +With the `--json` flag, a json object will be printed one per line with the +format: + +``` +{ + "service": "web", + "event": "create", + "container": "213cf75fc39a", + "image": "alpine:edge", + "time": "2015-11-20T18:01:03.615550", +} +``` diff --git a/docs/reference/index.md b/docs/reference/index.md index b2fb5bca..1635b60c 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -14,19 +14,20 @@ parent = "smn_compose_ref" The following pages describe the usage information for the [docker-compose](docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. * [build](build.md) +* [events](events.md) * [help](help.md) * [kill](kill.md) -* [ps](ps.md) -* [restart](restart.md) -* [run](run.md) -* [start](start.md) -* [up](up.md) * [logs](logs.md) * [port](port.md) +* [ps](ps.md) * [pull](pull.md) +* [restart](restart.md) * [rm](rm.md) +* [run](run.md) * [scale](scale.md) +* [start](start.md) * [stop](stop.md) +* [up](up.md) ## Where to go next diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6859c774..322d9b5a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import json import os import shlex import signal @@ -855,6 +856,16 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(get_port(3000, index=2), containers[1].get_local_port(3000)) self.assertEqual(get_port(3002), "") + def test_events_json(self): + events_proc = start_process(self.base_dir, ['events', '--json']) + self.dispatch(['up', '-d']) + wait_on_condition(ContainerCountCondition(self.project, 2)) + + os.kill(events_proc.pid, signal.SIGINT) + result = wait_on_process(events_proc, returncode=1) + lines = [json.loads(line) for line in result.stdout.rstrip().split('\n')] + assert [e['event'] for e in lines] == ['create', 'start', 'create', 'start'] + def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') self.dispatch(['-f', config_path, 'up', '-d'], None) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index a182680b..a4b61b64 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import datetime + import docker from .. import mock @@ -197,6 +199,83 @@ class ProjectTest(unittest.TestCase): project.get_service('test')._get_volumes_from(), [container_ids[0] + ':rw']) + def test_events(self): + services = [Service(name='web'), Service(name='db')] + project = Project('test', services, self.mock_client) + self.mock_client.events.return_value = iter([ + { + 'status': 'create', + 'from': 'example/image', + 'id': 'abcde', + 'time': 1420092061, + 'timeNano': 14200920610000002000, + }, + { + 'status': 'attach', + 'from': 'example/image', + 'id': 'abcde', + 'time': 1420092061, + 'timeNano': 14200920610000003000, + }, + { + 'status': 'create', + 'from': 'example/other', + 'id': 'bdbdbd', + 'time': 1420092061, + 'timeNano': 14200920610000005000, + }, + { + 'status': 'create', + 'from': 'example/db', + 'id': 'ababa', + 'time': 1420092061, + 'timeNano': 14200920610000004000, + }, + ]) + + def dt_with_microseconds(dt, us): + return datetime.datetime.fromtimestamp(dt).replace(microsecond=us) + + def get_container(cid): + if cid == 'abcde': + labels = {LABEL_SERVICE: 'web'} + elif cid == 'ababa': + labels = {LABEL_SERVICE: 'db'} + else: + labels = {} + return {'Id': cid, 'Config': {'Labels': labels}} + + self.mock_client.inspect_container.side_effect = get_container + + events = project.events() + + events_list = list(events) + # Assert the return value is a generator + assert not list(events) + assert events_list == [ + { + 'service': 'web', + 'event': 'create', + 'container': 'abcde', + 'image': 'example/image', + 'time': dt_with_microseconds(1420092061, 2), + }, + { + 'service': 'web', + 'event': 'attach', + 'container': 'abcde', + 'image': 'example/image', + 'time': dt_with_microseconds(1420092061, 3), + }, + { + 'service': 'db', + 'event': 'create', + 'container': 'ababa', + 'image': 'example/db', + 'time': dt_with_microseconds(1420092061, 4), + }, + ] + def test_net_unset(self): project = Project.from_config('test', Config(None, [ { From 21aae13e77b43ce1cd3a07c660bb411f15993c27 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Jan 2016 13:21:45 -0800 Subject: [PATCH 198/359] Move logging config changes to v2 spec Reorganize JSON schemas Update fixtures Update service validation function Signed-off-by: Joffrey F --- compose/config/config.py | 2 +- compose/config/fields_schema_v1.json | 161 +-------------- compose/config/fields_schema_v2.json | 2 +- compose/config/service_schema.json | 30 --- compose/config/service_schema_v1.json | 175 +++++++++++++++++ compose/config/service_schema_v2.json | 184 ++++++++++++++++++ compose/config/validation.py | 4 +- docker-compose.spec | 9 +- .../fixtures/logging-composefile/compose2.yml | 8 +- .../logging-composefile/docker-compose.yml | 26 +-- 10 files changed, 391 insertions(+), 210 deletions(-) delete mode 100644 compose/config/service_schema.json create mode 100644 compose/config/service_schema_v1.json create mode 100644 compose/config/service_schema_v2.json diff --git a/compose/config/config.py b/compose/config/config.py index 0012aefd..0e794259 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -287,7 +287,7 @@ def load_services(working_dir, config_files, version): service_dict = process_service(resolver.run()) # TODO: move to validate_service() - validate_against_service_schema(service_dict, service_config.name) + validate_against_service_schema(service_dict, service_config.name, version) validate_paths(service_dict) service_dict = finalize_service(service_config._replace(config=service_dict)) diff --git a/compose/config/fields_schema_v1.json b/compose/config/fields_schema_v1.json index 790ace34..8f6a8c0a 100644 --- a/compose/config/fields_schema_v1.json +++ b/compose/config/fields_schema_v1.json @@ -6,165 +6,8 @@ "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "#/definitions/service" + "$ref": "service_schema_v1.json#/definitions/service" } }, - "additionalProperties": false, - - "definitions": { - "service": { - "id": "#/definitions/service", - "type": "object", - - "properties": { - "build": {"type": "string"}, - "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "cgroup_parent": {"type": "string"}, - "command": { - "oneOf": [ - {"type": "string"}, - {"type": "array", "items": {"type": "string"}} - ] - }, - "container_name": {"type": "string"}, - "cpu_shares": {"type": ["number", "string"]}, - "cpu_quota": {"type": ["number", "string"]}, - "cpuset": {"type": "string"}, - "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "dns": {"$ref": "#/definitions/string_or_list"}, - "dns_search": {"$ref": "#/definitions/string_or_list"}, - "dockerfile": {"type": "string"}, - "domainname": {"type": "string"}, - "entrypoint": {"$ref": "#/definitions/string_or_list"}, - "env_file": {"$ref": "#/definitions/string_or_list"}, - "environment": {"$ref": "#/definitions/list_or_dict"}, - - "expose": { - "type": "array", - "items": { - "type": ["string", "number"], - "format": "expose" - }, - "uniqueItems": true - }, - - "extends": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "object", - - "properties": { - "service": {"type": "string"}, - "file": {"type": "string"} - }, - "required": ["service"], - "additionalProperties": false - } - ] - }, - - "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, - "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "hostname": {"type": "string"}, - "image": {"type": "string"}, - "ipc": {"type": "string"}, - "labels": {"$ref": "#/definitions/list_or_dict"}, - "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - - "logging": { - "type": "object", - - "properties": { - "driver": {"type": "string"}, - "options": {"type": "object"} - }, - "additionalProperties": false - }, - - "mac_address": {"type": "string"}, - "mem_limit": {"type": ["number", "string"]}, - "memswap_limit": {"type": ["number", "string"]}, - "net": {"type": "string"}, - "pid": {"type": ["string", "null"]}, - - "ports": { - "type": "array", - "items": { - "type": ["string", "number"], - "format": "ports" - }, - "uniqueItems": true - }, - - "privileged": {"type": "boolean"}, - "read_only": {"type": "boolean"}, - "restart": {"type": "string"}, - "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "stdin_open": {"type": "boolean"}, - "tty": {"type": "boolean"}, - "ulimits": { - "type": "object", - "patternProperties": { - "^[a-z]+$": { - "oneOf": [ - {"type": "integer"}, - { - "type":"object", - "properties": { - "hard": {"type": "integer"}, - "soft": {"type": "integer"} - }, - "required": ["soft", "hard"], - "additionalProperties": false - } - ] - } - } - }, - "user": {"type": "string"}, - "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "volume_driver": {"type": "string"}, - "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, - "working_dir": {"type": "string"} - }, - - "dependencies": { - "memswap_limit": ["mem_limit"] - }, - "additionalProperties": false - }, - - "string_or_list": { - "oneOf": [ - {"type": "string"}, - {"$ref": "#/definitions/list_of_strings"} - ] - }, - - "list_of_strings": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": true - }, - - "list_or_dict": { - "oneOf": [ - { - "type": "object", - "patternProperties": { - ".+": { - "type": ["string", "number", "boolean", "null"], - "format": "bool-value-in-mapping" - } - }, - "additionalProperties": false - }, - {"type": "array", "items": {"type": "string"}, "uniqueItems": true} - ] - } - } + "additionalProperties": false } diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 49cab367..22ff839f 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -12,7 +12,7 @@ "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "fields_schema_v1.json#/definitions/service" + "$ref": "service_schema_v2.json#/definitions/service" } }, "additionalProperties": false diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json deleted file mode 100644 index 91a1e005..00000000 --- a/compose/config/service_schema.json +++ /dev/null @@ -1,30 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "id": "service_schema.json", - - "type": "object", - - "allOf": [ - {"$ref": "fields_schema_v1.json#/definitions/service"}, - {"$ref": "#/definitions/constraints"} - ], - - "definitions": { - "constraints": { - "id": "#/definitions/constraints", - "anyOf": [ - { - "required": ["build"], - "not": {"required": ["image"]} - }, - { - "required": ["image"], - "not": {"anyOf": [ - {"required": ["build"]}, - {"required": ["dockerfile"]} - ]} - } - ] - } - } -} diff --git a/compose/config/service_schema_v1.json b/compose/config/service_schema_v1.json new file mode 100644 index 00000000..d51c7f73 --- /dev/null +++ b/compose/config/service_schema_v1.json @@ -0,0 +1,175 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "service_schema_v1.json", + + "type": "object", + + "allOf": [ + {"$ref": "#/definitions/service"}, + {"$ref": "#/definitions/constraints"} + ], + + "definitions": { + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "build": {"type": "string"}, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, + "cpuset": {"type": "string"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "dockerfile": {"type": "string"}, + "domainname": {"type": "string"}, + "entrypoint": {"$ref": "#/definitions/string_or_list"}, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "extends": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] + }, + + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "log_driver": {"type": "string"}, + "log_opt": {"type": "object"}, + "mac_address": {"type": "string"}, + "mem_limit": {"type": ["number", "string"]}, + "memswap_limit": {"type": ["number", "string"]}, + "net": {"type": "string"}, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "stdin_open": {"type": "boolean"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volume_driver": {"type": "string"}, + "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + + "dependencies": { + "memswap_limit": ["mem_limit"] + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "boolean", "null"], + "format": "bool-value-in-mapping" + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + "constraints": { + "id": "#/definitions/constraints", + "anyOf": [ + { + "required": ["build"], + "not": {"required": ["image"]} + }, + { + "required": ["image"], + "not": {"anyOf": [ + {"required": ["build"]}, + {"required": ["dockerfile"]} + ]} + } + ] + } + } +} diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json new file mode 100644 index 00000000..a64b3bdc --- /dev/null +++ b/compose/config/service_schema_v2.json @@ -0,0 +1,184 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "service_schema_v2.json", + + "type": "object", + + "allOf": [ + {"$ref": "#/definitions/service"}, + {"$ref": "#/definitions/constraints"} + ], + + "definitions": { + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "build": {"type": "string"}, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, + "cpuset": {"type": "string"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "dockerfile": {"type": "string"}, + "domainname": {"type": "string"}, + "entrypoint": {"$ref": "#/definitions/string_or_list"}, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "extends": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] + }, + + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": {"type": "object"} + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "mem_limit": {"type": ["number", "string"]}, + "memswap_limit": {"type": ["number", "string"]}, + "net": {"type": "string"}, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "stdin_open": {"type": "boolean"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volume_driver": {"type": "string"}, + "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + + "dependencies": { + "memswap_limit": ["mem_limit"] + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "boolean", "null"], + "format": "bool-value-in-mapping" + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + "constraints": { + "id": "#/definitions/constraints", + "anyOf": [ + { + "required": ["build"], + "not": {"required": ["image"]} + }, + { + "required": ["image"], + "not": {"anyOf": [ + {"required": ["build"]}, + {"required": ["dockerfile"]} + ]} + } + ] + } + } +} diff --git a/compose/config/validation.py b/compose/config/validation.py index 74dd461f..fea9a22b 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -298,10 +298,10 @@ def validate_against_fields_schema(config, filename, version): filename=filename) -def validate_against_service_schema(config, service_name): +def validate_against_service_schema(config, service_name, version): _validate_against_schema( config, - "service_schema.json", + "service_schema_v{0}.json".format(version), format_checker=["ports"], service_name=service_name) diff --git a/docker-compose.spec b/docker-compose.spec index c760d7b4..f7f2059f 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -28,8 +28,13 @@ exe = EXE(pyz, 'DATA' ), ( - 'compose/config/service_schema.json', - 'compose/config/service_schema.json', + 'compose/config/service_schema_v1.json', + 'compose/config/service_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/service_schema_v2.json', + 'compose/config/service_schema_v2.json', 'DATA' ), ( diff --git a/tests/fixtures/logging-composefile/compose2.yml b/tests/fixtures/logging-composefile/compose2.yml index ba582969..69889761 100644 --- a/tests/fixtures/logging-composefile/compose2.yml +++ b/tests/fixtures/logging-composefile/compose2.yml @@ -1,3 +1,5 @@ -another: - logging: - driver: "none" +version: 2 +services: + another: + logging: + driver: "none" diff --git a/tests/fixtures/logging-composefile/docker-compose.yml b/tests/fixtures/logging-composefile/docker-compose.yml index 877ee5e2..0a73030a 100644 --- a/tests/fixtures/logging-composefile/docker-compose.yml +++ b/tests/fixtures/logging-composefile/docker-compose.yml @@ -1,12 +1,14 @@ -simple: - image: busybox:latest - command: top - logging: - driver: "none" -another: - image: busybox:latest - command: top - logging: - driver: "json-file" - options: - max-size: "10m" +version: 2 +services: + simple: + image: busybox:latest + command: top + logging: + driver: "none" + another: + image: busybox:latest + command: top + logging: + driver: "json-file" + options: + max-size: "10m" From 46585fb8e17a4b6cb5763f055ef00d0aa3d952c9 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 8 Jan 2016 14:37:07 -0800 Subject: [PATCH 199/359] Support legacy logging options format Additional test for legacy compose file. Signed-off-by: Joffrey F --- compose/config/config.py | 10 ++++++++++ tests/acceptance/cli_test.py | 14 ++++++++++++++ .../logging-composefile-legacy/docker-compose.yml | 10 ++++++++++ 3 files changed, 34 insertions(+) create mode 100644 tests/fixtures/logging-composefile-legacy/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 0e794259..c59f384d 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -504,6 +504,16 @@ def finalize_service(service_config): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + if 'log_driver' in service_dict or 'log_opt' in service_dict: + if 'logging' not in service_dict: + service_dict['logging'] = {} + if 'log_driver' in service_dict: + service_dict['logging']['driver'] = service_dict['log_driver'] + del service_dict['log_driver'] + if 'log_opt' in service_dict: + service_dict['logging']['options'] = service_dict['log_opt'] + del service_dict['log_opt'] + return service_dict diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index c5df3079..8abdf785 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -730,6 +730,20 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(log_config.get('Type'), 'json-file') self.assertEqual(log_config.get('Config')['max-size'], '10m') + def test_up_logging_legacy(self): + self.base_dir = 'tests/fixtures/logging-composefile-legacy' + self.dispatch(['up', '-d']) + simple = self.project.get_service('simple').containers()[0] + log_config = simple.get('HostConfig.LogConfig') + self.assertTrue(log_config) + self.assertEqual(log_config.get('Type'), 'none') + + another = self.project.get_service('another').containers()[0] + log_config = another.get('HostConfig.LogConfig') + self.assertTrue(log_config) + self.assertEqual(log_config.get('Type'), 'json-file') + self.assertEqual(log_config.get('Config')['max-size'], '10m') + def test_up_logging_with_multiple_files(self): self.base_dir = 'tests/fixtures/logging-composefile' config_paths = [ diff --git a/tests/fixtures/logging-composefile-legacy/docker-compose.yml b/tests/fixtures/logging-composefile-legacy/docker-compose.yml new file mode 100644 index 00000000..ee994107 --- /dev/null +++ b/tests/fixtures/logging-composefile-legacy/docker-compose.yml @@ -0,0 +1,10 @@ +simple: + image: busybox:latest + command: top + log_driver: "none" +another: + image: busybox:latest + command: top + log_driver: "json-file" + log_opt: + max-size: "10m" From d3cd038b845d9708c42b9281253be342e2fe97d0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 8 Jan 2016 16:54:35 -0500 Subject: [PATCH 200/359] Update event field names to match the new API fields. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 7 ++++-- compose/project.py | 12 ++++++---- tests/acceptance/cli_test.py | 22 +++++++++++++++++- tests/unit/project_test.py | 43 ++++++++++++++++++++++++++---------- 4 files changed, 65 insertions(+), 19 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index d99816cf..d10b9582 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -256,8 +256,10 @@ class TopLevelCommand(DocoptCommand): --json Output events as a stream of json objects """ def format_event(event): - return ("{time}: service={service} event={event} " - "container={container} image={image}").format(**event) + attributes = ["%s=%s" % item for item in event['attributes'].items()] + return ("{time} {type} {action} {id} ({attrs})").format( + attrs=", ".join(sorted(attributes)), + **event) def json_format_event(event): event['time'] = event['time'].isoformat() @@ -266,6 +268,7 @@ class TopLevelCommand(DocoptCommand): for event in project.events(): formatter = json_format_event if options['--json'] else format_event print(formatter(event)) + sys.stdout.flush() def help(self, project, options): """ diff --git a/compose/project.py b/compose/project.py index b4eed7c8..50f991be 100644 --- a/compose/project.py +++ b/compose/project.py @@ -282,11 +282,15 @@ class Project(object): time = time.replace( microsecond=microseconds_from_time_nano(event['timeNano'])) return { - 'service': container.service, - 'event': event['status'], - 'container': container.id, - 'image': event['from'], 'time': time, + 'type': 'container', + 'action': event['status'], + 'id': container.id, + 'service': container.service, + 'attributes': { + 'name': container.name, + 'image': event['from'], + } } service_names = set(self.service_names) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 322d9b5a..db3e1b43 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1,6 +1,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +import datetime import json import os import shlex @@ -864,7 +865,26 @@ class CLITestCase(DockerClientTestCase): os.kill(events_proc.pid, signal.SIGINT) result = wait_on_process(events_proc, returncode=1) lines = [json.loads(line) for line in result.stdout.rstrip().split('\n')] - assert [e['event'] for e in lines] == ['create', 'start', 'create', 'start'] + assert [e['action'] for e in lines] == ['create', 'start', 'create', 'start'] + + def test_events_human_readable(self): + events_proc = start_process(self.base_dir, ['events']) + self.dispatch(['up', '-d', 'simple']) + wait_on_condition(ContainerCountCondition(self.project, 1)) + + os.kill(events_proc.pid, signal.SIGINT) + result = wait_on_process(events_proc, returncode=1) + lines = result.stdout.rstrip().split('\n') + assert len(lines) == 2 + + container, = self.project.containers() + expected_template = ( + ' container {} {} (image=busybox:latest, ' + 'name=simplecomposefile_simple_1)') + + assert expected_template.format('create', container.id) in lines[0] + assert expected_template.format('start', container.id) in lines[1] + assert lines[0].startswith(datetime.date.today().isoformat()) def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index a4b61b64..c8590a1f 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -238,12 +238,19 @@ class ProjectTest(unittest.TestCase): def get_container(cid): if cid == 'abcde': - labels = {LABEL_SERVICE: 'web'} + name = 'web' + labels = {LABEL_SERVICE: name} elif cid == 'ababa': - labels = {LABEL_SERVICE: 'db'} + name = 'db' + labels = {LABEL_SERVICE: name} else: labels = {} - return {'Id': cid, 'Config': {'Labels': labels}} + name = '' + return { + 'Id': cid, + 'Config': {'Labels': labels}, + 'Name': '/project_%s_1' % name, + } self.mock_client.inspect_container.side_effect = get_container @@ -254,24 +261,36 @@ class ProjectTest(unittest.TestCase): assert not list(events) assert events_list == [ { + 'type': 'container', 'service': 'web', - 'event': 'create', - 'container': 'abcde', - 'image': 'example/image', + 'action': 'create', + 'id': 'abcde', + 'attributes': { + 'name': 'project_web_1', + 'image': 'example/image', + }, 'time': dt_with_microseconds(1420092061, 2), }, { + 'type': 'container', 'service': 'web', - 'event': 'attach', - 'container': 'abcde', - 'image': 'example/image', + 'action': 'attach', + 'id': 'abcde', + 'attributes': { + 'name': 'project_web_1', + 'image': 'example/image', + }, 'time': dt_with_microseconds(1420092061, 3), }, { + 'type': 'container', 'service': 'db', - 'event': 'create', - 'container': 'ababa', - 'image': 'example/db', + 'action': 'create', + 'id': 'ababa', + 'attributes': { + 'name': 'project_db_1', + 'image': 'example/db', + }, 'time': dt_with_microseconds(1420092061, 4), }, ] From f7a7e68df63dad82d7bc382cc8832510856c5296 Mon Sep 17 00:00:00 2001 From: Thomas Hourlier Date: Mon, 11 Jan 2016 09:48:52 -0800 Subject: [PATCH 201/359] Mount $HOME in /root to share docker config. Fixes #2630 Signed-off-by: Thomas Hourlier --- script/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/run.sh b/script/run.sh index 514a5389..087c2692 100755 --- a/script/run.sh +++ b/script/run.sh @@ -40,7 +40,7 @@ if [ -n "$compose_dir" ]; then VOLUMES="$VOLUMES -v $compose_dir:$compose_dir" fi if [ -n "$HOME" ]; then - VOLUMES="$VOLUMES -v $HOME:$HOME" + VOLUMES="$VOLUMES -v $HOME:$HOME -v $HOME:/root" # mount $HOME in /root to share docker.config fi # Only allocate tty if we detect one From c32991a8d491f8af1e4f3502bdb7bc8131922143 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 11 Jan 2016 15:39:59 -0800 Subject: [PATCH 202/359] Remove superfluous service code Signed-off-by: Joffrey F --- compose/service.py | 9 --------- 1 file changed, 9 deletions(-) diff --git a/compose/service.py b/compose/service.py index 366833dd..9fd679f8 100644 --- a/compose/service.py +++ b/compose/service.py @@ -510,13 +510,6 @@ class Service(object): return volumes_from - def get_logging_options(self): - logging_dict = self.options.get('logging', {}) - return { - 'log_driver': logging_dict.get('driver', ""), - 'log_opt': logging_dict.get('options', None) - } - def _get_container_create_options( self, override_options, @@ -530,8 +523,6 @@ class Service(object): for k in DOCKER_CONFIG_KEYS if k in self.options) container_options.update(override_options) - container_options.update(self.get_logging_options()) - if self.custom_container_name() and not one_off: container_options['name'] = self.custom_container_name() elif not container_options.get('name'): From 46a474ecd9132fcf84eb5568df4245bd13e3ca26 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 11 Jan 2016 15:53:28 -0800 Subject: [PATCH 203/359] Move v1-v2 config normalization to separate function. Signed-off-by: Joffrey F --- compose/config/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index c59f384d..2ca5fa22 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -504,6 +504,10 @@ def finalize_service(service_config): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + return normalize_v1_service_format(service_dict) + + +def normalize_v1_service_format(service_dict): if 'log_driver' in service_dict or 'log_opt' in service_dict: if 'logging' not in service_dict: service_dict['logging'] = {} From ca634649bbdb92cd7c52a98ee693d0c55a1e153b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 11 Jan 2016 16:25:19 -0800 Subject: [PATCH 204/359] Changed logging override test into integration test Signed-off-by: Joffrey F --- tests/acceptance/cli_test.py | 23 -------- .../fixtures/logging-composefile/compose2.yml | 5 -- tests/integration/project_test.py | 53 +++++++++++++++++++ 3 files changed, 53 insertions(+), 28 deletions(-) delete mode 100644 tests/fixtures/logging-composefile/compose2.yml diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8abdf785..caadb62f 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -744,29 +744,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(log_config.get('Type'), 'json-file') self.assertEqual(log_config.get('Config')['max-size'], '10m') - def test_up_logging_with_multiple_files(self): - self.base_dir = 'tests/fixtures/logging-composefile' - config_paths = [ - 'docker-compose.yml', - 'compose2.yml', - ] - self._project = get_project(self.base_dir, config_paths) - self.dispatch( - [ - '-f', config_paths[0], - '-f', config_paths[1], - 'up', '-d', - ], - None) - - containers = self.project.containers() - self.assertEqual(len(containers), 2) - - another = self.project.get_service('another').containers()[0] - log_config = another.get('HostConfig.LogConfig') - self.assertTrue(log_config) - self.assertEqual(log_config.get('Type'), 'none') - def test_pause_unpause(self): self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') diff --git a/tests/fixtures/logging-composefile/compose2.yml b/tests/fixtures/logging-composefile/compose2.yml deleted file mode 100644 index 69889761..00000000 --- a/tests/fixtures/logging-composefile/compose2.yml +++ /dev/null @@ -1,5 +0,0 @@ -version: 2 -services: - another: - logging: - driver: "none" diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 2cf5f556..5a1444b6 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -3,6 +3,8 @@ from __future__ import unicode_literals import random +import py + from .testcases import DockerClientTestCase from compose.cli.docker_client import docker_client from compose.config import config @@ -534,6 +536,57 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + def test_project_up_logging_with_multiple_files(self): + base_file = config.ConfigFile( + 'base.yml', + { + 'version': 2, + 'services': { + 'simple': {'image': 'busybox:latest', 'command': 'top'}, + 'another': { + 'image': 'busybox:latest', + 'command': 'top', + 'logging': { + 'driver': "json-file", + 'options': { + 'max-size': "10m" + } + } + } + } + + }) + override_file = config.ConfigFile( + 'override.yml', + { + 'version': 2, + 'services': { + 'another': { + 'logging': { + 'driver': "none" + } + } + } + + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + tmpdir = py.test.ensuretemp('logging_test') + self.addCleanup(tmpdir.remove) + with tmpdir.as_cwd(): + config_data = config.load(details) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + containers = project.containers() + self.assertEqual(len(containers), 2) + + another = project.get_service('another').containers()[0] + log_config = another.get('HostConfig.LogConfig') + self.assertTrue(log_config) + self.assertEqual(log_config.get('Type'), 'none') + def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) From 12b5405420ebfd2a3b260f3a4421bfadd52e8a97 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 12 Jan 2016 13:27:18 -0500 Subject: [PATCH 205/359] Fix pre-commit on master. Signed-off-by: Daniel Nephin --- compose/cli/signals.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/compose/cli/signals.py b/compose/cli/signals.py index 38474ba4..68a0598e 100644 --- a/compose/cli/signals.py +++ b/compose/cli/signals.py @@ -1,3 +1,6 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + import signal From 6c205a8e017569f3a18ae697edf6310f221860f6 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Tue, 12 Jan 2016 10:51:04 -0800 Subject: [PATCH 206/359] Add bash completion for `docker-compose events` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index c18e4f6d..2b1d689c 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -133,6 +133,24 @@ _docker_compose_docker_compose() { } +_docker_compose_events() { + case "$prev" in + --json) + return + ;; + esac + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help --json" -- "$cur" ) ) + ;; + *) + __docker_compose_services_all + ;; + esac +} + + _docker_compose_help() { COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) } @@ -379,6 +397,7 @@ _docker_compose() { local commands=( build config + events help kill logs From ed4db542d6f6d4ec062bb29d8c99a6ea5c9523d7 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 12 Jan 2016 14:02:30 -0500 Subject: [PATCH 207/359] Fix pep8 errors from the new pep8 release. Signed-off-by: Daniel Nephin --- compose/project.py | 4 ++-- compose/service.py | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compose/project.py b/compose/project.py index 50f991be..d41b1f52 100644 --- a/compose/project.py +++ b/compose/project.py @@ -344,8 +344,8 @@ class Project(object): updated_dependencies = [ name for name in service.get_dependency_names() - if name in plans - and plans[name].action in ('recreate', 'create') + if name in plans and + plans[name].action in ('recreate', 'create') ] if updated_dependencies and strategy.allows_recreate: diff --git a/compose/service.py b/compose/service.py index 693e1980..bd8143e7 100644 --- a/compose/service.py +++ b/compose/service.py @@ -535,9 +535,9 @@ class Service(object): # unqualified hostname and a domainname unless domainname # was also given explicitly. This matches the behavior of # the official Docker CLI in that scenario. - if ('hostname' in container_options - and 'domainname' not in container_options - and '.' in container_options['hostname']): + if ('hostname' in container_options and + 'domainname' not in container_options and + '.' in container_options['hostname']): parts = container_options['hostname'].partition('.') container_options['hostname'] = parts[0] container_options['domainname'] = parts[2] From e98ab0e534a0a0b9ec86f18f0867b05188fc8b02 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 12 Jan 2016 11:24:06 -0500 Subject: [PATCH 208/359] Allow both image and build together. Signed-off-by: Daniel Nephin --- compose/config/config.py | 25 +++-- compose/config/service_schema_v2.json | 13 +-- compose/config/validation.py | 4 +- compose/service.py | 12 +-- tests/integration/service_test.py | 16 ++- tests/unit/config/config_test.py | 144 ++++++++++++++------------ tests/unit/service_test.py | 9 ++ 7 files changed, 121 insertions(+), 102 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index db2f67ea..918946b3 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -305,7 +305,8 @@ def load_services(working_dir, config_files, version): return { name: merge_service_dicts_from_files( base.get(name, {}), - override.get(name, {})) + override.get(name, {}), + version) for name in all_service_names } @@ -397,7 +398,10 @@ class ServiceExtendsResolver(object): service_name, ) - return merge_service_dicts(other_service_dict, self.service_config.config) + return merge_service_dicts( + other_service_dict, + self.service_config.config, + self.version) def get_extended_config_path(self, extends_options): """Service we are extending either has a value for 'file' set, which we @@ -521,12 +525,12 @@ def normalize_v1_service_format(service_dict): return service_dict -def merge_service_dicts_from_files(base, override): +def merge_service_dicts_from_files(base, override, version): """When merging services from multiple files we need to merge the `extends` field. This is not handled by `merge_service_dicts()` which is used to perform the `extends`. """ - new_service = merge_service_dicts(base, override) + new_service = merge_service_dicts(base, override, version) if 'extends' in override: new_service['extends'] = override['extends'] elif 'extends' in base: @@ -534,7 +538,7 @@ def merge_service_dicts_from_files(base, override): return new_service -def merge_service_dicts(base, override): +def merge_service_dicts(base, override, version): d = {} def merge_field(field, merge_func, default=None): @@ -545,7 +549,6 @@ def merge_service_dicts(base, override): merge_field('environment', merge_environment) merge_field('labels', merge_labels) - merge_image_or_build(base, override, d) for field in ['volumes', 'devices']: merge_field(field, merge_path_mappings) @@ -556,15 +559,19 @@ def merge_service_dicts(base, override): for field in ['dns', 'dns_search', 'env_file']: merge_field(field, merge_list_or_string) - already_merged_keys = set(d) | {'image', 'build'} - for field in set(ALLOWED_KEYS) - already_merged_keys: + for field in set(ALLOWED_KEYS) - set(d): if field in base or field in override: d[field] = override.get(field, base.get(field)) + if version == 1: + legacy_v1_merge_image_or_build(d, base, override) + return d -def merge_image_or_build(base, override, output): +def legacy_v1_merge_image_or_build(output, base, override): + output.pop('image', None) + output.pop('build', None) if 'image' in override: output['image'] = override['image'] elif 'build' in override: diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index a64b3bdc..d1d0854f 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -167,17 +167,8 @@ "constraints": { "id": "#/definitions/constraints", "anyOf": [ - { - "required": ["build"], - "not": {"required": ["image"]} - }, - { - "required": ["image"], - "not": {"anyOf": [ - {"required": ["build"]}, - {"required": ["dockerfile"]} - ]} - } + {"required": ["build"]}, + {"required": ["image"]} ] } } diff --git a/compose/config/validation.py b/compose/config/validation.py index fea9a22b..e7006d5a 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -151,6 +151,7 @@ def handle_error_for_schema_with_id(error, service_name): VALID_NAME_CHARS) if schema_id == '#/definitions/constraints': + # TODO: only applies to v1 if 'image' in error.instance and 'build' in error.instance: return ( "Service '{}' has both an image and build path specified. " @@ -159,7 +160,8 @@ def handle_error_for_schema_with_id(error, service_name): if 'image' not in error.instance and 'build' not in error.instance: return ( "Service '{}' has neither an image nor a build path " - "specified. Exactly one must be provided.".format(service_name)) + "specified. At least one must be provided.".format(service_name)) + # TODO: only applies to v1 if 'image' in error.instance and 'dockerfile' in error.instance: return ( "Service '{}' has both an image and alternate Dockerfile. " diff --git a/compose/service.py b/compose/service.py index bd8143e7..d5c36f1a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -275,10 +275,7 @@ class Service(object): @property def image_name(self): - if self.can_be_built(): - return self.full_name - else: - return self.options['image'] + return self.options.get('image', '{s.project}_{s.name}'.format(s=self)) def convergence_plan(self, strategy=ConvergenceStrategy.changed): containers = self.containers(stopped=True) @@ -665,13 +662,6 @@ class Service(object): def can_be_built(self): return 'build' in self.options - @property - def full_name(self): - """ - The tag to give to images built for this service. - """ - return '%s_%s' % (self.project, self.name) - def labels(self, one_off=False): return [ '{0}={1}'.format(LABEL_PROJECT, self.project), diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 86bc4d9d..539be5a9 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -491,7 +491,7 @@ class ServiceTest(DockerClientTestCase): f.write("FROM busybox\n") self.create_service('web', build=base_dir).build() - self.assertEqual(len(self.client.images(name='composetest_web')), 1) + assert self.client.inspect_image('composetest_web') def test_build_non_ascii_filename(self): base_dir = tempfile.mkdtemp() @@ -504,7 +504,19 @@ class ServiceTest(DockerClientTestCase): f.write("hello world\n") self.create_service('web', build=text_type(base_dir)).build() - self.assertEqual(len(self.client.images(name='composetest_web')), 1) + assert self.client.inspect_image('composetest_web') + + def test_build_with_image_name(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + + image_name = 'examples/composetest:latest' + self.addCleanup(self.client.remove_image, image_name) + self.create_service('web', build=base_dir, image=image_name).build() + assert self.client.inspect_image(image_name) def test_build_with_git_url(self): build_url = "https://github.com/dnephin/docker-build-from-url.git" diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 1c2876f5..a59d1d34 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -19,6 +19,9 @@ from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest +DEFAULT_VERSION = 2 +V1 = 1 + def make_service_dict(name, service_dict, working_dir, filename=None): """ @@ -238,8 +241,7 @@ class ConfigTest(unittest.TestCase): ) ) - @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') - def test_load_with_multiple_files(self): + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', { @@ -265,7 +267,7 @@ class ConfigTest(unittest.TestCase): expected = [ { 'name': 'web', - 'build': '/', + 'build': os.path.abspath('/'), 'links': ['db'], 'volumes': [VolumeSpec.parse('/home/user/project:/code')], }, @@ -274,7 +276,7 @@ class ConfigTest(unittest.TestCase): 'image': 'example/db', }, ] - self.assertEqual(service_sort(service_dicts), service_sort(expected)) + assert service_sort(service_dicts) == service_sort(expected) def test_load_with_multiple_files_and_empty_override(self): base_file = config.ConfigFile( @@ -428,6 +430,7 @@ class ConfigTest(unittest.TestCase): { 'name': 'web', 'build': os.path.abspath('/'), + 'image': 'example/web', 'links': ['db'], 'volumes': [VolumeSpec.parse('/home/user/project:/code')], }, @@ -436,7 +439,7 @@ class ConfigTest(unittest.TestCase): 'image': 'example/db', }, ] - self.assertEqual(service_sort(service_dicts), service_sort(expected)) + assert service_sort(service_dicts) == service_sort(expected) def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: @@ -525,16 +528,15 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_image_and_dockerfile_raise_validation_error(self): - expected_error_msg = "Service 'web' has both an image and alternate Dockerfile." - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): - config.load( - build_config_details( - {'web': {'image': 'busybox', 'dockerfile': 'Dockerfile.alt'}}, - 'working_dir', - 'filename.yml' - ) - ) + def test_load_config_dockerfile_without_build_raises_error(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'dockerfile': 'Dockerfile.alt' + } + })) + assert "Service 'web' has both an image and alternate Dockerfile." in exc.exconly() def test_config_extra_hosts_string_raises_validation_error(self): expected_error_msg = "Service 'web' configuration key 'extra_hosts' contains an invalid type" @@ -746,7 +748,10 @@ class ConfigTest(unittest.TestCase): override = { 'image': 'alpine:edge', } - actual = config.merge_service_dicts_from_files(base, override) + actual = config.merge_service_dicts_from_files( + base, + override, + DEFAULT_VERSION) assert actual == { 'image': 'alpine:edge', 'volumes': ['.:/app'], @@ -762,7 +767,10 @@ class ConfigTest(unittest.TestCase): 'image': 'alpine:edge', 'extends': {'service': 'foo'} } - actual = config.merge_service_dicts_from_files(base, override) + actual = config.merge_service_dicts_from_files( + base, + override, + DEFAULT_VERSION) assert actual == { 'image': 'alpine:edge', 'volumes': ['.:/app'], @@ -1023,43 +1031,43 @@ class MergePathMappingTest(object): return "" def test_empty(self): - service_dict = config.merge_service_dicts({}, {}) - self.assertNotIn(self.config_name(), service_dict) + service_dict = config.merge_service_dicts({}, {}, DEFAULT_VERSION) + assert self.config_name() not in service_dict def test_no_override(self): service_dict = config.merge_service_dicts( {self.config_name(): ['/foo:/code', '/data']}, {}, - ) - self.assertEqual(set(service_dict[self.config_name()]), set(['/foo:/code', '/data'])) + DEFAULT_VERSION) + assert set(service_dict[self.config_name()]) == set(['/foo:/code', '/data']) def test_no_base(self): service_dict = config.merge_service_dicts( {}, {self.config_name(): ['/bar:/code']}, - ) - self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code'])) + DEFAULT_VERSION) + assert set(service_dict[self.config_name()]) == set(['/bar:/code']) def test_override_explicit_path(self): service_dict = config.merge_service_dicts( {self.config_name(): ['/foo:/code', '/data']}, {self.config_name(): ['/bar:/code']}, - ) - self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code', '/data'])) + DEFAULT_VERSION) + assert set(service_dict[self.config_name()]) == set(['/bar:/code', '/data']) def test_add_explicit_path(self): service_dict = config.merge_service_dicts( {self.config_name(): ['/foo:/code', '/data']}, {self.config_name(): ['/bar:/code', '/quux:/data']}, - ) - self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code', '/quux:/data'])) + DEFAULT_VERSION) + assert set(service_dict[self.config_name()]) == set(['/bar:/code', '/quux:/data']) def test_remove_explicit_path(self): service_dict = config.merge_service_dicts( {self.config_name(): ['/foo:/code', '/quux:/data']}, {self.config_name(): ['/bar:/code', '/data']}, - ) - self.assertEqual(set(service_dict[self.config_name()]), set(['/bar:/code', '/data'])) + DEFAULT_VERSION) + assert set(service_dict[self.config_name()]) == set(['/bar:/code', '/data']) class MergeVolumesTest(unittest.TestCase, MergePathMappingTest): @@ -1075,63 +1083,62 @@ class MergeDevicesTest(unittest.TestCase, MergePathMappingTest): class BuildOrImageMergeTest(unittest.TestCase): def test_merge_build_or_image_no_override(self): self.assertEqual( - config.merge_service_dicts({'build': '.'}, {}), + config.merge_service_dicts({'build': '.'}, {}, V1), {'build': '.'}, ) self.assertEqual( - config.merge_service_dicts({'image': 'redis'}, {}), + config.merge_service_dicts({'image': 'redis'}, {}, V1), {'image': 'redis'}, ) def test_merge_build_or_image_override_with_same(self): self.assertEqual( - config.merge_service_dicts({'build': '.'}, {'build': './web'}), + config.merge_service_dicts({'build': '.'}, {'build': './web'}, V1), {'build': './web'}, ) self.assertEqual( - config.merge_service_dicts({'image': 'redis'}, {'image': 'postgres'}), + config.merge_service_dicts({'image': 'redis'}, {'image': 'postgres'}, V1), {'image': 'postgres'}, ) def test_merge_build_or_image_override_with_other(self): self.assertEqual( - config.merge_service_dicts({'build': '.'}, {'image': 'redis'}), - {'image': 'redis'} + config.merge_service_dicts({'build': '.'}, {'image': 'redis'}, V1), + {'image': 'redis'}, ) self.assertEqual( - config.merge_service_dicts({'image': 'redis'}, {'build': '.'}), - {'build': '.'} + config.merge_service_dicts({'image': 'redis'}, {'build': '.'}, V1), + {'build': '.'}, ) class MergeListsTest(unittest.TestCase): def test_empty(self): - service_dict = config.merge_service_dicts({}, {}) - self.assertNotIn('ports', service_dict) + assert 'ports' not in config.merge_service_dicts({}, {}, DEFAULT_VERSION) def test_no_override(self): service_dict = config.merge_service_dicts( {'ports': ['10:8000', '9000']}, {}, - ) - self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000'])) + DEFAULT_VERSION) + assert set(service_dict['ports']) == set(['10:8000', '9000']) def test_no_base(self): service_dict = config.merge_service_dicts( {}, {'ports': ['10:8000', '9000']}, - ) - self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000'])) + DEFAULT_VERSION) + assert set(service_dict['ports']) == set(['10:8000', '9000']) def test_add_item(self): service_dict = config.merge_service_dicts( {'ports': ['10:8000', '9000']}, {'ports': ['20:8000']}, - ) - self.assertEqual(set(service_dict['ports']), set(['10:8000', '9000', '20:8000'])) + DEFAULT_VERSION) + assert set(service_dict['ports']) == set(['10:8000', '9000', '20:8000']) class MergeStringsOrListsTest(unittest.TestCase): @@ -1139,70 +1146,69 @@ class MergeStringsOrListsTest(unittest.TestCase): service_dict = config.merge_service_dicts( {'dns': '8.8.8.8'}, {}, - ) - self.assertEqual(set(service_dict['dns']), set(['8.8.8.8'])) + DEFAULT_VERSION) + assert set(service_dict['dns']) == set(['8.8.8.8']) def test_no_base(self): service_dict = config.merge_service_dicts( {}, {'dns': '8.8.8.8'}, - ) - self.assertEqual(set(service_dict['dns']), set(['8.8.8.8'])) + DEFAULT_VERSION) + assert set(service_dict['dns']) == set(['8.8.8.8']) def test_add_string(self): service_dict = config.merge_service_dicts( {'dns': ['8.8.8.8']}, {'dns': '9.9.9.9'}, - ) - self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) + DEFAULT_VERSION) + assert set(service_dict['dns']) == set(['8.8.8.8', '9.9.9.9']) def test_add_list(self): service_dict = config.merge_service_dicts( {'dns': '8.8.8.8'}, {'dns': ['9.9.9.9']}, - ) - self.assertEqual(set(service_dict['dns']), set(['8.8.8.8', '9.9.9.9'])) + DEFAULT_VERSION) + assert set(service_dict['dns']) == set(['8.8.8.8', '9.9.9.9']) class MergeLabelsTest(unittest.TestCase): def test_empty(self): - service_dict = config.merge_service_dicts({}, {}) - self.assertNotIn('labels', service_dict) + assert 'labels' not in config.merge_service_dicts({}, {}, DEFAULT_VERSION) def test_no_override(self): service_dict = config.merge_service_dicts( make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar']}, 'tests/'), make_service_dict('foo', {'build': '.'}, 'tests/'), - ) - self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': ''}) + DEFAULT_VERSION) + assert service_dict['labels'] == {'foo': '1', 'bar': ''} def test_no_base(self): service_dict = config.merge_service_dicts( make_service_dict('foo', {'build': '.'}, 'tests/'), make_service_dict('foo', {'build': '.', 'labels': ['foo=2']}, 'tests/'), - ) - self.assertEqual(service_dict['labels'], {'foo': '2'}) + DEFAULT_VERSION) + assert service_dict['labels'] == {'foo': '2'} def test_override_explicit_value(self): service_dict = config.merge_service_dicts( make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar']}, 'tests/'), make_service_dict('foo', {'build': '.', 'labels': ['foo=2']}, 'tests/'), - ) - self.assertEqual(service_dict['labels'], {'foo': '2', 'bar': ''}) + DEFAULT_VERSION) + assert service_dict['labels'] == {'foo': '2', 'bar': ''} def test_add_explicit_value(self): service_dict = config.merge_service_dicts( make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar']}, 'tests/'), make_service_dict('foo', {'build': '.', 'labels': ['bar=2']}, 'tests/'), - ) - self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': '2'}) + DEFAULT_VERSION) + assert service_dict['labels'] == {'foo': '1', 'bar': '2'} def test_remove_explicit_value(self): service_dict = config.merge_service_dicts( make_service_dict('foo', {'build': '.', 'labels': ['foo=1', 'bar=2']}, 'tests/'), make_service_dict('foo', {'build': '.', 'labels': ['bar']}, 'tests/'), - ) - self.assertEqual(service_dict['labels'], {'foo': '1', 'bar': ''}) + DEFAULT_VERSION) + assert service_dict['labels'] == {'foo': '1', 'bar': ''} class MemoryOptionsTest(unittest.TestCase): @@ -1541,10 +1547,12 @@ class ExtendsTest(unittest.TestCase): self.assertEquals(service[0]['command'], "/bin/true") def test_extended_service_with_invalid_config(self): - expected_error_msg = "Service 'myweb' has neither an image nor a build path specified" - - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as exc: load_from_filename('tests/fixtures/extends/service-with-invalid-schema.yml') + assert ( + "Service 'myweb' has neither an image nor a build path specified" in + exc.exconly() + ) def test_extended_service_with_valid_config(self): service = load_from_filename('tests/fixtures/extends/service-with-valid-composite-extends.yml') diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index d7080fae..63cf658e 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -492,6 +492,15 @@ class ServiceTest(unittest.TestCase): use_networking=True) self.assertEqual(service._get_links(link_to_self=True), []) + def test_image_name_from_config(self): + image_name = 'example/web:latest' + service = Service('foo', image=image_name) + assert service.image_name == image_name + + def test_image_name_default(self): + service = Service('foo', project='testing') + assert service.image_name == 'testing_foo' + def sort_by_name(dictionary_list): return sorted(dictionary_list, key=lambda k: k['name']) From b59387401c0f5c2eea11153eb024932db2748855 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Tue, 12 Jan 2016 22:04:05 +0100 Subject: [PATCH 209/359] Add zsh completion for 'docker-compose events' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 35c2b996..710dadd4 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -203,6 +203,12 @@ __docker-compose_subcommand() { '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ '--services[Print the service names, one per line.]' && ret=0 ;; + (events) + _arguments \ + $opts_help \ + '--json[Output events as a stream of json objects.]' \ + '*:services:__docker-compose_services_all' && ret=0 + ;; (help) _arguments ':subcommand:__docker-compose_commands' && ret=0 ;; From b786b47bc8425dd75d5f6a257d948d5fd8fda022 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Jan 2016 17:35:48 +0000 Subject: [PATCH 210/359] Remove version checks from tests requiring API v1.21 Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 21 ++++++--------------- tests/integration/project_test.py | 23 +++++++---------------- tests/integration/service_test.py | 2 -- 3 files changed, 13 insertions(+), 33 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3a3c89b0..7348edb0 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -16,7 +16,6 @@ from docker import errors from .. import mock from compose.cli.command import get_project -from compose.cli.docker_client import docker_client from compose.container import Container from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links @@ -336,13 +335,10 @@ class CLITestCase(DockerClientTestCase): assert 'another_1 | another' in result.stdout def test_up_without_networking(self): - self.require_api_version('1.21') - self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d'], None) - client = docker_client(version='1.21') - networks = client.networks(names=[self.project.name]) + networks = self.client.networks(names=[self.project.name]) self.assertEqual(len(networks), 0) for service in self.project.get_services(): @@ -354,21 +350,18 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(web_container.get('HostConfig.Links')) def test_up_with_networking(self): - self.require_api_version('1.21') - self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['--x-networking', 'up', '-d'], None) - client = docker_client(version='1.21') services = self.project.get_services() - networks = client.networks(names=[self.project.name]) + networks = self.client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(client.remove_network, n['Id']) + self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['Driver'], 'bridge') - network = client.inspect_network(networks[0]['Id']) + network = self.client.inspect_network(networks[0]['Id']) self.assertEqual(len(network['Containers']), len(services)) for service in services: @@ -652,15 +645,13 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(container.name, name) def test_run_with_networking(self): - self.require_api_version('1.21') - client = docker_client(version='1.21') self.base_dir = 'tests/fixtures/simple-dockerfile' self.dispatch(['--x-networking', 'run', 'simple', 'true'], None) service = self.project.get_service('simple') container, = service.containers(stopped=True, one_off=True) - networks = client.networks(names=[self.project.name]) + networks = self.client.networks(names=[self.project.name]) for n in networks: - self.addCleanup(client.remove_network, n['Id']) + self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) self.assertEqual(container.human_readable_command, u'true') diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 5a1444b6..1a342c8a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -6,7 +6,6 @@ import random import py from .testcases import DockerClientTestCase -from compose.cli.docker_client import docker_client from compose.config import config from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec @@ -105,20 +104,14 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) def test_get_network_does_not_exist(self): - self.require_api_version('1.21') - client = docker_client(version='1.21') - - project = Project('composetest', [], client) + project = Project('composetest', [], self.client) assert project.get_network() is None def test_get_network(self): - self.require_api_version('1.21') - client = docker_client(version='1.21') - network_name = 'network_does_exist' - project = Project(network_name, [], client) - client.create_network(network_name) - self.addCleanup(client.remove_network, network_name) + project = Project(network_name, [], self.client) + self.client.create_network(network_name) + self.addCleanup(self.client.remove_network, network_name) assert project.get_network()['Name'] == network_name def test_net_from_service(self): @@ -476,15 +469,13 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('console').containers()), 0) def test_project_up_with_custom_network(self): - self.require_api_version('1.21') - client = docker_client(version='1.21') network_name = 'composetest-custom' - client.create_network(network_name) - self.addCleanup(client.remove_network, network_name) + self.client.create_network(network_name) + self.addCleanup(self.client.remove_network, network_name) web = self.create_service('web', net=Net(network_name)) - project = Project('composetest', [web], client, use_networking=True) + project = Project('composetest', [web], self.client, use_networking=True) project.up() assert project.get_network() is None diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 539be5a9..3eb50942 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -718,8 +718,6 @@ class ServiceTest(DockerClientTestCase): """Test that calling scale on a service that has a custom container name results in warning output. """ - # Disable this test against earlier versions because it is flaky - self.require_api_version('1.21') service = self.create_service('app', container_name='custom-container') self.assertEqual(service.custom_container_name(), 'custom-container') From 1a66543461c610ff0b6c1dcf117ed965edd5c249 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Jan 2016 16:25:15 +0000 Subject: [PATCH 211/359] Make the default network name '{project name}_default' Signed-off-by: Aanand Prasad --- compose/project.py | 14 +++++++++----- tests/acceptance/cli_test.py | 4 ++-- tests/integration/project_test.py | 8 ++++++-- tests/unit/project_test.py | 2 +- 4 files changed, 18 insertions(+), 10 deletions(-) diff --git a/compose/project.py b/compose/project.py index d41b1f52..71e353ec 100644 --- a/compose/project.py +++ b/compose/project.py @@ -185,7 +185,7 @@ class Project(object): net = service_dict.pop('net', None) if not net: if self.use_networking: - return Net(self.name) + return Net(self.default_network_name) return Net(None) net_name = get_service_name_from_net(net) @@ -383,7 +383,7 @@ class Project(object): def get_network(self): try: - return self.client.inspect_network(self.name) + return self.client.inspect_network(self.default_network_name) except NotFound: return None @@ -396,9 +396,9 @@ class Project(object): log.info( 'Creating network "{}" with {}' - .format(self.name, driver_name) + .format(self.default_network_name, driver_name) ) - self.client.create_network(self.name, driver=self.network_driver) + self.client.create_network(self.default_network_name, driver=self.network_driver) def remove_network(self): network = self.get_network() @@ -406,7 +406,11 @@ class Project(object): self.client.remove_network(network['Id']) def uses_default_network(self): - return any(service.net.mode == self.name for service in self.services) + return any(service.net.mode == self.default_network_name for service in self.services) + + @property + def default_network_name(self): + return '{}_default'.format(self.name) def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 7348edb0..46ed4237 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -355,7 +355,7 @@ class CLITestCase(DockerClientTestCase): services = self.project.get_services() - networks = self.client.networks(names=[self.project.name]) + networks = self.client.networks(names=[self.project.default_network_name]) for n in networks: self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) @@ -649,7 +649,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['--x-networking', 'run', 'simple', 'true'], None) service = self.project.get_service('simple') container, = service.containers(stopped=True, one_off=True) - networks = self.client.networks(names=[self.project.name]) + networks = self.client.networks(names=[self.project.default_network_name]) for n in networks: self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 1a342c8a..a3e0f33a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -108,10 +108,14 @@ class ProjectTest(DockerClientTestCase): assert project.get_network() is None def test_get_network(self): - network_name = 'network_does_exist' - project = Project(network_name, [], self.client) + project_name = 'network_does_exist' + network_name = '{}_default'.format(project_name) + + project = Project(project_name, [], self.client) self.client.create_network(network_name) self.addCleanup(self.client.remove_network, network_name) + + assert isinstance(project.get_network(), dict) assert project.get_network()['Name'] == network_name def test_net_from_service(self): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index c8590a1f..63953376 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -346,7 +346,7 @@ class ProjectTest(unittest.TestCase): self.assertEqual(service.net.mode, 'container:' + container_name) def test_uses_default_network_true(self): - web = Service('web', project='test', image="alpine", net=Net('test')) + web = Service('web', project='test', image="alpine", net=Net('test_default')) db = Service('web', project='test', image="alpine", net=Net('other')) project = Project('test', [web, db], None) assert project.uses_default_network() From a027a0079c17c87b1e531c82cf8250086dbfbf66 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Jan 2016 16:26:20 +0000 Subject: [PATCH 212/359] Use networking for version 2 Compose files - Remove --x-networking and --x-network-driver - There's now no way to set a network driver - this will be added back with the 'networks' key Signed-off-by: Aanand Prasad --- compose/cli/command.py | 18 ++++++------------ compose/cli/main.py | 4 ---- compose/project.py | 5 +++-- contrib/completion/bash/docker-compose | 9 +-------- contrib/completion/zsh/_docker-compose | 2 -- tests/acceptance/cli_test.py | 12 ++++++------ tests/fixtures/v2-simple/docker-compose.yml | 8 ++++++++ tests/integration/testcases.py | 7 +++++++ tests/unit/project_test.py | 20 ++++++++++++++++++-- 9 files changed, 49 insertions(+), 36 deletions(-) create mode 100644 tests/fixtures/v2-simple/docker-compose.yml diff --git a/compose/cli/command.py b/compose/cli/command.py index b278af3a..f14388c6 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -49,8 +49,6 @@ def project_from_options(base_dir, options): get_config_path_from_options(options), project_name=options.get('--project-name'), verbose=options.get('--verbose'), - use_networking=options.get('--x-networking'), - network_driver=options.get('--x-network-driver'), ) @@ -75,18 +73,14 @@ def get_client(verbose=False, version=None): return client -def get_project(base_dir, config_path=None, project_name=None, verbose=False, - use_networking=False, network_driver=None): +def get_project(base_dir, config_path=None, project_name=None, verbose=False): config_details = config.find(base_dir, config_path) + project_name = get_project_name(config_details.working_dir, project_name) + config_data = config.load(config_details) + api_version = '1.21' if config_data.version < 2 else None + client = get_client(verbose=verbose, version=api_version) - api_version = '1.21' if use_networking else None - return Project.from_config( - get_project_name(config_details.working_dir, project_name), - config.load(config_details), - get_client(verbose=verbose, version=api_version), - use_networking=use_networking, - network_driver=network_driver - ) + return Project.from_config(project_name, config_data, client) def get_project_name(working_dir, project_name=None): diff --git a/compose/cli/main.py b/compose/cli/main.py index 82cf05c9..7c6d1cb5 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -122,10 +122,6 @@ class TopLevelCommand(DocoptCommand): Options: -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) -p, --project-name NAME Specify an alternate project name (default: directory name) - --x-networking (EXPERIMENTAL) Use new Docker networking functionality. - Requires Docker 1.9 or later. - --x-network-driver DRIVER (EXPERIMENTAL) Specify a network driver (default: "bridge"). - Requires Docker 1.9 or later. --verbose Show more output -v, --version Print version and exit diff --git a/compose/project.py b/compose/project.py index 71e353ec..fa3eace2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -48,11 +48,12 @@ class Project(object): ] @classmethod - def from_config(cls, name, config_data, client, use_networking=False, network_driver=None): + def from_config(cls, name, config_data, client): """ Construct a Project from a config.Config object. """ - project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver) + use_networking = (config_data.version and config_data.version >= 2) + project = cls(name, [], client, use_networking=use_networking) if use_networking: remove_links(config_data.services) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index c18e4f6d..caea2a23 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -116,15 +116,11 @@ _docker_compose_docker_compose() { --project-name|-p) return ;; - --x-network-driver) - COMPREPLY=( $( compgen -W "bridge host none overlay" -- "$cur" ) ) - return - ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v --x-networking --x-network-driver" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -416,9 +412,6 @@ _docker_compose() { (( counter++ )) compose_project="${words[$counter]}" ;; - --x-network-driver) - (( counter++ )) - ;; -*) ;; *) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 35c2b996..2a19dd59 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -332,8 +332,6 @@ _docker-compose() { '(- :)'{-v,--version}'[Print version and exit]' \ '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ - '--x-networking[(EXPERIMENTAL) Use new Docker networking functionality. Requires Docker 1.9 or later.]' \ - '--x-network-driver[(EXPERIMENTAL) Specify a network driver (default: "bridge"). Requires Docker 1.9 or later.]:Network Driver:(bridge host none overlay)' \ '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 46ed4237..25808c94 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -338,7 +338,7 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d'], None) - networks = self.client.networks(names=[self.project.name]) + networks = self.client.networks(names=[self.project.default_network_name]) self.assertEqual(len(networks), 0) for service in self.project.get_services(): @@ -350,8 +350,8 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(web_container.get('HostConfig.Links')) def test_up_with_networking(self): - self.base_dir = 'tests/fixtures/links-composefile' - self.dispatch(['--x-networking', 'up', '-d'], None) + self.base_dir = 'tests/fixtures/v2-simple' + self.dispatch(['up', '-d'], None) services = self.project.get_services() @@ -369,7 +369,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn(containers[0].id, network['Containers']) - web_container = self.project.get_service('web').containers()[0] + web_container = self.project.get_service('simple').containers()[0] self.assertFalse(web_container.get('HostConfig.Links')) def test_up_with_links(self): @@ -645,8 +645,8 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(container.name, name) def test_run_with_networking(self): - self.base_dir = 'tests/fixtures/simple-dockerfile' - self.dispatch(['--x-networking', 'run', 'simple', 'true'], None) + self.base_dir = 'tests/fixtures/v2-simple' + self.dispatch(['run', 'simple', 'true'], None) service = self.project.get_service('simple') container, = service.containers(stopped=True, one_off=True) networks = self.client.networks(names=[self.project.default_network_name]) diff --git a/tests/fixtures/v2-simple/docker-compose.yml b/tests/fixtures/v2-simple/docker-compose.yml new file mode 100644 index 00000000..12a9de72 --- /dev/null +++ b/tests/fixtures/v2-simple/docker-compose.yml @@ -0,0 +1,8 @@ +version: 2 +services: + simple: + image: busybox:latest + command: top + another: + image: busybox:latest + command: top diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 8e0525ee..3002539e 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -36,14 +36,21 @@ class DockerClientTestCase(unittest.TestCase): all=True, filters={'label': '%s=composetest' % LABEL_PROJECT}): self.client.remove_container(c['Id'], force=True) + for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): self.client.remove_image(i) + volumes = self.client.volumes().get('Volumes') or [] for v in volumes: if 'composetest_' in v['Name']: self.client.remove_volume(v['Name']) + networks = self.client.networks() + for n in networks: + if 'composetest_' in n['Name']: + self.client.remove_network(n['Name']) + def create_service(self, name, **kwargs): if 'image' not in kwargs and 'build' not in kwargs: kwargs['image'] = 'busybox:latest' diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 63953376..f63135ae 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -39,7 +39,7 @@ class ProjectTest(unittest.TestCase): self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') def test_from_config(self): - dicts = Config(None, [ + config = Config(None, [ { 'name': 'web', 'image': 'busybox:latest', @@ -49,12 +49,28 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', }, ], None) - project = Project.from_config('composetest', dicts, None) + project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') self.assertEqual(project.get_service('db').name, 'db') self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') + self.assertFalse(project.use_networking) + + def test_from_config_v2(self): + config = Config(2, [ + { + 'name': 'web', + 'image': 'busybox:latest', + }, + { + 'name': 'db', + 'image': 'busybox:latest', + }, + ], None) + project = Project.from_config('composetest', config, None) + self.assertEqual(len(project.services), 2) + self.assertTrue(project.use_networking) def test_get_service(self): web = Service( From 9e17cff0efaff14f2b03d5d0ba70559661733b26 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Jan 2016 18:12:53 +0000 Subject: [PATCH 213/359] Refactor API version switching logic Signed-off-by: Aanand Prasad --- compose/cli/command.py | 6 +++++- compose/cli/docker_client.py | 7 ++----- compose/const.py | 7 +++++++ 3 files changed, 14 insertions(+), 6 deletions(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index f14388c6..c1681ffc 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -13,6 +13,7 @@ from requests.exceptions import SSLError from . import errors from . import verbose_proxy from .. import config +from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client from .utils import call_silently @@ -77,7 +78,10 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False): config_details = config.find(base_dir, config_path) project_name = get_project_name(config_details.working_dir, project_name) config_data = config.load(config_details) - api_version = '1.21' if config_data.version < 2 else None + + api_version = os.environ.get( + 'COMPOSE_API_VERSION', + API_VERSIONS[config_data.version]) client = get_client(verbose=verbose, version=api_version) return Project.from_config(project_name, config_data, client) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 48ba97bd..611997df 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -11,8 +11,6 @@ from ..const import HTTP_TIMEOUT log = logging.getLogger(__name__) -DEFAULT_API_VERSION = '1.21' - def docker_client(version=None): """ @@ -23,8 +21,7 @@ def docker_client(version=None): log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') kwargs = kwargs_from_env(assert_hostname=False) - kwargs['version'] = version or os.environ.get( - 'COMPOSE_API_VERSION', - DEFAULT_API_VERSION) + if version: + kwargs['version'] = version kwargs['timeout'] = HTTP_TIMEOUT return Client(**kwargs) diff --git a/compose/const.py b/compose/const.py index 84a5057a..331895b1 100644 --- a/compose/const.py +++ b/compose/const.py @@ -15,3 +15,10 @@ LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' COMPOSEFILE_VERSIONS = (1, 2) + +API_VERSIONS = { + 1: '1.21', + + # TODO: update to 1.22 when there's a Docker 1.10 build to test against + 2: '1.21', +} From 70cce961a8223403d36e8d209925e112f1ebd02f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 12 Jan 2016 17:26:18 +0000 Subject: [PATCH 214/359] Don't allow links or external_links in v2 files Signed-off-by: Aanand Prasad --- compose/config/service_schema_v2.json | 2 -- tests/acceptance/cli_test.py | 13 ++++++++++++- tests/fixtures/v2-simple/links-invalid.yml | 10 ++++++++++ tests/unit/config/config_test.py | 4 +--- 4 files changed, 23 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/v2-simple/links-invalid.yml diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index d1d0854f..47b195fc 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -66,12 +66,10 @@ }, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, - "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "hostname": {"type": "string"}, "image": {"type": "string"}, "ipc": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, - "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "logging": { "type": "object", diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 25808c94..eab4bdae 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -372,7 +372,18 @@ class CLITestCase(DockerClientTestCase): web_container = self.project.get_service('simple').containers()[0] self.assertFalse(web_container.get('HostConfig.Links')) - def test_up_with_links(self): + def test_up_with_links_is_invalid(self): + self.base_dir = 'tests/fixtures/v2-simple' + + result = self.dispatch( + ['-f', 'links-invalid.yml', 'up', '-d'], + returncode=1) + + # TODO: fix validation error messages for v2 files + # assert "Unsupported config option for service 'simple': 'links'" in result.stderr + assert "Unsupported config option" in result.stderr + + def test_up_with_links_v1(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'web'], None) web = self.project.get_service('web') diff --git a/tests/fixtures/v2-simple/links-invalid.yml b/tests/fixtures/v2-simple/links-invalid.yml new file mode 100644 index 00000000..422f9314 --- /dev/null +++ b/tests/fixtures/v2-simple/links-invalid.yml @@ -0,0 +1,10 @@ +version: 2 +services: + simple: + image: busybox:latest + command: top + links: + - another + another: + image: busybox:latest + command: top diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a59d1d34..25483e52 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -268,8 +268,8 @@ class ConfigTest(unittest.TestCase): { 'name': 'web', 'build': os.path.abspath('/'), - 'links': ['db'], 'volumes': [VolumeSpec.parse('/home/user/project:/code')], + 'links': ['db'], }, { 'name': 'db', @@ -405,7 +405,6 @@ class ConfigTest(unittest.TestCase): 'services': { 'web': { 'image': 'example/web', - 'links': ['db'], }, 'db': { 'image': 'example/db', @@ -431,7 +430,6 @@ class ConfigTest(unittest.TestCase): 'name': 'web', 'build': os.path.abspath('/'), 'image': 'example/web', - 'links': ['db'], 'volumes': [VolumeSpec.parse('/home/user/project:/code')], }, { From 2f07e2ac3628617b9817dc9f7815d6c3a730aaac Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Thu, 7 Jan 2016 21:21:47 +0200 Subject: [PATCH 215/359] Ulimits are now merged into extended services Signed-off-by: Dimitar Bonev --- compose/config/config.py | 25 +++++++++++++------ .../extends/common-env-labels-ulimits.yml | 13 ++++++++++ tests/unit/config/config_test.py | 18 +++++++++++++ 3 files changed, 48 insertions(+), 8 deletions(-) create mode 100644 tests/fixtures/extends/common-env-labels-ulimits.yml diff --git a/compose/config/config.py b/compose/config/config.py index 918946b3..11cc3ce9 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -547,8 +547,15 @@ def merge_service_dicts(base, override, version): base.get(field, default), override.get(field, default)) - merge_field('environment', merge_environment) - merge_field('labels', merge_labels) + def merge_mapping(mapping, parse_func): + if mapping in base or mapping in override: + merged = parse_func(base.get(mapping, None)) + merged.update(parse_func(override.get(mapping, None))) + d[mapping] = merged + + merge_mapping('environment', parse_environment) + merge_mapping('labels', parse_labels) + merge_mapping('ulimits', parse_ulimits) for field in ['volumes', 'devices']: merge_field(field, merge_path_mappings) @@ -723,12 +730,6 @@ def join_path_mapping(pair): return ":".join((host, container)) -def merge_labels(base, override): - labels = parse_labels(base) - labels.update(parse_labels(override)) - return labels - - def parse_labels(labels): if not labels: return {} @@ -747,6 +748,14 @@ def split_label(label): return label, '' +def parse_ulimits(ulimits): + if not ulimits: + return {} + + if isinstance(ulimits, dict): + return dict(ulimits) + + def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, os.path.expanduser(path))) diff --git a/tests/fixtures/extends/common-env-labels-ulimits.yml b/tests/fixtures/extends/common-env-labels-ulimits.yml new file mode 100644 index 00000000..09efb4e7 --- /dev/null +++ b/tests/fixtures/extends/common-env-labels-ulimits.yml @@ -0,0 +1,13 @@ +web: + extends: + file: common.yml + service: web + environment: + - FOO=2 + - BAZ=3 + labels: ['label=one'] + ulimits: + nproc: 65535 + memlock: + soft: 1024 + hard: 2048 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index a59d1d34..fb324c57 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1396,6 +1396,24 @@ class ExtendsTest(unittest.TestCase): } ])) + def test_merging_env_labels_ulimits(self): + service_dicts = load_from_filename('tests/fixtures/extends/common-env-labels-ulimits.yml') + + self.assertEqual(service_sort(service_dicts), service_sort([ + { + 'name': 'web', + 'image': 'busybox', + 'command': '/bin/true', + 'environment': { + "FOO": "2", + "BAR": "1", + "BAZ": "3", + }, + 'labels': {'label': 'one'}, + 'ulimits': {'nproc': 65535, 'memlock': {'soft': 1024, 'hard': 2048}} + } + ])) + def test_nested(self): service_dicts = load_from_filename('tests/fixtures/extends/nested.yml') From 3a46abd17fe0b631643a62e2ca5e51a6c9ced462 Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Sun, 10 Jan 2016 18:15:13 +0200 Subject: [PATCH 216/359] Allowed port range in exposed ports Signed-off-by: Dimitar Bonev --- compose/config/validation.py | 2 +- tests/acceptance/cli_test.py | 22 +++++++++++++++++++ .../expose-composefile/docker-compose.yml | 11 ++++++++++ 3 files changed, 34 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/expose-composefile/docker-compose.yml diff --git a/compose/config/validation.py b/compose/config/validation.py index e7006d5a..74ae5c9c 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -38,7 +38,7 @@ DOCKER_CONFIG_HINTS = { VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]' -VALID_EXPOSE_FORMAT = r'^\d+(\/[a-zA-Z]+)?$' +VALID_EXPOSE_FORMAT = r'^\d+(\-\d+)?(\/[a-zA-Z]+)?$' @FormatChecker.cls_checks(format="ports", raises=ValidationError) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 3a3c89b0..a882f9de 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -642,6 +642,28 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "127.0.0.1:30000") self.assertEqual(port_full, "127.0.0.1:30001") + def test_run_with_expose_ports(self): + # create one off container + self.base_dir = 'tests/fixtures/expose-composefile' + self.dispatch(['run', '-d', '--service-ports', 'simple']) + container = self.project.get_service('simple').containers(one_off=True)[0] + + ports = container.ports + self.assertEqual(len(ports), 9) + # exposed ports are not mapped to host ports + assert ports['3000/tcp'] is None + assert ports['3001/tcp'] is None + assert ports['3001/udp'] is None + assert ports['3002/tcp'] is None + assert ports['3003/tcp'] is None + assert ports['3004/tcp'] is None + assert ports['3005/tcp'] is None + assert ports['3006/udp'] is None + assert ports['3007/udp'] is None + + # close all one off containers we just created + container.stop() + def test_run_with_custom_name(self): self.base_dir = 'tests/fixtures/environment-composefile' name = 'the-container-name' diff --git a/tests/fixtures/expose-composefile/docker-compose.yml b/tests/fixtures/expose-composefile/docker-compose.yml new file mode 100644 index 00000000..d14a468d --- /dev/null +++ b/tests/fixtures/expose-composefile/docker-compose.yml @@ -0,0 +1,11 @@ + +simple: + image: busybox:latest + command: top + expose: + - '3000' + - '3001/tcp' + - '3001/udp' + - '3002-3003' + - '3004-3005/tcp' + - '3006-3007/udp' From 5d8c2d3cec432fd1853268804b21cdebe3ed81ce Mon Sep 17 00:00:00 2001 From: Jonathan Stewmon Date: Fri, 4 Dec 2015 16:40:09 -0600 Subject: [PATCH 217/359] add support for stop_signal to compose file Signed-off-by: Jonathan Stewmon --- compose/config/config.py | 1 + compose/config/service_schema_v1.json | 1 + compose/config/service_schema_v2.json | 1 + compose/container.py | 8 ++++++++ tests/acceptance/cli_test.py | 12 ++++++++++++ .../stop-signal-composefile/docker-compose.yml | 10 ++++++++++ tests/integration/service_test.py | 6 ++++++ 7 files changed, 39 insertions(+) create mode 100644 tests/fixtures/stop-signal-composefile/docker-compose.yml diff --git a/compose/config/config.py b/compose/config/config.py index 918946b3..cdd35e12 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -62,6 +62,7 @@ DOCKER_CONFIG_KEYS = [ 'restart', 'security_opt', 'stdin_open', + 'stop_signal', 'tty', 'user', 'volume_driver', diff --git a/compose/config/service_schema_v1.json b/compose/config/service_schema_v1.json index d51c7f73..43e7c8b8 100644 --- a/compose/config/service_schema_v1.json +++ b/compose/config/service_schema_v1.json @@ -94,6 +94,7 @@ "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "stdin_open": {"type": "boolean"}, + "stop_signal": {"type": "string"}, "tty": {"type": "boolean"}, "ulimits": { "type": "object", diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 47b195fc..10331dcc 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -101,6 +101,7 @@ "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "stdin_open": {"type": "boolean"}, + "stop_signal": {"type": "string"}, "tty": {"type": "boolean"}, "ulimits": { "type": "object", diff --git a/compose/container.py b/compose/container.py index 5730f224..2565c8ff 100644 --- a/compose/container.py +++ b/compose/container.py @@ -107,6 +107,10 @@ class Container(object): def labels(self): return self.get('Config.Labels') or {} + @property + def stop_signal(self): + return self.get('Config.StopSignal') + @property def log_config(self): return self.get('HostConfig.LogConfig') or None @@ -132,6 +136,10 @@ class Container(object): def environment(self): return dict(var.split("=", 1) for var in self.get('Config.Env') or []) + @property + def exit_code(self): + return self.get('State.ExitCode') + @property def is_running(self): return self.get('State.Running') diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index eab4bdae..90dca298 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -717,6 +717,18 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(service.containers(stopped=True)), 1) self.assertFalse(service.containers(stopped=True)[0].is_running) + def test_stop_signal(self): + self.base_dir = 'tests/fixtures/stop-signal-composefile' + self.dispatch(['up', '-d'], None) + service = self.project.get_service('simple') + self.assertEqual(len(service.containers()), 1) + self.assertTrue(service.containers()[0].is_running) + + self.dispatch(['stop', '-t', '1'], None) + self.assertEqual(len(service.containers(stopped=True)), 1) + self.assertFalse(service.containers(stopped=True)[0].is_running) + self.assertEqual(service.containers(stopped=True)[0].exit_code, 0) + def test_start_no_containers(self): result = self.dispatch(['start'], returncode=1) assert 'No containers to start' in result.stderr diff --git a/tests/fixtures/stop-signal-composefile/docker-compose.yml b/tests/fixtures/stop-signal-composefile/docker-compose.yml new file mode 100644 index 00000000..04f58aa9 --- /dev/null +++ b/tests/fixtures/stop-signal-composefile/docker-compose.yml @@ -0,0 +1,10 @@ +simple: + image: busybox:latest + command: + - sh + - '-c' + - | + trap 'exit 0' SIGINT + trap 'exit 1' SIGTERM + while true; do :; done + stop_signal: SIGINT diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 3eb50942..4818e47a 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -887,6 +887,12 @@ class ServiceTest(DockerClientTestCase): for name in labels_dict: self.assertIn((name, ''), labels) + def test_stop_signal(self): + stop_signal = 'SIGINT' + service = self.create_service('web', stop_signal=stop_signal) + container = create_and_start_container(service) + self.assertEqual(container.stop_signal, stop_signal) + def test_custom_container_name(self): service = self.create_service('web', container_name='my-web-container') self.assertEqual(service.custom_container_name(), 'my-web-container') From 05935b5e5448436f210cb5488a65338aae6e722f Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Mon, 4 Jan 2016 15:10:32 -0800 Subject: [PATCH 218/359] Don't recreate pre-existing volumes. During the initialize_volumes phase, if a volume using the non-namespaced name already exists, don't create the namespaced equivalent. Signed-off-by: Joffrey F --- compose/project.py | 6 ++++++ compose/volume.py | 11 +++++++++++ tests/integration/project_test.py | 26 ++++++++++++++++++++++++-- tests/integration/volume_test.py | 10 ++++++++++ 4 files changed, 51 insertions(+), 2 deletions(-) diff --git a/compose/project.py b/compose/project.py index fa3eace2..f5bb8beb 100644 --- a/compose/project.py +++ b/compose/project.py @@ -235,6 +235,12 @@ class Project(object): def initialize_volumes(self): try: for volume in self.volumes: + if volume.is_user_created: + log.info( + 'Found user-created volume "{0}". No new namespaced ' + 'volume will be created.'.format(volume.name) + ) + continue volume.create() except NotFound: raise ConfigurationError( diff --git a/compose/volume.py b/compose/volume.py index fb8bd580..9bd98fa5 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +from docker.errors import NotFound + class Volume(object): def __init__(self, client, project, name, driver=None, driver_opts=None): @@ -21,6 +23,15 @@ class Volume(object): def inspect(self): return self.client.inspect_volume(self.full_name) + @property + def is_user_created(self): + try: + self.client.inspect_volume(self.name) + except NotFound: + return False + + return True + @property def full_name(self): return '{0}_{1}'.format(self.project, self.name) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index a3e0f33a..d1b1fdf0 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import random import py +from docker.errors import NotFound from .testcases import DockerClientTestCase from compose.config import config @@ -624,7 +625,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') - def test_project_up_invalid_volume_driver(self): + def test_initialize_volumes_invalid_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( @@ -642,7 +643,7 @@ class ProjectTest(DockerClientTestCase): with self.assertRaises(config.ConfigurationError): project.initialize_volumes() - def test_project_up_updated_driver(self): + def test_initialize_volumes_updated_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -675,3 +676,24 @@ class ProjectTest(DockerClientTestCase): assert 'Configuration for volume {0} specifies driver smb'.format( vol_name ) in str(e.exception) + + def test_initialize_volumes_user_created_volumes(self): + # Use composetest_ prefix so it gets garbage-collected in tearDown() + vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + self.client.create_volume(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'local'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + with self.assertRaises(NotFound): + self.client.inspect_volume(full_vol_name) diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 8ae35378..8bcce0e1 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -54,3 +54,13 @@ class VolumeTest(DockerClientTestCase): vol.remove() volumes = self.client.volumes()['Volumes'] assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 + + def test_is_user_created(self): + vol = Volume(self.client, 'composetest', 'uservolume01') + try: + self.client.create_volume('uservolume01') + assert vol.is_user_created is True + finally: + self.client.remove_volume('uservolume01') + vol2 = Volume(self.client, 'composetest', 'volume01') + assert vol2.is_user_created is False From 9cb58b796e2ea70f37ea857909fa4a3044d8861b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 12 Jan 2016 16:53:49 -0800 Subject: [PATCH 219/359] Implement ability to specify external volumes External volumes are created and managed by the user. They are not namespaced. They are expected to exist at the beginning of the up phase. Signed-off-by: Joffrey F --- compose/config/fields_schema_v2.json | 7 +++++++ compose/project.py | 14 ++++++++++--- compose/volume.py | 21 ++++++++++++++----- tests/integration/project_test.py | 24 ++++++++++++++++++++-- tests/integration/volume_test.py | 30 ++++++++++++++++++---------- tests/unit/config/config_test.py | 18 +++++++++++++++++ 6 files changed, 93 insertions(+), 21 deletions(-) diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 22ff839f..61bd7628 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -41,6 +41,13 @@ "^.+$": {"type": ["string", "number"]} }, "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false } } } diff --git a/compose/project.py b/compose/project.py index f5bb8beb..49b54e10 100644 --- a/compose/project.py +++ b/compose/project.py @@ -77,7 +77,9 @@ class Project(object): project.volumes.append( Volume( client=client, project=name, name=vol_name, - driver=data.get('driver'), driver_opts=data.get('driver_opts') + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external=data.get('external', False) ) ) return project @@ -235,11 +237,17 @@ class Project(object): def initialize_volumes(self): try: for volume in self.volumes: - if volume.is_user_created: + if volume.external: log.info( - 'Found user-created volume "{0}". No new namespaced ' + 'Volume {0} declared as external. No new ' 'volume will be created.'.format(volume.name) ) + if not volume.exists(): + raise ConfigurationError( + 'Volume {0} declared as external, but could not be' + ' found. Please create the volume manually and try' + ' again.'.format(volume.full_name) + ) continue volume.create() except NotFound: diff --git a/compose/volume.py b/compose/volume.py index 9bd98fa5..64671ca9 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -5,12 +5,19 @@ from docker.errors import NotFound class Volume(object): - def __init__(self, client, project, name, driver=None, driver_opts=None): + def __init__(self, client, project, name, driver=None, driver_opts=None, + external=False): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts + self.external_name = None + if external: + if isinstance(external, dict): + self.external_name = external.get('name') + else: + self.external_name = self.name def create(self): return self.client.create_volume( @@ -23,15 +30,19 @@ class Volume(object): def inspect(self): return self.client.inspect_volume(self.full_name) - @property - def is_user_created(self): + def exists(self): try: - self.client.inspect_volume(self.name) + self.inspect() except NotFound: return False - return True + @property + def external(self): + return bool(self.external_name) + @property def full_name(self): + if self.external_name: + return self.external_name return '{0}_{1}'.format(self.project, self.name) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d1b1fdf0..36b736b4 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -677,7 +677,7 @@ class ProjectTest(DockerClientTestCase): vol_name ) in str(e.exception) - def test_initialize_volumes_user_created_volumes(self): + def test_initialize_volumes_external_volumes(self): # Use composetest_ prefix so it gets garbage-collected in tearDown() vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -687,7 +687,7 @@ class ProjectTest(DockerClientTestCase): 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {'driver': 'local'}} + }], volumes={vol_name: {'external': True}} ) project = Project.from_config( name='composetest', @@ -697,3 +697,23 @@ class ProjectTest(DockerClientTestCase): with self.assertRaises(NotFound): self.client.inspect_volume(full_vol_name) + + def test_initialize_volumes_inexistent_external_volume(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'external': True}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError) as e: + project.initialize_volumes() + assert 'Volume {0} declared as external'.format( + vol_name + ) in str(e.exception) diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 8bcce0e1..fbb4aaa2 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -18,9 +18,10 @@ class VolumeTest(DockerClientTestCase): except DockerException: pass - def create_volume(self, name, driver=None, opts=None): + def create_volume(self, name, driver=None, opts=None, external=False): vol = Volume( - self.client, 'composetest', name, driver=driver, driver_opts=opts + self.client, 'composetest', name, driver=driver, driver_opts=opts, + external=external ) self.tmp_volumes.append(vol) return vol @@ -55,12 +56,19 @@ class VolumeTest(DockerClientTestCase): volumes = self.client.volumes()['Volumes'] assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 - def test_is_user_created(self): - vol = Volume(self.client, 'composetest', 'uservolume01') - try: - self.client.create_volume('uservolume01') - assert vol.is_user_created is True - finally: - self.client.remove_volume('uservolume01') - vol2 = Volume(self.client, 'composetest', 'volume01') - assert vol2.is_user_created is False + def test_external_volume(self): + vol = self.create_volume('volume01', external=True) + assert vol.external is True + assert vol.full_name == vol.name + vol.create() + info = vol.inspect() + assert info['Name'] == vol.name + + def test_external_aliased_volume(self): + alias_name = 'alias01' + vol = self.create_volume('volume01', external={'name': alias_name}) + assert vol.external is True + assert vol.full_name == alias_name + vol.create() + info = vol.inspect() + assert info['Name'] == alias_name diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 25483e52..679125bc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -775,6 +775,24 @@ class ConfigTest(unittest.TestCase): 'extends': {'service': 'foo'} } + def test_external_volume_config(self): + config_details = build_config_details({ + 'version': 2, + 'services': { + 'bogus': {'image': 'busybox'} + }, + 'volumes': { + 'ext': {'external': True}, + 'ext2': {'external': {'name': 'aliased'}} + } + }) + config_result = config.load(config_details) + volumes = config_result.volumes + assert 'ext' in volumes + assert volumes['ext']['external'] is True + assert 'ext2' in volumes + assert volumes['ext2']['external']['name'] == 'aliased' + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ From f774422d18c32f3d9233e7c697ccf65e30a3fc06 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 12 Jan 2016 16:58:24 -0800 Subject: [PATCH 220/359] Test Volume.exists() behavior Signed-off-by: Joffrey F --- tests/integration/volume_test.py | 22 ++++++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index fbb4aaa2..2e65f0be 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -57,7 +57,7 @@ class VolumeTest(DockerClientTestCase): assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 def test_external_volume(self): - vol = self.create_volume('volume01', external=True) + vol = self.create_volume('composetest_volume_ext', external=True) assert vol.external is True assert vol.full_name == vol.name vol.create() @@ -65,10 +65,28 @@ class VolumeTest(DockerClientTestCase): assert info['Name'] == vol.name def test_external_aliased_volume(self): - alias_name = 'alias01' + alias_name = 'composetest_alias01' vol = self.create_volume('volume01', external={'name': alias_name}) assert vol.external is True assert vol.full_name == alias_name vol.create() info = vol.inspect() assert info['Name'] == alias_name + + def test_exists(self): + vol = self.create_volume('volume01') + assert vol.exists() is False + vol.create() + assert vol.exists() is True + + def test_exists_external(self): + vol = self.create_volume('volume01', external=True) + assert vol.exists() is False + vol.create() + assert vol.exists() is True + + def test_exists_external_aliased(self): + vol = self.create_volume('volume01', external={'name': 'composetest_alias01'}) + assert vol.exists() is False + vol.create() + assert vol.exists() is True From bf48a781dbc4d82e8b9fa940522b68b78e4c12e3 Mon Sep 17 00:00:00 2001 From: Evgeniy Dobrohvalov Date: Fri, 2 Oct 2015 20:07:44 +0300 Subject: [PATCH 221/359] Add flag for stops all containers if any container was stopped. Signed-off-by: Evgeniy Dobrohvalov --- compose/cli/log_printer.py | 5 +++-- compose/cli/main.py | 38 ++++++++++++++++++++-------------- compose/cli/multiplexer.py | 8 +++++-- docs/reference/up.md | 28 ++++++++++++++----------- tests/unit/cli/main_test.py | 4 ++-- tests/unit/multiplexer_test.py | 18 ++++++++++++++++ 6 files changed, 68 insertions(+), 33 deletions(-) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 864657a4..85fef794 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -13,10 +13,11 @@ from compose.utils import split_buffer class LogPrinter(object): """Print logs from many containers to a single output stream.""" - def __init__(self, containers, output=sys.stdout, monochrome=False): + def __init__(self, containers, output=sys.stdout, monochrome=False, cascade_stop=False): self.containers = containers self.output = utils.get_output_stream(output) self.monochrome = monochrome + self.cascade_stop = cascade_stop def run(self): if not self.containers: @@ -24,7 +25,7 @@ class LogPrinter(object): prefix_width = max_name_width(self.containers) generators = list(self._make_log_generators(self.monochrome, prefix_width)) - for line in Multiplexer(generators).loop(): + for line in Multiplexer(generators, cascade_stop=self.cascade_stop).loop(): self.output.write(line) self.output.flush() diff --git a/compose/cli/main.py b/compose/cli/main.py index 7c6d1cb5..a46521f3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -590,25 +590,33 @@ class TopLevelCommand(DocoptCommand): Usage: up [options] [SERVICE...] Options: - -d Detached mode: Run containers in the background, - print new container names. - --no-color Produce monochrome output. - --no-deps Don't start linked services. - --force-recreate Recreate containers even if their configuration and - image haven't changed. Incompatible with --no-recreate. - --no-recreate If containers already exist, don't recreate them. - Incompatible with --force-recreate. - --no-build Don't build an image, even if it's missing - -t, --timeout TIMEOUT Use this timeout in seconds for container shutdown - when attached or when containers are already - running. (default: 10) + -d Detached mode: Run containers in the background, + print new container names. + Incompatible with --abort-on-container-exit. + --no-color Produce monochrome output. + --no-deps Don't start linked services. + --force-recreate Recreate containers even if their configuration + and image haven't changed. + Incompatible with --no-recreate. + --no-recreate If containers already exist, don't recreate them. + Incompatible with --force-recreate. + --no-build Don't build an image, even if it's missing + --abort-on-container-exit Stops all containers if any container was stopped. + Incompatible with -d. + -t, --timeout TIMEOUT Use this timeout in seconds for container shutdown + when attached or when containers are already + running. (default: 10) """ monochrome = options['--no-color'] start_deps = not options['--no-deps'] + cascade_stop = options['--abort-on-container-exit'] service_names = options['SERVICE'] timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) detached = options.get('-d') + if detached and cascade_stop: + raise UserError("--abort-on-container-exit and -d cannot be combined.") + to_attach = project.up( service_names=service_names, start_deps=start_deps, @@ -619,7 +627,7 @@ class TopLevelCommand(DocoptCommand): ) if not detached: - log_printer = build_log_printer(to_attach, service_names, monochrome) + log_printer = build_log_printer(to_attach, service_names, monochrome, cascade_stop) attach_to_logs(project, log_printer, service_names, timeout) def version(self, project, options): @@ -695,13 +703,13 @@ def run_one_off_container(container_options, project, service, options): sys.exit(exit_code) -def build_log_printer(containers, service_names, monochrome): +def build_log_printer(containers, service_names, monochrome, cascade_stop): if service_names: containers = [ container for container in containers if container.service in service_names ] - return LogPrinter(containers, monochrome=monochrome) + return LogPrinter(containers, monochrome=monochrome, cascade_stop=cascade_stop) def attach_to_logs(project, log_printer, service_names, timeout): diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py index 5e8d91a4..e6e63f24 100644 --- a/compose/cli/multiplexer.py +++ b/compose/cli/multiplexer.py @@ -20,8 +20,9 @@ class Multiplexer(object): parallel and yielding results as they come in. """ - def __init__(self, iterators): + def __init__(self, iterators, cascade_stop=False): self.iterators = iterators + self.cascade_stop = cascade_stop self._num_running = len(iterators) self.queue = Queue() @@ -36,7 +37,10 @@ class Multiplexer(object): raise exception if item is STOP: - self._num_running -= 1 + if self.cascade_stop is True: + break + else: + self._num_running -= 1 else: yield item except Empty: diff --git a/docs/reference/up.md b/docs/reference/up.md index 966aff1e..a02358ec 100644 --- a/docs/reference/up.md +++ b/docs/reference/up.md @@ -15,18 +15,22 @@ parent = "smn_compose_cli" Usage: up [options] [SERVICE...] Options: --d Detached mode: Run containers in the background, - print new container names. ---no-color Produce monochrome output. ---no-deps Don't start linked services. ---force-recreate Recreate containers even if their configuration and - image haven't changed. Incompatible with --no-recreate. ---no-recreate If containers already exist, don't recreate them. - Incompatible with --force-recreate. ---no-build Don't build an image, even if it's missing --t, --timeout TIMEOUT Use this timeout in seconds for container shutdown - when attached or when containers are already - running. (default: 10) +-d Detached mode: Run containers in the background, + print new container names. + Incompatible with --abort-on-container-exit. +--no-color Produce monochrome output. +--no-deps Don't start linked services. +--force-recreate Recreate containers even if their configuration + and image haven't changed. + Incompatible with --no-recreate. +--no-recreate If containers already exist, don't recreate them. + Incompatible with --force-recreate. +--no-build Don't build an image, even if it's missing +--abort-on-container-exit Stops all containers if any container was stopped. + Incompatible with -d. +-t, --timeout TIMEOUT Use this timeout in seconds for container shutdown + when attached or when containers are already + running. (default: 10) ``` Builds, (re)creates, starts, and attaches to containers for a service. diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index f62b2bcc..6f5dd3ca 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -36,7 +36,7 @@ class CLIMainTestCase(unittest.TestCase): mock_container('another', 1), ] service_names = ['web', 'db'] - log_printer = build_log_printer(containers, service_names, True) + log_printer = build_log_printer(containers, service_names, True, False) self.assertEqual(log_printer.containers, containers[:3]) def test_build_log_printer_all_services(self): @@ -46,7 +46,7 @@ class CLIMainTestCase(unittest.TestCase): mock_container('other', 1), ] service_names = [] - log_printer = build_log_printer(containers, service_names, True) + log_printer = build_log_printer(containers, service_names, True, False) self.assertEqual(log_printer.containers, containers) def test_attach_to_logs(self): diff --git a/tests/unit/multiplexer_test.py b/tests/unit/multiplexer_test.py index c56ece1b..750faad8 100644 --- a/tests/unit/multiplexer_test.py +++ b/tests/unit/multiplexer_test.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import unittest +from time import sleep from compose.cli.multiplexer import Multiplexer @@ -46,3 +47,20 @@ class MultiplexerTest(unittest.TestCase): with self.assertRaises(Problem): list(mux.loop()) + + def test_cascade_stop(self): + mux = Multiplexer([ + ((lambda x: sleep(0.01) or x)(x) for x in ['after 0.01 sec T1', + 'after 0.02 sec T1', + 'after 0.03 sec T1']), + ((lambda x: sleep(0.02) or x)(x) for x in ['after 0.02 sec T2', + 'after 0.04 sec T2', + 'after 0.06 sec T2']), + ], cascade_stop=True) + + self.assertEqual( + ['after 0.01 sec T1', + 'after 0.02 sec T1', + 'after 0.02 sec T2', + 'after 0.03 sec T1'], + sorted(list(mux.loop()))) From 8616b2de515f64d0d2541bcc61f8abcac15ac070 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Jan 2016 11:22:57 -0800 Subject: [PATCH 222/359] Update error message when external volume is missing Signed-off-by: Joffrey F --- compose/project.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/compose/project.py b/compose/project.py index 49b54e10..08843a6e 100644 --- a/compose/project.py +++ b/compose/project.py @@ -238,15 +238,18 @@ class Project(object): try: for volume in self.volumes: if volume.external: - log.info( + log.debug( 'Volume {0} declared as external. No new ' 'volume will be created.'.format(volume.name) ) if not volume.exists(): raise ConfigurationError( - 'Volume {0} declared as external, but could not be' - ' found. Please create the volume manually and try' - ' again.'.format(volume.full_name) + 'Volume {name} declared as external, but could' + ' not be found. Please create the volume manually' + ' using `{command}{name}` and try again.'.format( + name=volume.full_name, + command='docker volume create --name=' + ) ) continue volume.create() From d601199eb5d31da117179c113d77f9d070b3507b Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Jan 2016 12:07:08 -0800 Subject: [PATCH 223/359] Normalize external_name Signed-off-by: Joffrey F --- compose/config/config.py | 7 +++++++ compose/project.py | 2 +- compose/volume.py | 9 ++------- tests/integration/project_test.py | 8 ++++++-- tests/integration/volume_test.py | 10 ++++++---- 5 files changed, 22 insertions(+), 14 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 918946b3..88722318 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -273,6 +273,13 @@ def load_volumes(config_files): for config_file in config_files: for name, volume_config in config_file.config.get('volumes', {}).items(): volumes.update({name: volume_config}) + external = volume_config.get('external') + if external: + if isinstance(external, dict): + volume_config['external_name'] = external.get('name') + else: + volume_config['external_name'] = name + return volumes diff --git a/compose/project.py b/compose/project.py index 08843a6e..e882713c 100644 --- a/compose/project.py +++ b/compose/project.py @@ -79,7 +79,7 @@ class Project(object): client=client, project=name, name=vol_name, driver=data.get('driver'), driver_opts=data.get('driver_opts'), - external=data.get('external', False) + external_name=data.get('external_name') ) ) return project diff --git a/compose/volume.py b/compose/volume.py index 64671ca9..b78aa029 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -6,18 +6,13 @@ from docker.errors import NotFound class Volume(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - external=False): + external_name=None): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts - self.external_name = None - if external: - if isinstance(external, dict): - self.external_name = external.get('name') - else: - self.external_name = self.name + self.external_name = external_name def create(self): return self.client.create_volume( diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 36b736b4..467eb786 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -687,7 +687,9 @@ class ProjectTest(DockerClientTestCase): 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {'external': True}} + }], volumes={ + vol_name: {'external': True, 'external_name': vol_name} + } ) project = Project.from_config( name='composetest', @@ -706,7 +708,9 @@ class ProjectTest(DockerClientTestCase): 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {'external': True}} + }], volumes={ + vol_name: {'external': True, 'external_name': vol_name} + } ) project = Project.from_config( name='composetest', diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 2e65f0be..706179ed 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -18,10 +18,12 @@ class VolumeTest(DockerClientTestCase): except DockerException: pass - def create_volume(self, name, driver=None, opts=None, external=False): + def create_volume(self, name, driver=None, opts=None, external=None): + if external and isinstance(external, bool): + external = name vol = Volume( self.client, 'composetest', name, driver=driver, driver_opts=opts, - external=external + external_name=external ) self.tmp_volumes.append(vol) return vol @@ -66,7 +68,7 @@ class VolumeTest(DockerClientTestCase): def test_external_aliased_volume(self): alias_name = 'composetest_alias01' - vol = self.create_volume('volume01', external={'name': alias_name}) + vol = self.create_volume('volume01', external=alias_name) assert vol.external is True assert vol.full_name == alias_name vol.create() @@ -86,7 +88,7 @@ class VolumeTest(DockerClientTestCase): assert vol.exists() is True def test_exists_external_aliased(self): - vol = self.create_volume('volume01', external={'name': 'composetest_alias01'}) + vol = self.create_volume('volume01', external='composetest_alias01') assert vol.exists() is False vol.create() assert vol.exists() is True From 85a210d9eb4d3f7f0de537469680724d94a8abe4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jan 2016 20:34:07 +0000 Subject: [PATCH 224/359] Increase timeout on signal-handling tests Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index eab4bdae..2f6dfba4 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -454,14 +454,14 @@ class CLITestCase(DockerClientTestCase): wait_on_condition(ContainerCountCondition(self.project, 2)) os.kill(proc.pid, signal.SIGINT) - wait_on_condition(ContainerCountCondition(self.project, 0)) + wait_on_condition(ContainerCountCondition(self.project, 0), timeout=30) def test_up_handles_sigterm(self): proc = start_process(self.base_dir, ['up', '-t', '2']) wait_on_condition(ContainerCountCondition(self.project, 2)) os.kill(proc.pid, signal.SIGTERM) - wait_on_condition(ContainerCountCondition(self.project, 0)) + wait_on_condition(ContainerCountCondition(self.project, 0), timeout=30) def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' From e76b2679eb8f3117c7c0e0444e70bfc4d7fa502d Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Jan 2016 13:52:59 -0800 Subject: [PATCH 225/359] external volume disallows other config keys Signed-off-by: Joffrey F --- compose/config/fields_schema_v2.json | 42 +++++++++++++++++----------- tests/unit/config/config_test.py | 13 +++++++++ 2 files changed, 38 insertions(+), 17 deletions(-) diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 61bd7628..310dbf96 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -32,24 +32,32 @@ "definitions": { "volume": { "id": "#/definitions/volume", - "type": "object", - "properties": { - "driver": {"type": "string"}, - "driver_opts": { - "type": "object", - "patternProperties": { - "^.+$": {"type": ["string", "number"]} - }, - "additionalProperties": false + "oneOf": [{ + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + }, + "additionalProperties": false + } }, - "external": { - "type": ["boolean", "object"], - "properties": { - "name": {"type": "string"} - }, - "additionalProperties": false - } - } + "additionalProperties": false + }, { + "type": "object", + "properties": { + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }] } }, "additionalProperties": false diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 679125bc..b1759880 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -793,6 +793,19 @@ class ConfigTest(unittest.TestCase): assert 'ext2' in volumes assert volumes['ext2']['external']['name'] == 'aliased' + def test_external_volume_invalid_config(self): + config_details = build_config_details({ + 'version': 2, + 'services': { + 'bogus': {'image': 'busybox'} + }, + 'volumes': { + 'ext': {'external': True, 'driver': 'foo'} + } + }) + with self.assertRaises(ConfigurationError): + config.load(config_details) + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ From b6618815b91a8e01f87c691d67c1ea9839377e69 Mon Sep 17 00:00:00 2001 From: Jonathan Stewmon Date: Wed, 13 Jan 2016 16:18:00 -0600 Subject: [PATCH 226/359] update docker-py requirement to use master branch Signed-off-by: Jonathan Stewmon --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 8c6d5f3a..313c6b7a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ PyYAML==3.11 -docker-py==1.6.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 +git+https://github.com/docker/docker-py.git@master#egg=docker-py jsonschema==2.5.1 requests==2.7.0 six==1.7.3 From 59a4ab9634ab227f6bd8a5f2e579661c64aa6813 Mon Sep 17 00:00:00 2001 From: "Michael A. Smith" Date: Tue, 12 Jan 2016 13:49:14 -0500 Subject: [PATCH 227/359] Allow Entrypoints to be Lists Signed-off-by: Michael A. Smith --- compose/config/service_schema_v1.json | 7 ++++++- compose/config/service_schema_v2.json | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/compose/config/service_schema_v1.json b/compose/config/service_schema_v1.json index d51c7f73..cee6ad74 100644 --- a/compose/config/service_schema_v1.json +++ b/compose/config/service_schema_v1.json @@ -34,7 +34,12 @@ "dns_search": {"$ref": "#/definitions/string_or_list"}, "dockerfile": {"type": "string"}, "domainname": {"type": "string"}, - "entrypoint": {"$ref": "#/definitions/string_or_list"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, "env_file": {"$ref": "#/definitions/string_or_list"}, "environment": {"$ref": "#/definitions/list_or_dict"}, diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 47b195fc..17a5387f 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -34,7 +34,12 @@ "dns_search": {"$ref": "#/definitions/string_or_list"}, "dockerfile": {"type": "string"}, "domainname": {"type": "string"}, - "entrypoint": {"$ref": "#/definitions/string_or_list"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, "env_file": {"$ref": "#/definitions/string_or_list"}, "environment": {"$ref": "#/definitions/list_or_dict"}, From 9bff308251d1629a9f7107adc5b25f71a3844e12 Mon Sep 17 00:00:00 2001 From: "Michael A. Smith" Date: Fri, 8 Jan 2016 15:46:49 -0500 Subject: [PATCH 228/359] Document Entrypoints and Commands as Lists Signed-off-by: Michael A. Smith --- docs/compose-file.md | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 40a3cf02..4759cde0 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -120,6 +120,10 @@ Override the default command. command: bundle exec thin -p 3000 +The command can also be a list, in a manner similar to [dockerfile](https://docs.docker.com/engine/reference/builder/#cmd): + + command: [bundle, exec, thin, -p, 3000] + ### cgroup_parent Specify an optional parent cgroup for the container. @@ -174,6 +178,22 @@ specified using the `build` key. Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. +### entrypoint + +Override the default entrypoint. + + entrypoint: /code/entrypoint.sh + +The entrypoint can also be a list, in a manner similar to [dockerfile](https://docs.docker.com/engine/reference/builder/#entrypoint): + + entrypoint: + - php + - -d + - zend_extension=/usr/local/lib/php/extensions/no-debug-non-zts-20100525/xdebug.so + - -d + - memory_limit=-1 + - vendor/bin/phpunit + ### env_file Add environment variables from a file. Can be a single value or a list. @@ -451,7 +471,7 @@ specifying read-only access(``ro``) or read-write(``rw``). - container_name - service_name:rw -### cpu\_shares, cpu\_quota, cpuset, domainname, entrypoint, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir +### cpu\_shares, cpu\_quota, cpuset, domainname, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir Each of these is a single value, analogous to its [docker run](https://docs.docker.com/engine/reference/run/) counterpart. @@ -460,7 +480,6 @@ Each of these is a single value, analogous to its cpu_quota: 50000 cpuset: 0,1 - entrypoint: /code/entrypoint.sh user: postgresql working_dir: /code From 6877c6ca06ef605e1227b79092cf8054996df27a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 13 Jan 2016 20:51:39 -0500 Subject: [PATCH 229/359] Fix flaky multiplex test. Signed-off-by: Daniel Nephin --- tests/unit/multiplexer_test.py | 23 +++++++++-------------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/tests/unit/multiplexer_test.py b/tests/unit/multiplexer_test.py index 750faad8..737ba25d 100644 --- a/tests/unit/multiplexer_test.py +++ b/tests/unit/multiplexer_test.py @@ -49,18 +49,13 @@ class MultiplexerTest(unittest.TestCase): list(mux.loop()) def test_cascade_stop(self): - mux = Multiplexer([ - ((lambda x: sleep(0.01) or x)(x) for x in ['after 0.01 sec T1', - 'after 0.02 sec T1', - 'after 0.03 sec T1']), - ((lambda x: sleep(0.02) or x)(x) for x in ['after 0.02 sec T2', - 'after 0.04 sec T2', - 'after 0.06 sec T2']), - ], cascade_stop=True) + def fast_stream(): + for num in range(3): + yield "stream1 %s" % num - self.assertEqual( - ['after 0.01 sec T1', - 'after 0.02 sec T1', - 'after 0.02 sec T2', - 'after 0.03 sec T1'], - sorted(list(mux.loop()))) + def slow_stream(): + sleep(5) + yield "stream2 FAIL" + + mux = Multiplexer([fast_stream(), slow_stream()], cascade_stop=True) + assert "stream2 FAIL" not in set(mux.loop()) From e41e6c1241a7e17fddd5c3abf536f462ed5ab1c4 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 13 Jan 2016 18:22:29 -0800 Subject: [PATCH 230/359] Properly validate volume definition Test valid empty volume definitions Signed-off-by: Joffrey F --- compose/config/config.py | 12 +++++++++ compose/config/fields_schema_v2.json | 38 +++++++++++----------------- tests/unit/config/config_test.py | 17 +++++++++++++ 3 files changed, 44 insertions(+), 23 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index d2b75e71..17fd2db4 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -272,9 +272,21 @@ def load_volumes(config_files): volumes = {} for config_file in config_files: for name, volume_config in config_file.config.get('volumes', {}).items(): + if volume_config is None: + volumes.update({name: {}}) + continue + volumes.update({name: volume_config}) external = volume_config.get('external') if external: + if len(volume_config.keys()) > 1: + raise ConfigurationError( + 'Volume {0} declared as external but specifies' + ' additional attributes ({1}). '.format( + name, + ', '.join([k for k in volume_config.keys() if k != 'external']) + ) + ) if isinstance(external, dict): volume_config['external_name'] = external.get('name') else: diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 310dbf96..25126ed1 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -32,32 +32,24 @@ "definitions": { "volume": { "id": "#/definitions/volume", - "oneOf": [{ - "type": "object", - "properties": { - "driver": {"type": "string"}, - "driver_opts": { - "type": "object", - "patternProperties": { - "^.+$": {"type": ["string", "number"]} - }, - "additionalProperties": false + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} } }, "additionalProperties": false - }, { - "type": "object", - "properties": { - "external": { - "type": ["boolean", "object"], - "properties": { - "name": {"type": "string"} - }, - "additionalProperties": false - } - }, - "additionalProperties": false - }] + }, + "additionalProperties": false } }, "additionalProperties": false diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 05fea27d..77c55ad9 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -112,6 +112,23 @@ class ConfigTest(unittest.TestCase): } }) + def test_named_volume_config_empty(self): + config_details = build_config_details({ + 'version': 2, + 'services': { + 'simple': {'image': 'busybox'} + }, + 'volumes': { + 'simple': None, + 'other': {}, + } + }) + config_result = config.load(config_details) + volumes = config_result.volumes + assert 'simple' in volumes + assert volumes['simple'] == {} + assert volumes['other'] == {} + def test_load_service_with_name_version(self): config_data = config.load( build_config_details({ From c8ed1568063d67595a0a38522d2d9355c637b899 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 27 Nov 2015 12:19:43 -0500 Subject: [PATCH 231/359] Adding docker-compose down Signed-off-by: Daniel Nephin --- compose/cli/main.py | 27 +++++++++++++++++++++++++++ compose/project.py | 20 ++++++++++++++++++++ compose/service.py | 21 +++++++++++++++++++++ tests/acceptance/cli_test.py | 8 ++++++++ tests/unit/service_test.py | 34 ++++++++++++++++++++++++++++++++++ 5 files changed, 110 insertions(+) diff --git a/compose/cli/main.py b/compose/cli/main.py index a46521f3..8c1a55ff 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -25,6 +25,7 @@ from ..progress_stream import StreamOutputError from ..project import NoSuchService from ..service import BuildError from ..service import ConvergenceStrategy +from ..service import ImageType from ..service import NeedsBuildError from .command import friendly_error_message from .command import get_config_path_from_options @@ -129,6 +130,7 @@ class TopLevelCommand(DocoptCommand): build Build or rebuild services config Validate and view the compose file create Create services + down Stop and remove containers, networks, images, and volumes events Receive real time events from containers help Get help on a command kill Kill containers @@ -242,6 +244,22 @@ class TopLevelCommand(DocoptCommand): do_build=not options['--no-build'] ) + def down(self, project, options): + """ + Stop containers and remove containers, networks, volumes, and images + created by `up`. + + Usage: down [options] + + Options: + --rmi type Remove images, type may be one of: 'all' to remove + all images, or 'local' to remove only images that + don't have an custom name set by the `image` field + -v, --volumes Remove data volumes + """ + image_type = image_type_from_opt('--rmi', options['--rmi']) + project.down(image_type, options['--volumes']) + def events(self, project, options): """ Receive real time events from containers. @@ -660,6 +678,15 @@ def convergence_strategy_from_opts(options): return ConvergenceStrategy.changed +def image_type_from_opt(flag, value): + if not value: + return ImageType.none + try: + return ImageType[value] + except KeyError: + raise UserError("%s flag must be one of: all, local" % flag) + + def run_one_off_container(container_options, project, service, options): if not options['--no-deps']: deps = service.get_linked_service_names() diff --git a/compose/project.py b/compose/project.py index e882713c..0774a400 100644 --- a/compose/project.py +++ b/compose/project.py @@ -270,6 +270,24 @@ class Project(object): ) ) + def down(self, remove_image_type, include_volumes): + self.stop() + self.remove_stopped() + self.remove_network() + + if include_volumes: + self.remove_volumes() + + self.remove_images(remove_image_type) + + def remove_images(self, remove_image_type): + for service in self.get_services(): + service.remove_image(remove_image_type) + + def remove_volumes(self): + for volume in self.volumes: + volume.remove() + def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -419,6 +437,8 @@ class Project(object): self.client.create_network(self.default_network_name, driver=self.network_driver) def remove_network(self): + if not self.use_networking: + return network = self.get_network() if network: self.client.remove_network(network['Id']) diff --git a/compose/service.py b/compose/service.py index d5c36f1a..1972b1b1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -98,6 +98,14 @@ class ConvergenceStrategy(enum.Enum): return self is not type(self).never +@enum.unique +class ImageType(enum.Enum): + """Enumeration for the types of images known to compose.""" + none = 0 + local = 1 + all = 2 + + class Service(object): def __init__( self, @@ -672,6 +680,19 @@ class Service(object): def custom_container_name(self): return self.options.get('container_name') + def remove_image(self, image_type): + if not image_type or image_type == ImageType.none: + return False + if image_type == ImageType.local and self.options.get('image'): + return False + + try: + self.client.remove_image(self.image_name) + return True + except APIError as e: + log.error("Failed to remove image for service %s: %s", self.name, e) + return False + def specifies_host_port(self): def has_host_port(binding): _, external_bindings = split_port(binding) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8b5892ab..a06f14dc 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -314,6 +314,14 @@ class CLITestCase(DockerClientTestCase): ['create', '--force-recreate', '--no-recreate'], returncode=1) + def test_down_invalid_rmi_flag(self): + result = self.dispatch(['down', '--rmi', 'bogus'], returncode=1) + assert '--rmi flag must be' in result.stderr + + def test_down(self): + result = self.dispatch(['down']) + # TODO: + def test_up_detached(self): self.dispatch(['up', '-d']) service = self.project.get_service('simple') diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 63cf658e..fa58929b 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker +from docker.errors import APIError from .. import mock from .. import unittest @@ -16,6 +17,7 @@ from compose.service import build_ulimits from compose.service import build_volume_binding from compose.service import ContainerNet from compose.service import get_container_data_volumes +from compose.service import ImageType from compose.service import merge_volume_bindings from compose.service import NeedsBuildError from compose.service import Net @@ -422,6 +424,38 @@ class ServiceTest(unittest.TestCase): } self.assertEqual(config_dict, expected) + def test_remove_image_none(self): + web = Service('web', image='example', client=self.mock_client) + assert not web.remove_image(ImageType.none) + assert not self.mock_client.remove_image.called + + def test_remove_image_local_with_image_name_doesnt_remove(self): + web = Service('web', image='example', client=self.mock_client) + assert not web.remove_image(ImageType.local) + assert not self.mock_client.remove_image.called + + def test_remove_image_local_without_image_name_does_remove(self): + web = Service('web', build='.', client=self.mock_client) + assert web.remove_image(ImageType.local) + self.mock_client.remove_image.assert_called_once_with(web.image_name) + + def test_remove_image_all_does_remove(self): + web = Service('web', image='example', client=self.mock_client) + assert web.remove_image(ImageType.all) + self.mock_client.remove_image.assert_called_once_with(web.image_name) + + def test_remove_image_with_error(self): + self.mock_client.remove_image.side_effect = error = APIError( + message="testing", + response={}, + explanation="Boom") + + web = Service('web', image='example', client=self.mock_client) + with mock.patch('compose.service.log', autospec=True) as mock_log: + assert not web.remove_image(ImageType.all) + mock_log.error.assert_called_once_with( + "Failed to remove image for service %s: %s", web.name, error) + def test_specifies_host_port_with_no_ports(self): service = Service( 'foo', From c64af0a459ed0a96c38547451a0b5da69c3da079 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 12 Jan 2016 17:33:17 -0500 Subject: [PATCH 232/359] Add an acceptance test and docs for the down subcommand Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- compose/parallel.py | 2 +- compose/project.py | 3 ++- compose/service.py | 1 + compose/volume.py | 9 ++++++++ docs/index.md | 3 +-- docs/reference/down.md | 26 ++++++++++++++++++++++ docs/reference/index.md | 5 +++++ tests/acceptance/cli_test.py | 12 ++++++++-- tests/fixtures/shutdown/Dockerfile | 4 ++++ tests/fixtures/shutdown/docker-compose.yml | 10 +++++++++ tests/integration/service_test.py | 12 +++++----- tests/unit/volume_test.py | 26 ++++++++++++++++++++++ 13 files changed, 102 insertions(+), 13 deletions(-) create mode 100644 docs/reference/down.md create mode 100644 tests/fixtures/shutdown/Dockerfile create mode 100644 tests/fixtures/shutdown/docker-compose.yml create mode 100644 tests/unit/volume_test.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 8c1a55ff..9957d391 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -247,7 +247,7 @@ class TopLevelCommand(DocoptCommand): def down(self, project, options): """ Stop containers and remove containers, networks, volumes, and images - created by `up`. + created by `up`. Only containers and networks are removed by default. Usage: down [options] diff --git a/compose/parallel.py b/compose/parallel.py index 2735a397..b8415e5e 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -24,7 +24,7 @@ def parallel_execute(objects, func, index_func, msg): object we give it. """ objects = list(objects) - stream = get_output_stream(sys.stdout) + stream = get_output_stream(sys.stderr) writer = ParallelStreamWriter(stream, msg) for obj in objects: diff --git a/compose/project.py b/compose/project.py index 0774a400..3ba9532f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -272,7 +272,7 @@ class Project(object): def down(self, remove_image_type, include_volumes): self.stop() - self.remove_stopped() + self.remove_stopped(v=include_volumes) self.remove_network() if include_volumes: @@ -441,6 +441,7 @@ class Project(object): return network = self.get_network() if network: + log.info("Removing network %s", self.default_network_name) self.client.remove_network(network['Id']) def uses_default_network(self): diff --git a/compose/service.py b/compose/service.py index 1972b1b1..1c848ca3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -686,6 +686,7 @@ class Service(object): if image_type == ImageType.local and self.options.get('image'): return False + log.info("Removing image %s", self.image_name) try: self.client.remove_image(self.image_name) return True diff --git a/compose/volume.py b/compose/volume.py index b78aa029..469e406a 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -1,9 +1,14 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging + from docker.errors import NotFound +log = logging.getLogger(__name__) + + class Volume(object): def __init__(self, client, project, name, driver=None, driver_opts=None, external_name=None): @@ -20,6 +25,10 @@ class Volume(object): ) def remove(self): + if self.external: + log.info("Volume %s is external, skipping", self.full_name) + return + log.info("Removing volume %s", self.full_name) return self.client.remove_volume(self.full_name) def inspect(self): diff --git a/docs/index.md b/docs/index.md index 6e8f2090..887df99d 100644 --- a/docs/index.md +++ b/docs/index.md @@ -154,8 +154,7 @@ environments in just a few commands: $ docker-compose up -d $ ./run_tests - $ docker-compose stop - $ docker-compose rm -f + $ docker-compose down ### Single host deployments diff --git a/docs/reference/down.md b/docs/reference/down.md new file mode 100644 index 00000000..428e4e58 --- /dev/null +++ b/docs/reference/down.md @@ -0,0 +1,26 @@ + + +# down + +``` +Stop containers and remove containers, networks, volumes, and images +created by `up`. Only containers and networks are removed by default. + +Usage: down [options] + +Options: + --rmi type Remove images, type may be one of: 'all' to remove + all images, or 'local' to remove only images that + don't have an custom name set by the `image` field + -v, --volumes Remove data volumes + +``` diff --git a/docs/reference/index.md b/docs/reference/index.md index 1635b60c..5406b9c7 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -14,10 +14,14 @@ parent = "smn_compose_ref" The following pages describe the usage information for the [docker-compose](docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. * [build](build.md) +* [config](config.md) +* [create](create.md) +* [down](down.md) * [events](events.md) * [help](help.md) * [kill](kill.md) * [logs](logs.md) +* [pause](pause.md) * [port](port.md) * [ps](ps.md) * [pull](pull.md) @@ -27,6 +31,7 @@ The following pages describe the usage information for the [docker-compose](dock * [scale](scale.md) * [start](start.md) * [stop](stop.md) +* [unpause](unpause.md) * [up](up.md) ## Where to go next diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a06f14dc..cb04918b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -319,8 +319,16 @@ class CLITestCase(DockerClientTestCase): assert '--rmi flag must be' in result.stderr def test_down(self): - result = self.dispatch(['down']) - # TODO: + self.base_dir = 'tests/fixtures/shutdown' + self.dispatch(['up', '-d']) + wait_on_condition(ContainerCountCondition(self.project, 1)) + + result = self.dispatch(['down', '--rmi=local', '--volumes']) + assert 'Stopping shutdown_web_1' in result.stderr + assert 'Removing shutdown_web_1' in result.stderr + assert 'Removing volume shutdown_data' in result.stderr + assert 'Removing image shutdown_web' in result.stderr + assert 'Removing network shutdown_default' in result.stderr def test_up_detached(self): self.dispatch(['up', '-d']) diff --git a/tests/fixtures/shutdown/Dockerfile b/tests/fixtures/shutdown/Dockerfile new file mode 100644 index 00000000..51ed0d90 --- /dev/null +++ b/tests/fixtures/shutdown/Dockerfile @@ -0,0 +1,4 @@ + +FROM busybox:latest +RUN echo something +CMD top diff --git a/tests/fixtures/shutdown/docker-compose.yml b/tests/fixtures/shutdown/docker-compose.yml new file mode 100644 index 00000000..c83c3d63 --- /dev/null +++ b/tests/fixtures/shutdown/docker-compose.yml @@ -0,0 +1,10 @@ + +version: 2 + +volumes: + data: + driver: local + +services: + web: + build: . diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4818e47a..314076cd 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -616,13 +616,13 @@ class ServiceTest(DockerClientTestCase): service.create_container(number=next_number) service.create_container(number=next_number + 1) - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) for container in service.containers(): self.assertTrue(container.is_running) self.assertTrue(container.number in valid_numbers) - captured_output = mock_stdout.getvalue() + captured_output = mock_stderr.getvalue() self.assertNotIn('Creating', captured_output) self.assertIn('Starting', captured_output) @@ -639,14 +639,14 @@ class ServiceTest(DockerClientTestCase): for container in service.containers(): self.assertFalse(container.is_running) - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(2) self.assertEqual(len(service.containers()), 2) for container in service.containers(): self.assertTrue(container.is_running) - captured_output = mock_stdout.getvalue() + captured_output = mock_stderr.getvalue() self.assertIn('Creating', captured_output) self.assertIn('Starting', captured_output) @@ -665,12 +665,12 @@ class ServiceTest(DockerClientTestCase): response={}, explanation="Boom")): - with mock.patch('sys.stdout', new_callable=StringIO) as mock_stdout: + with mock.patch('sys.stderr', new_callable=StringIO) as mock_stderr: service.scale(3) self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.assertIn("ERROR: for 2 Boom", mock_stdout.getvalue()) + self.assertIn("ERROR: for 2 Boom", mock_stderr.getvalue()) def test_scale_with_unexpected_exception(self): """Test that when scaling if the API returns an error, that is not of type diff --git a/tests/unit/volume_test.py b/tests/unit/volume_test.py new file mode 100644 index 00000000..d7ad0792 --- /dev/null +++ b/tests/unit/volume_test.py @@ -0,0 +1,26 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import docker +import pytest + +from compose import volume +from tests import mock + + +@pytest.fixture +def mock_client(): + return mock.create_autospec(docker.Client) + + +class TestVolume(object): + + def test_remove_local_volume(self, mock_client): + vol = volume.Volume(mock_client, 'foo', 'project') + vol.remove() + mock_client.remove_volume.assert_called_once_with('foo_project') + + def test_remove_external_volume(self, mock_client): + vol = volume.Volume(mock_client, 'foo', 'project', external_name='data') + vol.remove() + assert not mock_client.remove_volume.called From de949284f5b126d07e42c2322f1c80c62f9c68a5 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 13 Jan 2016 12:55:59 -0500 Subject: [PATCH 233/359] Refactor config loading for handling volumes_from in v2. Signed-off-by: Daniel Nephin --- compose/config/config.py | 60 +++++++++++++++++----------------------- 1 file changed, 25 insertions(+), 35 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7eb2ce2c..82ff40aa 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -122,6 +122,9 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def get_service_dicts(self, version): return self.config if version == 1 else self.config.get('services', {}) + def get_volumes(self, version): + return {} if version == 1 else self.config.get('volumes', {}) + class Config(namedtuple('_Config', 'version services volumes')): """ @@ -243,41 +246,29 @@ def load(config_details): if version not in COMPOSEFILE_VERSIONS: raise ConfigurationError('Invalid config version provided: {0}'.format(version)) - processed_files = [] - for config_file in config_details.config_files: - processed_files.append( - process_config_file(config_file, version=version) - ) + processed_files = [ + process_config_file(config_file, version=version) + for config_file in config_details.config_files + ] config_details = config_details._replace(config_files=processed_files) - if version == 1: - service_dicts = load_services( - config_details.working_dir, config_details.config_files, - version - ) - volumes = {} - elif version == 2: - config_files = [ - ConfigFile(f.filename, f.config.get('services', {})) - for f in config_details.config_files - ] - service_dicts = load_services( - config_details.working_dir, config_files, version - ) - volumes = load_volumes(config_details.config_files) - + volumes = load_volumes(config_details.config_files) + service_dicts = load_services( + config_details.working_dir, + config_details.config_files[0].filename, + [file.get_service_dicts(version) for file in config_details.config_files], + version) return Config(version, service_dicts, volumes) def load_volumes(config_files): volumes = {} for config_file in config_files: - for name, volume_config in config_file.config.get('volumes', {}).items(): - if volume_config is None: - volumes.update({name: {}}) + for name, volume_config in config_file.get_volumes().items(): + volumes[name] = volume_config or {} + if not volume_config: continue - volumes.update({name: volume_config}) external = volume_config.get('external') if external: if len(volume_config.keys()) > 1: @@ -296,8 +287,8 @@ def load_volumes(config_files): return volumes -def load_services(working_dir, config_files, version): - def build_service(filename, service_name, service_dict): +def load_services(working_dir, filename, service_configs, version): + def build_service(service_name, service_dict): service_config = ServiceConfig.with_abs_paths( working_dir, filename, @@ -314,10 +305,10 @@ def load_services(working_dir, config_files, version): service_dict['name'] = service_config.name return service_dict - def build_services(config_file): + def build_services(service_config): return sort_service_dicts([ - build_service(config_file.filename, name, service_dict) - for name, service_dict in config_file.config.items() + build_service(name, service_dict) + for name, service_dict in service_config.items() ]) def merge_services(base, override): @@ -330,12 +321,11 @@ def load_services(working_dir, config_files, version): for name in all_service_names } - config_file = config_files[0] - for next_file in config_files[1:]: - config = merge_services(config_file.config, next_file.config) - config_file = config_file._replace(config=config) + service_config = service_configs[0] + for next_config in service_configs[1:]: + service_config = merge_services(service_config, next_config) - return build_services(config_file) + return build_services(service_config) def process_config_file(config_file, version, service_name=None): From c3968a439fa5d00ecedeb37963f5e8c0763ef6d1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 13 Jan 2016 13:28:39 -0500 Subject: [PATCH 234/359] Refactor config loading to move version check into ConfigFile. Adds the cached_property package. Signed-off-by: Daniel Nephin --- compose/config/config.py | 92 +++++++++++++++++++----------------- compose/config/validation.py | 11 ++--- requirements.txt | 1 + setup.py | 1 + 4 files changed, 55 insertions(+), 50 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 82ff40aa..203c4e9e 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -10,6 +10,7 @@ from collections import namedtuple import six import yaml +from cached_property import cached_property from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference @@ -119,11 +120,23 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def from_filename(cls, filename): return cls(filename, load_yaml(filename)) - def get_service_dicts(self, version): - return self.config if version == 1 else self.config.get('services', {}) + @cached_property + def version(self): + if self.config is None: + return 1 + version = self.config.get('version', 1) + if isinstance(version, dict): + log.warn("Unexpected type for field 'version', in file {} assuming " + "version is the name of a service, and defaulting to " + "Compose file version 1".format(self.filename)) + return 1 + return version - def get_volumes(self, version): - return {} if version == 1 else self.config.get('volumes', {}) + def get_service_dicts(self): + return self.config if self.version == 1 else self.config.get('services', {}) + + def get_volumes(self): + return {} if self.version == 1 else self.config.get('volumes', {}) class Config(namedtuple('_Config', 'version services volumes')): @@ -168,32 +181,24 @@ def find(base_dir, filenames): [ConfigFile.from_filename(f) for f in filenames]) -def get_config_version(config_details): - def get_version(config): - if config.config is None: - return 1 - version = config.config.get('version', 1) - if isinstance(version, dict): - # in that case 'version' is probably a service name, so assume - # this is a legacy (version=1) file - version = 1 - return version - +def validate_config_version(config_details): main_file = config_details.config_files[0] validate_top_level_object(main_file) - version = get_version(main_file) for next_file in config_details.config_files[1:]: validate_top_level_object(next_file) - next_file_version = get_version(next_file) - if version != next_file_version and next_file_version is not None: + if main_file.version != next_file.version: raise ConfigurationError( - "Version mismatch: main file {0} specifies version {1} but " + "Version mismatch: file {0} specifies version {1} but " "extension file {2} uses version {3}".format( - main_file.filename, version, next_file.filename, next_file_version - ) - ) - return version + main_file.filename, + main_file.version, + next_file.filename, + next_file.version)) + + if main_file.version not in COMPOSEFILE_VERSIONS: + raise ConfigurationError( + 'Invalid Compose file version: {0}'.format(main_file.version)) def get_default_config_files(base_dir): @@ -242,23 +247,22 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ - version = get_config_version(config_details) - if version not in COMPOSEFILE_VERSIONS: - raise ConfigurationError('Invalid config version provided: {0}'.format(version)) + validate_config_version(config_details) processed_files = [ - process_config_file(config_file, version=version) + process_config_file(config_file) for config_file in config_details.config_files ] config_details = config_details._replace(config_files=processed_files) + main_file = config_details.config_files[0] volumes = load_volumes(config_details.config_files) service_dicts = load_services( config_details.working_dir, - config_details.config_files[0].filename, - [file.get_service_dicts(version) for file in config_details.config_files], - version) - return Config(version, service_dicts, volumes) + main_file.filename, + [file.get_service_dicts() for file in config_details.config_files], + main_file.version) + return Config(main_file.version, service_dicts, volumes) def load_volumes(config_files): @@ -328,27 +332,28 @@ def load_services(working_dir, filename, service_configs, version): return build_services(service_config) -def process_config_file(config_file, version, service_name=None): - service_dicts = config_file.get_service_dicts(version) - validate_top_level_service_objects( - config_file.filename, service_dicts - ) +def process_config_file(config_file, service_name=None): + service_dicts = config_file.get_service_dicts() + validate_top_level_service_objects(config_file.filename, service_dicts) + + # TODO: interpolate config in volumes/network sections as well interpolated_config = interpolate_environment_variables(service_dicts) - if version == 2: + + if config_file.version == 2: processed_config = dict(config_file.config) processed_config.update({'services': interpolated_config}) - if version == 1: + if config_file.version == 1: processed_config = interpolated_config - validate_against_fields_schema( - processed_config, config_file.filename, version - ) + + config_file = config_file._replace(config=processed_config) + validate_against_fields_schema(config_file) if service_name and service_name not in processed_config: raise ConfigurationError( "Cannot extend service '{}' in {}: Service not found".format( service_name, config_file.filename)) - return config_file._replace(config=processed_config) + return config_file class ServiceExtendsResolver(object): @@ -385,8 +390,7 @@ class ServiceExtendsResolver(object): extended_file = process_config_file( ConfigFile.from_filename(config_path), - version=self.version, service_name=service_name - ) + service_name=service_name) service_config = extended_file.config[service_name] return config_path, service_config, service_name diff --git a/compose/config/validation.py b/compose/config/validation.py index 74ae5c9c..0bf75691 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -105,8 +105,7 @@ def validate_top_level_service_objects(filename, service_dicts): def validate_top_level_object(config_file): if not isinstance(config_file.config, dict): raise ConfigurationError( - "Top level object in '{}' needs to be an object not '{}'. Check " - "that you have defined a service at the top level.".format( + "Top level object in '{}' needs to be an object not '{}'.".format( config_file.filename, type(config_file.config))) @@ -291,13 +290,13 @@ def process_errors(errors, service_name=None): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config, filename, version): - schema_filename = "fields_schema_v{0}.json".format(version) +def validate_against_fields_schema(config_file): + schema_filename = "fields_schema_v{0}.json".format(config_file.version) _validate_against_schema( - config, + config_file.config, schema_filename, format_checker=["ports", "expose", "bool-value-in-mapping"], - filename=filename) + filename=config_file.filename) def validate_against_service_schema(config, service_name, version): diff --git a/requirements.txt b/requirements.txt index 313c6b7a..563baa10 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,5 @@ PyYAML==3.11 +cached-property==1.2.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/setup.py b/setup.py index bd6f201d..f159d2b1 100644 --- a/setup.py +++ b/setup.py @@ -28,6 +28,7 @@ def find_version(*file_paths): install_requires = [ + 'cached-property >= 1.2.0', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', 'requests >= 2.6.1, < 2.8', From b76dc1e05ecfa2ab95ef58ae62072afbf354abda Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 13 Jan 2016 14:41:34 -0500 Subject: [PATCH 235/359] Require volumes_from a container to be explicit in V2 config. Signed-off-by: Daniel Nephin --- compose/config/config.py | 16 +++++--- compose/config/types.py | 44 +++++++++++++++++++-- compose/project.py | 52 ++++++++++++++----------- setup.py | 2 +- tests/integration/project_test.py | 2 +- tests/integration/service_test.py | 4 +- tests/unit/config/config_test.py | 2 +- tests/unit/config/sort_services_test.py | 6 +-- tests/unit/config/types_test.py | 45 +++++++++++++++++++++ tests/unit/project_test.py | 17 ++++---- tests/unit/service_test.py | 29 +++++++++++--- 11 files changed, 166 insertions(+), 53 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 203c4e9e..8383fb5c 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -292,7 +292,7 @@ def load_volumes(config_files): def load_services(working_dir, filename, service_configs, version): - def build_service(service_name, service_dict): + def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( working_dir, filename, @@ -305,13 +305,17 @@ def load_services(working_dir, filename, service_configs, version): validate_against_service_schema(service_dict, service_config.name, version) validate_paths(service_dict) - service_dict = finalize_service(service_config._replace(config=service_dict)) + service_dict = finalize_service( + service_config._replace(config=service_dict), + service_names, + version) service_dict['name'] = service_config.name return service_dict def build_services(service_config): + service_names = service_config.keys() return sort_service_dicts([ - build_service(name, service_dict) + build_service(name, service_dict, service_names) for name, service_dict in service_config.items() ]) @@ -504,7 +508,7 @@ def process_service(service_config): return service_dict -def finalize_service(service_config): +def finalize_service(service_config, service_names, version): service_dict = dict(service_config.config) if 'environment' in service_dict or 'env_file' in service_dict: @@ -513,7 +517,9 @@ def finalize_service(service_config): if 'volumes_from' in service_dict: service_dict['volumes_from'] = [ - VolumeFromSpec.parse(vf) for vf in service_dict['volumes_from']] + VolumeFromSpec.parse(vf, service_names, version) + for vf in service_dict['volumes_from'] + ] if 'volumes' in service_dict: service_dict['volumes'] = [ diff --git a/compose/config/types.py b/compose/config/types.py index cec1f6cf..64e356fa 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -11,10 +11,16 @@ from compose.config.errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM -class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode')): +class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode type')): + + # TODO: drop service_names arg when v1 is removed + @classmethod + def parse(cls, volume_from_config, service_names, version): + func = cls.parse_v1 if version == 1 else cls.parse_v2 + return func(service_names, volume_from_config) @classmethod - def parse(cls, volume_from_config): + def parse_v1(cls, service_names, volume_from_config): parts = volume_from_config.split(':') if len(parts) > 2: raise ConfigurationError( @@ -27,7 +33,39 @@ class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode')): else: source, mode = parts - return cls(source, mode) + type = 'service' if source in service_names else 'container' + return cls(source, mode, type) + + @classmethod + def parse_v2(cls, service_names, volume_from_config): + parts = volume_from_config.split(':') + if len(parts) > 3: + raise ConfigurationError( + "volume_from {} has incorrect format, should be one of " + "'[:]' or " + "'container:[:]'".format(volume_from_config)) + + if len(parts) == 1: + source = parts[0] + return cls(source, 'rw', 'service') + + if len(parts) == 2: + if parts[0] == 'container': + type, source = parts + return cls(source, 'rw', type) + + source, mode = parts + return cls(source, mode, 'service') + + if len(parts) == 3: + type, source, mode = parts + if type not in ('service', 'container'): + raise ConfigurationError( + "Unknown volumes_from type '{}' in '{}'".format( + type, + volume_from_config)) + + return cls(source, mode, type) def parse_restart_spec(restart_config): diff --git a/compose/project.py b/compose/project.py index e882713c..06f9eaea 100644 --- a/compose/project.py +++ b/compose/project.py @@ -60,7 +60,7 @@ class Project(object): for service_dict in config_data.services: links = project.get_links(service_dict) - volumes_from = project.get_volumes_from(service_dict) + volumes_from = get_volumes_from(project, service_dict) net = project.get_net(service_dict) project.services.append( @@ -162,28 +162,6 @@ class Project(object): del service_dict['links'] return links - def get_volumes_from(self, service_dict): - volumes_from = [] - if 'volumes_from' in service_dict: - for volume_from_spec in service_dict.get('volumes_from', []): - # Get service - try: - service = self.get_service(volume_from_spec.source) - volume_from_spec = volume_from_spec._replace(source=service) - except NoSuchService: - try: - container = Container.from_id(self.client, volume_from_spec.source) - volume_from_spec = volume_from_spec._replace(source=container) - except APIError: - raise ConfigurationError( - 'Service "%s" mounts volumes from "%s", which is ' - 'not the name of a service or container.' % ( - service_dict['name'], - volume_from_spec.source)) - volumes_from.append(volume_from_spec) - del service_dict['volumes_from'] - return volumes_from - def get_net(self, service_dict): net = service_dict.pop('net', None) if not net: @@ -465,6 +443,34 @@ def remove_links(service_dicts): del s['links'] +def get_volumes_from(project, service_dict): + volumes_from = service_dict.pop('volumes_from', None) + if not volumes_from: + return [] + + def build_volume_from(spec): + if spec.type == 'service': + try: + return spec._replace(source=project.get_service(spec.source)) + except NoSuchService: + pass + + if spec.type == 'container': + try: + container = Container.from_id(project.client, spec.source) + return spec._replace(source=container) + except APIError: + pass + + raise ConfigurationError( + "Service \"{}\" mounts volumes from \"{}\", which is not the name " + "of a service or container.".format( + service_dict['name'], + spec.source)) + + return [build_volume_from(vf) for vf in volumes_from] + + class NoSuchService(Exception): def __init__(self, name): self.name = name diff --git a/setup.py b/setup.py index f159d2b1..b365e05b 100644 --- a/setup.py +++ b/setup.py @@ -28,7 +28,7 @@ def find_version(*file_paths): install_requires = [ - 'cached-property >= 1.2.0', + 'cached-property >= 1.2.0, < 2', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', 'requests >= 2.6.1, < 2.8', diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 467eb786..535a9775 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -81,7 +81,7 @@ class ProjectTest(DockerClientTestCase): ) db = project.get_service('db') data = project.get_service('data') - self.assertEqual(db.volumes_from, [VolumeFromSpec(data, 'rw')]) + self.assertEqual(db.volumes_from, [VolumeFromSpec(data, 'rw', 'service')]) def test_volumes_from_container(self): data_container = Container.create( diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 4818e47a..4ba1b635 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -224,8 +224,8 @@ class ServiceTest(DockerClientTestCase): host_service = self.create_service( 'host', volumes_from=[ - VolumeFromSpec(volume_service, 'rw'), - VolumeFromSpec(volume_container_2, 'rw') + VolumeFromSpec(volume_service, 'rw', 'service'), + VolumeFromSpec(volume_container_2, 'rw', 'container') ] ) host_container = host_service.create_container() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 77c55ad9..f0d432d5 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -19,7 +19,7 @@ from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest -DEFAULT_VERSION = 2 +DEFAULT_VERSION = V2 = 2 V1 = 1 diff --git a/tests/unit/config/sort_services_test.py b/tests/unit/config/sort_services_test.py index c2ebbc67..ebe444fe 100644 --- a/tests/unit/config/sort_services_test.py +++ b/tests/unit/config/sort_services_test.py @@ -77,7 +77,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'parent', - 'volumes_from': [VolumeFromSpec('child', 'rw')] + 'volumes_from': [VolumeFromSpec('child', 'rw', 'service')] }, { 'links': ['parent'], @@ -120,7 +120,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'parent', - 'volumes_from': [VolumeFromSpec('child', 'ro')] + 'volumes_from': [VolumeFromSpec('child', 'ro', 'service')] }, { 'name': 'child' @@ -145,7 +145,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'two', - 'volumes_from': [VolumeFromSpec('one', 'rw')] + 'volumes_from': [VolumeFromSpec('one', 'rw', 'service')] }, { 'name': 'one' diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 245b854f..214a0e31 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -5,8 +5,11 @@ import pytest from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts +from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM +from tests.unit.config.config_test import V1 +from tests.unit.config.config_test import V2 def test_parse_extra_hosts_list(): @@ -67,3 +70,45 @@ class TestVolumeSpec(object): "/opt/shiny/config", "ro" ) + + +class TestVolumesFromSpec(object): + + services = ['servicea', 'serviceb'] + + def test_parse_v1_from_service(self): + volume_from = VolumeFromSpec.parse('servicea', self.services, V1) + assert volume_from == VolumeFromSpec('servicea', 'rw', 'service') + + def test_parse_v1_from_container(self): + volume_from = VolumeFromSpec.parse('foo:ro', self.services, V1) + assert volume_from == VolumeFromSpec('foo', 'ro', 'container') + + def test_parse_v1_invalid(self): + with pytest.raises(ConfigurationError): + VolumeFromSpec.parse('unknown:format:ro', self.services, V1) + + def test_parse_v2_from_service(self): + volume_from = VolumeFromSpec.parse('servicea', self.services, V2) + assert volume_from == VolumeFromSpec('servicea', 'rw', 'service') + + def test_parse_v2_from_service_with_mode(self): + volume_from = VolumeFromSpec.parse('servicea:ro', self.services, V2) + assert volume_from == VolumeFromSpec('servicea', 'ro', 'service') + + def test_parse_v2_from_container(self): + volume_from = VolumeFromSpec.parse('container:foo', self.services, V2) + assert volume_from == VolumeFromSpec('foo', 'rw', 'container') + + def test_parse_v2_from_container_with_mode(self): + volume_from = VolumeFromSpec.parse('container:foo:ro', self.services, V2) + assert volume_from == VolumeFromSpec('foo', 'ro', 'container') + + def test_parse_v2_invalid_type(self): + with pytest.raises(ConfigurationError) as exc: + VolumeFromSpec.parse('bogus:foo:ro', self.services, V2) + assert "Unknown volumes_from type 'bogus'" in exc.exconly() + + def test_parse_v2_invalid(self): + with pytest.raises(ConfigurationError): + VolumeFromSpec.parse('unknown:format:ro', self.services, V2) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f63135ae..861f9656 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -165,10 +165,10 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('aaa', 'rw')] + 'volumes_from': [VolumeFromSpec('aaa', 'rw', 'container')] } ], None), self.mock_client) - self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"]) + assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] def test_use_volumes_from_service_no_container(self): container_name = 'test_vol_1' @@ -188,10 +188,10 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('vol', 'rw')] + 'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')] } ], None), self.mock_client) - self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"]) + assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] @@ -204,16 +204,17 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('vol', 'rw')] + 'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')] } ], None), None) with mock.patch.object(Service, 'containers') as mock_return: mock_return.return_value = [ mock.Mock(id=container_id, spec=Container) for container_id in container_ids] - self.assertEqual( - project.get_service('test')._get_volumes_from(), - [container_ids[0] + ':rw']) + assert ( + project.get_service('test')._get_volumes_from() == + [container_ids[0] + ':rw'] + ) def test_events(self): services = [Service(name='web'), Service(name='db')] diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 63cf658e..9845ebc6 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -70,7 +70,11 @@ class ServiceTest(unittest.TestCase): service = Service( 'test', image='foo', - volumes_from=[VolumeFromSpec(mock.Mock(id=container_id, spec=Container), 'rw')]) + volumes_from=[ + VolumeFromSpec( + mock.Mock(id=container_id, spec=Container), + 'rw', + 'container')]) self.assertEqual(service._get_volumes_from(), [container_id + ':rw']) @@ -79,7 +83,11 @@ class ServiceTest(unittest.TestCase): service = Service( 'test', image='foo', - volumes_from=[VolumeFromSpec(mock.Mock(id=container_id, spec=Container), 'ro')]) + volumes_from=[ + VolumeFromSpec( + mock.Mock(id=container_id, spec=Container), + 'ro', + 'container')]) self.assertEqual(service._get_volumes_from(), [container_id + ':ro']) @@ -90,7 +98,10 @@ class ServiceTest(unittest.TestCase): mock.Mock(id=container_id, spec=Container) for container_id in container_ids ] - service = Service('test', volumes_from=[VolumeFromSpec(from_service, 'rw')], image='foo') + service = Service( + 'test', + volumes_from=[VolumeFromSpec(from_service, 'rw', 'service')], + image='foo') self.assertEqual(service._get_volumes_from(), [container_ids[0] + ":rw"]) @@ -102,7 +113,10 @@ class ServiceTest(unittest.TestCase): mock.Mock(id=container_id.split(':')[0], spec=Container) for container_id in container_ids ] - service = Service('test', volumes_from=[VolumeFromSpec(from_service, mode)], image='foo') + service = Service( + 'test', + volumes_from=[VolumeFromSpec(from_service, mode, 'service')], + image='foo') self.assertEqual(service._get_volumes_from(), [container_ids[0]]) @@ -113,7 +127,10 @@ class ServiceTest(unittest.TestCase): from_service.create_container.return_value = mock.Mock( id=container_id, spec=Container) - service = Service('test', image='foo', volumes_from=[VolumeFromSpec(from_service, 'rw')]) + service = Service( + 'test', + image='foo', + volumes_from=[VolumeFromSpec(from_service, 'rw', 'service')]) self.assertEqual(service._get_volumes_from(), [container_id + ':rw']) from_service.create_container.assert_called_once_with() @@ -389,7 +406,7 @@ class ServiceTest(unittest.TestCase): client=self.mock_client, net=ServiceNet(Service('other')), links=[(Service('one'), 'one')], - volumes_from=[VolumeFromSpec(Service('two'), 'rw')]) + volumes_from=[VolumeFromSpec(Service('two'), 'rw', 'service')]) config_dict = service.config_dict() expected = { From 69ed5f9c48a5fe419ac041239659fe69304fe4a9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 17 Dec 2015 17:49:48 +0000 Subject: [PATCH 236/359] Specify networks in Compose file There's not yet a proper way for services to join networks Signed-off-by: Aanand Prasad --- compose/cli/main.py | 3 +- compose/config/config.py | 43 +-- compose/config/fields_schema_v2.json | 13 + compose/network.py | 57 ++++ compose/project.py | 127 ++++----- tests/acceptance/cli_test.py | 19 +- tests/fixtures/networks/docker-compose.yml | 7 + .../no-links-composefile/docker-compose.yml | 9 + tests/integration/project_test.py | 113 ++++---- tests/unit/project_test.py | 248 +++++++++++------- 10 files changed, 400 insertions(+), 239 deletions(-) create mode 100644 compose/network.py create mode 100644 tests/fixtures/networks/docker-compose.yml create mode 100644 tests/fixtures/no-links-composefile/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 9957d391..62abcb10 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -696,8 +696,7 @@ def run_one_off_container(container_options, project, service, options): start_deps=True, strategy=ConvergenceStrategy.never) - if project.use_networking: - project.ensure_network_exists() + project.initialize_networks() container = service.create_container( quiet=True, diff --git a/compose/config/config.py b/compose/config/config.py index 8383fb5c..c8d93faf 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -139,14 +139,16 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): return {} if self.version == 1 else self.config.get('volumes', {}) -class Config(namedtuple('_Config', 'version services volumes')): +class Config(namedtuple('_Config', 'version services volumes networks')): """ :param version: configuration version :type version: int :param services: List of service description dictionaries :type services: :class:`list` - :param volumes: List of volume description dictionaries - :type volumes: :class:`list` + :param volumes: Dictionary mapping volume names to description dictionaries + :type volumes: :class:`dict` + :param networks: Dictionary mapping network names to description dictionaries + :type networks: :class:`dict` """ @@ -256,39 +258,44 @@ def load(config_details): config_details = config_details._replace(config_files=processed_files) main_file = config_details.config_files[0] - volumes = load_volumes(config_details.config_files) + volumes = load_mapping(config_details.config_files, 'volumes', 'Volume') + networks = load_mapping(config_details.config_files, 'networks', 'Network') service_dicts = load_services( config_details.working_dir, main_file.filename, [file.get_service_dicts() for file in config_details.config_files], main_file.version) - return Config(main_file.version, service_dicts, volumes) + return Config(main_file.version, service_dicts, volumes, networks) -def load_volumes(config_files): - volumes = {} +def load_mapping(config_files, key, entity_type): + mapping = {} + for config_file in config_files: - for name, volume_config in config_file.get_volumes().items(): - volumes[name] = volume_config or {} - if not volume_config: + for name, config in config_file.config.get(key, {}).items(): + mapping[name] = config or {} + if not config: continue - external = volume_config.get('external') + external = config.get('external') if external: - if len(volume_config.keys()) > 1: + if len(config.keys()) > 1: raise ConfigurationError( - 'Volume {0} declared as external but specifies' - ' additional attributes ({1}). '.format( + '{} {} declared as external but specifies' + ' additional attributes ({}). '.format( + entity_type, name, - ', '.join([k for k in volume_config.keys() if k != 'external']) + ', '.join([k for k in config.keys() if k != 'external']) ) ) if isinstance(external, dict): - volume_config['external_name'] = external.get('name') + config['external_name'] = external.get('name') else: - volume_config['external_name'] = name + config['external_name'] = name - return volumes + mapping[name] = config + + return mapping def load_services(working_dir, filename, service_configs, version): diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 25126ed1..b0f304e8 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -17,6 +17,15 @@ }, "additionalProperties": false }, + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, "volumes": { "id": "#/properties/volumes", "type": "object", @@ -30,6 +39,10 @@ }, "definitions": { + "network": { + "id": "#/definitions/network", + "type": "object" + }, "volume": { "id": "#/definitions/volume", "type": ["object", "null"], diff --git a/compose/network.py b/compose/network.py new file mode 100644 index 00000000..0b4e40c6 --- /dev/null +++ b/compose/network.py @@ -0,0 +1,57 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import logging + +from docker.errors import NotFound + +from .config import ConfigurationError + + +log = logging.getLogger(__name__) + + +class Network(object): + def __init__(self, client, project, name, driver=None, driver_opts=None): + self.client = client + self.project = project + self.name = name + self.driver = driver + self.driver_opts = driver_opts + + def ensure(self): + try: + data = self.inspect() + if self.driver and data['Driver'] != self.driver: + raise ConfigurationError( + 'Network {} needs to be recreated - driver has changed' + .format(self.full_name)) + if data['Options'] != (self.driver_opts or {}): + raise ConfigurationError( + 'Network {} needs to be recreated - options have changed' + .format(self.full_name)) + except NotFound: + driver_name = 'the default driver' + if self.driver: + driver_name = 'driver "{}"'.format(self.driver) + + log.info( + 'Creating network "{}" with {}' + .format(self.full_name, driver_name) + ) + + self.client.create_network( + self.full_name, self.driver, self.driver_opts + ) + + def remove(self): + # TODO: don't remove external networks + log.info("Removing network {}".format(self.full_name)) + self.client.remove_network(self.full_name) + + def inspect(self): + return self.client.inspect_network(self.full_name) + + @property + def full_name(self): + return '{0}_{1}'.format(self.project, self.name) diff --git a/compose/project.py b/compose/project.py index 3e7c5afd..10d457a7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -17,6 +17,7 @@ from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .container import Container +from .network import Network from .service import ContainerNet from .service import ConvergenceStrategy from .service import Net @@ -33,12 +34,14 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None): + def __init__(self, name, services, client, networks=None, volumes=None, + use_networking=False, network_driver=None): self.name = name self.services = services self.client = client self.use_networking = use_networking self.network_driver = network_driver + self.networks = networks or [] self.volumes = volumes or [] def labels(self, one_off=False): @@ -55,9 +58,6 @@ class Project(object): use_networking = (config_data.version and config_data.version >= 2) project = cls(name, [], client, use_networking=use_networking) - if use_networking: - remove_links(config_data.services) - for service_dict in config_data.services: links = project.get_links(service_dict) volumes_from = get_volumes_from(project, service_dict) @@ -72,6 +72,16 @@ class Project(object): net=net, volumes_from=volumes_from, **service_dict)) + + if config_data.networks: + for network_name, data in config_data.networks.items(): + project.networks.append( + Network( + client=client, project=name, name=network_name, + driver=data.get('driver'), driver_opts=data.get('driver_opts') + ) + ) + if config_data.volumes: for vol_name, data in config_data.volumes.items(): project.volumes.append( @@ -82,6 +92,7 @@ class Project(object): external_name=data.get('external_name') ) ) + return project @property @@ -124,20 +135,18 @@ class Project(object): Raises NoSuchService if any of the named services do not exist. """ if service_names is None or len(service_names) == 0: - return self.get_services( - service_names=self.service_names, - include_deps=include_deps - ) - else: - unsorted = [self.get_service(name) for name in service_names] - services = [s for s in self.services if s in unsorted] + service_names = self.service_names - if include_deps: - services = reduce(self._inject_deps, services, []) + unsorted = [self.get_service(name) for name in service_names] + services = [s for s in self.services if s in unsorted] - uniques = [] - [uniques.append(s) for s in services if s not in uniques] - return uniques + if include_deps: + services = reduce(self._inject_deps, services, []) + + uniques = [] + [uniques.append(s) for s in services if s not in uniques] + + return uniques def get_services_without_duplicate(self, service_names=None, include_deps=False): services = self.get_services(service_names, include_deps) @@ -166,7 +175,7 @@ class Project(object): net = service_dict.pop('net', None) if not net: if self.use_networking: - return Net(self.default_network_name) + return Net(self.default_network.full_name) return Net(None) net_name = get_service_name_from_net(net) @@ -251,7 +260,7 @@ class Project(object): def down(self, remove_image_type, include_volumes): self.stop() self.remove_stopped(v=include_volumes) - self.remove_network() + self.remove_default_network() if include_volumes: self.remove_volumes() @@ -262,10 +271,34 @@ class Project(object): for service in self.get_services(): service.remove_image(remove_image_type) + def remove_default_network(self): + if not self.use_networking: + return + if self.uses_default_network(): + self.default_network.remove() + def remove_volumes(self): for volume in self.volumes: volume.remove() + def initialize_networks(self): + networks = self.networks + if self.uses_default_network(): + networks.append(self.default_network) + + for network in networks: + network.ensure() + + def uses_default_network(self): + return any( + service.net.mode == self.default_network.full_name + for service in self.services + ) + + @property + def default_network(self): + return Network(client=self.client, project=self.name, name='default') + def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -335,9 +368,7 @@ class Project(object): plans = self._get_convergence_plans(services, strategy) - if self.use_networking and self.uses_default_network(): - self.ensure_network_exists() - + self.initialize_networks() self.initialize_volumes() return [ @@ -395,40 +426,6 @@ class Project(object): return [c for c in containers if matches_service_names(c)] - def get_network(self): - try: - return self.client.inspect_network(self.default_network_name) - except NotFound: - return None - - def ensure_network_exists(self): - # TODO: recreate network if driver has changed? - if self.get_network() is None: - driver_name = 'the default driver' - if self.network_driver: - driver_name = 'driver "{}"'.format(self.network_driver) - - log.info( - 'Creating network "{}" with {}' - .format(self.default_network_name, driver_name) - ) - self.client.create_network(self.default_network_name, driver=self.network_driver) - - def remove_network(self): - if not self.use_networking: - return - network = self.get_network() - if network: - log.info("Removing network %s", self.default_network_name) - self.client.remove_network(network['Id']) - - def uses_default_network(self): - return any(service.net.mode == self.default_network_name for service in self.services) - - @property - def default_network_name(self): - return '{}_default'.format(self.name) - def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() @@ -444,26 +441,6 @@ class Project(object): return acc + dep_services -def remove_links(service_dicts): - services_with_links = [s for s in service_dicts if 'links' in s] - if not services_with_links: - return - - if len(services_with_links) == 1: - prefix = '"{}" defines'.format(services_with_links[0]['name']) - else: - prefix = 'Some services ({}) define'.format( - ", ".join('"{}"'.format(s['name']) for s in services_with_links)) - - log.warn( - '\n{} links, which are not compatible with Docker networking and will be ignored.\n' - 'Future versions of Docker will not support links - you should remove them for ' - 'forwards-compatibility.\n'.format(prefix)) - - for s in services_with_links: - del s['links'] - - def get_volumes_from(project, service_dict): volumes_from = service_dict.pop('volumes_from', None) if not volumes_from: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index cb04918b..1e31988b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -354,7 +354,7 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d'], None) - networks = self.client.networks(names=[self.project.default_network_name]) + networks = self.client.networks(names=[self.project.default_network.full_name]) self.assertEqual(len(networks), 0) for service in self.project.get_services(): @@ -371,7 +371,7 @@ class CLITestCase(DockerClientTestCase): services = self.project.get_services() - networks = self.client.networks(names=[self.project.default_network_name]) + networks = self.client.networks(names=[self.project.default_network.full_name]) for n in networks: self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) @@ -388,6 +388,19 @@ class CLITestCase(DockerClientTestCase): web_container = self.project.get_service('simple').containers()[0] self.assertFalse(web_container.get('HostConfig.Links')) + def test_up_with_networks(self): + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['up', '-d'], None) + + networks = self.client.networks(names=[ + '{}_{}'.format(self.project.name, n) + for n in ['foo', 'bar']]) + + self.assertEqual(len(networks), 2) + + for net in networks: + self.assertEqual(net['Driver'], 'bridge') + def test_up_with_links_is_invalid(self): self.base_dir = 'tests/fixtures/v2-simple' @@ -698,7 +711,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['run', 'simple', 'true'], None) service = self.project.get_service('simple') container, = service.containers(stopped=True, one_off=True) - networks = self.client.networks(names=[self.project.default_network_name]) + networks = self.client.networks(names=[self.project.default_network.full_name]) for n in networks: self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml new file mode 100644 index 00000000..c0795526 --- /dev/null +++ b/tests/fixtures/networks/docker-compose.yml @@ -0,0 +1,7 @@ +version: 2 + +networks: + foo: + driver: + + bar: {} diff --git a/tests/fixtures/no-links-composefile/docker-compose.yml b/tests/fixtures/no-links-composefile/docker-compose.yml new file mode 100644 index 00000000..75a6a085 --- /dev/null +++ b/tests/fixtures/no-links-composefile/docker-compose.yml @@ -0,0 +1,9 @@ +db: + image: busybox:latest + command: top +web: + image: busybox:latest + command: top +console: + image: busybox:latest + command: top diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 535a9775..ef8a084b 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -14,7 +14,6 @@ from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project from compose.service import ConvergenceStrategy -from compose.service import Net def build_service_dicts(service_config): @@ -104,21 +103,6 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) - def test_get_network_does_not_exist(self): - project = Project('composetest', [], self.client) - assert project.get_network() is None - - def test_get_network(self): - project_name = 'network_does_exist' - network_name = '{}_default'.format(project_name) - - project = Project(project_name, [], self.client) - self.client.create_network(network_name) - self.addCleanup(self.client.remove_network, network_name) - - assert isinstance(project.get_network(), dict) - assert project.get_network()['Name'] == network_name - def test_net_from_service(self): project = Project.from_config( name='composetest', @@ -473,18 +457,6 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('data').containers(stopped=True)), 1) self.assertEqual(len(project.get_service('console').containers()), 0) - def test_project_up_with_custom_network(self): - network_name = 'composetest-custom' - - self.client.create_network(network_name) - self.addCleanup(self.client.remove_network, network_name) - - web = self.create_service('web', net=Net(network_name)) - project = Project('composetest', [web], self.client, use_networking=True) - project.up() - - assert project.get_network() is None - def test_unscale_after_restart(self): web = self.create_service('web') project = Project('composetest', [web], self.client) @@ -510,15 +482,50 @@ class ProjectTest(DockerClientTestCase): service = project.get_service('web') self.assertEqual(len(service.containers()), 1) + def test_project_up_networks(self): + config_data = config.Config( + version=2, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top', + }], + volumes={}, + networks={ + 'foo': {'driver': 'bridge'}, + 'bar': {'driver': None}, + 'baz': {}, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + self.assertEqual(len(project.containers()), 1) + + for net_name in ['foo', 'bar', 'baz']: + full_net_name = 'composetest_{}'.format(net_name) + network_data = self.client.inspect_network(full_net_name) + self.assertEqual(network_data['Name'], full_net_name) + + foo_data = self.client.inspect_network('composetest_foo') + self.assertEqual(foo_data['Driver'], 'bridge') + def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {'driver': 'local'}} + }], + volumes={vol_name: {'driver': 'local'}}, + networks={}, ) project = Project.from_config( @@ -587,11 +594,14 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {}} + }], + volumes={vol_name: {}}, + networks={}, ) project = Project.from_config( @@ -608,11 +618,14 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {}} + }], + volumes={vol_name: {}}, + networks={}, ) project = Project.from_config( @@ -629,11 +642,14 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {'driver': 'foobar'}} + }], + volumes={vol_name: {'driver': 'foobar'}}, + networks={}, ) project = Project.from_config( @@ -648,11 +664,14 @@ class ProjectTest(DockerClientTestCase): full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={vol_name: {'driver': 'local'}} + }], + volumes={vol_name: {'driver': 'local'}}, + networks={}, ) project = Project.from_config( name='composetest', @@ -683,13 +702,16 @@ class ProjectTest(DockerClientTestCase): full_vol_name = 'composetest_{0}'.format(vol_name) self.client.create_volume(vol_name) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={ + }], + volumes={ vol_name: {'external': True, 'external_name': vol_name} - } + }, + networks=None, ) project = Project.from_config( name='composetest', @@ -704,13 +726,16 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( - version=2, services=[{ + version=2, + services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' - }], volumes={ + }], + volumes={ vol_name: {'external': True, 'external_name': vol_name} - } + }, + networks=None, ) project = Project.from_config( name='composetest', diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 861f9656..470e51ad 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -21,35 +21,27 @@ class ProjectTest(unittest.TestCase): def setUp(self): self.mock_client = mock.create_autospec(docker.Client) - def test_from_dict(self): - project = Project.from_config('composetest', Config(None, [ - { - 'name': 'web', - 'image': 'busybox:latest' - }, - { - 'name': 'db', - 'image': 'busybox:latest' - }, - ], None), None) - self.assertEqual(len(project.services), 2) - self.assertEqual(project.get_service('web').name, 'web') - self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') - self.assertEqual(project.get_service('db').name, 'db') - self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') - def test_from_config(self): - config = Config(None, [ - { - 'name': 'web', - 'image': 'busybox:latest', - }, - { - 'name': 'db', - 'image': 'busybox:latest', - }, - ], None) - project = Project.from_config('composetest', config, None) + config = Config( + version=None, + services=[ + { + 'name': 'web', + 'image': 'busybox:latest', + }, + { + 'name': 'db', + 'image': 'busybox:latest', + }, + ], + networks=None, + volumes=None, + ) + project = Project.from_config( + name='composetest', + config_data=config, + client=None, + ) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -58,16 +50,21 @@ class ProjectTest(unittest.TestCase): self.assertFalse(project.use_networking) def test_from_config_v2(self): - config = Config(2, [ - { - 'name': 'web', - 'image': 'busybox:latest', - }, - { - 'name': 'db', - 'image': 'busybox:latest', - }, - ], None) + config = Config( + version=2, + services=[ + { + 'name': 'web', + 'image': 'busybox:latest', + }, + { + 'name': 'db', + 'image': 'busybox:latest', + }, + ], + networks=None, + volumes=None, + ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) self.assertTrue(project.use_networking) @@ -161,13 +158,20 @@ class ProjectTest(unittest.TestCase): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_config('test', Config(None, [ - { - 'name': 'test', - 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('aaa', 'rw', 'container')] - } - ], None), self.mock_client) + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=None, + services=[{ + 'name': 'test', + 'image': 'busybox:latest', + 'volumes_from': [VolumeFromSpec('aaa', 'rw', 'container')] + }], + networks=None, + volumes=None, + ), + ) assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] def test_use_volumes_from_service_no_container(self): @@ -180,33 +184,51 @@ class ProjectTest(unittest.TestCase): "Image": 'busybox:latest' } ] - project = Project.from_config('test', Config(None, [ - { - 'name': 'vol', - 'image': 'busybox:latest' - }, - { - 'name': 'test', - 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')] - } - ], None), self.mock_client) + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=None, + services=[ + { + 'name': 'vol', + 'image': 'busybox:latest' + }, + { + 'name': 'test', + 'image': 'busybox:latest', + 'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')] + } + ], + networks=None, + volumes=None, + ), + ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] - project = Project.from_config('test', Config(None, [ - { - 'name': 'vol', - 'image': 'busybox:latest' - }, - { - 'name': 'test', - 'image': 'busybox:latest', - 'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')] - } - ], None), None) + project = Project.from_config( + name='test', + client=None, + config_data=Config( + version=None, + services=[ + { + 'name': 'vol', + 'image': 'busybox:latest' + }, + { + 'name': 'test', + 'image': 'busybox:latest', + 'volumes_from': [VolumeFromSpec('vol', 'rw', 'service')] + } + ], + networks=None, + volumes=None, + ), + ) with mock.patch.object(Service, 'containers') as mock_return: mock_return.return_value = [ mock.Mock(id=container_id, spec=Container) @@ -313,12 +335,21 @@ class ProjectTest(unittest.TestCase): ] def test_net_unset(self): - project = Project.from_config('test', Config(None, [ - { - 'name': 'test', - 'image': 'busybox:latest', - } - ], None), self.mock_client) + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=None, + services=[ + { + 'name': 'test', + 'image': 'busybox:latest', + } + ], + networks=None, + volumes=None, + ), + ) service = project.get_service('test') self.assertEqual(service.net.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) @@ -327,13 +358,22 @@ class ProjectTest(unittest.TestCase): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_config('test', Config(None, [ - { - 'name': 'test', - 'image': 'busybox:latest', - 'net': 'container:aaa' - } - ], None), self.mock_client) + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=None, + services=[ + { + 'name': 'test', + 'image': 'busybox:latest', + 'net': 'container:aaa' + }, + ], + networks=None, + volumes=None, + ), + ) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_id) @@ -347,17 +387,26 @@ class ProjectTest(unittest.TestCase): "Image": 'busybox:latest' } ] - project = Project.from_config('test', Config(None, [ - { - 'name': 'aaa', - 'image': 'busybox:latest' - }, - { - 'name': 'test', - 'image': 'busybox:latest', - 'net': 'container:aaa' - } - ], None), self.mock_client) + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=None, + services=[ + { + 'name': 'aaa', + 'image': 'busybox:latest' + }, + { + 'name': 'test', + 'image': 'busybox:latest', + 'net': 'container:aaa' + }, + ], + networks=None, + volumes=None, + ), + ) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_name) @@ -403,11 +452,16 @@ class ProjectTest(unittest.TestCase): }, } project = Project.from_config( - 'test', - Config(None, [{ - 'name': 'web', - 'image': 'busybox:latest', - }], None), - self.mock_client, + name='test', + client=self.mock_client, + config_data=Config( + version=None, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + }], + networks=None, + volumes=None, + ), ) self.assertEqual([c.id for c in project.containers()], ['1']) From 35e347cf92b13a374ff767653aa8629ccdb64a71 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jan 2016 14:05:30 +0000 Subject: [PATCH 237/359] Disable the use of 'net' in v2 Signed-off-by: Aanand Prasad --- compose/config/service_schema_v2.json | 1 - tests/acceptance/cli_test.py | 24 +++++++++++++++++++ .../fixtures/net-container/docker-compose.yml | 7 ++++++ tests/fixtures/net-container/v2-invalid.yml | 10 ++++++++ 4 files changed, 41 insertions(+), 1 deletion(-) create mode 100644 tests/fixtures/net-container/docker-compose.yml create mode 100644 tests/fixtures/net-container/v2-invalid.yml diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index c911502f..1e54c666 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -89,7 +89,6 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, - "net": {"type": "string"}, "pid": {"type": ["string", "null"]}, "ports": { diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 1e31988b..ceec400c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -422,6 +422,30 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) + def test_up_with_net_is_invalid(self): + self.base_dir = 'tests/fixtures/net-container' + + result = self.dispatch( + ['-f', 'v2-invalid.yml', 'up', '-d'], + returncode=1) + + # TODO: fix validation error messages for v2 files + # assert "Unsupported config option for service 'web': 'net'" in exc.exconly() + assert "Unsupported config option" in result.stderr + + def test_up_with_net_v1(self): + self.base_dir = 'tests/fixtures/net-container' + self.dispatch(['up', '-d'], None) + + bar = self.project.get_service('bar') + bar_container = bar.containers()[0] + + foo = self.project.get_service('foo') + foo_container = foo.containers()[0] + + assert foo_container.get('HostConfig.NetworkMode') == \ + 'container:{}'.format(bar_container.id) + def test_up_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', '--no-deps', 'web'], None) diff --git a/tests/fixtures/net-container/docker-compose.yml b/tests/fixtures/net-container/docker-compose.yml new file mode 100644 index 00000000..b5506e0e --- /dev/null +++ b/tests/fixtures/net-container/docker-compose.yml @@ -0,0 +1,7 @@ +foo: + image: busybox + command: top + net: "container:bar" +bar: + image: busybox + command: top diff --git a/tests/fixtures/net-container/v2-invalid.yml b/tests/fixtures/net-container/v2-invalid.yml new file mode 100644 index 00000000..eac4b5f1 --- /dev/null +++ b/tests/fixtures/net-container/v2-invalid.yml @@ -0,0 +1,10 @@ +version: 2 + +services: + foo: + image: busybox + command: top + bar: + image: busybox + command: top + net: "container:foo" From 3f9038aea9abfde49ee2b8745369d7d8647f3262 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jan 2016 14:15:02 +0000 Subject: [PATCH 238/359] Remove test duplication Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 33 ++++++++++++++------------------- 1 file changed, 14 insertions(+), 19 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ceec400c..52e3d354 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -350,22 +350,7 @@ class CLITestCase(DockerClientTestCase): assert 'simple_1 | simple' in result.stdout assert 'another_1 | another' in result.stdout - def test_up_without_networking(self): - self.base_dir = 'tests/fixtures/links-composefile' - self.dispatch(['up', '-d'], None) - - networks = self.client.networks(names=[self.project.default_network.full_name]) - self.assertEqual(len(networks), 0) - - for service in self.project.get_services(): - containers = service.containers() - self.assertEqual(len(containers), 1) - self.assertNotEqual(containers[0].get('Config.Hostname'), service.name) - - web_container = self.project.get_service('web').containers()[0] - self.assertTrue(web_container.get('HostConfig.Links')) - - def test_up_with_networking(self): + def test_up(self): self.base_dir = 'tests/fixtures/v2-simple' self.dispatch(['up', '-d'], None) @@ -385,9 +370,6 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 1) self.assertIn(containers[0].id, network['Containers']) - web_container = self.project.get_service('simple').containers()[0] - self.assertFalse(web_container.get('HostConfig.Links')) - def test_up_with_networks(self): self.base_dir = 'tests/fixtures/networks' self.dispatch(['up', '-d'], None) @@ -415,13 +397,26 @@ class CLITestCase(DockerClientTestCase): def test_up_with_links_v1(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'web'], None) + + # No network was created + networks = self.client.networks(names=[self.project.default_network.full_name]) + for n in networks: + self.addCleanup(self.client.remove_network, n['Id']) + self.assertEqual(len(networks), 0) + web = self.project.get_service('web') db = self.project.get_service('db') console = self.project.get_service('console') + + # console was not started self.assertEqual(len(web.containers()), 1) self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) + # web has links + web_container = web.containers()[0] + self.assertTrue(web_container.get('HostConfig.Links')) + def test_up_with_net_is_invalid(self): self.base_dir = 'tests/fixtures/net-container' From 3eafdbb01bc9c2e72098aa30eb16f5e29d2228ab Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 13 Jan 2016 17:00:31 +0000 Subject: [PATCH 239/359] Connect services to networks with the 'networks' key Signed-off-by: Aanand Prasad --- compose/cli/main.py | 3 +- compose/config/service_schema_v2.json | 7 ++ compose/network.py | 1 + compose/project.py | 48 ++++++++++--- compose/service.py | 19 ++++-- tests/acceptance/cli_test.py | 68 +++++++++++++++---- tests/fixtures/networks/docker-compose.yml | 20 ++++-- tests/fixtures/networks/missing-network.yml | 10 +++ tests/fixtures/no-services/docker-compose.yml | 5 ++ tests/integration/resilience_test.py | 4 +- tests/integration/service_test.py | 33 ++++----- tests/unit/project_test.py | 55 +++++++++------ 12 files changed, 195 insertions(+), 78 deletions(-) create mode 100644 tests/fixtures/networks/missing-network.yml create mode 100644 tests/fixtures/no-services/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 62abcb10..473c6d60 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -704,7 +704,7 @@ def run_one_off_container(container_options, project, service, options): **container_options) if options['-d']: - container.start() + service.start_container(container) print(container.name) return @@ -716,6 +716,7 @@ def run_one_off_container(container_options, project, service, options): try: try: dockerpty.start(project.client, container.id, interactive=not options['-T']) + service.connect_container_to_networks(container) exit_code = container.wait() except signals.ShutdownException: project.client.stop(container.id) diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 1e54c666..5f4e0478 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -89,6 +89,13 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, + + "networks": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + "pid": {"type": ["string", "null"]}, "ports": { diff --git a/compose/network.py b/compose/network.py index 0b4e40c6..a8f7e918 100644 --- a/compose/network.py +++ b/compose/network.py @@ -11,6 +11,7 @@ from .config import ConfigurationError log = logging.getLogger(__name__) +# TODO: support external networks class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None): self.client = client diff --git a/compose/project.py b/compose/project.py index 10d457a7..292bf2f2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -58,7 +58,21 @@ class Project(object): use_networking = (config_data.version and config_data.version >= 2) project = cls(name, [], client, use_networking=use_networking) + custom_networks = [] + if config_data.networks: + for network_name, data in config_data.networks.items(): + custom_networks.append( + Network( + client=client, project=name, name=network_name, + driver=data.get('driver'), driver_opts=data.get('driver_opts') + ) + ) + for service_dict in config_data.services: + networks = project.get_networks( + service_dict, + custom_networks + [project.default_network]) + links = project.get_links(service_dict) volumes_from = get_volumes_from(project, service_dict) net = project.get_net(service_dict) @@ -68,19 +82,15 @@ class Project(object): client=client, project=name, use_networking=use_networking, + networks=networks, links=links, net=net, volumes_from=volumes_from, **service_dict)) - if config_data.networks: - for network_name, data in config_data.networks.items(): - project.networks.append( - Network( - client=client, project=name, name=network_name, - driver=data.get('driver'), driver_opts=data.get('driver_opts') - ) - ) + project.networks += custom_networks + if project.uses_default_network(): + project.networks.append(project.default_network) if config_data.volumes: for vol_name, data in config_data.volumes.items(): @@ -154,6 +164,18 @@ class Project(object): service.remove_duplicate_containers() return services + def get_networks(self, service_dict, network_definitions): + networks = [] + for name in service_dict.pop('networks', ['default']): + matches = [n for n in network_definitions if n.name == name] + if matches: + networks.append(matches[0].full_name) + else: + raise ConfigurationError( + 'Service "{}" uses an undefined network "{}"' + .format(service_dict['name'], name)) + return networks + def get_links(self, service_dict): links = [] if 'links' in service_dict: @@ -172,10 +194,11 @@ class Project(object): return links def get_net(self, service_dict): + if self.use_networking: + return Net(None) + net = service_dict.pop('net', None) if not net: - if self.use_networking: - return Net(self.default_network.full_name) return Net(None) net_name = get_service_name_from_net(net) @@ -282,6 +305,9 @@ class Project(object): volume.remove() def initialize_networks(self): + if not self.use_networking: + return + networks = self.networks if self.uses_default_network(): networks.append(self.default_network) @@ -291,7 +317,7 @@ class Project(object): def uses_default_network(self): return any( - service.net.mode == self.default_network.full_name + self.default_network.full_name in service.networks for service in self.services ) diff --git a/compose/service.py b/compose/service.py index 1c848ca3..0a7f0d8e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -116,6 +116,7 @@ class Service(object): links=None, volumes_from=None, net=None, + networks=None, **options ): self.name = name @@ -125,6 +126,7 @@ class Service(object): self.links = links or [] self.volumes_from = volumes_from or [] self.net = net or Net(None) + self.networks = networks or [] self.options = options def containers(self, stopped=False, one_off=False, filters={}): @@ -175,7 +177,7 @@ class Service(object): def create_and_start(service, number): container = service.create_container(number=number, quiet=True) - container.start() + service.start_container(container) return container running_containers = self.containers(stopped=False) @@ -348,7 +350,7 @@ class Service(object): container.attach_log_stream() if start: - container.start() + self.start_container(container) return [container] @@ -406,7 +408,7 @@ class Service(object): if attach_logs: new_container.attach_log_stream() if start_new_container: - new_container.start() + self.start_container(new_container) container.remove() return new_container @@ -415,9 +417,18 @@ class Service(object): log.info("Starting %s" % container.name) if attach_logs: container.attach_log_stream() - container.start() + return self.start_container(container) + + def start_container(self, container): + container.start() + self.connect_container_to_networks(container) return container + def connect_container_to_networks(self, container): + for network in self.networks: + log.debug('Connecting "{}" to "{}"'.format(container.name, network)) + self.client.connect_container_to_network(container.id, network) + def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): for c in self.duplicate_containers(): log.info('Removing %s' % c.name) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 52e3d354..ff9c34f1 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -103,8 +103,15 @@ class CLITestCase(DockerClientTestCase): if self.base_dir: self.project.kill() self.project.remove_stopped() + for container in self.project.containers(stopped=True, one_off=True): container.remove(force=True) + + networks = self.client.networks() + for n in networks: + if n['Name'].startswith('{}_'.format(self.project.name)): + self.client.remove_network(n['Name']) + super(CLITestCase, self).tearDown() @property @@ -357,12 +364,11 @@ class CLITestCase(DockerClientTestCase): services = self.project.get_services() networks = self.client.networks(names=[self.project.default_network.full_name]) - for n in networks: - self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['Driver'], 'bridge') network = self.client.inspect_network(networks[0]['Id']) + # print self.project.services[0].containers()[0].get('NetworkSettings') self.assertEqual(len(network['Containers']), len(services)) for service in services: @@ -374,14 +380,52 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/networks' self.dispatch(['up', '-d'], None) - networks = self.client.networks(names=[ - '{}_{}'.format(self.project.name, n) - for n in ['foo', 'bar']]) + back_name = '{}_back'.format(self.project.name) + front_name = '{}_front'.format(self.project.name) - self.assertEqual(len(networks), 2) + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] - for net in networks: - self.assertEqual(net['Driver'], 'bridge') + # Two networks were created: back and front + assert sorted(n['Name'] for n in networks) == [back_name, front_name] + + back_network = [n for n in networks if n['Name'] == back_name][0] + front_network = [n for n in networks if n['Name'] == front_name][0] + + web_container = self.project.get_service('web').containers()[0] + app_container = self.project.get_service('app').containers()[0] + db_container = self.project.get_service('db').containers()[0] + + # db and app joined the back network + assert sorted(back_network['Containers']) == sorted([db_container.id, app_container.id]) + + # web and app joined the front network + assert sorted(front_network['Containers']) == sorted([web_container.id, app_container.id]) + + def test_up_missing_network(self): + self.base_dir = 'tests/fixtures/networks' + + result = self.dispatch( + ['-f', 'missing-network.yml', 'up', '-d'], + returncode=1) + + assert 'Service "web" uses an undefined network "foo"' in result.stderr + + def test_up_no_services(self): + self.base_dir = 'tests/fixtures/no-services' + self.dispatch(['up', '-d'], None) + + network_names = [ + n['Name'] for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + + assert sorted(network_names) == [ + '{}_{}'.format(self.project.name, name) + for name in ['bar', 'foo'] + ] def test_up_with_links_is_invalid(self): self.base_dir = 'tests/fixtures/v2-simple' @@ -400,9 +444,7 @@ class CLITestCase(DockerClientTestCase): # No network was created networks = self.client.networks(names=[self.project.default_network.full_name]) - for n in networks: - self.addCleanup(self.client.remove_network, n['Id']) - self.assertEqual(len(networks), 0) + assert networks == [] web = self.project.get_service('web') db = self.project.get_service('db') @@ -731,8 +773,6 @@ class CLITestCase(DockerClientTestCase): service = self.project.get_service('simple') container, = service.containers(stopped=True, one_off=True) networks = self.client.networks(names=[self.project.default_network.full_name]) - for n in networks: - self.addCleanup(self.client.remove_network, n['Id']) self.assertEqual(len(networks), 1) self.assertEqual(container.human_readable_command, u'true') @@ -890,7 +930,7 @@ class CLITestCase(DockerClientTestCase): def test_restart(self): service = self.project.get_service('simple') container = service.create_container() - container.start() + service.start_container(container) started_at = container.dictionary['State']['StartedAt'] self.dispatch(['restart', '-t', '1'], None) container.inspect() diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml index c0795526..f1b79df0 100644 --- a/tests/fixtures/networks/docker-compose.yml +++ b/tests/fixtures/networks/docker-compose.yml @@ -1,7 +1,19 @@ version: 2 -networks: - foo: - driver: +services: + web: + image: busybox + command: top + networks: ["front"] + app: + image: busybox + command: top + networks: ["front", "back"] + db: + image: busybox + command: top + networks: ["back"] - bar: {} +networks: + front: {} + back: {} diff --git a/tests/fixtures/networks/missing-network.yml b/tests/fixtures/networks/missing-network.yml new file mode 100644 index 00000000..666f7d34 --- /dev/null +++ b/tests/fixtures/networks/missing-network.yml @@ -0,0 +1,10 @@ +version: 2 + +services: + web: + image: busybox + command: top + networks: ["foo"] + +networks: + bar: {} diff --git a/tests/fixtures/no-services/docker-compose.yml b/tests/fixtures/no-services/docker-compose.yml new file mode 100644 index 00000000..fa498784 --- /dev/null +++ b/tests/fixtures/no-services/docker-compose.yml @@ -0,0 +1,5 @@ +version: 2 + +networks: + foo: {} + bar: {} diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 5df751c7..b544783a 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -17,7 +17,7 @@ class ResilienceTest(DockerClientTestCase): self.project = Project('composetest', [self.db], self.client) container = self.db.create_container() - container.start() + self.db.start_container(container) self.host_path = container.get_mount('/var/db')['Source'] def test_successful_recreate(self): @@ -35,7 +35,7 @@ class ResilienceTest(DockerClientTestCase): self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_start_failure(self): - with mock.patch('compose.container.Container.start', crash): + with mock.patch('compose.service.Service.start_container', crash): with self.assertRaises(Crash): self.project.up(strategy=ConvergenceStrategy.always) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 0ce4103e..37ceb65c 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -32,14 +32,7 @@ from compose.service import Service def create_and_start_container(service, **override_options): container = service.create_container(**override_options) - container.start() - return container - - -def remove_stopped(service): - containers = [c for c in service.containers(stopped=True) if not c.is_running] - for container in containers: - container.remove() + return service.start_container(container) class ServiceTest(DockerClientTestCase): @@ -88,19 +81,19 @@ class ServiceTest(DockerClientTestCase): def test_create_container_with_unspecified_volume(self): service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) container = service.create_container() - container.start() + service.start_container(container) assert container.get_mount('/var/db') def test_create_container_with_volume_driver(self): service = self.create_service('db', volume_driver='foodriver') container = service.create_container() - container.start() + service.start_container(container) self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver')) def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(container.get('HostConfig.CpuShares'), 73) def test_create_container_with_cpu_quota(self): @@ -113,7 +106,7 @@ class ServiceTest(DockerClientTestCase): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts)) def test_create_container_with_extra_hosts_dicts(self): @@ -121,33 +114,33 @@ class ServiceTest(DockerClientTestCase): extra_hosts_list = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(set(container.get('HostConfig.ExtraHosts')), set(extra_hosts_list)) def test_create_container_with_cpu_set(self): service = self.create_service('db', cpuset='0') container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(container.get('HostConfig.CpusetCpus'), '0') def test_create_container_with_read_only_root_fs(self): read_only = True service = self.create_service('db', read_only=read_only) container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(container.get('HostConfig.ReadonlyRootfs'), read_only, container.get('HostConfig')) def test_create_container_with_security_opt(self): security_opt = ['label:disable'] service = self.create_service('db', security_opt=security_opt) container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(set(container.get('HostConfig.SecurityOpt')), set(security_opt)) def test_create_container_with_mac_address(self): service = self.create_service('db', mac_address='02:42:ac:11:65:43') container = service.create_container() - container.start() + service.start_container(container) self.assertEqual(container.inspect()['Config']['MacAddress'], '02:42:ac:11:65:43') def test_create_container_with_specified_volume(self): @@ -158,7 +151,7 @@ class ServiceTest(DockerClientTestCase): 'db', volumes=[VolumeSpec(host_path, container_path, 'rw')]) container = service.create_container() - container.start() + service.start_container(container) assert container.get_mount(container_path) # Match the last component ("host-path"), because boot2docker symlinks /tmp @@ -229,7 +222,7 @@ class ServiceTest(DockerClientTestCase): ] ) host_container = host_service.create_container() - host_container.start() + host_service.start_container(host_container) self.assertIn(volume_container_1.id + ':rw', host_container.get('HostConfig.VolumesFrom')) self.assertIn(volume_container_2.id + ':rw', @@ -248,7 +241,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=1', old_container.get('Config.Env')) self.assertEqual(old_container.name, 'composetest_db_1') - old_container.start() + service.start_container(old_container) old_container.inspect() # reload volume data volume_path = old_container.get_mount('/etc')['Source'] diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 470e51ad..ffd4455f 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -12,8 +12,6 @@ from compose.config.types import VolumeFromSpec from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project -from compose.service import ContainerNet -from compose.service import Net from compose.service import Service @@ -412,29 +410,42 @@ class ProjectTest(unittest.TestCase): self.assertEqual(service.net.mode, 'container:' + container_name) def test_uses_default_network_true(self): - web = Service('web', project='test', image="alpine", net=Net('test_default')) - db = Service('web', project='test', image="alpine", net=Net('other')) - project = Project('test', [web, db], None) + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=2, + services=[ + { + 'name': 'foo', + 'image': 'busybox:latest' + }, + ], + networks=None, + volumes=None, + ), + ) + assert project.uses_default_network() - def test_uses_default_network_custom_name(self): - web = Service('web', project='test', image="alpine", net=Net('other')) - project = Project('test', [web], None) - assert not project.uses_default_network() + def test_uses_default_network_false(self): + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version=2, + services=[ + { + 'name': 'foo', + 'image': 'busybox:latest', + 'networks': ['custom'] + }, + ], + networks={'custom': {}}, + volumes=None, + ), + ) - def test_uses_default_network_host(self): - web = Service('web', project='test', image="alpine", net=Net('host')) - project = Project('test', [web], None) - assert not project.uses_default_network() - - def test_uses_default_network_container(self): - container = mock.Mock(id='test') - web = Service( - 'web', - project='test', - image="alpine", - net=ContainerNet(container)) - project = Project('test', [web], None) assert not project.uses_default_network() def test_container_without_name(self): From 9c91cf29674f4e294f9d24a135933d2e7df4e546 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 12:18:20 +0000 Subject: [PATCH 240/359] Test discoverability across multiple networks Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index ff9c34f1..4549a3cb 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -126,6 +126,20 @@ class CLITestCase(DockerClientTestCase): proc = start_process(self.base_dir, project_options + options) return wait_on_process(proc, returncode=returncode) + def execute(self, container, cmd): + # Remove once Hijack and CloseNotifier sign a peace treaty + self.client.close() + exc = self.client.exec_create(container.id, cmd) + self.client.exec_start(exc) + return self.client.exec_inspect(exc)['ExitCode'] + + def lookup(self, container, service_name): + exit_code = self.execute(container, [ + "nslookup", + "{}_{}_1".format(self.project.name, service_name) + ]) + return exit_code == 0 + def test_help(self): self.base_dir = 'tests/fixtures/no-composefile' result = self.dispatch(['help', 'up'], returncode=1) @@ -404,6 +418,13 @@ class CLITestCase(DockerClientTestCase): # web and app joined the front network assert sorted(front_network['Containers']) == sorted([web_container.id, app_container.id]) + # web can see app but not db + assert self.lookup(web_container, "app") + assert not self.lookup(web_container, "db") + + # app can see db + assert self.lookup(app_container, "db") + def test_up_missing_network(self): self.base_dir = 'tests/fixtures/networks' From e75629392d5274697a668e3f405cbb5e82d99676 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 13:01:17 +0000 Subject: [PATCH 241/359] Don't join the bridge network by default in v2 Signed-off-by: Aanand Prasad --- compose/project.py | 18 ++++++++++-------- tests/acceptance/cli_test.py | 9 ++++++--- 2 files changed, 16 insertions(+), 11 deletions(-) diff --git a/compose/project.py b/compose/project.py index 292bf2f2..8606c11e 100644 --- a/compose/project.py +++ b/compose/project.py @@ -69,13 +69,18 @@ class Project(object): ) for service_dict in config_data.services: - networks = project.get_networks( - service_dict, - custom_networks + [project.default_network]) + if use_networking: + networks = project.get_networks( + service_dict, + custom_networks + [project.default_network]) + net = Net(networks[0]) if networks else Net("none") + links = [] + else: + networks = [] + net = project.get_net(service_dict) + links = project.get_links(service_dict) - links = project.get_links(service_dict) volumes_from = get_volumes_from(project, service_dict) - net = project.get_net(service_dict) project.services.append( Service( @@ -194,9 +199,6 @@ class Project(object): return links def get_net(self, service_dict): - if self.use_networking: - return Net(None) - net = service_dict.pop('net', None) if not net: return Net(None) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 4549a3cb..8db9727b 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -382,13 +382,16 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(networks[0]['Driver'], 'bridge') network = self.client.inspect_network(networks[0]['Id']) - # print self.project.services[0].containers()[0].get('NetworkSettings') - self.assertEqual(len(network['Containers']), len(services)) for service in services: containers = service.containers() self.assertEqual(len(containers), 1) - self.assertIn(containers[0].id, network['Containers']) + + container = containers[0] + self.assertIn(container.id, network['Containers']) + + networks = container.get('NetworkSettings.Networks').keys() + self.assertEqual(networks, [network['Name']]) def test_up_with_networks(self): self.base_dir = 'tests/fixtures/networks' From ca68c9faa4d4a4e217c948c2dd5abf25f595c8d7 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 13:16:02 +0000 Subject: [PATCH 242/359] Services can join 'bridge' or 'host' Signed-off-by: Aanand Prasad --- compose/project.py | 15 +++++++++------ tests/acceptance/cli_test.py | 19 +++++++++++++++++++ .../fixtures/networks/predefined-networks.yml | 17 +++++++++++++++++ 3 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/fixtures/networks/predefined-networks.yml diff --git a/compose/project.py b/compose/project.py index 8606c11e..1b5d2eb9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -172,13 +172,16 @@ class Project(object): def get_networks(self, service_dict, network_definitions): networks = [] for name in service_dict.pop('networks', ['default']): - matches = [n for n in network_definitions if n.name == name] - if matches: - networks.append(matches[0].full_name) + if name in ['bridge', 'host']: + networks.append(name) else: - raise ConfigurationError( - 'Service "{}" uses an undefined network "{}"' - .format(service_dict['name'], name)) + matches = [n for n in network_definitions if n.name == name] + if matches: + networks.append(matches[0].full_name) + else: + raise ConfigurationError( + 'Service "{}" uses an undefined network "{}"' + .format(service_dict['name'], name)) return networks def get_links(self, service_dict): diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8db9727b..90c50769 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -437,6 +437,25 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr + def test_up_predefined_networks(self): + filename = 'predefined-networks.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + self.dispatch(['-f', filename, 'up', '-d'], None) + + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + assert not networks + + for name in ['bridge', 'host', 'none']: + container = self.project.get_service(name).containers()[0] + assert container.get('NetworkSettings.Networks').keys() == [name] + assert container.get('HostConfig.NetworkMode') == name + def test_up_no_services(self): self.base_dir = 'tests/fixtures/no-services' self.dispatch(['up', '-d'], None) diff --git a/tests/fixtures/networks/predefined-networks.yml b/tests/fixtures/networks/predefined-networks.yml new file mode 100644 index 00000000..d0fac377 --- /dev/null +++ b/tests/fixtures/networks/predefined-networks.yml @@ -0,0 +1,17 @@ +version: 2 + +services: + bridge: + image: busybox + command: top + networks: ["bridge"] + + host: + image: busybox + command: top + networks: ["host"] + + none: + image: busybox + command: top + networks: [] From 73fbd01cfe7c1e8ad7d297d7f7cfb8704aeb501d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 13:39:44 +0000 Subject: [PATCH 243/359] Support the 'external' option for networks Signed-off-by: Aanand Prasad --- compose/network.py | 25 +++++++++++++++++-- compose/project.py | 4 ++- tests/acceptance/cli_test.py | 23 +++++++++++++++++ tests/fixtures/networks/external-networks.yml | 16 ++++++++++++ 4 files changed, 65 insertions(+), 3 deletions(-) create mode 100644 tests/fixtures/networks/external-networks.yml diff --git a/compose/network.py b/compose/network.py index a8f7e918..b2ba2e9b 100644 --- a/compose/network.py +++ b/compose/network.py @@ -11,16 +11,35 @@ from .config import ConfigurationError log = logging.getLogger(__name__) -# TODO: support external networks class Network(object): - def __init__(self, client, project, name, driver=None, driver_opts=None): + def __init__(self, client, project, name, driver=None, driver_opts=None, + external_name=None): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts + self.external_name = external_name def ensure(self): + if self.external_name: + try: + self.inspect() + log.debug( + 'Network {0} declared as external. No new ' + 'network will be created.'.format(self.name) + ) + except NotFound: + raise ConfigurationError( + 'Network {name} declared as external, but could' + ' not be found. Please create the network manually' + ' using `{command} {name}` and try again.'.format( + name=self.external_name, + command='docker network create' + ) + ) + return + try: data = self.inspect() if self.driver and data['Driver'] != self.driver: @@ -55,4 +74,6 @@ class Network(object): @property def full_name(self): + if self.external_name: + return self.external_name return '{0}_{1}'.format(self.project, self.name) diff --git a/compose/project.py b/compose/project.py index 1b5d2eb9..6a171d51 100644 --- a/compose/project.py +++ b/compose/project.py @@ -64,7 +64,9 @@ class Project(object): custom_networks.append( Network( client=client, project=name, name=network_name, - driver=data.get('driver'), driver_opts=data.get('driver_opts') + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external_name=data.get('external_name'), ) ) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 90c50769..a7d5dbaa 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -456,6 +456,29 @@ class CLITestCase(DockerClientTestCase): assert container.get('NetworkSettings.Networks').keys() == [name] assert container.get('HostConfig.NetworkMode') == name + def test_up_external_networks(self): + filename = 'external-networks.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + result = self.dispatch(['-f', filename, 'up', '-d'], returncode=1) + assert 'declared as external, but could not be found' in result.stderr + + networks = [ + n['Name'] for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + assert not networks + + network_names = ['{}_{}'.format(self.project.name, n) for n in ['foo', 'bar']] + for name in network_names: + self.client.create_network(name) + + self.dispatch(['-f', filename, 'up', '-d']) + container = self.project.containers()[0] + assert sorted(container.get('NetworkSettings.Networks').keys()) == sorted(network_names) + def test_up_no_services(self): self.base_dir = 'tests/fixtures/no-services' self.dispatch(['up', '-d'], None) diff --git a/tests/fixtures/networks/external-networks.yml b/tests/fixtures/networks/external-networks.yml new file mode 100644 index 00000000..644e3dda --- /dev/null +++ b/tests/fixtures/networks/external-networks.yml @@ -0,0 +1,16 @@ +version: 2 + +services: + web: + image: busybox + command: top + networks: + - networks_foo + - bar + +networks: + networks_foo: + external: true + bar: + external: + name: networks_bar From 87326c00ebc8380729f1999d5f20eddc265164e8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 14:51:42 +0000 Subject: [PATCH 244/359] Python 3 fixes Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index a7d5dbaa..9c9ced8c 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -390,7 +390,7 @@ class CLITestCase(DockerClientTestCase): container = containers[0] self.assertIn(container.id, network['Containers']) - networks = container.get('NetworkSettings.Networks').keys() + networks = list(container.get('NetworkSettings.Networks')) self.assertEqual(networks, [network['Name']]) def test_up_with_networks(self): @@ -453,7 +453,7 @@ class CLITestCase(DockerClientTestCase): for name in ['bridge', 'host', 'none']: container = self.project.get_service(name).containers()[0] - assert container.get('NetworkSettings.Networks').keys() == [name] + assert list(container.get('NetworkSettings.Networks')) == [name] assert container.get('HostConfig.NetworkMode') == name def test_up_external_networks(self): @@ -477,7 +477,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['-f', filename, 'up', '-d']) container = self.project.containers()[0] - assert sorted(container.get('NetworkSettings.Networks').keys()) == sorted(network_names) + assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted(network_names) def test_up_no_services(self): self.base_dir = 'tests/fixtures/no-services' From 4e61377c6d912a4f5e454b6afcb7bde0416a83b3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 18:01:44 +0000 Subject: [PATCH 245/359] Move get_networks() out of Project class Signed-off-by: Aanand Prasad --- compose/project.py | 33 +++++++++++++++++---------------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/compose/project.py b/compose/project.py index 6a171d51..933849c2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -72,7 +72,7 @@ class Project(object): for service_dict in config_data.services: if use_networking: - networks = project.get_networks( + networks = get_networks( service_dict, custom_networks + [project.default_network]) net = Net(networks[0]) if networks else Net("none") @@ -171,21 +171,6 @@ class Project(object): service.remove_duplicate_containers() return services - def get_networks(self, service_dict, network_definitions): - networks = [] - for name in service_dict.pop('networks', ['default']): - if name in ['bridge', 'host']: - networks.append(name) - else: - matches = [n for n in network_definitions if n.name == name] - if matches: - networks.append(matches[0].full_name) - else: - raise ConfigurationError( - 'Service "{}" uses an undefined network "{}"' - .format(service_dict['name'], name)) - return networks - def get_links(self, service_dict): links = [] if 'links' in service_dict: @@ -474,6 +459,22 @@ class Project(object): return acc + dep_services +def get_networks(service_dict, network_definitions): + networks = [] + for name in service_dict.pop('networks', ['default']): + if name in ['bridge', 'host']: + networks.append(name) + else: + matches = [n for n in network_definitions if n.name == name] + if matches: + networks.append(matches[0].full_name) + else: + raise ConfigurationError( + 'Service "{}" uses an undefined network "{}"' + .format(service_dict['name'], name)) + return networks + + def get_volumes_from(project, service_dict): volumes_from = service_dict.pop('volumes_from', None) if not volumes_from: From d98b64f6e7f75e801973e59a6723b5f75e724988 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 18:09:45 +0000 Subject: [PATCH 246/359] Remove duplicated logic from initialize_networks() Signed-off-by: Aanand Prasad --- compose/project.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/compose/project.py b/compose/project.py index 933849c2..12d52cc2 100644 --- a/compose/project.py +++ b/compose/project.py @@ -300,11 +300,7 @@ class Project(object): if not self.use_networking: return - networks = self.networks - if self.uses_default_network(): - networks.append(self.default_network) - - for network in networks: + for network in self.networks: network.ensure() def uses_default_network(self): From a7be0afa5b3a1b62324e22b0d81c51dc939966d6 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 14 Jan 2016 10:32:06 -0800 Subject: [PATCH 247/359] bash completion for `docker-compose down` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 3f672517..36d1ef27 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -129,6 +129,22 @@ _docker_compose_docker_compose() { } +_docker_compose_down() { + case "$prev" in + --rmi) + COMPREPLY=( $( compgen -W "all local" -- "$cur" ) ) + return + ;; + esac + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help --rmi --volumes -v" -- "$cur" ) ) + ;; + esac +} + + _docker_compose_events() { case "$prev" in --json) @@ -393,6 +409,7 @@ _docker_compose() { local commands=( build config + down events help kill From fca3e47a7519844704c79dac4fc55b6ce79e4923 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Thu, 14 Jan 2016 10:43:53 -0800 Subject: [PATCH 248/359] bash completion for `docker-compose up --abort-on-container-exit` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 3f672517..b3ed8411 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -368,7 +368,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --help --no-build --no-color --no-deps --no-recreate --force-recreate --timeout -t" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--abort-on-container-exit -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t" -- "$cur" ) ) ;; *) __docker_compose_services_all From 79df2ebe1bbe81232acd84eeca7bf66af8e3004b Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Wed, 13 Jan 2016 15:19:02 -0500 Subject: [PATCH 249/359] Support variable interpolation for volumes and networks sections. Signed-off-by: Daniel Nephin --- compose/config/config.py | 21 +++++--- compose/config/interpolation.py | 31 +++++------ tests/unit/config/config_test.py | 16 +++--- tests/unit/config/interpolation_test.py | 69 +++++++++++++++++++++++++ 4 files changed, 104 insertions(+), 33 deletions(-) create mode 100644 tests/unit/config/interpolation_test.py diff --git a/compose/config/config.py b/compose/config/config.py index c8d93faf..f6df3d3b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -138,6 +138,9 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def get_volumes(self): return {} if self.version == 1 else self.config.get('volumes', {}) + def get_networks(self): + return {} if self.version == 1 else self.config.get('networks', {}) + class Config(namedtuple('_Config', 'version services volumes networks')): """ @@ -258,8 +261,8 @@ def load(config_details): config_details = config_details._replace(config_files=processed_files) main_file = config_details.config_files[0] - volumes = load_mapping(config_details.config_files, 'volumes', 'Volume') - networks = load_mapping(config_details.config_files, 'networks', 'Network') + volumes = load_mapping(config_details.config_files, 'get_volumes', 'Volume') + networks = load_mapping(config_details.config_files, 'get_networks', 'Network') service_dicts = load_services( config_details.working_dir, main_file.filename, @@ -268,11 +271,11 @@ def load(config_details): return Config(main_file.version, service_dicts, volumes, networks) -def load_mapping(config_files, key, entity_type): +def load_mapping(config_files, get_func, entity_type): mapping = {} for config_file in config_files: - for name, config in config_file.config.get(key, {}).items(): + for name, config in getattr(config_file, get_func)().items(): mapping[name] = config or {} if not config: continue @@ -347,12 +350,16 @@ def process_config_file(config_file, service_name=None): service_dicts = config_file.get_service_dicts() validate_top_level_service_objects(config_file.filename, service_dicts) - # TODO: interpolate config in volumes/network sections as well - interpolated_config = interpolate_environment_variables(service_dicts) + interpolated_config = interpolate_environment_variables(service_dicts, 'service') if config_file.version == 2: processed_config = dict(config_file.config) - processed_config.update({'services': interpolated_config}) + processed_config['services'] = interpolated_config + processed_config['volumes'] = interpolate_environment_variables( + config_file.get_volumes(), 'volume') + processed_config['networks'] = interpolate_environment_variables( + config_file.get_networks(), 'network') + if config_file.version == 1: processed_config = interpolated_config diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index 7a757644..e1c781fe 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -11,35 +11,32 @@ from .errors import ConfigurationError log = logging.getLogger(__name__) -def interpolate_environment_variables(service_dicts): +def interpolate_environment_variables(config, section): mapping = BlankDefaultDict(os.environ) + def process_item(name, config_dict): + return dict( + (key, interpolate_value(name, key, val, section, mapping)) + for key, val in (config_dict or {}).items() + ) + return dict( - (service_name, process_service(service_name, service_dict, mapping)) - for (service_name, service_dict) in service_dicts.items() + (name, process_item(name, config_dict)) + for name, config_dict in config.items() ) -def process_service(service_name, service_dict, mapping): - return dict( - (key, interpolate_value(service_name, key, val, mapping)) - for (key, val) in service_dict.items() - ) - - -def interpolate_value(service_name, config_key, value, mapping): +def interpolate_value(name, config_key, value, section, mapping): try: return recursive_interpolate(value, mapping) except InvalidInterpolation as e: raise ConfigurationError( 'Invalid interpolation format for "{config_key}" option ' - 'in service "{service_name}": "{string}"' - .format( + 'in {section} "{name}": "{string}"'.format( config_key=config_key, - service_name=service_name, - string=e.string, - ) - ) + name=name, + section=section, + string=e.string)) def recursive_interpolate(obj, mapping): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f0d432d5..f8816643 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -686,8 +686,8 @@ class ConfigTest(unittest.TestCase): ) ) - self.assertTrue(mock_logging.warn.called) - self.assertTrue(expected_warning_msg in mock_logging.warn.call_args[0][0]) + assert mock_logging.warn.called + assert expected_warning_msg in mock_logging.warn.call_args[0][0] def test_config_valid_environment_dict_key_contains_dashes(self): services = config.load( @@ -1664,15 +1664,13 @@ class ExtendsTest(unittest.TestCase): load_from_filename('tests/fixtures/extends/invalid-net.yml') @mock.patch.dict(os.environ) - def test_valid_interpolation_in_extended_service(self): - os.environ.update( - HOSTNAME_VALUE="penguin", - ) + def test_load_config_runs_interpolation_in_extended_service(self): + os.environ.update(HOSTNAME_VALUE="penguin") expected_interpolated_value = "host-penguin" - - service_dicts = load_from_filename('tests/fixtures/extends/valid-interpolation.yml') + service_dicts = load_from_filename( + 'tests/fixtures/extends/valid-interpolation.yml') for service in service_dicts: - self.assertTrue(service['hostname'], expected_interpolated_value) + assert service['hostname'] == expected_interpolated_value @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') def test_volume_path(self): diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py new file mode 100644 index 00000000..0691e886 --- /dev/null +++ b/tests/unit/config/interpolation_test.py @@ -0,0 +1,69 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import os + +import mock +import pytest + +from compose.config.interpolation import interpolate_environment_variables + + +@pytest.yield_fixture +def mock_env(): + with mock.patch.dict(os.environ): + os.environ['USER'] = 'jenny' + os.environ['FOO'] = 'bar' + yield + + +def test_interpolate_environment_variables_in_services(mock_env): + services = { + 'servivea': { + 'image': 'example:${USER}', + 'volumes': ['$FOO:/target'], + 'logging': { + 'driver': '${FOO}', + 'options': { + 'user': '$USER', + } + } + } + } + expected = { + 'servivea': { + 'image': 'example:jenny', + 'volumes': ['bar:/target'], + 'logging': { + 'driver': 'bar', + 'options': { + 'user': 'jenny', + } + } + } + } + assert interpolate_environment_variables(services, 'service') == expected + + +def test_interpolate_environment_variables_in_volumes(mock_env): + volumes = { + 'data': { + 'driver': '$FOO', + 'driver_opts': { + 'max': 2, + 'user': '${USER}' + } + }, + 'other': None, + } + expected = { + 'data': { + 'driver': 'bar', + 'driver_opts': { + 'max': 2, + 'user': 'jenny' + } + }, + 'other': {}, + } + assert interpolate_environment_variables(volumes, 'volume') == expected From 9cfa71ceee3cb164119b448edef8ac0bda63f751 Mon Sep 17 00:00:00 2001 From: Garrett Heel Date: Fri, 11 Dec 2015 15:19:51 -0800 Subject: [PATCH 250/359] Add support for build arguments Allows 'build' configuration option to be specified as an object and adds support for build args. Signed-off-by: Garrett Heel --- compose/config/config.py | 112 +++++++++++++++++++++----- compose/config/service_schema_v2.json | 15 +++- compose/config/validation.py | 17 +++- compose/service.py | 6 +- docs/compose-file.md | 67 ++++++++++++++- tests/integration/service_test.py | 32 +++++--- tests/integration/state_test.py | 6 +- tests/unit/config/config_test.py | 82 +++++++++++++++++-- tests/unit/service_test.py | 9 ++- 9 files changed, 297 insertions(+), 49 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index c8d93faf..8200900f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import codecs +import functools import logging import operator import os @@ -455,6 +456,12 @@ def resolve_environment(service_dict): return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) +def resolve_build_args(build): + args = {} + args.update(parse_build_arguments(build.get('args'))) + return dict(resolve_env_var(k, v) for k, v in six.iteritems(args)) + + def validate_extended_service_dict(service_dict, filename, service): error_prefix = "Cannot extend service '%s' in %s:" % (service, filename) @@ -492,12 +499,16 @@ def process_service(service_config): for path in to_list(service_dict['env_file']) ] + if 'build' in service_dict: + if isinstance(service_dict['build'], six.string_types): + service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) + elif isinstance(service_dict['build'], dict) and 'context' in service_dict['build']: + path = service_dict['build']['context'] + service_dict['build']['context'] = resolve_build_path(working_dir, path) + if 'volumes' in service_dict and service_dict.get('volume_driver') is None: service_dict['volumes'] = resolve_volume_paths(working_dir, service_dict) - if 'build' in service_dict: - service_dict['build'] = resolve_build_path(working_dir, service_dict['build']) - if 'labels' in service_dict: service_dict['labels'] = parse_labels(service_dict['labels']) @@ -535,6 +546,8 @@ def finalize_service(service_config, service_names, version): if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) + normalize_build(service_dict, service_config.working_dir) + return normalize_v1_service_format(service_dict) @@ -599,10 +612,31 @@ def merge_service_dicts(base, override, version): if version == 1: legacy_v1_merge_image_or_build(d, base, override) + else: + merge_build(d, base, override) return d +def merge_build(output, base, override): + build = {} + + if 'build' in base: + if isinstance(base['build'], six.string_types): + build['context'] = base['build'] + else: + build.update(base['build']) + + if 'build' in override: + if isinstance(override['build'], six.string_types): + build['context'] = override['build'] + else: + build.update(override['build']) + + if build: + output['build'] = build + + def legacy_v1_merge_image_or_build(output, base, override): output.pop('image', None) output.pop('build', None) @@ -622,22 +656,6 @@ def merge_environment(base, override): return env -def parse_environment(environment): - if not environment: - return {} - - if isinstance(environment, list): - return dict(split_env(e) for e in environment) - - if isinstance(environment, dict): - return dict(environment) - - raise ConfigurationError( - "environment \"%s\" must be a list or mapping," % - environment - ) - - def split_env(env): if isinstance(env, six.binary_type): env = env.decode('utf-8', 'replace') @@ -647,6 +665,34 @@ def split_env(env): return env, None +def split_label(label): + if '=' in label: + return label.split('=', 1) + else: + return label, '' + + +def parse_dict_or_list(split_func, type_name, arguments): + if not arguments: + return {} + + if isinstance(arguments, list): + return dict(split_func(e) for e in arguments) + + if isinstance(arguments, dict): + return dict(arguments) + + raise ConfigurationError( + "%s \"%s\" must be a list or mapping," % + (type_name, arguments) + ) + + +parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments') +parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment') +parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels') + + def resolve_env_var(key, val): if val is not None: return key, val @@ -690,6 +736,26 @@ def resolve_volume_path(working_dir, volume): return container_path +def normalize_build(service_dict, working_dir): + build = {} + + # supported in V1 only + if 'dockerfile' in service_dict: + build['dockerfile'] = service_dict.pop('dockerfile') + + if 'build' in service_dict: + # Shortcut where specifying a string is treated as the build context + if isinstance(service_dict['build'], six.string_types): + build['context'] = service_dict.pop('build') + else: + build.update(service_dict['build']) + if 'args' in build: + build['args'] = resolve_build_args(build) + + if build: + service_dict['build'] = build + + def resolve_build_path(working_dir, build_path): if is_url(build_path): return build_path @@ -702,7 +768,13 @@ def is_url(build_path): def validate_paths(service_dict): if 'build' in service_dict: - build_path = service_dict['build'] + build = service_dict.get('build', {}) + + if isinstance(build, six.string_types): + build_path = build + elif isinstance(build, dict) and 'context' in build: + build_path = build['context'] + if ( not is_url(build_path) and (not os.path.exists(build_path) or not os.access(build_path, os.R_OK)) diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 5f4e0478..23d0381c 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -15,7 +15,20 @@ "type": "object", "properties": { - "build": {"type": "string"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "cgroup_parent": {"type": "string"}, diff --git a/compose/config/validation.py b/compose/config/validation.py index 0bf75691..639e8bed 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -150,18 +150,29 @@ def handle_error_for_schema_with_id(error, service_name): VALID_NAME_CHARS) if schema_id == '#/definitions/constraints': + # Build context could in 'build' or 'build.context' and dockerfile could be + # in 'dockerfile' or 'build.dockerfile' + context = False + dockerfile = 'dockerfile' in error.instance + if 'build' in error.instance: + if isinstance(error.instance['build'], six.string_types): + context = True + else: + context = 'context' in error.instance['build'] + dockerfile = dockerfile or 'dockerfile' in error.instance['build'] + # TODO: only applies to v1 - if 'image' in error.instance and 'build' in error.instance: + if 'image' in error.instance and context: return ( "Service '{}' has both an image and build path specified. " "A service can either be built to image or use an existing " "image, not both.".format(service_name)) - if 'image' not in error.instance and 'build' not in error.instance: + if 'image' not in error.instance and not context: return ( "Service '{}' has neither an image nor a build path " "specified. At least one must be provided.".format(service_name)) # TODO: only applies to v1 - if 'image' in error.instance and 'dockerfile' in error.instance: + if 'image' in error.instance and dockerfile: return ( "Service '{}' has both an image and alternate Dockerfile. " "A service can either be built to image or use an existing " diff --git a/compose/service.py b/compose/service.py index 0a7f0d8e..c91c3a58 100644 --- a/compose/service.py +++ b/compose/service.py @@ -638,7 +638,8 @@ class Service(object): def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) - path = self.options['build'] + build_opts = self.options.get('build', {}) + path = build_opts.get('context') # python2 os.path() doesn't support unicode, so we need to encode it to # a byte string if not six.PY3: @@ -652,7 +653,8 @@ class Service(object): forcerm=force_rm, pull=pull, nocache=no_cache, - dockerfile=self.options.get('dockerfile', None), + dockerfile=build_opts.get('dockerfile', None), + buildargs=build_opts.get('args', None), ) try: diff --git a/docs/compose-file.md b/docs/compose-file.md index 4759cde0..a9e54014 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -37,7 +37,8 @@ those files, all the [services](#service-configuration-reference) are declared at the root of the document. Version 1 files do not support the declaration of -named [volumes](#volume-configuration-reference) +named [volumes](#volume-configuration-reference) or +[build arguments](#args). Example: @@ -89,6 +90,30 @@ definition. ### build +Configuration options that are applied at build time. + +In version 1 this must be given as a string representing the context. + + build: . + +In version 2 this can alternatively be given as an object with extra options. + + version: 2 + services: + web: + build: . + + version: 2 + services: + web: + build: + context: . + dockerfile: Dockerfile-alternate + args: + buildno: 1 + +#### context + Either a path to a directory containing a Dockerfile, or a url to a git repository. When the value supplied is a relative path, it is interpreted as relative to the @@ -99,9 +124,46 @@ Compose will build and tag it with a generated name, and use that image thereaft build: /path/to/build/dir -Using `build` together with `image` is not allowed. Attempting to do so results in + build: + context: /path/to/build/dir + +Using `context` together with `image` is not allowed. Attempting to do so results in an error. +#### dockerfile + +Alternate Dockerfile. + +Compose will use an alternate file to build with. A build path must also be +specified using the `build` key. + + build: + context: /path/to/build/dir + dockerfile: Dockerfile-alternate + +Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. + +#### args + +Add build arguments. You can use either an array or a dictionary. Any +boolean values; true, false, yes, no, need to be enclosed in quotes to ensure +they are not converted to True or False by the YML parser. + +Build arguments with only a key are resolved to their environment value on the +machine Compose is running on. + +> **Note:** Introduced in version 2 of the compose file format. + + build: + args: + buildno: 1 + user: someuser + + build: + args: + - buildno=1 + - user=someuser + ### cap_add, cap_drop Add or drop container capabilities. @@ -194,6 +256,7 @@ The entrypoint can also be a list, in a manner similar to [dockerfile](https://d - memory_limit=-1 - vendor/bin/phpunit + ### env_file Add environment variables from a file. Can be a single value or a list. diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 37ceb65c..0e91dcf7 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -294,7 +294,7 @@ class ServiceTest(DockerClientTestCase): project='composetest', name='db', client=self.client, - build='tests/fixtures/dockerfile-with-volume', + build={'context': 'tests/fixtures/dockerfile-with-volume'}, ) old_container = create_and_start_container(service) @@ -315,7 +315,7 @@ class ServiceTest(DockerClientTestCase): def test_execute_convergence_plan_when_image_volume_masks_config(self): service = self.create_service( 'db', - build='tests/fixtures/dockerfile-with-volume', + build={'context': 'tests/fixtures/dockerfile-with-volume'}, ) old_container = create_and_start_container(service) @@ -346,7 +346,7 @@ class ServiceTest(DockerClientTestCase): def test_execute_convergence_plan_without_start(self): service = self.create_service( 'db', - build='tests/fixtures/dockerfile-with-volume' + build={'context': 'tests/fixtures/dockerfile-with-volume'} ) containers = service.execute_convergence_plan(ConvergencePlan('create', []), start=False) @@ -450,7 +450,7 @@ class ServiceTest(DockerClientTestCase): service = Service( name='test', client=self.client, - build='tests/fixtures/simple-dockerfile', + build={'context': 'tests/fixtures/simple-dockerfile'}, project='composetest', ) container = create_and_start_container(service) @@ -463,7 +463,7 @@ class ServiceTest(DockerClientTestCase): service = Service( name='test', client=self.client, - build='this/does/not/exist/and/will/throw/error', + build={'context': 'this/does/not/exist/and/will/throw/error'}, project='composetest', ) container = create_and_start_container(service) @@ -483,7 +483,7 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: f.write("FROM busybox\n") - self.create_service('web', build=base_dir).build() + self.create_service('web', build={'context': base_dir}).build() assert self.client.inspect_image('composetest_web') def test_build_non_ascii_filename(self): @@ -496,7 +496,7 @@ class ServiceTest(DockerClientTestCase): with open(os.path.join(base_dir.encode('utf8'), b'foo\xE2bar'), 'w') as f: f.write("hello world\n") - self.create_service('web', build=text_type(base_dir)).build() + self.create_service('web', build={'context': text_type(base_dir)}).build() assert self.client.inspect_image('composetest_web') def test_build_with_image_name(self): @@ -508,16 +508,30 @@ class ServiceTest(DockerClientTestCase): image_name = 'examples/composetest:latest' self.addCleanup(self.client.remove_image, image_name) - self.create_service('web', build=base_dir, image=image_name).build() + self.create_service('web', build={'context': base_dir}, image=image_name).build() assert self.client.inspect_image(image_name) def test_build_with_git_url(self): build_url = "https://github.com/dnephin/docker-build-from-url.git" - service = self.create_service('buildwithurl', build=build_url) + service = self.create_service('buildwithurl', build={'context': build_url}) self.addCleanup(self.client.remove_image, service.image_name) service.build() assert service.image() + def test_build_with_build_args(self): + base_dir = tempfile.mkdtemp() + self.addCleanup(shutil.rmtree, base_dir) + + with open(os.path.join(base_dir, 'Dockerfile'), 'w') as f: + f.write("FROM busybox\n") + f.write("ARG build_version\n") + + service = self.create_service('buildwithargs', + build={'context': text_type(base_dir), + 'args': {"build_version": "1"}}) + service.build() + assert service.image() + def test_start_container_stays_unpriviliged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 6e656c29..36099d2d 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -266,13 +266,13 @@ class ServiceStateTest(DockerClientTestCase): dockerfile = context.join('Dockerfile') dockerfile.write(base_image) - web = self.create_service('web', build=str(context)) + web = self.create_service('web', build={'context': str(context)}) container = web.create_container() dockerfile.write(base_image + 'CMD echo hello world\n') web.build() - web = self.create_service('web', build=str(context)) + web = self.create_service('web', build={'context': str(context)}) self.assertEqual(('recreate', [container]), web.convergence_plan()) def test_image_changed_to_build(self): @@ -286,7 +286,7 @@ class ServiceStateTest(DockerClientTestCase): web = self.create_service('web', image='busybox') container = web.create_container() - web = self.create_service('web', build=str(context)) + web = self.create_service('web', build={'context': str(context)}) plan = web.convergence_plan() self.assertEqual(('recreate', [container]), plan) containers = web.execute_convergence_plan(plan) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f0d432d5..ddb992fd 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -12,6 +12,7 @@ import py import pytest from compose.config import config +from compose.config.config import resolve_build_args from compose.config.config import resolve_environment from compose.config.errors import ConfigurationError from compose.config.types import VolumeSpec @@ -284,7 +285,7 @@ class ConfigTest(unittest.TestCase): expected = [ { 'name': 'web', - 'build': os.path.abspath('/'), + 'build': {'context': os.path.abspath('/')}, 'volumes': [VolumeSpec.parse('/home/user/project:/code')], 'links': ['db'], }, @@ -414,6 +415,59 @@ class ConfigTest(unittest.TestCase): assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_config_build_configuration(self): + service = config.load( + build_config_details( + {'web': { + 'build': '.', + 'dockerfile': 'Dockerfile-alt' + }}, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services + self.assertTrue('context' in service[0]['build']) + self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + + def test_config_build_configuration_v2(self): + service = config.load( + build_config_details( + { + 'version': 2, + 'services': { + 'web': { + 'build': '.', + 'dockerfile': 'Dockerfile-alt' + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services + self.assertTrue('context' in service[0]['build']) + self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + + service = config.load( + build_config_details( + { + 'version': 2, + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt' + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services + self.assertTrue('context' in service[0]['build']) + self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + def test_load_with_multiple_files_v2(self): base_file = config.ConfigFile( 'base.yaml', @@ -445,7 +499,7 @@ class ConfigTest(unittest.TestCase): expected = [ { 'name': 'web', - 'build': os.path.abspath('/'), + 'build': {'context': os.path.abspath('/')}, 'image': 'example/web', 'volumes': [VolumeSpec.parse('/home/user/project:/code')], }, @@ -1157,7 +1211,7 @@ class BuildOrImageMergeTest(unittest.TestCase): self.assertEqual( config.merge_service_dicts({'image': 'redis'}, {'build': '.'}, V1), - {'build': '.'}, + {'build': '.'} ) @@ -1388,6 +1442,24 @@ class EnvTest(unittest.TestCase): }, ) + @mock.patch.dict(os.environ) + def test_resolve_build_args(self): + os.environ['env_arg'] = 'value2' + + build = { + 'context': '.', + 'args': { + 'arg1': 'value1', + 'empty_arg': '', + 'env_arg': None, + 'no_env': None + } + } + self.assertEqual( + resolve_build_args(build), + {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': ''}, + ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @mock.patch.dict(os.environ) def test_resolve_path(self): @@ -1873,7 +1945,7 @@ class BuildPathTest(unittest.TestCase): def test_from_file(self): service_dict = load_from_filename('tests/fixtures/build-path/docker-compose.yml') - self.assertEquals(service_dict, [{'name': 'foo', 'build': self.abs_context_path}]) + self.assertEquals(service_dict, [{'name': 'foo', 'build': {'context': self.abs_context_path}}]) def test_valid_url_in_build_path(self): valid_urls = [ @@ -1888,7 +1960,7 @@ class BuildPathTest(unittest.TestCase): service_dict = config.load(build_config_details({ 'validurl': {'build': valid_url}, }, '.', None)).services - assert service_dict[0]['build'] == valid_url + assert service_dict[0]['build'] == {'context': valid_url} def test_invalid_url_in_build_path(self): invalid_urls = [ diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 9a3e13b4..c9244a47 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -355,7 +355,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@")) def test_create_container_with_build(self): - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = [ NoSuchImageError, {'Id': 'abc123'}, @@ -374,17 +374,18 @@ class ServiceTest(unittest.TestCase): forcerm=False, nocache=False, rm=True, + buildargs=None, ) def test_create_container_no_build(self): - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.return_value = {'Id': 'abc123'} service.create_container(do_build=False) self.assertFalse(self.mock_client.build.called) def test_create_container_no_build_but_needs_build(self): - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = NoSuchImageError with self.assertRaises(NeedsBuildError): service.create_container(do_build=False) @@ -394,7 +395,7 @@ class ServiceTest(unittest.TestCase): b'{"stream": "Successfully built 12345"}', ] - service = Service('foo', client=self.mock_client, build='.') + service = Service('foo', client=self.mock_client, build={'context': '.'}) service.build() self.assertEqual(self.mock_client.build.call_count, 1) From 13063a96cbbc7848a87b1b3137fedbadc8fef188 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 14 Jan 2016 12:10:42 -0800 Subject: [PATCH 251/359] Fix handling of service.dockerfile key Made invalid in v2 format Doesn't break build config anymore Signed-off-by: Joffrey F --- compose/config/config.py | 6 ++++ compose/config/service_schema_v2.json | 1 - tests/unit/config/config_test.py | 42 +++++++++++++++++---------- 3 files changed, 33 insertions(+), 16 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 8200900f..86f0aa3b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -562,6 +562,12 @@ def normalize_v1_service_format(service_dict): service_dict['logging']['options'] = service_dict['log_opt'] del service_dict['log_opt'] + if 'dockerfile' in service_dict: + service_dict['build'] = service_dict.get('build', {}) + service_dict['build'].update({ + 'dockerfile': service_dict.pop('dockerfile') + }) + return service_dict diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 23d0381c..8623507a 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -45,7 +45,6 @@ "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, - "dockerfile": {"type": "string"}, "domainname": {"type": "string"}, "entrypoint": { "oneOf": [ diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index ddb992fd..5146df1e 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -430,23 +430,35 @@ class ConfigTest(unittest.TestCase): self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') def test_config_build_configuration_v2(self): - service = config.load( - build_config_details( - { - 'version': 2, - 'services': { - 'web': { - 'build': '.', - 'dockerfile': 'Dockerfile-alt' + # service.dockerfile is invalid in v2 + with self.assertRaises(ConfigurationError): + config.load( + build_config_details( + { + 'version': 2, + 'services': { + 'web': { + 'build': '.', + 'dockerfile': 'Dockerfile-alt' + } } - } - }, - 'tests/fixtures/extends', - 'filename.yml' + }, + 'tests/fixtures/extends', + 'filename.yml' + ) ) - ).services - self.assertTrue('context' in service[0]['build']) - self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + + service = config.load( + build_config_details({ + 'version': 2, + 'services': { + 'web': { + 'build': '.' + } + } + }, 'tests/fixtures/extends', 'filename.yml') + ).services[0] + self.assertTrue('context' in service['build']) service = config.load( build_config_details( From ce9f2681a2be99ed97915674de9dd26e441298d4 Mon Sep 17 00:00:00 2001 From: Clemens Gutweiler Date: Thu, 7 Jan 2016 17:59:51 +0100 Subject: [PATCH 252/359] Fixes #1422: ipv6 addr contains colons, so we split only by the first char. Signed-off-by: Clemens Gutweiler --- compose/config/types.py | 2 +- tests/unit/config/types_test.py | 6 ++++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index cec1f6cf..437f4120 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -58,7 +58,7 @@ def parse_extra_hosts(extra_hosts_config): extra_hosts_dict = {} for extra_hosts_line in extra_hosts_config: # TODO: validate string contains ':' ? - host, ip = extra_hosts_line.split(':') + host, ip = extra_hosts_line.split(':', 1) extra_hosts_dict[host.strip()] = ip.strip() return extra_hosts_dict diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 4df66548..702aa977 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -16,11 +16,13 @@ def test_parse_extra_hosts_list(): assert parse_extra_hosts([ "www.example.com: 192.168.0.17", "static.example.com:192.168.0.19", - "api.example.com: 192.168.0.18" + "api.example.com: 192.168.0.18", + "v6.example.com: ::1" ]) == { 'www.example.com': '192.168.0.17', 'static.example.com': '192.168.0.19', - 'api.example.com': '192.168.0.18' + 'api.example.com': '192.168.0.18', + 'v6.example.com': '::1' } From 1ae57d92d4081380759fc4f816975d1a3a4d459c Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 14 Jan 2016 13:12:39 -0800 Subject: [PATCH 253/359] Remove duplicate functions Signed-off-by: Joffrey F --- compose/config/config.py | 44 +++++++++------------------------------- docs/compose-file.md | 12 ----------- 2 files changed, 10 insertions(+), 46 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 86f0aa3b..58b73dbe 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -457,8 +457,7 @@ def resolve_environment(service_dict): def resolve_build_args(build): - args = {} - args.update(parse_build_arguments(build.get('args'))) + args = parse_build_arguments(build.get('args')) return dict(resolve_env_var(k, v) for k, v in six.iteritems(args)) @@ -699,6 +698,14 @@ parse_environment = functools.partial(parse_dict_or_list, split_env, 'environmen parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels') +def parse_ulimits(ulimits): + if not ulimits: + return {} + + if isinstance(ulimits, dict): + return dict(ulimits) + + def resolve_env_var(key, val): if val is not None: return key, val @@ -743,13 +750,9 @@ def resolve_volume_path(working_dir, volume): def normalize_build(service_dict, working_dir): - build = {} - - # supported in V1 only - if 'dockerfile' in service_dict: - build['dockerfile'] = service_dict.pop('dockerfile') if 'build' in service_dict: + build = {} # Shortcut where specifying a string is treated as the build context if isinstance(service_dict['build'], six.string_types): build['context'] = service_dict.pop('build') @@ -758,7 +761,6 @@ def normalize_build(service_dict, working_dir): if 'args' in build: build['args'] = resolve_build_args(build) - if build: service_dict['build'] = build @@ -835,32 +837,6 @@ def join_path_mapping(pair): return ":".join((host, container)) -def parse_labels(labels): - if not labels: - return {} - - if isinstance(labels, list): - return dict(split_label(e) for e in labels) - - if isinstance(labels, dict): - return dict(labels) - - -def split_label(label): - if '=' in label: - return label.split('=', 1) - else: - return label, '' - - -def parse_ulimits(ulimits): - if not ulimits: - return {} - - if isinstance(ulimits, dict): - return dict(ulimits) - - def expand_path(working_dir, path): return os.path.abspath(os.path.join(working_dir, os.path.expanduser(path))) diff --git a/docs/compose-file.md b/docs/compose-file.md index a9e54014..ecd135f1 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -228,18 +228,6 @@ Custom DNS search domains. Can be a single value or a list. - dc1.example.com - dc2.example.com -### dockerfile - -Alternate Dockerfile. - -Compose will use an alternate file to build with. A build path must also be -specified using the `build` key. - - build: /path/to/build/dir - dockerfile: Dockerfile-alternate - -Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. - ### entrypoint Override the default entrypoint. From b689c4a21888b751eb566630f6d441128d196cdd Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 14 Jan 2016 11:39:16 -0500 Subject: [PATCH 254/359] Move service validation to validate_service(). Signed-off-by: Daniel Nephin --- compose/config/config.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 4684161e..a378f276 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -312,15 +312,11 @@ def load_services(working_dir, filename, service_configs, version): resolver = ServiceExtendsResolver(service_config, version) service_dict = process_service(resolver.run()) - # TODO: move to validate_service() - validate_against_service_schema(service_dict, service_config.name, version) - validate_paths(service_dict) - + validate_service(service_dict, service_config.name, version) service_dict = finalize_service( service_config._replace(config=service_dict), service_names, version) - service_dict['name'] = service_config.name return service_dict def build_services(service_config): @@ -494,7 +490,14 @@ def validate_ulimits(ulimit_config): "than 'hard' value".format(ulimit_config)) -# TODO: rename to normalize_service +def validate_service(service_dict, service_name, version): + validate_against_service_schema(service_dict, service_name, version) + validate_paths(service_dict) + + if 'ulimits' in service_dict: + validate_ulimits(service_dict['ulimits']) + + def process_service(service_config): working_dir = service_config.working_dir service_dict = dict(service_config.config) @@ -525,10 +528,6 @@ def process_service(service_config): if field in service_dict: service_dict[field] = to_list(service_dict[field]) - # TODO: move to a validate_service() - if 'ulimits' in service_dict: - validate_ulimits(service_dict['ulimits']) - return service_dict @@ -554,6 +553,7 @@ def finalize_service(service_config, service_names, version): normalize_build(service_dict, service_config.working_dir) + service_dict['name'] = service_config.name return normalize_v1_service_format(service_dict) From b98e2169e6aebd786dab5554e91dce55c248abb9 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 14 Jan 2016 13:28:53 -0500 Subject: [PATCH 255/359] Error when the project name is invalid for the image name. Signed-off-by: Daniel Nephin --- compose/config/config.py | 12 ++++++++++++ tests/unit/config/config_test.py | 7 +++++++ 2 files changed, 19 insertions(+) diff --git a/compose/config/config.py b/compose/config/config.py index a378f276..b3894225 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -6,6 +6,7 @@ import functools import logging import operator import os +import string import sys from collections import namedtuple @@ -497,6 +498,13 @@ def validate_service(service_dict, service_name, version): if 'ulimits' in service_dict: validate_ulimits(service_dict['ulimits']) + if not service_dict.get('image') and has_uppercase(service_name): + raise ConfigurationError( + "Service '{name}' contains uppercase characters which are not valid " + "as part of an image name. Either use a lowercase service name or " + "use the `image` field to set a custom name for the service image." + .format(name=service_name)) + def process_service(service_config): working_dir = service_config.working_dir @@ -861,6 +869,10 @@ def to_list(value): return value +def has_uppercase(name): + return any(char in string.ascii_uppercase for char in name) + + def load_yaml(filename): try: with open(filename, 'r') as fh: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 10c5bbb7..e24dc904 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -544,6 +544,13 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_errors_on_uppercase_with_no_image(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details({ + 'Foo': {'build': '.'}, + }, 'tests/fixtures/build-ctx')) + assert "Service 'Foo' contains uppercase characters" in exc.exconly() + def test_invalid_config_build_and_image_specified(self): expected_error_msg = "Service 'foo' has both an image and build path specified." with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): From 0f234154c24da87d524e39255e659ae340227278 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 14 Jan 2016 14:35:02 -0500 Subject: [PATCH 256/359] Remove all non-external networks on down. Also moves the shutdown test fixtures to be a more general v2-full fixture. Signed-off-by: Daniel Nephin --- compose/network.py | 5 ++++- compose/project.py | 8 ++++---- tests/acceptance/cli_test.py | 18 ++++++++++------- tests/fixtures/shutdown/docker-compose.yml | 10 ---------- .../fixtures/{shutdown => v2-full}/Dockerfile | 0 tests/fixtures/v2-full/docker-compose.yml | 20 +++++++++++++++++++ 6 files changed, 39 insertions(+), 22 deletions(-) delete mode 100644 tests/fixtures/shutdown/docker-compose.yml rename tests/fixtures/{shutdown => v2-full}/Dockerfile (100%) create mode 100644 tests/fixtures/v2-full/docker-compose.yml diff --git a/compose/network.py b/compose/network.py index b2ba2e9b..eaad770c 100644 --- a/compose/network.py +++ b/compose/network.py @@ -65,7 +65,10 @@ class Network(object): ) def remove(self): - # TODO: don't remove external networks + if self.external_name: + log.info("Network %s is external, skipping", self.full_name) + return + log.info("Removing network {}".format(self.full_name)) self.client.remove_network(self.full_name) diff --git a/compose/project.py b/compose/project.py index 12d52cc2..1322c990 100644 --- a/compose/project.py +++ b/compose/project.py @@ -275,7 +275,7 @@ class Project(object): def down(self, remove_image_type, include_volumes): self.stop() self.remove_stopped(v=include_volumes) - self.remove_default_network() + self.remove_networks() if include_volumes: self.remove_volumes() @@ -286,11 +286,11 @@ class Project(object): for service in self.get_services(): service.remove_image(remove_image_type) - def remove_default_network(self): + def remove_networks(self): if not self.use_networking: return - if self.uses_default_network(): - self.default_network.remove() + for network in self.networks: + network.remove() def remove_volumes(self): for volume in self.volumes: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 9c9ced8c..548c6b93 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -340,16 +340,20 @@ class CLITestCase(DockerClientTestCase): assert '--rmi flag must be' in result.stderr def test_down(self): - self.base_dir = 'tests/fixtures/shutdown' + self.base_dir = 'tests/fixtures/v2-full' self.dispatch(['up', '-d']) - wait_on_condition(ContainerCountCondition(self.project, 1)) + wait_on_condition(ContainerCountCondition(self.project, 2)) result = self.dispatch(['down', '--rmi=local', '--volumes']) - assert 'Stopping shutdown_web_1' in result.stderr - assert 'Removing shutdown_web_1' in result.stderr - assert 'Removing volume shutdown_data' in result.stderr - assert 'Removing image shutdown_web' in result.stderr - assert 'Removing network shutdown_default' in result.stderr + assert 'Stopping v2full_web_1' in result.stderr + assert 'Stopping v2full_other_1' in result.stderr + assert 'Removing v2full_web_1' in result.stderr + assert 'Removing v2full_other_1' in result.stderr + assert 'Removing volume v2full_data' in result.stderr + assert 'Removing image v2full_web' in result.stderr + assert 'Removing image busybox' not in result.stderr + assert 'Removing network v2full_default' in result.stderr + assert 'Removing network v2full_front' in result.stderr def test_up_detached(self): self.dispatch(['up', '-d']) diff --git a/tests/fixtures/shutdown/docker-compose.yml b/tests/fixtures/shutdown/docker-compose.yml deleted file mode 100644 index c83c3d63..00000000 --- a/tests/fixtures/shutdown/docker-compose.yml +++ /dev/null @@ -1,10 +0,0 @@ - -version: 2 - -volumes: - data: - driver: local - -services: - web: - build: . diff --git a/tests/fixtures/shutdown/Dockerfile b/tests/fixtures/v2-full/Dockerfile similarity index 100% rename from tests/fixtures/shutdown/Dockerfile rename to tests/fixtures/v2-full/Dockerfile diff --git a/tests/fixtures/v2-full/docker-compose.yml b/tests/fixtures/v2-full/docker-compose.yml new file mode 100644 index 00000000..86d1c2c2 --- /dev/null +++ b/tests/fixtures/v2-full/docker-compose.yml @@ -0,0 +1,20 @@ + +version: 2 + +volumes: + data: + driver: local + +networks: + front: {} + +services: + web: + build: . + networks: + - front + - default + + other: + image: busybox:latest + command: top From 3021ee12fe021092673930bd0ad578783a51dffa Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 14 Jan 2016 15:09:04 -0500 Subject: [PATCH 257/359] Fix `config` command to print the new sections of the config Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- tests/acceptance/cli_test.py | 37 +++++++++++++++-------- tests/fixtures/v2-full/docker-compose.yml | 4 +++ 3 files changed, 30 insertions(+), 13 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 473c6d60..661c91f2 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -217,7 +217,7 @@ class TopLevelCommand(DocoptCommand): compose_config = dict( (service.pop('name'), service) for service in compose_config.services) - print(yaml.dump( + print(yaml.safe_dump( compose_config, default_flow_style=False, indent=2, diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 548c6b93..d9388199 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -10,8 +10,8 @@ import subprocess import time from collections import namedtuple from operator import attrgetter -from textwrap import dedent +import yaml from docker import errors from .. import mock @@ -148,8 +148,9 @@ class CLITestCase(DockerClientTestCase): self.base_dir = None def test_config_list_services(self): + self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--services']) - assert set(result.stdout.rstrip().split('\n')) == {'simple', 'another'} + assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} def test_config_quiet_with_error(self): self.base_dir = None @@ -160,20 +161,32 @@ class CLITestCase(DockerClientTestCase): assert "'notaservice' doesn't have any configuration" in result.stderr def test_config_quiet(self): + self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '-q']).stdout == '' def test_config_default(self): + self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config']) - assert dedent(""" - simple: - command: top - image: busybox:latest - """).lstrip() in result.stdout - assert dedent(""" - another: - command: top - image: busybox:latest - """).lstrip() in result.stdout + # assert there are no python objects encoded in the output + assert '!!' not in result.stdout + + output = yaml.load(result.stdout) + expected = { + 'version': 2, + 'volumes': {'data': {'driver': 'local'}}, + 'networks': {'front': {}}, + 'services': { + 'web': { + 'build': os.path.abspath(self.base_dir), + 'networks': ['front', 'default'], + }, + 'other': { + 'image': 'busybox:latest', + 'command': 'top', + }, + }, + } + assert output == expected def test_ps(self): self.project.get_service('simple').create_container() diff --git a/tests/fixtures/v2-full/docker-compose.yml b/tests/fixtures/v2-full/docker-compose.yml index 86d1c2c2..725296c9 100644 --- a/tests/fixtures/v2-full/docker-compose.yml +++ b/tests/fixtures/v2-full/docker-compose.yml @@ -14,7 +14,11 @@ services: networks: - front - default + volumes_from: + - other other: image: busybox:latest command: top + volumes: + - /data From 1bfbba36b27df69302f3a196834d32d3dc64987e Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 14 Jan 2016 17:30:48 -0500 Subject: [PATCH 258/359] Ensure that the config output by config command never contains python objects. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 10 ++-------- compose/config/serialize.py | 30 ++++++++++++++++++++++++++++++ compose/config/types.py | 7 +++++++ compose/service.py | 20 +++++++++----------- tests/acceptance/cli_test.py | 6 +++++- 5 files changed, 53 insertions(+), 20 deletions(-) create mode 100644 compose/config/serialize.py diff --git a/compose/cli/main.py b/compose/cli/main.py index 661c91f2..4be8536f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -9,7 +9,6 @@ import sys from inspect import getdoc from operator import attrgetter -import yaml from docker.errors import APIError from requests.exceptions import ReadTimeout @@ -18,6 +17,7 @@ from .. import __version__ from ..config import config from ..config import ConfigurationError from ..config import parse_environment +from ..config.serialize import serialize_config from ..const import DEFAULT_TIMEOUT from ..const import HTTP_TIMEOUT from ..const import IS_WINDOWS_PLATFORM @@ -215,13 +215,7 @@ class TopLevelCommand(DocoptCommand): print('\n'.join(service['name'] for service in compose_config.services)) return - compose_config = dict( - (service.pop('name'), service) for service in compose_config.services) - print(yaml.safe_dump( - compose_config, - default_flow_style=False, - indent=2, - width=80)) + print(serialize_config(compose_config)) def create(self, project, options): """ diff --git a/compose/config/serialize.py b/compose/config/serialize.py new file mode 100644 index 00000000..06e0a027 --- /dev/null +++ b/compose/config/serialize.py @@ -0,0 +1,30 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import six +import yaml + +from compose.config import types + + +def serialize_config_type(dumper, data): + representer = dumper.represent_str if six.PY3 else dumper.represent_unicode + return representer(data.repr()) + + +yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) +yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) + + +def serialize_config(config): + output = { + 'version': config.version, + 'services': {service.pop('name'): service for service in config.services}, + 'networks': config.networks, + 'volumes': config.volumes, + } + return yaml.safe_dump( + output, + default_flow_style=False, + indent=2, + width=80) diff --git a/compose/config/types.py b/compose/config/types.py index c0adca6c..b872cba9 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -67,6 +67,9 @@ class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode type')): return cls(source, mode, type) + def repr(self): + return '{v.type}:{v.source}:{v.mode}'.format(v=self) + def parse_restart_spec(restart_config): if not restart_config: @@ -156,3 +159,7 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): mode = parts[2] return cls(external, internal, mode) + + def repr(self): + external = self.external + ':' if self.external else '' + return '{ext}{v.internal}:{v.mode}'.format(ext=external, v=self) diff --git a/compose/service.py b/compose/service.py index c91c3a58..0866b83b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -460,7 +460,8 @@ class Service(object): 'links': self.get_link_names(), 'net': self.net.id, 'volumes_from': [ - (v.source.name, v.mode) for v in self.volumes_from if isinstance(v.source, Service) + (v.source.name, v.mode) + for v in self.volumes_from if isinstance(v.source, Service) ], } @@ -519,12 +520,7 @@ class Service(object): return links def _get_volumes_from(self): - volumes_from = [] - for volume_from_spec in self.volumes_from: - volumes = build_volume_from(volume_from_spec) - volumes_from.extend(volumes) - - return volumes_from + return [build_volume_from(spec) for spec in self.volumes_from] def _get_container_create_options( self, @@ -927,7 +923,7 @@ def warn_on_masked_volume(volumes_option, container_volumes, service): def build_volume_binding(volume_spec): - return volume_spec.internal, "{}:{}:{}".format(*volume_spec) + return volume_spec.internal, volume_spec.repr() def build_volume_from(volume_from_spec): @@ -938,12 +934,14 @@ def build_volume_from(volume_from_spec): if isinstance(volume_from_spec.source, Service): containers = volume_from_spec.source.containers(stopped=True) if not containers: - return ["{}:{}".format(volume_from_spec.source.create_container().id, volume_from_spec.mode)] + return "{}:{}".format( + volume_from_spec.source.create_container().id, + volume_from_spec.mode) container = containers[0] - return ["{}:{}".format(container.id, volume_from_spec.mode)] + return "{}:{}".format(container.id, volume_from_spec.mode) elif isinstance(volume_from_spec.source, Container): - return ["{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode)] + return "{}:{}".format(volume_from_spec.source.id, volume_from_spec.mode) # Labels diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index d9388199..d910473a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -177,12 +177,16 @@ class CLITestCase(DockerClientTestCase): 'networks': {'front': {}}, 'services': { 'web': { - 'build': os.path.abspath(self.base_dir), + 'build': { + 'context': os.path.abspath(self.base_dir), + }, 'networks': ['front', 'default'], + 'volumes_from': ['service:other:rw'], }, 'other': { 'image': 'busybox:latest', 'command': 'top', + 'volumes': ['/data:rw'], }, }, } From abd031cb3d0c49ca933d24d2d03a982692cfc6c4 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 14 Jan 2016 18:33:35 +0000 Subject: [PATCH 259/359] Containers join each network aliased to their service's name Signed-off-by: Aanand Prasad --- compose/const.py | 4 +--- compose/service.py | 12 +++++++++++- tests/acceptance/cli_test.py | 11 +++++------ 3 files changed, 17 insertions(+), 10 deletions(-) diff --git a/compose/const.py b/compose/const.py index 331895b1..6ff108fb 100644 --- a/compose/const.py +++ b/compose/const.py @@ -18,7 +18,5 @@ COMPOSEFILE_VERSIONS = (1, 2) API_VERSIONS = { 1: '1.21', - - # TODO: update to 1.22 when there's a Docker 1.10 build to test against - 2: '1.21', + 2: '1.22', } diff --git a/compose/service.py b/compose/service.py index 0866b83b..4409f903 100644 --- a/compose/service.py +++ b/compose/service.py @@ -427,7 +427,9 @@ class Service(object): def connect_container_to_networks(self, container): for network in self.networks: log.debug('Connecting "{}" to "{}"'.format(container.name, network)) - self.client.connect_container_to_network(container.id, network) + self.client.connect_container_to_network( + container.id, network, + aliases=[self.name]) def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): for c in self.duplicate_containers(): @@ -597,6 +599,8 @@ class Service(object): override_options, one_off=one_off) + container_options['networking_config'] = self._get_container_networking_config() + return container_options def _get_container_host_config(self, override_options, one_off=False): @@ -631,6 +635,12 @@ class Service(object): cpu_quota=options.get('cpu_quota'), ) + def _get_container_networking_config(self): + return self.client.create_networking_config({ + network_name: self.client.create_endpoint_config(aliases=[self.name]) + for network_name in self.networks + }) + def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index d910473a..8f3cdf50 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -133,12 +133,8 @@ class CLITestCase(DockerClientTestCase): self.client.exec_start(exc) return self.client.exec_inspect(exc)['ExitCode'] - def lookup(self, container, service_name): - exit_code = self.execute(container, [ - "nslookup", - "{}_{}_1".format(self.project.name, service_name) - ]) - return exit_code == 0 + def lookup(self, container, hostname): + return self.execute(container, ["nslookup", hostname]) == 0 def test_help(self): self.base_dir = 'tests/fixtures/no-composefile' @@ -414,6 +410,9 @@ class CLITestCase(DockerClientTestCase): networks = list(container.get('NetworkSettings.Networks')) self.assertEqual(networks, [network['Name']]) + for service in services: + assert self.lookup(container, service.name) + def test_up_with_networks(self): self.base_dir = 'tests/fixtures/networks' self.dispatch(['up', '-d'], None) From 406b6b28f498d814e3517fdf66eecae137bcd985 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 Jan 2016 00:10:57 +0000 Subject: [PATCH 260/359] Tag v2-only tests - Don't run them against Engine < 1.10 - Set the API version appropriately for the Engine version, so all tests use API version 1.22 against Engine 1.10 Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 9 +++++++++ tests/integration/project_test.py | 10 ++++++++++ tests/integration/testcases.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 49 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 8f3cdf50..5978dd5d 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -20,6 +20,7 @@ from compose.container import Container from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox +from tests.integration.testcases import v2_only ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -388,6 +389,7 @@ class CLITestCase(DockerClientTestCase): assert 'simple_1 | simple' in result.stdout assert 'another_1 | another' in result.stdout + @v2_only() def test_up(self): self.base_dir = 'tests/fixtures/v2-simple' self.dispatch(['up', '-d'], None) @@ -413,6 +415,7 @@ class CLITestCase(DockerClientTestCase): for service in services: assert self.lookup(container, service.name) + @v2_only() def test_up_with_networks(self): self.base_dir = 'tests/fixtures/networks' self.dispatch(['up', '-d'], None) @@ -448,6 +451,7 @@ class CLITestCase(DockerClientTestCase): # app can see db assert self.lookup(app_container, "db") + @v2_only() def test_up_missing_network(self): self.base_dir = 'tests/fixtures/networks' @@ -457,6 +461,7 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr + @v2_only() def test_up_predefined_networks(self): filename = 'predefined-networks.yml' @@ -476,6 +481,7 @@ class CLITestCase(DockerClientTestCase): assert list(container.get('NetworkSettings.Networks')) == [name] assert container.get('HostConfig.NetworkMode') == name + @v2_only() def test_up_external_networks(self): filename = 'external-networks.yml' @@ -499,6 +505,7 @@ class CLITestCase(DockerClientTestCase): container = self.project.containers()[0] assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted(network_names) + @v2_only() def test_up_no_services(self): self.base_dir = 'tests/fixtures/no-services' self.dispatch(['up', '-d'], None) @@ -513,6 +520,7 @@ class CLITestCase(DockerClientTestCase): for name in ['bar', 'foo'] ] + @v2_only() def test_up_with_links_is_invalid(self): self.base_dir = 'tests/fixtures/v2-simple' @@ -853,6 +861,7 @@ class CLITestCase(DockerClientTestCase): container, = service.containers(stopped=True, one_off=True) self.assertEqual(container.name, name) + @v2_only() def test_run_with_networking(self): self.base_dir = 'tests/fixtures/v2-simple' self.dispatch(['run', 'simple', 'true'], None) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ef8a084b..d29d9f1e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -14,6 +14,7 @@ from compose.const import LABEL_PROJECT from compose.container import Container from compose.project import Project from compose.service import ConvergenceStrategy +from tests.integration.testcases import v2_only def build_service_dicts(service_config): @@ -482,6 +483,7 @@ class ProjectTest(DockerClientTestCase): service = project.get_service('web') self.assertEqual(len(service.containers()), 1) + @v2_only() def test_project_up_networks(self): config_data = config.Config( version=2, @@ -514,6 +516,7 @@ class ProjectTest(DockerClientTestCase): foo_data = self.client.inspect_network('composetest_foo') self.assertEqual(foo_data['Driver'], 'bridge') + @v2_only() def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -539,6 +542,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v2_only() def test_project_up_logging_with_multiple_files(self): base_file = config.ConfigFile( 'base.yml', @@ -590,6 +594,7 @@ class ProjectTest(DockerClientTestCase): self.assertTrue(log_config) self.assertEqual(log_config.get('Type'), 'none') + @v2_only() def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -614,6 +619,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v2_only() def test_project_up_implicit_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -638,6 +644,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v2_only() def test_initialize_volumes_invalid_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) @@ -659,6 +666,7 @@ class ProjectTest(DockerClientTestCase): with self.assertRaises(config.ConfigurationError): project.initialize_volumes() + @v2_only() def test_initialize_volumes_updated_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) @@ -696,6 +704,7 @@ class ProjectTest(DockerClientTestCase): vol_name ) in str(e.exception) + @v2_only() def test_initialize_volumes_external_volumes(self): # Use composetest_ prefix so it gets garbage-collected in tearDown() vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) @@ -722,6 +731,7 @@ class ProjectTest(DockerClientTestCase): with self.assertRaises(NotFound): self.client.inspect_volume(full_vol_name) + @v2_only() def test_initialize_volumes_inexistent_external_volume(self): vol_name = '{0:x}'.format(random.getrandbits(32)) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 3002539e..5870946d 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -1,12 +1,16 @@ from __future__ import absolute_import from __future__ import unicode_literals +import functools +import os + from docker.utils import version_lt from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment +from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output from compose.service import Service @@ -26,10 +30,35 @@ def get_links(container): return [format_link(link) for link in links] +def engine_version_too_low_for_v2(): + if 'DOCKER_VERSION' not in os.environ: + return False + version = os.environ['DOCKER_VERSION'].partition('-')[0] + return version_lt(version, '1.10') + + +def v2_only(): + def decorator(f): + @functools.wraps(f) + def wrapper(self, *args, **kwargs): + if engine_version_too_low_for_v2(): + skip("Engine version is too low") + return + return f(self, *args, **kwargs) + return wrapper + + return decorator + + class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - cls.client = docker_client() + if engine_version_too_low_for_v2(): + version = API_VERSIONS[1] + else: + version = API_VERSIONS[2] + + cls.client = docker_client(version) def tearDown(self): for c in self.client.containers( From ab2d18851f9c42153d8460494c4020ea0ca5f079 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 Jan 2016 00:12:07 +0000 Subject: [PATCH 261/359] Test against a dev build of Engine 1.10 Signed-off-by: Aanand Prasad --- script/test-versions | 12 +++++++++--- tox.ini | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/script/test-versions b/script/test-versions index 76e55e11..24412b91 100755 --- a/script/test-versions +++ b/script/test-versions @@ -18,8 +18,7 @@ get_versions="docker run --rm if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - # TODO: `-n 2` when engine 1.10 releases - DOCKER_VERSIONS="$($get_versions recent -n 1)" + DOCKER_VERSIONS="1.9.1 1.10.0-dev" fi @@ -39,12 +38,18 @@ for version in $DOCKER_VERSIONS; do trap "on_exit" EXIT + if [[ $version == *"-dev" ]]; then + repo="dnephin/dind" + else + repo="dockerswarm/dind" + fi + docker run \ -d \ --name "$daemon_container" \ --privileged \ --volume="/var/lib/docker" \ - dockerswarm/dind:$version \ + "$repo:$version" \ docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ 2>&1 | tail -n 10 @@ -52,6 +57,7 @@ for version in $DOCKER_VERSIONS; do --rm \ --link="$daemon_container:docker" \ --env="DOCKER_HOST=tcp://docker:2375" \ + --env="DOCKER_VERSION=$version" \ --entrypoint="tox" \ "$TAG" \ -e py27,py34 -- "$@" diff --git a/tox.ini b/tox.ini index 9d45b0c7..dc85bc6d 100644 --- a/tox.ini +++ b/tox.ini @@ -8,6 +8,7 @@ passenv = DOCKER_HOST DOCKER_CERT_PATH DOCKER_TLS_VERIFY + DOCKER_VERSION setenv = HOME=/tmp deps = From cba75627e18adf80f66b6d090800b2204cfa97e0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 Jan 2016 00:12:46 +0000 Subject: [PATCH 262/359] Fix error when joining host/bridge network Signed-off-by: Aanand Prasad --- compose/service.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/service.py b/compose/service.py index 4409f903..f5db07fb 100644 --- a/compose/service.py +++ b/compose/service.py @@ -639,6 +639,7 @@ class Service(object): return self.client.create_networking_config({ network_name: self.client.create_endpoint_config(aliases=[self.name]) for network_name in self.networks + if network_name not in ['host', 'bridge'] }) def build(self, no_cache=False, pull=False, force_rm=False): From fbc275e06b39a9fe02d57cd23aae23d25a5a73c9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 Jan 2016 00:33:04 +0000 Subject: [PATCH 263/359] Work around error message change in Engine Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 5978dd5d..39e154ad 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -238,7 +238,8 @@ class CLITestCase(DockerClientTestCase): assert 'Pulling simple (busybox:latest)...' in result.stderr assert 'Pulling another (nonexisting-image:latest)...' in result.stderr - assert 'Error: image library/nonexisting-image:latest not found' in result.stderr + assert 'Error: image library/nonexisting-image' in result.stderr + assert 'not found' in result.stderr def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' From 4772815491e3fb1ae838f1e834cb160aedc1c19a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 Jan 2016 00:33:44 +0000 Subject: [PATCH 264/359] Disable tests until Engine 1.10 change has been worked around Signed-off-by: Aanand Prasad --- tests/integration/service_test.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 0e91dcf7..bce3999b 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -7,6 +7,7 @@ import tempfile from os import path from docker.errors import APIError +from pytest import mark from six import StringIO from six import text_type @@ -371,6 +372,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) self.assertEqual(db.containers()[0].environment['FOO'], 'BAR') + @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_container_creates_links(self): db = self.create_service('db') web = self.create_service('web', links=[(db, None)]) @@ -387,6 +389,7 @@ class ServiceTest(DockerClientTestCase): 'db']) ) + @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_container_creates_links_with_names(self): db = self.create_service('db') web = self.create_service('web', links=[(db, 'custom_link_name')]) @@ -430,6 +433,7 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db) self.assertEqual(set(get_links(c)), set([])) + @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') From de6d6a42d7ea112342a5ea419fc93af0fb5ecad6 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 15 Jan 2016 01:49:54 +0000 Subject: [PATCH 265/359] Tag some more v2-dependent tests Not clear why the config tests are v2-dependent; needs investigating Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 39e154ad..231b78db 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -144,11 +144,15 @@ class CLITestCase(DockerClientTestCase): # Prevent tearDown from trying to create a project self.base_dir = None + # TODO: this shouldn't be v2-dependent + @v2_only() def test_config_list_services(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--services']) assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} + # TODO: this shouldn't be v2-dependent + @v2_only() def test_config_quiet_with_error(self): self.base_dir = None result = self.dispatch([ @@ -157,10 +161,14 @@ class CLITestCase(DockerClientTestCase): ], returncode=1) assert "'notaservice' doesn't have any configuration" in result.stderr + # TODO: this shouldn't be v2-dependent + @v2_only() def test_config_quiet(self): self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '-q']).stdout == '' + # TODO: this shouldn't be v2-dependent + @v2_only() def test_config_default(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config']) @@ -354,6 +362,7 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['down', '--rmi', 'bogus'], returncode=1) assert '--rmi flag must be' in result.stderr + @v2_only() def test_down(self): self.base_dir = 'tests/fixtures/v2-full' self.dispatch(['up', '-d']) @@ -939,6 +948,7 @@ class CLITestCase(DockerClientTestCase): result = self.dispatch(['start'], returncode=1) assert 'No containers to start' in result.stderr + @v2_only() def test_up_logging(self): self.base_dir = 'tests/fixtures/logging-composefile' self.dispatch(['up', '-d']) From 82b288b25b4306157b3edc3dfc5fca8b8285044a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 15 Jan 2016 15:02:19 -0500 Subject: [PATCH 266/359] Fix linux master build. Signed-off-by: Daniel Nephin --- script/travis/build-binary | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/script/travis/build-binary b/script/travis/build-binary index 0becee7f..7cc1092d 100755 --- a/script/travis/build-binary +++ b/script/travis/build-binary @@ -4,8 +4,8 @@ set -ex if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then script/build-linux - script/build-image master - # TODO: requires auth + # TODO: requires auth to push, so disable for now + # script/build-image master # docker push docker/compose:master else script/prepare-osx From 89e31f7a8d50d743398815d3c2f0bde3c7fd9c01 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 14 Jan 2016 22:28:20 -0500 Subject: [PATCH 267/359] Validate that an extended config file has the same version as the base. Signed-off-by: Daniel Nephin --- compose/config/config.py | 40 ++++++++++++++++---------------- tests/unit/config/config_test.py | 37 +++++++++++++++++++++++------ 2 files changed, 50 insertions(+), 27 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index b3894225..ac5e8d17 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -188,10 +188,10 @@ def find(base_dir, filenames): [ConfigFile.from_filename(f) for f in filenames]) -def validate_config_version(config_details): - main_file = config_details.config_files[0] +def validate_config_version(config_files): + main_file = config_files[0] validate_top_level_object(main_file) - for next_file in config_details.config_files[1:]: + for next_file in config_files[1:]: validate_top_level_object(next_file) if main_file.version != next_file.version: @@ -254,7 +254,7 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ - validate_config_version(config_details) + validate_config_version(config_details.config_files) processed_files = [ process_config_file(config_file) @@ -267,9 +267,8 @@ def load(config_details): networks = load_mapping(config_details.config_files, 'get_networks', 'Network') service_dicts = load_services( config_details.working_dir, - main_file.filename, - [file.get_service_dicts() for file in config_details.config_files], - main_file.version) + main_file, + [file.get_service_dicts() for file in config_details.config_files]) return Config(main_file.version, service_dicts, volumes, networks) @@ -303,21 +302,21 @@ def load_mapping(config_files, get_func, entity_type): return mapping -def load_services(working_dir, filename, service_configs, version): +def load_services(working_dir, config_file, service_configs): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( working_dir, - filename, + config_file.filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config, version) + resolver = ServiceExtendsResolver(service_config, config_file) service_dict = process_service(resolver.run()) - validate_service(service_dict, service_config.name, version) + validate_service(service_dict, service_config.name, config_file.version) service_dict = finalize_service( service_config._replace(config=service_dict), service_names, - version) + config_file.version) return service_dict def build_services(service_config): @@ -333,7 +332,7 @@ def load_services(working_dir, filename, service_configs, version): name: merge_service_dicts_from_files( base.get(name, {}), override.get(name, {}), - version) + config_file.version) for name in all_service_names } @@ -373,11 +372,11 @@ def process_config_file(config_file, service_name=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, version, already_seen=None): + def __init__(self, service_config, config_file, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] - self.version = version + self.config_file = config_file @property def signature(self): @@ -404,8 +403,10 @@ class ServiceExtendsResolver(object): config_path = self.get_extended_config_path(extends) service_name = extends['service'] + extends_file = ConfigFile.from_filename(config_path) + validate_config_version([self.config_file, extends_file]) extended_file = process_config_file( - ConfigFile.from_filename(config_path), + extends_file, service_name=service_name) service_config = extended_file.config[service_name] return config_path, service_config, service_name @@ -417,7 +418,7 @@ class ServiceExtendsResolver(object): extended_config_path, service_name, service_dict), - self.version, + self.config_file, already_seen=self.already_seen + [self.signature]) service_config = resolver.run() @@ -425,13 +426,12 @@ class ServiceExtendsResolver(object): validate_extended_service_dict( other_service_dict, extended_config_path, - service_name, - ) + service_name) return merge_service_dicts( other_service_dict, self.service_config.config, - self.version) + self.config_file.version) def get_extended_config_path(self, extends_options): """Service we are extending either has a value for 'file' set, which we diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e24dc904..cc205136 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -25,14 +25,15 @@ V1 = 1 def make_service_dict(name, service_dict, working_dir, filename=None): + """Test helper function to construct a ServiceExtendsResolver """ - Test helper function to construct a ServiceExtendsResolver - """ - resolver = config.ServiceExtendsResolver(config.ServiceConfig( - working_dir=working_dir, - filename=filename, - name=name, - config=service_dict), version=1) + resolver = config.ServiceExtendsResolver( + config.ServiceConfig( + working_dir=working_dir, + filename=filename, + name=name, + config=service_dict), + config.ConfigFile(filename=filename, config={})) return config.process_service(resolver.run()) @@ -1888,6 +1889,28 @@ class ExtendsTest(unittest.TestCase): assert config == expected + def test_extends_with_mixed_versions_is_error(self): + tmpdir = py.test.ensuretemp('test_extends_with_mixed_version') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: 2 + services: + web: + extends: + file: base.yml + service: base + image: busybox + """) + tmpdir.join('base.yml').write(""" + base: + volumes: ['/foo'] + ports: ['3000:3000'] + """) + + with pytest.raises(ConfigurationError) as exc: + load_from_filename(str(tmpdir.join('docker-compose.yml'))) + assert 'Version mismatch' in exc.exconly() + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From 34fd042dbf0e3de5223f1cc333cf94437d311845 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 15 Jan 2016 10:14:55 -0500 Subject: [PATCH 268/359] Increase the timeout for all acceptance tests. Signed-off-by: Daniel Nephin --- tests/acceptance/cli_test.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 231b78db..700e9cdf 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -48,7 +48,7 @@ def wait_on_process(proc, returncode=0): return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) -def wait_on_condition(condition, delay=0.1, timeout=20): +def wait_on_condition(condition, delay=0.1, timeout=40): start_time = time.time() while not condition(): if time.time() - start_time > timeout: @@ -648,14 +648,14 @@ class CLITestCase(DockerClientTestCase): wait_on_condition(ContainerCountCondition(self.project, 2)) os.kill(proc.pid, signal.SIGINT) - wait_on_condition(ContainerCountCondition(self.project, 0), timeout=30) + wait_on_condition(ContainerCountCondition(self.project, 0)) def test_up_handles_sigterm(self): proc = start_process(self.base_dir, ['up', '-t', '2']) wait_on_condition(ContainerCountCondition(self.project, 2)) os.kill(proc.pid, signal.SIGTERM) - wait_on_condition(ContainerCountCondition(self.project, 0), timeout=30) + wait_on_condition(ContainerCountCondition(self.project, 0)) def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' From fffedfc87b4aa53ee2fecc9ba950309025b8b8be Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 15 Jan 2016 19:04:01 -0500 Subject: [PATCH 269/359] Test against 1.10rc1. Signed-off-by: Daniel Nephin --- script/test-versions | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/script/test-versions b/script/test-versions index 24412b91..2e9c9167 100755 --- a/script/test-versions +++ b/script/test-versions @@ -18,7 +18,7 @@ get_versions="docker run --rm if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="1.9.1 1.10.0-dev" + DOCKER_VERSIONS=$($get_versions -n 2 recent) fi @@ -38,11 +38,7 @@ for version in $DOCKER_VERSIONS; do trap "on_exit" EXIT - if [[ $version == *"-dev" ]]; then - repo="dnephin/dind" - else - repo="dockerswarm/dind" - fi + repo="dockerswarm/dind" docker run \ -d \ From a22d2483908889c627fe5cb87f32b6b252111048 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 18 Jan 2016 16:15:52 +0000 Subject: [PATCH 270/359] Quote network names in error messages Signed-off-by: Aanand Prasad --- compose/network.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/compose/network.py b/compose/network.py index eaad770c..34159fd6 100644 --- a/compose/network.py +++ b/compose/network.py @@ -44,11 +44,11 @@ class Network(object): data = self.inspect() if self.driver and data['Driver'] != self.driver: raise ConfigurationError( - 'Network {} needs to be recreated - driver has changed' + 'Network "{}" needs to be recreated - driver has changed' .format(self.full_name)) if data['Options'] != (self.driver_opts or {}): raise ConfigurationError( - 'Network {} needs to be recreated - options have changed' + 'Network "{}" needs to be recreated - options have changed' .format(self.full_name)) except NotFound: driver_name = 'the default driver' From 64fc2b85cb0e1d6d8910f648f03c4f05f7e6b50a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 18 Jan 2016 16:16:24 +0000 Subject: [PATCH 271/359] Allow overriding of config for the default network Signed-off-by: Aanand Prasad --- compose/project.py | 31 ++++++++++--------- tests/acceptance/cli_test.py | 13 ++++++++ .../networks/default-network-config.yml | 13 ++++++++ 3 files changed, 42 insertions(+), 15 deletions(-) create mode 100644 tests/fixtures/networks/default-network-config.yml diff --git a/compose/project.py b/compose/project.py index 1322c990..777e8f82 100644 --- a/compose/project.py +++ b/compose/project.py @@ -58,23 +58,24 @@ class Project(object): use_networking = (config_data.version and config_data.version >= 2) project = cls(name, [], client, use_networking=use_networking) - custom_networks = [] - if config_data.networks: - for network_name, data in config_data.networks.items(): - custom_networks.append( - Network( - client=client, project=name, name=network_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - external_name=data.get('external_name'), - ) - ) + network_config = config_data.networks or {} + custom_networks = [ + Network( + client=client, project=name, name=network_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external_name=data.get('external_name'), + ) + for network_name, data in network_config.items() + ] + + all_networks = custom_networks[:] + if 'default' not in network_config: + all_networks.append(project.default_network) for service_dict in config_data.services: if use_networking: - networks = get_networks( - service_dict, - custom_networks + [project.default_network]) + networks = get_networks(service_dict, all_networks) net = Net(networks[0]) if networks else Net("none") links = [] else: @@ -96,7 +97,7 @@ class Project(object): **service_dict)) project.networks += custom_networks - if project.uses_default_network(): + if 'default' not in network_config and project.uses_default_network(): project.networks.append(project.default_network) if config_data.volumes: diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 700e9cdf..f5c16380 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -409,6 +409,7 @@ class CLITestCase(DockerClientTestCase): networks = self.client.networks(names=[self.project.default_network.full_name]) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['Driver'], 'bridge') + assert 'com.docker.network.bridge.enable_icc' not in networks[0]['Options'] network = self.client.inspect_network(networks[0]['Id']) @@ -425,6 +426,18 @@ class CLITestCase(DockerClientTestCase): for service in services: assert self.lookup(container, service.name) + @v2_only() + def test_up_with_default_network_config(self): + filename = 'default-network-config.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + self.dispatch(['-f', filename, 'up', '-d'], None) + + networks = self.client.networks(names=[self.project.default_network.full_name]) + assert networks[0]['Options']['com.docker.network.bridge.enable_icc'] == 'false' + @v2_only() def test_up_with_networks(self): self.base_dir = 'tests/fixtures/networks' diff --git a/tests/fixtures/networks/default-network-config.yml b/tests/fixtures/networks/default-network-config.yml new file mode 100644 index 00000000..275fae98 --- /dev/null +++ b/tests/fixtures/networks/default-network-config.yml @@ -0,0 +1,13 @@ +version: 2 +services: + simple: + image: busybox:latest + command: top + another: + image: busybox:latest + command: top +networks: + default: + driver: bridge + driver_opts: + "com.docker.network.bridge.enable_icc": "false" From d4720f85ef98c5002140527a844284b86a9718d3 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 18 Jan 2016 15:04:10 +0000 Subject: [PATCH 272/359] Update networking docs Signed-off-by: Aanand Prasad --- docs/networking.md | 136 +++++++++++++++++++++++++++++---------------- 1 file changed, 89 insertions(+), 47 deletions(-) diff --git a/docs/networking.md b/docs/networking.md index 91ac0b73..f111f730 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -12,83 +12,125 @@ weight=6 # Networking in Compose -> **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. +> **Note:** This document only applies if you're using v2 of the [Compose file format](compose-file.md). Networking features are not supported for legacy Compose files. -Compose sets up a single default +By default Compose sets up a single [network](/engine/reference/commandline/network_create.md) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the container name. -> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. +> **Note:** Your app's network is given a name based on the "project name", which is based on the name of the directory it lives in. You can override the project name with either the [`--project-name` flag](reference/docker-compose.md) or the [`COMPOSE_PROJECT_NAME` environment variable](reference/overview.md#compose-project-name). For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: - web: - build: . - ports: - - "8000:8000" - db: - image: postgres + version: 2 -When you run `docker-compose --x-networking up`, the following happens: + services: + web: + build: . + ports: + - "8000:8000" + db: + image: postgres -1. A network called `myapp` is created. -2. A container is created using `web`'s configuration. It joins the network -`myapp` under the name `myapp_web_1`. -3. A container is created using `db`'s configuration. It joins the network -`myapp` under the name `myapp_db_1`. +When you run `docker-compose up`, the following happens: -Each container can now look up the hostname `myapp_web_1` or `myapp_db_1` and +1. A network called `myapp_default` is created. +2. A container is created using `web`'s configuration. It joins the network + `myapp_default` under the name `web`. +3. A container is created using `db`'s configuration. It joins the network + `myapp_default` under the name `db`. + +Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s -application code could connect to the URL `postgres://myapp_db_1:5432` and start +application code could connect to the URL `postgres://db:5432` and start using the Postgres database. Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. -> **Note:** in the next release there will be additional aliases for the -> container, including a short name without the project name and container -> index. The full container name will remain as one of the alias for backwards -> compatibility. - ## Updating containers If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. If any containers have connections open to the old container, they will be closed. It is a container's responsibility to detect this condition, look up the name again and reconnect. -## Configure how services are published - -By default, containers for each service are published on the network with the -container name. If you want to change the name, or stop containers from being -discoverable at all, you can use the `container_name` option: - - web: - build: . - container_name: "my-web-application" - ## Links -Docker links are a one-way, single-host communication system. They should now be considered deprecated, and you should update your app to use networking instead. In the majority of cases, this will simply involve removing the `links` sections from your `docker-compose.yml`. +Docker links are a one-way, single-host communication system. They should now be considered deprecated, and as part of upgrading your app to the v2 format, you must remove any `links` sections from your `docker-compose.yml` and use service names (e.g. `web`, `db`) as the hostnames to connect to. -## Specifying the network driver - -By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](/engine/extend/plugins_network.md). - -You can specify which one to use with the `--x-network-driver` flag: - - $ docker-compose --x-networking --x-network-driver=overlay up - - +When deploying a Compose application to a Swarm cluster, you can make use of the built-in `overlay` driver to enable multi-host communication between containers with no changes to application code. Consult the [Getting started with multi-host networking](/engine/userguide/networking/get-started-overlay.md) to see how to set up the overlay driver, and then specify `driver: overlay` in your networking config (see the sections below for how to do this). + +## Specifying custom networks + +Instead of just using the default app network, you can specify your own networks with the top-level `networks` key. This lets you create more complex topologies and specify [custom network drivers](/engine/extend/plugins_network.md) and options. You can also use it to connect services to externally-created networks which aren't managed by Compose. + +Each service can specify what networks to connect to with the *service-level* `networks` key, which is a list of names referencing entries under the *top-level* `networks` key. + +Here's an example Compose file defining several networks. The `proxy` service is the gateway to the outside world, via a network called `outside` which is expected to already exist. `proxy` is isolated from the `db` service, because they do not share a network in common - only `app` can talk to both. + + version: 2 + + services: + proxy: + build: ./proxy + networks: + - outside + - front + app: + build: ./app + networks: + - front + - back + db: + image: postgres + networks: + - back + + networks: + front: + # Use the overlay driver for multi-host communication + driver: overlay + back: + # Use a custom driver which takes special options + driver: my-custom-driver + options: + foo: "1" + bar: "2" + outside: + # The 'outside' network is expected to already exist - Compose will not + # attempt to create it + external: true + +## Configuring the default network + +Instead of (or as well as) specifying your own networks, you can also change the settings of the app-wide default network by defining an entry under `networks` named `default`: + + version: 2 + + services: + web: + build: . + ports: + - "8000:8000" + db: + image: postgres + + networks: + default: + # Use the overlay driver for multi-host communication + driver: overlay ## Custom container network modes -Compose allows you to specify a custom network mode for a service with the `net` option - for example, `net: "host"` specifies that its containers should use the same network namespace as the Docker host, and `net: "none"` specifies that they should have no networking capabilities. +The `docker` CLI command allows you to specify a custom network mode for a container with the `--net` option - for example, `--net=host` specifies that the container should use the same network namespace as the Docker host, and `--net=none` specifies that it should have no networking capabilities. -If a service specifies the `net` option, its containers will *not* join the app’s network and will not be able to communicate with other services in the app. +To make use of this in Compose, specify a `networks` list with a single item `host`, `bridge` or `none`: -If *all* services in an app specify the `net` option, a network will not be created at all. + app: + build: ./app + networks: ["host"] + +There is no equivalent to `--net=container:CONTAINER_NAME` in the v2 Compose file format. You should instead use networks to enable communication. From 237f134a0096210f43abc1cb9c73ab40c7e8853e Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 18 Jan 2016 17:08:50 +0000 Subject: [PATCH 273/359] Allow custom ipam config Signed-off-by: Aanand Prasad --- compose/network.py | 28 +++++++++++++-- compose/project.py | 1 + tests/integration/project_test.py | 57 +++++++++++++++++++++++++++++++ 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/compose/network.py b/compose/network.py index 34159fd6..4f4f5522 100644 --- a/compose/network.py +++ b/compose/network.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals import logging from docker.errors import NotFound +from docker.utils import create_ipam_config +from docker.utils import create_ipam_pool from .config import ConfigurationError @@ -13,12 +15,13 @@ log = logging.getLogger(__name__) class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - external_name=None): + ipam=None, external_name=None): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts + self.ipam = create_ipam_config_from_dict(ipam) self.external_name = external_name def ensure(self): @@ -61,7 +64,10 @@ class Network(object): ) self.client.create_network( - self.full_name, self.driver, self.driver_opts + name=self.full_name, + driver=self.driver, + options=self.driver_opts, + ipam=self.ipam, ) def remove(self): @@ -80,3 +86,21 @@ class Network(object): if self.external_name: return self.external_name return '{0}_{1}'.format(self.project, self.name) + + +def create_ipam_config_from_dict(ipam_dict): + if not ipam_dict: + return None + + return create_ipam_config( + driver=ipam_dict.get('driver'), + pool_configs=[ + create_ipam_pool( + subnet=config.get('subnet'), + iprange=config.get('ip_range'), + gateway=config.get('gateway'), + aux_addresses=config.get('aux_addresses'), + ) + for config in ipam_dict.get('config', []) + ], + ) diff --git a/compose/project.py b/compose/project.py index 777e8f82..080c4c49 100644 --- a/compose/project.py +++ b/compose/project.py @@ -64,6 +64,7 @@ class Project(object): client=client, project=name, name=network_name, driver=data.get('driver'), driver_opts=data.get('driver_opts'), + ipam=data.get('ipam'), external_name=data.get('external_name'), ) for network_name, data in network_config.items() diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d29d9f1e..d3fbb71e 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -516,6 +516,63 @@ class ProjectTest(DockerClientTestCase): foo_data = self.client.inspect_network('composetest_foo') self.assertEqual(foo_data['Driver'], 'bridge') + @v2_only() + def test_up_with_ipam_config(self): + config_data = config.Config( + version=2, + services=[], + volumes={}, + networks={ + 'front': { + 'driver': 'bridge', + 'driver_opts': { + "com.docker.network.bridge.enable_icc": "false", + }, + 'ipam': { + 'driver': 'default', + 'config': [{ + "subnet": "172.28.0.0/16", + "ip_range": "172.28.5.0/24", + "gateway": "172.28.5.254", + "aux_addresses": { + "a": "172.28.1.5", + "b": "172.28.1.6", + "c": "172.28.1.7", + }, + }], + }, + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + + network = self.client.networks(names=['composetest_front'])[0] + + assert network['Options'] == { + "com.docker.network.bridge.enable_icc": "false" + } + + assert network['IPAM'] == { + 'Driver': 'default', + 'Options': None, + 'Config': [{ + 'Subnet': "172.28.0.0/16", + 'IPRange': "172.28.5.0/24", + 'Gateway': "172.28.5.254", + 'AuxiliaryAddresses': { + 'a': '172.28.1.5', + 'b': '172.28.1.6', + 'c': '172.28.1.7', + }, + }], + } + @v2_only() def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) From d2556a1347a7a93171431b0d17da9f127142f6a0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 15 Jan 2016 17:51:02 -0500 Subject: [PATCH 274/359] Bump 1.6.0-rc1 Signed-off-by: Daniel Nephin --- CHANGELOG.md | 108 ++++++++++++++++++++++++++++++++++++++++++++ compose/__init__.py | 2 +- docs/install.md | 6 +-- script/run.ps1 | 2 +- script/run.sh | 2 +- 5 files changed, 114 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 79aee75f..0251d669 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,114 @@ Change log ========== +1.6.0 (2016-01-15) +------------------ + +Major Features: + +- Compose 1.6 introduces a new format for `docker-compose.yml` which lets + you define networks and volumes in the Compose file as well as services. It + also makes a few changes to the structure of some configuration options. + + You don't have to use it - your existing Compose files will run on Compose + 1.6 exactly as they do today. + + Check the upgrade guide for full details: + https://docs.docker.com/compose/compose-file/upgrading + +- Support for networking has exited experimental status and is the recommended + way to enable communication between containers. + + If you use the new file format, your app will use networking. If you want to + keep using links, just leave your Compose file as it is and it'll continue + to work just the same. + + By default, you don't have to configure any networks. In fact, using + networking with Compose involves even less configuration than using links. + Consult the networking guide for how to use it: + https://docs.docker.com/compose/networking + + The experimental flags `--x-networking` and `--x-network-driver`, introduced + in Compose 1.5, have been removed. + +- You can now pass arguments to a build if you're using the new file format: + + build: + context: . + args: + buildno: 1 + +- You can now specify both a `build` and an `image` key if you're using the + new file format. `docker-compose build` will build the image and tag it with + the name you've specified, while `docker-compose pull` will attempt to pull + it. + +- There's a new `events` command for monitoring container events from + the application, much like `docker events`. This is a good primitive for + building tools on top of Compose for performing actions when particular + things happen, such as containers starting and stopping. + + +New Features: + +- Added a new command `config` which validates and prints the Compose + configuration after interpolating variables, resolving relative paths, and + merging multiple files and `extends`. + +- Added a new command `create` for creating containers without starting them. + +- Added a new command `down` to stop and remove all the resources created by + `up` in a single command. + +- Added support for the `cpu_quota` configuration option. + +- Added support for the `stop_signal` configuration option. + +- Commands `start`, `restart`, `pause`, and `unpause` now exit with an + error status code if no containers were modified. + +- Added a new `--abort-on-container-exit` flag to `up` which causes `up` to + stop all container and exit once the first container exits. + +- Removed support for `FIG_FILE`, `FIG_PROJECT_NAME`, and no longer reads + `fig.yml` as a default Compose file location. + +- Removed the `migrate-to-labels` command. + +- Removed the `--allow-insecure-ssl` flag. + + +Bug Fixes: + +- Fixed a validation bug that prevented the use of a range of ports in + the `expose` field. + +- Fixed a validation bug that prevented the use of arrays in the `entrypoint` + field if they contained duplicate entries. + +- Fixed a bug that caused `ulimits` to be ignored when used with `extends`. + +- Fixed a bug that prevented ipv6 addresses in `extra_hosts`. + +- Fixed a bug that caused `extends` to be ignored when included from + multiple Compose files. + +- Fixed an incorrect warning when a container volume was defined in + the Compose file. + +- Fixed a bug that prevented the force shutdown behaviour of `up` and + `logs`. + +- Fixed a bug that caused `None` to be printed as the network driver name + when the default network driver was used. + +- Fixed a bug where using the string form of `dns` or `dns_search` would + cause an error. + +- Fixed a bug where a container would be reported as "Up" when it was + in the restarting state. + + 1.5.2 (2015-12-03) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 3ba90fde..52d0e7bf 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.6.0dev' +__version__ = '1.6.0rc1' diff --git a/docs/install.md b/docs/install.md index 417a48c1..944e9190 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,7 +39,7 @@ which the release page specifies, in your terminal. The following is an example command illustrating the format: - curl -L https://github.com/docker/compose/releases/download/1.5.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.6.0rc1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose If you have problems installing with `curl`, see [Alternative Install Options](#alternative-install-options). @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.5.2 + docker-compose version: 1.6.0rc1 ## Alternative install options @@ -77,7 +77,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.2/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.6.0rc1/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.ps1 b/script/run.ps1 index 47ec5469..f4ff2abb 100644 --- a/script/run.ps1 +++ b/script/run.ps1 @@ -5,7 +5,7 @@ # $Env:DOCKER_COMPOSE_OPTIONS. if ($Env:DOCKER_COMPOSE_VERSION -eq $null -or $Env:DOCKER_COMPOSE_VERSION.Length -eq 0) { - $Env:DOCKER_COMPOSE_VERSION = "latest" + $Env:DOCKER_COMPOSE_VERSION = "1.6.0rc1" } if ($Env:DOCKER_COMPOSE_OPTIONS -eq $null) { diff --git a/script/run.sh b/script/run.sh index 087c2692..3fbc60e0 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.2" +VERSION="1.6.0rc1" IMAGE="docker/compose:$VERSION" From 20c936a25179cad087522e49e409e0ac3db7f96a Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 18 Jan 2016 15:55:30 -0500 Subject: [PATCH 275/359] Fix some bugs in release scripts. Signed-off-by: Daniel Nephin --- script/release/contributors | 11 +++++++---- script/release/push-release | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/script/release/contributors b/script/release/contributors index bb9fe871..1e69b143 100755 --- a/script/release/contributors +++ b/script/release/contributors @@ -18,10 +18,13 @@ PREV_RELEASE=$1 VERSION=HEAD URL="https://api.github.com/repos/docker/compose/compare" -curl -sf "$URL/$PREV_RELEASE...$VERSION" | \ +contribs=$(curl -sf "$URL/$PREV_RELEASE...$VERSION" | \ jq -r '.commits[].author.login' | \ sort | \ uniq -c | \ - sort -nr | \ - awk '{print "@"$2","}' | \ - xargs echo + sort -nr) + +echo "Contributions by user: " +echo "$contribs" +echo +echo "$contribs" | awk '{print "@"$2","}' | xargs diff --git a/script/release/push-release b/script/release/push-release index b754d40f..7d9ec0a2 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -60,7 +60,7 @@ sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/maste ./script/write-git-sha python setup.py sdist if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION}.tar.gz + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz else python setup.py upload fi From d5f3826ec7374ce5b2bafc592e25268edddd9cce Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Sat, 16 Jan 2016 13:53:16 +0100 Subject: [PATCH 276/359] Fix zsh completion to ensure we have enough commands to store in the cache Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 01e5f896..a8a60b59 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -179,7 +179,7 @@ __docker-compose_commands() { local -a lines lines=(${(f)"$(_call_program commands docker-compose 2>&1)"}) _docker_compose_subcommands=(${${${lines[$((${lines[(i)Commands:]} + 1)),${lines[(I) *]}]}## #}/ ##/:}) - _store_cache docker_compose_subcommands _docker_compose_subcommands + (( $#_docker_compose_subcommands > 0 )) && _store_cache docker_compose_subcommands _docker_compose_subcommands fi _describe -t docker-compose-commands "docker-compose command" _docker_compose_subcommands } From 7dd5fd5763e68d607978367775475d47117f30e5 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Sat, 16 Jan 2016 13:47:48 +0100 Subject: [PATCH 277/359] Add zsh completion for 'docker-compose down' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index a8a60b59..c9a4f3dc 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -203,6 +203,12 @@ __docker-compose_subcommand() { '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ '--services[Print the service names, one per line.]' && ret=0 ;; + (down) + _arguments \ + $opts_help \ + "--rmi[Remove images, type may be one of: 'all' to remove all images, or 'local' to remove only images that don't have an custom name set by the 'image' field]:type:(all local)" \ + '(-v --volumes)'{-v,--volumes}"[Remove data volumes]" && ret=0 + ;; (events) _arguments \ $opts_help \ From 8398382b65bd88033c28f831e22608d53915a4f6 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Sat, 16 Jan 2016 13:22:02 +0100 Subject: [PATCH 278/359] Add zsh completion for 'docker-compose up --abort-on-container-exit' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index c9a4f3dc..43017022 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -304,12 +304,13 @@ __docker-compose_subcommand() { (up) _arguments \ $opts_help \ - '-d[Detached mode: Run containers in the background, print new container names.]' \ + '(--abort-on-container-exit)-d[Detached mode: Run containers in the background, print new container names.]' \ '--no-color[Produce monochrome output.]' \ "--no-deps[Don't start linked services.]" \ "--force-recreate[Recreate containers even if their configuration and image haven't changed. Incompatible with --no-recreate.]" \ "--no-recreate[If containers already exist, don't recreate them.]" \ "--no-build[Don't build an image, even if it's missing]" \ + "(-d)--abort-on-container-exit[Stops all containers if any container was stopped. Incompatible with -d.]" \ '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ '*:services:__docker-compose_services_all' && ret=0 ;; From fbee4ce4b323a29052ffbb6441c0a1e283a1c2ee Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 18 Jan 2016 19:22:04 +0000 Subject: [PATCH 279/359] Catch TLSParameterErrors from docker-py Signed-off-by: Aanand Prasad --- compose/cli/docker_client.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 611997df..b680616e 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -5,9 +5,11 @@ import logging import os from docker import Client +from docker.errors import TLSParameterError from docker.utils import kwargs_from_env from ..const import HTTP_TIMEOUT +from .errors import UserError log = logging.getLogger(__name__) @@ -20,8 +22,16 @@ def docker_client(version=None): if 'DOCKER_CLIENT_TIMEOUT' in os.environ: log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') - kwargs = kwargs_from_env(assert_hostname=False) + try: + kwargs = kwargs_from_env(assert_hostname=False) + except TLSParameterError: + raise UserError( + 'TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY and DOCKER_CERT_PATH are set correctly.\n' + 'You might need to run `eval "$(docker-machine env default)"`') + if version: kwargs['version'] = version + kwargs['timeout'] = HTTP_TIMEOUT + return Client(**kwargs) From 7442b416e8dcb1083fd6b570adcf340497045eb0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 19 Jan 2016 10:57:12 +0000 Subject: [PATCH 280/359] Test that you can set the default network to be external Signed-off-by: Aanand Prasad --- tests/acceptance/cli_test.py | 23 ++++++++++++++++++++ tests/fixtures/networks/external-default.yml | 12 ++++++++++ 2 files changed, 35 insertions(+) create mode 100644 tests/fixtures/networks/external-default.yml diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index f5c16380..4c278aa4 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -528,6 +528,29 @@ class CLITestCase(DockerClientTestCase): container = self.project.containers()[0] assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted(network_names) + @v2_only() + def test_up_with_external_default_network(self): + filename = 'external-default.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + result = self.dispatch(['-f', filename, 'up', '-d'], returncode=1) + assert 'declared as external, but could not be found' in result.stderr + + networks = [ + n['Name'] for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + assert not networks + + network_name = 'composetest_external_network' + self.client.create_network(network_name) + + self.dispatch(['-f', filename, 'up', '-d']) + container = self.project.containers()[0] + assert list(container.get('NetworkSettings.Networks')) == [network_name] + @v2_only() def test_up_no_services(self): self.base_dir = 'tests/fixtures/no-services' diff --git a/tests/fixtures/networks/external-default.yml b/tests/fixtures/networks/external-default.yml new file mode 100644 index 00000000..7b0797e5 --- /dev/null +++ b/tests/fixtures/networks/external-default.yml @@ -0,0 +1,12 @@ +version: 2 +services: + simple: + image: busybox:latest + command: top + another: + image: busybox:latest + command: top +networks: + default: + external: + name: composetest_external_network From bb377d3fe600bd6e89dfb720cec4d309332aa626 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 19 Jan 2016 11:27:27 +0000 Subject: [PATCH 281/359] Fix "name is reserved" with Engine 1.10 RC1 Ensure link aliases are unique (this deduping was previously performed on the server). Signed-off-by: Aanand Prasad --- compose/service.py | 25 ++++++++++++++++--------- tests/integration/service_test.py | 4 ---- 2 files changed, 16 insertions(+), 13 deletions(-) diff --git a/compose/service.py b/compose/service.py index f5db07fb..f72863c9 100644 --- a/compose/service.py +++ b/compose/service.py @@ -502,24 +502,31 @@ class Service(object): if self.use_networking: return [] - links = [] + links = {} + for service, link_name in self.links: for container in service.containers(): - links.append((container.name, link_name or service.name)) - links.append((container.name, container.name)) - links.append((container.name, container.name_without_project)) + links[link_name or service.name] = container.name + links[container.name] = container.name + links[container.name_without_project] = container.name + if link_to_self: for container in self.containers(): - links.append((container.name, self.name)) - links.append((container.name, container.name)) - links.append((container.name, container.name_without_project)) + links[self.name] = container.name + links[container.name] = container.name + links[container.name_without_project] = container.name + for external_link in self.options.get('external_links') or []: if ':' not in external_link: link_name = external_link else: external_link, link_name = external_link.split(':') - links.append((external_link, link_name)) - return links + links[link_name] = external_link + + return [ + (alias, container_name) + for (container_name, alias) in links.items() + ] def _get_volumes_from(self): return [build_volume_from(spec) for spec in self.volumes_from] diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index bce3999b..0e91dcf7 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -7,7 +7,6 @@ import tempfile from os import path from docker.errors import APIError -from pytest import mark from six import StringIO from six import text_type @@ -372,7 +371,6 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) self.assertEqual(db.containers()[0].environment['FOO'], 'BAR') - @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_container_creates_links(self): db = self.create_service('db') web = self.create_service('web', links=[(db, None)]) @@ -389,7 +387,6 @@ class ServiceTest(DockerClientTestCase): 'db']) ) - @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_container_creates_links_with_names(self): db = self.create_service('db') web = self.create_service('web', links=[(db, 'custom_link_name')]) @@ -433,7 +430,6 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db) self.assertEqual(set(get_links(c)), set([])) - @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') From 33bb7c4e026d46dda184d682c89fad7481ab1a77 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 18 Jan 2016 14:02:25 -0500 Subject: [PATCH 282/359] Add migration script. Signed-off-by: Daniel Nephin --- .../migrate-compose-file-v1-to-v2.py | 99 +++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100755 contrib/migration/migrate-compose-file-v1-to-v2.py diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py new file mode 100755 index 00000000..a961b0bd --- /dev/null +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -0,0 +1,99 @@ +#!/usr/bin/env python +""" +Migrate a Compose file from the V1 format in Compose 1.5 to the V2 format +supported by Compose 1.6+ +""" +from __future__ import absolute_import +from __future__ import unicode_literals + +import argparse +import logging +import sys + +import ruamel.yaml + + +log = logging.getLogger('migrate') + + +def migrate(content): + data = ruamel.yaml.load(content, ruamel.yaml.RoundTripLoader) + + service_names = data.keys() + for name, service in data.items(): + # remove links and external links + service.pop('links', None) + external_links = service.pop('external_links', None) + if external_links: + log.warn( + "Service {name} has external_links: {ext}, which are no longer " + "supported. See https://docs.docker.com/compose/networking/ " + "for options on how to connect external containers to the " + "compose network.".format(name=name, ext=external_links)) + + # net is now networks + if 'net' in service: + service['networks'] = [service.pop('net')] + + # create build section + if 'dockerfile' in service: + service['build'] = { + 'context': service.pop('build'), + 'dockerfile': service.pop('dockerfile'), + } + + # create logging section + if 'log_driver' in service: + service['logging'] = {'driver': service.pop('log_driver')} + if 'log_opt' in service: + service['logging']['options'] = service.pop('log_opt') + + # volumes_from prefix with 'container:' + for idx, volume_from in enumerate(service.get('volumes_from', [])): + if volume_from.split(':', 1)[0] not in service_names: + service['volumes_from'][idx] = 'container:%s' % volume_from + + data['services'] = {name: data.pop(name) for name in data.keys()} + data['version'] = 2 + return data + + +def write(stream, new_format, indent, width): + ruamel.yaml.dump( + new_format, + stream, + Dumper=ruamel.yaml.RoundTripDumper, + indent=indent, + width=width) + + +def parse_opts(args): + parser = argparse.ArgumentParser() + parser.add_argument("filename", help="Compose file filename.") + parser.add_argument("-i", "--in-place", action='store_true') + parser.add_argument( + "--indent", type=int, default=2, + help="Number of spaces used to indent the output yaml.") + parser.add_argument( + "--width", type=int, default=80, + help="Number of spaces used as the output width.") + return parser.parse_args() + + +def main(args): + logging.basicConfig() + + opts = parse_opts(args) + + with open(opts.filename, 'r') as fh: + new_format = migrate(fh.read()) + + if opts.in_place: + output = open(opts.filename, 'w') + else: + output = sys.stdout + write(output, new_format, opts.indent, opts.width) + + +if __name__ == "__main__": + main(sys.argv) From 6e5c312768dbcdd7c6432b26e14361117adf22d0 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Jan 2016 15:41:45 -0500 Subject: [PATCH 283/359] Implement depends_on to define an order for services in the v2 format. Signed-off-by: Daniel Nephin --- compose/config/config.py | 15 ++++++++++--- compose/config/service_schema_v2.json | 1 + compose/config/sort_services.py | 9 +++++--- compose/config/validation.py | 8 +++++++ compose/service.py | 3 ++- tests/unit/config/config_test.py | 28 ++++++++++++++++++++++++- tests/unit/config/sort_services_test.py | 14 +++++++++++++ 7 files changed, 70 insertions(+), 8 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ac5e8d17..6bba53b8 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -27,6 +27,7 @@ from .types import VolumeFromSpec from .types import VolumeSpec from .validation import validate_against_fields_schema from .validation import validate_against_service_schema +from .validation import validate_depends_on from .validation import validate_extends_file_path from .validation import validate_top_level_object from .validation import validate_top_level_service_objects @@ -312,9 +313,10 @@ def load_services(working_dir, config_file, service_configs): resolver = ServiceExtendsResolver(service_config, config_file) service_dict = process_service(resolver.run()) - validate_service(service_dict, service_config.name, config_file.version) + service_config = service_config._replace(config=service_dict) + validate_service(service_config, service_names, config_file.version) service_dict = finalize_service( - service_config._replace(config=service_dict), + service_config, service_names, config_file.version) return service_dict @@ -481,6 +483,10 @@ def validate_extended_service_dict(service_dict, filename, service): raise ConfigurationError( "%s services with 'net: container' cannot be extended" % error_prefix) + if 'depends_on' in service_dict: + raise ConfigurationError( + "%s services with 'depends_on' cannot be extended" % error_prefix) + def validate_ulimits(ulimit_config): for limit_name, soft_hard_values in six.iteritems(ulimit_config): @@ -491,13 +497,16 @@ def validate_ulimits(ulimit_config): "than 'hard' value".format(ulimit_config)) -def validate_service(service_dict, service_name, version): +def validate_service(service_config, service_names, version): + service_dict, service_name = service_config.config, service_config.name validate_against_service_schema(service_dict, service_name, version) validate_paths(service_dict) if 'ulimits' in service_dict: validate_ulimits(service_dict['ulimits']) + validate_depends_on(service_config, service_names) + if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( "Service '{name}' contains uppercase characters which are not valid " diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 8623507a..d4ec575a 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -42,6 +42,7 @@ "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py index 05552122..ac0fa458 100644 --- a/compose/config/sort_services.py +++ b/compose/config/sort_services.py @@ -33,7 +33,8 @@ def sort_service_dicts(services): service for service in services if (name in get_service_names(service.get('links', [])) or name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or - name == get_service_name_from_net(service.get('net'))) + name == get_service_name_from_net(service.get('net')) or + name in service.get('depends_on', [])) ] def visit(n): @@ -42,8 +43,10 @@ def sort_service_dicts(services): raise DependencyError('A service can not link to itself: %s' % n['name']) if n['name'] in n.get('volumes_from', []): raise DependencyError('A service can not mount itself as volume: %s' % n['name']) - else: - raise DependencyError('Circular import between %s' % ' and '.join(temporary_marked)) + if n['name'] in n.get('depends_on', []): + raise DependencyError('A service can not depend on itself: %s' % n['name']) + raise DependencyError('Circular dependency between %s' % ' and '.join(temporary_marked)) + if n in unmarked: temporary_marked.add(n['name']) for m in get_service_dependents(n, services): diff --git a/compose/config/validation.py b/compose/config/validation.py index 639e8bed..6cce1105 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -123,6 +123,14 @@ def validate_extends_file_path(service_name, extends_options, filename): ) +def validate_depends_on(service_config, service_names): + for dependency in service_config.config.get('depends_on', []): + if dependency not in service_names: + raise ConfigurationError( + "Service '{s.name}' depends on service '{dep}' which is " + "undefined.".format(s=service_config, dep=dependency)) + + def get_unsupported_config_msg(service_name, error_key): msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) if error_key in DOCKER_CONFIG_HINTS: diff --git a/compose/service.py b/compose/service.py index f72863c9..1dfda06a 100644 --- a/compose/service.py +++ b/compose/service.py @@ -471,7 +471,8 @@ class Service(object): net_name = self.net.service_name return (self.get_linked_service_names() + self.get_volumes_from_names() + - ([net_name] if net_name else [])) + ([net_name] if net_name else []) + + self.options.get('depends_on', [])) def get_linked_service_names(self): return [service.name for (service, _) in self.links] diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index cc205136..0416d5b7 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -894,9 +894,35 @@ class ConfigTest(unittest.TestCase): 'ext': {'external': True, 'driver': 'foo'} } }) - with self.assertRaises(ConfigurationError): + with pytest.raises(ConfigurationError): config.load(config_details) + def test_depends_on_orders_services(self): + config_details = build_config_details({ + 'version': 2, + 'services': { + 'one': {'image': 'busybox', 'depends_on': ['three', 'two']}, + 'two': {'image': 'busybox', 'depends_on': ['three']}, + 'three': {'image': 'busybox'}, + }, + }) + actual = config.load(config_details) + assert ( + [service['name'] for service in actual.services] == + ['three', 'two', 'one'] + ) + + def test_depends_on_unknown_service_errors(self): + config_details = build_config_details({ + 'version': 2, + 'services': { + 'one': {'image': 'busybox', 'depends_on': ['three']}, + }, + }) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert "Service 'one' depends on service 'three'" in exc.exconly() + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ diff --git a/tests/unit/config/sort_services_test.py b/tests/unit/config/sort_services_test.py index ebe444fe..3279ece4 100644 --- a/tests/unit/config/sort_services_test.py +++ b/tests/unit/config/sort_services_test.py @@ -1,6 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals +import pytest + from compose.config.errors import DependencyError from compose.config.sort_services import sort_service_dicts from compose.config.types import VolumeFromSpec @@ -240,3 +242,15 @@ class SortServiceTest(unittest.TestCase): self.assertIn('web', e.msg) else: self.fail('Should have thrown an DependencyError') + + def test_sort_service_dicts_depends_on_self(self): + services = [ + { + 'depends_on': ['web'], + 'name': 'web' + }, + ] + + with pytest.raises(DependencyError) as exc: + sort_service_dicts(services) + assert 'A service can not depend on itself: web' in exc.exconly() From 3f65bdcf46556e7e309fbccc8f531cba00115fd1 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Jan 2016 15:47:57 -0500 Subject: [PATCH 284/359] Move ulimits validation to validation.py and improve the error message. Signed-off-by: Daniel Nephin --- compose/config/config.py | 14 ++------------ compose/config/validation.py | 12 ++++++++++++ tests/unit/config/config_test.py | 2 +- 3 files changed, 15 insertions(+), 13 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 6bba53b8..72ad50af 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -31,6 +31,7 @@ from .validation import validate_depends_on from .validation import validate_extends_file_path from .validation import validate_top_level_object from .validation import validate_top_level_service_objects +from .validation import validate_ulimits DOCKER_CONFIG_KEYS = [ @@ -488,23 +489,12 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'depends_on' cannot be extended" % error_prefix) -def validate_ulimits(ulimit_config): - for limit_name, soft_hard_values in six.iteritems(ulimit_config): - if isinstance(soft_hard_values, dict): - if not soft_hard_values['soft'] <= soft_hard_values['hard']: - raise ConfigurationError( - "ulimit_config \"{}\" cannot contain a 'soft' value higher " - "than 'hard' value".format(ulimit_config)) - - def validate_service(service_config, service_names, version): service_dict, service_name = service_config.config, service_config.name validate_against_service_schema(service_dict, service_name, version) validate_paths(service_dict) - if 'ulimits' in service_dict: - validate_ulimits(service_dict['ulimits']) - + validate_ulimits(service_config) validate_depends_on(service_config, service_names) if not service_dict.get('image') and has_uppercase(service_name): diff --git a/compose/config/validation.py b/compose/config/validation.py index 6cce1105..ecf8d4f9 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -110,6 +110,18 @@ def validate_top_level_object(config_file): type(config_file.config))) +def validate_ulimits(service_config): + ulimit_config = service_config.config.get('ulimits', {}) + for limit_name, soft_hard_values in six.iteritems(ulimit_config): + if isinstance(soft_hard_values, dict): + if not soft_hard_values['soft'] <= soft_hard_values['hard']: + raise ConfigurationError( + "Service '{s.name}' has invalid ulimit '{ulimit}'. " + "'soft' value can not be greater than 'hard' value ".format( + s=service_config, + ulimit=ulimit_config)) + + def validate_extends_file_path(service_name, extends_options, filename): """ The service to be extended must either be defined in the config key 'file', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 0416d5b7..3c3c6326 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -700,7 +700,7 @@ class ConfigTest(unittest.TestCase): assert "'hard' is a required property" in exc.exconly() def test_config_ulimits_soft_greater_than_hard_error(self): - expected = "cannot contain a 'soft' value higher than 'hard' value" + expected = "'soft' value can not be greater than 'hard' value" with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( From c52eed66b7291d4d7a4c70d07d90ea58db9f1c03 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Jan 2016 15:52:17 -0500 Subject: [PATCH 285/359] Update tests in sort_services_test.py to use pytest. Signed-off-by: Daniel Nephin --- tests/unit/config/sort_services_test.py | 95 +++++++++++-------------- 1 file changed, 41 insertions(+), 54 deletions(-) diff --git a/tests/unit/config/sort_services_test.py b/tests/unit/config/sort_services_test.py index 3279ece4..f5990664 100644 --- a/tests/unit/config/sort_services_test.py +++ b/tests/unit/config/sort_services_test.py @@ -6,10 +6,9 @@ import pytest from compose.config.errors import DependencyError from compose.config.sort_services import sort_service_dicts from compose.config.types import VolumeFromSpec -from tests import unittest -class SortServiceTest(unittest.TestCase): +class TestSortService(object): def test_sort_service_dicts_1(self): services = [ { @@ -25,10 +24,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'grunt') - self.assertEqual(sorted_services[1]['name'], 'redis') - self.assertEqual(sorted_services[2]['name'], 'web') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'grunt' + assert sorted_services[1]['name'] == 'redis' + assert sorted_services[2]['name'] == 'web' def test_sort_service_dicts_2(self): services = [ @@ -46,10 +45,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'redis') - self.assertEqual(sorted_services[1]['name'], 'postgres') - self.assertEqual(sorted_services[2]['name'], 'web') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'redis' + assert sorted_services[1]['name'] == 'postgres' + assert sorted_services[2]['name'] == 'web' def test_sort_service_dicts_3(self): services = [ @@ -67,10 +66,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_4(self): services = [ @@ -88,10 +87,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_5(self): services = [ @@ -109,10 +108,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_6(self): services = [ @@ -130,10 +129,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_7(self): services = [ @@ -155,11 +154,11 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 4) - self.assertEqual(sorted_services[0]['name'], 'one') - self.assertEqual(sorted_services[1]['name'], 'two') - self.assertEqual(sorted_services[2]['name'], 'three') - self.assertEqual(sorted_services[3]['name'], 'four') + assert len(sorted_services) == 4 + assert sorted_services[0]['name'] == 'one' + assert sorted_services[1]['name'] == 'two' + assert sorted_services[2]['name'] == 'three' + assert sorted_services[3]['name'] == 'four' def test_sort_service_dicts_circular_imports(self): services = [ @@ -173,13 +172,10 @@ class SortServiceTest(unittest.TestCase): }, ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('redis', e.msg) - self.assertIn('web', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'redis' in exc.exconly() + assert 'web' in exc.exconly() def test_sort_service_dicts_circular_imports_2(self): services = [ @@ -196,13 +192,10 @@ class SortServiceTest(unittest.TestCase): } ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('redis', e.msg) - self.assertIn('web', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'redis' in exc.exconly() + assert 'web' in exc.exconly() def test_sort_service_dicts_circular_imports_3(self): services = [ @@ -220,13 +213,10 @@ class SortServiceTest(unittest.TestCase): } ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('a', e.msg) - self.assertIn('b', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'a' in exc.exconly() + assert 'b' in exc.exconly() def test_sort_service_dicts_self_imports(self): services = [ @@ -236,12 +226,9 @@ class SortServiceTest(unittest.TestCase): }, ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('web', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'web' in exc.exconly() def test_sort_service_dicts_depends_on_self(self): services = [ From c9ef1fa32f034a7a6bd84821187a2e8014e37c51 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jure=20=C5=BDvelc?= Date: Wed, 20 Jan 2016 13:01:04 +0100 Subject: [PATCH 286/359] =?UTF-8?q?Fix=20for=20extending=20services=20writ?= =?UTF-8?q?ten=20in=20v2=20format.=20Signed-off-by:=20Jure=20=C5=BDvelc=20?= =?UTF-8?q??= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- compose/config/config.py | 12 ++++++++---- tests/unit/config/config_test.py | 24 ++++++++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 72ad50af..961d36bb 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -136,6 +136,9 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): return 1 return version + def get_service(self, name): + return self.get_service_dicts()[name] + def get_service_dicts(self): return self.config if self.version == 1 else self.config.get('services', {}) @@ -354,19 +357,19 @@ def process_config_file(config_file, service_name=None): if config_file.version == 2: processed_config = dict(config_file.config) - processed_config['services'] = interpolated_config + processed_config['services'] = services = interpolated_config processed_config['volumes'] = interpolate_environment_variables( config_file.get_volumes(), 'volume') processed_config['networks'] = interpolate_environment_variables( config_file.get_networks(), 'network') if config_file.version == 1: - processed_config = interpolated_config + processed_config = services = interpolated_config config_file = config_file._replace(config=processed_config) validate_against_fields_schema(config_file) - if service_name and service_name not in processed_config: + if service_name and service_name not in services: raise ConfigurationError( "Cannot extend service '{}' in {}: Service not found".format( service_name, config_file.filename)) @@ -411,7 +414,8 @@ class ServiceExtendsResolver(object): extended_file = process_config_file( extends_file, service_name=service_name) - service_config = extended_file.config[service_name] + service_config = extended_file.get_service(service_name) + return config_path, service_config, service_name def resolve_extends(self, extended_config_path, service_dict, service_name): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3c3c6326..eb8ed2c7 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1937,6 +1937,30 @@ class ExtendsTest(unittest.TestCase): load_from_filename(str(tmpdir.join('docker-compose.yml'))) assert 'Version mismatch' in exc.exconly() + def test_extends_with_defined_version_passes(self): + tmpdir = py.test.ensuretemp('test_extends_with_defined_version') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: 2 + services: + web: + extends: + file: base.yml + service: base + image: busybox + """) + tmpdir.join('base.yml').write(""" + version: 2 + services: + base: + volumes: ['/foo'] + ports: ['3000:3000'] + command: top + """) + + service = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + self.assertEquals(service[0]['command'], "top") + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From 76bc06b7295dd1ba765dfda78ba4cb7333a1397a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 11:38:01 +0000 Subject: [PATCH 287/359] Fix Windows build failures when installing dependencies from git Signed-off-by: Aanand Prasad --- script/build-windows.ps1 | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 28011b1d..4a2bc1f7 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -41,6 +41,9 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } # Create virtualenv virtualenv .\venv +# pip and pyinstaller generate lots of warnings, so we need to ignore them +$ErrorActionPreference = "Continue" + # Install dependencies .\venv\Scripts\pip install pypiwin32==219 .\venv\Scripts\pip install -r requirements.txt @@ -50,8 +53,6 @@ virtualenv .\venv git rev-parse --short HEAD | out-file -encoding ASCII compose\GITSHA # Build binary -# pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue -$ErrorActionPreference = "Continue" .\venv\Scripts\pyinstaller .\docker-compose.spec $ErrorActionPreference = "Stop" From cbec6f88348783ca49ae66f95aaef2994c5b4866 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 20 Jan 2016 17:08:24 +0000 Subject: [PATCH 288/359] Support links in v2 files Signed-off-by: Aanand Prasad --- compose/config/service_schema_v2.json | 1 + compose/project.py | 3 +-- compose/service.py | 13 +++++++------ tests/acceptance/cli_test.py | 19 +++++++------------ tests/fixtures/networks/docker-compose.yml | 2 ++ tests/unit/service_test.py | 8 -------- 6 files changed, 18 insertions(+), 28 deletions(-) diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index d4ec575a..94046d5b 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -88,6 +88,7 @@ "image": {"type": "string"}, "ipc": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "logging": { "type": "object", diff --git a/compose/project.py b/compose/project.py index 080c4c49..0fea875a 100644 --- a/compose/project.py +++ b/compose/project.py @@ -78,12 +78,11 @@ class Project(object): if use_networking: networks = get_networks(service_dict, all_networks) net = Net(networks[0]) if networks else Net("none") - links = [] else: networks = [] net = project.get_net(service_dict) - links = project.get_links(service_dict) + links = project.get_links(service_dict) volumes_from = get_volumes_from(project, service_dict) project.services.append( diff --git a/compose/service.py b/compose/service.py index 1dfda06a..ea4e57d0 100644 --- a/compose/service.py +++ b/compose/service.py @@ -426,10 +426,11 @@ class Service(object): def connect_container_to_networks(self, container): for network in self.networks: - log.debug('Connecting "{}" to "{}"'.format(container.name, network)) self.client.connect_container_to_network( container.id, network, - aliases=[self.name]) + aliases=[self.name], + links=self._get_links(False), + ) def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): for c in self.duplicate_containers(): @@ -500,9 +501,6 @@ class Service(object): return 1 if not numbers else max(numbers) + 1 def _get_links(self, link_to_self): - if self.use_networking: - return [] - links = {} for service, link_name in self.links: @@ -645,7 +643,10 @@ class Service(object): def _get_container_networking_config(self): return self.client.create_networking_config({ - network_name: self.client.create_endpoint_config(aliases=[self.name]) + network_name: self.client.create_endpoint_config( + aliases=[self.name], + links=self._get_links(False), + ) for network_name in self.networks if network_name not in ['host', 'bridge'] }) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 4c278aa4..1806e707 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -461,6 +461,10 @@ class CLITestCase(DockerClientTestCase): app_container = self.project.get_service('app').containers()[0] db_container = self.project.get_service('db').containers()[0] + for net_name in [front_name, back_name]: + links = app_container.get('NetworkSettings.Networks.{}.Links'.format(net_name)) + assert '{}:database'.format(db_container.name) in links + # db and app joined the back network assert sorted(back_network['Containers']) == sorted([db_container.id, app_container.id]) @@ -474,6 +478,9 @@ class CLITestCase(DockerClientTestCase): # app can see db assert self.lookup(app_container, "db") + # app has aliased db to "database" + assert self.lookup(app_container, "database") + @v2_only() def test_up_missing_network(self): self.base_dir = 'tests/fixtures/networks' @@ -566,18 +573,6 @@ class CLITestCase(DockerClientTestCase): for name in ['bar', 'foo'] ] - @v2_only() - def test_up_with_links_is_invalid(self): - self.base_dir = 'tests/fixtures/v2-simple' - - result = self.dispatch( - ['-f', 'links-invalid.yml', 'up', '-d'], - returncode=1) - - # TODO: fix validation error messages for v2 files - # assert "Unsupported config option for service 'simple': 'links'" in result.stderr - assert "Unsupported config option" in result.stderr - def test_up_with_links_v1(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'web'], None) diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml index f1b79df0..5351c0f0 100644 --- a/tests/fixtures/networks/docker-compose.yml +++ b/tests/fixtures/networks/docker-compose.yml @@ -9,6 +9,8 @@ services: image: busybox command: top networks: ["front", "back"] + links: + - "db:database" db: image: busybox command: top diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c9244a47..4d9aec65 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -536,14 +536,6 @@ class ServiceTest(unittest.TestCase): ports=["127.0.0.1:1000-2000:2000-3000"]) self.assertEqual(service.specifies_host_port(), True) - def test_get_links_with_networking(self): - service = Service( - 'foo', - image='foo', - links=[(Service('one'), 'one')], - use_networking=True) - self.assertEqual(service._get_links(link_to_self=True), []) - def test_image_name_from_config(self): image_name = 'example/web:latest' service = Service('foo', image=image_name) From 513a6b35cca1ab49c15723c72e8db6a3c8b7155d Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 15:19:55 +0000 Subject: [PATCH 289/359] Fix scale when containers exit immediately Signed-off-by: Aanand Prasad --- compose/service.py | 8 +++----- tests/integration/service_test.py | 5 +++++ 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index ea4e57d0..2df95fc3 100644 --- a/compose/service.py +++ b/compose/service.py @@ -228,11 +228,9 @@ class Service(object): sorted_running_containers = sorted( running_containers, key=attrgetter('number')) - parallel_stop( - sorted_running_containers[-num_to_stop:], - dict(timeout=timeout)) - - parallel_remove(self.containers(stopped=True), {}) + containers_to_stop = sorted_running_containers[-num_to_stop:] + parallel_stop(containers_to_stop, dict(timeout=timeout)) + parallel_remove(containers_to_stop, {}) def create_container(self, one_off=False, diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 0e91dcf7..379e51ea 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -746,6 +746,11 @@ class ServiceTest(DockerClientTestCase): for container in containers: self.assertEqual(list(container.inspect()['HostConfig']['PortBindings'].keys()), ['8000/tcp']) + def test_scale_with_immediate_exit(self): + service = self.create_service('web', image='busybox', command='true') + service.scale(2) + assert len(service.containers(stopped=True)) == 2 + def test_network_mode_none(self): service = self.create_service('web', net=Net('none')) container = create_and_start_container(service) From 5cfd947f380632461380b1b627facea89a659d95 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 15:28:40 +0000 Subject: [PATCH 290/359] Stop and remove containers in parallel when scaling down Signed-off-by: Aanand Prasad --- compose/service.py | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/compose/service.py b/compose/service.py index 2df95fc3..8afc59c1 100644 --- a/compose/service.py +++ b/compose/service.py @@ -27,9 +27,7 @@ from .const import LABEL_SERVICE from .const import LABEL_VERSION from .container import Container from .parallel import parallel_execute -from .parallel import parallel_remove from .parallel import parallel_start -from .parallel import parallel_stop from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash @@ -180,6 +178,10 @@ class Service(object): service.start_container(container) return container + def stop_and_remove(container): + container.stop(timeout=timeout) + container.remove() + running_containers = self.containers(stopped=False) num_running = len(running_containers) @@ -225,12 +227,17 @@ class Service(object): if desired_num < num_running: num_to_stop = num_running - desired_num + sorted_running_containers = sorted( running_containers, key=attrgetter('number')) - containers_to_stop = sorted_running_containers[-num_to_stop:] - parallel_stop(containers_to_stop, dict(timeout=timeout)) - parallel_remove(containers_to_stop, {}) + + parallel_execute( + sorted_running_containers[-num_to_stop:], + stop_and_remove, + lambda c: c.name, + "Stopping and removing", + ) def create_container(self, one_off=False, From 6fe54f5c24adc839683efb449b09580bf4cac299 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 19 Jan 2016 13:23:25 +0000 Subject: [PATCH 291/359] Update Compose file documentation for version 2 - Explain each version in its own section - Explain how to upgrade from version 1 to 2 - Note which keys are restricted to particular versions - A few corrections to the docs for version-specific keys Signed-off-by: Aanand Prasad --- docs/compose-file.md | 518 ++++++++++++++++++++++++++++++++++--------- docs/networking.md | 20 +- 2 files changed, 422 insertions(+), 116 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index ecd135f1..ba6b4cb4 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -12,79 +12,33 @@ parent="smn_compose_ref" # Compose file reference -The compose file is a [YAML](http://yaml.org/) file where all the top level -keys are the name of a service, and the values are the service definition. -The default path for a compose file is `./docker-compose.yml`. +The Compose file is a [YAML](http://yaml.org/) file defining +[services](#service-configuration-reference), +[networks](#network-configuration-reference) and +[volumes](#volume-configuration-reference). +The default path for a Compose file is `./docker-compose.yml`. -Each service defined in `docker-compose.yml` must specify exactly one of -`image` or `build`. Other keys are optional, and are analogous to their -`docker run` command-line counterparts. +A service definition contains configuration which will be applied to each +container started for that service, much like passing command-line parameters to +`docker run`. Likewise, network and volume definitions are analogous to +`docker network create` and `docker volume create`. As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. -## Versioning - -It is possible to use different versions of the `compose.yml` format. -Below are the formats currently supported by compose. - - -### Version 1 - -Compose files that do not declare a version are considered "version 1". In -those files, all the [services](#service-configuration-reference) are declared -at the root of the document. - -Version 1 files do not support the declaration of -named [volumes](#volume-configuration-reference) or -[build arguments](#args). - -Example: - - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - - logvolume01:/var/log - links: - - redis - redis: - image: redis - - -### Version 2 - -Compose files using the version 2 syntax must indicate the version number at -the root of the document. All [services](#service-configuration-reference) -must be declared under the `services` key. -Named [volumes](#volume-configuration-reference) must be declared under the -`volumes` key. - -Example: - - version: 2 - services: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - - logvolume01:/var/log - links: - - redis - redis: - image: redis - volumes: - logvolume01: - driver: default +You can use environment variables in configuration values with a Bash-like +`${VARIABLE}` syntax - see [variable substitution](#variable-substitution) for +full details. ## Service configuration reference +> **Note:** There are two versions of the Compose file format – version 1 (the +> legacy format, which does not support volumes or networks) and version 2 (the +> most up-to-date). For more information, see the [Versioning](#versioning) +> section. + This section contains a list of all configuration options supported by a service definition. @@ -92,28 +46,30 @@ definition. Configuration options that are applied at build time. -In version 1 this must be given as a string representing the context. +`build` can be specified either as a string containing a path to the build +context, or an object with the path specified under [context](#context) and +optionally [dockerfile](#dockerfile) and [args](#args). - build: . + build: ./dir -In version 2 this can alternatively be given as an object with extra options. + build: + context: ./dir + dockerfile: Dockerfile-alternate + args: + buildno: 1 - version: 2 - services: - web: - build: . - - version: 2 - services: - web: - build: - context: . - dockerfile: Dockerfile-alternate - args: - buildno: 1 +> **Note**: In the [version 1 file format](#version-1), `build` is different in +> two ways: +> +> - Only the string form (`build: .`) is allowed - not the object form. +> - Using `build` together with `image` is not allowed. Attempting to do so +> results in an error. #### context +> [Version 2 file format](#version-2) only. In version 1, just use +> [build](#build). + Either a path to a directory containing a Dockerfile, or a url to a git repository. When the value supplied is a relative path, it is interpreted as relative to the @@ -122,29 +78,34 @@ sent to the Docker daemon. Compose will build and tag it with a generated name, and use that image thereafter. - build: /path/to/build/dir - build: - context: /path/to/build/dir - -Using `context` together with `image` is not allowed. Attempting to do so results in -an error. + context: ./dir #### dockerfile Alternate Dockerfile. Compose will use an alternate file to build with. A build path must also be -specified using the `build` key. +specified. build: - context: /path/to/build/dir + context: . dockerfile: Dockerfile-alternate -Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. +> **Note**: In the [version 1 file format](#version-1), `dockerfile` is +> different in two ways: +> +> - It appears alongside `build`, not as a sub-option: +> +> build: . +> dockerfile: Dockerfile-alternate +> - Using `dockerfile` together with `image` is not allowed. Attempting to do +> so results in an error. #### args +> [Version 2 file format](#version-2) only. + Add build arguments. You can use either an array or a dictionary. Any boolean values; true, false, yes, no, need to be enclosed in quotes to ensure they are not converted to True or False by the YML parser. @@ -152,8 +113,6 @@ they are not converted to True or False by the YML parser. Build arguments with only a key are resolved to their environment value on the machine Compose is running on. -> **Note:** Introduced in version 2 of the compose file format. - build: args: buildno: 1 @@ -376,6 +335,9 @@ It's recommended that you use reverse-DNS notation to prevent your labels from c ### links +> [Version 1 file format](#version-1) only. In version 2 files, use +> [networking](networking.md) for communication between containers. + Link to containers in another service. Either specify both the service name and the link alias (`SERVICE:ALIAS`), or just the service name (which will also be used for the alias). @@ -397,13 +359,15 @@ reference](env.md) for details. ### logging -Logging configuration for the service. This configuration replaces the previous -`log_driver` and `log_opt` keys. +> [Version 2 file format](#version-2) only. In version 1, use +> [log_driver](#log_driver) and [log_opt](#log_opt). + +Logging configuration for the service. logging: - driver: log_driver - options: - syslog-address: "tcp://192.168.0.42:123" + driver: syslog + options: + syslog-address: "tcp://192.168.0.42:123" The `driver` name specifies a logging driver for the service's containers, as with the ``--log-driver`` option for docker run @@ -421,15 +385,36 @@ The default value is json-file. Specify logging options for the logging driver with the ``options`` key, as with the ``--log-opt`` option for `docker run`. - -Logging options are key value pairs. An example of `syslog` options: +Logging options are key-value pairs. An example of `syslog` options: driver: "syslog" options: syslog-address: "tcp://192.168.0.42:123" +### log_driver + +> [Version 1 file format](#version-1) only. In version 2, use +> [logging](#logging). + +Specify a log driver. The default is `json-file`. + + log_driver: syslog + +### log_opt + +> [Version 1 file format](#version-1) only. In version 2, use +> [logging](#logging). + +Specify logging options as key-value pairs. An example of `syslog` options: + + log_opt: + syslog-address: "tcp://192.168.0.42:123" + ### net +> [Version 1 file format](#version-1) only. In version 2, use +> [networks](#networks). + Networking mode. Use the same values as the docker client `--net` parameter. net: "bridge" @@ -437,6 +422,22 @@ Networking mode. Use the same values as the docker client `--net` parameter. net: "container:[name or id]" net: "host" +### networks + +> [Version 2 file format](#version-2) only. In version 1, use [net](#net). + +Networks to join, referencing entries under the +[top-level `networks` key](#network-configuration-reference). + + networks: + - some-network + - other-network + +The values `bridge`, `host` and `none` can also be used, and are equivalent to +`net: "bridge"`, `net: "host"` or `net: "none"` in version 1. + +There is no equivalent to `net: "container:[name or id]"`. + ### pid pid: "host" @@ -487,24 +488,37 @@ limit as an integer or soft/hard limits as a mapping. ### volumes, volume\_driver -Mount paths as volumes, optionally specifying a path on the host machine -(`HOST:CONTAINER`), or an access mode (`HOST:CONTAINER:ro`). - - volumes: - - /var/lib/mysql - - ./cache:/tmp/cache - - ~/configs:/etc/configs/:ro +Mount paths or named volumes, optionally specifying a path on the host machine +(`HOST:CONTAINER`), or an access mode (`HOST:CONTAINER:ro`). Named volumes can +be specified with the +[top-level `volumes` key](#volume-configuration-reference), but this is +optional - the Docker Engine will create the volume if it doesn't exist. You can mount a relative path on the host, which will expand relative to the directory of the Compose configuration file being used. Relative paths should always begin with `.` or `..`. + volumes: + # Just specify a path and let the Engine create a volume + - /var/lib/mysql + + # Specify an absolute path mapping + - /opt/data:/var/lib/mysql + + # Path on the host, relative to the Compose file + - ./cache:/tmp/cache + + # User-relative path + - ~/configs:/etc/configs/:ro + + # Named volume + - datavolume:/var/lib/mysql + If you use a volume name (instead of a volume path), you may also specify a `volume_driver`. volume_driver: mydriver - > Note: No path expansion will be done if you have also specified a > `volume_driver`. @@ -519,8 +533,18 @@ specifying read-only access(``ro``) or read-write(``rw``). volumes_from: - service_name - - container_name - - service_name:rw + - service_name:ro + - container:container_name + - container:container_name:rw + +> **Note:** The `container:...` formats are only supported in the +> [version 2 file format](#version-2). In [version 1](#version-1), you can use +> container names without marking them as such: +> +> - service_name +> - service_name:ro +> - container_name +> - container_name:rw ### cpu\_shares, cpu\_quota, cpuset, domainname, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir @@ -562,20 +586,296 @@ subcommand documentation for more information. ### driver Specify which volume driver should be used for this volume. Defaults to -`local`. An exception will be raised if the driver is not available. +`local`. The Docker Engine will return an error if the driver is not available. driver: foobar ### driver_opts Specify a list of options as key-value pairs to pass to the driver for this -volume. Those options are driver dependent - consult the driver's +volume. Those options are driver-dependent - consult the driver's documentation for more information. Optional. driver_opts: foo: "bar" baz: 1 +## external + +If set to `true`, specifies that this volume has been created outside of +Compose. + +In the example below, instead of attemping to create a volume called +`[projectname]_data`, Compose will look for an existing volume simply +called `data` and mount it into the `db` service's containers. + + version: 2 + + services: + db: + image: postgres + volumes: + - data:/var/lib/postgres/data + + volumes: + data: + external: true + +You can also specify the name of the volume separately from the name used to +refer to it within the Compose file: + + volumes + data: + external: + name: actual-name-of-volume + + +## Network configuration reference + +The top-level `networks` key lets you specify networks to be created. For a full +explanation of Compose's use of Docker networking features, see the +[Networking guide](networking.md). + +### driver + +Specify which driver should be used for this network. + +The default driver depends on how the Docker Engine you're using is configured, +but in most instances it will be `bridge` on a single host and `overlay` on a +Swarm. + +The Docker Engine will return an error if the driver is not available. + + driver: overlay + +### driver_opts + +Specify a list of options as key-value pairs to pass to the driver for this +network. Those options are driver-dependent - consult the driver's +documentation for more information. Optional. + + driver_opts: + foo: "bar" + baz: 1 + +### ipam + +Specify custom IPAM config. This is an object with several properties, each of +which is optional: + +- `driver`: Custom IPAM driver, instead of the default. +- `config`: A list with zero or more config blocks, each containing any of + the following keys: + - `subnet`: Subnet in CIDR format that represents a network segment + - `ip_range`: Range of IPs from which to allocate container IPs + - `gateway`: IPv4 or IPv6 gateway for the master subnet + - `aux_addresses`: Auxiliary IPv4 or IPv6 addresses used by Network driver, + as a mapping from hostname to IP + +A full example: + + ipam: + driver: default + config: + - subnet: 172.28.0.0/16 + ip_range: 172.28.5.0/24 + gateway: 172.28.5.254 + aux_addresses: + host1: 172.28.1.5 + host2: 172.28.1.6 + host3: 172.28.1.7 + +### external + +If set to `true`, specifies that this network has been created outside of +Compose. + +In the example below, `proxy` is the gateway to the outside world. Instead of +attemping to create a network called `[projectname]_outside`, Compose will +look for an existing network simply called `outside` and connect the `proxy` +service's containers to it. + + version: 2 + + services: + proxy: + build: ./proxy + networks: + - outside + - default + app: + build: ./app + networks: + - default + + networks + outside: + external: true + +You can also specify the name of the network separately from the name used to +refer to it within the Compose file: + + networks + outside: + external: + name: actual-name-of-network + + +## Versioning + +There are two versions of the Compose file format: + +- Version 1, the legacy format. This is specified by omitting a `version` key at + the root of the YAML. +- Version 2, the recommended format. This is specified with a `version: 2` entry + at the root of the YAML. + +To move your project from version 1 to 2, see the [Upgrading](#upgrading) +section. + +> **Note:** If you're using +> [multiple Compose files](extends.md#different-environments) or +> [extending services](extends.md#extending-services), each file must be of the +> same version - you cannot mix version 1 and 2 in a single project. + +Several things differ depending on which version you use: + +- The structure and permitted configuration keys +- The minimum Docker Engine version you must be running +- Compose's behaviour with regards to networking + +These differences are explained below. + + +### Version 1 + +Compose files that do not declare a version are considered "version 1". In +those files, all the [services](#service-configuration-reference) are declared +at the root of the document. + +Version 1 is supported by **Compose up to 1.6.x**. It will be deprecated in a +future Compose release. + +Version 1 files cannot declare named +[volumes](#volume-configuration-reference), [networks](networking.md) or +[build arguments](#args). They *can*, however, define [links](#links). + +Example: + + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + links: + - redis + redis: + image: redis + + +### Version 2 + +Compose files using the version 2 syntax must indicate the version number at +the root of the document. All [services](#service-configuration-reference) +must be declared under the `services` key. + +Version 2 files are supported by **Compose 1.6.0+** and require a Docker Engine +of version **1.10.0+**. + +Named [volumes](#volume-configuration-reference) can be declared under the +`volumes` key, and [networks](#network-configuration-reference) can be declared +under the `networks` key. + +You cannot define links when using version 2. Instead, you should use +[networking](networking.md) for communication between containers. In most cases, +this will involve less configuration than links. + +Simple example: + + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + redis: + image: redis + +A more extended example, defining volumes and networks: + + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + networks: + - front + - back + redis: + image: redis + volumes: + - data:/var/lib/redis + networks: + - back + volumes: + data: + driver: local + networks: + front: + driver: bridge + back: + driver: bridge + + +### Upgrading + +In the majority of cases, moving from version 1 to 2 is a very simple process: + +1. Indent the whole file by one level and put a `services:` key at the top. +2. Add a `version: 2` line at the top of the file. +3. Delete all `links` entries. + +It's more complicated if you're using particular configuration features: + +- `dockerfile`: This now lives under the `build` key: + + build: + context: . + dockerfile: Dockerfile-alternate + +- `log_driver`, `log_opt`: These now live under the `logging` key: + + logging: + driver: syslog + options: + syslog-address: "tcp://192.168.0.42:123" + +- `links` with aliases: If you've defined a link with an alias such as + `myservice:db`, there's currently no equivalent to this in version 2. You + will have to refer to the service using its name (in this example, + `myservice`). + +- `external_links`: Links are deprecated, so you should use + [external networks](networking.md#using-externally-created-networks) to + communicate with containers outside the app. + +- `net`: If you're using `host`, `bridge` or `none`, this is now replaced by + `networks`: + + net: host -> networks: ["host"] + net: bridge -> networks: ["bridge"] + net: none -> networks: ["none"] + + If you're using `net: "container:"`, there is no equivalent to this in + version 2 - you should use [Docker networks](networking.md) for + communication instead. + ## Variable substitution diff --git a/docs/networking.md b/docs/networking.md index f111f730..a5b49c1f 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -12,7 +12,7 @@ weight=6 # Networking in Compose -> **Note:** This document only applies if you're using v2 of the [Compose file format](compose-file.md). Networking features are not supported for legacy Compose files. +> **Note:** This document only applies if you're using [version 2 of the Compose file format](compose-file.md#versioning). Networking features are not supported for version 1 (legacy) Compose files. By default Compose sets up a single [network](/engine/reference/commandline/network_create.md) for your app. Each @@ -69,7 +69,7 @@ Instead of just using the default app network, you can specify your own networks Each service can specify what networks to connect to with the *service-level* `networks` key, which is a list of names referencing entries under the *top-level* `networks` key. -Here's an example Compose file defining several networks. The `proxy` service is the gateway to the outside world, via a network called `outside` which is expected to already exist. `proxy` is isolated from the `db` service, because they do not share a network in common - only `app` can talk to both. +Here's an example Compose file defining two custom networks. The `proxy` service is isolated from the `db` service, because they do not share a network in common - only `app` can talk to both. version: 2 @@ -77,7 +77,6 @@ Here's an example Compose file defining several networks. The `proxy` service is proxy: build: ./proxy networks: - - outside - front app: build: ./app @@ -99,10 +98,6 @@ Here's an example Compose file defining several networks. The `proxy` service is options: foo: "1" bar: "2" - outside: - # The 'outside' network is expected to already exist - Compose will not - # attempt to create it - external: true ## Configuring the default network @@ -123,6 +118,17 @@ Instead of (or as well as) specifying your own networks, you can also change the # Use the overlay driver for multi-host communication driver: overlay +## Using a pre-existing network + +If you want your containers to join a pre-existing network, use the [`external` option](compose-file.md#network-configuration-reference): + + networks: + default: + external: + name: my-pre-existing-network + +Instead of attemping to create a network called `[projectname]_default`, Compose will look for a network called `my-pre-existing-network` and connect your app's containers to it. + ## Custom container network modes The `docker` CLI command allows you to specify a custom network mode for a container with the `--net` option - for example, `--net=host` specifies that the container should use the same network namespace as the Docker host, and `--net=none` specifies that it should have no networking capabilities. From e40de088f382d7f88b541084d8829c92cae576db Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 13:55:40 +0000 Subject: [PATCH 292/359] Document depends_on Signed-off-by: Aanand Prasad --- docs/compose-file.md | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/docs/compose-file.md b/docs/compose-file.md index ba6b4cb4..d1c4f36f 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -169,6 +169,29 @@ client create option. devices: - "/dev/ttyUSB0:/dev/ttyUSB0" +### depends_on + +Express dependency between services, which has two effects: + +- `docker-compose up` will start services in dependency order. In the following + example, `db` and `redis` will be started before `web`. + +- `docker-compose up SERVICE` will automatically include `SERVICE`'s + dependencies. In the following example, `docker-compose up web` will also + create and start `db` and `redis`. + + version: 2 + services: + web: + build: . + depends_on: + - db + - redis + redis: + image: redis + db: + image: postgres + ### dns Custom DNS servers. Can be a single value or a list. From 18a1829db03aeeeac698e9e5f24ab27e75402a5a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 13:55:54 +0000 Subject: [PATCH 293/359] Update documentation for links Signed-off-by: Aanand Prasad --- docs/compose-file.md | 62 +++++++++++++++++++++++++++----------------- docs/env.md | 4 ++- docs/networking.md | 13 +++++++++- 3 files changed, 53 insertions(+), 26 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index d1c4f36f..5969b855 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -318,6 +318,10 @@ container name and the link alias (`CONTAINER:ALIAS`). - project_db_1:mysql - project_db_1:postgresql +> **Note:** If you're using the [version 2 file format](#version-2), the +> externally-created containers must be connected to at least one of the same +> networks as the service which is linking to them. + ### extra_hosts Add hostname mappings. Use the same values as the docker client `--add-host` parameter. @@ -358,27 +362,24 @@ It's recommended that you use reverse-DNS notation to prevent your labels from c ### links -> [Version 1 file format](#version-1) only. In version 2 files, use -> [networking](networking.md) for communication between containers. - Link to containers in another service. Either specify both the service name and -the link alias (`SERVICE:ALIAS`), or just the service name (which will also be -used for the alias). +a link alias (`SERVICE:ALIAS`), or just the service name. - links: - - db - - db:database - - redis + web: + links: + - db + - db:database + - redis -An entry with the alias' name will be created in `/etc/hosts` inside containers -for this service, e.g: +Containers for the linked service will be reachable at a hostname identical to +the alias, or the service name if no alias was specified. - 172.17.2.186 db - 172.17.2.186 database - 172.17.2.187 redis +Links also express dependency between services in the same way as +[depends_on](#depends-on), so they determine the order of service startup. -Environment variables will also be created - see the [environment variable -reference](env.md) for details. +> **Note:** If you define both links and [networks](#networks), services with +> links between them must share at least one network in common in order to +> communicate. ### logging @@ -862,7 +863,6 @@ In the majority of cases, moving from version 1 to 2 is a very simple process: 1. Indent the whole file by one level and put a `services:` key at the top. 2. Add a `version: 2` line at the top of the file. -3. Delete all `links` entries. It's more complicated if you're using particular configuration features: @@ -879,14 +879,28 @@ It's more complicated if you're using particular configuration features: options: syslog-address: "tcp://192.168.0.42:123" -- `links` with aliases: If you've defined a link with an alias such as - `myservice:db`, there's currently no equivalent to this in version 2. You - will have to refer to the service using its name (in this example, - `myservice`). +- `links` with environment variables: As documented in the + [environment variables reference](env.md), environment variables created by + links have been deprecated for some time. In the new Docker network system, + they have been removed. You should either connect directly to the + appropriate hostname or set the relevant environment variable yourself, + using the link hostname: -- `external_links`: Links are deprecated, so you should use - [external networks](networking.md#using-externally-created-networks) to - communicate with containers outside the app. + web: + links: + - db + environment: + - DB_PORT=tcp://db:5432 + +- `external_links`: Compose uses Docker networks when running version 2 + projects, so links behave slightly differently. In particular, two + containers must be connected to at least one network in common in order to + communicate, even if explicitly linked together. + + Either connect the external container to your app's + [default network](networking.md), or connect both the external container and + your service's containers to an + [external network](networking.md#using-a-pre-existing-network). - `net`: If you're using `host`, `bridge` or `none`, this is now replaced by `networks`: diff --git a/docs/env.md b/docs/env.md index c0e03a4e..7b7e1bc7 100644 --- a/docs/env.md +++ b/docs/env.md @@ -11,7 +11,9 @@ weight=3 # Compose environment variables reference -**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. +> **Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. +> +> Environment variables will only be populated if you're using the [legacy version 1 Compose file format](compose-file.md#versioning). Compose uses [Docker links] to expose services' containers to one another. Each linked container injects a set of environment variables, each of which begins with the uppercase name of the container. diff --git a/docs/networking.md b/docs/networking.md index a5b49c1f..1bdd7fe0 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -57,7 +57,18 @@ If any containers have connections open to the old container, they will be close ## Links -Docker links are a one-way, single-host communication system. They should now be considered deprecated, and as part of upgrading your app to the v2 format, you must remove any `links` sections from your `docker-compose.yml` and use service names (e.g. `web`, `db`) as the hostnames to connect to. +Links allow you to define extra aliases by which a service is reachable from another service. They are not required to enable services to communicate - by default, any service can reach any other service at that service's name. In the following example, `db` is reachable from `web` at the hostnames `db` and `database`: + + version: 2 + services: + web: + build: . + links: + - "db:database" + db: + image: postgres + +See the [links reference](compose-file.md#links) for more information. ## Multi-host networking From 0952c1bb51e85baf2942473c7261a52928f9d5c9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 13:58:01 +0000 Subject: [PATCH 294/359] Add links to networks key references Signed-off-by: Aanand Prasad --- docs/networking.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/docs/networking.md b/docs/networking.md index 1bdd7fe0..bcfd39e5 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -110,6 +110,11 @@ Here's an example Compose file defining two custom networks. The `proxy` service foo: "1" bar: "2" +For full details of the network configuration options available, see the following references: + +- [Top-level `networks` key](compose-file.md#network-configuration-reference) +- [Service-level `networks` key](compose-file.md#networks) + ## Configuring the default network Instead of (or as well as) specifying your own networks, you can also change the settings of the app-wide default network by defining an entry under `networks` named `default`: From 38a6d04852e4056f7ee07fdba343f40875a6ef10 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Wed, 20 Jan 2016 11:07:28 -0800 Subject: [PATCH 295/359] Update documentation for `external` param Signed-off-by: Joffrey F Conflicts: docs/compose-file.md --- docs/compose-file.md | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 5969b855..67adeb82 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -627,7 +627,11 @@ documentation for more information. Optional. ## external If set to `true`, specifies that this volume has been created outside of -Compose. +Compose. `docker-compose up` will not attempt to create it, and will raise +an error if it doesn't exist. + +`external` cannot be used in conjunction with other volume configuration keys +(`driver`, `driver_opts`). In the example below, instead of attemping to create a volume called `[projectname]_data`, Compose will look for an existing volume simply @@ -712,7 +716,11 @@ A full example: ### external If set to `true`, specifies that this network has been created outside of -Compose. +Compose. `docker-compose up` will not attempt to create it, and will raise +an error if it doesn't exist. + +`external` cannot be used in conjunction with other network configuration keys +(`driver`, `driver_opts`, `ipam`). In the example below, `proxy` is the gateway to the outside world. Instead of attemping to create a network called `[projectname]_outside`, Compose will From 6ca410fd6b993fae0d17b43e9e34e1f149098626 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 16:49:50 +0000 Subject: [PATCH 296/359] Fix 'run' behaviour with networks - Test that one-off containers join all networks - Don't set any aliases Signed-off-by: Aanand Prasad --- compose/service.py | 17 ++++++++++--- tests/acceptance/cli_test.py | 49 ++++++++++++++++++++++++++++++------ 2 files changed, 54 insertions(+), 12 deletions(-) diff --git a/compose/service.py b/compose/service.py index 8afc59c1..ebe7978c 100644 --- a/compose/service.py +++ b/compose/service.py @@ -430,10 +430,12 @@ class Service(object): return container def connect_container_to_networks(self, container): + one_off = (container.labels.get(LABEL_ONE_OFF) == "True") + for network in self.networks: self.client.connect_container_to_network( container.id, network, - aliases=[self.name], + aliases=self._get_aliases(one_off=one_off), links=self._get_links(False), ) @@ -505,6 +507,12 @@ class Service(object): numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 + def _get_aliases(self, one_off): + if one_off: + return [] + + return [self.name] + def _get_links(self, link_to_self): links = {} @@ -610,7 +618,8 @@ class Service(object): override_options, one_off=one_off) - container_options['networking_config'] = self._get_container_networking_config() + container_options['networking_config'] = self._get_container_networking_config( + one_off=one_off) return container_options @@ -646,10 +655,10 @@ class Service(object): cpu_quota=options.get('cpu_quota'), ) - def _get_container_networking_config(self): + def _get_container_networking_config(self, one_off=False): return self.client.create_networking_config({ network_name: self.client.create_endpoint_config( - aliases=[self.name], + aliases=self._get_aliases(one_off=one_off), links=self._get_links(False), ) for network_name in self.networks diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 1806e707..6ae04ee5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -903,14 +903,47 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(container.name, name) @v2_only() - def test_run_with_networking(self): - self.base_dir = 'tests/fixtures/v2-simple' - self.dispatch(['run', 'simple', 'true'], None) - service = self.project.get_service('simple') - container, = service.containers(stopped=True, one_off=True) - networks = self.client.networks(names=[self.project.default_network.full_name]) - self.assertEqual(len(networks), 1) - self.assertEqual(container.human_readable_command, u'true') + def test_run_interactive_connects_to_network(self): + self.base_dir = 'tests/fixtures/networks' + + self.dispatch(['up', '-d']) + self.dispatch(['run', 'app', 'nslookup', 'app']) + self.dispatch(['run', 'app', 'nslookup', 'db']) + + containers = self.project.get_service('app').containers( + stopped=True, one_off=True) + assert len(containers) == 2 + + for container in containers: + networks = container.get('NetworkSettings.Networks') + + assert sorted(list(networks)) == [ + '{}_{}'.format(self.project.name, name) + for name in ['back', 'front'] + ] + + for _, config in networks.items(): + assert not config['Aliases'] + + @v2_only() + def test_run_detached_connects_to_network(self): + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['up', '-d']) + self.dispatch(['run', '-d', 'app', 'top']) + + container = self.project.get_service('app').containers(one_off=True)[0] + networks = container.get('NetworkSettings.Networks') + + assert sorted(list(networks)) == [ + '{}_{}'.format(self.project.name, name) + for name in ['back', 'front'] + ] + + for _, config in networks.items(): + assert not config['Aliases'] + + assert self.lookup(container, 'app') + assert self.lookup(container, 'db') def test_run_handles_sigint(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) From 836ec709793c5a417a47063f262dce229f5cd144 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 18:04:50 +0000 Subject: [PATCH 297/359] Stop connecting to all networks on container creation This relies on an Engine behaviour which is a bug, not an intentional feature - we have to connect to networks one at a time Signed-off-by: Aanand Prasad --- compose/service.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/compose/service.py b/compose/service.py index ebe7978c..18322846 100644 --- a/compose/service.py +++ b/compose/service.py @@ -656,13 +656,17 @@ class Service(object): ) def _get_container_networking_config(self, one_off=False): + if self.net.mode in ['host', 'bridge']: + return None + + if self.net.mode not in self.networks: + return None + return self.client.create_networking_config({ - network_name: self.client.create_endpoint_config( + self.net.mode: self.client.create_endpoint_config( aliases=self._get_aliases(one_off=one_off), links=self._get_links(False), ) - for network_name in self.networks - if network_name not in ['host', 'bridge'] }) def build(self, no_cache=False, pull=False, force_rm=False): From b1ebf5ce17288b95638dc8202c8eefb7d9c8b2cf Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 18:34:18 +0000 Subject: [PATCH 298/359] Fix interactive run with networking Make sure we connect the container to all required networks *after* starting the container and *before* hijacking the terminal. Signed-off-by: Aanand Prasad --- compose/cli/main.py | 8 +++++--- requirements.txt | 2 +- tests/unit/cli_test.py | 4 ++-- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 4be8536f..7409107d 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -41,7 +41,7 @@ from .utils import yesno if not IS_WINDOWS_PLATFORM: - import dockerpty + from dockerpty.pty import PseudoTerminal log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) @@ -709,8 +709,10 @@ def run_one_off_container(container_options, project, service, options): signals.set_signal_handler_to_shutdown() try: try: - dockerpty.start(project.client, container.id, interactive=not options['-T']) - service.connect_container_to_networks(container) + pty = PseudoTerminal(project.client, container.id, interactive=not options['-T']) + sockets = pty.sockets() + service.start_container(container) + pty.start(sockets) exit_code = container.wait() except signals.ShutdownException: project.client.stop(container.id) diff --git a/requirements.txt b/requirements.txt index 563baa10..45e18528 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ PyYAML==3.11 cached-property==1.2.0 -dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 +git+https://github.com/d11wtq/dockerpty.git@29b1394108b017ef3e3deaf00604a9eb99880d5e#egg=dockerpty git+https://github.com/docker/docker-py.git@master#egg=docker-py jsonschema==2.5.1 requests==2.7.0 diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index a5767097..f9e3fb8f 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -72,8 +72,8 @@ class CLITestCase(unittest.TestCase): TopLevelCommand().dispatch(['help', 'nonexistent'], None) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") - @mock.patch('compose.cli.main.dockerpty', autospec=True) - def test_run_with_environment_merged_with_options_list(self, mock_dockerpty): + @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) + def test_run_with_environment_merged_with_options_list(self, mock_pseudo_terminal): command = TopLevelCommand() mock_client = mock.create_autospec(docker.Client) mock_project = mock.Mock(client=mock_client) From bf068a828743caa7aee49b36ed205d7ffbb5a5d4 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Jan 2016 15:03:41 -0500 Subject: [PATCH 299/359] Add stop signal to the docs. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 32 ++++++++++++++++++++------------ docs/index.md | 3 +-- 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 67adeb82..b2675ac9 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -494,9 +494,17 @@ port (a random host port will be chosen). Override the default labeling scheme for each container. - security_opt: - - label:user:USER - - label:role:ROLE + security_opt: + - label:user:USER + - label:role:ROLE + +### stop_signal + +Sets an alternative signal to stop the container. By default `stop` uses +SIGTERM. Setting an alternative signal using `stop_signal` will cause +`stop` to send that signal instead. + + stop_signal: SIGUSR1 ### ulimits @@ -504,11 +512,11 @@ Override the default ulimits for a container. You can either specify a single limit as an integer or soft/hard limits as a mapping. - ulimits: - nproc: 65535 - nofile: - soft: 20000 - hard: 40000 + ulimits: + nproc: 65535 + nofile: + soft: 20000 + hard: 40000 ### volumes, volume\_driver @@ -612,7 +620,7 @@ subcommand documentation for more information. Specify which volume driver should be used for this volume. Defaults to `local`. The Docker Engine will return an error if the driver is not available. - driver: foobar + driver: foobar ### driver_opts @@ -620,9 +628,9 @@ Specify a list of options as key-value pairs to pass to the driver for this volume. Those options are driver-dependent - consult the driver's documentation for more information. Optional. - driver_opts: - foo: "bar" - baz: 1 + driver_opts: + foo: "bar" + baz: 1 ## external diff --git a/docs/index.md b/docs/index.md index 887df99d..9cb594a7 100644 --- a/docs/index.md +++ b/docs/index.md @@ -45,8 +45,7 @@ A `docker-compose.yml` looks like this: redis: image: redis volumes: - logvolume01: - driver: default + logvolume01: {} For more information about the Compose file, see the [Compose file reference](compose-file.md) From 833e16117ec942184f145d3388a9843123fa7349 Mon Sep 17 00:00:00 2001 From: Alf Lervag Date: Tue, 22 Dec 2015 11:43:48 +0100 Subject: [PATCH 300/359] Fixes #2448 Signed-off-by: Alf Lervag Conflicts: compose/cli/main.py requirements.txt --- compose/cli/main.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 7409107d..14febe25 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -709,7 +709,12 @@ def run_one_off_container(container_options, project, service, options): signals.set_signal_handler_to_shutdown() try: try: - pty = PseudoTerminal(project.client, container.id, interactive=not options['-T']) + pty = PseudoTerminal( + project.client, + container.id, + interactive=not options['-T'], + logs=False, + ) sockets = pty.sockets() service.start_container(container) pty.start(sockets) From d399b7893f14a9089720bd882c70c2a98d18c4ed Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 21 Jan 2016 19:15:15 +0000 Subject: [PATCH 301/359] Add test for logs=False Signed-off-by: Aanand Prasad Conflicts: compose/cli/main.py --- tests/unit/cli_test.py | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index f9e3fb8f..fd52a3c1 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -71,6 +71,37 @@ class CLITestCase(unittest.TestCase): with self.assertRaises(NoSuchCommand): TopLevelCommand().dispatch(['help', 'nonexistent'], None) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") + @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) + def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal): + command = TopLevelCommand() + mock_client = mock.create_autospec(docker.Client) + mock_project = mock.Mock(client=mock_client) + mock_project.get_service.return_value = Service( + 'service', + client=mock_client, + environment=['FOO=ONE', 'BAR=TWO'], + image='someimage') + + with pytest.raises(SystemExit): + command.run(mock_project, { + 'SERVICE': 'service', + 'COMMAND': None, + '-e': ['BAR=NEW', 'OTHER=bär'.encode('utf-8')], + '--user': None, + '--no-deps': None, + '-d': False, + '-T': None, + '--entrypoint': None, + '--service-ports': None, + '--publish': [], + '--rm': None, + '--name': None, + }) + + _, _, call_kwargs = mock_pseudo_terminal.mock_calls[0] + assert call_kwargs['logs'] is False + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) def test_run_with_environment_merged_with_options_list(self, mock_pseudo_terminal): From 227fa5c0dee51683d34495ac1ca97ab63c94f2e6 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jan 2016 18:07:14 -0800 Subject: [PATCH 302/359] Use latest docker-py rc Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 45e18528..ed3b8686 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ PyYAML==3.11 cached-property==1.2.0 +docker-py==1.7.0rc2 docopt==0.6.1 enum34==1.0.4 git+https://github.com/d11wtq/dockerpty.git@29b1394108b017ef3e3deaf00604a9eb99880d5e#egg=dockerpty -git+https://github.com/docker/docker-py.git@master#egg=docker-py jsonschema==2.5.1 requests==2.7.0 six==1.7.3 From 5545c55eccac1619bed2618621a2856bb79bc026 Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Fri, 22 Jan 2016 01:18:15 +0200 Subject: [PATCH 303/359] Network fields schema validation Signed-off-by: Dimitar Bonev --- compose/config/fields_schema_v2.json | 29 +++++++++++++++++++- docs/networking.md | 2 +- tests/unit/config/config_test.py | 41 ++++++++++++++++++++++++++++ 3 files changed, 70 insertions(+), 2 deletions(-) diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index b0f304e8..c001df68 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -41,7 +41,34 @@ "definitions": { "network": { "id": "#/definitions/network", - "type": "object" + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array" + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false }, "volume": { "id": "#/definitions/volume", diff --git a/docs/networking.md b/docs/networking.md index bcfd39e5..1e662dd2 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -106,7 +106,7 @@ Here's an example Compose file defining two custom networks. The `proxy` service back: # Use a custom driver which takes special options driver: my-custom-driver - options: + driver_opts: foo: "1" bar: "2" diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index eb8ed2c7..fe609820 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -88,11 +88,26 @@ class ConfigTest(unittest.TestCase): 'driver': 'default', 'driver_opts': {'beep': 'boop'} } + }, + 'networks': { + 'default': { + 'driver': 'bridge', + 'driver_opts': {'beep': 'boop'} + }, + 'with_ipam': { + 'ipam': { + 'driver': 'default', + 'config': [ + {'subnet': '172.28.0.0/16'} + ] + } + } } }, 'working_dir', 'filename.yml') ) service_dicts = config_data.services volume_dict = config_data.volumes + networks_dict = config_data.networks self.assertEqual( service_sort(service_dicts), service_sort([ @@ -113,6 +128,20 @@ class ConfigTest(unittest.TestCase): 'driver_opts': {'beep': 'boop'} } }) + self.assertEqual(networks_dict, { + 'default': { + 'driver': 'bridge', + 'driver_opts': {'beep': 'boop'} + }, + 'with_ipam': { + 'ipam': { + 'driver': 'default', + 'config': [ + {'subnet': '172.28.0.0/16'} + ] + } + } + }) def test_named_volume_config_empty(self): config_details = build_config_details({ @@ -191,6 +220,18 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_throws_error_with_invalid_network_fields(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 2, + 'services': {'web': 'busybox:latest'}, + 'networks': { + 'invalid': {'foo', 'bar'} + } + }, 'working_dir', 'filename.yml') + ) + def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: From 883227c4d841805c631c650bb96835515d59e0e9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 25 Jan 2016 16:14:21 +0000 Subject: [PATCH 304/359] Alias containers by short id Signed-off-by: Aanand Prasad --- compose/service.py | 31 +++++++++---------------------- tests/acceptance/cli_test.py | 8 ++++++-- 2 files changed, 15 insertions(+), 24 deletions(-) diff --git a/compose/service.py b/compose/service.py index 18322846..166fb0b2 100644 --- a/compose/service.py +++ b/compose/service.py @@ -430,12 +430,16 @@ class Service(object): return container def connect_container_to_networks(self, container): - one_off = (container.labels.get(LABEL_ONE_OFF) == "True") + connected_networks = container.get('NetworkSettings.Networks') for network in self.networks: + if network in connected_networks: + self.client.disconnect_container_from_network( + container.id, network) + self.client.connect_container_to_network( container.id, network, - aliases=self._get_aliases(one_off=one_off), + aliases=self._get_aliases(container), links=self._get_links(False), ) @@ -507,11 +511,11 @@ class Service(object): numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 - def _get_aliases(self, one_off): - if one_off: + def _get_aliases(self, container): + if container.labels.get(LABEL_ONE_OFF) == "True": return [] - return [self.name] + return [self.name, container.short_id] def _get_links(self, link_to_self): links = {} @@ -618,9 +622,6 @@ class Service(object): override_options, one_off=one_off) - container_options['networking_config'] = self._get_container_networking_config( - one_off=one_off) - return container_options def _get_container_host_config(self, override_options, one_off=False): @@ -655,20 +656,6 @@ class Service(object): cpu_quota=options.get('cpu_quota'), ) - def _get_container_networking_config(self, one_off=False): - if self.net.mode in ['host', 'bridge']: - return None - - if self.net.mode not in self.networks: - return None - - return self.client.create_networking_config({ - self.net.mode: self.client.create_endpoint_config( - aliases=self._get_aliases(one_off=one_off), - links=self._get_links(False), - ) - }) - def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6ae04ee5..7bc72305 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -420,8 +420,12 @@ class CLITestCase(DockerClientTestCase): container = containers[0] self.assertIn(container.id, network['Containers']) - networks = list(container.get('NetworkSettings.Networks')) - self.assertEqual(networks, [network['Name']]) + networks = container.get('NetworkSettings.Networks') + self.assertEqual(list(networks), [network['Name']]) + + self.assertEqual( + sorted(networks[network['Name']]['Aliases']), + sorted([service.name, container.short_id])) for service in services: assert self.lookup(container, service.name) From a66bf72199f2b8cb14a9256f17c97c344535e323 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Thu, 21 Jan 2016 18:03:58 -0800 Subject: [PATCH 305/359] Match named volumes in service definitions with declared volumes Raise ConfigurationError for undeclared named volumes Test new behavior Signed-off-by: Joffrey F --- compose/config/types.py | 6 +++++ compose/project.py | 45 ++++++++++++++++++++++--------- tests/integration/project_test.py | 36 +++++++++++++++++++++++++ tests/unit/project_test.py | 43 +++++++++++++++++++++++++++++ 4 files changed, 118 insertions(+), 12 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index b872cba9..2bb2519d 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -163,3 +163,9 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): def repr(self): external = self.external + ':' if self.external else '' return '{ext}{v.internal}:{v.mode}'.format(ext=external, v=self) + + @property + def is_named_volume(self): + return self.external and not ( + self.external.startswith('.') or self.external.startswith('/') + ) diff --git a/compose/project.py b/compose/project.py index 0fea875a..bbc61a5b 100644 --- a/compose/project.py +++ b/compose/project.py @@ -74,6 +74,17 @@ class Project(object): if 'default' not in network_config: all_networks.append(project.default_network) + if config_data.volumes: + for vol_name, data in config_data.volumes.items(): + project.volumes.append( + Volume( + client=client, project=name, name=vol_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external_name=data.get('external_name') + ) + ) + for service_dict in config_data.services: if use_networking: networks = get_networks(service_dict, all_networks) @@ -85,6 +96,9 @@ class Project(object): links = project.get_links(service_dict) volumes_from = get_volumes_from(project, service_dict) + if config_data.version == 2: + match_named_volumes(service_dict, project.volumes) + project.services.append( Service( client=client, @@ -94,23 +108,13 @@ class Project(object): links=links, net=net, volumes_from=volumes_from, - **service_dict)) + **service_dict) + ) project.networks += custom_networks if 'default' not in network_config and project.uses_default_network(): project.networks.append(project.default_network) - if config_data.volumes: - for vol_name, data in config_data.volumes.items(): - project.volumes.append( - Volume( - client=client, project=name, name=vol_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - external_name=data.get('external_name') - ) - ) - return project @property @@ -472,6 +476,23 @@ def get_networks(service_dict, network_definitions): return networks +def match_named_volumes(service_dict, project_volumes): + for volume_spec in service_dict.get('volumes', []): + if volume_spec.is_named_volume: + declared_volume = next( + (v for v in project_volumes if v.name == volume_spec.external), + None + ) + if not declared_volume: + raise ConfigurationError( + 'Named volume "{0}" is used in service "{1}" but no' + ' declaration was found in the volumes section.'.format( + volume_spec.repr(), service_dict.get('name') + ) + ) + volume_spec._replace(external=declared_volume.full_name) + + def get_volumes_from(project, service_dict): volumes_from = service_dict.pop('volumes_from', None) if not volumes_from: diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d3fbb71e..aa0dd33f 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -813,3 +813,39 @@ class ProjectTest(DockerClientTestCase): assert 'Volume {0} declared as external'.format( vol_name ) in str(e.exception) + + @v2_only() + def test_project_up_named_volumes_in_binds(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + + base_file = config.ConfigFile( + 'base.yml', + { + 'version': 2, + 'services': { + 'simple': { + 'image': 'busybox:latest', + 'command': 'top', + 'volumes': ['{0}:/data'.format(vol_name)] + }, + }, + 'volumes': { + vol_name: {'driver': 'local'} + } + + }) + config_details = config.ConfigDetails('.', [base_file]) + config_data = config.load(config_details) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + service = project.services[0] + self.assertEqual(service.name, 'simple') + volumes = service.options.get('volumes') + self.assertEqual(len(volumes), 1) + self.assertEqual(volumes[0].external, full_vol_name) + project.up() + engine_volumes = self.client.volumes() + self.assertIsNone(next(v for v in engine_volumes if v['Name'] == vol_name)) + self.assertIsNotNone(next(v for v in engine_volumes if v['Name'] == full_vol_name)) diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index ffd4455f..f587d697 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -7,8 +7,10 @@ import docker from .. import mock from .. import unittest +from compose.config import ConfigurationError from compose.config.config import Config from compose.config.types import VolumeFromSpec +from compose.config.types import VolumeSpec from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project @@ -476,3 +478,44 @@ class ProjectTest(unittest.TestCase): ), ) self.assertEqual([c.id for c in project.containers()], ['1']) + + def test_undeclared_volume_v2(self): + config = Config( + version=2, + services=[ + { + 'name': 'web', + 'image': 'busybox:latest', + 'volumes': [VolumeSpec.parse('data0028:/data:ro')], + }, + ], + networks=None, + volumes=None, + ) + with self.assertRaises(ConfigurationError): + Project.from_config('composetest', config, None) + + config = Config( + version=2, + services=[ + { + 'name': 'web', + 'image': 'busybox:latest', + 'volumes': [VolumeSpec.parse('./data0028:/data:ro')], + }, + ], networks=None, volumes=None, + ) + Project.from_config('composetest', config, None) + + def test_undeclared_volume_v1(self): + config = Config( + version=1, + services=[ + { + 'name': 'web', + 'image': 'busybox:latest', + 'volumes': [VolumeSpec.parse('data0028:/data:ro')], + }, + ], networks=None, volumes=None, + ) + Project.from_config('composetest', config, None) From 3da25aa463ade5813f32cbca6caedcb329d51b30 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 22 Jan 2016 16:05:21 -0800 Subject: [PATCH 306/359] is_named_volume also tests for home paths ~ Fix bug with VolumeSpec not being updated Fix integration test Signed-off-by: Joffrey F --- compose/config/types.py | 4 +--- compose/project.py | 7 +++++-- tests/integration/project_test.py | 7 ++++--- 3 files changed, 10 insertions(+), 8 deletions(-) diff --git a/compose/config/types.py b/compose/config/types.py index 2bb2519d..2e648e5a 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -166,6 +166,4 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): @property def is_named_volume(self): - return self.external and not ( - self.external.startswith('.') or self.external.startswith('/') - ) + return self.external and not self.external.startswith(('.', '/', '~')) diff --git a/compose/project.py b/compose/project.py index bbc61a5b..8e763abf 100644 --- a/compose/project.py +++ b/compose/project.py @@ -477,7 +477,8 @@ def get_networks(service_dict, network_definitions): def match_named_volumes(service_dict, project_volumes): - for volume_spec in service_dict.get('volumes', []): + service_volumes = service_dict.get('volumes', []) + for volume_spec in service_volumes: if volume_spec.is_named_volume: declared_volume = next( (v for v in project_volumes if v.name == volume_spec.external), @@ -490,7 +491,9 @@ def match_named_volumes(service_dict, project_volumes): volume_spec.repr(), service_dict.get('name') ) ) - volume_spec._replace(external=declared_volume.full_name) + service_volumes[service_volumes.index(volume_spec)] = ( + volume_spec._replace(external=declared_volume.full_name) + ) def get_volumes_from(project, service_dict): diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index aa0dd33f..586f9444 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -846,6 +846,7 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(volumes), 1) self.assertEqual(volumes[0].external, full_vol_name) project.up() - engine_volumes = self.client.volumes() - self.assertIsNone(next(v for v in engine_volumes if v['Name'] == vol_name)) - self.assertIsNotNone(next(v for v in engine_volumes if v['Name'] == full_vol_name)) + engine_volumes = self.client.volumes()['Volumes'] + container = service.get_container() + assert [mount['Name'] for mount in container.get('Mounts')] == [full_vol_name] + assert next((v for v in engine_volumes if v['Name'] == vol_name), None) is None From 3f28472ebc136d82731ca5a88c20cabc13864925 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Fri, 22 Jan 2016 17:42:24 -0800 Subject: [PATCH 307/359] Move named volumes matching to config validation phase Signed-off-by: Joffrey F --- compose/config/config.py | 6 ++++ compose/config/validation.py | 12 ++++++++ compose/project.py | 46 ++++++++++------------------- tests/unit/config/config_test.py | 50 ++++++++++++++++++++++++++++++++ tests/unit/project_test.py | 43 --------------------------- 5 files changed, 83 insertions(+), 74 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 961d36bb..8e7d96e2 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -25,6 +25,7 @@ from .types import parse_extra_hosts from .types import parse_restart_spec from .types import VolumeFromSpec from .types import VolumeSpec +from .validation import match_named_volumes from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_depends_on @@ -274,6 +275,11 @@ def load(config_details): config_details.working_dir, main_file, [file.get_service_dicts() for file in config_details.config_files]) + + if main_file.version >= 2: + for service_dict in service_dicts: + match_named_volumes(service_dict, volumes) + return Config(main_file.version, service_dicts, volumes, networks) diff --git a/compose/config/validation.py b/compose/config/validation.py index ecf8d4f9..5c2d69ec 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -77,6 +77,18 @@ def format_boolean_in_environment(instance): return True +def match_named_volumes(service_dict, project_volumes): + service_volumes = service_dict.get('volumes', []) + for volume_spec in service_volumes: + if volume_spec.is_named_volume and volume_spec.external not in project_volumes: + raise ConfigurationError( + 'Named volume "{0}" is used in service "{1}" but no' + ' declaration was found in the volumes section.'.format( + volume_spec.repr(), service_dict.get('name') + ) + ) + + def validate_top_level_service_objects(filename, service_dicts): """Perform some high level validation of the service name and value. diff --git a/compose/project.py b/compose/project.py index 8e763abf..b51fcd7e 100644 --- a/compose/project.py +++ b/compose/project.py @@ -42,7 +42,7 @@ class Project(object): self.use_networking = use_networking self.network_driver = network_driver self.networks = networks or [] - self.volumes = volumes or [] + self.volumes = volumes or {} def labels(self, one_off=False): return [ @@ -76,13 +76,11 @@ class Project(object): if config_data.volumes: for vol_name, data in config_data.volumes.items(): - project.volumes.append( - Volume( - client=client, project=name, name=vol_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - external_name=data.get('external_name') - ) + project.volumes[vol_name] = Volume( + client=client, project=name, name=vol_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external_name=data.get('external_name') ) for service_dict in config_data.services: @@ -97,7 +95,13 @@ class Project(object): volumes_from = get_volumes_from(project, service_dict) if config_data.version == 2: - match_named_volumes(service_dict, project.volumes) + service_volumes = service_dict.get('volumes', []) + for volume_spec in service_volumes: + if volume_spec.is_named_volume: + declared_volume = project.volumes[volume_spec.external] + service_volumes[service_volumes.index(volume_spec)] = ( + volume_spec._replace(external=declared_volume.full_name) + ) project.services.append( Service( @@ -243,7 +247,7 @@ class Project(object): def initialize_volumes(self): try: - for volume in self.volumes: + for volume in self.volumes.values(): if volume.external: log.debug( 'Volume {0} declared as external. No new ' @@ -298,7 +302,7 @@ class Project(object): network.remove() def remove_volumes(self): - for volume in self.volumes: + for volume in self.volumes.values(): volume.remove() def initialize_networks(self): @@ -476,26 +480,6 @@ def get_networks(service_dict, network_definitions): return networks -def match_named_volumes(service_dict, project_volumes): - service_volumes = service_dict.get('volumes', []) - for volume_spec in service_volumes: - if volume_spec.is_named_volume: - declared_volume = next( - (v for v in project_volumes if v.name == volume_spec.external), - None - ) - if not declared_volume: - raise ConfigurationError( - 'Named volume "{0}" is used in service "{1}" but no' - ' declaration was found in the volumes section.'.format( - volume_spec.repr(), service_dict.get('name') - ) - ) - service_volumes[service_volumes.index(volume_spec)] = ( - volume_spec._replace(external=declared_volume.full_name) - ) - - def get_volumes_from(project, service_dict): volumes_from = service_dict.pop('volumes_from', None) if not volumes_from: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index fe609820..98fe7758 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -564,6 +564,56 @@ class ConfigTest(unittest.TestCase): ] assert service_sort(service_dicts) == service_sort(expected) + def test_undeclared_volume_v2(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': ['data0028:/data:ro'], + }, + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + with self.assertRaises(ConfigurationError): + config.load(details) + + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': ['./data0028:/data:ro'], + }, + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + volume = config_data.services[0].get('volumes')[0] + assert not volume.is_named_volume + + def test_undeclared_volume_v1(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': { + 'image': 'busybox:latest', + 'volumes': ['data0028:/data:ro'], + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + volume = config_data.services[0].get('volumes')[0] + assert volume.external == 'data0028' + assert volume.is_named_volume + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f587d697..ffd4455f 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -7,10 +7,8 @@ import docker from .. import mock from .. import unittest -from compose.config import ConfigurationError from compose.config.config import Config from compose.config.types import VolumeFromSpec -from compose.config.types import VolumeSpec from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project @@ -478,44 +476,3 @@ class ProjectTest(unittest.TestCase): ), ) self.assertEqual([c.id for c in project.containers()], ['1']) - - def test_undeclared_volume_v2(self): - config = Config( - version=2, - services=[ - { - 'name': 'web', - 'image': 'busybox:latest', - 'volumes': [VolumeSpec.parse('data0028:/data:ro')], - }, - ], - networks=None, - volumes=None, - ) - with self.assertRaises(ConfigurationError): - Project.from_config('composetest', config, None) - - config = Config( - version=2, - services=[ - { - 'name': 'web', - 'image': 'busybox:latest', - 'volumes': [VolumeSpec.parse('./data0028:/data:ro')], - }, - ], networks=None, volumes=None, - ) - Project.from_config('composetest', config, None) - - def test_undeclared_volume_v1(self): - config = Config( - version=1, - services=[ - { - 'name': 'web', - 'image': 'busybox:latest', - 'volumes': [VolumeSpec.parse('data0028:/data:ro')], - }, - ], networks=None, volumes=None, - ) - Project.from_config('composetest', config, None) From 2b7306967bd0672646002e938c63377e0a833732 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 25 Jan 2016 12:45:30 +0000 Subject: [PATCH 308/359] Implement network_mode in v2 Signed-off-by: Aanand Prasad --- compose/config/config.py | 18 ++- compose/config/service_schema_v2.json | 1 + compose/config/sort_services.py | 12 +- compose/config/validation.py | 19 +++ compose/project.py | 43 +++--- docs/compose-file.md | 49 +++++-- docs/networking.md | 12 -- tests/acceptance/cli_test.py | 35 ++++- tests/fixtures/extends/invalid-net-v2.yml | 12 ++ tests/fixtures/networks/bridge.yml | 9 ++ tests/fixtures/networks/network-mode.yml | 27 ++++ .../fixtures/networks/predefined-networks.yml | 17 --- tests/integration/project_test.py | 99 +++++++++++-- tests/unit/config/config_test.py | 131 +++++++++++++++++- tests/unit/config/sort_services_test.py | 4 +- tests/unit/project_test.py | 4 +- 16 files changed, 405 insertions(+), 87 deletions(-) create mode 100644 tests/fixtures/extends/invalid-net-v2.yml create mode 100644 tests/fixtures/networks/bridge.yml create mode 100644 tests/fixtures/networks/network-mode.yml delete mode 100644 tests/fixtures/networks/predefined-networks.yml diff --git a/compose/config/config.py b/compose/config/config.py index 8e7d96e2..c1391c2f 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -19,6 +19,7 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables +from .sort_services import get_container_name_from_net from .sort_services import get_service_name_from_net from .sort_services import sort_service_dicts from .types import parse_extra_hosts @@ -30,6 +31,7 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_depends_on from .validation import validate_extends_file_path +from .validation import validate_network_mode from .validation import validate_top_level_object from .validation import validate_top_level_service_objects from .validation import validate_ulimits @@ -490,10 +492,15 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'volumes_from' cannot be extended" % error_prefix) if 'net' in service_dict: - if get_service_name_from_net(service_dict['net']) is not None: + if get_container_name_from_net(service_dict['net']): raise ConfigurationError( "%s services with 'net: container' cannot be extended" % error_prefix) + if 'network_mode' in service_dict: + if get_service_name_from_net(service_dict['network_mode']): + raise ConfigurationError( + "%s services with 'network_mode: service' cannot be extended" % error_prefix) + if 'depends_on' in service_dict: raise ConfigurationError( "%s services with 'depends_on' cannot be extended" % error_prefix) @@ -505,6 +512,7 @@ def validate_service(service_config, service_names, version): validate_paths(service_dict) validate_ulimits(service_config) + validate_network_mode(service_config, service_names) validate_depends_on(service_config, service_names) if not service_dict.get('image') and has_uppercase(service_name): @@ -565,6 +573,14 @@ def finalize_service(service_config, service_names, version): service_dict['volumes'] = [ VolumeSpec.parse(v) for v in service_dict['volumes']] + if 'net' in service_dict: + network_mode = service_dict.pop('net') + container_name = get_container_name_from_net(network_mode) + if container_name and container_name in service_names: + service_dict['network_mode'] = 'service:{}'.format(container_name) + else: + service_dict['network_mode'] = network_mode + if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 94046d5b..56c0cbf5 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -103,6 +103,7 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, + "network_mode": {"type": "string"}, "networks": { "type": "array", diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py index ac0fa458..cf38a603 100644 --- a/compose/config/sort_services.py +++ b/compose/config/sort_services.py @@ -5,10 +5,18 @@ from compose.config.errors import DependencyError def get_service_name_from_net(net_config): + return get_source_name_from_net(net_config, 'service') + + +def get_container_name_from_net(net_config): + return get_source_name_from_net(net_config, 'container') + + +def get_source_name_from_net(net_config, source_type): if not net_config: return - if not net_config.startswith('container:'): + if not net_config.startswith(source_type+':'): return _, net_name = net_config.split(':', 1) @@ -33,7 +41,7 @@ def sort_service_dicts(services): service for service in services if (name in get_service_names(service.get('links', [])) or name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or - name == get_service_name_from_net(service.get('net')) or + name == get_service_name_from_net(service.get('network_mode')) or name in service.get('depends_on', [])) ] diff --git a/compose/config/validation.py b/compose/config/validation.py index 5c2d69ec..dfc34d57 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -15,6 +15,7 @@ from jsonschema import RefResolver from jsonschema import ValidationError from .errors import ConfigurationError +from .sort_services import get_service_name_from_net log = logging.getLogger(__name__) @@ -147,6 +148,24 @@ def validate_extends_file_path(service_name, extends_options, filename): ) +def validate_network_mode(service_config, service_names): + network_mode = service_config.config.get('network_mode') + if not network_mode: + return + + if 'networks' in service_config.config: + raise ConfigurationError("'network_mode' and 'networks' cannot be combined") + + dependency = get_service_name_from_net(network_mode) + if not dependency: + return + + if dependency not in service_names: + raise ConfigurationError( + "Service '{s.name}' uses the network stack of service '{dep}' which " + "is undefined.".format(s=service_config, dep=dependency)) + + def validate_depends_on(service_config, service_names): for dependency in service_config.config.get('depends_on', []): if dependency not in service_names: diff --git a/compose/project.py b/compose/project.py index b51fcd7e..ef913d63 100644 --- a/compose/project.py +++ b/compose/project.py @@ -10,6 +10,7 @@ from docker.errors import NotFound from . import parallel from .config import ConfigurationError +from .config.sort_services import get_container_name_from_net from .config.sort_services import get_service_name_from_net from .const import DEFAULT_TIMEOUT from .const import IMAGE_EVENTS @@ -86,12 +87,11 @@ class Project(object): for service_dict in config_data.services: if use_networking: networks = get_networks(service_dict, all_networks) - net = Net(networks[0]) if networks else Net("none") else: networks = [] - net = project.get_net(service_dict) links = project.get_links(service_dict) + net = project.get_net(service_dict, networks) volumes_from = get_volumes_from(project, service_dict) if config_data.version == 2: @@ -197,27 +197,27 @@ class Project(object): del service_dict['links'] return links - def get_net(self, service_dict): - net = service_dict.pop('net', None) + def get_net(self, service_dict, networks): + net = service_dict.pop('network_mode', None) if not net: + if self.use_networking: + return Net(networks[0]) if networks else Net('none') return Net(None) - net_name = get_service_name_from_net(net) - if not net_name: - return Net(net) + service_name = get_service_name_from_net(net) + if service_name: + return ServiceNet(self.get_service(service_name)) - try: - return ServiceNet(self.get_service(net_name)) - except NoSuchService: - pass - try: - return ContainerNet(Container.from_id(self.client, net_name)) - except APIError: - raise ConfigurationError( - 'Service "%s" is trying to use the network of "%s", ' - 'which is not the name of a service or container.' % ( - service_dict['name'], - net_name)) + container_name = get_container_name_from_net(net) + if container_name: + try: + return ContainerNet(Container.from_id(self.client, container_name)) + except APIError: + raise ConfigurationError( + "Service '{name}' uses the network stack of container '{dep}' which " + "does not exist.".format(name=service_dict['name'], dep=container_name)) + + return Net(net) def start(self, service_names=None, **options): containers = [] @@ -465,9 +465,12 @@ class Project(object): def get_networks(service_dict, network_definitions): + if 'network_mode' in service_dict: + return [] + networks = [] for name in service_dict.pop('networks', ['default']): - if name in ['bridge', 'host']: + if name in ['bridge']: networks.append(name) else: matches = [n for n in network_definitions if n.name == name] diff --git a/docs/compose-file.md b/docs/compose-file.md index b2675ac9..6b61755f 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -437,14 +437,29 @@ Specify logging options as key-value pairs. An example of `syslog` options: ### net > [Version 1 file format](#version-1) only. In version 2, use -> [networks](#networks). +> [network_mode](#network_mode). -Networking mode. Use the same values as the docker client `--net` parameter. +Network mode. Use the same values as the docker client `--net` parameter. +The `container:...` form can take a service name instead of a container name or +id. net: "bridge" - net: "none" - net: "container:[name or id]" net: "host" + net: "none" + net: "container:[service name or container name/id]" + +### network_mode + +> [Version 2 file format](#version-1) only. In version 1, use [net](#net). + +Network mode. Use the same values as the docker client `--net` parameter, plus +the special form `service:[service name]`. + + network_mode: "bridge" + network_mode: "host" + network_mode: "none" + network_mode: "service:[service name]" + network_mode: "container:[container name/id]" ### networks @@ -457,8 +472,8 @@ Networks to join, referencing entries under the - some-network - other-network -The values `bridge`, `host` and `none` can also be used, and are equivalent to -`net: "bridge"`, `net: "host"` or `net: "none"` in version 1. +The value `bridge` can also be used to make containers join the pre-defined +`bridge` network. There is no equivalent to `net: "container:[name or id]"`. @@ -918,16 +933,22 @@ It's more complicated if you're using particular configuration features: your service's containers to an [external network](networking.md#using-a-pre-existing-network). -- `net`: If you're using `host`, `bridge` or `none`, this is now replaced by - `networks`: +- `net`: This is now replaced by [network_mode](#network_mode): - net: host -> networks: ["host"] - net: bridge -> networks: ["bridge"] - net: none -> networks: ["none"] + net: host -> network_mode: host + net: bridge -> network_mode: bridge + net: none -> network_mode: none - If you're using `net: "container:"`, there is no equivalent to this in - version 2 - you should use [Docker networks](networking.md) for - communication instead. + If you're using `net: "container:[service name]"`, you must now use + `network_mode: "service:[service name]"` instead. + + net: "container:web" -> network_mode: "service:web" + + If you're using `net: "container:[container name/id]"`, the value does not + need to change. + + net: "container:cont-name" -> network_mode: "container:cont-name" + net: "container:abc12345" -> network_mode: "container:abc12345" ## Variable substitution diff --git a/docs/networking.md b/docs/networking.md index 1e662dd2..93533e9d 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -144,15 +144,3 @@ If you want your containers to join a pre-existing network, use the [`external` name: my-pre-existing-network Instead of attemping to create a network called `[projectname]_default`, Compose will look for a network called `my-pre-existing-network` and connect your app's containers to it. - -## Custom container network modes - -The `docker` CLI command allows you to specify a custom network mode for a container with the `--net` option - for example, `--net=host` specifies that the container should use the same network namespace as the Docker host, and `--net=none` specifies that it should have no networking capabilities. - -To make use of this in Compose, specify a `networks` list with a single item `host`, `bridge` or `none`: - - app: - build: ./app - networks: ["host"] - -There is no equivalent to `--net=container:CONTAINER_NAME` in the v2 Compose file format. You should instead use networks to enable communication. diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 7bc72305..4b560efa 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -496,8 +496,29 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr @v2_only() - def test_up_predefined_networks(self): - filename = 'predefined-networks.yml' + def test_up_with_bridge_network_plus_default(self): + filename = 'bridge.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + self.dispatch(['-f', filename, 'up', '-d'], None) + + container = self.project.containers()[0] + + assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted([ + 'bridge', + self.project.default_network.full_name, + ]) + + @v2_only() + def test_up_with_network_mode(self): + c = self.client.create_container('busybox', 'top', name='composetest_network_mode_container') + self.addCleanup(self.client.remove_container, c, force=True) + self.client.start(c) + container_mode_source = 'container:{}'.format(c['Id']) + + filename = 'network-mode.yml' self.base_dir = 'tests/fixtures/networks' self._project = get_project(self.base_dir, [filename]) @@ -515,6 +536,16 @@ class CLITestCase(DockerClientTestCase): assert list(container.get('NetworkSettings.Networks')) == [name] assert container.get('HostConfig.NetworkMode') == name + service_mode_source = 'container:{}'.format( + self.project.get_service('bridge').containers()[0].id) + service_mode_container = self.project.get_service('service').containers()[0] + assert not service_mode_container.get('NetworkSettings.Networks') + assert service_mode_container.get('HostConfig.NetworkMode') == service_mode_source + + container_mode_container = self.project.get_service('container').containers()[0] + assert not container_mode_container.get('NetworkSettings.Networks') + assert container_mode_container.get('HostConfig.NetworkMode') == container_mode_source + @v2_only() def test_up_external_networks(self): filename = 'external-networks.yml' diff --git a/tests/fixtures/extends/invalid-net-v2.yml b/tests/fixtures/extends/invalid-net-v2.yml new file mode 100644 index 00000000..0a04f468 --- /dev/null +++ b/tests/fixtures/extends/invalid-net-v2.yml @@ -0,0 +1,12 @@ +version: 2 +services: + myweb: + build: '.' + extends: + service: web + command: top + web: + build: '.' + network_mode: "service:net" + net: + build: '.' diff --git a/tests/fixtures/networks/bridge.yml b/tests/fixtures/networks/bridge.yml new file mode 100644 index 00000000..95098372 --- /dev/null +++ b/tests/fixtures/networks/bridge.yml @@ -0,0 +1,9 @@ +version: 2 + +services: + web: + image: busybox + command: top + networks: + - bridge + - default diff --git a/tests/fixtures/networks/network-mode.yml b/tests/fixtures/networks/network-mode.yml new file mode 100644 index 00000000..7ab63df8 --- /dev/null +++ b/tests/fixtures/networks/network-mode.yml @@ -0,0 +1,27 @@ +version: 2 + +services: + bridge: + image: busybox + command: top + network_mode: bridge + + service: + image: busybox + command: top + network_mode: "service:bridge" + + container: + image: busybox + command: top + network_mode: "container:composetest_network_mode_container" + + host: + image: busybox + command: top + network_mode: host + + none: + image: busybox + command: top + network_mode: none diff --git a/tests/fixtures/networks/predefined-networks.yml b/tests/fixtures/networks/predefined-networks.yml deleted file mode 100644 index d0fac377..00000000 --- a/tests/fixtures/networks/predefined-networks.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 - -services: - bridge: - image: busybox - command: top - networks: ["bridge"] - - host: - image: busybox - command: top - networks: ["host"] - - none: - image: busybox - command: top - networks: [] diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 586f9444..0945ebb8 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -4,10 +4,12 @@ from __future__ import unicode_literals import random import py +import pytest from docker.errors import NotFound from .testcases import DockerClientTestCase from compose.config import config +from compose.config import ConfigurationError from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT @@ -104,7 +106,71 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) - def test_net_from_service(self): + @v2_only() + def test_network_mode_from_service(self): + project = Project.from_config( + name='composetest', + client=self.client, + config_data=build_service_dicts({ + 'version': 2, + 'services': { + 'net': { + 'image': 'busybox:latest', + 'command': ["top"] + }, + 'web': { + 'image': 'busybox:latest', + 'network_mode': 'service:net', + 'command': ["top"] + }, + }, + }), + ) + + project.up() + + web = project.get_service('web') + net = project.get_service('net') + self.assertEqual(web.net.mode, 'container:' + net.containers()[0].id) + + @v2_only() + def test_network_mode_from_container(self): + def get_project(): + return Project.from_config( + name='composetest', + config_data=build_service_dicts({ + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox:latest', + 'network_mode': 'container:composetest_net_container' + }, + }, + }), + client=self.client, + ) + + with pytest.raises(ConfigurationError) as excinfo: + get_project() + + assert "container 'composetest_net_container' which does not exist" in excinfo.exconly() + + net_container = Container.create( + self.client, + image='busybox:latest', + name='composetest_net_container', + command='top', + labels={LABEL_PROJECT: 'composetest'}, + ) + net_container.start() + + project = get_project() + project.up() + + web = project.get_service('web') + self.assertEqual(web.net.mode, 'container:' + net_container.id) + + def test_net_from_service_v1(self): project = Project.from_config( name='composetest', config_data=build_service_dicts({ @@ -127,7 +193,24 @@ class ProjectTest(DockerClientTestCase): net = project.get_service('net') self.assertEqual(web.net.mode, 'container:' + net.containers()[0].id) - def test_net_from_container(self): + def test_net_from_container_v1(self): + def get_project(): + return Project.from_config( + name='composetest', + config_data=build_service_dicts({ + 'web': { + 'image': 'busybox:latest', + 'net': 'container:composetest_net_container' + }, + }), + client=self.client, + ) + + with pytest.raises(ConfigurationError) as excinfo: + get_project() + + assert "container 'composetest_net_container' which does not exist" in excinfo.exconly() + net_container = Container.create( self.client, image='busybox:latest', @@ -137,17 +220,7 @@ class ProjectTest(DockerClientTestCase): ) net_container.start() - project = Project.from_config( - name='composetest', - config_data=build_service_dicts({ - 'web': { - 'image': 'busybox:latest', - 'net': 'container:composetest_net_container' - }, - }), - client=self.client, - ) - + project = get_project() project.up() web = project.get_service('web') diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 98fe7758..0d8f7224 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1015,6 +1015,126 @@ class ConfigTest(unittest.TestCase): assert "Service 'one' depends on service 'three'" in exc.exconly() +class NetworkModeTest(unittest.TestCase): + def test_network_mode_standard(self): + config_data = config.load(build_config_details({ + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'bridge', + }, + }, + })) + + assert config_data.services[0]['network_mode'] == 'bridge' + + def test_network_mode_standard_v1(self): + config_data = config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'command': "top", + 'net': 'bridge', + }, + })) + + assert config_data.services[0]['network_mode'] == 'bridge' + assert 'net' not in config_data.services[0] + + def test_network_mode_container(self): + config_data = config.load(build_config_details({ + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'container:foo', + }, + }, + })) + + assert config_data.services[0]['network_mode'] == 'container:foo' + + def test_network_mode_container_v1(self): + config_data = config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'command': "top", + 'net': 'container:foo', + }, + })) + + assert config_data.services[0]['network_mode'] == 'container:foo' + + def test_network_mode_service(self): + config_data = config.load(build_config_details({ + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'service:foo', + }, + 'foo': { + 'image': 'busybox', + 'command': "top", + }, + }, + })) + + assert config_data.services[1]['network_mode'] == 'service:foo' + + def test_network_mode_service_v1(self): + config_data = config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'command': "top", + 'net': 'container:foo', + }, + 'foo': { + 'image': 'busybox', + 'command': "top", + }, + })) + + assert config_data.services[1]['network_mode'] == 'service:foo' + + def test_network_mode_service_nonexistent(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load(build_config_details({ + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'service:foo', + }, + }, + })) + + assert "service 'foo' which is undefined" in excinfo.exconly() + + def test_network_mode_plus_networks_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load(build_config_details({ + 'version': 2, + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'bridge', + 'networks': ['front'], + }, + }, + 'networks': { + 'front': None, + } + })) + + assert "'network_mode' and 'networks' cannot be combined" in excinfo.exconly() + + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ {"1": "8000"}, @@ -1867,11 +1987,18 @@ class ExtendsTest(unittest.TestCase): load_from_filename('tests/fixtures/extends/invalid-volumes.yml') def test_invalid_net_in_extended_service(self): - expected_error_msg = "services with 'net: container' cannot be extended" + with pytest.raises(ConfigurationError) as excinfo: + load_from_filename('tests/fixtures/extends/invalid-net-v2.yml') - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + assert 'network_mode: service' in excinfo.exconly() + assert 'cannot be extended' in excinfo.exconly() + + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-net.yml') + assert 'net: container' in excinfo.exconly() + assert 'cannot be extended' in excinfo.exconly() + @mock.patch.dict(os.environ) def test_load_config_runs_interpolation_in_extended_service(self): os.environ.update(HOSTNAME_VALUE="penguin") diff --git a/tests/unit/config/sort_services_test.py b/tests/unit/config/sort_services_test.py index f5990664..c39ac022 100644 --- a/tests/unit/config/sort_services_test.py +++ b/tests/unit/config/sort_services_test.py @@ -100,7 +100,7 @@ class TestSortService(object): }, { 'name': 'parent', - 'net': 'container:child' + 'network_mode': 'service:child' }, { 'name': 'child' @@ -137,7 +137,7 @@ class TestSortService(object): def test_sort_service_dicts_7(self): services = [ { - 'net': 'container:three', + 'network_mode': 'service:three', 'name': 'four' }, { diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index ffd4455f..3ad131f3 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -365,7 +365,7 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'net': 'container:aaa' + 'network_mode': 'container:aaa' }, ], networks=None, @@ -398,7 +398,7 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'net': 'container:aaa' + 'network_mode': 'service:aaa' }, ], networks=None, From 2b466858556efb990959dce36fd1301e87534700 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 25 Jan 2016 14:26:36 +0000 Subject: [PATCH 309/359] Test that net can be extended Signed-off-by: Aanand Prasad --- tests/fixtures/extends/common.yml | 1 + tests/fixtures/extends/docker-compose.yml | 1 + tests/unit/config/config_test.py | 3 +++ 3 files changed, 5 insertions(+) diff --git a/tests/fixtures/extends/common.yml b/tests/fixtures/extends/common.yml index 358ef5bc..b2d86aa4 100644 --- a/tests/fixtures/extends/common.yml +++ b/tests/fixtures/extends/common.yml @@ -1,6 +1,7 @@ web: image: busybox command: /bin/true + net: host environment: - FOO=1 - BAR=1 diff --git a/tests/fixtures/extends/docker-compose.yml b/tests/fixtures/extends/docker-compose.yml index c51be49e..8e37d404 100644 --- a/tests/fixtures/extends/docker-compose.yml +++ b/tests/fixtures/extends/docker-compose.yml @@ -11,6 +11,7 @@ myweb: BAR: "2" # add BAZ BAZ: "2" + net: bridge mydb: image: busybox command: top diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 0d8f7224..5f8b097b 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1762,6 +1762,7 @@ class ExtendsTest(unittest.TestCase): 'name': 'myweb', 'image': 'busybox', 'command': 'top', + 'network_mode': 'bridge', 'links': ['mydb:db'], 'environment': { "FOO": "1", @@ -1779,6 +1780,7 @@ class ExtendsTest(unittest.TestCase): 'name': 'web', 'image': 'busybox', 'command': '/bin/true', + 'network_mode': 'host', 'environment': { "FOO": "2", "BAR": "1", @@ -1797,6 +1799,7 @@ class ExtendsTest(unittest.TestCase): 'name': 'myweb', 'image': 'busybox', 'command': '/bin/true', + 'network_mode': 'host', 'environment': { "FOO": "2", "BAR": "2", From 52e74ab7ade1dd70c98480d28bc05cd6fccc5ffc Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Mon, 25 Jan 2016 14:27:12 +0000 Subject: [PATCH 310/359] Rename 'net' to 'network mode' in various classes/methods Signed-off-by: Aanand Prasad --- compose/config/config.py | 10 +++--- compose/config/sort_services.py | 18 +++++------ compose/config/validation.py | 4 +-- compose/project.py | 34 ++++++++++---------- compose/service.py | 23 +++++++------- tests/integration/project_test.py | 8 ++--- tests/integration/service_test.py | 8 ++--- tests/unit/project_test.py | 6 ++-- tests/unit/service_test.py | 52 +++++++++++++++---------------- 9 files changed, 81 insertions(+), 82 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index c1391c2f..ffd805ad 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -19,8 +19,8 @@ from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError from .interpolation import interpolate_environment_variables -from .sort_services import get_container_name_from_net -from .sort_services import get_service_name_from_net +from .sort_services import get_container_name_from_network_mode +from .sort_services import get_service_name_from_network_mode from .sort_services import sort_service_dicts from .types import parse_extra_hosts from .types import parse_restart_spec @@ -492,12 +492,12 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'volumes_from' cannot be extended" % error_prefix) if 'net' in service_dict: - if get_container_name_from_net(service_dict['net']): + if get_container_name_from_network_mode(service_dict['net']): raise ConfigurationError( "%s services with 'net: container' cannot be extended" % error_prefix) if 'network_mode' in service_dict: - if get_service_name_from_net(service_dict['network_mode']): + if get_service_name_from_network_mode(service_dict['network_mode']): raise ConfigurationError( "%s services with 'network_mode: service' cannot be extended" % error_prefix) @@ -575,7 +575,7 @@ def finalize_service(service_config, service_names, version): if 'net' in service_dict: network_mode = service_dict.pop('net') - container_name = get_container_name_from_net(network_mode) + container_name = get_container_name_from_network_mode(network_mode) if container_name and container_name in service_names: service_dict['network_mode'] = 'service:{}'.format(container_name) else: diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py index cf38a603..9d29f329 100644 --- a/compose/config/sort_services.py +++ b/compose/config/sort_services.py @@ -4,22 +4,22 @@ from __future__ import unicode_literals from compose.config.errors import DependencyError -def get_service_name_from_net(net_config): - return get_source_name_from_net(net_config, 'service') +def get_service_name_from_network_mode(network_mode): + return get_source_name_from_network_mode(network_mode, 'service') -def get_container_name_from_net(net_config): - return get_source_name_from_net(net_config, 'container') +def get_container_name_from_network_mode(network_mode): + return get_source_name_from_network_mode(network_mode, 'container') -def get_source_name_from_net(net_config, source_type): - if not net_config: +def get_source_name_from_network_mode(network_mode, source_type): + if not network_mode: return - if not net_config.startswith(source_type+':'): + if not network_mode.startswith(source_type+':'): return - _, net_name = net_config.split(':', 1) + _, net_name = network_mode.split(':', 1) return net_name @@ -41,7 +41,7 @@ def sort_service_dicts(services): service for service in services if (name in get_service_names(service.get('links', [])) or name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or - name == get_service_name_from_net(service.get('network_mode')) or + name == get_service_name_from_network_mode(service.get('network_mode')) or name in service.get('depends_on', [])) ] diff --git a/compose/config/validation.py b/compose/config/validation.py index dfc34d57..05982020 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -15,7 +15,7 @@ from jsonschema import RefResolver from jsonschema import ValidationError from .errors import ConfigurationError -from .sort_services import get_service_name_from_net +from .sort_services import get_service_name_from_network_mode log = logging.getLogger(__name__) @@ -156,7 +156,7 @@ def validate_network_mode(service_config, service_names): if 'networks' in service_config.config: raise ConfigurationError("'network_mode' and 'networks' cannot be combined") - dependency = get_service_name_from_net(network_mode) + dependency = get_service_name_from_network_mode(network_mode) if not dependency: return diff --git a/compose/project.py b/compose/project.py index ef913d63..e5b6faef 100644 --- a/compose/project.py +++ b/compose/project.py @@ -10,8 +10,8 @@ from docker.errors import NotFound from . import parallel from .config import ConfigurationError -from .config.sort_services import get_container_name_from_net -from .config.sort_services import get_service_name_from_net +from .config.sort_services import get_container_name_from_network_mode +from .config.sort_services import get_service_name_from_network_mode from .const import DEFAULT_TIMEOUT from .const import IMAGE_EVENTS from .const import LABEL_ONE_OFF @@ -19,11 +19,11 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .container import Container from .network import Network -from .service import ContainerNet +from .service import ContainerNetworkMode from .service import ConvergenceStrategy -from .service import Net +from .service import NetworkMode from .service import Service -from .service import ServiceNet +from .service import ServiceNetworkMode from .utils import microseconds_from_time_nano from .volume import Volume @@ -91,7 +91,7 @@ class Project(object): networks = [] links = project.get_links(service_dict) - net = project.get_net(service_dict, networks) + network_mode = project.get_network_mode(service_dict, networks) volumes_from = get_volumes_from(project, service_dict) if config_data.version == 2: @@ -110,7 +110,7 @@ class Project(object): use_networking=use_networking, networks=networks, links=links, - net=net, + network_mode=network_mode, volumes_from=volumes_from, **service_dict) ) @@ -197,27 +197,27 @@ class Project(object): del service_dict['links'] return links - def get_net(self, service_dict, networks): - net = service_dict.pop('network_mode', None) - if not net: + def get_network_mode(self, service_dict, networks): + network_mode = service_dict.pop('network_mode', None) + if not network_mode: if self.use_networking: - return Net(networks[0]) if networks else Net('none') - return Net(None) + return NetworkMode(networks[0]) if networks else NetworkMode('none') + return NetworkMode(None) - service_name = get_service_name_from_net(net) + service_name = get_service_name_from_network_mode(network_mode) if service_name: - return ServiceNet(self.get_service(service_name)) + return ServiceNetworkMode(self.get_service(service_name)) - container_name = get_container_name_from_net(net) + container_name = get_container_name_from_network_mode(network_mode) if container_name: try: - return ContainerNet(Container.from_id(self.client, container_name)) + return ContainerNetworkMode(Container.from_id(self.client, container_name)) except APIError: raise ConfigurationError( "Service '{name}' uses the network stack of container '{dep}' which " "does not exist.".format(name=service_dict['name'], dep=container_name)) - return Net(net) + return NetworkMode(network_mode) def start(self, service_names=None, **options): containers = [] diff --git a/compose/service.py b/compose/service.py index 166fb0b2..106d5b26 100644 --- a/compose/service.py +++ b/compose/service.py @@ -47,7 +47,6 @@ DOCKER_START_KEYS = [ 'extra_hosts', 'ipc', 'read_only', - 'net', 'log_driver', 'log_opt', 'mem_limit', @@ -113,7 +112,7 @@ class Service(object): use_networking=False, links=None, volumes_from=None, - net=None, + network_mode=None, networks=None, **options ): @@ -123,7 +122,7 @@ class Service(object): self.use_networking = use_networking self.links = links or [] self.volumes_from = volumes_from or [] - self.net = net or Net(None) + self.network_mode = network_mode or NetworkMode(None) self.networks = networks or [] self.options = options @@ -472,7 +471,7 @@ class Service(object): 'options': self.options, 'image_id': self.image()['Id'], 'links': self.get_link_names(), - 'net': self.net.id, + 'net': self.network_mode.id, 'volumes_from': [ (v.source.name, v.mode) for v in self.volumes_from if isinstance(v.source, Service) @@ -480,7 +479,7 @@ class Service(object): } def get_dependency_names(self): - net_name = self.net.service_name + net_name = self.network_mode.service_name return (self.get_linked_service_names() + self.get_volumes_from_names() + ([net_name] if net_name else []) + @@ -636,7 +635,7 @@ class Service(object): binds=options.get('binds'), volumes_from=self._get_volumes_from(), privileged=options.get('privileged', False), - network_mode=self.net.mode, + network_mode=self.network_mode.mode, devices=options.get('devices'), dns=options.get('dns'), dns_search=options.get('dns_search'), @@ -774,22 +773,22 @@ class Service(object): log.error(six.text_type(e)) -class Net(object): +class NetworkMode(object): """A `standard` network mode (ex: host, bridge)""" service_name = None - def __init__(self, net): - self.net = net + def __init__(self, network_mode): + self.network_mode = network_mode @property def id(self): - return self.net + return self.network_mode mode = id -class ContainerNet(object): +class ContainerNetworkMode(object): """A network mode that uses a container's network stack.""" service_name = None @@ -806,7 +805,7 @@ class ContainerNet(object): return 'container:' + self.container.id -class ServiceNet(object): +class ServiceNetworkMode(object): """A network mode that uses a service's network stack.""" def __init__(self, service): diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 0945ebb8..0c8c9a6a 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -131,7 +131,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') net = project.get_service('net') - self.assertEqual(web.net.mode, 'container:' + net.containers()[0].id) + self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) @v2_only() def test_network_mode_from_container(self): @@ -168,7 +168,7 @@ class ProjectTest(DockerClientTestCase): project.up() web = project.get_service('web') - self.assertEqual(web.net.mode, 'container:' + net_container.id) + self.assertEqual(web.network_mode.mode, 'container:' + net_container.id) def test_net_from_service_v1(self): project = Project.from_config( @@ -191,7 +191,7 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') net = project.get_service('net') - self.assertEqual(web.net.mode, 'container:' + net.containers()[0].id) + self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) def test_net_from_container_v1(self): def get_project(): @@ -224,7 +224,7 @@ class ProjectTest(DockerClientTestCase): project.up() web = project.get_service('web') - self.assertEqual(web.net.mode, 'container:' + net_container.id) + self.assertEqual(web.network_mode.mode, 'container:' + net_container.id) def test_start_pause_unpause_stop_kill_remove(self): web = self.create_service('web') diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 379e51ea..cde50b10 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -26,7 +26,7 @@ from compose.const import LABEL_VERSION from compose.container import Container from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy -from compose.service import Net +from compose.service import NetworkMode from compose.service import Service @@ -752,17 +752,17 @@ class ServiceTest(DockerClientTestCase): assert len(service.containers(stopped=True)) == 2 def test_network_mode_none(self): - service = self.create_service('web', net=Net('none')) + service = self.create_service('web', network_mode=NetworkMode('none')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'none') def test_network_mode_bridged(self): - service = self.create_service('web', net=Net('bridge')) + service = self.create_service('web', network_mode=NetworkMode('bridge')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'bridge') def test_network_mode_host(self): - service = self.create_service('web', net=Net('host')) + service = self.create_service('web', network_mode=NetworkMode('host')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'host') diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 3ad131f3..21c6be47 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -349,7 +349,7 @@ class ProjectTest(unittest.TestCase): ), ) service = project.get_service('test') - self.assertEqual(service.net.id, None) + self.assertEqual(service.network_mode.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) def test_use_net_from_container(self): @@ -373,7 +373,7 @@ class ProjectTest(unittest.TestCase): ), ) service = project.get_service('test') - self.assertEqual(service.net.mode, 'container:' + container_id) + self.assertEqual(service.network_mode.mode, 'container:' + container_id) def test_use_net_from_service(self): container_name = 'test_aaa_1' @@ -407,7 +407,7 @@ class ProjectTest(unittest.TestCase): ) service = project.get_service('test') - self.assertEqual(service.net.mode, 'container:' + container_name) + self.assertEqual(service.network_mode.mode, 'container:' + container_name) def test_uses_default_network_true(self): project = Project.from_config( diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 4d9aec65..74e9f0f5 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -15,16 +15,16 @@ from compose.const import LABEL_SERVICE from compose.container import Container from compose.service import build_ulimits from compose.service import build_volume_binding -from compose.service import ContainerNet +from compose.service import ContainerNetworkMode from compose.service import get_container_data_volumes from compose.service import ImageType from compose.service import merge_volume_bindings from compose.service import NeedsBuildError -from compose.service import Net +from compose.service import NetworkMode from compose.service import NoSuchImageError from compose.service import parse_repository_tag from compose.service import Service -from compose.service import ServiceNet +from compose.service import ServiceNetworkMode from compose.service import warn_on_masked_volume @@ -407,7 +407,7 @@ class ServiceTest(unittest.TestCase): 'foo', image='example.com/foo', client=self.mock_client, - net=ServiceNet(Service('other')), + network_mode=ServiceNetworkMode(Service('other')), links=[(Service('one'), 'one')], volumes_from=[VolumeFromSpec(Service('two'), 'rw', 'service')]) @@ -421,7 +421,7 @@ class ServiceTest(unittest.TestCase): } self.assertEqual(config_dict, expected) - def test_config_dict_with_net_from_container(self): + def test_config_dict_with_network_mode_from_container(self): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} container = Container( self.mock_client, @@ -430,7 +430,7 @@ class ServiceTest(unittest.TestCase): 'foo', image='example.com/foo', client=self.mock_client, - net=container) + network_mode=ContainerNetworkMode(container)) config_dict = service.config_dict() expected = { @@ -589,20 +589,20 @@ class BuildUlimitsTestCase(unittest.TestCase): class NetTestCase(unittest.TestCase): - def test_net(self): - net = Net('host') - self.assertEqual(net.id, 'host') - self.assertEqual(net.mode, 'host') - self.assertEqual(net.service_name, None) + def test_network_mode(self): + network_mode = NetworkMode('host') + self.assertEqual(network_mode.id, 'host') + self.assertEqual(network_mode.mode, 'host') + self.assertEqual(network_mode.service_name, None) - def test_net_container(self): + def test_network_mode_container(self): container_id = 'abcd' - net = ContainerNet(Container(None, {'Id': container_id})) - self.assertEqual(net.id, container_id) - self.assertEqual(net.mode, 'container:' + container_id) - self.assertEqual(net.service_name, None) + network_mode = ContainerNetworkMode(Container(None, {'Id': container_id})) + self.assertEqual(network_mode.id, container_id) + self.assertEqual(network_mode.mode, 'container:' + container_id) + self.assertEqual(network_mode.service_name, None) - def test_net_service(self): + def test_network_mode_service(self): container_id = 'bbbb' service_name = 'web' mock_client = mock.create_autospec(docker.Client) @@ -611,23 +611,23 @@ class NetTestCase(unittest.TestCase): ] service = Service(name=service_name, client=mock_client) - net = ServiceNet(service) + network_mode = ServiceNetworkMode(service) - self.assertEqual(net.id, service_name) - self.assertEqual(net.mode, 'container:' + container_id) - self.assertEqual(net.service_name, service_name) + self.assertEqual(network_mode.id, service_name) + self.assertEqual(network_mode.mode, 'container:' + container_id) + self.assertEqual(network_mode.service_name, service_name) - def test_net_service_no_containers(self): + def test_network_mode_service_no_containers(self): service_name = 'web' mock_client = mock.create_autospec(docker.Client) mock_client.containers.return_value = [] service = Service(name=service_name, client=mock_client) - net = ServiceNet(service) + network_mode = ServiceNetworkMode(service) - self.assertEqual(net.id, service_name) - self.assertEqual(net.mode, None) - self.assertEqual(net.service_name, service_name) + self.assertEqual(network_mode.id, service_name) + self.assertEqual(network_mode.mode, None) + self.assertEqual(network_mode.service_name, service_name) def build_mount(destination, source, mode='rw'): From c0fe5459472b0a1c770cf0704cc58c1a1c4b2271 Mon Sep 17 00:00:00 2001 From: Tobias Munk Date: Mon, 25 Jan 2016 19:04:03 +0100 Subject: [PATCH 311/359] fixed documentation about traversing yml files Signed-off-by: Tobias Munk --- docs/reference/docker-compose.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index c19e4284..44a466e3 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -85,12 +85,12 @@ stdin. When stdin is used all paths in the configuration are relative to the current working directory. The `-f` flag is optional. If you don't provide this flag on the command line, -Compose traverses the working directory and its subdirectories looking for a +Compose traverses the working directory and its parent directories looking for a `docker-compose.yml` and a `docker-compose.override.yml` file. You must -supply at least the `docker-compose.yml` file. If both files are present, -Compose combines the two files into a single configuration. The configuration -in the `docker-compose.override.yml` file is applied over and in addition to -the values in the `docker-compose.yml` file. +supply at least the `docker-compose.yml` file. If both files are present on the +same directory level, Compose combines the two files into a single configuration. +The configuration in the `docker-compose.override.yml` file is applied over and +in addition to the values in the `docker-compose.yml` file. See also the `COMPOSE_FILE` [environment variable](overview.md#compose-file). From 7f4a94514bb48b18fdc5dbf1c282d094a097ce36 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 23:22:21 +0000 Subject: [PATCH 312/359] Fix trailing whitespace in docker-compose.md Signed-off-by: Aanand Prasad --- docs/reference/docker-compose.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md index 44a466e3..6e7e4901 100644 --- a/docs/reference/docker-compose.md +++ b/docs/reference/docker-compose.md @@ -87,9 +87,9 @@ relative to the current working directory. The `-f` flag is optional. If you don't provide this flag on the command line, Compose traverses the working directory and its parent directories looking for a `docker-compose.yml` and a `docker-compose.override.yml` file. You must -supply at least the `docker-compose.yml` file. If both files are present on the -same directory level, Compose combines the two files into a single configuration. -The configuration in the `docker-compose.override.yml` file is applied over and +supply at least the `docker-compose.yml` file. If both files are present on the +same directory level, Compose combines the two files into a single configuration. +The configuration in the `docker-compose.override.yml` file is applied over and in addition to the values in the `docker-compose.yml` file. See also the `COMPOSE_FILE` [environment variable](overview.md#compose-file). From d765a3fb912f174d5b11b0b3f69d9a5caaa50c03 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 18:48:43 +0000 Subject: [PATCH 313/359] Add back external links in v2 Signed-off-by: Aanand Prasad --- compose/config/service_schema_v2.json | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.json index 56c0cbf5..ca9bb671 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.json @@ -83,6 +83,7 @@ ] }, + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, "hostname": {"type": "string"}, "image": {"type": "string"}, From 16ef3d0eb882cfb96072ff92d13f124d6b5695df Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 26 Jan 2016 11:12:35 -0800 Subject: [PATCH 314/359] Bump docker-py version to latest RC Signed-off-by: Joffrey F --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index ed3b8686..68ef9f35 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ PyYAML==3.11 cached-property==1.2.0 -docker-py==1.7.0rc2 +docker-py==1.7.0rc3 docopt==0.6.1 enum34==1.0.4 git+https://github.com/d11wtq/dockerpty.git@29b1394108b017ef3e3deaf00604a9eb99880d5e#egg=dockerpty From 297d20f085ec6ef6858212cf09eea2d7e51e7a43 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 Jan 2016 00:42:04 +0000 Subject: [PATCH 315/359] Remove ability to join bridge network + user-defined networks Containers connected to the bridge network can't have aliases, so it's simpler to rule that they can *either* be connected to the bridge network (via `network_mode: bridge`) *or* be connected to user-defined networks (via `networks` or the default network). Signed-off-by: Aanand Prasad --- compose/project.py | 15 ++++++--------- docs/compose-file.md | 5 ----- tests/acceptance/cli_test.py | 16 ---------------- 3 files changed, 6 insertions(+), 30 deletions(-) diff --git a/compose/project.py b/compose/project.py index e5b6faef..d2787ecf 100644 --- a/compose/project.py +++ b/compose/project.py @@ -470,16 +470,13 @@ def get_networks(service_dict, network_definitions): networks = [] for name in service_dict.pop('networks', ['default']): - if name in ['bridge']: - networks.append(name) + matches = [n for n in network_definitions if n.name == name] + if matches: + networks.append(matches[0].full_name) else: - matches = [n for n in network_definitions if n.name == name] - if matches: - networks.append(matches[0].full_name) - else: - raise ConfigurationError( - 'Service "{}" uses an undefined network "{}"' - .format(service_dict['name'], name)) + raise ConfigurationError( + 'Service "{}" uses an undefined network "{}"' + .format(service_dict['name'], name)) return networks diff --git a/docs/compose-file.md b/docs/compose-file.md index 6b61755f..afef0b1a 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -472,11 +472,6 @@ Networks to join, referencing entries under the - some-network - other-network -The value `bridge` can also be used to make containers join the pre-defined -`bridge` network. - -There is no equivalent to `net: "container:[name or id]"`. - ### pid pid: "host" diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 4b560efa..30589bad 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -495,22 +495,6 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr - @v2_only() - def test_up_with_bridge_network_plus_default(self): - filename = 'bridge.yml' - - self.base_dir = 'tests/fixtures/networks' - self._project = get_project(self.base_dir, [filename]) - - self.dispatch(['-f', filename, 'up', '-d'], None) - - container = self.project.containers()[0] - - assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted([ - 'bridge', - self.project.default_network.full_name, - ]) - @v2_only() def test_up_with_network_mode(self): c = self.client.create_container('busybox', 'top', name='composetest_network_mode_container') From 695c692be6347d47669270958bb0bbe1c383773a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 11:09:28 +0000 Subject: [PATCH 316/359] Bump 1.6.0-rc2 Signed-off-by: Aanand Prasad --- CHANGELOG.md | 15 ++++++++++++--- compose/__init__.py | 2 +- docs/install.md | 6 +++--- script/run.sh | 2 +- 4 files changed, 17 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0251d669..d115f05d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -19,9 +19,9 @@ Major Features: - Support for networking has exited experimental status and is the recommended way to enable communication between containers. - If you use the new file format, your app will use networking. If you want to - keep using links, just leave your Compose file as it is and it'll continue - to work just the same. + If you use the new file format, your app will use networking. If you aren't + ready yet, just leave your Compose file as it is and it'll continue to work + just the same. By default, you don't have to configure any networks. In fact, using networking with Compose involves even less configuration than using links. @@ -48,6 +48,10 @@ Major Features: building tools on top of Compose for performing actions when particular things happen, such as containers starting and stopping. +- There's a new `depends_on` option for specifying dependencies between + services. This enforces the order of startup, and ensures that when you run + `docker-compose up SERVICE` on a service with dependencies, those are started + as well. New Features: @@ -108,6 +112,11 @@ Bug Fixes: - Fixed a bug where a container would be reported as "Up" when it was in the restarting state. +- Fixed a confusing error message when DOCKER_CERT_PATH was not set properly. + +- Fixed a bug where attaching to a container would fail if it was using a + non-standard logging driver (or none at all). + 1.5.2 (2015-12-03) ------------------ diff --git a/compose/__init__.py b/compose/__init__.py index 52d0e7bf..3c52ff7d 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.6.0rc1' +__version__ = '1.6.0rc2' diff --git a/docs/install.md b/docs/install.md index 944e9190..c1dd9a49 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,7 +39,7 @@ which the release page specifies, in your terminal. The following is an example command illustrating the format: - curl -L https://github.com/docker/compose/releases/download/1.6.0rc1/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.6.0rc2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose If you have problems installing with `curl`, see [Alternative Install Options](#alternative-install-options). @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.6.0rc1 + docker-compose version: 1.6.0rc2 ## Alternative install options @@ -77,7 +77,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.6.0rc1/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.6.0rc2/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 3fbc60e0..89655053 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.6.0rc1" +VERSION="1.6.0rc2" IMAGE="docker/compose:$VERSION" From 4537ec70cc0823e79c132b59c9767950b3d26a4f Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 Jan 2016 02:34:59 +0000 Subject: [PATCH 317/359] Remove outdated warnings about links from docs Signed-off-by: Aanand Prasad --- docs/compose-file.md | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index afef0b1a..28fedf42 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -808,7 +808,7 @@ future Compose release. Version 1 files cannot declare named [volumes](#volume-configuration-reference), [networks](networking.md) or -[build arguments](#args). They *can*, however, define [links](#links). +[build arguments](#args). Example: @@ -837,10 +837,6 @@ Named [volumes](#volume-configuration-reference) can be declared under the `volumes` key, and [networks](#network-configuration-reference) can be declared under the `networks` key. -You cannot define links when using version 2. Instead, you should use -[networking](networking.md) for communication between containers. In most cases, -this will involve less configuration than links. - Simple example: version: 2 From 9249ec62c2304620433eb3a5ca4ae6a4dd44b5b8 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 15:51:09 +0000 Subject: [PATCH 318/359] Add note about named volumes to upgrade guide Signed-off-by: Aanand Prasad --- docs/compose-file.md | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/docs/compose-file.md b/docs/compose-file.md index 28fedf42..5b72cc28 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -941,6 +941,27 @@ It's more complicated if you're using particular configuration features: net: "container:cont-name" -> network_mode: "container:cont-name" net: "container:abc12345" -> network_mode: "container:abc12345" +- `volumes` with named volumes: these must now be explicitly declared in a + top-level `volumes` section of your Compose file. If a service mounts a + named volume called `data`, you must declare a `data` volume in your + top-level `volumes` section. The whole file might look like this: + + version: 2 + services: + db: + image: postgres + volumes: + - data:/var/lib/postgresql/data + volumes: + data: {} + + By default, Compose creates a volume whose name is prefixed with your + project name. If you want it to just be called `data`, declared it as + external: + + volumes: + data: + external: true ## Variable substitution From 36a10f8dd523cc97fdd66d4e4a48556e68c51eca Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 15:53:02 +0000 Subject: [PATCH 319/359] Update for links, external_links, network_mode Signed-off-by: Aanand Prasad --- .../migrate-compose-file-v1-to-v2.py | 43 +++++++++++++++---- 1 file changed, 34 insertions(+), 9 deletions(-) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py index a961b0bd..011ce338 100755 --- a/contrib/migration/migrate-compose-file-v1-to-v2.py +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -20,20 +20,45 @@ def migrate(content): data = ruamel.yaml.load(content, ruamel.yaml.RoundTripLoader) service_names = data.keys() + for name, service in data.items(): - # remove links and external links - service.pop('links', None) - external_links = service.pop('external_links', None) + links = service.get('links') + if links: + example_service = links[0].partition(':')[0] + log.warn( + "Service {name} has links, which no longer create environment " + "variables such as {example_service_upper}_PORT. " + "If you are using those in your application code, you should " + "instead connect directly to the hostname, e.g. " + "'{example_service}'." + .format(name=name, example_service=example_service, + example_service_upper=example_service.upper())) + + external_links = service.get('external_links') if external_links: log.warn( - "Service {name} has external_links: {ext}, which are no longer " - "supported. See https://docs.docker.com/compose/networking/ " - "for options on how to connect external containers to the " - "compose network.".format(name=name, ext=external_links)) + "Service {name} has external_links: {ext}, which now work " + "slightly differently. In particular, two containers must be " + "connected to at least one network in common in order to " + "communicate, even if explicitly linked together.\n\n" + "Either connect the external container to your app's default " + "network, or connect both the external container and your " + "service's containers to a pre-existing network. See " + "https://docs.docker.com/compose/networking/ " + "for more on how to do this." + .format(name=name, ext=external_links)) - # net is now networks + # net is now network_mode if 'net' in service: - service['networks'] = [service.pop('net')] + network_mode = service.pop('net') + + # "container:" is now "service:" + if network_mode.startswith('container:'): + name = network_mode.partition(':')[2] + if name in service_names: + network_mode = 'service:{}'.format(name) + + service['network_mode'] = network_mode # create build section if 'dockerfile' in service: From 86bdab64abe6cea20d9b1ced97b3326f41f20f74 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 15:53:25 +0000 Subject: [PATCH 320/359] Make warnings a bit more readable Signed-off-by: Aanand Prasad --- contrib/migration/migrate-compose-file-v1-to-v2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py index 011ce338..e156d9e1 100755 --- a/contrib/migration/migrate-compose-file-v1-to-v2.py +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -106,7 +106,7 @@ def parse_opts(args): def main(args): - logging.basicConfig() + logging.basicConfig(format='\033[33m%(levelname)s:\033[37m %(message)s\n') opts = parse_opts(args) From 1772909fe2a81adc818ed2e31229566917b48359 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 15:54:18 +0000 Subject: [PATCH 321/359] Make sure version line is at the top of the file Signed-off-by: Aanand Prasad --- contrib/migration/migrate-compose-file-v1-to-v2.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py index e156d9e1..fb73811c 100755 --- a/contrib/migration/migrate-compose-file-v1-to-v2.py +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -78,8 +78,11 @@ def migrate(content): if volume_from.split(':', 1)[0] not in service_names: service['volumes_from'][idx] = 'container:%s' % volume_from - data['services'] = {name: data.pop(name) for name in data.keys()} + services = {name: data.pop(name) for name in data.keys()} + data['version'] = 2 + data['services'] = services + return data From be66779fe9ef3265e0453dba692f26b08cef2ded Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 16:01:43 +0000 Subject: [PATCH 322/359] Create declarations for named volumes Signed-off-by: Aanand Prasad --- .../migrate-compose-file-v1-to-v2.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py index fb73811c..23f6be3b 100755 --- a/contrib/migration/migrate-compose-file-v1-to-v2.py +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -12,6 +12,8 @@ import sys import ruamel.yaml +from compose.config.types import VolumeSpec + log = logging.getLogger('migrate') @@ -82,10 +84,39 @@ def migrate(content): data['version'] = 2 data['services'] = services + create_volumes_section(data) return data +def create_volumes_section(data): + named_volumes = get_named_volumes(data['services']) + if named_volumes: + log.warn( + "Named volumes ({names}) must be explicitly declared. Creating a " + "'volumes' section with declarations.\n\n" + "For backwards-compatibility, they've been declared as external. " + "If you don't mind the volume names being prefixed with the " + "project name, you can remove the 'external' option from each one." + .format(names=', '.join(list(named_volumes)))) + + data['volumes'] = named_volumes + + +def get_named_volumes(services): + volume_specs = [ + VolumeSpec.parse(volume) + for service in services.values() + for volume in service.get('volumes', []) + ] + names = { + spec.external + for spec in volume_specs + if spec.is_named_volume + } + return {name: {'external': True} for name in names} + + def write(stream, new_format, indent, width): ruamel.yaml.dump( new_format, From 89cca7bcb2acbf994c3936787927e7d315e5b4c9 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 16:08:34 +0000 Subject: [PATCH 323/359] Extract helper methods Signed-off-by: Aanand Prasad --- .../migrate-compose-file-v1-to-v2.py | 125 ++++++++++-------- 1 file changed, 70 insertions(+), 55 deletions(-) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py index 23f6be3b..4f9be97f 100755 --- a/contrib/migration/migrate-compose-file-v1-to-v2.py +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -24,61 +24,12 @@ def migrate(content): service_names = data.keys() for name, service in data.items(): - links = service.get('links') - if links: - example_service = links[0].partition(':')[0] - log.warn( - "Service {name} has links, which no longer create environment " - "variables such as {example_service_upper}_PORT. " - "If you are using those in your application code, you should " - "instead connect directly to the hostname, e.g. " - "'{example_service}'." - .format(name=name, example_service=example_service, - example_service_upper=example_service.upper())) - - external_links = service.get('external_links') - if external_links: - log.warn( - "Service {name} has external_links: {ext}, which now work " - "slightly differently. In particular, two containers must be " - "connected to at least one network in common in order to " - "communicate, even if explicitly linked together.\n\n" - "Either connect the external container to your app's default " - "network, or connect both the external container and your " - "service's containers to a pre-existing network. See " - "https://docs.docker.com/compose/networking/ " - "for more on how to do this." - .format(name=name, ext=external_links)) - - # net is now network_mode - if 'net' in service: - network_mode = service.pop('net') - - # "container:" is now "service:" - if network_mode.startswith('container:'): - name = network_mode.partition(':')[2] - if name in service_names: - network_mode = 'service:{}'.format(name) - - service['network_mode'] = network_mode - - # create build section - if 'dockerfile' in service: - service['build'] = { - 'context': service.pop('build'), - 'dockerfile': service.pop('dockerfile'), - } - - # create logging section - if 'log_driver' in service: - service['logging'] = {'driver': service.pop('log_driver')} - if 'log_opt' in service: - service['logging']['options'] = service.pop('log_opt') - - # volumes_from prefix with 'container:' - for idx, volume_from in enumerate(service.get('volumes_from', [])): - if volume_from.split(':', 1)[0] not in service_names: - service['volumes_from'][idx] = 'container:%s' % volume_from + warn_for_links(name, service) + warn_for_external_links(name, service) + rewrite_net(service, service_names) + rewrite_build(service) + rewrite_logging(service) + rewrite_volumes_from(service, service_names) services = {name: data.pop(name) for name in data.keys()} @@ -89,6 +40,70 @@ def migrate(content): return data +def warn_for_links(name, service): + links = service.get('links') + if links: + example_service = links[0].partition(':')[0] + log.warn( + "Service {name} has links, which no longer create environment " + "variables such as {example_service_upper}_PORT. " + "If you are using those in your application code, you should " + "instead connect directly to the hostname, e.g. " + "'{example_service}'." + .format(name=name, example_service=example_service, + example_service_upper=example_service.upper())) + + +def warn_for_external_links(name, service): + external_links = service.get('external_links') + if external_links: + log.warn( + "Service {name} has external_links: {ext}, which now work " + "slightly differently. In particular, two containers must be " + "connected to at least one network in common in order to " + "communicate, even if explicitly linked together.\n\n" + "Either connect the external container to your app's default " + "network, or connect both the external container and your " + "service's containers to a pre-existing network. See " + "https://docs.docker.com/compose/networking/ " + "for more on how to do this." + .format(name=name, ext=external_links)) + + +def rewrite_net(service, service_names): + if 'net' in service: + network_mode = service.pop('net') + + # "container:" is now "service:" + if network_mode.startswith('container:'): + name = network_mode.partition(':')[2] + if name in service_names: + network_mode = 'service:{}'.format(name) + + service['network_mode'] = network_mode + + +def rewrite_build(service): + if 'dockerfile' in service: + service['build'] = { + 'context': service.pop('build'), + 'dockerfile': service.pop('dockerfile'), + } + + +def rewrite_logging(service): + if 'log_driver' in service: + service['logging'] = {'driver': service.pop('log_driver')} + if 'log_opt' in service: + service['logging']['options'] = service.pop('log_opt') + + +def rewrite_volumes_from(service, service_names): + for idx, volume_from in enumerate(service.get('volumes_from', [])): + if volume_from.split(':', 1)[0] not in service_names: + service['volumes_from'][idx] = 'container:%s' % volume_from + + def create_volumes_section(data): named_volumes = get_named_volumes(data['services']) if named_volumes: From fbe8484377888f305a50ffdd09c9b62381ebc36f Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Sun, 24 Jan 2016 12:03:44 -0800 Subject: [PATCH 324/359] New navigation for 1.10 release Updating with Joffrey's comments Signed-off-by: Mary Anthony --- docs/README.md | 4 +- docs/completion.md | 4 +- docs/compose-file.md | 3 +- docs/django.md | 6 +- docs/env.md | 6 +- docs/extends.md | 2 +- docs/faq.md | 4 +- docs/gettingstarted.md | 4 +- docs/index.md | 175 ++---------------------------------- docs/install.md | 6 +- docs/networking.md | 3 +- docs/overview.md | 191 ++++++++++++++++++++++++++++++++++++++++ docs/production.md | 2 +- docs/rails.md | 6 +- docs/reference/index.md | 7 +- docs/wordpress.md | 6 +- 16 files changed, 230 insertions(+), 199 deletions(-) create mode 100644 docs/overview.md diff --git a/docs/README.md b/docs/README.md index d8ab7c3e..e60fa48c 100644 --- a/docs/README.md +++ b/docs/README.md @@ -58,7 +58,7 @@ The top of each Docker Compose documentation file contains TOML metadata. The me description = "How to use Docker Compose's extends keyword to share configuration between files and projects" keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] [menu.main] - parent="smn_workw_compose" + parent="workw_compose" weight=2 +++ @@ -70,7 +70,7 @@ The metadata alone has this structure: description = "How to use Docker Compose's extends keyword to share configuration between files and projects" keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] [menu.main] - parent="smn_workw_compose" + parent="workw_compose" weight=2 +++ diff --git a/docs/completion.md b/docs/completion.md index cac0d1a6..2076d512 100644 --- a/docs/completion.md +++ b/docs/completion.md @@ -4,8 +4,8 @@ title = "Command-line Completion" description = "Compose CLI reference" keywords = ["fig, composition, compose, docker, orchestration, cli, reference"] [menu.main] -parent="smn_workw_compose" -weight=10 +parent="workw_compose" +weight=88 +++ diff --git a/docs/compose-file.md b/docs/compose-file.md index 5b72cc28..cc21bd8a 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -5,7 +5,8 @@ description = "Compose file reference" keywords = ["fig, composition, compose, docker"] aliases = ["/compose/yml"] [menu.main] -parent="smn_compose_ref" +parent="workw_compose" +weight=70 +++ diff --git a/docs/django.md b/docs/django.md index 2d4fdaf9..573ea3d9 100644 --- a/docs/django.md +++ b/docs/django.md @@ -1,16 +1,16 @@ -# Quickstart Guide: Compose and Django +# Quickstart: Compose and Django This quick-start guide demonstrates how to use Compose to set up and run a simple Django/PostgreSQL app. Before starting, you'll need to have diff --git a/docs/env.md b/docs/env.md index 7b7e1bc7..c7b93c77 100644 --- a/docs/env.md +++ b/docs/env.md @@ -1,11 +1,11 @@ diff --git a/docs/extends.md b/docs/extends.md index 011a7350..c9b65db8 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -4,7 +4,7 @@ title = "Extending services in Compose" description = "How to use Docker Compose's extends keyword to share configuration between files and projects" keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] [menu.main] -parent="smn_workw_compose" +parent="workw_compose" weight=2 +++ diff --git a/docs/faq.md b/docs/faq.md index b36eb5ac..264a27ee 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -4,8 +4,8 @@ title = "Frequently Asked Questions" description = "Docker Compose FAQ" keywords = "documentation, docs, docker, compose, faq" [menu.main] -parent="smn_workw_compose" -weight=9 +parent="workw_compose" +weight=90 +++ diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md index bb3d51e9..1939500c 100644 --- a/docs/gettingstarted.md +++ b/docs/gettingstarted.md @@ -4,8 +4,8 @@ title = "Getting Started" description = "Getting started with Docker Compose" keywords = ["documentation, docs, docker, compose, orchestration, containers"] [menu.main] -parent="smn_workw_compose" -weight=3 +parent="workw_compose" +weight=-85 +++ diff --git a/docs/index.md b/docs/index.md index 9cb594a7..61bc41ee 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,65 +1,21 @@ -# Overview of Docker Compose +# Docker Compose -Compose is a tool for defining and running multi-container Docker applications. -With Compose, you use a Compose file to configure your application's services. -Then, using a single command, you create and start all the services -from your configuration. To learn more about all the features of Compose -see [the list of features](#features). +Compose is a tool for defining and running multi-container Docker applications. To learn more about Compose refer to the following documentation: -Compose is great for development, testing, and staging environments, as well as -CI workflows. You can learn more about each case in -[Common Use Cases](#common-use-cases). - -Using Compose is basically a three-step process. - -1. Define your app's environment with a `Dockerfile` so it can be -reproduced anywhere. -2. Define the services that make up your app in `docker-compose.yml` so -they can be run together in an isolated environment. -3. Lastly, run `docker-compose up` and Compose will start and run your entire app. - -A `docker-compose.yml` looks like this: - - version: 2 - services: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - - logvolume01:/var/log - links: - - redis - redis: - image: redis - volumes: - logvolume01: {} - -For more information about the Compose file, see the -[Compose file reference](compose-file.md) - -Compose has commands for managing the whole lifecycle of your application: - - * Start, stop and rebuild services - * View the status of running services - * Stream the log output of running services - * Run a one-off command on a service - -## Compose documentation - -- [Installing Compose](install.md) +- [Compose Overview](overview.md) +- [Install Compose](install.md) - [Getting Started](gettingstarted.md) - [Get started with Django](django.md) - [Get started with Rails](rails.md) @@ -68,123 +24,6 @@ Compose has commands for managing the whole lifecycle of your application: - [Command line reference](./reference/index.md) - [Compose file reference](compose-file.md) -## Features - -The features of Compose that make it effective are: - -* [Multiple isolated environments on a single host](#Multiple-isolated-environments-on-a-single-host) -* [Preserve volume data when containers are created](#preserve-volume-data-when-containers-are-created) -* [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) -* [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) - -#### Multiple isolated environments on a single host - -Compose uses a project name to isolate environments from each other. You can use -this project name to: - -* on a dev host, to create multiple copies of a single environment (ex: you want - to run a stable copy for each feature branch of a project) -* on a CI server, to keep builds from interfering with each other, you can set - the project name to a unique build number -* on a shared host or dev host, to prevent different projects which may use the - same service names, from interfering with each other - -The default project name is the basename of the project directory. You can set -a custom project name by using the -[`-p` command line option](./reference/docker-compose.md) or the -[`COMPOSE_PROJECT_NAME` environment variable](./reference/overview.md#compose-project-name). - -#### Preserve volume data when containers are created - -Compose preserves all volumes used by your services. When `docker-compose up` -runs, if it finds any containers from previous runs, it copies the volumes from -the old container to the new container. This process ensures that any data -you've created in volumes isn't lost. - - -#### Only recreate containers that have changed - -Compose caches the configuration used to create a container. When you -restart a service that has not changed, Compose re-uses the existing -containers. Re-using containers means that you can make changes to your -environment very quickly. - - -#### Variables and moving a composition between environments - -Compose supports variables in the Compose file. You can use these variables -to customize your composition for different environments, or different users. -See [Variable substitution](compose-file.md#variable-substitution) for more -details. - -You can extend a Compose file using the `extends` field or by creating multiple -Compose files. See [extends](extends.md) for more details. - - -## Common Use Cases - -Compose can be used in many different ways. Some common use cases are outlined -below. - -### Development environments - -When you're developing software, the ability to run an application in an -isolated environment and interact with it is crucial. The Compose command -line tool can be used to create the environment and interact with it. - -The [Compose file](compose-file.md) provides a way to document and configure -all of the application's service dependencies (databases, queues, caches, -web service APIs, etc). Using the Compose command line tool you can create -and start one or more containers for each dependency with a single command -(`docker-compose up`). - -Together, these features provide a convenient way for developers to get -started on a project. Compose can reduce a multi-page "developer getting -started guide" to a single machine readable Compose file and a few commands. - -### Automated testing environments - -An important part of any Continuous Deployment or Continuous Integration process -is the automated test suite. Automated end-to-end testing requires an -environment in which to run tests. Compose provides a convenient way to create -and destroy isolated testing environments for your test suite. By defining the full -environment in a [Compose file](compose-file.md) you can create and destroy these -environments in just a few commands: - - $ docker-compose up -d - $ ./run_tests - $ docker-compose down - -### Single host deployments - -Compose has traditionally been focused on development and testing workflows, -but with each release we're making progress on more production-oriented features. -You can use Compose to deploy to a remote Docker Engine. The Docker Engine may -be a single instance provisioned with -[Docker Machine](https://docs.docker.com/machine/) or an entire -[Docker Swarm](https://docs.docker.com/swarm/) cluster. - -For details on using production-oriented features, see -[compose in production](production.md) in this documentation. - - -## Release Notes - To see a detailed list of changes for past and current releases of Docker Compose, please refer to the [CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). - -## Getting help - -Docker Compose is under active development. If you need help, would like to -contribute, or simply want to talk about the project with like-minded -individuals, we have a number of open channels for communication. - -* To report bugs or file feature requests: please use the [issue tracker on Github](https://github.com/docker/compose/issues). - -* To talk about the project with people in real time: please join the - `#docker-compose` channel on freenode IRC. - -* To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). - -For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/opensource/get-help/). diff --git a/docs/install.md b/docs/install.md index c1dd9a49..7563db6b 100644 --- a/docs/install.md +++ b/docs/install.md @@ -1,11 +1,11 @@ diff --git a/docs/networking.md b/docs/networking.md index 93533e9d..fb34e3de 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -4,8 +4,7 @@ title = "Networking in Compose" description = "How Compose sets up networking between containers" keywords = ["documentation, docs, docker, compose, orchestration, containers, networking"] [menu.main] -parent="smn_workw_compose" -weight=6 +parent="workw_compose" +++ diff --git a/docs/overview.md b/docs/overview.md new file mode 100644 index 00000000..19f51ab4 --- /dev/null +++ b/docs/overview.md @@ -0,0 +1,191 @@ + + + +# Overview of Docker Compose + +Compose is a tool for defining and running multi-container Docker applications. +With Compose, you use a Compose file to configure your application's services. +Then, using a single command, you create and start all the services +from your configuration. To learn more about all the features of Compose +see [the list of features](#features). + +Compose is great for development, testing, and staging environments, as well as +CI workflows. You can learn more about each case in +[Common Use Cases](#common-use-cases). + +Using Compose is basically a three-step process. + +1. Define your app's environment with a `Dockerfile` so it can be +reproduced anywhere. +2. Define the services that make up your app in `docker-compose.yml` so +they can be run together in an isolated environment. +3. Lastly, run `docker-compose up` and Compose will start and run your entire app. + +A `docker-compose.yml` looks like this: + + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: {} + +For more information about the Compose file, see the +[Compose file reference](compose-file.md) + +Compose has commands for managing the whole lifecycle of your application: + + * Start, stop and rebuild services + * View the status of running services + * Stream the log output of running services + * Run a one-off command on a service + +## Compose documentation + +- [Installing Compose](install.md) +- [Getting Started](gettingstarted.md) +- [Get started with Django](django.md) +- [Get started with Rails](rails.md) +- [Get started with WordPress](wordpress.md) +- [Frequently asked questions](faq.md) +- [Command line reference](./reference/index.md) +- [Compose file reference](compose-file.md) + +## Features + +The features of Compose that make it effective are: + +* [Multiple isolated environments on a single host](#Multiple-isolated-environments-on-a-single-host) +* [Preserve volume data when containers are created](#preserve-volume-data-when-containers-are-created) +* [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) +* [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) + +### Multiple isolated environments on a single host + +Compose uses a project name to isolate environments from each other. You can use +this project name to: + +* on a dev host, to create multiple copies of a single environment (ex: you want + to run a stable copy for each feature branch of a project) +* on a CI server, to keep builds from interfering with each other, you can set + the project name to a unique build number +* on a shared host or dev host, to prevent different projects which may use the + same service names, from interfering with each other + +The default project name is the basename of the project directory. You can set +a custom project name by using the +[`-p` command line option](./reference/docker-compose.md) or the +[`COMPOSE_PROJECT_NAME` environment variable](./reference/overview.md#compose-project-name). + +### Preserve volume data when containers are created + +Compose preserves all volumes used by your services. When `docker-compose up` +runs, if it finds any containers from previous runs, it copies the volumes from +the old container to the new container. This process ensures that any data +you've created in volumes isn't lost. + + +### Only recreate containers that have changed + +Compose caches the configuration used to create a container. When you +restart a service that has not changed, Compose re-uses the existing +containers. Re-using containers means that you can make changes to your +environment very quickly. + + +### Variables and moving a composition between environments + +Compose supports variables in the Compose file. You can use these variables +to customize your composition for different environments, or different users. +See [Variable substitution](compose-file.md#variable-substitution) for more +details. + +You can extend a Compose file using the `extends` field or by creating multiple +Compose files. See [extends](extends.md) for more details. + + +## Common Use Cases + +Compose can be used in many different ways. Some common use cases are outlined +below. + +### Development environments + +When you're developing software, the ability to run an application in an +isolated environment and interact with it is crucial. The Compose command +line tool can be used to create the environment and interact with it. + +The [Compose file](compose-file.md) provides a way to document and configure +all of the application's service dependencies (databases, queues, caches, +web service APIs, etc). Using the Compose command line tool you can create +and start one or more containers for each dependency with a single command +(`docker-compose up`). + +Together, these features provide a convenient way for developers to get +started on a project. Compose can reduce a multi-page "developer getting +started guide" to a single machine readable Compose file and a few commands. + +### Automated testing environments + +An important part of any Continuous Deployment or Continuous Integration process +is the automated test suite. Automated end-to-end testing requires an +environment in which to run tests. Compose provides a convenient way to create +and destroy isolated testing environments for your test suite. By defining the full +environment in a [Compose file](compose-file.md) you can create and destroy these +environments in just a few commands: + + $ docker-compose up -d + $ ./run_tests + $ docker-compose down + +### Single host deployments + +Compose has traditionally been focused on development and testing workflows, +but with each release we're making progress on more production-oriented features. +You can use Compose to deploy to a remote Docker Engine. The Docker Engine may +be a single instance provisioned with +[Docker Machine](https://docs.docker.com/machine/) or an entire +[Docker Swarm](https://docs.docker.com/swarm/) cluster. + +For details on using production-oriented features, see +[compose in production](production.md) in this documentation. + + +## Release Notes + +To see a detailed list of changes for past and current releases of Docker +Compose, please refer to the +[CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). + +## Getting help + +Docker Compose is under active development. If you need help, would like to +contribute, or simply want to talk about the project with like-minded +individuals, we have a number of open channels for communication. + +* To report bugs or file feature requests: please use the [issue tracker on Github](https://github.com/docker/compose/issues). + +* To talk about the project with people in real time: please join the + `#docker-compose` channel on freenode IRC. + +* To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). + +For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/opensource/get-help/). diff --git a/docs/production.md b/docs/production.md index 46e221bb..d51ca549 100644 --- a/docs/production.md +++ b/docs/production.md @@ -4,7 +4,7 @@ title = "Using Compose in production" description = "Guide to using Docker Compose in production" keywords = ["documentation, docs, docker, compose, orchestration, containers, production"] [menu.main] -parent="smn_workw_compose" +parent="workw_compose" weight=1 +++ diff --git a/docs/rails.md b/docs/rails.md index e3daff25..f7634a6d 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -1,15 +1,15 @@ -## Quickstart Guide: Compose and Rails +## Quickstart: Compose and Rails This Quickstart guide will show you how to use Compose to set up and run a Rails/PostgreSQL app. Before starting, you'll need to have [Compose installed](install.md). diff --git a/docs/reference/index.md b/docs/reference/index.md index 5406b9c7..d1bd61c2 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -1,15 +1,16 @@ -## Compose CLI reference +## Compose command-line reference The following pages describe the usage information for the [docker-compose](docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. diff --git a/docs/wordpress.md b/docs/wordpress.md index 84010491..50362253 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -1,16 +1,16 @@ -# Quickstart Guide: Compose and WordPress +# Quickstart: Compose and WordPress You can use Compose to easily run WordPress in an isolated environment built with Docker containers. From 24e71db345e031af1247fd7bafe9b09cb2f5ad73 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Tue, 19 Jan 2016 14:41:21 -0500 Subject: [PATCH 325/359] Don't copy over volumes that were previously host volumes, and are now container volumes. Signed-off-by: Daniel Nephin --- compose/service.py | 4 ++++ tests/integration/service_test.py | 25 +++++++++++++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/compose/service.py b/compose/service.py index 106d5b26..2ca0e64d 100644 --- a/compose/service.py +++ b/compose/service.py @@ -917,6 +917,10 @@ def get_container_data_volumes(container, volumes_option): if not mount: continue + # Volume was previously a host volume, now it's a container volume + if not mount.get('Name'): + continue + # Copy existing volume from old container volume = volume._replace(external=mount['Source']) volumes.append(volume) diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index cde50b10..189eb9da 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -343,6 +343,31 @@ class ServiceTest(DockerClientTestCase): ) self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) + def test_execute_convergence_plan_when_host_volume_is_removed(self): + host_path = '/tmp/host-path' + service = self.create_service( + 'db', + build={'context': 'tests/fixtures/dockerfile-with-volume'}, + volumes=[VolumeSpec(host_path, '/data', 'rw')]) + + old_container = create_and_start_container(service) + assert ( + [mount['Destination'] for mount in old_container.get('Mounts')] == + ['/data'] + ) + service.options['volumes'] = [] + + with mock.patch('compose.service.log', autospec=True) as mock_log: + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [old_container])) + + assert not mock_log.warn.called + assert ( + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] + ) + assert new_container.get_mount('/data')['Source'] != host_path + def test_execute_convergence_plan_without_start(self): service = self.create_service( 'db', From 8fb90bd73267fe880d7389ebc3b0af6da73df914 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 Jan 2016 13:26:32 +0000 Subject: [PATCH 326/359] Default to vim if EDITOR is not set Signed-off-by: Aanand Prasad --- script/release/make-branch | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/script/release/make-branch b/script/release/make-branch index 48fa771b..82b6ada0 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -63,15 +63,17 @@ git merge --strategy=ours --no-edit $REMOTE/release git config "branch.${BRANCH}.release" $VERSION +editor=${EDITOR:-vim} + echo "Update versions in docs/install.md, compose/__init__.py, script/run.sh" -$EDITOR docs/install.md -$EDITOR compose/__init__.py -$EDITOR script/run.sh +$editor docs/install.md +$editor compose/__init__.py +$editor script/run.sh echo "Write release notes in CHANGELOG.md" browser "https://github.com/docker/compose/issues?q=milestone%3A$VERSION+is%3Aclosed" -$EDITOR CHANGELOG.md +$editor CHANGELOG.md git diff From 110401b6f045dd9c9a7db40204654c06a450d9b7 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 Jan 2016 13:27:12 +0000 Subject: [PATCH 327/359] Let the user specify any repo as their fork Signed-off-by: Aanand Prasad --- script/release/make-branch | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/script/release/make-branch b/script/release/make-branch index 82b6ada0..46ba6bbc 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -86,10 +86,10 @@ echo "Push branch to user remote" GITHUB_USER=$USER USER_REMOTE="$(find_remote $GITHUB_USER/compose)" if [ -z "$USER_REMOTE" ]; then - echo "No user remote found for $GITHUB_USER" - read -r -p "Enter the name of your github user: " GITHUB_USER + echo "$GITHUB_USER/compose not found" + read -r -p "Enter the name of your GitHub fork (username/repo): " GITHUB_REPO # assumes there is already a user remote somewhere - USER_REMOTE=$(find_remote $GITHUB_USER/compose) + USER_REMOTE=$(find_remote $GITHUB_REPO) fi if [ -z "$USER_REMOTE" ]; then >&2 echo "No user remote found. You need to 'git push' your branch." From e925b8272b90aa39a27b104439c4b0abb08f97f4 Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Mon, 25 Jan 2016 10:34:29 +0100 Subject: [PATCH 328/359] Fix computation of service list in bash completion The previous approach assumed that the service list could be extracted from a single file. It did not follow extends and overrides. Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2b020c37..2c18bc04 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -17,6 +17,10 @@ # . ~/.docker-compose-completion.sh +__docker_compose_q() { + docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} "$@" +} + # suppress trailing whitespace __docker_compose_nospace() { # compopt is not available in ancient bash versions @@ -39,7 +43,7 @@ __docker_compose_compose_file() { # Extracts all service names from the compose file. ___docker_compose_all_services_in_compose_file() { - awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker_compose_compose_file)}" 2>/dev/null + __docker_compose_q config --services } # All services, even those without an existing container @@ -49,8 +53,12 @@ __docker_compose_services_all() { # All services that have an entry with the given key in their compose_file section ___docker_compose_services_with_key() { - # flatten sections to one line, then filter lines containing the key and return section name. - awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker_compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' + # flatten sections under "services" to one line, then filter lines containing the key and return section name + __docker_compose_q config \ + | sed -n -e '/^services:/,/^[^ ]/p' \ + | sed -n 's/^ //p' \ + | awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' \ + | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' } # All services that are defined by a Dockerfile reference @@ -67,11 +75,9 @@ __docker_compose_services_from_image() { # by a boolean expression passed in as argument. __docker_compose_services_with() { local containers names - containers="$(docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} ps -q)" - names=( $(docker 2>/dev/null inspect --format "{{if ${1:-true}}} {{ .Name }} {{end}}" $containers) ) - names=( ${names[@]%_*} ) # strip trailing numbers - names=( ${names[@]#*_} ) # strip project name - COMPREPLY=( $(compgen -W "${names[*]}" -- "$cur") ) + containers="$(__docker_compose_q ps -q)" + names=$(docker 2>/dev/null inspect -f "{{if ${1:-true}}}{{range \$k, \$v := .Config.Labels}}{{if eq \$k \"com.docker.compose.service\"}}{{\$v}}{{end}}{{end}}{{end}}" $containers) + COMPREPLY=( $(compgen -W "$names" -- "$cur") ) } # The services for which at least one paused container exists From bbaae11a0f53ae7be59bf8f53428fd8b16fb8d19 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 12:50:52 -0500 Subject: [PATCH 329/359] Fix race condition with up and setting signal handlers. Also print stdout on wait_for_container(). Signed-off-by: Daniel Nephin --- compose/cli/main.py | 36 +++++++++++-------- tests/acceptance/cli_test.py | 15 ++++++-- .../sleeps-composefile/docker-compose.yml | 10 ++++++ tests/unit/cli/main_test.py | 18 ---------- 4 files changed, 44 insertions(+), 35 deletions(-) create mode 100644 tests/fixtures/sleeps-composefile/docker-compose.yml diff --git a/compose/cli/main.py b/compose/cli/main.py index 14febe25..93c4729f 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import contextlib import json import logging import re @@ -53,7 +54,7 @@ def main(): command = TopLevelCommand() command.sys_dispatch() except KeyboardInterrupt: - log.error("\nAborting.") + log.error("Aborting.") sys.exit(1) except (UserError, NoSuchService, ConfigurationError) as e: log.error(e.msg) @@ -629,18 +630,20 @@ class TopLevelCommand(DocoptCommand): if detached and cascade_stop: raise UserError("--abort-on-container-exit and -d cannot be combined.") - to_attach = project.up( - service_names=service_names, - start_deps=start_deps, - strategy=convergence_strategy_from_opts(options), - do_build=not options['--no-build'], - timeout=timeout, - detached=detached - ) + with up_shutdown_context(project, service_names, timeout, detached): + to_attach = project.up( + service_names=service_names, + start_deps=start_deps, + strategy=convergence_strategy_from_opts(options), + do_build=not options['--no-build'], + timeout=timeout, + detached=detached) - if not detached: + if detached: + return log_printer = build_log_printer(to_attach, service_names, monochrome, cascade_stop) - attach_to_logs(project, log_printer, service_names, timeout) + print("Attaching to", list_containers(log_printer.containers)) + log_printer.run() def version(self, project, options): """ @@ -740,13 +743,16 @@ def build_log_printer(containers, service_names, monochrome, cascade_stop): return LogPrinter(containers, monochrome=monochrome, cascade_stop=cascade_stop) -def attach_to_logs(project, log_printer, service_names, timeout): - print("Attaching to", list_containers(log_printer.containers)) - signals.set_signal_handler_to_shutdown() +@contextlib.contextmanager +def up_shutdown_context(project, service_names, timeout, detached): + if detached: + yield + return + signals.set_signal_handler_to_shutdown() try: try: - log_printer.run() + yield except signals.ShutdownException: print("Gracefully stopping... (press Ctrl+C again to force)") project.stop(service_names=service_names, timeout=timeout) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 30589bad..b69ce8aa 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -43,7 +43,8 @@ def start_process(base_dir, options): def wait_on_process(proc, returncode=0): stdout, stderr = proc.communicate() if proc.returncode != returncode: - print(stderr.decode('utf-8')) + print("Stderr: {}".format(stderr)) + print("Stdout: {}".format(stdout)) assert proc.returncode == returncode return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) @@ -81,7 +82,6 @@ class ContainerStateCondition(object): self.name = name self.running = running - # State.Running == true def __call__(self): try: container = self.client.inspect_container(self.name) @@ -707,6 +707,17 @@ class CLITestCase(DockerClientTestCase): os.kill(proc.pid, signal.SIGTERM) wait_on_condition(ContainerCountCondition(self.project, 0)) + @v2_only() + def test_up_handles_force_shutdown(self): + self.base_dir = 'tests/fixtures/sleeps-composefile' + proc = start_process(self.base_dir, ['up', '-t', '200']) + wait_on_condition(ContainerCountCondition(self.project, 2)) + + os.kill(proc.pid, signal.SIGTERM) + time.sleep(0.1) + os.kill(proc.pid, signal.SIGTERM) + wait_on_condition(ContainerCountCondition(self.project, 0)) + def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['run', 'console', '/bin/true']) diff --git a/tests/fixtures/sleeps-composefile/docker-compose.yml b/tests/fixtures/sleeps-composefile/docker-compose.yml new file mode 100644 index 00000000..1eff7b73 --- /dev/null +++ b/tests/fixtures/sleeps-composefile/docker-compose.yml @@ -0,0 +1,10 @@ + +version: 2 + +services: + simple: + image: busybox:latest + command: sleep 200 + another: + image: busybox:latest + command: sleep 200 diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index 6f5dd3ca..fd6c5002 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -6,12 +6,9 @@ import logging from compose import container from compose.cli.errors import UserError from compose.cli.formatter import ConsoleWarningFormatter -from compose.cli.log_printer import LogPrinter -from compose.cli.main import attach_to_logs from compose.cli.main import build_log_printer from compose.cli.main import convergence_strategy_from_opts from compose.cli.main import setup_console_handler -from compose.project import Project from compose.service import ConvergenceStrategy from tests import mock from tests import unittest @@ -49,21 +46,6 @@ class CLIMainTestCase(unittest.TestCase): log_printer = build_log_printer(containers, service_names, True, False) self.assertEqual(log_printer.containers, containers) - def test_attach_to_logs(self): - project = mock.create_autospec(Project) - log_printer = mock.create_autospec(LogPrinter, containers=[]) - service_names = ['web', 'db'] - timeout = 12 - - with mock.patch('compose.cli.main.signals.signal', autospec=True) as mock_signal: - attach_to_logs(project, log_printer, service_names, timeout) - - assert mock_signal.signal.mock_calls == [ - mock.call(mock_signal.SIGINT, mock.ANY), - mock.call(mock_signal.SIGTERM, mock.ANY), - ] - log_printer.run.assert_called_once_with() - class SetupConsoleHandlerTestCase(unittest.TestCase): From d3cd9213c1e8a47337c8720c1af01466ce314006 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 26 Jan 2016 17:41:26 +0000 Subject: [PATCH 330/359] Fix rebase-bump-commit script Trim whitespace from wc's output before constructing arguments to `git rebase` Signed-off-by: Aanand Prasad --- script/release/rebase-bump-commit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/script/release/rebase-bump-commit b/script/release/rebase-bump-commit index 23877bb5..3c2ae72b 100755 --- a/script/release/rebase-bump-commit +++ b/script/release/rebase-bump-commit @@ -32,7 +32,7 @@ if [[ "$sha" == "$(git rev-parse HEAD)" ]]; then exit 0 fi -commits=$(git log --format="%H" "$sha..HEAD" | wc -l) +commits=$(git log --format="%H" "$sha..HEAD" | wc -l | xargs echo) git rebase --onto $sha~1 HEAD~$commits $BRANCH git cherry-pick $sha From c98c617c3004a5bb71dabdda1ff357bcfe1773d5 Mon Sep 17 00:00:00 2001 From: Steve Durrheimer Date: Mon, 25 Jan 2016 10:27:21 +0100 Subject: [PATCH 331/359] Add zsh completion for 'docker-compose create' Signed-off-by: Steve Durrheimer --- contrib/completion/zsh/_docker-compose | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 43017022..f67bc9f6 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -203,6 +203,14 @@ __docker-compose_subcommand() { '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ '--services[Print the service names, one per line.]' && ret=0 ;; + (create) + _arguments \ + $opts_help \ + "(--no-recreate --no-build)--force-recreate[Recreate containers even if their configuration and image haven't changed. Incompatible with --no-recreate.]" \ + "(--force-recreate)--no-build[If containers already exist, don't recreate them. Incompatible with --force-recreate.]" \ + "(--force-recreate)--no-recreate[Don't build an image, even if it's missing]" \ + '*:services:__docker-compose_services_all' && ret=0 + ;; (down) _arguments \ $opts_help \ From 25df0d81472fdfa5c8cd54b7e8d344bdd2a2ec0c Mon Sep 17 00:00:00 2001 From: Harald Albers Date: Mon, 25 Jan 2016 10:15:14 +0100 Subject: [PATCH 332/359] bash completion for `docker-compose create` Signed-off-by: Harald Albers --- contrib/completion/bash/docker-compose | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2c18bc04..3b135311 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -113,6 +113,18 @@ _docker_compose_config() { } +_docker_compose_create() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--force-recreate --help --no-build --no-recreate" -- "$cur" ) ) + ;; + *) + __docker_compose_services_all + ;; + esac +} + + _docker_compose_docker_compose() { case "$prev" in --file|-f) @@ -415,6 +427,7 @@ _docker_compose() { local commands=( build config + create down events help From 9e9b36460ce90d8fc99badfd79096dc789d4267b Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 Jan 2016 14:44:51 +0000 Subject: [PATCH 333/359] Convert validation error tests to pytest style Signed-off-by: Aanand Prasad --- tests/unit/config/config_test.py | 139 +++++++++++++++++-------------- 1 file changed, 76 insertions(+), 63 deletions(-) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 5f8b097b..87f10afb 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -274,9 +274,7 @@ class ConfigTest(unittest.TestCase): assert error_msg in exc.exconly() def test_config_integer_service_name_raise_validation_error(self): - expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " - "be a string, eg '1'") - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {1: {'image': 'busybox'}}, @@ -285,11 +283,11 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_integer_service_name_raise_validation_error_v2(self): - expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " - "be a string, eg '1'") + assert "In file 'filename.yml' service name: 1 needs to be a string, eg '1'" \ + in excinfo.exconly() - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_config_integer_service_name_raise_validation_error_v2(self): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -301,6 +299,9 @@ class ConfigTest(unittest.TestCase): ) ) + assert "In file 'filename.yml' service name: 1 needs to be a string, eg '1'" \ + in excinfo.exconly() + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', @@ -624,8 +625,7 @@ class ConfigTest(unittest.TestCase): assert services[0]['name'] == valid_name def test_config_hint(self): - expected_error_msg = "(did you mean 'privileged'?)" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -636,6 +636,8 @@ class ConfigTest(unittest.TestCase): ) ) + assert "(did you mean 'privileged'?)" in excinfo.exconly() + def test_load_errors_on_uppercase_with_no_image(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details({ @@ -643,9 +645,8 @@ class ConfigTest(unittest.TestCase): }, 'tests/fixtures/build-ctx')) assert "Service 'Foo' contains uppercase characters" in exc.exconly() - def test_invalid_config_build_and_image_specified(self): - expected_error_msg = "Service 'foo' has both an image and build path specified." - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_invalid_config_build_and_image_specified_v1(self): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -656,9 +657,10 @@ class ConfigTest(unittest.TestCase): ) ) + assert "Service 'foo' has both an image and build path specified." in excinfo.exconly() + def test_invalid_config_type_should_be_an_array(self): - expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -669,10 +671,11 @@ class ConfigTest(unittest.TestCase): ) ) + assert "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" \ + in excinfo.exconly() + def test_invalid_config_not_a_dictionary(self): - expected_error_msg = ("Top level object in 'filename.yml' needs to be " - "an object.") - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( ['foo', 'lol'], @@ -681,9 +684,11 @@ class ConfigTest(unittest.TestCase): ) ) + assert "Top level object in 'filename.yml' needs to be an object" \ + in excinfo.exconly() + def test_invalid_config_not_unique_items(self): - expected_error_msg = "has non-unique elements" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -694,10 +699,10 @@ class ConfigTest(unittest.TestCase): ) ) + assert "has non-unique elements" in excinfo.exconly() + def test_invalid_list_of_strings_format(self): - expected_error_msg = "Service 'web' configuration key 'command' contains 1" - expected_error_msg += ", which is an invalid type, it should be a string" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -708,7 +713,10 @@ class ConfigTest(unittest.TestCase): ) ) - def test_load_config_dockerfile_without_build_raises_error(self): + assert "Service 'web' configuration key 'command' contains 1, which is an invalid type, it should be a string" \ + in excinfo.exconly() + + def test_load_config_dockerfile_without_build_raises_error_v1(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details({ 'web': { @@ -716,12 +724,11 @@ class ConfigTest(unittest.TestCase): 'dockerfile': 'Dockerfile.alt' } })) + assert "Service 'web' has both an image and alternate Dockerfile." in exc.exconly() def test_config_extra_hosts_string_raises_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'extra_hosts' contains an invalid type" - - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {'web': { @@ -733,12 +740,11 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_extra_hosts_list_of_dicts_validation_error(self): - expected_error_msg = ( - "key 'extra_hosts' contains {\"somehost\": \"162.242.195.82\"}, " - "which is an invalid type, it should be a string") + assert "Service 'web' configuration key 'extra_hosts' contains an invalid type" \ + in excinfo.exconly() - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_config_extra_hosts_list_of_dicts_validation_error(self): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {'web': { @@ -753,10 +759,11 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_ulimits_invalid_keys_validation_error(self): - expected = ("Service 'web' configuration key 'ulimits' 'nofile' contains " - "unsupported option: 'not_soft_or_hard'") + assert "key 'extra_hosts' contains {\"somehost\": \"162.242.195.82\"}, " \ + "which is an invalid type, it should be a string" \ + in excinfo.exconly() + def test_config_ulimits_invalid_keys_validation_error(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( { @@ -773,10 +780,11 @@ class ConfigTest(unittest.TestCase): }, 'working_dir', 'filename.yml')) - assert expected in exc.exconly() + + assert "Service 'web' configuration key 'ulimits' 'nofile' contains unsupported option: 'not_soft_or_hard'" \ + in exc.exconly() def test_config_ulimits_required_keys_validation_error(self): - with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( { @@ -1574,11 +1582,7 @@ class MemoryOptionsTest(unittest.TestCase): When you set a 'memswap_limit' it is invalid config unless you also set a mem_limit """ - expected_error_msg = ( - "Service 'foo' configuration key 'memswap_limit' is invalid: when " - "defining 'memswap_limit' you must set 'mem_limit' as well" - ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1589,6 +1593,10 @@ class MemoryOptionsTest(unittest.TestCase): ) ) + assert "Service 'foo' configuration key 'memswap_limit' is invalid: when defining " \ + "'memswap_limit' you must set 'mem_limit' as well" \ + in excinfo.exconly() + def test_validation_with_correct_memswap_values(self): service_dict = config.load( build_config_details( @@ -1851,7 +1859,7 @@ class ExtendsTest(unittest.TestCase): self.assertEqual(path, expected) def test_extends_validation_empty_dictionary(self): - with self.assertRaisesRegexp(ConfigurationError, 'service'): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1862,8 +1870,10 @@ class ExtendsTest(unittest.TestCase): ) ) + assert 'service' in excinfo.exconly() + def test_extends_validation_missing_service_key(self): - with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1874,12 +1884,10 @@ class ExtendsTest(unittest.TestCase): ) ) + assert "'service' is a required property" in excinfo.exconly() + def test_extends_validation_invalid_key(self): - expected_error_msg = ( - "Service 'web' configuration key 'extends' " - "contains unsupported option: 'rogue_key'" - ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1897,12 +1905,11 @@ class ExtendsTest(unittest.TestCase): ) ) + assert "Service 'web' configuration key 'extends' contains unsupported option: 'rogue_key'" \ + in excinfo.exconly() + def test_extends_validation_sub_property_key(self): - expected_error_msg = ( - "Service 'web' configuration key 'extends' 'file' contains 1, " - "which is an invalid type, it should be a string" - ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1919,13 +1926,16 @@ class ExtendsTest(unittest.TestCase): ) ) + assert "Service 'web' configuration key 'extends' 'file' contains 1, which is an invalid type, it should be a string" \ + in excinfo.exconly() + def test_extends_validation_no_file_key_no_filename_set(self): dictionary = {'extends': {'service': 'web'}} - def load_config(): - return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') + with pytest.raises(ConfigurationError) as excinfo: + make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - self.assertRaisesRegexp(ConfigurationError, 'file', load_config) + assert 'file' in excinfo.exconly() def test_extends_validation_valid_config(self): service = config.load( @@ -1979,16 +1989,17 @@ class ExtendsTest(unittest.TestCase): ])) def test_invalid_links_in_extended_service(self): - expected_error_msg = "services with 'links' cannot be extended" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-links.yml') - def test_invalid_volumes_from_in_extended_service(self): - expected_error_msg = "services with 'volumes_from' cannot be extended" + assert "services with 'links' cannot be extended" in excinfo.exconly() - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_invalid_volumes_from_in_extended_service(self): + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-volumes.yml') + assert "services with 'volumes_from' cannot be extended" in excinfo.exconly() + def test_invalid_net_in_extended_service(self): with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-net-v2.yml') @@ -2044,10 +2055,12 @@ class ExtendsTest(unittest.TestCase): ]) def test_load_throws_error_when_base_service_does_not_exist(self): - err_msg = r'''Cannot extend service 'foo' in .*: Service not found''' - with self.assertRaisesRegexp(ConfigurationError, err_msg): + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/nonexistent-service.yml') + assert "Cannot extend service 'foo'" in excinfo.exconly() + assert "Service not found" in excinfo.exconly() + def test_partial_service_config_in_extends_is_still_valid(self): dicts = load_from_filename('tests/fixtures/extends/valid-common-config.yml') self.assertEqual(dicts[0]['environment'], {'FOO': '1'}) From e9ba06ed4b1cb4f51da7bd2ec4ee1c93b417bfe0 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 Jan 2016 15:20:46 +0000 Subject: [PATCH 334/359] Normalise/fix config field designators in validation messages - Instead of "Service 'web' configuration key 'image'", just say "web.image" - Fix the "Service 'services'" bug in the v2 file format Signed-off-by: Aanand Prasad --- compose/config/validation.py | 96 +++++++++++++++++--------------- tests/unit/config/config_test.py | 91 +++++++++++++++++++++++------- 2 files changed, 121 insertions(+), 66 deletions(-) diff --git a/compose/config/validation.py b/compose/config/validation.py index 05982020..ba1ac521 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -174,8 +174,8 @@ def validate_depends_on(service_config, service_names): "undefined.".format(s=service_config, dep=dependency)) -def get_unsupported_config_msg(service_name, error_key): - msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) +def get_unsupported_config_msg(path, error_key): + msg = "Unsupported config option for {}: '{}'".format(path_string(path), error_key) if error_key in DOCKER_CONFIG_HINTS: msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key]) return msg @@ -191,7 +191,7 @@ def is_service_dict_schema(schema_id): return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services' -def handle_error_for_schema_with_id(error, service_name): +def handle_error_for_schema_with_id(error, path): schema_id = error.schema['id'] if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': @@ -215,62 +215,64 @@ def handle_error_for_schema_with_id(error, service_name): # TODO: only applies to v1 if 'image' in error.instance and context: return ( - "Service '{}' has both an image and build path specified. " + "{} has both an image and build path specified. " "A service can either be built to image or use an existing " - "image, not both.".format(service_name)) + "image, not both.".format(path_string(path))) if 'image' not in error.instance and not context: return ( - "Service '{}' has neither an image nor a build path " - "specified. At least one must be provided.".format(service_name)) + "{} has neither an image nor a build path specified. " + "At least one must be provided.".format(path_string(path))) # TODO: only applies to v1 if 'image' in error.instance and dockerfile: return ( - "Service '{}' has both an image and alternate Dockerfile. " + "{} has both an image and alternate Dockerfile. " "A service can either be built to image or use an existing " - "image, not both.".format(service_name)) + "image, not both.".format(path_string(path))) if schema_id == '#/definitions/service': if error.validator == 'additionalProperties': invalid_config_key = parse_key_from_error_msg(error) - return get_unsupported_config_msg(service_name, invalid_config_key) + return get_unsupported_config_msg(path, invalid_config_key) -def handle_generic_service_error(error, service_name): - config_key = " ".join("'%s'" % k for k in error.path) +def handle_generic_service_error(error, path): msg_format = None error_msg = error.message if error.validator == 'oneOf': - msg_format = "Service '{}' configuration key {} {}" - error_msg = _parse_oneof_validator(error) + msg_format = "{path} {msg}" + config_key, error_msg = _parse_oneof_validator(error) + if config_key: + path.append(config_key) elif error.validator == 'type': - msg_format = ("Service '{}' configuration key {} contains an invalid " - "type, it should be {}") + msg_format = "{path} contains an invalid type, it should be {msg}" error_msg = _parse_valid_types_from_validator(error.validator_value) # TODO: no test case for this branch, there are no config options # which exercise this branch elif error.validator == 'required': - msg_format = "Service '{}' configuration key '{}' is invalid, {}" + msg_format = "{path} is invalid, {msg}" elif error.validator == 'dependencies': - msg_format = "Service '{}' configuration key '{}' is invalid: {}" config_key = list(error.validator_value.keys())[0] required_keys = ",".join(error.validator_value[config_key]) + + msg_format = "{path} is invalid: {msg}" + path.append(config_key) error_msg = "when defining '{}' you must set '{}' as well".format( config_key, required_keys) elif error.cause: error_msg = six.text_type(error.cause) - msg_format = "Service '{}' configuration key {} is invalid: {}" + msg_format = "{path} is invalid: {msg}" elif error.path: - msg_format = "Service '{}' configuration key {} value {}" + msg_format = "{path} value {msg}" if msg_format: - return msg_format.format(service_name, config_key, error_msg) + return msg_format.format(path=path_string(path), msg=error_msg) return error.message @@ -279,6 +281,10 @@ def parse_key_from_error_msg(error): return error.message.split("'")[1] +def path_string(path): + return ".".join(c for c in path if isinstance(c, six.string_types)) + + def _parse_valid_types_from_validator(validator): """A validator value can be either an array of valid types or a string of a valid type. Parse the valid types and prefix with the correct article. @@ -304,52 +310,52 @@ def _parse_oneof_validator(error): for context in error.context: if context.validator == 'required': - return context.message + return (None, context.message) if context.validator == 'additionalProperties': invalid_config_key = parse_key_from_error_msg(context) - return "contains unsupported option: '{}'".format(invalid_config_key) + return (None, "contains unsupported option: '{}'".format(invalid_config_key)) if context.path: - invalid_config_key = " ".join( - "'{}' ".format(fragment) for fragment in context.path - if isinstance(fragment, six.string_types) + return ( + path_string(context.path), + "contains {}, which is an invalid type, it should be {}".format( + json.dumps(context.instance), + _parse_valid_types_from_validator(context.validator_value)), ) - return "{}contains {}, which is an invalid type, it should be {}".format( - invalid_config_key, - # Always print the json repr of the invalid value - json.dumps(context.instance), - _parse_valid_types_from_validator(context.validator_value)) if context.validator == 'uniqueItems': - return "contains non unique items, please remove duplicates from {}".format( - context.instance) + return ( + None, + "contains non unique items, please remove duplicates from {}".format( + context.instance), + ) if context.validator == 'type': types.append(context.validator_value) valid_types = _parse_valid_types_from_validator(types) - return "contains an invalid type, it should be {}".format(valid_types) + return (None, "contains an invalid type, it should be {}".format(valid_types)) -def process_errors(errors, service_name=None): +def process_errors(errors, path_prefix=None): """jsonschema gives us an error tree full of information to explain what has gone wrong. Process each error and pull out relevant information and re-write helpful error messages that are relevant. """ - def format_error_message(error, service_name): - if not service_name and error.path: - # field_schema errors will have service name on the path - service_name = error.path.popleft() + path_prefix = path_prefix or [] + + def format_error_message(error): + path = path_prefix + list(error.path) if 'id' in error.schema: - error_msg = handle_error_for_schema_with_id(error, service_name) + error_msg = handle_error_for_schema_with_id(error, path) if error_msg: return error_msg - return handle_generic_service_error(error, service_name) + return handle_generic_service_error(error, path) - return '\n'.join(format_error_message(error, service_name) for error in errors) + return '\n'.join(format_error_message(error) for error in errors) def validate_against_fields_schema(config_file): @@ -366,14 +372,14 @@ def validate_against_service_schema(config, service_name, version): config, "service_schema_v{0}.json".format(version), format_checker=["ports"], - service_name=service_name) + path_prefix=[service_name]) def _validate_against_schema( config, schema_filename, format_checker=(), - service_name=None, + path_prefix=None, filename=None): config_source_dir = os.path.dirname(os.path.abspath(__file__)) @@ -399,7 +405,7 @@ def _validate_against_schema( if not errors: return - error_msg = process_errors(errors, service_name) + error_msg = process_errors(errors, path_prefix=path_prefix) file_msg = " in file '{}'".format(filename) if filename else '' raise ConfigurationError("Validation failed{}, reason(s):\n{}".format( file_msg, diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 87f10afb..44f5c684 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -253,15 +253,31 @@ class ConfigTest(unittest.TestCase): assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() def test_load_with_invalid_field_name(self): - config_details = build_config_details( - {'web': {'image': 'busybox', 'name': 'bogus'}}, - 'working_dir', - 'filename.yml') with pytest.raises(ConfigurationError) as exc: - config.load(config_details) - error_msg = "Unsupported config option for 'web' service: 'name'" - assert error_msg in exc.exconly() - assert "Validation failed in file 'filename.yml'" in exc.exconly() + config.load(build_config_details( + { + 'version': 2, + 'services': { + 'web': {'image': 'busybox', 'name': 'bogus'}, + } + }, + 'working_dir', + 'filename.yml', + )) + + assert "Unsupported config option for services.web: 'name'" in exc.exconly() + + def test_load_with_invalid_field_name_v1(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + { + 'web': {'image': 'busybox', 'name': 'bogus'}, + }, + 'working_dir', + 'filename.yml', + )) + + assert "Unsupported config option for web: 'name'" in exc.exconly() def test_load_invalid_service_definition(self): config_details = build_config_details( @@ -645,6 +661,39 @@ class ConfigTest(unittest.TestCase): }, 'tests/fixtures/build-ctx')) assert "Service 'Foo' contains uppercase characters" in exc.exconly() + def test_invalid_config_v1(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'foo': {'image': 1}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + assert "foo.image contains an invalid type, it should be a string" \ + in excinfo.exconly() + + def test_invalid_config_v2(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': 2, + 'services': { + 'foo': {'image': 1}, + }, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + assert "services.foo.image contains an invalid type, it should be a string" \ + in excinfo.exconly() + def test_invalid_config_build_and_image_specified_v1(self): with pytest.raises(ConfigurationError) as excinfo: config.load( @@ -657,7 +706,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert "Service 'foo' has both an image and build path specified." in excinfo.exconly() + assert "foo has both an image and build path specified." in excinfo.exconly() def test_invalid_config_type_should_be_an_array(self): with pytest.raises(ConfigurationError) as excinfo: @@ -671,7 +720,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" \ + assert "foo.links contains an invalid type, it should be an array" \ in excinfo.exconly() def test_invalid_config_not_a_dictionary(self): @@ -713,7 +762,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert "Service 'web' configuration key 'command' contains 1, which is an invalid type, it should be a string" \ + assert "web.command contains 1, which is an invalid type, it should be a string" \ in excinfo.exconly() def test_load_config_dockerfile_without_build_raises_error_v1(self): @@ -725,7 +774,7 @@ class ConfigTest(unittest.TestCase): } })) - assert "Service 'web' has both an image and alternate Dockerfile." in exc.exconly() + assert "web has both an image and alternate Dockerfile." in exc.exconly() def test_config_extra_hosts_string_raises_validation_error(self): with pytest.raises(ConfigurationError) as excinfo: @@ -740,7 +789,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert "Service 'web' configuration key 'extra_hosts' contains an invalid type" \ + assert "web.extra_hosts contains an invalid type" \ in excinfo.exconly() def test_config_extra_hosts_list_of_dicts_validation_error(self): @@ -759,7 +808,7 @@ class ConfigTest(unittest.TestCase): ) ) - assert "key 'extra_hosts' contains {\"somehost\": \"162.242.195.82\"}, " \ + assert "web.extra_hosts contains {\"somehost\": \"162.242.195.82\"}, " \ "which is an invalid type, it should be a string" \ in excinfo.exconly() @@ -781,7 +830,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml')) - assert "Service 'web' configuration key 'ulimits' 'nofile' contains unsupported option: 'not_soft_or_hard'" \ + assert "web.ulimits.nofile contains unsupported option: 'not_soft_or_hard'" \ in exc.exconly() def test_config_ulimits_required_keys_validation_error(self): @@ -795,7 +844,7 @@ class ConfigTest(unittest.TestCase): }, 'working_dir', 'filename.yml')) - assert "Service 'web' configuration key 'ulimits' 'nofile'" in exc.exconly() + assert "web.ulimits.nofile" in exc.exconly() assert "'hard' is a required property" in exc.exconly() def test_config_ulimits_soft_greater_than_hard_error(self): @@ -896,7 +945,7 @@ class ConfigTest(unittest.TestCase): 'extra_hosts': "www.example.com: 192.168.0.17", } })) - assert "'extra_hosts' contains an invalid type" in exc.exconly() + assert "web.extra_hosts contains an invalid type" in exc.exconly() def test_validate_extra_hosts_invalid_list(self): with pytest.raises(ConfigurationError) as exc: @@ -1593,7 +1642,7 @@ class MemoryOptionsTest(unittest.TestCase): ) ) - assert "Service 'foo' configuration key 'memswap_limit' is invalid: when defining " \ + assert "foo.memswap_limit is invalid: when defining " \ "'memswap_limit' you must set 'mem_limit' as well" \ in excinfo.exconly() @@ -1905,7 +1954,7 @@ class ExtendsTest(unittest.TestCase): ) ) - assert "Service 'web' configuration key 'extends' contains unsupported option: 'rogue_key'" \ + assert "web.extends contains unsupported option: 'rogue_key'" \ in excinfo.exconly() def test_extends_validation_sub_property_key(self): @@ -1926,7 +1975,7 @@ class ExtendsTest(unittest.TestCase): ) ) - assert "Service 'web' configuration key 'extends' 'file' contains 1, which is an invalid type, it should be a string" \ + assert "web.extends.file contains 1, which is an invalid type, it should be a string" \ in excinfo.exconly() def test_extends_validation_no_file_key_no_filename_set(self): @@ -1956,7 +2005,7 @@ class ExtendsTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: load_from_filename('tests/fixtures/extends/service-with-invalid-schema.yml') assert ( - "Service 'myweb' has neither an image nor a build path specified" in + "myweb has neither an image nor a build path specified" in exc.exconly() ) From b2ee08f439bc2fb1bf699eb9ca5356e2485ba743 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 28 Jan 2016 16:26:25 +0000 Subject: [PATCH 335/359] Remove redundant check - self.config should never be None Signed-off-by: Aanand Prasad --- compose/config/config.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index ffd805ad..34168df5 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -129,8 +129,6 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): @cached_property def version(self): - if self.config is None: - return 1 version = self.config.get('version', 1) if isinstance(version, dict): log.warn("Unexpected type for field 'version', in file {} assuming " From ce0d469c18c4495d8420e47f81a7fd31a1a24795 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 Jan 2016 15:58:38 +0000 Subject: [PATCH 336/359] Make 'version' a string Signed-off-by: Aanand Prasad --- compose/config/config.py | 33 +++--- ...schema_v2.json => fields_schema_v2.0.json} | 6 +- ...chema_v2.json => service_schema_v2.0.json} | 2 +- compose/config/types.py | 3 +- compose/const.py | 9 +- compose/project.py | 5 +- docker-compose.spec | 8 +- tests/acceptance/cli_test.py | 2 +- tests/fixtures/extends/invalid-net-v2.yml | 2 +- .../logging-composefile/docker-compose.yml | 2 +- tests/fixtures/net-container/v2-invalid.yml | 2 +- tests/fixtures/networks/bridge.yml | 2 +- .../networks/default-network-config.yml | 2 +- tests/fixtures/networks/docker-compose.yml | 2 +- tests/fixtures/networks/external-default.yml | 2 +- tests/fixtures/networks/external-networks.yml | 2 +- tests/fixtures/networks/missing-network.yml | 2 +- tests/fixtures/networks/network-mode.yml | 2 +- tests/fixtures/no-services/docker-compose.yml | 2 +- .../sleeps-composefile/docker-compose.yml | 2 +- tests/fixtures/v2-full/docker-compose.yml | 2 +- tests/fixtures/v2-simple/docker-compose.yml | 2 +- tests/fixtures/v2-simple/links-invalid.yml | 2 +- tests/integration/project_test.py | 29 ++--- tests/integration/testcases.py | 6 +- tests/unit/config/config_test.py | 109 +++++++++++++----- tests/unit/config/types_test.py | 16 +-- 27 files changed, 160 insertions(+), 98 deletions(-) rename compose/config/{fields_schema_v2.json => fields_schema_v2.0.json} (94%) rename compose/config/{service_schema_v2.json => service_schema_v2.0.json} (99%) diff --git a/compose/config/config.py b/compose/config/config.py index 34168df5..bda086b9 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -14,6 +14,8 @@ import six import yaml from cached_property import cached_property +from ..const import COMPOSEFILE_V1 as V1 +from ..const import COMPOSEFILE_V2_0 as V2_0 from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference from .errors import ComposeFileNotFound @@ -129,25 +131,34 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): @cached_property def version(self): - version = self.config.get('version', 1) + version = self.config.get('version', V1) + if isinstance(version, dict): log.warn("Unexpected type for field 'version', in file {} assuming " "version is the name of a service, and defaulting to " "Compose file version 1".format(self.filename)) - return 1 + return V1 + + if version == '2': + version = V2_0 + + if version not in COMPOSEFILE_VERSIONS: + raise ConfigurationError( + 'Invalid Compose file version: {0}'.format(version)) + return version def get_service(self, name): return self.get_service_dicts()[name] def get_service_dicts(self): - return self.config if self.version == 1 else self.config.get('services', {}) + return self.config if self.version == V1 else self.config.get('services', {}) def get_volumes(self): - return {} if self.version == 1 else self.config.get('volumes', {}) + return {} if self.version == V1 else self.config.get('volumes', {}) def get_networks(self): - return {} if self.version == 1 else self.config.get('networks', {}) + return {} if self.version == V1 else self.config.get('networks', {}) class Config(namedtuple('_Config', 'version services volumes networks')): @@ -209,10 +220,6 @@ def validate_config_version(config_files): next_file.filename, next_file.version)) - if main_file.version not in COMPOSEFILE_VERSIONS: - raise ConfigurationError( - 'Invalid Compose file version: {0}'.format(main_file.version)) - def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) @@ -276,7 +283,7 @@ def load(config_details): main_file, [file.get_service_dicts() for file in config_details.config_files]) - if main_file.version >= 2: + if main_file.version != V1: for service_dict in service_dicts: match_named_volumes(service_dict, volumes) @@ -361,7 +368,7 @@ def process_config_file(config_file, service_name=None): interpolated_config = interpolate_environment_variables(service_dicts, 'service') - if config_file.version == 2: + if config_file.version == V2_0: processed_config = dict(config_file.config) processed_config['services'] = services = interpolated_config processed_config['volumes'] = interpolate_environment_variables( @@ -369,7 +376,7 @@ def process_config_file(config_file, service_name=None): processed_config['networks'] = interpolate_environment_variables( config_file.get_networks(), 'network') - if config_file.version == 1: + if config_file.version == V1: processed_config = services = interpolated_config config_file = config_file._replace(config=processed_config) @@ -653,7 +660,7 @@ def merge_service_dicts(base, override, version): if field in base or field in override: d[field] = override.get(field, base.get(field)) - if version == 1: + if version == V1: legacy_v1_merge_image_or_build(d, base, override) else: merge_build(d, base, override) diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.0.json similarity index 94% rename from compose/config/fields_schema_v2.json rename to compose/config/fields_schema_v2.0.json index c001df68..7703adcd 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.0.json @@ -1,18 +1,18 @@ { "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "id": "fields_schema_v2.json", + "id": "fields_schema_v2.0.json", "properties": { "version": { - "enum": [2] + "type": "string" }, "services": { "id": "#/properties/services", "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "service_schema_v2.json#/definitions/service" + "$ref": "service_schema_v2.0.json#/definitions/service" } }, "additionalProperties": false diff --git a/compose/config/service_schema_v2.json b/compose/config/service_schema_v2.0.json similarity index 99% rename from compose/config/service_schema_v2.json rename to compose/config/service_schema_v2.0.json index ca9bb671..8dd4faf5 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/service_schema_v2.0.json @@ -1,6 +1,6 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "service_schema_v2.json", + "id": "service_schema_v2.0.json", "type": "object", diff --git a/compose/config/types.py b/compose/config/types.py index 2e648e5a..9bda7180 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals import os from collections import namedtuple +from compose.config.config import V1 from compose.config.errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM @@ -16,7 +17,7 @@ class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode type')): # TODO: drop service_names arg when v1 is removed @classmethod def parse(cls, volume_from_config, service_names, version): - func = cls.parse_v1 if version == 1 else cls.parse_v2 + func = cls.parse_v1 if version == V1 else cls.parse_v2 return func(service_names, volume_from_config) @classmethod diff --git a/compose/const.py b/compose/const.py index 6ff108fb..d78a5fa7 100644 --- a/compose/const.py +++ b/compose/const.py @@ -14,9 +14,12 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' -COMPOSEFILE_VERSIONS = (1, 2) + +COMPOSEFILE_V1 = '1' +COMPOSEFILE_V2_0 = '2.0' +COMPOSEFILE_VERSIONS = (COMPOSEFILE_V1, COMPOSEFILE_V2_0) API_VERSIONS = { - 1: '1.21', - 2: '1.22', + COMPOSEFILE_V1: '1.21', + COMPOSEFILE_V2_0: '1.22', } diff --git a/compose/project.py b/compose/project.py index d2787ecf..6411f7cc 100644 --- a/compose/project.py +++ b/compose/project.py @@ -10,6 +10,7 @@ from docker.errors import NotFound from . import parallel from .config import ConfigurationError +from .config.config import V1 from .config.sort_services import get_container_name_from_network_mode from .config.sort_services import get_service_name_from_network_mode from .const import DEFAULT_TIMEOUT @@ -56,7 +57,7 @@ class Project(object): """ Construct a Project from a config.Config object. """ - use_networking = (config_data.version and config_data.version >= 2) + use_networking = (config_data.version and config_data.version != V1) project = cls(name, [], client, use_networking=use_networking) network_config = config_data.networks or {} @@ -94,7 +95,7 @@ class Project(object): network_mode = project.get_network_mode(service_dict, networks) volumes_from = get_volumes_from(project, service_dict) - if config_data.version == 2: + if config_data.version != V1: service_volumes = service_dict.get('volumes', []) for volume_spec in service_volumes: if volume_spec.is_named_volume: diff --git a/docker-compose.spec b/docker-compose.spec index f7f2059f..b3d8db39 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -23,8 +23,8 @@ exe = EXE(pyz, 'DATA' ), ( - 'compose/config/fields_schema_v2.json', - 'compose/config/fields_schema_v2.json', + 'compose/config/fields_schema_v2.0.json', + 'compose/config/fields_schema_v2.0.json', 'DATA' ), ( @@ -33,8 +33,8 @@ exe = EXE(pyz, 'DATA' ), ( - 'compose/config/service_schema_v2.json', - 'compose/config/service_schema_v2.json', + 'compose/config/service_schema_v2.0.json', + 'compose/config/service_schema_v2.0.json', 'DATA' ), ( diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index b69ce8aa..447b1e32 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -177,7 +177,7 @@ class CLITestCase(DockerClientTestCase): output = yaml.load(result.stdout) expected = { - 'version': 2, + 'version': '2.0', 'volumes': {'data': {'driver': 'local'}}, 'networks': {'front': {}}, 'services': { diff --git a/tests/fixtures/extends/invalid-net-v2.yml b/tests/fixtures/extends/invalid-net-v2.yml index 0a04f468..7ba714e8 100644 --- a/tests/fixtures/extends/invalid-net-v2.yml +++ b/tests/fixtures/extends/invalid-net-v2.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: myweb: build: '.' diff --git a/tests/fixtures/logging-composefile/docker-compose.yml b/tests/fixtures/logging-composefile/docker-compose.yml index 0a73030a..466d13e5 100644 --- a/tests/fixtures/logging-composefile/docker-compose.yml +++ b/tests/fixtures/logging-composefile/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/net-container/v2-invalid.yml b/tests/fixtures/net-container/v2-invalid.yml index eac4b5f1..9b846295 100644 --- a/tests/fixtures/net-container/v2-invalid.yml +++ b/tests/fixtures/net-container/v2-invalid.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: foo: diff --git a/tests/fixtures/networks/bridge.yml b/tests/fixtures/networks/bridge.yml index 95098372..9fa7db82 100644 --- a/tests/fixtures/networks/bridge.yml +++ b/tests/fixtures/networks/bridge.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: diff --git a/tests/fixtures/networks/default-network-config.yml b/tests/fixtures/networks/default-network-config.yml index 275fae98..4bd0989b 100644 --- a/tests/fixtures/networks/default-network-config.yml +++ b/tests/fixtures/networks/default-network-config.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml index 5351c0f0..c11fa682 100644 --- a/tests/fixtures/networks/docker-compose.yml +++ b/tests/fixtures/networks/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: diff --git a/tests/fixtures/networks/external-default.yml b/tests/fixtures/networks/external-default.yml index 7b0797e5..5c9426b8 100644 --- a/tests/fixtures/networks/external-default.yml +++ b/tests/fixtures/networks/external-default.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/networks/external-networks.yml b/tests/fixtures/networks/external-networks.yml index 644e3dda..db75b780 100644 --- a/tests/fixtures/networks/external-networks.yml +++ b/tests/fixtures/networks/external-networks.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: diff --git a/tests/fixtures/networks/missing-network.yml b/tests/fixtures/networks/missing-network.yml index 666f7d34..41012535 100644 --- a/tests/fixtures/networks/missing-network.yml +++ b/tests/fixtures/networks/missing-network.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: diff --git a/tests/fixtures/networks/network-mode.yml b/tests/fixtures/networks/network-mode.yml index 7ab63df8..e4d070b4 100644 --- a/tests/fixtures/networks/network-mode.yml +++ b/tests/fixtures/networks/network-mode.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: bridge: diff --git a/tests/fixtures/no-services/docker-compose.yml b/tests/fixtures/no-services/docker-compose.yml index fa498784..6e76ec0c 100644 --- a/tests/fixtures/no-services/docker-compose.yml +++ b/tests/fixtures/no-services/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" networks: foo: {} diff --git a/tests/fixtures/sleeps-composefile/docker-compose.yml b/tests/fixtures/sleeps-composefile/docker-compose.yml index 1eff7b73..7c8d84f8 100644 --- a/tests/fixtures/sleeps-composefile/docker-compose.yml +++ b/tests/fixtures/sleeps-composefile/docker-compose.yml @@ -1,5 +1,5 @@ -version: 2 +version: "2" services: simple: diff --git a/tests/fixtures/v2-full/docker-compose.yml b/tests/fixtures/v2-full/docker-compose.yml index 725296c9..a973dd0c 100644 --- a/tests/fixtures/v2-full/docker-compose.yml +++ b/tests/fixtures/v2-full/docker-compose.yml @@ -1,5 +1,5 @@ -version: 2 +version: "2" volumes: data: diff --git a/tests/fixtures/v2-simple/docker-compose.yml b/tests/fixtures/v2-simple/docker-compose.yml index 12a9de72..c99ae02f 100644 --- a/tests/fixtures/v2-simple/docker-compose.yml +++ b/tests/fixtures/v2-simple/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/v2-simple/links-invalid.yml b/tests/fixtures/v2-simple/links-invalid.yml index 422f9314..481aa404 100644 --- a/tests/fixtures/v2-simple/links-invalid.yml +++ b/tests/fixtures/v2-simple/links-invalid.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 0c8c9a6a..180c9df1 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -10,6 +10,7 @@ from docker.errors import NotFound from .testcases import DockerClientTestCase from compose.config import config from compose.config import ConfigurationError +from compose.config.config import V2_0 from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT @@ -112,7 +113,7 @@ class ProjectTest(DockerClientTestCase): name='composetest', client=self.client, config_data=build_service_dicts({ - 'version': 2, + 'version': V2_0, 'services': { 'net': { 'image': 'busybox:latest', @@ -139,7 +140,7 @@ class ProjectTest(DockerClientTestCase): return Project.from_config( name='composetest', config_data=build_service_dicts({ - 'version': 2, + 'version': V2_0, 'services': { 'web': { 'image': 'busybox:latest', @@ -559,7 +560,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_project_up_networks(self): config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -592,7 +593,7 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_up_with_ipam_config(self): config_data = config.Config( - version=2, + version=V2_0, services=[], volumes={}, networks={ @@ -651,7 +652,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -677,7 +678,7 @@ class ProjectTest(DockerClientTestCase): base_file = config.ConfigFile( 'base.yml', { - 'version': 2, + 'version': V2_0, 'services': { 'simple': {'image': 'busybox:latest', 'command': 'top'}, 'another': { @@ -696,7 +697,7 @@ class ProjectTest(DockerClientTestCase): override_file = config.ConfigFile( 'override.yml', { - 'version': 2, + 'version': V2_0, 'services': { 'another': { 'logging': { @@ -729,7 +730,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -754,7 +755,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -779,7 +780,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -802,7 +803,7 @@ class ProjectTest(DockerClientTestCase): full_vol_name = 'composetest_{0}'.format(vol_name) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -841,7 +842,7 @@ class ProjectTest(DockerClientTestCase): full_vol_name = 'composetest_{0}'.format(vol_name) self.client.create_volume(vol_name) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -866,7 +867,7 @@ class ProjectTest(DockerClientTestCase): vol_name = '{0:x}'.format(random.getrandbits(32)) config_data = config.Config( - version=2, + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -895,7 +896,7 @@ class ProjectTest(DockerClientTestCase): base_file = config.ConfigFile( 'base.yml', { - 'version': 2, + 'version': V2_0, 'services': { 'simple': { 'image': 'busybox:latest', diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 5870946d..8e2f2593 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -10,6 +10,8 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment +from compose.config.config import V1 +from compose.config.config import V2_0 from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output @@ -54,9 +56,9 @@ class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): if engine_version_too_low_for_v2(): - version = API_VERSIONS[1] + version = API_VERSIONS[V1] else: - version = API_VERSIONS[2] + version = API_VERSIONS[V2_0] cls.client = docker_client(version) diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 44f5c684..302f4703 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -14,14 +14,15 @@ import pytest from compose.config import config from compose.config.config import resolve_build_args from compose.config.config import resolve_environment +from compose.config.config import V1 +from compose.config.config import V2_0 from compose.config.errors import ConfigurationError from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from tests import mock from tests import unittest -DEFAULT_VERSION = V2 = 2 -V1 = 1 +DEFAULT_VERSION = V2_0 def make_service_dict(name, service_dict, working_dir, filename=None): @@ -78,7 +79,7 @@ class ConfigTest(unittest.TestCase): def test_load_v2(self): config_data = config.load( build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'foo': {'image': 'busybox'}, 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, @@ -143,9 +144,55 @@ class ConfigTest(unittest.TestCase): } }) + def test_valid_versions(self): + for version in ['2', '2.0']: + cfg = config.load(build_config_details({'version': version})) + assert cfg.version == V2_0 + + def test_v1_file_version(self): + cfg = config.load(build_config_details({'web': {'image': 'busybox'}})) + assert cfg.version == V1 + assert list(s['name'] for s in cfg.services) == ['web'] + + cfg = config.load(build_config_details({'version': {'image': 'busybox'}})) + assert cfg.version == V1 + assert list(s['name'] for s in cfg.services) == ['version'] + + def test_wrong_version_type(self): + for version in [None, 2, 2.0]: + with pytest.raises(ConfigurationError): + config.load( + build_config_details( + {'version': version}, + filename='filename.yml', + ) + ) + + def test_unsupported_version(self): + with pytest.raises(ConfigurationError): + config.load( + build_config_details( + {'version': '2.1'}, + filename='filename.yml', + ) + ) + + def test_v1_file_with_version_is_invalid(self): + for version in [1, "1"]: + with pytest.raises(ConfigurationError): + config.load( + build_config_details( + { + 'version': version, + 'web': {'image': 'busybox'}, + }, + filename='filename.yml', + ) + ) + def test_named_volume_config_empty(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'simple': {'image': 'busybox'} }, @@ -214,7 +261,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaises(ConfigurationError): config.load( build_config_details( - {'version': 2, 'services': {'web': 'busybox:latest'}}, + {'version': '2', 'services': {'web': 'busybox:latest'}}, 'working_dir', 'filename.yml' ) @@ -224,7 +271,7 @@ class ConfigTest(unittest.TestCase): with self.assertRaises(ConfigurationError): config.load( build_config_details({ - 'version': 2, + 'version': '2', 'services': {'web': 'busybox:latest'}, 'networks': { 'invalid': {'foo', 'bar'} @@ -246,7 +293,7 @@ class ConfigTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: config.load( build_config_details({ - 'version': 2, + 'version': '2', 'services': {invalid_name: {'image': 'busybox'}} }, 'working_dir', 'filename.yml') ) @@ -256,7 +303,7 @@ class ConfigTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( { - 'version': 2, + 'version': '2', 'services': { 'web': {'image': 'busybox', 'name': 'bogus'}, } @@ -307,7 +354,7 @@ class ConfigTest(unittest.TestCase): config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': {1: {'image': 'busybox'}} }, 'working_dir', @@ -370,7 +417,7 @@ class ConfigTest(unittest.TestCase): def test_load_with_multiple_files_and_empty_override_v2(self): base_file = config.ConfigFile( 'base.yml', - {'version': 2, 'services': {'web': {'image': 'example/web'}}}) + {'version': '2', 'services': {'web': {'image': 'example/web'}}}) override_file = config.ConfigFile('override.yml', None) details = config.ConfigDetails('.', [base_file, override_file]) @@ -394,7 +441,7 @@ class ConfigTest(unittest.TestCase): base_file = config.ConfigFile('base.yml', None) override_file = config.ConfigFile( 'override.tml', - {'version': 2, 'services': {'web': {'image': 'example/web'}}} + {'version': '2', 'services': {'web': {'image': 'example/web'}}} ) details = config.ConfigDetails('.', [base_file, override_file]) with pytest.raises(ConfigurationError) as exc: @@ -494,7 +541,7 @@ class ConfigTest(unittest.TestCase): config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': '.', @@ -509,7 +556,7 @@ class ConfigTest(unittest.TestCase): service = config.load( build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': '.' @@ -522,7 +569,7 @@ class ConfigTest(unittest.TestCase): service = config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': { @@ -543,7 +590,7 @@ class ConfigTest(unittest.TestCase): base_file = config.ConfigFile( 'base.yaml', { - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'example/web', @@ -556,7 +603,7 @@ class ConfigTest(unittest.TestCase): override_file = config.ConfigFile( 'override.yaml', { - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': '/', @@ -585,7 +632,7 @@ class ConfigTest(unittest.TestCase): base_file = config.ConfigFile( 'base.yaml', { - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox:latest', @@ -601,7 +648,7 @@ class ConfigTest(unittest.TestCase): base_file = config.ConfigFile( 'base.yaml', { - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox:latest', @@ -681,7 +728,7 @@ class ConfigTest(unittest.TestCase): config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': { 'foo': {'image': 1}, }, @@ -1016,7 +1063,7 @@ class ConfigTest(unittest.TestCase): def test_external_volume_config(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'bogus': {'image': 'busybox'} }, @@ -1034,7 +1081,7 @@ class ConfigTest(unittest.TestCase): def test_external_volume_invalid_config(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'bogus': {'image': 'busybox'} }, @@ -1047,7 +1094,7 @@ class ConfigTest(unittest.TestCase): def test_depends_on_orders_services(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'one': {'image': 'busybox', 'depends_on': ['three', 'two']}, 'two': {'image': 'busybox', 'depends_on': ['three']}, @@ -1062,7 +1109,7 @@ class ConfigTest(unittest.TestCase): def test_depends_on_unknown_service_errors(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'one': {'image': 'busybox', 'depends_on': ['three']}, }, @@ -1075,7 +1122,7 @@ class ConfigTest(unittest.TestCase): class NetworkModeTest(unittest.TestCase): def test_network_mode_standard(self): config_data = config.load(build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox', @@ -1101,7 +1148,7 @@ class NetworkModeTest(unittest.TestCase): def test_network_mode_container(self): config_data = config.load(build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox', @@ -1126,7 +1173,7 @@ class NetworkModeTest(unittest.TestCase): def test_network_mode_service(self): config_data = config.load(build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox', @@ -1160,7 +1207,7 @@ class NetworkModeTest(unittest.TestCase): def test_network_mode_service_nonexistent(self): with pytest.raises(ConfigurationError) as excinfo: config.load(build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox', @@ -1175,7 +1222,7 @@ class NetworkModeTest(unittest.TestCase): def test_network_mode_plus_networks_is_invalid(self): with pytest.raises(ConfigurationError) as excinfo: config.load(build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'busybox', @@ -2202,7 +2249,7 @@ class ExtendsTest(unittest.TestCase): tmpdir = py.test.ensuretemp('test_extends_with_mixed_version') self.addCleanup(tmpdir.remove) tmpdir.join('docker-compose.yml').write(""" - version: 2 + version: "2" services: web: extends: @@ -2224,7 +2271,7 @@ class ExtendsTest(unittest.TestCase): tmpdir = py.test.ensuretemp('test_extends_with_defined_version') self.addCleanup(tmpdir.remove) tmpdir.join('docker-compose.yml').write(""" - version: 2 + version: "2" services: web: extends: @@ -2233,7 +2280,7 @@ class ExtendsTest(unittest.TestCase): image: busybox """) tmpdir.join('base.yml').write(""" - version: 2 + version: "2" services: base: volumes: ['/foo'] diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 50b7efcb..c741a339 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -3,13 +3,13 @@ from __future__ import unicode_literals import pytest +from compose.config.config import V1 +from compose.config.config import V2_0 from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM -from tests.unit.config.config_test import V1 -from tests.unit.config.config_test import V2 def test_parse_extra_hosts_list(): @@ -91,26 +91,26 @@ class TestVolumesFromSpec(object): VolumeFromSpec.parse('unknown:format:ro', self.services, V1) def test_parse_v2_from_service(self): - volume_from = VolumeFromSpec.parse('servicea', self.services, V2) + volume_from = VolumeFromSpec.parse('servicea', self.services, V2_0) assert volume_from == VolumeFromSpec('servicea', 'rw', 'service') def test_parse_v2_from_service_with_mode(self): - volume_from = VolumeFromSpec.parse('servicea:ro', self.services, V2) + volume_from = VolumeFromSpec.parse('servicea:ro', self.services, V2_0) assert volume_from == VolumeFromSpec('servicea', 'ro', 'service') def test_parse_v2_from_container(self): - volume_from = VolumeFromSpec.parse('container:foo', self.services, V2) + volume_from = VolumeFromSpec.parse('container:foo', self.services, V2_0) assert volume_from == VolumeFromSpec('foo', 'rw', 'container') def test_parse_v2_from_container_with_mode(self): - volume_from = VolumeFromSpec.parse('container:foo:ro', self.services, V2) + volume_from = VolumeFromSpec.parse('container:foo:ro', self.services, V2_0) assert volume_from == VolumeFromSpec('foo', 'ro', 'container') def test_parse_v2_invalid_type(self): with pytest.raises(ConfigurationError) as exc: - VolumeFromSpec.parse('bogus:foo:ro', self.services, V2) + VolumeFromSpec.parse('bogus:foo:ro', self.services, V2_0) assert "Unknown volumes_from type 'bogus'" in exc.exconly() def test_parse_v2_invalid(self): with pytest.raises(ConfigurationError): - VolumeFromSpec.parse('unknown:format:ro', self.services, V2) + VolumeFromSpec.parse('unknown:format:ro', self.services, V2_0) From f081376067b6fc1cb3e8fc52f52ff9148cab919a Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 Jan 2016 16:15:16 +0000 Subject: [PATCH 337/359] Improve error messages for invalid versions Signed-off-by: Aanand Prasad --- compose/config/config.py | 23 ++++++++-- compose/config/errors.py | 8 ++++ compose/config/validation.py | 8 +++- compose/const.py | 1 - tests/unit/config/config_test.py | 73 +++++++++++++++++--------------- 5 files changed, 71 insertions(+), 42 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index bda086b9..094c8b3a 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -16,10 +16,10 @@ from cached_property import cached_property from ..const import COMPOSEFILE_V1 as V1 from ..const import COMPOSEFILE_V2_0 as V2_0 -from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError +from .errors import VERSION_EXPLANATION from .interpolation import interpolate_environment_variables from .sort_services import get_container_name_from_network_mode from .sort_services import get_service_name_from_network_mode @@ -105,6 +105,7 @@ SUPPORTED_FILENAMES = [ DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' + log = logging.getLogger(__name__) @@ -131,7 +132,10 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): @cached_property def version(self): - version = self.config.get('version', V1) + if 'version' not in self.config: + return V1 + + version = self.config['version'] if isinstance(version, dict): log.warn("Unexpected type for field 'version', in file {} assuming " @@ -139,12 +143,23 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): "Compose file version 1".format(self.filename)) return V1 + if not isinstance(version, six.string_types): + raise ConfigurationError( + 'Version in "{}" is invalid - it should be a string.' + .format(self.filename)) + + if version == '1': + raise ConfigurationError( + 'Version in "{}" is invalid. {}' + .format(self.filename, VERSION_EXPLANATION)) + if version == '2': version = V2_0 - if version not in COMPOSEFILE_VERSIONS: + if version != V2_0: raise ConfigurationError( - 'Invalid Compose file version: {0}'.format(version)) + 'Version in "{}" is unsupported. {}' + .format(self.filename, VERSION_EXPLANATION)) return version diff --git a/compose/config/errors.py b/compose/config/errors.py index 99129f3d..f94ac7ac 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -2,6 +2,14 @@ from __future__ import absolute_import from __future__ import unicode_literals +VERSION_EXPLANATION = ( + 'Either specify a version of "2" (or "2.0") and place your service ' + 'definitions under the `services` key, or omit the `version` key and place ' + 'your service definitions at the root of the file to use version 1.\n' + 'For more on the Compose file format versions, see ' + 'https://docs.docker.com/compose/compose-file/') + + class ConfigurationError(Exception): def __init__(self, msg): self.msg = msg diff --git a/compose/config/validation.py b/compose/config/validation.py index ba1ac521..6b240135 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -15,6 +15,7 @@ from jsonschema import RefResolver from jsonschema import ValidationError from .errors import ConfigurationError +from .errors import VERSION_EXPLANATION from .sort_services import get_service_name_from_network_mode @@ -229,11 +230,14 @@ def handle_error_for_schema_with_id(error, path): "A service can either be built to image or use an existing " "image, not both.".format(path_string(path))) - if schema_id == '#/definitions/service': - if error.validator == 'additionalProperties': + if error.validator == 'additionalProperties': + if schema_id == '#/definitions/service': invalid_config_key = parse_key_from_error_msg(error) return get_unsupported_config_msg(path, invalid_config_key) + if not error.path: + return '{}\n{}'.format(error.message, VERSION_EXPLANATION) + def handle_generic_service_error(error, path): msg_format = None diff --git a/compose/const.py b/compose/const.py index d78a5fa7..0e307835 100644 --- a/compose/const.py +++ b/compose/const.py @@ -17,7 +17,6 @@ LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' COMPOSEFILE_V1 = '1' COMPOSEFILE_V2_0 = '2.0' -COMPOSEFILE_VERSIONS = (COMPOSEFILE_V1, COMPOSEFILE_V2_0) API_VERSIONS = { COMPOSEFILE_V1: '1.21', diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 302f4703..3c012ea7 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -17,6 +17,7 @@ from compose.config.config import resolve_environment from compose.config.config import V1 from compose.config.config import V2_0 from compose.config.errors import ConfigurationError +from compose.config.errors import VERSION_EXPLANATION from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM from tests import mock @@ -159,8 +160,8 @@ class ConfigTest(unittest.TestCase): assert list(s['name'] for s in cfg.services) == ['version'] def test_wrong_version_type(self): - for version in [None, 2, 2.0]: - with pytest.raises(ConfigurationError): + for version in [None, 1, 2, 2.0]: + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {'version': version}, @@ -168,8 +169,11 @@ class ConfigTest(unittest.TestCase): ) ) + assert 'Version in "filename.yml" is invalid - it should be a string.' \ + in excinfo.exconly() + def test_unsupported_version(self): - with pytest.raises(ConfigurationError): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {'version': '2.1'}, @@ -177,18 +181,38 @@ class ConfigTest(unittest.TestCase): ) ) - def test_v1_file_with_version_is_invalid(self): - for version in [1, "1"]: - with pytest.raises(ConfigurationError): - config.load( - build_config_details( - { - 'version': version, - 'web': {'image': 'busybox'}, - }, - filename='filename.yml', - ) + assert 'Version in "filename.yml" is unsupported' in excinfo.exconly() + assert VERSION_EXPLANATION in excinfo.exconly() + + def test_version_1_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': '1', + 'web': {'image': 'busybox'}, + }, + filename='filename.yml', ) + ) + + assert 'Version in "filename.yml" is invalid' in excinfo.exconly() + assert VERSION_EXPLANATION in excinfo.exconly() + + def test_v1_file_with_version_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': '2', + 'web': {'image': 'busybox'}, + }, + filename='filename.yml', + ) + ) + + assert 'Additional properties are not allowed' in excinfo.exconly() + assert VERSION_EXPLANATION in excinfo.exconly() def test_named_volume_config_empty(self): config_details = build_config_details({ @@ -226,27 +250,6 @@ class ConfigTest(unittest.TestCase): ]) ) - def test_load_invalid_version(self): - with self.assertRaises(ConfigurationError): - config.load( - build_config_details({ - 'version': 18, - 'services': { - 'foo': {'image': 'busybox'} - } - }, 'working_dir', 'filename.yml') - ) - - with self.assertRaises(ConfigurationError): - config.load( - build_config_details({ - 'version': 'two point oh', - 'services': { - 'foo': {'image': 'busybox'} - } - }, 'working_dir', 'filename.yml') - ) - def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( From 8024f2f09e2969d6e359d62d2dc6488f25a8e248 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Fri, 29 Jan 2016 16:38:23 +0000 Subject: [PATCH 338/359] Tweak and test warning shown when version is a dict Signed-off-by: Aanand Prasad --- compose/config/config.py | 6 +++--- tests/unit/config/config_test.py | 19 ++++++++++++------- 2 files changed, 15 insertions(+), 10 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 094c8b3a..be680503 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -138,9 +138,9 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): version = self.config['version'] if isinstance(version, dict): - log.warn("Unexpected type for field 'version', in file {} assuming " - "version is the name of a service, and defaulting to " - "Compose file version 1".format(self.filename)) + log.warn('Unexpected type for "version" key in "{}". Assuming ' + '"version" is the name of a service, and defaulting to ' + 'Compose file version 1.'.format(self.filename)) return V1 if not isinstance(version, six.string_types): diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 3c012ea7..af256f20 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -232,13 +232,18 @@ class ConfigTest(unittest.TestCase): assert volumes['other'] == {} def test_load_service_with_name_version(self): - config_data = config.load( - build_config_details({ - 'version': { - 'image': 'busybox' - } - }, 'working_dir', 'filename.yml') - ) + with mock.patch('compose.config.config.log') as mock_logging: + config_data = config.load( + build_config_details({ + 'version': { + 'image': 'busybox' + } + }, 'working_dir', 'filename.yml') + ) + + assert 'Unexpected type for "version" key in "filename.yml"' \ + in mock_logging.warn.call_args[0][0] + service_dicts = config_data.services self.assertEqual( service_sort(service_dicts), From 0c87e0b18fa354ddfe1543f5a2391db375fe80be Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 14:41:18 -0500 Subject: [PATCH 339/359] Refactor project network initlization. Signed-off-by: Daniel Nephin --- compose/network.py | 64 ++++++++++++++++++++++++++ compose/project.py | 93 ++++++++------------------------------ tests/unit/project_test.py | 8 ++-- 3 files changed, 88 insertions(+), 77 deletions(-) diff --git a/compose/network.py b/compose/network.py index 4f4f5522..4f4e06b2 100644 --- a/compose/network.py +++ b/compose/network.py @@ -104,3 +104,67 @@ def create_ipam_config_from_dict(ipam_dict): for config in ipam_dict.get('config', []) ], ) + + +def build_networks(name, config_data, client): + network_config = config_data.networks or {} + networks = { + network_name: Network( + client=client, project=name, name=network_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + ipam=data.get('ipam'), + external_name=data.get('external_name'), + ) + for network_name, data in network_config.items() + } + + if 'default' not in networks: + networks['default'] = Network(client, name, 'default') + + return networks + + +class ProjectNetworks(object): + + def __init__(self, networks, use_networking): + self.networks = networks or {} + self.use_networking = use_networking + + @classmethod + def from_services(cls, services, networks, use_networking): + networks = { + network: networks[network] + for service in services + for network in service.get('networks', ['default']) + } + return cls(networks, use_networking) + + def remove(self): + if not self.use_networking: + return + for network in self.networks.values(): + network.remove() + + def initialize(self): + if not self.use_networking: + return + + for network in self.networks.values(): + network.ensure() + + +def get_networks(service_dict, network_definitions): + if 'network_mode' in service_dict: + return [] + + networks = [] + for name in service_dict.pop('networks', ['default']): + network = network_definitions.get(name) + if network: + networks.append(network.full_name) + else: + raise ConfigurationError( + 'Service "{}" uses an undefined network "{}"' + .format(service_dict['name'], name)) + return networks diff --git a/compose/project.py b/compose/project.py index 6411f7cc..2e9cfb8f 100644 --- a/compose/project.py +++ b/compose/project.py @@ -19,7 +19,9 @@ from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .container import Container -from .network import Network +from .network import build_networks +from .network import get_networks +from .network import ProjectNetworks from .service import ContainerNetworkMode from .service import ConvergenceStrategy from .service import NetworkMode @@ -36,15 +38,12 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, networks=None, volumes=None, - use_networking=False, network_driver=None): + def __init__(self, name, services, client, networks=None, volumes=None): self.name = name self.services = services self.client = client - self.use_networking = use_networking - self.network_driver = network_driver - self.networks = networks or [] self.volumes = volumes or {} + self.networks = networks or ProjectNetworks({}, False) def labels(self, one_off=False): return [ @@ -58,23 +57,12 @@ class Project(object): Construct a Project from a config.Config object. """ use_networking = (config_data.version and config_data.version != V1) - project = cls(name, [], client, use_networking=use_networking) - - network_config = config_data.networks or {} - custom_networks = [ - Network( - client=client, project=name, name=network_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - ipam=data.get('ipam'), - external_name=data.get('external_name'), - ) - for network_name, data in network_config.items() - ] - - all_networks = custom_networks[:] - if 'default' not in network_config: - all_networks.append(project.default_network) + networks = build_networks(name, config_data, client) + project_networks = ProjectNetworks.from_services( + config_data.services, + networks, + use_networking) + project = cls(name, [], client, project_networks) if config_data.volumes: for vol_name, data in config_data.volumes.items(): @@ -86,13 +74,15 @@ class Project(object): ) for service_dict in config_data.services: + service_dict = dict(service_dict) if use_networking: - networks = get_networks(service_dict, all_networks) + service_networks = get_networks(service_dict, networks) else: - networks = [] + service_networks = [] + service_dict.pop('networks', None) links = project.get_links(service_dict) - network_mode = project.get_network_mode(service_dict, networks) + network_mode = project.get_network_mode(service_dict, service_networks) volumes_from = get_volumes_from(project, service_dict) if config_data.version != V1: @@ -109,17 +99,13 @@ class Project(object): client=client, project=name, use_networking=use_networking, - networks=networks, + networks=service_networks, links=links, network_mode=network_mode, volumes_from=volumes_from, **service_dict) ) - project.networks += custom_networks - if 'default' not in network_config and project.uses_default_network(): - project.networks.append(project.default_network) - return project @property @@ -201,7 +187,7 @@ class Project(object): def get_network_mode(self, service_dict, networks): network_mode = service_dict.pop('network_mode', None) if not network_mode: - if self.use_networking: + if self.networks.use_networking: return NetworkMode(networks[0]) if networks else NetworkMode('none') return NetworkMode(None) @@ -285,7 +271,7 @@ class Project(object): def down(self, remove_image_type, include_volumes): self.stop() self.remove_stopped(v=include_volumes) - self.remove_networks() + self.networks.remove() if include_volumes: self.remove_volumes() @@ -296,33 +282,10 @@ class Project(object): for service in self.get_services(): service.remove_image(remove_image_type) - def remove_networks(self): - if not self.use_networking: - return - for network in self.networks: - network.remove() - def remove_volumes(self): for volume in self.volumes.values(): volume.remove() - def initialize_networks(self): - if not self.use_networking: - return - - for network in self.networks: - network.ensure() - - def uses_default_network(self): - return any( - self.default_network.full_name in service.networks - for service in self.services - ) - - @property - def default_network(self): - return Network(client=self.client, project=self.name, name='default') - def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -392,7 +355,7 @@ class Project(object): plans = self._get_convergence_plans(services, strategy) - self.initialize_networks() + self.networks.initialize() self.initialize_volumes() return [ @@ -465,22 +428,6 @@ class Project(object): return acc + dep_services -def get_networks(service_dict, network_definitions): - if 'network_mode' in service_dict: - return [] - - networks = [] - for name in service_dict.pop('networks', ['default']): - matches = [n for n in network_definitions if n.name == name] - if matches: - networks.append(matches[0].full_name) - else: - raise ConfigurationError( - 'Service "{}" uses an undefined network "{}"' - .format(service_dict['name'], name)) - return networks - - def get_volumes_from(project, service_dict): volumes_from = service_dict.pop('volumes_from', None) if not volumes_from: diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index 21c6be47..bec238de 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -45,7 +45,7 @@ class ProjectTest(unittest.TestCase): self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') self.assertEqual(project.get_service('db').name, 'db') self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') - self.assertFalse(project.use_networking) + self.assertFalse(project.networks.use_networking) def test_from_config_v2(self): config = Config( @@ -65,7 +65,7 @@ class ProjectTest(unittest.TestCase): ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) - self.assertTrue(project.use_networking) + self.assertTrue(project.networks.use_networking) def test_get_service(self): web = Service( @@ -426,7 +426,7 @@ class ProjectTest(unittest.TestCase): ), ) - assert project.uses_default_network() + assert 'default' in project.networks.networks def test_uses_default_network_false(self): project = Project.from_config( @@ -446,7 +446,7 @@ class ProjectTest(unittest.TestCase): ), ) - assert not project.uses_default_network() + assert 'default' not in project.networks.networks def test_container_without_name(self): self.mock_client.containers.return_value = [ From 3e8a4a5dc3000cfb4b64950e1f26248508b48e95 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 16:00:50 -0500 Subject: [PATCH 340/359] Don't initialize networks that aren't used by any services. Signed-off-by: Daniel Nephin --- compose/cli/main.py | 2 +- compose/network.py | 11 ++++++++--- compose/project.py | 13 ++++++++----- tests/acceptance/cli_test.py | 16 ++++++++-------- tests/integration/project_test.py | 7 ++++++- 5 files changed, 31 insertions(+), 18 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 93c4729f..5121d5c3 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -693,7 +693,7 @@ def run_one_off_container(container_options, project, service, options): start_deps=True, strategy=ConvergenceStrategy.never) - project.initialize_networks() + project.initialize() container = service.create_container( quiet=True, diff --git a/compose/network.py b/compose/network.py index 4f4e06b2..36de4502 100644 --- a/compose/network.py +++ b/compose/network.py @@ -133,12 +133,17 @@ class ProjectNetworks(object): @classmethod def from_services(cls, services, networks, use_networking): - networks = { - network: networks[network] + service_networks = { + network: networks.get(network) for service in services for network in service.get('networks', ['default']) } - return cls(networks, use_networking) + unused = set(networks) - set(service_networks) - {'default'} + if unused: + log.warn( + "Some networks were defined but are not used by any service: " + "{}".format(", ".join(unused))) + return cls(service_networks, use_networking) def remove(self): if not self.use_networking: diff --git a/compose/project.py b/compose/project.py index 2e9cfb8f..291e32b1 100644 --- a/compose/project.py +++ b/compose/project.py @@ -351,13 +351,12 @@ class Project(object): timeout=DEFAULT_TIMEOUT, detached=False): - services = self.get_services_without_duplicate(service_names, include_deps=start_deps) + self.initialize() + services = self.get_services_without_duplicate( + service_names, + include_deps=start_deps) plans = self._get_convergence_plans(services, strategy) - - self.networks.initialize() - self.initialize_volumes() - return [ container for service in services @@ -369,6 +368,10 @@ class Project(object): ) ] + def initialize(self): + self.networks.initialize() + self.initialize_volumes() + def _get_convergence_plans(self, services, strategy): plans = {} diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 447b1e32..6b9a28e5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -406,7 +406,8 @@ class CLITestCase(DockerClientTestCase): services = self.project.get_services() - networks = self.client.networks(names=[self.project.default_network.full_name]) + network_name = self.project.networks.networks['default'] + networks = self.client.networks(names=[network_name]) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['Driver'], 'bridge') assert 'com.docker.network.bridge.enable_icc' not in networks[0]['Options'] @@ -439,7 +440,9 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['-f', filename, 'up', '-d'], None) - networks = self.client.networks(names=[self.project.default_network.full_name]) + network_name = self.project.networks.networks['default'] + networks = self.client.networks(names=[network_name]) + assert networks[0]['Options']['com.docker.network.bridge.enable_icc'] == 'false' @v2_only() @@ -586,18 +589,15 @@ class CLITestCase(DockerClientTestCase): n['Name'] for n in self.client.networks() if n['Name'].startswith('{}_'.format(self.project.name)) ] - - assert sorted(network_names) == [ - '{}_{}'.format(self.project.name, name) - for name in ['bar', 'foo'] - ] + assert network_names == [] def test_up_with_links_v1(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'web'], None) # No network was created - networks = self.client.networks(names=[self.project.default_network.full_name]) + network_name = self.project.networks.networks['default'] + networks = self.client.networks(names=[network_name]) assert networks == [] web = self.project.get_service('web') diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 180c9df1..79644e59 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -565,6 +565,7 @@ class ProjectTest(DockerClientTestCase): 'name': 'web', 'image': 'busybox:latest', 'command': 'top', + 'networks': ['foo', 'bar', 'baz'], }], volumes={}, networks={ @@ -594,7 +595,11 @@ class ProjectTest(DockerClientTestCase): def test_up_with_ipam_config(self): config_data = config.Config( version=V2_0, - services=[], + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': ['front'], + }], volumes={}, networks={ 'front': { From 3b1276bd4481247c5b6ce379bd6c8255a952b146 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 16:30:24 -0500 Subject: [PATCH 341/359] Include networks in the config_hash. Signed-off-by: Daniel Nephin --- compose/network.py | 10 +++++++--- compose/project.py | 1 + compose/service.py | 1 + tests/acceptance/cli_test.py | 6 +++--- tests/integration/project_test.py | 3 ++- tests/unit/service_test.py | 8 +++++--- 6 files changed, 19 insertions(+), 10 deletions(-) diff --git a/compose/network.py b/compose/network.py index 36de4502..82a78f3b 100644 --- a/compose/network.py +++ b/compose/network.py @@ -136,7 +136,7 @@ class ProjectNetworks(object): service_networks = { network: networks.get(network) for service in services - for network in service.get('networks', ['default']) + for network in get_network_names_for_service(service) } unused = set(networks) - set(service_networks) - {'default'} if unused: @@ -159,12 +159,15 @@ class ProjectNetworks(object): network.ensure() -def get_networks(service_dict, network_definitions): +def get_network_names_for_service(service_dict): if 'network_mode' in service_dict: return [] + return service_dict.get('networks', ['default']) + +def get_networks(service_dict, network_definitions): networks = [] - for name in service_dict.pop('networks', ['default']): + for name in get_network_names_for_service(service_dict): network = network_definitions.get(name) if network: networks.append(network.full_name) @@ -172,4 +175,5 @@ def get_networks(service_dict, network_definitions): raise ConfigurationError( 'Service "{}" uses an undefined network "{}"' .format(service_dict['name'], name)) + return networks diff --git a/compose/project.py b/compose/project.py index 291e32b1..a90ec2c3 100644 --- a/compose/project.py +++ b/compose/project.py @@ -96,6 +96,7 @@ class Project(object): project.services.append( Service( + service_dict.pop('name'), client=client, project=name, use_networking=use_networking, diff --git a/compose/service.py b/compose/service.py index 2ca0e64d..e6aad3ae 100644 --- a/compose/service.py +++ b/compose/service.py @@ -472,6 +472,7 @@ class Service(object): 'image_id': self.image()['Id'], 'links': self.get_link_names(), 'net': self.network_mode.id, + 'networks': self.networks, 'volumes_from': [ (v.source.name, v.mode) for v in self.volumes_from if isinstance(v.source, Service) diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 6b9a28e5..032900d5 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -406,7 +406,7 @@ class CLITestCase(DockerClientTestCase): services = self.project.get_services() - network_name = self.project.networks.networks['default'] + network_name = self.project.networks.networks['default'].full_name networks = self.client.networks(names=[network_name]) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['Driver'], 'bridge') @@ -440,7 +440,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['-f', filename, 'up', '-d'], None) - network_name = self.project.networks.networks['default'] + network_name = self.project.networks.networks['default'].full_name networks = self.client.networks(names=[network_name]) assert networks[0]['Options']['com.docker.network.bridge.enable_icc'] == 'false' @@ -596,7 +596,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d', 'web'], None) # No network was created - network_name = self.project.networks.networks['default'] + network_name = self.project.networks.networks['default'].full_name networks = self.client.networks(names=[network_name]) assert networks == [] diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 79644e59..45bae2c3 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -832,7 +832,8 @@ class ProjectTest(DockerClientTestCase): ) project = Project.from_config( name='composetest', - config_data=config_data, client=self.client + config_data=config_data, + client=self.client ) with self.assertRaises(config.ConfigurationError) as e: project.initialize_volumes() diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 74e9f0f5..f34de3bf 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -266,7 +266,7 @@ class ServiceTest(unittest.TestCase): self.assertEqual( opts['labels'][LABEL_CONFIG_HASH], - '3c85881a8903b9d73a06c41860c8be08acce1494ab4cf8408375966dccd714de') + 'f8bfa1058ad1f4231372a0b1639f0dfdb574dafff4e8d7938049ae993f7cf1fc') self.assertEqual( opts['environment'], { @@ -417,9 +417,10 @@ class ServiceTest(unittest.TestCase): 'options': {'image': 'example.com/foo'}, 'links': [('one', 'one')], 'net': 'other', + 'networks': [], 'volumes_from': [('two', 'rw')], } - self.assertEqual(config_dict, expected) + assert config_dict == expected def test_config_dict_with_network_mode_from_container(self): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} @@ -437,10 +438,11 @@ class ServiceTest(unittest.TestCase): 'image_id': 'abcd', 'options': {'image': 'example.com/foo'}, 'links': [], + 'networks': [], 'net': 'aaabbb', 'volumes_from': [], } - self.assertEqual(config_dict, expected) + assert config_dict == expected def test_remove_image_none(self): web = Service('web', image='example', client=self.mock_client) From 61d00ebee4229dc1ead4721c5d4501eb1a86a297 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 17:31:27 -0500 Subject: [PATCH 342/359] Extract volume init and removal from project. Signed-off-by: Daniel Nephin --- compose/project.py | 72 +++++-------------------------- compose/volume.py | 70 ++++++++++++++++++++++++++++++ tests/integration/project_test.py | 12 +++--- 3 files changed, 86 insertions(+), 68 deletions(-) diff --git a/compose/project.py b/compose/project.py index a90ec2c3..62e1d2cd 100644 --- a/compose/project.py +++ b/compose/project.py @@ -6,7 +6,6 @@ import logging from functools import reduce from docker.errors import APIError -from docker.errors import NotFound from . import parallel from .config import ConfigurationError @@ -28,7 +27,7 @@ from .service import NetworkMode from .service import Service from .service import ServiceNetworkMode from .utils import microseconds_from_time_nano -from .volume import Volume +from .volume import ProjectVolumes log = logging.getLogger(__name__) @@ -42,7 +41,7 @@ class Project(object): self.name = name self.services = services self.client = client - self.volumes = volumes or {} + self.volumes = volumes or ProjectVolumes({}) self.networks = networks or ProjectNetworks({}, False) def labels(self, one_off=False): @@ -62,16 +61,8 @@ class Project(object): config_data.services, networks, use_networking) - project = cls(name, [], client, project_networks) - - if config_data.volumes: - for vol_name, data in config_data.volumes.items(): - project.volumes[vol_name] = Volume( - client=client, project=name, name=vol_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - external_name=data.get('external_name') - ) + volumes = ProjectVolumes.from_config(name, config_data, client) + project = cls(name, [], client, project_networks, volumes) for service_dict in config_data.services: service_dict = dict(service_dict) @@ -86,13 +77,10 @@ class Project(object): volumes_from = get_volumes_from(project, service_dict) if config_data.version != V1: - service_volumes = service_dict.get('volumes', []) - for volume_spec in service_volumes: - if volume_spec.is_named_volume: - declared_volume = project.volumes[volume_spec.external] - service_volumes[service_volumes.index(volume_spec)] = ( - volume_spec._replace(external=declared_volume.full_name) - ) + service_dict['volumes'] = [ + volumes.namespace_spec(volume_spec) + for volume_spec in service_dict.get('volumes', []) + ] project.services.append( Service( @@ -233,49 +221,13 @@ class Project(object): def remove_stopped(self, service_names=None, **options): parallel.parallel_remove(self.containers(service_names, stopped=True), options) - def initialize_volumes(self): - try: - for volume in self.volumes.values(): - if volume.external: - log.debug( - 'Volume {0} declared as external. No new ' - 'volume will be created.'.format(volume.name) - ) - if not volume.exists(): - raise ConfigurationError( - 'Volume {name} declared as external, but could' - ' not be found. Please create the volume manually' - ' using `{command}{name}` and try again.'.format( - name=volume.full_name, - command='docker volume create --name=' - ) - ) - continue - volume.create() - except NotFound: - raise ConfigurationError( - 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) - ) - except APIError as e: - if 'Choose a different volume name' in str(e): - raise ConfigurationError( - 'Configuration for volume {0} specifies driver {1}, but ' - 'a volume with the same name uses a different driver ' - '({3}). If you wish to use the new configuration, please ' - 'remove the existing volume "{2}" first:\n' - '$ docker volume rm {2}'.format( - volume.name, volume.driver, volume.full_name, - volume.inspect()['Driver'] - ) - ) - def down(self, remove_image_type, include_volumes): self.stop() self.remove_stopped(v=include_volumes) self.networks.remove() if include_volumes: - self.remove_volumes() + self.volumes.remove() self.remove_images(remove_image_type) @@ -283,10 +235,6 @@ class Project(object): for service in self.get_services(): service.remove_image(remove_image_type) - def remove_volumes(self): - for volume in self.volumes.values(): - volume.remove() - def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -371,7 +319,7 @@ class Project(object): def initialize(self): self.networks.initialize() - self.initialize_volumes() + self.volumes.initialize() def _get_convergence_plans(self, services, strategy): plans = {} diff --git a/compose/volume.py b/compose/volume.py index 469e406a..2713fd32 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -3,8 +3,10 @@ from __future__ import unicode_literals import logging +from docker.errors import APIError from docker.errors import NotFound +from .config import ConfigurationError log = logging.getLogger(__name__) @@ -50,3 +52,71 @@ class Volume(object): if self.external_name: return self.external_name return '{0}_{1}'.format(self.project, self.name) + + +class ProjectVolumes(object): + + def __init__(self, volumes): + self.volumes = volumes + + @classmethod + def from_config(cls, name, config_data, client): + config_volumes = config_data.volumes or {} + volumes = { + vol_name: Volume( + client=client, + project=name, + name=vol_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external_name=data.get('external_name')) + for vol_name, data in config_volumes.items() + } + return cls(volumes) + + def remove(self): + for volume in self.volumes.values(): + volume.remove() + + def initialize(self): + try: + for volume in self.volumes.values(): + if volume.external: + log.debug( + 'Volume {0} declared as external. No new ' + 'volume will be created.'.format(volume.name) + ) + if not volume.exists(): + raise ConfigurationError( + 'Volume {name} declared as external, but could' + ' not be found. Please create the volume manually' + ' using `{command}{name}` and try again.'.format( + name=volume.full_name, + command='docker volume create --name=' + ) + ) + continue + volume.create() + except NotFound: + raise ConfigurationError( + 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) + ) + except APIError as e: + if 'Choose a different volume name' in str(e): + raise ConfigurationError( + 'Configuration for volume {0} specifies driver {1}, but ' + 'a volume with the same name uses a different driver ' + '({3}). If you wish to use the new configuration, please ' + 'remove the existing volume "{2}" first:\n' + '$ docker volume rm {2}'.format( + volume.name, volume.driver, volume.full_name, + volume.inspect()['Driver'] + ) + ) + + def namespace_spec(self, volume_spec): + if not volume_spec.is_named_volume: + return volume_spec + + volume = self.volumes[volume_spec.external] + return volume_spec._replace(external=volume.full_name) diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 45bae2c3..6bb076a3 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -749,7 +749,7 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data, client=self.client ) - project.initialize_volumes() + project.volumes.initialize() volume_data = self.client.inspect_volume(full_vol_name) self.assertEqual(volume_data['Name'], full_vol_name) @@ -800,7 +800,7 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, client=self.client ) with self.assertRaises(config.ConfigurationError): - project.initialize_volumes() + project.volumes.initialize() @v2_only() def test_initialize_volumes_updated_driver(self): @@ -821,7 +821,7 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data, client=self.client ) - project.initialize_volumes() + project.volumes.initialize() volume_data = self.client.inspect_volume(full_vol_name) self.assertEqual(volume_data['Name'], full_vol_name) @@ -836,7 +836,7 @@ class ProjectTest(DockerClientTestCase): client=self.client ) with self.assertRaises(config.ConfigurationError) as e: - project.initialize_volumes() + project.volumes.initialize() assert 'Configuration for volume {0} specifies driver smb'.format( vol_name ) in str(e.exception) @@ -863,7 +863,7 @@ class ProjectTest(DockerClientTestCase): name='composetest', config_data=config_data, client=self.client ) - project.initialize_volumes() + project.volumes.initialize() with self.assertRaises(NotFound): self.client.inspect_volume(full_vol_name) @@ -889,7 +889,7 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, client=self.client ) with self.assertRaises(config.ConfigurationError) as e: - project.initialize_volumes() + project.volumes.initialize() assert 'Volume {0} declared as external'.format( vol_name ) in str(e.exception) From 6b59ba0c31cf5e9a5adff5d4770064159c10c6dc Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 28 Jan 2016 18:50:05 -0500 Subject: [PATCH 343/359] Re-order compose docs so that quickstart guides come before other documentation. Signed-off-by: Daniel Nephin --- docs/extends.md | 2 +- docs/networking.md | 1 + docs/production.md | 2 +- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index c9b65db8..252ffdfc 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -5,7 +5,7 @@ description = "How to use Docker Compose's extends keyword to share configuratio keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] [menu.main] parent="workw_compose" -weight=2 +weight=20 +++ diff --git a/docs/networking.md b/docs/networking.md index fb34e3de..96cfa810 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -5,6 +5,7 @@ description = "How Compose sets up networking between containers" keywords = ["documentation, docs, docker, compose, orchestration, containers, networking"] [menu.main] parent="workw_compose" +weight=21 +++ diff --git a/docs/production.md b/docs/production.md index d51ca549..27c686dd 100644 --- a/docs/production.md +++ b/docs/production.md @@ -5,7 +5,7 @@ description = "Guide to using Docker Compose in production" keywords = ["documentation, docs, docker, compose, orchestration, containers, production"] [menu.main] parent="workw_compose" -weight=1 +weight=22 +++ From 009dbbe97116d6534c91c70e92eb2645c1b85f81 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 28 Jan 2016 18:53:40 -0500 Subject: [PATCH 344/359] Use the same capitalization for all menu items in the docs. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 2 +- docs/env.md | 2 +- docs/extends.md | 2 +- docs/production.md | 2 +- docs/reference/index.md | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index cc21bd8a..9bc72561 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -1,6 +1,6 @@ -# Introduction to the CLI - -This section describes the subcommands you can use with the `docker-compose` command. You can run subcommand against one or more services. To run against a specific service, you supply the service name from your compose configuration. If you do not specify the service name, the command runs against all the services in your configuration. - - -## Commands - -* [docker-compose Command](docker-compose.md) -* [CLI Reference](index.md) - - -## Environment Variables +# CLI Environment Variables Several environment variables are available for you to configure the Docker Compose command-line behaviour. Variables starting with `DOCKER_` are the same as those used to configure the Docker command-line client. If you're using `docker-machine`, then the `eval "$(docker-machine env my-docker-vm)"` command should set them to their correct values. (In this example, `my-docker-vm` is the name of a machine you created.) -### COMPOSE\_PROJECT\_NAME +## COMPOSE\_PROJECT\_NAME Sets the project name. This value is prepended along with the service name to the container container on start up. For example, if you project name is `myapp` and it includes two services `db` and `web` then compose starts containers named `myapp_db_1` and `myapp_web_1` respectively. @@ -36,14 +26,14 @@ Setting this is optional. If you do not set this, the `COMPOSE_PROJECT_NAME` defaults to the `basename` of the project directory. See also the `-p` [command-line option](docker-compose.md). -### COMPOSE\_FILE +## COMPOSE\_FILE Specify the file containing the compose configuration. If not provided, Compose looks for a file named `docker-compose.yml` in the current directory and then each parent directory in succession until a file by that name is found. See also the `-f` [command-line option](docker-compose.md). -### COMPOSE\_API\_VERSION +## COMPOSE\_API\_VERSION The Docker API only supports requests from clients which report a specific version. If you receive a `client and server don't have same version error` using @@ -63,20 +53,20 @@ If you run into problems running with this set, resolve the mismatch through upgrade and remove this setting to see if your problems resolve before notifying support. -### DOCKER\_HOST +## DOCKER\_HOST Sets the URL of the `docker` daemon. As with the Docker client, defaults to `unix:///var/run/docker.sock`. -### DOCKER\_TLS\_VERIFY +## DOCKER\_TLS\_VERIFY When set to anything other than an empty string, enables TLS communication with the `docker` daemon. -### DOCKER\_CERT\_PATH +## DOCKER\_CERT\_PATH Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TLS verification. Defaults to `~/.docker`. -### COMPOSE\_HTTP\_TIMEOUT +## COMPOSE\_HTTP\_TIMEOUT Configures the time (in seconds) a request to the Docker daemon is allowed to hang before Compose considers it failed. Defaults to 60 seconds. diff --git a/docs/reference/index.md b/docs/reference/index.md index d6fa2948..528550c7 100644 --- a/docs/reference/index.md +++ b/docs/reference/index.md @@ -14,6 +14,7 @@ weight=80 The following pages describe the usage information for the [docker-compose](docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. +* [docker-compose](docker-compose.md) * [build](build.md) * [config](config.md) * [create](create.md) @@ -37,5 +38,5 @@ The following pages describe the usage information for the [docker-compose](dock ## Where to go next -* [CLI environment variables](overview.md) +* [CLI environment variables](envvars.md) * [docker-compose Command](docker-compose.md) From 44c7d080bd5f6c3a0132034c90a11839466b4595 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 13:00:43 -0500 Subject: [PATCH 346/359] Rename the old environment variable page to link environment variables. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 3 ++- docs/{env.md => link-env-deprecated.md} | 5 +++-- 2 files changed, 5 insertions(+), 3 deletions(-) rename docs/{env.md => link-env-deprecated.md} (94%) diff --git a/docs/compose-file.md b/docs/compose-file.md index 9bc72561..e5326566 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -903,7 +903,8 @@ It's more complicated if you're using particular configuration features: syslog-address: "tcp://192.168.0.42:123" - `links` with environment variables: As documented in the - [environment variables reference](env.md), environment variables created by + [environment variables reference](link-env-deprecated.md), environment variables + created by links have been deprecated for some time. In the new Docker network system, they have been removed. You should either connect directly to the appropriate hostname or set the relevant environment variable yourself, diff --git a/docs/env.md b/docs/link-env-deprecated.md similarity index 94% rename from docs/env.md rename to docs/link-env-deprecated.md index f4eeedeb..55ba5f2d 100644 --- a/docs/env.md +++ b/docs/link-env-deprecated.md @@ -1,15 +1,16 @@ -# Compose environment variables reference +# Link environment variables reference > **Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. > From 7f009aeeb959b7962542d5674819c57c5130fd1d Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 1 Feb 2016 12:10:21 -0500 Subject: [PATCH 347/359] Move command reference to overview. Signed-off-by: Daniel Nephin --- docs/extends.md | 2 +- docs/faq.md | 2 +- docs/index.md | 2 +- docs/networking.md | 2 +- docs/overview.md | 2 +- docs/reference/envvars.md | 5 ++--- docs/reference/index.md | 6 +++--- docs/reference/{docker-compose.md => overview.md} | 8 ++++---- 8 files changed, 14 insertions(+), 15 deletions(-) rename docs/reference/{docker-compose.md => overview.md} (95%) diff --git a/docs/extends.md b/docs/extends.md index 95a3d64f..4067a4f0 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -42,7 +42,7 @@ are copied. To use multiple override files, or an override file with a different name, you can use the `-f` option to specify the list of files. Compose merges files in the order they're specified on the command line. See the [`docker-compose` -command reference](./reference/docker-compose.md) for more information about +command reference](./reference/overview.md) for more information about using `-f`. When you use multiple configuration files, you must make sure all paths in the diff --git a/docs/faq.md b/docs/faq.md index 3cb6a553..8fa629de 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -50,7 +50,7 @@ handling `SIGTERM` properly. Compose uses the project name to create unique identifiers for all of a project's containers and other resources. To run multiple copies of a project, set a custom project name using the [`-p` command line -option](./reference/docker-compose.md) or the [`COMPOSE_PROJECT_NAME` +option](./reference/overview.md) or the [`COMPOSE_PROJECT_NAME` environment variable](./reference/envvars.md#compose-project-name). ## What's the difference between `up`, `run`, and `start`? diff --git a/docs/index.md b/docs/index.md index 61bc41ee..f5d84218 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,6 +1,6 @@ -# docker-compose Command +# Overview of docker-compose CLI This page provides the usage information for the `docker-compose` Command. You can also see this information by running `docker-compose --help` from the @@ -114,4 +115,3 @@ envvars.md#compose-project-name) ## Where to go next * [CLI environment variables](envvars.md) -* [Command line reference](index.md) From 3b7471ae848fd6405b0272448a854df94699c0cf Mon Sep 17 00:00:00 2001 From: Ryan Taylor Long Date: Thu, 28 Jan 2016 06:19:03 +0000 Subject: [PATCH 348/359] Add depends_on to ALLOWED_KEYS This ensures and already existing `depends_on` is not deleted when the service on which it is defined also employs `extends`. Signed-off-by: Ryan Taylor Long --- compose/config/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/compose/config/config.py b/compose/config/config.py index be680503..7a8e5533 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -84,6 +84,7 @@ DOCKER_CONFIG_KEYS = [ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'build', 'container_name', + 'depends_on', 'dockerfile', 'expose', 'external_links', From aa5ff0546301c608e96e7eb73ef2aa081f5c1d93 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Fri, 29 Jan 2016 13:36:51 -0500 Subject: [PATCH 349/359] Fix merging of lists with multiple files. Signed-off-by: Daniel Nephin --- compose/config/config.py | 12 ++++++++---- tests/unit/config/config_test.py | 28 ++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 7a8e5533..07f62290 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -84,10 +84,7 @@ DOCKER_CONFIG_KEYS = [ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'build', 'container_name', - 'depends_on', 'dockerfile', - 'expose', - 'external_links', 'logging', ] @@ -666,7 +663,14 @@ def merge_service_dicts(base, override, version): for field in ['volumes', 'devices']: merge_field(field, merge_path_mappings) - for field in ['ports', 'expose', 'external_links']: + for field in [ + 'depends_on', + 'expose', + 'external_links', + 'links', + 'ports', + 'volumes_from', + ]: merge_field(field, operator.add, default=[]) for field in ['dns', 'dns_search', 'env_file']: diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index af256f20..f667b3bb 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -602,6 +602,7 @@ class ConfigTest(unittest.TestCase): 'services': { 'web': { 'image': 'example/web', + 'depends_on': ['db'], }, 'db': { 'image': 'example/db', @@ -616,7 +617,11 @@ class ConfigTest(unittest.TestCase): 'web': { 'build': '/', 'volumes': ['/home/user/project:/code'], + 'depends_on': ['other'], }, + 'other': { + 'image': 'example/other', + } } }) details = config.ConfigDetails('.', [base_file, override_file]) @@ -628,11 +633,16 @@ class ConfigTest(unittest.TestCase): 'build': {'context': os.path.abspath('/')}, 'image': 'example/web', 'volumes': [VolumeSpec.parse('/home/user/project:/code')], + 'depends_on': ['db', 'other'], }, { 'name': 'db', 'image': 'example/db', }, + { + 'name': 'other', + 'image': 'example/other', + }, ] assert service_sort(service_dicts) == service_sort(expected) @@ -2299,6 +2309,24 @@ class ExtendsTest(unittest.TestCase): service = load_from_filename(str(tmpdir.join('docker-compose.yml'))) self.assertEquals(service[0]['command'], "top") + def test_extends_with_depends_on(self): + tmpdir = py.test.ensuretemp('test_extends_with_defined_version') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: 2 + services: + base: + image: example + web: + extends: base + image: busybox + depends_on: ['other'] + other: + image: example + """) + services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + assert service_sort(services)[2]['depends_on'] == ['other'] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): From dc718eae65dda425cf04e8efc8cba74e494d5509 Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 1 Feb 2016 13:47:13 -0500 Subject: [PATCH 350/359] Make links unique-by-alias when merging Factor out MergeDict from merge_service_dicts to reduce complexity below limit. Signed-off-by: Daniel Nephin --- compose/config/config.py | 85 ++++++++++++++++++++++++++++------------ compose/config/types.py | 19 +++++++++ 2 files changed, 78 insertions(+), 26 deletions(-) diff --git a/compose/config/config.py b/compose/config/config.py index 07f62290..f362f1b8 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -26,6 +26,7 @@ from .sort_services import get_service_name_from_network_mode from .sort_services import sort_service_dicts from .types import parse_extra_hosts from .types import parse_restart_spec +from .types import ServiceLink from .types import VolumeFromSpec from .types import VolumeSpec from .validation import match_named_volumes @@ -641,51 +642,79 @@ def merge_service_dicts_from_files(base, override, version): return new_service +class MergeDict(dict): + """A dict-like object responsible for merging two dicts into one.""" + + def __init__(self, base, override): + self.base = base + self.override = override + + def needs_merge(self, field): + return field in self.base or field in self.override + + def merge_field(self, field, merge_func, default=None): + if not self.needs_merge(field): + return + + self[field] = merge_func( + self.base.get(field, default), + self.override.get(field, default)) + + def merge_mapping(self, field, parse_func): + if not self.needs_merge(field): + return + + self[field] = parse_func(self.base.get(field)) + self[field].update(parse_func(self.override.get(field))) + + def merge_sequence(self, field, parse_func): + def parse_sequence_func(seq): + return to_mapping((parse_func(item) for item in seq), 'merge_field') + + if not self.needs_merge(field): + return + + merged = parse_sequence_func(self.base.get(field, [])) + merged.update(parse_sequence_func(self.override.get(field, []))) + self[field] = [item.repr() for item in merged.values()] + + def merge_scalar(self, field): + if self.needs_merge(field): + self[field] = self.override.get(field, self.base.get(field)) + + def merge_service_dicts(base, override, version): - d = {} + md = MergeDict(base, override) - def merge_field(field, merge_func, default=None): - if field in base or field in override: - d[field] = merge_func( - base.get(field, default), - override.get(field, default)) - - def merge_mapping(mapping, parse_func): - if mapping in base or mapping in override: - merged = parse_func(base.get(mapping, None)) - merged.update(parse_func(override.get(mapping, None))) - d[mapping] = merged - - merge_mapping('environment', parse_environment) - merge_mapping('labels', parse_labels) - merge_mapping('ulimits', parse_ulimits) + md.merge_mapping('environment', parse_environment) + md.merge_mapping('labels', parse_labels) + md.merge_mapping('ulimits', parse_ulimits) + md.merge_sequence('links', ServiceLink.parse) for field in ['volumes', 'devices']: - merge_field(field, merge_path_mappings) + md.merge_field(field, merge_path_mappings) for field in [ 'depends_on', 'expose', 'external_links', - 'links', 'ports', 'volumes_from', ]: - merge_field(field, operator.add, default=[]) + md.merge_field(field, operator.add, default=[]) for field in ['dns', 'dns_search', 'env_file']: - merge_field(field, merge_list_or_string) + md.merge_field(field, merge_list_or_string) - for field in set(ALLOWED_KEYS) - set(d): - if field in base or field in override: - d[field] = override.get(field, base.get(field)) + for field in set(ALLOWED_KEYS) - set(md): + md.merge_scalar(field) if version == V1: - legacy_v1_merge_image_or_build(d, base, override) + legacy_v1_merge_image_or_build(md, base, override) else: - merge_build(d, base, override) + merge_build(md, base, override) - return d + return dict(md) def merge_build(output, base, override): @@ -919,6 +948,10 @@ def to_list(value): return value +def to_mapping(sequence, key_field): + return {getattr(item, key_field): item for item in sequence} + + def has_uppercase(name): return any(char in string.ascii_uppercase for char in name) diff --git a/compose/config/types.py b/compose/config/types.py index 9bda7180..fc3347c8 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -168,3 +168,22 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): @property def is_named_volume(self): return self.external and not self.external.startswith(('.', '/', '~')) + + +class ServiceLink(namedtuple('_ServiceLink', 'target alias')): + + @classmethod + def parse(cls, link_spec): + target, _, alias = link_spec.partition(':') + if not alias: + alias = target + return cls(target, alias) + + def repr(self): + if self.target == self.alias: + return self.target + return '{s.target}:{s.alias}'.format(s=self) + + @property + def merge_field(self): + return self.alias From 46f33f12b022fb9970e41029e3173e557b86f97c Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Mon, 1 Feb 2016 14:04:53 -0500 Subject: [PATCH 351/359] Update merge docs with depends_on, and correction about how links and volumes_from are merged. Signed-off-by: Daniel Nephin --- docs/extends.md | 27 ++++++++++----------------- tests/unit/config/config_test.py | 2 +- 2 files changed, 11 insertions(+), 18 deletions(-) diff --git a/docs/extends.md b/docs/extends.md index 4067a4f0..bceb0257 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -32,12 +32,9 @@ contains your base configuration. The override file, as its name implies, can contain configuration overrides for existing services or entirely new services. -If a service is defined in both files, Compose merges the configurations using -the same rules as the `extends` field (see [Adding and overriding -configuration](#adding-and-overriding-configuration)), with one exception. If a -service contains `links` or `volumes_from` those fields are copied over and -replace any values in the original service, in the same way single-valued fields -are copied. +If a service is defined in both files Compose merges the configurations using +the rules described in [Adding and overriding +configuration](#adding-and-overriding-configuration). To use multiple override files, or an override file with a different name, you can use the `-f` option to specify the list of files. Compose merges files in @@ -176,10 +173,12 @@ is useful if you have several services that reuse a common set of configuration options. Using `extends` you can define a common set of service options in one place and refer to it from anywhere. -> **Note:** `links` and `volumes_from` are never shared between services using -> `extends`. See -> [Adding and overriding configuration](#adding-and-overriding-configuration) - > for more information. +> **Note:** `links`, `volumes_from`, and `depends_on` are never shared between +> services using >`extends`. These exceptions exist to avoid +> implicit dependencies—you always define `links` and `volumes_from` +> locally. This ensures dependencies between services are clearly visible when +> reading the current file. Defining these locally also ensures changes to the +> referenced file don't result in breakage. ### Understand the extends configuration @@ -275,13 +274,7 @@ common configuration: ## Adding and overriding configuration -Compose copies configurations from the original service over to the local one, -**except** for `links` and `volumes_from`. These exceptions exist to avoid -implicit dependencies—you always define `links` and `volumes_from` -locally. This ensures dependencies between services are clearly visible when -reading the current file. Defining these locally also ensures changes to the -referenced file don't result in breakage. - +Compose copies configurations from the original service over to the local one. If a configuration option is defined in both the original service the local service, the local value *replaces* or *extends* the original value. diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index f667b3bb..8c9b73dc 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -2313,7 +2313,7 @@ class ExtendsTest(unittest.TestCase): tmpdir = py.test.ensuretemp('test_extends_with_defined_version') self.addCleanup(tmpdir.remove) tmpdir.join('docker-compose.yml').write(""" - version: 2 + version: "2" services: base: image: example From a59982eb1116df1e08ad0a1e807180719f2eb08c Mon Sep 17 00:00:00 2001 From: Mary Anthony Date: Tue, 2 Feb 2016 12:04:13 -0800 Subject: [PATCH 352/359] Fixing duplicate identifiers Signed-off-by: Mary Anthony --- docs/faq.md | 1 + docs/reference/down.md | 2 +- 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/faq.md b/docs/faq.md index 8fa629de..73596c18 100644 --- a/docs/faq.md +++ b/docs/faq.md @@ -4,6 +4,7 @@ title = "Frequently Asked Questions" description = "Docker Compose FAQ" keywords = "documentation, docs, docker, compose, faq" [menu.main] +identifier="faq.compose" parent="workw_compose" weight=90 +++ diff --git a/docs/reference/down.md b/docs/reference/down.md index 428e4e58..2495abea 100644 --- a/docs/reference/down.md +++ b/docs/reference/down.md @@ -4,7 +4,7 @@ title = "down" description = "down" keywords = ["fig, composition, compose, docker, orchestration, cli, down"] [menu.main] -identifier="build.compose" +identifier="down.compose" parent = "smn_compose_cli" +++ From 64336615cfc4b5cf690f426e313c1a2c0c6e7b0c Mon Sep 17 00:00:00 2001 From: Dimitar Bonev Date: Fri, 29 Jan 2016 14:15:38 +0200 Subject: [PATCH 353/359] Falling back to default project name when COMPOSE_PROJECT_NAME is set to empty Signed-off-by: Dimitar Bonev --- compose/cli/command.py | 2 +- tests/unit/cli_test.py | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/compose/cli/command.py b/compose/cli/command.py index c1681ffc..2a0d8698 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -92,7 +92,7 @@ def get_project_name(working_dir, project_name=None): return re.sub(r'[^a-z0-9]', '', name.lower()) project_name = project_name or os.environ.get('COMPOSE_PROJECT_NAME') - if project_name is not None: + if project_name: return normalize_name(project_name) project = os.path.basename(os.path.abspath(working_dir)) diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index fd52a3c1..fec7cdba 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -49,6 +49,13 @@ class CLITestCase(unittest.TestCase): project_name = get_project_name(None) self.assertEquals(project_name, name) + def test_project_name_with_empty_environment_var(self): + base_dir = 'tests/fixtures/simple-composefile' + with mock.patch.dict(os.environ): + os.environ['COMPOSE_PROJECT_NAME'] = '' + project_name = get_project_name(base_dir) + self.assertEquals('simplecomposefile', project_name) + def test_get_project(self): base_dir = 'tests/fixtures/longer-filename-composefile' project = get_project(base_dir) From 413a55aa715b85b5e8f5cbbdf4f240ff377df930 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Tue, 2 Feb 2016 19:31:49 +0000 Subject: [PATCH 354/359] Connect container to networks before starting it Signed-off-by: Aanand Prasad --- compose/service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/compose/service.py b/compose/service.py index e6aad3ae..652ee794 100644 --- a/compose/service.py +++ b/compose/service.py @@ -424,8 +424,8 @@ class Service(object): return self.start_container(container) def start_container(self, container): - container.start() self.connect_container_to_networks(container) + container.start() return container def connect_container_to_networks(self, container): From 6caa188730e66dfc328ecb0186c60c72ec1f0f1c Mon Sep 17 00:00:00 2001 From: Spencer Rinehart Date: Wed, 3 Feb 2016 14:34:36 -0600 Subject: [PATCH 355/359] Fix example formatting for depends_on. Markdown was acting against expectations here by merging the example indented YAML into the previous list item instead of treating it as a code block. I decided that a better way of handling this would be to add a "Simple example:" line that is also used elsewhere in this file. It resets the markdown indentation in a way that works. Signed-off-by: Spencer Rinehart --- docs/compose-file.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/compose-file.md b/docs/compose-file.md index e5326566..c8f55992 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -181,6 +181,8 @@ Express dependency between services, which has two effects: dependencies. In the following example, `docker-compose up web` will also create and start `db` and `redis`. +Simple example: + version: 2 services: web: From a55210413c63f173b996bb85273ce7951b8683af Mon Sep 17 00:00:00 2001 From: Daniel Nephin Date: Thu, 4 Feb 2016 12:17:20 -0500 Subject: [PATCH 356/359] Update docs for version being a string. Signed-off-by: Daniel Nephin --- docs/compose-file.md | 16 ++++++++-------- docs/networking.md | 8 ++++---- docs/overview.md | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index c8f55992..0b7c6dcb 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -183,7 +183,7 @@ Express dependency between services, which has two effects: Simple example: - version: 2 + version: '2' services: web: build: . @@ -658,7 +658,7 @@ In the example below, instead of attemping to create a volume called `[projectname]_data`, Compose will look for an existing volume simply called `data` and mount it into the `db` service's containers. - version: 2 + version: '2' services: db: @@ -748,7 +748,7 @@ attemping to create a network called `[projectname]_outside`, Compose will look for an existing network simply called `outside` and connect the `proxy` service's containers to it. - version: 2 + version: '2' services: proxy: @@ -780,7 +780,7 @@ There are two versions of the Compose file format: - Version 1, the legacy format. This is specified by omitting a `version` key at the root of the YAML. -- Version 2, the recommended format. This is specified with a `version: 2` entry +- Version 2, the recommended format. This is specified with a `version: '2'` entry at the root of the YAML. To move your project from version 1 to 2, see the [Upgrading](#upgrading) @@ -842,7 +842,7 @@ under the `networks` key. Simple example: - version: 2 + version: '2' services: web: build: . @@ -855,7 +855,7 @@ Simple example: A more extended example, defining volumes and networks: - version: 2 + version: '2' services: web: build: . @@ -887,7 +887,7 @@ A more extended example, defining volumes and networks: In the majority of cases, moving from version 1 to 2 is a very simple process: 1. Indent the whole file by one level and put a `services:` key at the top. -2. Add a `version: 2` line at the top of the file. +2. Add a `version: '2'` line at the top of the file. It's more complicated if you're using particular configuration features: @@ -950,7 +950,7 @@ It's more complicated if you're using particular configuration features: named volume called `data`, you must declare a `data` volume in your top-level `volumes` section. The whole file might look like this: - version: 2 + version: '2' services: db: image: postgres diff --git a/docs/networking.md b/docs/networking.md index 7d819d18..d625ca19 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -28,7 +28,7 @@ identical to the container name. For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: - version: 2 + version: '2' services: web: @@ -63,7 +63,7 @@ If any containers have connections open to the old container, they will be close Links allow you to define extra aliases by which a service is reachable from another service. They are not required to enable services to communicate - by default, any service can reach any other service at that service's name. In the following example, `db` is reachable from `web` at the hostnames `db` and `database`: - version: 2 + version: '2' services: web: build: . @@ -86,7 +86,7 @@ Each service can specify what networks to connect to with the *service-level* `n Here's an example Compose file defining two custom networks. The `proxy` service is isolated from the `db` service, because they do not share a network in common - only `app` can talk to both. - version: 2 + version: '2' services: proxy: @@ -123,7 +123,7 @@ For full details of the network configuration options available, see the followi Instead of (or as well as) specifying your own networks, you can also change the settings of the app-wide default network by defining an entry under `networks` named `default`: - version: 2 + version: '2' services: web: diff --git a/docs/overview.md b/docs/overview.md index f837d82f..bb3c5d71 100644 --- a/docs/overview.md +++ b/docs/overview.md @@ -32,7 +32,7 @@ they can be run together in an isolated environment. A `docker-compose.yml` looks like this: - version: 2 + version: '2' services: web: build: . From 19ae76a442078803e326a37abbbe3a991833123b Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Thu, 4 Feb 2016 16:01:11 +0000 Subject: [PATCH 357/359] Update docker-py and dockerpty Signed-off-by: Aanand Prasad --- compose/cli/main.py | 5 +++-- requirements.txt | 4 ++-- setup.py | 4 ++-- tests/unit/cli_test.py | 5 +++-- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/compose/cli/main.py b/compose/cli/main.py index 5121d5c3..deb1e912 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -42,7 +42,7 @@ from .utils import yesno if not IS_WINDOWS_PLATFORM: - from dockerpty.pty import PseudoTerminal + from dockerpty.pty import PseudoTerminal, RunOperation log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) @@ -712,12 +712,13 @@ def run_one_off_container(container_options, project, service, options): signals.set_signal_handler_to_shutdown() try: try: - pty = PseudoTerminal( + operation = RunOperation( project.client, container.id, interactive=not options['-T'], logs=False, ) + pty = PseudoTerminal(project.client, operation) sockets = pty.sockets() service.start_container(container) pty.start(sockets) diff --git a/requirements.txt b/requirements.txt index 68ef9f35..3fdd34ed 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,9 +1,9 @@ PyYAML==3.11 cached-property==1.2.0 -docker-py==1.7.0rc3 +docker-py==1.7.0 +dockerpty==0.4.1 docopt==0.6.1 enum34==1.0.4 -git+https://github.com/d11wtq/dockerpty.git@29b1394108b017ef3e3deaf00604a9eb99880d5e#egg=dockerpty jsonschema==2.5.1 requests==2.7.0 six==1.7.3 diff --git a/setup.py b/setup.py index b365e05b..df4172ce 100644 --- a/setup.py +++ b/setup.py @@ -34,8 +34,8 @@ install_requires = [ 'requests >= 2.6.1, < 2.8', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.5.0, < 2', - 'dockerpty >= 0.3.4, < 0.4', + 'docker-py >= 1.7.0, < 2', + 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', ] diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index fec7cdba..69236e2e 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -79,8 +79,9 @@ class CLITestCase(unittest.TestCase): TopLevelCommand().dispatch(['help', 'nonexistent'], None) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") + @mock.patch('compose.cli.main.RunOperation', autospec=True) @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) - def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal): + def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal, mock_run_operation): command = TopLevelCommand() mock_client = mock.create_autospec(docker.Client) mock_project = mock.Mock(client=mock_client) @@ -106,7 +107,7 @@ class CLITestCase(unittest.TestCase): '--name': None, }) - _, _, call_kwargs = mock_pseudo_terminal.mock_calls[0] + _, _, call_kwargs = mock_run_operation.mock_calls[0] assert call_kwargs['logs'] is False @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") From 8199c4a6e1bb1d332e5c4d305e7dd19c9235c5fa Mon Sep 17 00:00:00 2001 From: Ben Firshman Date: Wed, 3 Feb 2016 17:42:57 -0800 Subject: [PATCH 358/359] Improve names in Compose file 2 example Just makes it a bit clearer what's going on. Signed-off-by: Ben Firshman --- docs/compose-file.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/compose-file.md b/docs/compose-file.md index 0b7c6dcb..ec90ddcd 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -864,21 +864,21 @@ A more extended example, defining volumes and networks: volumes: - .:/code networks: - - front - - back + - front-tier + - back-tier redis: image: redis volumes: - - data:/var/lib/redis + - redis-data:/var/lib/redis networks: - - back + - back-tier volumes: - data: + redis-data: driver: local networks: - front: + front-tier: driver: bridge - back: + back-tier: driver: bridge From d99cad60e73ad81be3f621b6e9dfef9c9d9facb5 Mon Sep 17 00:00:00 2001 From: Aanand Prasad Date: Wed, 27 Jan 2016 13:30:41 +0000 Subject: [PATCH 359/359] Bump 1.6.0 Signed-off-by: Aanand Prasad --- compose/__init__.py | 2 +- docs/install.md | 6 +++--- script/run.sh | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/compose/__init__.py b/compose/__init__.py index 3c52ff7d..268bb719 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.6.0rc2' +__version__ = '1.6.0' diff --git a/docs/install.md b/docs/install.md index 7563db6b..b8979b95 100644 --- a/docs/install.md +++ b/docs/install.md @@ -39,7 +39,7 @@ which the release page specifies, in your terminal. The following is an example command illustrating the format: - curl -L https://github.com/docker/compose/releases/download/1.6.0rc2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + curl -L https://github.com/docker/compose/releases/download/1.6.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose If you have problems installing with `curl`, see [Alternative Install Options](#alternative-install-options). @@ -54,7 +54,7 @@ which the release page specifies, in your terminal. 7. Test the installation. $ docker-compose --version - docker-compose version: 1.6.0rc2 + docker-compose version: 1.6.0 ## Alternative install options @@ -77,7 +77,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.6.0rc2/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.6.0/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/script/run.sh b/script/run.sh index 89655053..784228e1 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.6.0rc2" +VERSION="1.6.0" IMAGE="docker/compose:$VERSION"