diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index db2b6506..0e7b9d5f 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -10,7 +10,7 @@ - id: end-of-file-fixer - id: flake8 - id: name-tests-test - exclude: 'tests/integration/testcases.py' + exclude: 'tests/(integration/testcases\.py|helpers\.py)' - id: requirements-txt-fixer - id: trailing-whitespace - repo: git://github.com/asottile/reorder_python_imports diff --git a/.travis.yml b/.travis.yml index 3bb365a1..fbf26964 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,3 +25,5 @@ deploy: key: '$BINTRAY_API_KEY' file: ./bintray.json skip_cleanup: true + on: + all_branches: true diff --git a/CHANGELOG.md b/CHANGELOG.md index 79aee75f..9c07e9ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,570 @@ Change log ========== +1.11.0 (2017-02-08) +------------------- + +### New Features + +#### Compose file version 3.1 + +- Introduced version 3.1 of the `docker-compose.yml` specification. This + version requires Docker Engine 1.13.0 or above. It introduces support + for secrets. See the documentation for more information + +#### Compose file version 2.0 and up + +- Introduced the `docker-compose top` command that displays processes running + for the different services managed by Compose. + +### Bugfixes + +- Fixed a bug where extending a service defining a healthcheck dictionary + would cause `docker-compose` to error out. + +- Fixed an issue where the `pid` entry in a service definition was being + ignored when using multiple Compose files. + +1.10.1 (2017-02-01) +------------------ + +### Bugfixes + +- Fixed an issue where presence of older versions of the docker-py + package would cause unexpected crashes while running Compose + +- Fixed an issue where healthcheck dependencies would be lost when + using multiple compose files for a project + +- Fixed a few issues that made the output of the `config` command + invalid + +- Fixed an issue where adding volume labels to v3 Compose files would + result in an error + +- Fixed an issue on Windows where build context paths containing unicode + characters were being improperly encoded + +- Fixed a bug where Compose would occasionally crash while streaming logs + when containers would stop or restart + +1.10.0 (2017-01-18) +------------------- + +### New Features + +#### Compose file version 3.0 + +- Introduced version 3.0 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.13 or above and is + specifically designed to work with the `docker stack` commands. + +#### Compose file version 2.1 and up + +- Healthcheck configuration can now be done in the service definition using + the `healthcheck` parameter + +- Containers dependencies can now be set up to wait on positive healthchecks + when declared using `depends_on`. See the documentation for the updated + syntax. + **Note:** This feature will not be ported to version 3 Compose files. + +- Added support for the `sysctls` parameter in service definitions + +- Added support for the `userns_mode` parameter in service definitions + +- Compose now adds identifying labels to networks and volumes it creates + +#### Compose file version 2.0 and up + +- Added support for the `stop_grace_period` option in service definitions. + +### Bugfixes + +- Colored output now works properly on Windows. + +- Fixed a bug where docker-compose run would fail to set up link aliases + in interactive mode on Windows. + +- Networks created by Compose are now always made attachable + (Compose files v2.1 and up). + +- Fixed a bug where falsy values of `COMPOSE_CONVERT_WINDOWS_PATHS` + (`0`, `false`, empty value) were being interpreted as true. + +- Fixed a bug where forward slashes in some .dockerignore patterns weren't + being parsed correctly on Windows + + +1.9.0 (2016-11-16) +----------------- + +**Breaking changes** + +- When using Compose with Docker Toolbox/Machine on Windows, volume paths are + no longer converted from `C:\Users` to `/c/Users`-style by default. To + re-enable this conversion so that your volumes keep working, set the + environment variable `COMPOSE_CONVERT_WINDOWS_PATHS=1`. Users of + Docker for Windows are not affected and do not need to set the variable. + +New Features + +- Interactive mode for `docker-compose run` and `docker-compose exec` is + now supported on Windows platforms. Please note that the `docker` binary + is required to be present on the system for this feature to work. + +- Introduced version 2.1 of the `docker-compose.yml` specification. This + version requires to be used with Docker Engine 1.12 or above. + - Added support for setting volume labels and network labels in + `docker-compose.yml`. + - Added support for the `isolation` parameter in service definitions. + - Added support for link-local IPs in the service networks definitions. + - Added support for shell-style inline defaults in variable interpolation. + The supported forms are `${FOO-default}` (fall back if FOO is unset) and + `${FOO:-default}` (fall back if FOO is unset or empty). + +- Added support for the `group_add` and `oom_score_adj` parameters in + service definitions. + +- Added support for the `internal` and `enable_ipv6` parameters in network + definitions. + +- Compose now defaults to using the `npipe` protocol on Windows. + +- Overriding a `logging` configuration will now properly merge the `options` + mappings if the `driver` values do not conflict. + +Bug Fixes + +- Fixed several bugs related to `npipe` protocol support on Windows. + +- Fixed an issue with Windows paths being incorrectly converted when + using Docker on Windows Server. + +- Fixed a bug where an empty `restart` value would sometimes result in an + exception being raised. + +- Fixed an issue where service logs containing unicode characters would + sometimes cause an error to occur. + +- Fixed a bug where unicode values in environment variables would sometimes + raise a unicode exception when retrieved. + +- Fixed an issue where Compose would incorrectly detect a configuration + mismatch for overlay networks. + + +1.8.1 (2016-09-22) +----------------- + +Bug Fixes + +- Fixed a bug where users using a credentials store were not able + to access their private images. + +- Fixed a bug where users using identity tokens to authenticate + were not able to access their private images. + +- Fixed a bug where an `HttpHeaders` entry in the docker configuration + file would cause Compose to crash when trying to build an image. + +- Fixed a few bugs related to the handling of Windows paths in volume + binding declarations. + +- Fixed a bug where Compose would sometimes crash while trying to + read a streaming response from the engine. + +- Fixed an issue where Compose would crash when encountering an API error + while streaming container logs. + +- Fixed an issue where Compose would erroneously try to output logs from + drivers not handled by the Engine's API. + +- Fixed a bug where options from the `docker-machine config` command would + not be properly interpreted by Compose. + +- Fixed a bug where the connection to the Docker Engine would + sometimes fail when running a large number of services simultaneously. + +- Fixed an issue where Compose would sometimes print a misleading + suggestion message when running the `bundle` command. + +- Fixed a bug where connection errors would not be handled properly by + Compose during the project initialization phase. + +- Fixed a bug where a misleading error would appear when encountering + a connection timeout. + + +1.8.0 (2016-06-14) +----------------- + +**Breaking Changes** + +- As announced in 1.7.0, `docker-compose rm` now removes containers + created by `docker-compose run` by default. + +- Setting `entrypoint` on a service now empties out any default + command that was set on the image (i.e. any `CMD` instruction in the + Dockerfile used to build it). This makes it consistent with + the `--entrypoint` flag to `docker run`. + +New Features + +- Added `docker-compose bundle`, a command that builds a bundle file + to be consumed by the new *Docker Stack* commands in Docker 1.12. + +- Added `docker-compose push`, a command that pushes service images + to a registry. + +- Compose now supports specifying a custom TLS version for + interaction with the Docker Engine using the `COMPOSE_TLS_VERSION` + environment variable. + +Bug Fixes + +- Fixed a bug where Compose would erroneously try to read `.env` + at the project's root when it is a directory. + +- `docker-compose run -e VAR` now passes `VAR` through from the shell + to the container, as with `docker run -e VAR`. + +- Improved config merging when multiple compose files are involved + for several service sub-keys. + +- Fixed a bug where volume mappings containing Windows drives would + sometimes be parsed incorrectly. + +- Fixed a bug in Windows environment where volume mappings of the + host's root directory would be parsed incorrectly. + +- Fixed a bug where `docker-compose config` would output an invalid + Compose file if external networks were specified. + +- Fixed an issue where unset buildargs would be assigned a string + containing `'None'` instead of the expected empty value. + +- Fixed a bug where yes/no prompts on Windows would not show before + receiving input. + +- Fixed a bug where trying to `docker-compose exec` on Windows + without the `-d` option would exit with a stacktrace. This will + still fail for the time being, but should do so gracefully. + +- Fixed a bug where errors during `docker-compose up` would show + an unrelated stacktrace at the end of the process. + +- `docker-compose create` and `docker-compose start` show more + descriptive error messages when something goes wrong. + + +1.7.1 (2016-05-04) +----------------- + +Bug Fixes + +- Fixed a bug where the output of `docker-compose config` for v1 files + would be an invalid configuration file. + +- Fixed a bug where `docker-compose config` would not check the validity + of links. + +- Fixed an issue where `docker-compose help` would not output a list of + available commands and generic options as expected. + +- Fixed an issue where filtering by service when using `docker-compose logs` + would not apply for newly created services. + +- Fixed a bug where unchanged services would sometimes be recreated in + in the up phase when using Compose with Python 3. + +- Fixed an issue where API errors encountered during the up phase would + not be recognized as a failure state by Compose. + +- Fixed a bug where Compose would raise a NameError because of an undefined + exception name on non-Windows platforms. + +- Fixed a bug where the wrong version of `docker-py` would sometimes be + installed alongside Compose. + +- Fixed a bug where the host value output by `docker-machine config default` + would not be recognized as valid options by the `docker-compose` + command line. + +- Fixed an issue where Compose would sometimes exit unexpectedly while + reading events broadcasted by a Swarm cluster. + +- Corrected a statement in the docs about the location of the `.env` file, + which is indeed read from the current directory, instead of in the same + location as the Compose file. + + +1.7.0 (2016-04-13) +------------------ + +**Breaking Changes** + +- `docker-compose logs` no longer follows log output by default. It now + matches the behaviour of `docker logs` and exits after the current logs + are printed. Use `-f` to get the old default behaviour. + +- Booleans are no longer allows as values for mappings in the Compose file + (for keys `environment`, `labels` and `extra_hosts`). Previously this + was a warning. Boolean values should be quoted so they become string values. + +New Features + +- Compose now looks for a `.env` file in the directory where it's run and + reads any environment variables defined inside, if they're not already + set in the shell environment. This lets you easily set defaults for + variables used in the Compose file, or for any of the `COMPOSE_*` or + `DOCKER_*` variables. + +- Added a `--remove-orphans` flag to both `docker-compose up` and + `docker-compose down` to remove containers for services that were removed + from the Compose file. + +- Added a `--all` flag to `docker-compose rm` to include containers created + by `docker-compose run`. This will become the default behavior in the next + version of Compose. + +- Added support for all the same TLS configuration flags used by the `docker` + client: `--tls`, `--tlscert`, `--tlskey`, etc. + +- Compose files now support the `tmpfs` and `shm_size` options. + +- Added the `--workdir` flag to `docker-compose run` + +- `docker-compose logs` now shows logs for new containers that are created + after it starts. + +- The `COMPOSE_FILE` environment variable can now contain multiple files, + separated by the host system's standard path separator (`:` on Mac/Linux, + `;` on Windows). + +- You can now specify a static IP address when connecting a service to a + network with the `ipv4_address` and `ipv6_address` options. + +- Added `--follow`, `--timestamp`, and `--tail` flags to the + `docker-compose logs` command. + +- `docker-compose up`, and `docker-compose start` will now start containers + in parallel where possible. + +- `docker-compose stop` now stops containers in reverse dependency order + instead of all at once. + +- Added the `--build` flag to `docker-compose up` to force it to build a new + image. It now shows a warning if an image is automatically built when the + flag is not used. + +- Added the `docker-compose exec` command for executing a process in a running + container. + + +Bug Fixes + +- `docker-compose down` now removes containers created by + `docker-compose run`. + +- A more appropriate error is shown when a timeout is hit during `up` when + using a tty. + +- Fixed a bug in `docker-compose down` where it would abort if some resources + had already been removed. + +- Fixed a bug where changes to network aliases would not trigger a service + to be recreated. + +- Fix a bug where a log message was printed about creating a new volume + when it already existed. + +- Fixed a bug where interrupting `up` would not always shut down containers. + +- Fixed a bug where `log_opt` and `log_driver` were not properly carried over + when extending services in the v1 Compose file format. + +- Fixed a bug where empty values for build args would cause file validation + to fail. + +1.6.2 (2016-02-23) +------------------ + +- Fixed a bug where connecting to a TLS-enabled Docker Engine would fail with + a certificate verification error. + +1.6.1 (2016-02-23) +------------------ + +Bug Fixes + +- Fixed a bug where recreating a container multiple times would cause the + new container to be started without the previous volumes. + +- Fixed a bug where Compose would set the value of unset environment variables + to an empty string, instead of a key without a value. + +- Provide a better error message when Compose requires a more recent version + of the Docker API. + +- Add a missing config field `network.aliases` which allows setting a network + scoped alias for a service. + +- Fixed a bug where `run` would not start services listed in `depends_on`. + +- Fixed a bug where `networks` and `network_mode` where not merged when using + extends or multiple Compose files. + +- Fixed a bug with service aliases where the short container id alias was + only contained 10 characters, instead of the 12 characters used in previous + versions. + +- Added a missing log message when creating a new named volume. + +- Fixed a bug where `build.args` was not merged when using `extends` or + multiple Compose files. + +- Fixed some bugs with config validation when null values or incorrect types + were used instead of a mapping. + +- Fixed a bug where a `build` section without a `context` would show a stack + trace instead of a helpful validation message. + +- Improved compatibility with swarm by only setting a container affinity to + the previous instance of a services' container when the service uses an + anonymous container volume. Previously the affinity was always set on all + containers. + +- Fixed the validation of some `driver_opts` would cause an error if a number + was used instead of a string. + +- Some improvements to the `run.sh` script used by the Compose container install + option. + +- Fixed a bug with `up --abort-on-container-exit` where Compose would exit, + but would not stop other containers. + +- Corrected the warning message that is printed when a boolean value is used + as a value in a mapping. + + +1.6.0 (2016-01-15) +------------------ + +Major Features: + +- Compose 1.6 introduces a new format for `docker-compose.yml` which lets + you define networks and volumes in the Compose file as well as services. It + also makes a few changes to the structure of some configuration options. + + You don't have to use it - your existing Compose files will run on Compose + 1.6 exactly as they do today. + + Check the upgrade guide for full details: + https://docs.docker.com/compose/compose-file#upgrading + +- Support for networking has exited experimental status and is the recommended + way to enable communication between containers. + + If you use the new file format, your app will use networking. If you aren't + ready yet, just leave your Compose file as it is and it'll continue to work + just the same. + + By default, you don't have to configure any networks. In fact, using + networking with Compose involves even less configuration than using links. + Consult the networking guide for how to use it: + https://docs.docker.com/compose/networking + + The experimental flags `--x-networking` and `--x-network-driver`, introduced + in Compose 1.5, have been removed. + +- You can now pass arguments to a build if you're using the new file format: + + build: + context: . + args: + buildno: 1 + +- You can now specify both a `build` and an `image` key if you're using the + new file format. `docker-compose build` will build the image and tag it with + the name you've specified, while `docker-compose pull` will attempt to pull + it. + +- There's a new `events` command for monitoring container events from + the application, much like `docker events`. This is a good primitive for + building tools on top of Compose for performing actions when particular + things happen, such as containers starting and stopping. + +- There's a new `depends_on` option for specifying dependencies between + services. This enforces the order of startup, and ensures that when you run + `docker-compose up SERVICE` on a service with dependencies, those are started + as well. + +New Features: + +- Added a new command `config` which validates and prints the Compose + configuration after interpolating variables, resolving relative paths, and + merging multiple files and `extends`. + +- Added a new command `create` for creating containers without starting them. + +- Added a new command `down` to stop and remove all the resources created by + `up` in a single command. + +- Added support for the `cpu_quota` configuration option. + +- Added support for the `stop_signal` configuration option. + +- Commands `start`, `restart`, `pause`, and `unpause` now exit with an + error status code if no containers were modified. + +- Added a new `--abort-on-container-exit` flag to `up` which causes `up` to + stop all container and exit once the first container exits. + +- Removed support for `FIG_FILE`, `FIG_PROJECT_NAME`, and no longer reads + `fig.yml` as a default Compose file location. + +- Removed the `migrate-to-labels` command. + +- Removed the `--allow-insecure-ssl` flag. + + +Bug Fixes: + +- Fixed a validation bug that prevented the use of a range of ports in + the `expose` field. + +- Fixed a validation bug that prevented the use of arrays in the `entrypoint` + field if they contained duplicate entries. + +- Fixed a bug that caused `ulimits` to be ignored when used with `extends`. + +- Fixed a bug that prevented ipv6 addresses in `extra_hosts`. + +- Fixed a bug that caused `extends` to be ignored when included from + multiple Compose files. + +- Fixed an incorrect warning when a container volume was defined in + the Compose file. + +- Fixed a bug that prevented the force shutdown behaviour of `up` and + `logs`. + +- Fixed a bug that caused `None` to be printed as the network driver name + when the default network driver was used. + +- Fixed a bug where using the string form of `dns` or `dns_search` would + cause an error. + +- Fixed a bug where a container would be reported as "Up" when it was + in the restarting state. + +- Fixed a confusing error message when DOCKER_CERT_PATH was not set properly. + +- Fixed a bug where attaching to a container would fail if it was using a + non-standard logging driver (or none at all). + + 1.5.2 (2015-12-03) ------------------ @@ -56,7 +620,7 @@ Change log if at least one container is using the network. - When printings logs during `up` or `logs`, flush the output buffer after - each line to prevent buffering issues from hideing logs. + each line to prevent buffering issues from hiding logs. - Recreate a container if one of its dependencies is being created. Previously a container was only recreated if it's dependencies already @@ -345,7 +909,7 @@ Fig has been renamed to Docker Compose, or just Compose for short. This has seve - The command you type is now `docker-compose`, not `fig`. - You should rename your fig.yml to docker-compose.yml. -- If you’re installing via PyPi, the package is now `docker-compose`, so install it with `pip install docker-compose`. +- If you’re installing via PyPI, the package is now `docker-compose`, so install it with `pip install docker-compose`. Besides that, there’s a lot of new stuff in this release: diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 66224752..16bccf98 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -35,7 +35,7 @@ that should get you started. This step is optional, but recommended. Pre-commit hooks will run style checks and in some cases fix style issues for you, when you commit code. -Install the git pre-commit hooks using [tox](https://tox.readthedocs.org) by +Install the git pre-commit hooks using [tox](https://tox.readthedocs.io) by running `tox -e pre-commit` or by following the [pre-commit install guide](http://pre-commit.com/#install). @@ -50,22 +50,22 @@ See Docker's [basic contribution workflow](https://docs.docker.com/opensource/wo Use the test script to run linting checks and then the full test suite against different Python interpreters: - $ script/test + $ script/test/default Tests are run against a Docker daemon inside a container, so that we can test against multiple Docker versions. By default they'll run against only the latest Docker version - set the `DOCKER_VERSIONS` environment variable to "all" to run against all supported versions: - $ DOCKER_VERSIONS=all script/test + $ DOCKER_VERSIONS=all script/test/default -Arguments to `script/test` are passed through to the `nosetests` executable, so +Arguments to `script/test/default` are passed through to the `tox` executable, so you can specify a test directory, file, module, class or method: - $ script/test tests/unit - $ script/test tests/unit/cli_test.py - $ script/test tests/unit/config_test.py::ConfigTest - $ script/test tests/unit/config_test.py::ConfigTest::test_load + $ script/test/default tests/unit + $ script/test/default tests/unit/cli_test.py + $ script/test/default tests/unit/config_test.py::ConfigTest + $ script/test/default tests/unit/config_test.py::ConfigTest::test_load ## Finding things to work on diff --git a/Dockerfile b/Dockerfile index acf9b6ae..a03e1510 100644 --- a/Dockerfile +++ b/Dockerfile @@ -13,6 +13,7 @@ RUN set -ex; \ ca-certificates \ curl \ libsqlite3-dev \ + libbz2-dev \ ; \ rm -rf /var/lib/apt/lists/* @@ -20,40 +21,32 @@ RUN curl https://get.docker.com/builds/Linux/x86_64/docker-1.8.3 \ -o /usr/local/bin/docker && \ chmod +x /usr/local/bin/docker -# Build Python 2.7.9 from source +# Build Python 2.7.13 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/2.7.9/Python-2.7.9.tgz | tar -xz; \ - cd Python-2.7.9; \ + curl -L https://www.python.org/ftp/python/2.7.13/Python-2.7.13.tgz | tar -xz; \ + cd Python-2.7.13; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-2.7.9 + rm -rf /Python-2.7.13 # Build python 3.4 from source RUN set -ex; \ - curl -L https://www.python.org/ftp/python/3.4.3/Python-3.4.3.tgz | tar -xz; \ - cd Python-3.4.3; \ + curl -L https://www.python.org/ftp/python/3.4.6/Python-3.4.6.tgz | tar -xz; \ + cd Python-3.4.6; \ ./configure --enable-shared; \ make; \ make install; \ cd ..; \ - rm -rf /Python-3.4.3 + rm -rf /Python-3.4.6 # Make libpython findable ENV LD_LIBRARY_PATH /usr/local/lib -# Install setuptools -RUN set -ex; \ - curl -L https://bootstrap.pypa.io/ez_setup.py | python - # Install pip RUN set -ex; \ - curl -L https://pypi.python.org/packages/source/p/pip/pip-7.0.1.tar.gz | tar -xz; \ - cd pip-7.0.1; \ - python setup.py install; \ - cd ..; \ - rm -rf pip-7.0.1 + curl -L https://bootstrap.pypa.io/get-pip.py | python # Python3 requires a valid locale RUN echo "en_US.UTF-8 UTF-8" > /etc/locale.gen && locale-gen diff --git a/Dockerfile.run b/Dockerfile.run index 792077ad..de46e35e 100644 --- a/Dockerfile.run +++ b/Dockerfile.run @@ -1,5 +1,6 @@ -FROM alpine:edge +FROM alpine:3.4 +ARG version RUN apk -U add \ python \ py-pip @@ -7,7 +8,7 @@ RUN apk -U add \ COPY requirements.txt /code/requirements.txt RUN pip install -r /code/requirements.txt -ADD dist/docker-compose-release.tar.gz /code/docker-compose -RUN pip install --no-deps /code/docker-compose/docker-compose-* +COPY dist/docker_compose-${version}-py2.py3-none-any.whl /code/ +RUN pip install --no-deps /code/docker_compose-${version}-py2.py3-none-any.whl ENTRYPOINT ["/usr/bin/docker-compose"] diff --git a/Jenkinsfile b/Jenkinsfile new file mode 100644 index 00000000..51136b1f --- /dev/null +++ b/Jenkinsfile @@ -0,0 +1,64 @@ +#!groovy + +def image + +def buildImage = { -> + wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { + stage("build image") { + checkout(scm) + def imageName = "dockerbuildbot/compose:${gitCommit()}" + image = docker.image(imageName) + try { + image.pull() + } catch (Exception exc) { + image = docker.build(imageName, ".") + image.push() + } + } + } +} + +def runTests = { Map settings -> + def dockerVersions = settings.get("dockerVersions", null) + def pythonVersions = settings.get("pythonVersions", null) + + if (!pythonVersions) { + throw new Exception("Need Python versions to test. e.g.: `runTests(pythonVersions: 'py27,py34')`") + } + if (!dockerVersions) { + throw new Exception("Need Docker versions to test. e.g.: `runTests(dockerVersions: 'all')`") + } + + { -> + wrappedNode(label: "ubuntu && !zfs", cleanWorkspace: true) { + stage("test python=${pythonVersions} / docker=${dockerVersions}") { + checkout(scm) + def storageDriver = sh(script: 'docker info | awk -F \': \' \'$1 == "Storage Driver" { print $2; exit }\'', returnStdout: true).trim() + echo "Using local system's storage driver: ${storageDriver}" + sh """docker run \\ + -t \\ + --rm \\ + --privileged \\ + --volume="\$(pwd)/.git:/code/.git" \\ + --volume="/var/run/docker.sock:/var/run/docker.sock" \\ + -e "TAG=${image.id}" \\ + -e "STORAGE_DRIVER=${storageDriver}" \\ + -e "DOCKER_VERSIONS=${dockerVersions}" \\ + -e "BUILD_NUMBER=\$BUILD_TAG" \\ + -e "PY_TEST_VERSIONS=${pythonVersions}" \\ + --entrypoint="script/ci" \\ + ${image.id} \\ + --verbose + """ + } + } + } +} + +buildImage() +// TODO: break this out into meaningful "DOCKER_VERSIONS" values instead of all +parallel( + failFast: true, + all_py27: runTests(pythonVersions: "py27", dockerVersions: "all"), + all_py34: runTests(pythonVersions: "py34", dockerVersions: "all"), +) diff --git a/README.md b/README.md index b60a7eee..35a10b90 100644 --- a/README.md +++ b/README.md @@ -6,11 +6,11 @@ Compose is a tool for defining and running multi-container Docker applications. With Compose, you use a Compose file to configure your application's services. Then, using a single command, you create and start all the services from your configuration. To learn more about all the features of Compose -see [the list of features](docs/index.md#features). +see [the list of features](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#features). Compose is great for development, testing, and staging environments, as well as CI workflows. You can learn more about each case in -[Common Use Cases](docs/index.md#common-use-cases). +[Common Use Cases](https://github.com/docker/docker.github.io/blob/master/compose/overview.md#common-use-cases). Using Compose is basically a three-step process. @@ -22,19 +22,20 @@ they can be run together in an isolated environment: A `docker-compose.yml` looks like this: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis + version: '2' + + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + redis: + image: redis For more information about the Compose file, see the -[Compose file reference](docs/compose-file.md) +[Compose file reference](https://github.com/docker/docker.github.io/blob/master/compose/compose-file/compose-versioning.md) Compose has commands for managing the whole lifecycle of your application: diff --git a/ROADMAP.md b/ROADMAP.md index 67903492..287e5468 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -1,13 +1,21 @@ # Roadmap +## An even better tool for development environments + +Compose is a great tool for development environments, but it could be even better. For example: + +- It should be possible to define hostnames for containers which work from the host machine, e.g. “mywebcontainer.local”. This is needed by apps comprising multiple web services which generate links to one another (e.g. a frontend website and a separate admin webapp) + ## More than just development environments -Over time we will extend Compose's remit to cover test, staging and production environments. This is not a simple task, and will take many incremental improvements such as: +Compose currently works really well in development, but we want to make the Compose file format better for test, staging, and production environments. To support these use cases, there will need to be improvements to the file format, improvements to the command-line tool, integrations with other tools, and perhaps new tools altogether. + +Some specific things we are considering: - Compose currently will attempt to get your application into the correct state when running `up`, but it has a number of shortcomings: - It should roll back to a known good state if it fails. - It should allow a user to check the actions it is about to perform before running them. -- It should be possible to partially modify the config file for different environments (dev/test/staging/prod), passing in e.g. custom ports or volume mount paths. ([#1377](https://github.com/docker/compose/issues/1377)) +- It should be possible to partially modify the config file for different environments (dev/test/staging/prod), passing in e.g. custom ports, volume mount paths, or volume drivers. ([#1377](https://github.com/docker/compose/issues/1377)) - Compose should recommend a technique for zero-downtime deploys. - It should be possible to continuously attempt to keep an application in the correct state, instead of just performing `up` a single time. @@ -22,10 +30,3 @@ The current state of integration is documented in [SWARM.md](SWARM.md). Compose works well for applications that are in a single repository and depend on services that are hosted on Docker Hub. If your application depends on another application within your organisation, Compose doesn't work as well. There are several ideas about how this could work, such as [including external files](https://github.com/docker/fig/issues/318). - -## An even better tool for development environments - -Compose is a great tool for development environments, but it could be even better. For example: - -- [Compose could watch your code and automatically kick off builds when something changes.](https://github.com/docker/fig/issues/184) -- It should be possible to define hostnames for containers which work from the host machine, e.g. “mywebcontainer.local”. This is needed by apps comprising multiple web services which generate links to one another (e.g. a frontend website and a separate admin webapp) diff --git a/SWARM.md b/SWARM.md index 1ea4e25f..c6f378a9 100644 --- a/SWARM.md +++ b/SWARM.md @@ -1,39 +1 @@ -Docker Compose/Swarm integration -================================ - -Eventually, Compose and Swarm aim to have full integration, meaning you can point a Compose app at a Swarm cluster and have it all just work as if you were using a single Docker host. - -However, integration is currently incomplete: Compose can create containers on a Swarm cluster, but the majority of Compose apps won’t work out of the box unless all containers are scheduled on one host, because links between containers do not work across hosts. - -Docker networking is [getting overhauled](https://github.com/docker/libnetwork) in such a way that it’ll fit the multi-host model much better. For now, linked containers are automatically scheduled on the same host. - -Building --------- - -Swarm can build an image from a Dockerfile just like a single-host Docker instance can, but the resulting image will only live on a single node and won't be distributed to other nodes. - -If you want to use Compose to scale the service in question to multiple nodes, you'll have to build it yourself, push it to a registry (e.g. the Docker Hub) and reference it from `docker-compose.yml`: - - $ docker build -t myusername/web . - $ docker push myusername/web - - $ cat docker-compose.yml - web: - image: myusername/web - - $ docker-compose up -d - $ docker-compose scale web=3 - -Scheduling ----------- - -Swarm offers a rich set of scheduling and affinity hints, enabling you to control where containers are located. They are specified via container environment variables, so you can use Compose's `environment` option to set them. - - environment: - # Schedule containers on a node that has the 'storage' label set to 'ssd' - - "constraint:storage==ssd" - - # Schedule containers where the 'redis' image is already pulled - - "affinity:image==redis" - -For the full set of available filters and expressions, see the [Swarm documentation](https://docs.docker.com/swarm/scheduler/filter/). +This file has moved to: https://docs.docker.com/compose/swarm/ diff --git a/appveyor.yml b/appveyor.yml index b162db1e..e4f39544 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -9,20 +9,16 @@ install: # Build the binary after tests build: false -environment: - BINTRAY_USER: "docker-compose-roleuser" - BINTRAY_PATH: "docker-compose/master/windows/master/docker-compose-Windows-x86_64.exe" - test_script: - "tox -e py27,py34 -- tests/unit" - - ps: ".\\script\\build-windows.ps1" - -deploy_script: - - "curl -sS - -u \"%BINTRAY_USER%:%BINTRAY_API_KEY%\" - -X PUT \"https://api.bintray.com/content/%BINTRAY_PATH%?override=1&publish=1\" - --data-binary @dist\\docker-compose-Windows-x86_64.exe" + - ps: ".\\script\\build\\windows.ps1" artifacts: - path: .\dist\docker-compose-Windows-x86_64.exe name: "Compose Windows binary" + +deploy: + - provider: Environment + name: master-builds + on: + branch: master diff --git a/compose/__init__.py b/compose/__init__.py index 3ba90fde..b2ca86f8 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,4 +1,4 @@ from __future__ import absolute_import from __future__ import unicode_literals -__version__ = '1.6.0dev' +__version__ = '1.12.0dev' diff --git a/compose/bundle.py b/compose/bundle.py new file mode 100644 index 00000000..505ce91f --- /dev/null +++ b/compose/bundle.py @@ -0,0 +1,258 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import json +import logging + +import six +from docker.utils import split_command +from docker.utils.ports import split_port + +from .cli.errors import UserError +from .config.serialize import denormalize_config +from .network import get_network_defs_for_service +from .service import format_environment +from .service import NoSuchImageError +from .service import parse_repository_tag + + +log = logging.getLogger(__name__) + + +SERVICE_KEYS = { + 'working_dir': 'WorkingDir', + 'user': 'User', + 'labels': 'Labels', +} + +IGNORED_KEYS = {'build'} + +SUPPORTED_KEYS = { + 'image', + 'ports', + 'expose', + 'networks', + 'command', + 'environment', + 'entrypoint', +} | set(SERVICE_KEYS) + +VERSION = '0.1' + + +class NeedsPush(Exception): + def __init__(self, image_name): + self.image_name = image_name + + +class NeedsPull(Exception): + def __init__(self, image_name, service_name): + self.image_name = image_name + self.service_name = service_name + + +class MissingDigests(Exception): + def __init__(self, needs_push, needs_pull): + self.needs_push = needs_push + self.needs_pull = needs_pull + + +def serialize_bundle(config, image_digests): + return json.dumps(to_bundle(config, image_digests), indent=2, sort_keys=True) + + +def get_image_digests(project, allow_push=False): + digests = {} + needs_push = set() + needs_pull = set() + + for service in project.services: + try: + digests[service.name] = get_image_digest( + service, + allow_push=allow_push, + ) + except NeedsPush as e: + needs_push.add(e.image_name) + except NeedsPull as e: + needs_pull.add(e.service_name) + + if needs_push or needs_pull: + raise MissingDigests(needs_push, needs_pull) + + return digests + + +def get_image_digest(service, allow_push=False): + if 'image' not in service.options: + raise UserError( + "Service '{s.name}' doesn't define an image tag. An image name is " + "required to generate a proper image digest for the bundle. Specify " + "an image repo and tag with the 'image' option.".format(s=service)) + + _, _, separator = parse_repository_tag(service.options['image']) + # Compose file already uses a digest, no lookup required + if separator == '@': + return service.options['image'] + + try: + image = service.image() + except NoSuchImageError: + action = 'build' if 'build' in service.options else 'pull' + raise UserError( + "Image not found for service '{service}'. " + "You might need to run `docker-compose {action} {service}`." + .format(service=service.name, action=action)) + + if image['RepoDigests']: + # TODO: pick a digest based on the image tag if there are multiple + # digests + return image['RepoDigests'][0] + + if 'build' not in service.options: + raise NeedsPull(service.image_name, service.name) + + if not allow_push: + raise NeedsPush(service.image_name) + + return push_image(service) + + +def push_image(service): + try: + digest = service.push() + except: + log.error( + "Failed to push image for service '{s.name}'. Please use an " + "image tag that can be pushed to a Docker " + "registry.".format(s=service)) + raise + + if not digest: + raise ValueError("Failed to get digest for %s" % service.name) + + repo, _, _ = parse_repository_tag(service.options['image']) + identifier = '{repo}@{digest}'.format(repo=repo, digest=digest) + + # only do this if RepoDigests isn't already populated + image = service.image() + if not image['RepoDigests']: + # Pull by digest so that image['RepoDigests'] is populated for next time + # and we don't have to pull/push again + service.client.pull(identifier) + log.info("Stored digest for {}".format(service.image_name)) + + return identifier + + +def to_bundle(config, image_digests): + if config.networks: + log.warn("Unsupported top level key 'networks' - ignoring") + + if config.volumes: + log.warn("Unsupported top level key 'volumes' - ignoring") + + config = denormalize_config(config) + + return { + 'Version': VERSION, + 'Services': { + name: convert_service_to_bundle( + name, + service_dict, + image_digests[name], + ) + for name, service_dict in config['services'].items() + }, + } + + +def convert_service_to_bundle(name, service_dict, image_digest): + container_config = {'Image': image_digest} + + for key, value in service_dict.items(): + if key in IGNORED_KEYS: + continue + + if key not in SUPPORTED_KEYS: + log.warn("Unsupported key '{}' in services.{} - ignoring".format(key, name)) + continue + + if key == 'environment': + container_config['Env'] = format_environment({ + envkey: envvalue for envkey, envvalue in value.items() + if envvalue + }) + continue + + if key in SERVICE_KEYS: + container_config[SERVICE_KEYS[key]] = value + continue + + set_command_and_args( + container_config, + service_dict.get('entrypoint', []), + service_dict.get('command', [])) + container_config['Networks'] = make_service_networks(name, service_dict) + + ports = make_port_specs(service_dict) + if ports: + container_config['Ports'] = ports + + return container_config + + +# See https://github.com/docker/swarmkit/blob/agent/exec/container/container.go#L95 +def set_command_and_args(config, entrypoint, command): + if isinstance(entrypoint, six.string_types): + entrypoint = split_command(entrypoint) + if isinstance(command, six.string_types): + command = split_command(command) + + if entrypoint: + config['Command'] = entrypoint + command + return + + if command: + config['Args'] = command + + +def make_service_networks(name, service_dict): + networks = [] + + for network_name, network_def in get_network_defs_for_service(service_dict).items(): + for key in network_def.keys(): + log.warn( + "Unsupported key '{}' in services.{}.networks.{} - ignoring" + .format(key, name, network_name)) + + networks.append(network_name) + + return networks + + +def make_port_specs(service_dict): + ports = [] + + internal_ports = [ + internal_port + for port_def in service_dict.get('ports', []) + for internal_port in split_port(port_def)[0] + ] + + internal_ports += service_dict.get('expose', []) + + for internal_port in internal_ports: + spec = make_port_spec(internal_port) + if spec not in ports: + ports.append(spec) + + return ports + + +def make_port_spec(value): + components = six.text_type(value).partition('/') + return { + 'Protocol': components[2] or 'tcp', + 'Port': int(components[0]), + } diff --git a/compose/cli/__init__.py b/compose/cli/__init__.py index e69de29b..c5db4455 100644 --- a/compose/cli/__init__.py +++ b/compose/cli/__init__.py @@ -0,0 +1,37 @@ +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +import subprocess +import sys + +# Attempt to detect https://github.com/docker/compose/issues/4344 +try: + # We don't try importing pip because it messes with package imports + # on some Linux distros (Ubuntu, Fedora) + # https://github.com/docker/compose/issues/4425 + # https://github.com/docker/compose/issues/4481 + # https://github.com/pypa/pip/blob/master/pip/_vendor/__init__.py + s_cmd = subprocess.Popen( + ['pip', 'freeze'], stderr=subprocess.PIPE, stdout=subprocess.PIPE + ) + packages = s_cmd.communicate()[0].splitlines() + dockerpy_installed = len( + list(filter(lambda p: p.startswith(b'docker-py=='), packages)) + ) > 0 + if dockerpy_installed: + from .colors import red + print( + red('ERROR:'), + "Dependency conflict: an older version of the 'docker-py' package " + "is polluting the namespace. " + "Run the following command to remedy the issue:\n" + "pip uninstall docker docker-py; pip install docker", + file=sys.stderr + ) + sys.exit(1) + +except OSError: + # pip command is not available, which indicates it's probably the binary + # distribution of Compose which is not affected + pass diff --git a/compose/cli/colors.py b/compose/cli/colors.py index 3c18886f..f1251e43 100644 --- a/compose/cli/colors.py +++ b/compose/cli/colors.py @@ -1,5 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals + +import colorama + NAMES = [ 'grey', 'red', @@ -30,6 +33,7 @@ def make_color_fn(code): return lambda s: ansi_color(code, s) +colorama.init(strip=False) for (name, code) in get_pairs(): globals()[name] = make_color_fn(code) diff --git a/compose/cli/command.py b/compose/cli/command.py index c1681ffc..02035428 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -1,69 +1,87 @@ from __future__ import absolute_import from __future__ import unicode_literals -import contextlib import logging import os import re +import ssl import six -from requests.exceptions import ConnectionError -from requests.exceptions import SSLError from . import errors from . import verbose_proxy from .. import config +from ..config.environment import Environment from ..const import API_VERSIONS from ..project import Project from .docker_client import docker_client -from .utils import call_silently +from .docker_client import tls_config_from_options from .utils import get_version_info -from .utils import is_mac -from .utils import is_ubuntu log = logging.getLogger(__name__) -@contextlib.contextmanager -def friendly_error_message(): - try: - yield - except SSLError as e: - raise errors.UserError('SSL error: %s' % e) - except ConnectionError: - if call_silently(['which', 'docker']) != 0: - if is_mac(): - raise errors.DockerNotFoundMac() - elif is_ubuntu(): - raise errors.DockerNotFoundUbuntu() - else: - raise errors.DockerNotFoundGeneric() - elif call_silently(['which', 'docker-machine']) == 0: - raise errors.ConnectionErrorDockerMachine() - else: - raise errors.ConnectionErrorGeneric(get_client().base_url) - - -def project_from_options(base_dir, options): +def project_from_options(project_dir, options): + environment = Environment.from_env_file(project_dir) + host = options.get('--host') + if host is not None: + host = host.lstrip('=') return get_project( - base_dir, - get_config_path_from_options(options), + project_dir, + get_config_path_from_options(project_dir, options, environment), project_name=options.get('--project-name'), verbose=options.get('--verbose'), + host=host, + tls_config=tls_config_from_options(options), + environment=environment ) -def get_config_path_from_options(options): +def get_config_from_options(base_dir, options): + environment = Environment.from_env_file(base_dir) + config_path = get_config_path_from_options( + base_dir, options, environment + ) + return config.load( + config.find(base_dir, config_path, environment) + ) + + +def get_config_path_from_options(base_dir, options, environment): file_option = options.get('--file') if file_option: return file_option - config_file = os.environ.get('COMPOSE_FILE') - return [config_file] if config_file else None + config_files = environment.get('COMPOSE_FILE') + if config_files: + return config_files.split(os.pathsep) + return None -def get_client(verbose=False, version=None): - client = docker_client(version=version) +def get_tls_version(environment): + compose_tls_version = environment.get('COMPOSE_TLS_VERSION', None) + if not compose_tls_version: + return None + + tls_attr_name = "PROTOCOL_{}".format(compose_tls_version) + if not hasattr(ssl, tls_attr_name): + log.warn( + 'The "{}" protocol is unavailable. You may need to update your ' + 'version of Python or OpenSSL. Falling back to TLSv1 (default).' + .format(compose_tls_version) + ) + return None + + return getattr(ssl, tls_attr_name) + + +def get_client(environment, verbose=False, version=None, tls_config=None, host=None, + tls_version=None): + + client = docker_client( + version=version, tls_config=tls_config, host=host, + environment=environment, tls_version=get_tls_version(environment) + ) if verbose: version_info = six.iteritems(client.version()) log.info(get_version_info('full')) @@ -74,25 +92,37 @@ def get_client(verbose=False, version=None): return client -def get_project(base_dir, config_path=None, project_name=None, verbose=False): - config_details = config.find(base_dir, config_path) - project_name = get_project_name(config_details.working_dir, project_name) +def get_project(project_dir, config_path=None, project_name=None, verbose=False, + host=None, tls_config=None, environment=None): + if not environment: + environment = Environment.from_env_file(project_dir) + config_details = config.find(project_dir, config_path, environment) + project_name = get_project_name( + config_details.working_dir, project_name, environment + ) config_data = config.load(config_details) - api_version = os.environ.get( + api_version = environment.get( 'COMPOSE_API_VERSION', API_VERSIONS[config_data.version]) - client = get_client(verbose=verbose, version=api_version) - return Project.from_config(project_name, config_data, client) + client = get_client( + verbose=verbose, version=api_version, tls_config=tls_config, + host=host, environment=environment + ) + + with errors.handle_connection_errors(client): + return Project.from_config(project_name, config_data, client) -def get_project_name(working_dir, project_name=None): +def get_project_name(working_dir, project_name=None, environment=None): def normalize_name(name): return re.sub(r'[^a-z0-9]', '', name.lower()) - project_name = project_name or os.environ.get('COMPOSE_PROJECT_NAME') - if project_name is not None: + if not environment: + environment = Environment.from_env_file(working_dir) + project_name = project_name or environment.get('COMPOSE_PROJECT_NAME') + if project_name: return normalize_name(project_name) project = os.path.basename(os.path.abspath(working_dir)) diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 611997df..018d2451 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -2,26 +2,73 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -import os -from docker import Client +from docker import APIClient +from docker.errors import TLSParameterError +from docker.tls import TLSConfig from docker.utils import kwargs_from_env from ..const import HTTP_TIMEOUT +from .errors import UserError +from .utils import generate_user_agent +from .utils import unquote_path log = logging.getLogger(__name__) -def docker_client(version=None): +def tls_config_from_options(options): + tls = options.get('--tls', False) + ca_cert = unquote_path(options.get('--tlscacert')) + cert = unquote_path(options.get('--tlscert')) + key = unquote_path(options.get('--tlskey')) + verify = options.get('--tlsverify') + skip_hostname_check = options.get('--skip-hostname-check', False) + + advanced_opts = any([ca_cert, cert, key, verify]) + + if tls is True and not advanced_opts: + return True + elif advanced_opts: # --tls is a noop + client_cert = None + if cert or key: + client_cert = (cert, key) + + return TLSConfig( + client_cert=client_cert, verify=verify, ca_cert=ca_cert, + assert_hostname=False if skip_hostname_check else None + ) + + return None + + +def docker_client(environment, version=None, tls_config=None, host=None, + tls_version=None): """ Returns a docker-py client configured using environment variables according to the same logic as the official Docker client. """ - if 'DOCKER_CLIENT_TIMEOUT' in os.environ: - log.warn('The DOCKER_CLIENT_TIMEOUT environment variable is deprecated. Please use COMPOSE_HTTP_TIMEOUT instead.') + try: + kwargs = kwargs_from_env(environment=environment, ssl_version=tls_version) + except TLSParameterError: + raise UserError( + "TLS configuration is invalid - make sure your DOCKER_TLS_VERIFY " + "and DOCKER_CERT_PATH are set correctly.\n" + "You might need to run `eval \"$(docker-machine env default)\"`") + + if host: + kwargs['base_url'] = host + if tls_config: + kwargs['tls'] = tls_config - kwargs = kwargs_from_env(assert_hostname=False) if version: kwargs['version'] = version - kwargs['timeout'] = HTTP_TIMEOUT - return Client(**kwargs) + + timeout = environment.get('COMPOSE_HTTP_TIMEOUT') + if timeout: + kwargs['timeout'] = int(timeout) + else: + kwargs['timeout'] = HTTP_TIMEOUT + + kwargs['user_agent'] = generate_user_agent() + + return APIClient(**kwargs) diff --git a/compose/cli/docopt_command.py b/compose/cli/docopt_command.py index e3f4aa9e..809a4b74 100644 --- a/compose/cli/docopt_command.py +++ b/compose/cli/docopt_command.py @@ -1,7 +1,6 @@ from __future__ import absolute_import from __future__ import unicode_literals -import sys from inspect import getdoc from docopt import docopt @@ -15,24 +14,21 @@ def docopt_full_help(docstring, *args, **kwargs): raise SystemExit(docstring) -class DocoptCommand(object): - def docopt_options(self): - return {'options_first': True} +class DocoptDispatcher(object): - def sys_dispatch(self): - self.dispatch(sys.argv[1:], None) + def __init__(self, command_class, options): + self.command_class = command_class + self.options = options - def dispatch(self, argv, global_options): - self.perform_command(*self.parse(argv, global_options)) - - def parse(self, argv, global_options): - options = docopt_full_help(getdoc(self), argv, **self.docopt_options()) + def parse(self, argv): + command_help = getdoc(self.command_class) + options = docopt_full_help(command_help, argv, **self.options) command = options['COMMAND'] if command is None: - raise SystemExit(getdoc(self)) + raise SystemExit(command_help) - handler = self.get_handler(command) + handler = get_handler(self.command_class, command) docstring = getdoc(handler) if docstring is None: @@ -41,13 +37,18 @@ class DocoptCommand(object): command_options = docopt_full_help(docstring, options['ARGS'], options_first=True) return options, handler, command_options - def get_handler(self, command): - command = command.replace('-', '_') - if not hasattr(self, command): - raise NoSuchCommand(command, self) +def get_handler(command_class, command): + command = command.replace('-', '_') + # we certainly want to have "exec" command, since that's what docker client has + # but in python exec is a keyword + if command == "exec": + command = "exec_command" - return getattr(self, command) + if not hasattr(command_class, command): + raise NoSuchCommand(command, command_class) + + return getattr(command_class, command) class NoSuchCommand(Exception): diff --git a/compose/cli/errors.py b/compose/cli/errors.py index 03d6a50c..5b977095 100644 --- a/compose/cli/errors.py +++ b/compose/cli/errors.py @@ -1,10 +1,30 @@ from __future__ import absolute_import from __future__ import unicode_literals +import contextlib +import logging +import socket +from distutils.spawn import find_executable from textwrap import dedent +from docker.errors import APIError +from requests.exceptions import ConnectionError as RequestsConnectionError +from requests.exceptions import ReadTimeout +from requests.exceptions import SSLError +from requests.packages.urllib3.exceptions import ReadTimeoutError + +from ..const import API_VERSION_TO_ENGINE_VERSION +from .utils import is_docker_for_mac_installed +from .utils import is_mac +from .utils import is_ubuntu +from .utils import is_windows + + +log = logging.getLogger(__name__) + class UserError(Exception): + def __init__(self, msg): self.msg = dedent(msg).strip() @@ -14,44 +34,104 @@ class UserError(Exception): __str__ = __unicode__ -class DockerNotFoundMac(UserError): - def __init__(self): - super(DockerNotFoundMac, self).__init__(""" - Couldn't connect to Docker daemon. You might need to install docker-osx: - - https://github.com/noplay/docker-osx - """) +class ConnectionError(Exception): + pass -class DockerNotFoundUbuntu(UserError): - def __init__(self): - super(DockerNotFoundUbuntu, self).__init__(""" - Couldn't connect to Docker daemon. You might need to install Docker: - - https://docs.docker.com/engine/installation/ubuntulinux/ - """) +@contextlib.contextmanager +def handle_connection_errors(client): + try: + yield + except SSLError as e: + log.error('SSL error: %s' % e) + raise ConnectionError() + except RequestsConnectionError as e: + if e.args and isinstance(e.args[0], ReadTimeoutError): + log_timeout_error(client.timeout) + raise ConnectionError() + exit_with_error(get_conn_error_message(client.base_url)) + except APIError as e: + log_api_error(e, client.api_version) + raise ConnectionError() + except (ReadTimeout, socket.timeout) as e: + log_timeout_error(client.timeout) + raise ConnectionError() -class DockerNotFoundGeneric(UserError): - def __init__(self): - super(DockerNotFoundGeneric, self).__init__(""" - Couldn't connect to Docker daemon. You might need to install Docker: - - https://docs.docker.com/engine/installation/ - """) +def log_timeout_error(timeout): + log.error( + "An HTTP request took too long to complete. Retry with --verbose to " + "obtain debug information.\n" + "If you encounter this issue regularly because of slow network " + "conditions, consider setting COMPOSE_HTTP_TIMEOUT to a higher " + "value (current value: %s)." % timeout) -class ConnectionErrorDockerMachine(UserError): - def __init__(self): - super(ConnectionErrorDockerMachine, self).__init__(""" - Couldn't connect to Docker daemon - you might need to run `docker-machine start default`. - """) +def log_api_error(e, client_version): + if b'client is newer than server' not in e.explanation: + log.error(e.explanation) + return + + version = API_VERSION_TO_ENGINE_VERSION.get(client_version) + if not version: + # They've set a custom API version + log.error(e.explanation) + return + + log.error( + "The Docker Engine version is less than the minimum required by " + "Compose. Your current project requires a Docker Engine of " + "version {version} or greater.".format(version=version)) -class ConnectionErrorGeneric(UserError): - def __init__(self, url): - super(ConnectionErrorGeneric, self).__init__(""" - Couldn't connect to Docker daemon at %s - is it running? +def exit_with_error(msg): + log.error(dedent(msg).strip()) + raise ConnectionError() - If it's at a non-standard location, specify the URL with the DOCKER_HOST environment variable. - """ % url) + +def get_conn_error_message(url): + if find_executable('docker') is None: + return docker_not_found_msg("Couldn't connect to Docker daemon.") + if is_docker_for_mac_installed(): + return conn_error_docker_for_mac + if find_executable('docker-machine') is not None: + return conn_error_docker_machine + return conn_error_generic.format(url=url) + + +def docker_not_found_msg(problem): + return "{} You might need to install Docker:\n\n{}".format( + problem, docker_install_url()) + + +def docker_install_url(): + if is_mac(): + return docker_install_url_mac + elif is_ubuntu(): + return docker_install_url_ubuntu + elif is_windows(): + return docker_install_url_windows + else: + return docker_install_url_generic + + +docker_install_url_mac = "https://docs.docker.com/engine/installation/mac/" +docker_install_url_ubuntu = "https://docs.docker.com/engine/installation/ubuntulinux/" +docker_install_url_windows = "https://docs.docker.com/engine/installation/windows/" +docker_install_url_generic = "https://docs.docker.com/engine/installation/" + + +conn_error_docker_machine = """ + Couldn't connect to Docker daemon - you might need to run `docker-machine start default`. +""" + +conn_error_docker_for_mac = """ + Couldn't connect to Docker daemon. You might need to start Docker for Mac. +""" + + +conn_error_generic = """ + Couldn't connect to Docker daemon at {url} - is it running? + + If it's at a non-standard location, specify the URL with the DOCKER_HOST environment variable. +""" diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index d0ed0f87..5f580645 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import logging import os +import six import texttable from compose.cli import colors @@ -44,5 +45,7 @@ class ConsoleWarningFormatter(logging.Formatter): return '' def format(self, record): + if isinstance(record.msg, six.binary_type): + record.msg = record.msg.decode('utf-8') message = super(ConsoleWarningFormatter, self).format(record) - return self.get_level_message(record) + message + return '{0}{1}'.format(self.get_level_message(record), message) diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 85fef794..299ddea4 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -2,60 +2,153 @@ from __future__ import absolute_import from __future__ import unicode_literals import sys +from collections import namedtuple from itertools import cycle +from threading import Thread + +from docker.errors import APIError +from six.moves import _thread as thread +from six.moves.queue import Empty +from six.moves.queue import Queue from . import colors -from .multiplexer import Multiplexer from compose import utils +from compose.cli.signals import ShutdownException from compose.utils import split_buffer -class LogPrinter(object): - """Print logs from many containers to a single output stream.""" +class LogPresenter(object): - def __init__(self, containers, output=sys.stdout, monochrome=False, cascade_stop=False): - self.containers = containers - self.output = utils.get_output_stream(output) - self.monochrome = monochrome - self.cascade_stop = cascade_stop + def __init__(self, prefix_width, color_func): + self.prefix_width = prefix_width + self.color_func = color_func - def run(self): - if not self.containers: - return - - prefix_width = max_name_width(self.containers) - generators = list(self._make_log_generators(self.monochrome, prefix_width)) - for line in Multiplexer(generators, cascade_stop=self.cascade_stop).loop(): - self.output.write(line) - self.output.flush() - - def _make_log_generators(self, monochrome, prefix_width): - def no_color(text): - return text - - if monochrome: - color_funcs = cycle([no_color]) - else: - color_funcs = cycle(colors.rainbow()) - - for color_func, container in zip(color_funcs, self.containers): - generator_func = get_log_generator(container) - prefix = color_func(build_log_prefix(container, prefix_width)) - yield generator_func(container, prefix, color_func) + def present(self, container, line): + prefix = container.name_without_project.ljust(self.prefix_width) + return '{prefix} {line}'.format( + prefix=self.color_func(prefix + ' |'), + line=line) -def build_log_prefix(container, prefix_width): - return container.name_without_project.ljust(prefix_width) + ' | ' +def build_log_presenters(service_names, monochrome): + """Return an iterable of functions. + + Each function can be used to format the logs output of a container. + """ + prefix_width = max_name_width(service_names) + + def no_color(text): + return text + + for color_func in cycle([no_color] if monochrome else colors.rainbow()): + yield LogPresenter(prefix_width, color_func) -def max_name_width(containers): +def max_name_width(service_names, max_index_width=3): """Calculate the maximum width of container names so we can make the log prefixes line up like so: db_1 | Listening web_1 | Listening """ - return max(len(container.name_without_project) for container in containers) + return max(len(name) for name in service_names) + max_index_width + + +class LogPrinter(object): + """Print logs from many containers to a single output stream.""" + + def __init__(self, + containers, + presenters, + event_stream, + output=sys.stdout, + cascade_stop=False, + log_args=None): + self.containers = containers + self.presenters = presenters + self.event_stream = event_stream + self.output = utils.get_output_stream(output) + self.cascade_stop = cascade_stop + self.log_args = log_args or {} + + def run(self): + if not self.containers: + return + + queue = Queue() + thread_args = queue, self.log_args + thread_map = build_thread_map(self.containers, self.presenters, thread_args) + start_producer_thread(( + thread_map, + self.event_stream, + self.presenters, + thread_args)) + + for line in consume_queue(queue, self.cascade_stop): + remove_stopped_threads(thread_map) + + if not line: + if not thread_map: + # There are no running containers left to tail, so exit + return + # We got an empty line because of a timeout, but there are still + # active containers to tail, so continue + continue + + self.output.write(line) + self.output.flush() + + +def remove_stopped_threads(thread_map): + for container_id, tailer_thread in list(thread_map.items()): + if not tailer_thread.is_alive(): + thread_map.pop(container_id, None) + + +def build_thread(container, presenter, queue, log_args): + tailer = Thread( + target=tail_container_logs, + args=(container, presenter, queue, log_args)) + tailer.daemon = True + tailer.start() + return tailer + + +def build_thread_map(initial_containers, presenters, thread_args): + return { + container.id: build_thread(container, next(presenters), *thread_args) + for container in initial_containers + } + + +class QueueItem(namedtuple('_QueueItem', 'item is_stop exc')): + + @classmethod + def new(cls, item): + return cls(item, None, None) + + @classmethod + def exception(cls, exc): + return cls(None, None, exc) + + @classmethod + def stop(cls): + return cls(None, True, None) + + +def tail_container_logs(container, presenter, queue, log_args): + generator = get_log_generator(container) + + try: + for item in generator(container, log_args): + queue.put(QueueItem.new(presenter.present(container, item))) + except Exception as e: + queue.put(QueueItem.exception(e)) + return + + if log_args.get('follow'): + queue.put(QueueItem.new(presenter.color_func(wait_on_exit(container)))) + queue.put(QueueItem.stop()) def get_log_generator(container): @@ -64,30 +157,81 @@ def get_log_generator(container): return build_no_log_generator -def build_no_log_generator(container, prefix, color_func): +def build_no_log_generator(container, log_args): """Return a generator that prints a warning about logs and waits for container to exit. """ - yield "{} WARNING: no logs are available with the '{}' log driver\n".format( - prefix, + yield "WARNING: no logs are available with the '{}' log driver\n".format( container.log_driver) - yield color_func(wait_on_exit(container)) -def build_log_generator(container, prefix, color_func): +def build_log_generator(container, log_args): # if the container doesn't have a log_stream we need to attach to container # before log printer starts running if container.log_stream is None: - stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) - line_generator = split_buffer(stream) + stream = container.logs(stdout=True, stderr=True, stream=True, **log_args) else: - line_generator = split_buffer(container.log_stream) + stream = container.log_stream - for line in line_generator: - yield prefix + line - yield color_func(wait_on_exit(container)) + return split_buffer(stream) def wait_on_exit(container): - exit_code = container.wait() - return "%s exited with code %s\n" % (container.name, exit_code) + try: + exit_code = container.wait() + return "%s exited with code %s\n" % (container.name, exit_code) + except APIError as e: + return "Unexpected API error for %s (HTTP code %s)\nResponse body:\n%s\n" % ( + container.name, e.response.status_code, + e.response.text or '[empty]' + ) + + +def start_producer_thread(thread_args): + producer = Thread(target=watch_events, args=thread_args) + producer.daemon = True + producer.start() + + +def watch_events(thread_map, event_stream, presenters, thread_args): + for event in event_stream: + if event['action'] == 'stop': + thread_map.pop(event['id'], None) + + if event['action'] != 'start': + continue + + if event['id'] in thread_map: + if thread_map[event['id']].is_alive(): + continue + # Container was stopped and started, we need a new thread + thread_map.pop(event['id'], None) + + thread_map[event['id']] = build_thread( + event['container'], + next(presenters), + *thread_args) + + +def consume_queue(queue, cascade_stop): + """Consume the queue by reading lines off of it and yielding them.""" + while True: + try: + item = queue.get(timeout=0.1) + except Empty: + yield None + continue + # See https://github.com/docker/compose/issues/189 + except thread.error: + raise ShutdownException() + + if item.exc: + raise item.exc + + if item.is_stop: + if cascade_stop: + raise StopIteration + else: + continue + + yield item.item diff --git a/compose/cli/main.py b/compose/cli/main.py index 4be8536f..51ba36a0 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -2,69 +2,73 @@ from __future__ import absolute_import from __future__ import print_function from __future__ import unicode_literals +import contextlib +import functools import json import logging +import pipes import re +import subprocess import sys +from distutils.spawn import find_executable from inspect import getdoc from operator import attrgetter -from docker.errors import APIError -from requests.exceptions import ReadTimeout - +from . import errors from . import signals from .. import __version__ -from ..config import config +from ..bundle import get_image_digests +from ..bundle import MissingDigests +from ..bundle import serialize_bundle from ..config import ConfigurationError from ..config import parse_environment +from ..config.environment import Environment from ..config.serialize import serialize_config -from ..const import DEFAULT_TIMEOUT -from ..const import HTTP_TIMEOUT from ..const import IS_WINDOWS_PLATFORM +from ..errors import StreamParseError from ..progress_stream import StreamOutputError from ..project import NoSuchService +from ..project import OneOffFilter +from ..project import ProjectError +from ..service import BuildAction from ..service import BuildError from ..service import ConvergenceStrategy from ..service import ImageType from ..service import NeedsBuildError -from .command import friendly_error_message -from .command import get_config_path_from_options +from ..service import OperationFailedError +from .command import get_config_from_options from .command import project_from_options -from .docopt_command import DocoptCommand +from .docopt_command import DocoptDispatcher +from .docopt_command import get_handler from .docopt_command import NoSuchCommand from .errors import UserError from .formatter import ConsoleWarningFormatter from .formatter import Formatter +from .log_printer import build_log_presenters from .log_printer import LogPrinter from .utils import get_version_info from .utils import yesno if not IS_WINDOWS_PLATFORM: - import dockerpty + from dockerpty.pty import PseudoTerminal, RunOperation, ExecOperation log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) def main(): - setup_logging() + command = dispatch() + try: - command = TopLevelCommand() - command.sys_dispatch() - except KeyboardInterrupt: - log.error("\nAborting.") + command() + except (KeyboardInterrupt, signals.ShutdownException): + log.error("Aborting.") sys.exit(1) - except (UserError, NoSuchService, ConfigurationError) as e: + except (UserError, NoSuchService, ConfigurationError, + ProjectError, OperationFailedError) as e: log.error(e.msg) sys.exit(1) - except NoSuchCommand as e: - commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) - log.error("No such command: %s\n\n%s", e.command, commands) - sys.exit(1) - except APIError as e: - log.error(e.explanation) - sys.exit(1) except BuildError as e: log.error("Service '%s' failed to build: %s" % (e.service.name, e.reason)) sys.exit(1) @@ -74,15 +78,44 @@ def main(): except NeedsBuildError as e: log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name) sys.exit(1) - except ReadTimeout as e: - log.error( - "An HTTP request took too long to complete. Retry with --verbose to obtain debug information.\n" - "If you encounter this issue regularly because of slow network conditions, consider setting " - "COMPOSE_HTTP_TIMEOUT to a higher value (current value: %s)." % HTTP_TIMEOUT - ) + except (errors.ConnectionError, StreamParseError): sys.exit(1) +def dispatch(): + setup_logging() + dispatcher = DocoptDispatcher( + TopLevelCommand, + {'options_first': True, 'version': get_version_info('compose')}) + + try: + options, handler, command_options = dispatcher.parse(sys.argv[1:]) + except NoSuchCommand as e: + commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) + log.error("No such command: %s\n\n%s", e.command, commands) + sys.exit(1) + + setup_console_handler(console_handler, options.get('--verbose')) + return functools.partial(perform_command, options, handler, command_options) + + +def perform_command(options, handler, command_options): + if options['COMMAND'] in ('help', 'version'): + # Skip looking up the compose file. + handler(command_options) + return + + if options['COMMAND'] in ('config', 'bundle'): + command = TopLevelCommand(None) + handler(command, options, command_options) + return + + project = project_from_options('.', options) + command = TopLevelCommand(project) + with errors.handle_connection_errors(project.client): + handler(command, command_options) + + def setup_logging(): root_logger = logging.getLogger() root_logger.addHandler(console_handler) @@ -113,66 +146,62 @@ def parse_doc_section(name, source): return [s.strip() for s in pattern.findall(source)] -class TopLevelCommand(DocoptCommand): +class TopLevelCommand(object): """Define and run multi-container applications with Docker. Usage: - docker-compose [-f=...] [options] [COMMAND] [ARGS...] + docker-compose [-f ...] [options] [COMMAND] [ARGS...] docker-compose -h|--help Options: - -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) - -p, --project-name NAME Specify an alternate project name (default: directory name) - --verbose Show more output - -v, --version Print version and exit + -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) + -p, --project-name NAME Specify an alternate project name (default: directory name) + --verbose Show more output + -v, --version Print version and exit + -H, --host HOST Daemon socket to connect to + + --tls Use TLS; implied by --tlsverify + --tlscacert CA_PATH Trust certs signed only by this CA + --tlscert CLIENT_CERT_PATH Path to TLS certificate file + --tlskey TLS_KEY_PATH Path to TLS key file + --tlsverify Use TLS and verify the remote + --skip-hostname-check Don't check the daemon's hostname against the name specified + in the client certificate (for example if your docker host + is an IP address) Commands: build Build or rebuild services + bundle Generate a Docker bundle from the Compose file config Validate and view the compose file create Create services down Stop and remove containers, networks, images, and volumes events Receive real time events from containers + exec Execute a command in a running container help Get help on a command kill Kill containers logs View output from containers pause Pause services port Print the public port for a port binding ps List containers - pull Pulls service images + pull Pull service images + push Push service images restart Restart services rm Remove stopped containers run Run a one-off command scale Set number of containers for a service start Start services stop Stop services + top Display the running processes unpause Unpause services up Create and start containers version Show the Docker-Compose version information """ - base_dir = '.' - def docopt_options(self): - options = super(TopLevelCommand, self).docopt_options() - options['version'] = get_version_info('compose') - return options + def __init__(self, project, project_dir='.'): + self.project = project + self.project_dir = '.' - def perform_command(self, options, handler, command_options): - setup_console_handler(console_handler, options.get('--verbose')) - - if options['COMMAND'] in ('help', 'version'): - # Skip looking up the compose file. - handler(None, command_options) - return - - if options['COMMAND'] == 'config': - handler(options, command_options) - return - - project = project_from_options(self.base_dir, options) - with friendly_error_message(): - handler(project, command_options) - - def build(self, project, options): + def build(self, options): """ Build or rebuild services. @@ -187,12 +216,81 @@ class TopLevelCommand(DocoptCommand): --no-cache Do not use cache when building the image. --pull Always attempt to pull a newer version of the image. """ - project.build( + self.project.build( service_names=options['SERVICE'], no_cache=bool(options.get('--no-cache', False)), pull=bool(options.get('--pull', False)), force_rm=bool(options.get('--force-rm', False))) + def bundle(self, config_options, options): + """ + Generate a Distributed Application Bundle (DAB) from the Compose file. + + Images must have digests stored, which requires interaction with a + Docker registry. If digests aren't stored for all images, you can fetch + them with `docker-compose pull` or `docker-compose push`. To push images + automatically when bundling, pass `--push-images`. Only services with + a `build` option specified will have their images pushed. + + Usage: bundle [options] + + Options: + --push-images Automatically push images for any services + which have a `build` option specified. + + -o, --output PATH Path to write the bundle file to. + Defaults to ".dab". + """ + self.project = project_from_options('.', config_options) + compose_config = get_config_from_options(self.project_dir, config_options) + + output = options["--output"] + if not output: + output = "{}.dab".format(self.project.name) + + with errors.handle_connection_errors(self.project.client): + try: + image_digests = get_image_digests( + self.project, + allow_push=options['--push-images'], + ) + except MissingDigests as e: + def list_images(images): + return "\n".join(" {}".format(name) for name in sorted(images)) + + paras = ["Some images are missing digests."] + + if e.needs_push: + command_hint = ( + "Use `docker-compose push {}` to push them. " + "You can do this automatically with `docker-compose bundle --push-images`." + .format(" ".join(sorted(e.needs_push))) + ) + paras += [ + "The following images can be pushed:", + list_images(e.needs_push), + command_hint, + ] + + if e.needs_pull: + command_hint = ( + "Use `docker-compose pull {}` to pull them. " + .format(" ".join(sorted(e.needs_pull))) + ) + + paras += [ + "The following images need to be pulled:", + list_images(e.needs_pull), + command_hint, + ] + + raise UserError("\n\n".join(paras)) + + with open(output, 'w') as f: + f.write(serialize_bundle(compose_config, image_digests)) + + log.info("Wrote bundle to {}".format(output)) + def config(self, config_options, options): """ Validate and view the compose file. @@ -205,8 +303,7 @@ class TopLevelCommand(DocoptCommand): --services Print the service names, one per line. """ - config_path = get_config_path_from_options(config_options) - compose_config = config.load(config.find(self.base_dir, config_path)) + compose_config = get_config_from_options(self.project_dir, config_options) if options['--quiet']: return @@ -217,7 +314,7 @@ class TopLevelCommand(DocoptCommand): print(serialize_config(compose_config)) - def create(self, project, options): + def create(self, options): """ Creates containers for a service. @@ -228,33 +325,47 @@ class TopLevelCommand(DocoptCommand): image haven't changed. Incompatible with --no-recreate. --no-recreate If containers already exist, don't recreate them. Incompatible with --force-recreate. - --no-build Don't build an image, even if it's missing + --no-build Don't build an image, even if it's missing. + --build Build images before creating containers. """ service_names = options['SERVICE'] - project.create( + self.project.create( service_names=service_names, strategy=convergence_strategy_from_opts(options), - do_build=not options['--no-build'] + do_build=build_action_from_opts(options), ) - def down(self, project, options): + def down(self, options): """ - Stop containers and remove containers, networks, volumes, and images - created by `up`. Only containers and networks are removed by default. + Stops containers and removes containers, networks, volumes, and images + created by `up`. + + By default, the only things removed are: + + - Containers for services defined in the Compose file + - Networks defined in the `networks` section of the Compose file + - The default network, if one is used + + Networks and volumes defined as `external` are never removed. Usage: down [options] Options: - --rmi type Remove images, type may be one of: 'all' to remove - all images, or 'local' to remove only images that - don't have an custom name set by the `image` field - -v, --volumes Remove data volumes + --rmi type Remove images. Type must be one of: + 'all': Remove all images used by any service. + 'local': Remove only images that don't have a custom tag + set by the `image` field. + -v, --volumes Remove named volumes declared in the `volumes` section + of the Compose file and anonymous volumes + attached to containers. + --remove-orphans Remove containers for services not defined in the + Compose file """ image_type = image_type_from_opt('--rmi', options['--rmi']) - project.down(image_type, options['--volumes']) + self.project.down(image_type, options['--volumes'], options['--remove-orphans']) - def events(self, project, options): + def events(self, options): """ Receive real time events from containers. @@ -271,23 +382,104 @@ class TopLevelCommand(DocoptCommand): def json_format_event(event): event['time'] = event['time'].isoformat() + event.pop('container') return json.dumps(event) - for event in project.events(): + for event in self.project.events(): formatter = json_format_event if options['--json'] else format_event print(formatter(event)) sys.stdout.flush() - def help(self, project, options): + def exec_command(self, options): + """ + Execute a command in a running container + + Usage: exec [options] SERVICE COMMAND [ARGS...] + + Options: + -d Detached mode: Run command in the background. + --privileged Give extended privileges to the process. + --user USER Run the command as this user. + -T Disable pseudo-tty allocation. By default `docker-compose exec` + allocates a TTY. + --index=index index of the container if there are multiple + instances of a service [default: 1] + """ + index = int(options.get('--index')) + service = self.project.get_service(options['SERVICE']) + detach = options['-d'] + + try: + container = service.get_container(number=index) + except ValueError as e: + raise UserError(str(e)) + command = [options['COMMAND']] + options['ARGS'] + tty = not options["-T"] + + if IS_WINDOWS_PLATFORM and not detach: + args = ["exec"] + + if options["-d"]: + args += ["--detach"] + else: + args += ["--interactive"] + + if not options["-T"]: + args += ["--tty"] + + if options["--privileged"]: + args += ["--privileged"] + + if options["--user"]: + args += ["--user", options["--user"]] + + args += [container.id] + args += command + + sys.exit(call_docker(args)) + + create_exec_options = { + "privileged": options["--privileged"], + "user": options["--user"], + "tty": tty, + "stdin": tty, + } + + exec_id = container.create_exec(command, **create_exec_options) + + if detach: + container.start_exec(exec_id, tty=tty) + return + + signals.set_signal_handler_to_shutdown() + try: + operation = ExecOperation( + self.project.client, + exec_id, + interactive=tty, + ) + pty = PseudoTerminal(self.project.client, operation) + pty.start() + except signals.ShutdownException: + log.info("received shutdown exception: closing") + exit_code = self.project.client.exec_inspect(exec_id).get("ExitCode") + sys.exit(exit_code) + + @classmethod + def help(cls, options): """ Get help on a command. - Usage: help COMMAND + Usage: help [COMMAND] """ - handler = self.get_handler(options['COMMAND']) - raise SystemExit(getdoc(handler)) + if options['COMMAND']: + subject = get_handler(cls, options['COMMAND']) + else: + subject = cls - def kill(self, project, options): + print(getdoc(subject)) + + def kill(self, options): """ Force stop service containers. @@ -299,33 +491,52 @@ class TopLevelCommand(DocoptCommand): """ signal = options.get('-s', 'SIGKILL') - project.kill(service_names=options['SERVICE'], signal=signal) + self.project.kill(service_names=options['SERVICE'], signal=signal) - def logs(self, project, options): + def logs(self, options): """ View output from containers. Usage: logs [options] [SERVICE...] Options: - --no-color Produce monochrome output. + --no-color Produce monochrome output. + -f, --follow Follow log output. + -t, --timestamps Show timestamps. + --tail="all" Number of lines to show from the end of the logs + for each container. """ - containers = project.containers(service_names=options['SERVICE'], stopped=True) + containers = self.project.containers(service_names=options['SERVICE'], stopped=True) - monochrome = options['--no-color'] + tail = options['--tail'] + if tail is not None: + if tail.isdigit(): + tail = int(tail) + elif tail != 'all': + raise UserError("tail flag must be all or a number") + log_args = { + 'follow': options['--follow'], + 'tail': tail, + 'timestamps': options['--timestamps'] + } print("Attaching to", list_containers(containers)) - LogPrinter(containers, monochrome=monochrome).run() + log_printer_from_project( + self.project, + containers, + options['--no-color'], + log_args, + event_stream=self.project.events(service_names=options['SERVICE'])).run() - def pause(self, project, options): + def pause(self, options): """ Pause services. Usage: pause [SERVICE...] """ - containers = project.pause(service_names=options['SERVICE']) + containers = self.project.pause(service_names=options['SERVICE']) exit_if(not containers, 'No containers to pause', 1) - def port(self, project, options): + def port(self, options): """ Print the public port for a port binding. @@ -337,7 +548,7 @@ class TopLevelCommand(DocoptCommand): instances of a service [default: 1] """ index = int(options.get('--index')) - service = project.get_service(options['SERVICE']) + service = self.project.get_service(options['SERVICE']) try: container = service.get_container(number=index) except ValueError as e: @@ -346,7 +557,7 @@ class TopLevelCommand(DocoptCommand): options['PRIVATE_PORT'], protocol=options.get('--protocol') or 'tcp') or '') - def ps(self, project, options): + def ps(self, options): """ List containers. @@ -356,8 +567,8 @@ class TopLevelCommand(DocoptCommand): -q Only display IDs """ containers = sorted( - project.containers(service_names=options['SERVICE'], stopped=True) + - project.containers(service_names=options['SERVICE'], one_off=True), + self.project.containers(service_names=options['SERVICE'], stopped=True) + + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), key=attrgetter('name')) if options['-q']: @@ -383,7 +594,7 @@ class TopLevelCommand(DocoptCommand): ]) print(Formatter().table(headers, rows)) - def pull(self, project, options): + def pull(self, options): """ Pulls images for services. @@ -392,17 +603,31 @@ class TopLevelCommand(DocoptCommand): Options: --ignore-pull-failures Pull what it can and ignores images with pull failures. """ - project.pull( + self.project.pull( service_names=options['SERVICE'], ignore_pull_failures=options.get('--ignore-pull-failures') ) - def rm(self, project, options): + def push(self, options): """ - Remove stopped service containers. + Pushes images for services. - By default, volumes attached to containers will not be removed. You can see all - volumes with `docker volume ls`. + Usage: push [options] [SERVICE...] + + Options: + --ignore-push-failures Push what it can and ignores images with push failures. + """ + self.project.push( + service_names=options['SERVICE'], + ignore_push_failures=options.get('--ignore-push-failures') + ) + + def rm(self, options): + """ + Removes stopped service containers. + + By default, anonymous volumes attached to containers will not be removed. You + can override this with `-v`. To list all volumes, use `docker volume ls`. Any data which is not in a volume will be lost. @@ -410,23 +635,34 @@ class TopLevelCommand(DocoptCommand): Options: -f, --force Don't ask to confirm removal - -v Remove volumes associated with containers + -v Remove any anonymous volumes attached to containers + -a, --all Deprecated - no effect. """ - all_containers = project.containers(service_names=options['SERVICE'], stopped=True) + if options.get('--all'): + log.warn( + '--all flag is obsolete. This is now the default behavior ' + 'of `docker-compose rm`' + ) + one_off = OneOffFilter.include + + all_containers = self.project.containers( + service_names=options['SERVICE'], stopped=True, one_off=one_off + ) stopped_containers = [c for c in all_containers if not c.is_running] if len(stopped_containers) > 0: print("Going to remove", list_containers(stopped_containers)) if options.get('--force') \ or yesno("Are you sure? [yN] ", default=False): - project.remove_stopped( + self.project.remove_stopped( service_names=options['SERVICE'], - v=options.get('-v', False) + v=options.get('-v', False), + one_off=one_off ) else: print("No stopped containers") - def run(self, project, options): + def run(self, options): """ Run a one-off command on a service. @@ -454,58 +690,28 @@ class TopLevelCommand(DocoptCommand): to the host. -T Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY. + -w, --workdir="" Working directory inside the container """ - service = project.get_service(options['SERVICE']) + service = self.project.get_service(options['SERVICE']) detach = options['-d'] - if IS_WINDOWS_PLATFORM and not detach: - raise UserError( - "Interactive mode is not yet supported on Windows.\n" - "Please pass the -d flag when using `docker-compose run`." - ) - - if options['COMMAND']: - command = [options['COMMAND']] + options['ARGS'] - else: - command = service.options.get('command') - - container_options = { - 'command': command, - 'tty': not (detach or options['-T'] or not sys.stdin.isatty()), - 'stdin_open': not detach, - 'detach': detach, - } - - if options['-e']: - container_options['environment'] = parse_environment(options['-e']) - - if options['--entrypoint']: - container_options['entrypoint'] = options.get('--entrypoint') - - if options['--rm']: - container_options['restart'] = None - - if options['--user']: - container_options['user'] = options.get('--user') - - if not options['--service-ports']: - container_options['ports'] = [] - - if options['--publish']: - container_options['ports'] = options.get('--publish') - if options['--publish'] and options['--service-ports']: raise UserError( 'Service port mapping and manual port mapping ' - 'can not be used togather' + 'can not be used together' ) - if options['--name']: - container_options['name'] = options['--name'] + if options['COMMAND'] is not None: + command = [options['COMMAND']] + options['ARGS'] + elif options['--entrypoint'] is not None: + command = [] + else: + command = service.options.get('command') - run_one_off_container(container_options, project, service, options) + container_options = build_container_options(options, detach, command) + run_one_off_container(container_options, self.project, service, options) - def scale(self, project, options): + def scale(self, options): """ Set number of containers to run for a service. @@ -520,7 +726,7 @@ class TopLevelCommand(DocoptCommand): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = timeout_from_opts(options) for s in options['SERVICE=NUM']: if '=' not in s: @@ -531,18 +737,18 @@ class TopLevelCommand(DocoptCommand): except ValueError: raise UserError('Number of containers for service "%s" is not a ' 'number' % service_name) - project.get_service(service_name).scale(num, timeout=timeout) + self.project.get_service(service_name).scale(num, timeout=timeout) - def start(self, project, options): + def start(self, options): """ Start existing containers. Usage: start [SERVICE...] """ - containers = project.start(service_names=options['SERVICE']) + containers = self.project.start(service_names=options['SERVICE']) exit_if(not containers, 'No containers to start', 1) - def stop(self, project, options): + def stop(self, options): """ Stop running containers without removing them. @@ -554,10 +760,10 @@ class TopLevelCommand(DocoptCommand): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) - project.stop(service_names=options['SERVICE'], timeout=timeout) + timeout = timeout_from_opts(options) + self.project.stop(service_names=options['SERVICE'], timeout=timeout) - def restart(self, project, options): + def restart(self, options): """ Restart running containers. @@ -567,20 +773,47 @@ class TopLevelCommand(DocoptCommand): -t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) """ - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) - containers = project.restart(service_names=options['SERVICE'], timeout=timeout) + timeout = timeout_from_opts(options) + containers = self.project.restart(service_names=options['SERVICE'], timeout=timeout) exit_if(not containers, 'No containers to restart', 1) - def unpause(self, project, options): + def top(self, options): + """ + Display the running processes + + Usage: top [SERVICE...] + + """ + containers = sorted( + self.project.containers(service_names=options['SERVICE'], stopped=False) + + self.project.containers(service_names=options['SERVICE'], one_off=OneOffFilter.only), + key=attrgetter('name') + ) + + for idx, container in enumerate(containers): + if idx > 0: + print() + + top_data = self.project.client.top(container.name) + headers = top_data.get("Titles") + rows = [] + + for process in top_data.get("Processes", []): + rows.append(process) + + print(container.name) + print(Formatter().table(headers, rows)) + + def unpause(self, options): """ Unpause services. Usage: unpause [SERVICE...] """ - containers = project.unpause(service_names=options['SERVICE']) + containers = self.project.unpause(service_names=options['SERVICE']) exit_if(not containers, 'No containers to unpause', 1) - def up(self, project, options): + def up(self, options): """ Builds, (re)creates, starts, and attaches to containers for a service. @@ -612,37 +845,55 @@ class TopLevelCommand(DocoptCommand): Incompatible with --no-recreate. --no-recreate If containers already exist, don't recreate them. Incompatible with --force-recreate. - --no-build Don't build an image, even if it's missing + --no-build Don't build an image, even if it's missing. + --build Build images before starting containers. --abort-on-container-exit Stops all containers if any container was stopped. Incompatible with -d. -t, --timeout TIMEOUT Use this timeout in seconds for container shutdown when attached or when containers are already running. (default: 10) + --remove-orphans Remove containers for services not + defined in the Compose file """ - monochrome = options['--no-color'] start_deps = not options['--no-deps'] cascade_stop = options['--abort-on-container-exit'] service_names = options['SERVICE'] - timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + timeout = timeout_from_opts(options) + remove_orphans = options['--remove-orphans'] detached = options.get('-d') if detached and cascade_stop: raise UserError("--abort-on-container-exit and -d cannot be combined.") - to_attach = project.up( - service_names=service_names, - start_deps=start_deps, - strategy=convergence_strategy_from_opts(options), - do_build=not options['--no-build'], - timeout=timeout, - detached=detached - ) + with up_shutdown_context(self.project, service_names, timeout, detached): + to_attach = self.project.up( + service_names=service_names, + start_deps=start_deps, + strategy=convergence_strategy_from_opts(options), + do_build=build_action_from_opts(options), + timeout=timeout, + detached=detached, + remove_orphans=remove_orphans) - if not detached: - log_printer = build_log_printer(to_attach, service_names, monochrome, cascade_stop) - attach_to_logs(project, log_printer, service_names, timeout) + if detached: + return - def version(self, project, options): + log_printer = log_printer_from_project( + self.project, + filter_containers_to_service_names(to_attach, service_names), + options['--no-color'], + {'follow': True}, + cascade_stop, + event_stream=self.project.events(service_names=service_names)) + print("Attaching to", list_containers(log_printer.containers)) + log_printer.run() + + if cascade_stop: + print("Aborting on container exit...") + self.project.stop(service_names=service_names, timeout=timeout) + + @classmethod + def version(cls, options): """ Show version informations @@ -672,6 +923,11 @@ def convergence_strategy_from_opts(options): return ConvergenceStrategy.changed +def timeout_from_opts(options): + timeout = options.get('--timeout') + return None if timeout is None else int(timeout) + + def image_type_from_opt(flag, value): if not value: return ImageType.none @@ -681,16 +937,66 @@ def image_type_from_opt(flag, value): raise UserError("%s flag must be one of: all, local" % flag) +def build_action_from_opts(options): + if options['--build'] and options['--no-build']: + raise UserError("--build and --no-build can not be combined.") + + if options['--build']: + return BuildAction.force + + if options['--no-build']: + return BuildAction.skip + + return BuildAction.none + + +def build_container_options(options, detach, command): + container_options = { + 'command': command, + 'tty': not (detach or options['-T'] or not sys.stdin.isatty()), + 'stdin_open': not detach, + 'detach': detach, + } + + if options['-e']: + container_options['environment'] = Environment.from_command_line( + parse_environment(options['-e']) + ) + + if options['--entrypoint']: + container_options['entrypoint'] = options.get('--entrypoint') + + if options['--rm']: + container_options['restart'] = None + + if options['--user']: + container_options['user'] = options.get('--user') + + if not options['--service-ports']: + container_options['ports'] = [] + + if options['--publish']: + container_options['ports'] = options.get('--publish') + + if options['--name']: + container_options['name'] = options['--name'] + + if options['--workdir']: + container_options['working_dir'] = options['--workdir'] + + return container_options + + def run_one_off_container(container_options, project, service, options): if not options['--no-deps']: - deps = service.get_linked_service_names() + deps = service.get_dependency_names() if deps: project.up( service_names=deps, start_deps=True, strategy=ConvergenceStrategy.never) - project.initialize_networks() + project.initialize() container = service.create_container( quiet=True, @@ -709,9 +1015,21 @@ def run_one_off_container(container_options, project, service, options): signals.set_signal_handler_to_shutdown() try: try: - dockerpty.start(project.client, container.id, interactive=not options['-T']) - service.connect_container_to_networks(container) - exit_code = container.wait() + if IS_WINDOWS_PLATFORM: + service.connect_container_to_networks(container) + exit_code = call_docker(["start", "--attach", "--interactive", container.id]) + else: + operation = RunOperation( + project.client, + container.id, + interactive=not options['-T'], + logs=False, + ) + pty = PseudoTerminal(project.client, operation) + sockets = pty.sockets() + service.start_container(container) + pty.start(sockets) + exit_code = container.wait() except signals.ShutdownException: project.client.stop(container.id) exit_code = 1 @@ -724,22 +1042,42 @@ def run_one_off_container(container_options, project, service, options): sys.exit(exit_code) -def build_log_printer(containers, service_names, monochrome, cascade_stop): - if service_names: - containers = [ - container - for container in containers if container.service in service_names - ] - return LogPrinter(containers, monochrome=monochrome, cascade_stop=cascade_stop) +def log_printer_from_project( + project, + containers, + monochrome, + log_args, + cascade_stop=False, + event_stream=None, +): + return LogPrinter( + containers, + build_log_presenters(project.service_names, monochrome), + event_stream or project.events(), + cascade_stop=cascade_stop, + log_args=log_args) -def attach_to_logs(project, log_printer, service_names, timeout): - print("Attaching to", list_containers(log_printer.containers)) +def filter_containers_to_service_names(containers, service_names): + if not service_names: + return containers + + return [ + container + for container in containers if container.service in service_names + ] + + +@contextlib.contextmanager +def up_shutdown_context(project, service_names, timeout, detached): + if detached: + yield + return + signals.set_signal_handler_to_shutdown() - try: try: - log_printer.run() + yield except signals.ShutdownException: print("Gracefully stopping... (press Ctrl+C again to force)") project.stop(service_names=service_names, timeout=timeout) @@ -756,3 +1094,14 @@ def exit_if(condition, message, exit_code): if condition: log.error(message) raise SystemExit(exit_code) + + +def call_docker(args): + executable_path = find_executable('docker') + if not executable_path: + raise UserError(errors.docker_not_found_msg("Couldn't find `docker` binary.")) + + args = [executable_path] + args + log.debug(" ".join(map(pipes.quote, args))) + + return subprocess.call(args) diff --git a/compose/cli/multiplexer.py b/compose/cli/multiplexer.py deleted file mode 100644 index e6e63f24..00000000 --- a/compose/cli/multiplexer.py +++ /dev/null @@ -1,65 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -from threading import Thread - -from six.moves import _thread as thread - -try: - from Queue import Queue, Empty -except ImportError: - from queue import Queue, Empty # Python 3.x - - -STOP = object() - - -class Multiplexer(object): - """ - Create a single iterator from several iterators by running all of them in - parallel and yielding results as they come in. - """ - - def __init__(self, iterators, cascade_stop=False): - self.iterators = iterators - self.cascade_stop = cascade_stop - self._num_running = len(iterators) - self.queue = Queue() - - def loop(self): - self._init_readers() - - while self._num_running > 0: - try: - item, exception = self.queue.get(timeout=0.1) - - if exception: - raise exception - - if item is STOP: - if self.cascade_stop is True: - break - else: - self._num_running -= 1 - else: - yield item - except Empty: - pass - # See https://github.com/docker/compose/issues/189 - except thread.error: - raise KeyboardInterrupt() - - def _init_readers(self): - for iterator in self.iterators: - t = Thread(target=_enqueue_output, args=(iterator, self.queue)) - t.daemon = True - t.start() - - -def _enqueue_output(iterator, queue): - try: - for item in iterator: - queue.put((item, None)) - queue.put((STOP, None)) - except Exception as e: - queue.put((None, e)) diff --git a/compose/cli/utils.py b/compose/cli/utils.py index dd859edc..580bd1b0 100644 --- a/compose/cli/utils.py +++ b/compose/cli/utils.py @@ -6,11 +6,19 @@ import os import platform import ssl import subprocess +import sys import docker -from six.moves import input import compose +from ..const import IS_WINDOWS_PLATFORM + +# WindowsError is not defined on non-win32 platforms. Avoid runtime errors by +# defining it as OSError (its parent class) if missing. +try: + WindowsError +except NameError: + WindowsError = OSError def yesno(prompt, default=None): @@ -35,6 +43,16 @@ def yesno(prompt, default=None): return None +def input(prompt): + """ + Version of input (raw_input in Python 2) which forces a flush of sys.stdout + to avoid problems where the prompt fails to appear due to line buffering + """ + sys.stdout.write(prompt) + sys.stdout.flush() + return sys.stdin.readline().rstrip('\n') + + def call_silently(*args, **kwargs): """ Like subprocess.call(), but redirects stdout and stderr to /dev/null. @@ -56,6 +74,10 @@ def is_ubuntu(): return platform.system() == 'Linux' and platform.linux_distribution()[0] == 'Ubuntu' +def is_windows(): + return IS_WINDOWS_PLATFORM + + def get_version_info(scope): versioninfo = 'docker-compose version {}, build {}'.format( compose.__version__, @@ -86,3 +108,30 @@ def get_build_version(): with open(filename) as fh: return fh.read().strip() + + +def is_docker_for_mac_installed(): + return is_mac() and os.path.isdir('/Applications/Docker.app') + + +def generate_user_agent(): + parts = [ + "docker-compose/{}".format(compose.__version__), + "docker-py/{}".format(docker.__version__), + ] + try: + p_system = platform.system() + p_release = platform.release() + except IOError: + pass + else: + parts.append("{}/{}".format(p_system, p_release)) + return " ".join(parts) + + +def unquote_path(s): + if not s: + return s + if s[0] == '"' and s[-1] == '"': + return s[1:-1] + return s diff --git a/compose/config/__init__.py b/compose/config/__init__.py index dd01f221..7cf71eb9 100644 --- a/compose/config/__init__.py +++ b/compose/config/__init__.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals +from . import environment from .config import ConfigurationError from .config import DOCKER_CONFIG_KEYS from .config import find diff --git a/compose/config/config.py b/compose/config/config.py index b3894225..4c9cf423 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,10 +1,8 @@ from __future__ import absolute_import from __future__ import unicode_literals -import codecs import functools import logging -import operator import os import string import sys @@ -14,22 +12,41 @@ import six import yaml from cached_property import cached_property -from ..const import COMPOSEFILE_VERSIONS +from . import types +from ..const import COMPOSEFILE_V1 as V1 +from ..const import COMPOSEFILE_V2_0 as V2_0 +from ..const import COMPOSEFILE_V2_1 as V2_1 +from ..const import COMPOSEFILE_V3_0 as V3_0 +from ..const import COMPOSEFILE_V3_1 as V3_1 +from ..utils import build_string_dict +from ..utils import parse_nanoseconds_int +from ..utils import splitdrive +from .environment import env_vars_from_file +from .environment import Environment +from .environment import split_env from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError +from .errors import VERSION_EXPLANATION from .interpolation import interpolate_environment_variables -from .sort_services import get_service_name_from_net +from .sort_services import get_container_name_from_network_mode +from .sort_services import get_service_name_from_network_mode from .sort_services import sort_service_dicts from .types import parse_extra_hosts from .types import parse_restart_spec +from .types import ServiceLink from .types import VolumeFromSpec from .types import VolumeSpec -from .validation import validate_against_fields_schema -from .validation import validate_against_service_schema +from .validation import match_named_volumes +from .validation import validate_against_config_schema +from .validation import validate_config_section +from .validation import validate_depends_on from .validation import validate_extends_file_path +from .validation import validate_links +from .validation import validate_network_mode +from .validation import validate_service_constraints from .validation import validate_top_level_object -from .validation import validate_top_level_service_objects +from .validation import validate_ulimits DOCKER_CONFIG_KEYS = [ @@ -49,7 +66,9 @@ DOCKER_CONFIG_KEYS = [ 'env_file', 'environment', 'extra_hosts', + 'group_add', 'hostname', + 'healthcheck', 'image', 'ipc', 'labels', @@ -57,17 +76,23 @@ DOCKER_CONFIG_KEYS = [ 'mac_address', 'mem_limit', 'memswap_limit', + 'mem_swappiness', 'net', + 'oom_score_adj', 'pid', 'ports', 'privileged', 'read_only', 'restart', + 'secrets', 'security_opt', + 'shm_size', 'stdin_open', 'stop_signal', + 'sysctls', 'tty', 'user', + 'userns_mode', 'volume_driver', 'volumes', 'volumes_from', @@ -78,9 +103,10 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [ 'build', 'container_name', 'dockerfile', - 'expose', - 'external_links', + 'log_driver', + 'log_opt', 'logging', + 'network_mode', ] DOCKER_VALID_URL_PREFIXES = ( @@ -98,16 +124,25 @@ SUPPORTED_FILENAMES = [ DEFAULT_OVERRIDE_FILENAME = 'docker-compose.override.yml' + log = logging.getLogger(__name__) -class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files')): +class ConfigDetails(namedtuple('_ConfigDetails', 'working_dir config_files environment')): """ :param working_dir: the directory to use for relative paths in the config :type working_dir: string :param config_files: list of configuration files to load :type config_files: list of :class:`ConfigFile` + :param environment: computed environment values for this project + :type environment: :class:`environment.Environment` """ + def __new__(cls, working_dir, config_files, environment=None): + if environment is None: + environment = Environment.from_env_file(working_dir) + return super(ConfigDetails, cls).__new__( + cls, working_dir, config_files, environment + ) class ConfigFile(namedtuple('_ConfigFile', 'filename config')): @@ -124,27 +159,52 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): @cached_property def version(self): - if self.config is None: - return 1 - version = self.config.get('version', 1) + if 'version' not in self.config: + return V1 + + version = self.config['version'] + if isinstance(version, dict): - log.warn("Unexpected type for field 'version', in file {} assuming " - "version is the name of a service, and defaulting to " - "Compose file version 1".format(self.filename)) - return 1 + log.warn('Unexpected type for "version" key in "{}". Assuming ' + '"version" is the name of a service, and defaulting to ' + 'Compose file version 1.'.format(self.filename)) + return V1 + + if not isinstance(version, six.string_types): + raise ConfigurationError( + 'Version in "{}" is invalid - it should be a string.' + .format(self.filename)) + + if version == '1': + raise ConfigurationError( + 'Version in "{}" is invalid. {}' + .format(self.filename, VERSION_EXPLANATION)) + + if version == '2': + version = V2_0 + + if version == '3': + version = V3_0 + return version + def get_service(self, name): + return self.get_service_dicts()[name] + def get_service_dicts(self): - return self.config if self.version == 1 else self.config.get('services', {}) + return self.config if self.version == V1 else self.config.get('services', {}) def get_volumes(self): - return {} if self.version == 1 else self.config.get('volumes', {}) + return {} if self.version == V1 else self.config.get('volumes', {}) def get_networks(self): - return {} if self.version == 1 else self.config.get('networks', {}) + return {} if self.version == V1 else self.config.get('networks', {}) + + def get_secrets(self): + return {} if self.version < V3_1 else self.config.get('secrets', {}) -class Config(namedtuple('_Config', 'version services volumes networks')): +class Config(namedtuple('_Config', 'version services volumes networks secrets')): """ :param version: configuration version :type version: int @@ -171,11 +231,13 @@ class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name conf config) -def find(base_dir, filenames): +def find(base_dir, filenames, environment): if filenames == ['-']: return ConfigDetails( os.getcwd(), - [ConfigFile(None, yaml.safe_load(sys.stdin))]) + [ConfigFile(None, yaml.safe_load(sys.stdin))], + environment + ) if filenames: filenames = [os.path.join(base_dir, f) for f in filenames] @@ -185,13 +247,15 @@ def find(base_dir, filenames): log.debug("Using configuration files: {}".format(",".join(filenames))) return ConfigDetails( os.path.dirname(filenames[0]), - [ConfigFile.from_filename(f) for f in filenames]) + [ConfigFile.from_filename(f) for f in filenames], + environment + ) -def validate_config_version(config_details): - main_file = config_details.config_files[0] +def validate_config_version(config_files): + main_file = config_files[0] validate_top_level_object(main_file) - for next_file in config_details.config_files[1:]: + for next_file in config_files[1:]: validate_top_level_object(next_file) if main_file.version != next_file.version: @@ -203,10 +267,6 @@ def validate_config_version(config_details): next_file.filename, next_file.version)) - if main_file.version not in COMPOSEFILE_VERSIONS: - raise ConfigurationError( - 'Invalid Compose file version: {0}'.format(main_file.version)) - def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) @@ -254,23 +314,37 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ - validate_config_version(config_details) + validate_config_version(config_details.config_files) processed_files = [ - process_config_file(config_file) + process_config_file(config_file, config_details.environment) for config_file in config_details.config_files ] config_details = config_details._replace(config_files=processed_files) main_file = config_details.config_files[0] - volumes = load_mapping(config_details.config_files, 'get_volumes', 'Volume') - networks = load_mapping(config_details.config_files, 'get_networks', 'Network') - service_dicts = load_services( - config_details.working_dir, - main_file.filename, - [file.get_service_dicts() for file in config_details.config_files], - main_file.version) - return Config(main_file.version, service_dicts, volumes, networks) + volumes = load_mapping( + config_details.config_files, 'get_volumes', 'Volume' + ) + networks = load_mapping( + config_details.config_files, 'get_networks', 'Network' + ) + secrets = load_secrets(config_details.config_files, config_details.working_dir) + service_dicts = load_services(config_details, main_file) + + if main_file.version != V1: + for service_dict in service_dicts: + match_named_volumes(service_dict, volumes) + + services_using_deploy = [s for s in service_dicts if s.get('deploy')] + if services_using_deploy: + log.warn( + "Some services ({}) use the 'deploy' key, which will be ignored. " + "Compose does not support deploy configuration - use " + "`docker stack deploy` to deploy to a swarm." + .format(", ".join(sorted(s['name'] for s in services_using_deploy)))) + + return Config(main_file.version, service_dicts, volumes, networks, secrets) def load_mapping(config_files, get_func, entity_type): @@ -284,40 +358,75 @@ def load_mapping(config_files, get_func, entity_type): external = config.get('external') if external: - if len(config.keys()) > 1: - raise ConfigurationError( - '{} {} declared as external but specifies' - ' additional attributes ({}). '.format( - entity_type, - name, - ', '.join([k for k in config.keys() if k != 'external']) - ) - ) + validate_external(entity_type, name, config) if isinstance(external, dict): config['external_name'] = external.get('name') else: config['external_name'] = name - mapping[name] = config + if 'driver_opts' in config: + config['driver_opts'] = build_string_dict( + config['driver_opts'] + ) + + if 'labels' in config: + config['labels'] = parse_labels(config['labels']) return mapping -def load_services(working_dir, filename, service_configs, version): +def validate_external(entity_type, name, config): + if len(config.keys()) <= 1: + return + + raise ConfigurationError( + "{} {} declared as external but specifies additional attributes " + "({}).".format( + entity_type, name, ', '.join(k for k in config if k != 'external'))) + + +def load_secrets(config_files, working_dir): + mapping = {} + + for config_file in config_files: + for name, config in config_file.get_secrets().items(): + mapping[name] = config or {} + if not config: + continue + + external = config.get('external') + if external: + validate_external('Secret', name, config) + if isinstance(external, dict): + config['external_name'] = external.get('name') + else: + config['external_name'] = name + + if 'file' in config: + config['file'] = expand_path(working_dir, config['file']) + + return mapping + + +def load_services(config_details, config_file): def build_service(service_name, service_dict, service_names): service_config = ServiceConfig.with_abs_paths( - working_dir, - filename, + config_details.working_dir, + config_file.filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config, version) + resolver = ServiceExtendsResolver( + service_config, config_file, environment=config_details.environment + ) service_dict = process_service(resolver.run()) - validate_service(service_dict, service_config.name, version) + service_config = service_config._replace(config=service_dict) + validate_service(service_config, service_names, config_file.version) service_dict = finalize_service( - service_config._replace(config=service_dict), + service_config, service_names, - version) + config_file.version, + config_details.environment) return service_dict def build_services(service_config): @@ -333,10 +442,14 @@ def load_services(working_dir, filename, service_configs, version): name: merge_service_dicts_from_files( base.get(name, {}), override.get(name, {}), - version) + config_file.version) for name in all_service_names } + service_configs = [ + file.get_service_dicts() for file in config_details.config_files + ] + service_config = service_configs[0] for next_config in service_configs[1:]: service_config = merge_services(service_config, next_config) @@ -344,27 +457,47 @@ def load_services(working_dir, filename, service_configs, version): return build_services(service_config) -def process_config_file(config_file, service_name=None): - service_dicts = config_file.get_service_dicts() - validate_top_level_service_objects(config_file.filename, service_dicts) +def interpolate_config_section(config_file, config, section, environment): + validate_config_section(config_file.filename, config, section) + return interpolate_environment_variables( + config_file.version, + config, + section, + environment + ) - interpolated_config = interpolate_environment_variables(service_dicts, 'service') - if config_file.version == 2: +def process_config_file(config_file, environment, service_name=None): + services = interpolate_config_section( + config_file, + config_file.get_service_dicts(), + 'service', + environment) + + if config_file.version in (V2_0, V2_1, V3_0, V3_1): processed_config = dict(config_file.config) - processed_config['services'] = interpolated_config - processed_config['volumes'] = interpolate_environment_variables( - config_file.get_volumes(), 'volume') - processed_config['networks'] = interpolate_environment_variables( - config_file.get_networks(), 'network') - - if config_file.version == 1: - processed_config = interpolated_config + processed_config['services'] = services + processed_config['volumes'] = interpolate_config_section( + config_file, + config_file.get_volumes(), + 'volume', + environment) + processed_config['networks'] = interpolate_config_section( + config_file, + config_file.get_networks(), + 'network', + environment) + elif config_file.version == V1: + processed_config = services + else: + raise ConfigurationError( + 'Version in "{}" is unsupported. {}' + .format(config_file.filename, VERSION_EXPLANATION)) config_file = config_file._replace(config=processed_config) - validate_against_fields_schema(config_file) + validate_against_config_schema(config_file) - if service_name and service_name not in processed_config: + if service_name and service_name not in services: raise ConfigurationError( "Cannot extend service '{}' in {}: Service not found".format( service_name, config_file.filename)) @@ -373,11 +506,12 @@ def process_config_file(config_file, service_name=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, version, already_seen=None): + def __init__(self, service_config, config_file, environment, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] - self.version = version + self.config_file = config_file + self.environment = environment @property def signature(self): @@ -404,10 +538,13 @@ class ServiceExtendsResolver(object): config_path = self.get_extended_config_path(extends) service_name = extends['service'] + extends_file = ConfigFile.from_filename(config_path) + validate_config_version([self.config_file, extends_file]) extended_file = process_config_file( - ConfigFile.from_filename(config_path), - service_name=service_name) - service_config = extended_file.config[service_name] + extends_file, self.environment, service_name=service_name + ) + service_config = extended_file.get_service(service_name) + return config_path, service_config, service_name def resolve_extends(self, extended_config_path, service_dict, service_name): @@ -417,21 +554,22 @@ class ServiceExtendsResolver(object): extended_config_path, service_name, service_dict), - self.version, - already_seen=self.already_seen + [self.signature]) + self.config_file, + already_seen=self.already_seen + [self.signature], + environment=self.environment + ) service_config = resolver.run() other_service_dict = process_service(service_config) validate_extended_service_dict( other_service_dict, extended_config_path, - service_name, - ) + service_name) return merge_service_dicts( other_service_dict, self.service_config.config, - self.version) + self.config_file.version) def get_extended_config_path(self, extends_options): """Service we are extending either has a value for 'file' set, which we @@ -448,7 +586,7 @@ class ServiceExtendsResolver(object): return filename -def resolve_environment(service_dict): +def resolve_environment(service_dict, environment=None): """Unpack any environment variables from an env_file, if set. Interpolate environment values if set. """ @@ -457,12 +595,12 @@ def resolve_environment(service_dict): env.update(env_vars_from_file(env_file)) env.update(parse_environment(service_dict.get('environment'))) - return dict(resolve_env_var(k, v) for k, v in six.iteritems(env)) + return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(env)) -def resolve_build_args(build): +def resolve_build_args(build, environment): args = parse_build_arguments(build.get('args')) - return dict(resolve_env_var(k, v) for k, v in six.iteritems(args)) + return dict(resolve_env_var(k, v, environment) for k, v in six.iteritems(args)) def validate_extended_service_dict(service_dict, filename, service): @@ -477,26 +615,29 @@ def validate_extended_service_dict(service_dict, filename, service): "%s services with 'volumes_from' cannot be extended" % error_prefix) if 'net' in service_dict: - if get_service_name_from_net(service_dict['net']) is not None: + if get_container_name_from_network_mode(service_dict['net']): raise ConfigurationError( "%s services with 'net: container' cannot be extended" % error_prefix) + if 'network_mode' in service_dict: + if get_service_name_from_network_mode(service_dict['network_mode']): + raise ConfigurationError( + "%s services with 'network_mode: service' cannot be extended" % error_prefix) -def validate_ulimits(ulimit_config): - for limit_name, soft_hard_values in six.iteritems(ulimit_config): - if isinstance(soft_hard_values, dict): - if not soft_hard_values['soft'] <= soft_hard_values['hard']: - raise ConfigurationError( - "ulimit_config \"{}\" cannot contain a 'soft' value higher " - "than 'hard' value".format(ulimit_config)) + if 'depends_on' in service_dict: + raise ConfigurationError( + "%s services with 'depends_on' cannot be extended" % error_prefix) -def validate_service(service_dict, service_name, version): - validate_against_service_schema(service_dict, service_name, version) +def validate_service(service_config, service_names, version): + service_dict, service_name = service_config.config, service_config.name + validate_service_constraints(service_dict, service_name, version) validate_paths(service_dict) - if 'ulimits' in service_dict: - validate_ulimits(service_dict['ulimits']) + validate_ulimits(service_config) + validate_network_mode(service_config, service_names) + validate_depends_on(service_config, service_names) + validate_links(service_config, service_names) if not service_dict.get('image') and has_uppercase(service_name): raise ConfigurationError( @@ -532,18 +673,67 @@ def process_service(service_config): if 'extra_hosts' in service_dict: service_dict['extra_hosts'] = parse_extra_hosts(service_dict['extra_hosts']) - for field in ['dns', 'dns_search']: + if 'sysctls' in service_dict: + service_dict['sysctls'] = build_string_dict(parse_sysctls(service_dict['sysctls'])) + + service_dict = process_depends_on(service_dict) + + for field in ['dns', 'dns_search', 'tmpfs']: if field in service_dict: service_dict[field] = to_list(service_dict[field]) + service_dict = process_healthcheck(service_dict, service_config.name) + return service_dict -def finalize_service(service_config, service_names, version): +def process_depends_on(service_dict): + if 'depends_on' in service_dict and not isinstance(service_dict['depends_on'], dict): + service_dict['depends_on'] = dict([ + (svc, {'condition': 'service_started'}) for svc in service_dict['depends_on'] + ]) + return service_dict + + +def process_healthcheck(service_dict, service_name): + if 'healthcheck' not in service_dict: + return service_dict + + hc = {} + raw = service_dict['healthcheck'] + + if raw.get('disable'): + if len(raw) > 1: + raise ConfigurationError( + 'Service "{}" defines an invalid healthcheck: ' + '"disable: true" cannot be combined with other options' + .format(service_name)) + hc['test'] = ['NONE'] + elif 'test' in raw: + hc['test'] = raw['test'] + + if 'interval' in raw: + if not isinstance(raw['interval'], six.integer_types): + hc['interval'] = parse_nanoseconds_int(raw['interval']) + else: # Conversion has been done previously + hc['interval'] = raw['interval'] + if 'timeout' in raw: + if not isinstance(raw['timeout'], six.integer_types): + hc['timeout'] = parse_nanoseconds_int(raw['timeout']) + else: # Conversion has been done previously + hc['timeout'] = raw['timeout'] + if 'retries' in raw: + hc['retries'] = raw['retries'] + + service_dict['healthcheck'] = hc + return service_dict + + +def finalize_service(service_config, service_names, version, environment): service_dict = dict(service_config.config) if 'environment' in service_dict or 'env_file' in service_dict: - service_dict['environment'] = resolve_environment(service_dict) + service_dict['environment'] = resolve_environment(service_dict, environment) service_dict.pop('env_file', None) if 'volumes_from' in service_dict: @@ -554,12 +744,31 @@ def finalize_service(service_config, service_names, version): if 'volumes' in service_dict: service_dict['volumes'] = [ - VolumeSpec.parse(v) for v in service_dict['volumes']] + VolumeSpec.parse( + v, environment.get_boolean('COMPOSE_CONVERT_WINDOWS_PATHS') + ) for v in service_dict['volumes'] + ] + + if 'net' in service_dict: + network_mode = service_dict.pop('net') + container_name = get_container_name_from_network_mode(network_mode) + if container_name and container_name in service_names: + service_dict['network_mode'] = 'service:{}'.format(container_name) + else: + service_dict['network_mode'] = network_mode + + if 'networks' in service_dict: + service_dict['networks'] = parse_networks(service_dict['networks']) if 'restart' in service_dict: service_dict['restart'] = parse_restart_spec(service_dict['restart']) - normalize_build(service_dict, service_config.working_dir) + if 'secrets' in service_dict: + service_dict['secrets'] = [ + types.ServiceSecret.parse(s) for s in service_dict['secrets'] + ] + + normalize_build(service_dict, service_config.working_dir, environment) service_dict['name'] = service_config.name return normalize_v1_service_format(service_dict) @@ -598,63 +807,110 @@ def merge_service_dicts_from_files(base, override, version): return new_service +class MergeDict(dict): + """A dict-like object responsible for merging two dicts into one.""" + + def __init__(self, base, override): + self.base = base + self.override = override + + def needs_merge(self, field): + return field in self.base or field in self.override + + def merge_field(self, field, merge_func, default=None): + if not self.needs_merge(field): + return + + self[field] = merge_func( + self.base.get(field, default), + self.override.get(field, default)) + + def merge_mapping(self, field, parse_func): + if not self.needs_merge(field): + return + + self[field] = parse_func(self.base.get(field)) + self[field].update(parse_func(self.override.get(field))) + + def merge_sequence(self, field, parse_func): + def parse_sequence_func(seq): + return to_mapping((parse_func(item) for item in seq), 'merge_field') + + if not self.needs_merge(field): + return + + merged = parse_sequence_func(self.base.get(field, [])) + merged.update(parse_sequence_func(self.override.get(field, []))) + self[field] = [item.repr() for item in sorted(merged.values())] + + def merge_scalar(self, field): + if self.needs_merge(field): + self[field] = self.override.get(field, self.base.get(field)) + + def merge_service_dicts(base, override, version): - d = {} + md = MergeDict(base, override) - def merge_field(field, merge_func, default=None): - if field in base or field in override: - d[field] = merge_func( - base.get(field, default), - override.get(field, default)) - - def merge_mapping(mapping, parse_func): - if mapping in base or mapping in override: - merged = parse_func(base.get(mapping, None)) - merged.update(parse_func(override.get(mapping, None))) - d[mapping] = merged - - merge_mapping('environment', parse_environment) - merge_mapping('labels', parse_labels) - merge_mapping('ulimits', parse_ulimits) + md.merge_mapping('environment', parse_environment) + md.merge_mapping('labels', parse_labels) + md.merge_mapping('ulimits', parse_ulimits) + md.merge_mapping('networks', parse_networks) + md.merge_mapping('sysctls', parse_sysctls) + md.merge_mapping('depends_on', parse_depends_on) + md.merge_sequence('links', ServiceLink.parse) + md.merge_sequence('secrets', types.ServiceSecret.parse) for field in ['volumes', 'devices']: - merge_field(field, merge_path_mappings) + md.merge_field(field, merge_path_mappings) - for field in ['ports', 'expose', 'external_links']: - merge_field(field, operator.add, default=[]) + for field in [ + 'ports', 'cap_add', 'cap_drop', 'expose', 'external_links', + 'security_opt', 'volumes_from', + ]: + md.merge_field(field, merge_unique_items_lists, default=[]) - for field in ['dns', 'dns_search', 'env_file']: - merge_field(field, merge_list_or_string) + for field in ['dns', 'dns_search', 'env_file', 'tmpfs']: + md.merge_field(field, merge_list_or_string) - for field in set(ALLOWED_KEYS) - set(d): - if field in base or field in override: - d[field] = override.get(field, base.get(field)) + md.merge_field('logging', merge_logging, default={}) - if version == 1: - legacy_v1_merge_image_or_build(d, base, override) - else: - merge_build(d, base, override) + for field in set(ALLOWED_KEYS) - set(md): + md.merge_scalar(field) - return d + if version == V1: + legacy_v1_merge_image_or_build(md, base, override) + elif md.needs_merge('build'): + md['build'] = merge_build(md, base, override) + + return dict(md) + + +def merge_unique_items_lists(base, override): + return sorted(set().union(base, override)) def merge_build(output, base, override): - build = {} + def to_dict(service): + build_config = service.get('build', {}) + if isinstance(build_config, six.string_types): + return {'context': build_config} + return build_config - if 'build' in base: - if isinstance(base['build'], six.string_types): - build['context'] = base['build'] - else: - build.update(base['build']) + md = MergeDict(to_dict(base), to_dict(override)) + md.merge_scalar('context') + md.merge_scalar('dockerfile') + md.merge_mapping('args', parse_build_arguments) + return dict(md) - if 'build' in override: - if isinstance(override['build'], six.string_types): - build['context'] = override['build'] - else: - build.update(override['build']) - if build: - output['build'] = build +def merge_logging(base, override): + md = MergeDict(base, override) + md.merge_scalar('driver') + if md.get('driver') == base.get('driver') or base.get('driver') is None: + md.merge_mapping('options', lambda m: m or {}) + else: + md['options'] = override.get('options') + return dict(md) def legacy_v1_merge_image_or_build(output, base, override): @@ -676,20 +932,11 @@ def merge_environment(base, override): return env -def split_env(env): - if isinstance(env, six.binary_type): - env = env.decode('utf-8', 'replace') - if '=' in env: - return env.split('=', 1) +def split_kv(kvpair): + if '=' in kvpair: + return kvpair.split('=', 1) else: - return env, None - - -def split_label(label): - if '=' in label: - return label.split('=', 1) - else: - return label, '' + return kvpair, '' def parse_dict_or_list(split_func, type_name, arguments): @@ -710,7 +957,12 @@ def parse_dict_or_list(split_func, type_name, arguments): parse_build_arguments = functools.partial(parse_dict_or_list, split_env, 'build arguments') parse_environment = functools.partial(parse_dict_or_list, split_env, 'environment') -parse_labels = functools.partial(parse_dict_or_list, split_label, 'labels') +parse_labels = functools.partial(parse_dict_or_list, split_kv, 'labels') +parse_networks = functools.partial(parse_dict_or_list, lambda k: (k, None), 'networks') +parse_sysctls = functools.partial(parse_dict_or_list, split_kv, 'sysctls') +parse_depends_on = functools.partial( + parse_dict_or_list, lambda k: (k, {'condition': 'service_started'}), 'depends_on' +) def parse_ulimits(ulimits): @@ -721,28 +973,13 @@ def parse_ulimits(ulimits): return dict(ulimits) -def resolve_env_var(key, val): +def resolve_env_var(key, val, environment): if val is not None: return key, val - elif key in os.environ: - return key, os.environ[key] + elif environment and key in environment: + return key, environment[key] else: - return key, '' - - -def env_vars_from_file(filename): - """ - Read in a line delimited file of environment variables. - """ - if not os.path.exists(filename): - raise ConfigurationError("Couldn't find env file: %s" % filename) - env = {} - for line in codecs.open(filename, 'r', 'utf-8'): - line = line.strip() - if line and not line.startswith('#'): - k, v = split_env(line) - env[k] = v - return env + return key, None def resolve_volume_paths(working_dir, service_dict): @@ -764,7 +1001,7 @@ def resolve_volume_path(working_dir, volume): return container_path -def normalize_build(service_dict, working_dir): +def normalize_build(service_dict, working_dir, environment): if 'build' in service_dict: build = {} @@ -774,7 +1011,9 @@ def normalize_build(service_dict, working_dir): else: build.update(service_dict['build']) if 'args' in build: - build['args'] = resolve_build_args(build) + build['args'] = build_string_dict( + resolve_build_args(build, environment) + ) service_dict['build'] = build @@ -797,6 +1036,9 @@ def validate_paths(service_dict): build_path = build elif isinstance(build, dict) and 'context' in build: build_path = build['context'] + else: + # We have a build section but no context, so nothing to validate + return if ( not is_url(build_path) and @@ -821,7 +1063,7 @@ def dict_from_path_mappings(path_mappings): def path_mappings_from_dict(d): - return [join_path_mapping(v) for v in d.items()] + return [join_path_mapping(v) for v in sorted(d.items())] def split_path_mapping(volume_path): @@ -830,12 +1072,7 @@ def split_path_mapping(volume_path): path. Using splitdrive so windows absolute paths won't cause issues with splitting on ':'. """ - # splitdrive has limitations when it comes to relative paths, so when it's - # relative, handle special case to set the drive to '' - if volume_path.startswith('.') or volume_path.startswith('~'): - drive, volume_config = '', volume_path - else: - drive, volume_config = os.path.splitdrive(volume_path) + drive, volume_config = splitdrive(volume_path) if ':' in volume_config: (host, container) = volume_config.split(':', 1) @@ -869,6 +1106,10 @@ def to_list(value): return value +def to_mapping(sequence, key_field): + return {getattr(item, key_field): item for item in sequence} + + def has_uppercase(name): return any(char in string.ascii_uppercase for char in name) diff --git a/compose/config/service_schema_v1.json b/compose/config/config_schema_v1.json similarity index 87% rename from compose/config/service_schema_v1.json rename to compose/config/config_schema_v1.json index d220ec54..94354cda 100644 --- a/compose/config/service_schema_v1.json +++ b/compose/config/config_schema_v1.json @@ -1,13 +1,16 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "service_schema_v1.json", + "id": "config_schema_v1.json", "type": "object", - "allOf": [ - {"$ref": "#/definitions/service"}, - {"$ref": "#/definitions/constraints"} - ], + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + + "additionalProperties": false, "definitions": { "service": { @@ -82,6 +85,7 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, + "mem_swappiness": {"type": "integer"}, "net": {"type": "string"}, "pid": {"type": ["string", "null"]}, @@ -98,6 +102,7 @@ "read_only": {"type": "boolean"}, "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, "stop_signal": {"type": "string"}, "tty": {"type": "boolean"}, @@ -152,8 +157,7 @@ "type": "object", "patternProperties": { ".+": { - "type": ["string", "number", "boolean", "null"], - "format": "bool-value-in-mapping" + "type": ["string", "number", "null"] } }, "additionalProperties": false @@ -161,21 +165,24 @@ {"type": "array", "items": {"type": "string"}, "uniqueItems": true} ] }, + "constraints": { - "id": "#/definitions/constraints", - "anyOf": [ - { - "required": ["build"], - "not": {"required": ["image"]} - }, - { - "required": ["image"], - "not": {"anyOf": [ - {"required": ["build"]}, - {"required": ["dockerfile"]} - ]} - } - ] + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + { + "required": ["build"], + "not": {"required": ["image"]} + }, + { + "required": ["image"], + "not": {"anyOf": [ + {"required": ["build"]}, + {"required": ["dockerfile"]} + ]} + } + ] + } } } } diff --git a/compose/config/service_schema_v2.json b/compose/config/config_schema_v2.0.json similarity index 56% rename from compose/config/service_schema_v2.json rename to compose/config/config_schema_v2.0.json index 8623507a..59c7b30c 100644 --- a/compose/config/service_schema_v2.json +++ b/compose/config/config_schema_v2.0.json @@ -1,15 +1,50 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "id": "service_schema_v2.json", - + "id": "config_schema_v2.0.json", "type": "object", - "allOf": [ - {"$ref": "#/definitions/service"}, - {"$ref": "#/definitions/constraints"} - ], + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, "definitions": { + "service": { "id": "#/definitions/service", "type": "object", @@ -42,6 +77,7 @@ "cpu_shares": {"type": ["number", "string"]}, "cpu_quota": {"type": ["number", "string"]}, "cpuset": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "dns": {"$ref": "#/definitions/string_or_list"}, "dns_search": {"$ref": "#/definitions/string_or_list"}, @@ -82,11 +118,13 @@ ] }, + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, "hostname": {"type": "string"}, "image": {"type": "string"}, "ipc": {"type": "string"}, "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, "logging": { "type": "object", @@ -101,13 +139,42 @@ "mac_address": {"type": "string"}, "mem_limit": {"type": ["number", "string"]}, "memswap_limit": {"type": ["number", "string"]}, + "mem_swappiness": {"type": "integer"}, + "network_mode": {"type": "string"}, "networks": { - "type": "array", - "items": {"type": "string"}, - "uniqueItems": true + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true }, - "pid": {"type": ["string", "null"]}, "ports": { @@ -123,8 +190,11 @@ "read_only": {"type": "boolean"}, "restart": {"type": "string"}, "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, "tty": {"type": "boolean"}, "ulimits": { "type": "object", @@ -158,6 +228,61 @@ "additionalProperties": false }, + "network": { + "id": "#/definitions/network", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array" + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + "string_or_list": { "oneOf": [ {"type": "string"}, @@ -177,8 +302,7 @@ "type": "object", "patternProperties": { ".+": { - "type": ["string", "number", "boolean", "null"], - "format": "bool-value-in-mapping" + "type": ["string", "number", "null"] } }, "additionalProperties": false @@ -186,12 +310,20 @@ {"type": "array", "items": {"type": "string"}, "uniqueItems": true} ] }, + "constraints": { - "id": "#/definitions/constraints", - "anyOf": [ + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ {"required": ["build"]}, {"required": ["image"]} - ] + ], + "properties": { + "build": { + "required": ["context"] + } + } + } } } } diff --git a/compose/config/config_schema_v2.1.json b/compose/config/config_schema_v2.1.json new file mode 100644 index 00000000..d1ffff89 --- /dev/null +++ b/compose/config/config_schema_v2.1.json @@ -0,0 +1,376 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v2.1.json", + "type": "object", + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "cpu_shares": {"type": ["number", "string"]}, + "cpu_quota": {"type": ["number", "string"]}, + "cpuset": {"type": "string"}, + "depends_on": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "additionalProperties": false, + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "type": "object", + "additionalProperties": false, + "properties": { + "condition": { + "type": "string", + "enum": ["service_started", "service_healthy"] + } + }, + "required": ["condition"] + } + } + } + ] + }, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "extends": { + "oneOf": [ + { + "type": "string" + }, + { + "type": "object", + + "properties": { + "service": {"type": "string"}, + "file": {"type": "string"} + }, + "required": ["service"], + "additionalProperties": false + } + ] + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "isolation": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": {"type": "object"} + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "mem_limit": {"type": ["number", "string"]}, + "memswap_limit": {"type": ["number", "string"]}, + "mem_swappiness": {"type": "integer"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"}, + "link_local_ips": {"$ref": "#/definitions/list_of_strings"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "oom_score_adj": {"type": "integer", "minimum": -1000, "maximum": 1000}, + "group_add": { + "type": "array", + "items": { + "type": ["string", "number"] + }, + "uniqueItems": true + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "volume_driver": {"type": "string"}, + "volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + + "dependencies": { + "memswap_limit": ["mem_limit"] + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + + "network": { + "id": "#/definitions/network", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array" + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "enable_ipv6": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/config_schema_v3.0.json b/compose/config/config_schema_v3.0.json new file mode 100644 index 00000000..fbcd8bb8 --- /dev/null +++ b/compose/config/config_schema_v3.0.json @@ -0,0 +1,383 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.0.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/config_schema_v3.1.json b/compose/config/config_schema_v3.1.json new file mode 100644 index 00000000..b7037485 --- /dev/null +++ b/compose/config/config_schema_v3.1.json @@ -0,0 +1,428 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "id": "config_schema_v3.1.json", + "type": "object", + "required": ["version"], + + "properties": { + "version": { + "type": "string" + }, + + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/service" + } + }, + "additionalProperties": false + }, + + "networks": { + "id": "#/properties/networks", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/network" + } + } + }, + + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + }, + + "secrets": { + "id": "#/properties/secrets", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/secret" + } + }, + "additionalProperties": false + } + }, + + "additionalProperties": false, + + "definitions": { + + "service": { + "id": "#/definitions/service", + "type": "object", + + "properties": { + "deploy": {"$ref": "#/definitions/deployment"}, + "build": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "context": {"type": "string"}, + "dockerfile": {"type": "string"}, + "args": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + } + ] + }, + "cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "cgroup_parent": {"type": "string"}, + "command": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "container_name": {"type": "string"}, + "depends_on": {"$ref": "#/definitions/list_of_strings"}, + "devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "dns": {"$ref": "#/definitions/string_or_list"}, + "dns_search": {"$ref": "#/definitions/string_or_list"}, + "domainname": {"type": "string"}, + "entrypoint": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "env_file": {"$ref": "#/definitions/string_or_list"}, + "environment": {"$ref": "#/definitions/list_or_dict"}, + + "expose": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "expose" + }, + "uniqueItems": true + }, + + "external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "extra_hosts": {"$ref": "#/definitions/list_or_dict"}, + "healthcheck": {"$ref": "#/definitions/healthcheck"}, + "hostname": {"type": "string"}, + "image": {"type": "string"}, + "ipc": {"type": "string"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + + "logging": { + "type": "object", + + "properties": { + "driver": {"type": "string"}, + "options": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number", "null"]} + } + } + }, + "additionalProperties": false + }, + + "mac_address": {"type": "string"}, + "network_mode": {"type": "string"}, + + "networks": { + "oneOf": [ + {"$ref": "#/definitions/list_of_strings"}, + { + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "oneOf": [ + { + "type": "object", + "properties": { + "aliases": {"$ref": "#/definitions/list_of_strings"}, + "ipv4_address": {"type": "string"}, + "ipv6_address": {"type": "string"} + }, + "additionalProperties": false + }, + {"type": "null"} + ] + } + }, + "additionalProperties": false + } + ] + }, + "pid": {"type": ["string", "null"]}, + + "ports": { + "type": "array", + "items": { + "type": ["string", "number"], + "format": "ports" + }, + "uniqueItems": true + }, + + "privileged": {"type": "boolean"}, + "read_only": {"type": "boolean"}, + "restart": {"type": "string"}, + "security_opt": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "shm_size": {"type": ["number", "string"]}, + "secrets": { + "type": "array", + "items": { + "oneOf": [ + {"type": "string"}, + { + "type": "object", + "properties": { + "source": {"type": "string"}, + "target": {"type": "string"}, + "uid": {"type": "string"}, + "gid": {"type": "string"}, + "mode": {"type": "number"} + } + } + ] + } + }, + "sysctls": {"$ref": "#/definitions/list_or_dict"}, + "stdin_open": {"type": "boolean"}, + "stop_grace_period": {"type": "string", "format": "duration"}, + "stop_signal": {"type": "string"}, + "tmpfs": {"$ref": "#/definitions/string_or_list"}, + "tty": {"type": "boolean"}, + "ulimits": { + "type": "object", + "patternProperties": { + "^[a-z]+$": { + "oneOf": [ + {"type": "integer"}, + { + "type":"object", + "properties": { + "hard": {"type": "integer"}, + "soft": {"type": "integer"} + }, + "required": ["soft", "hard"], + "additionalProperties": false + } + ] + } + } + }, + "user": {"type": "string"}, + "userns_mode": {"type": "string"}, + "volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true}, + "working_dir": {"type": "string"} + }, + "additionalProperties": false + }, + + "healthcheck": { + "id": "#/definitions/healthcheck", + "type": "object", + "additionalProperties": false, + "properties": { + "disable": {"type": "boolean"}, + "interval": {"type": "string"}, + "retries": {"type": "number"}, + "test": { + "oneOf": [ + {"type": "string"}, + {"type": "array", "items": {"type": "string"}} + ] + }, + "timeout": {"type": "string"} + } + }, + "deployment": { + "id": "#/definitions/deployment", + "type": ["object", "null"], + "properties": { + "mode": {"type": "string"}, + "replicas": {"type": "integer"}, + "labels": {"$ref": "#/definitions/list_or_dict"}, + "update_config": { + "type": "object", + "properties": { + "parallelism": {"type": "integer"}, + "delay": {"type": "string", "format": "duration"}, + "failure_action": {"type": "string"}, + "monitor": {"type": "string", "format": "duration"}, + "max_failure_ratio": {"type": "number"} + }, + "additionalProperties": false + }, + "resources": { + "type": "object", + "properties": { + "limits": {"$ref": "#/definitions/resource"}, + "reservations": {"$ref": "#/definitions/resource"} + } + }, + "restart_policy": { + "type": "object", + "properties": { + "condition": {"type": "string"}, + "delay": {"type": "string", "format": "duration"}, + "max_attempts": {"type": "integer"}, + "window": {"type": "string", "format": "duration"} + }, + "additionalProperties": false + }, + "placement": { + "type": "object", + "properties": { + "constraints": {"type": "array", "items": {"type": "string"}} + }, + "additionalProperties": false + } + }, + "additionalProperties": false + }, + + "resource": { + "id": "#/definitions/resource", + "type": "object", + "properties": { + "cpus": {"type": "string"}, + "memory": {"type": "string"} + }, + "additionalProperties": false + }, + + "network": { + "id": "#/definitions/network", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "ipam": { + "type": "object", + "properties": { + "driver": {"type": "string"}, + "config": { + "type": "array", + "items": { + "type": "object", + "properties": { + "subnet": {"type": "string"} + }, + "additionalProperties": false + } + } + }, + "additionalProperties": false + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "internal": {"type": "boolean"}, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "volume": { + "id": "#/definitions/volume", + "type": ["object", "null"], + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + } + }, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + }, + "additionalProperties": false + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "secret": { + "id": "#/definitions/secret", + "type": "object", + "properties": { + "file": {"type": "string"}, + "external": { + "type": ["boolean", "object"], + "properties": { + "name": {"type": "string"} + } + }, + "labels": {"$ref": "#/definitions/list_or_dict"} + }, + "additionalProperties": false + }, + + "string_or_list": { + "oneOf": [ + {"type": "string"}, + {"$ref": "#/definitions/list_of_strings"} + ] + }, + + "list_of_strings": { + "type": "array", + "items": {"type": "string"}, + "uniqueItems": true + }, + + "list_or_dict": { + "oneOf": [ + { + "type": "object", + "patternProperties": { + ".+": { + "type": ["string", "number", "null"] + } + }, + "additionalProperties": false + }, + {"type": "array", "items": {"type": "string"}, "uniqueItems": true} + ] + }, + + "constraints": { + "service": { + "id": "#/definitions/constraints/service", + "anyOf": [ + {"required": ["build"]}, + {"required": ["image"]} + ], + "properties": { + "build": { + "required": ["context"] + } + } + } + } + } +} diff --git a/compose/config/environment.py b/compose/config/environment.py new file mode 100644 index 00000000..4ba228c8 --- /dev/null +++ b/compose/config/environment.py @@ -0,0 +1,120 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import codecs +import contextlib +import logging +import os + +import six + +from ..const import IS_WINDOWS_PLATFORM +from .errors import ConfigurationError + +log = logging.getLogger(__name__) + + +def split_env(env): + if isinstance(env, six.binary_type): + env = env.decode('utf-8', 'replace') + if '=' in env: + return env.split('=', 1) + else: + return env, None + + +def env_vars_from_file(filename): + """ + Read in a line delimited file of environment variables. + """ + if not os.path.exists(filename): + raise ConfigurationError("Couldn't find env file: %s" % filename) + elif not os.path.isfile(filename): + raise ConfigurationError("%s is not a file." % (filename)) + env = {} + with contextlib.closing(codecs.open(filename, 'r', 'utf-8')) as fileobj: + for line in fileobj: + line = line.strip() + if line and not line.startswith('#'): + k, v = split_env(line) + env[k] = v + return env + + +class Environment(dict): + def __init__(self, *args, **kwargs): + super(Environment, self).__init__(*args, **kwargs) + self.missing_keys = [] + + @classmethod + def from_env_file(cls, base_dir): + def _initialize(): + result = cls() + if base_dir is None: + return result + env_file_path = os.path.join(base_dir, '.env') + try: + return cls(env_vars_from_file(env_file_path)) + except ConfigurationError: + pass + return result + instance = _initialize() + instance.update(os.environ) + return instance + + @classmethod + def from_command_line(cls, parsed_env_opts): + result = cls() + for k, v in parsed_env_opts.items(): + # Values from the command line take priority, unless they're unset + # in which case they take the value from the system's environment + if v is None and k in os.environ: + result[k] = os.environ[k] + else: + result[k] = v + return result + + def __getitem__(self, key): + try: + return super(Environment, self).__getitem__(key) + except KeyError: + if IS_WINDOWS_PLATFORM: + try: + return super(Environment, self).__getitem__(key.upper()) + except KeyError: + pass + if key not in self.missing_keys: + log.warn( + "The {} variable is not set. Defaulting to a blank string." + .format(key) + ) + self.missing_keys.append(key) + + return "" + + def __contains__(self, key): + result = super(Environment, self).__contains__(key) + if IS_WINDOWS_PLATFORM: + return ( + result or super(Environment, self).__contains__(key.upper()) + ) + return result + + def get(self, key, *args, **kwargs): + if IS_WINDOWS_PLATFORM: + return super(Environment, self).get( + key, + super(Environment, self).get(key.upper(), *args, **kwargs) + ) + return super(Environment, self).get(key, *args, **kwargs) + + def get_boolean(self, key): + # Convert a value to a boolean using "common sense" rules. + # Unset, empty, "0" and "false" (i-case) yield False. + # All other values yield True. + value = self.get(key) + if not value: + return False + if value.lower() in ['0', 'false']: + return False + return True diff --git a/compose/config/errors.py b/compose/config/errors.py index 99129f3d..16ed01b8 100644 --- a/compose/config/errors.py +++ b/compose/config/errors.py @@ -2,6 +2,15 @@ from __future__ import absolute_import from __future__ import unicode_literals +VERSION_EXPLANATION = ( + 'You might be seeing this error because you\'re using the wrong Compose file version. ' + 'Either specify a supported version ("2.0", "2.1", "3.0") and place your ' + 'service definitions under the `services` key, or omit the `version` key ' + 'and place your service definitions at the root of the file to use ' + 'version 1.\nFor more on the Compose file format versions, see ' + 'https://docs.docker.com/compose/compose-file/') + + class ConfigurationError(Exception): def __init__(self, msg): self.msg = msg @@ -30,7 +39,8 @@ class CircularReference(ConfigurationError): class ComposeFileNotFound(ConfigurationError): def __init__(self, supported_filenames): super(ComposeFileNotFound, self).__init__(""" - Can't find a suitable configuration file in this directory or any parent. Are you in the right directory? + Can't find a suitable configuration file in this directory or any + parent. Are you in the right directory? Supported filenames: %s """ % ", ".join(supported_filenames)) diff --git a/compose/config/fields_schema_v1.json b/compose/config/fields_schema_v1.json deleted file mode 100644 index 8f6a8c0a..00000000 --- a/compose/config/fields_schema_v1.json +++ /dev/null @@ -1,13 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - - "type": "object", - "id": "fields_schema_v1.json", - - "patternProperties": { - "^[a-zA-Z0-9._-]+$": { - "$ref": "service_schema_v1.json#/definitions/service" - } - }, - "additionalProperties": false -} diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json deleted file mode 100644 index b0f304e8..00000000 --- a/compose/config/fields_schema_v2.json +++ /dev/null @@ -1,69 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "id": "fields_schema_v2.json", - - "properties": { - "version": { - "enum": [2] - }, - "services": { - "id": "#/properties/services", - "type": "object", - "patternProperties": { - "^[a-zA-Z0-9._-]+$": { - "$ref": "service_schema_v2.json#/definitions/service" - } - }, - "additionalProperties": false - }, - "networks": { - "id": "#/properties/networks", - "type": "object", - "patternProperties": { - "^[a-zA-Z0-9._-]+$": { - "$ref": "#/definitions/network" - } - } - }, - "volumes": { - "id": "#/properties/volumes", - "type": "object", - "patternProperties": { - "^[a-zA-Z0-9._-]+$": { - "$ref": "#/definitions/volume" - } - }, - "additionalProperties": false - } - }, - - "definitions": { - "network": { - "id": "#/definitions/network", - "type": "object" - }, - "volume": { - "id": "#/definitions/volume", - "type": ["object", "null"], - "properties": { - "driver": {"type": "string"}, - "driver_opts": { - "type": "object", - "patternProperties": { - "^.+$": {"type": ["string", "number"]} - } - }, - "external": { - "type": ["boolean", "object"], - "properties": { - "name": {"type": "string"} - } - }, - "additionalProperties": false - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index e1c781fe..1b270b9e 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -2,33 +2,52 @@ from __future__ import absolute_import from __future__ import unicode_literals import logging -import os from string import Template import six from .errors import ConfigurationError +from compose.const import COMPOSEFILE_V1 as V1 +from compose.const import COMPOSEFILE_V2_0 as V2_0 + + log = logging.getLogger(__name__) -def interpolate_environment_variables(config, section): - mapping = BlankDefaultDict(os.environ) +class Interpolator(object): + + def __init__(self, templater, mapping): + self.templater = templater + self.mapping = mapping + + def interpolate(self, string): + try: + return self.templater(string).substitute(self.mapping) + except ValueError: + raise InvalidInterpolation(string) + + +def interpolate_environment_variables(version, config, section, environment): + if version in (V2_0, V1): + interpolator = Interpolator(Template, environment) + else: + interpolator = Interpolator(TemplateWithDefaults, environment) def process_item(name, config_dict): return dict( - (key, interpolate_value(name, key, val, section, mapping)) + (key, interpolate_value(name, key, val, section, interpolator)) for key, val in (config_dict or {}).items() ) return dict( - (name, process_item(name, config_dict)) + (name, process_item(name, config_dict or {})) for name, config_dict in config.items() ) -def interpolate_value(name, config_key, value, section, mapping): +def interpolate_value(name, config_key, value, section, interpolator): try: - return recursive_interpolate(value, mapping) + return recursive_interpolate(value, interpolator) except InvalidInterpolation as e: raise ConfigurationError( 'Invalid interpolation format for "{config_key}" option ' @@ -39,44 +58,44 @@ def interpolate_value(name, config_key, value, section, mapping): string=e.string)) -def recursive_interpolate(obj, mapping): +def recursive_interpolate(obj, interpolator): if isinstance(obj, six.string_types): - return interpolate(obj, mapping) - elif isinstance(obj, dict): + return interpolator.interpolate(obj) + if isinstance(obj, dict): return dict( - (key, recursive_interpolate(val, mapping)) + (key, recursive_interpolate(val, interpolator)) for (key, val) in obj.items() ) - elif isinstance(obj, list): - return [recursive_interpolate(val, mapping) for val in obj] - else: - return obj + if isinstance(obj, list): + return [recursive_interpolate(val, interpolator) for val in obj] + return obj -def interpolate(string, mapping): - try: - return Template(string).substitute(mapping) - except ValueError: - raise InvalidInterpolation(string) +class TemplateWithDefaults(Template): + idpattern = r'[_a-z][_a-z0-9]*(?::?-[^}]+)?' - -class BlankDefaultDict(dict): - def __init__(self, *args, **kwargs): - super(BlankDefaultDict, self).__init__(*args, **kwargs) - self.missing_keys = [] - - def __getitem__(self, key): - try: - return super(BlankDefaultDict, self).__getitem__(key) - except KeyError: - if key not in self.missing_keys: - log.warn( - "The {} variable is not set. Defaulting to a blank string." - .format(key) - ) - self.missing_keys.append(key) - - return "" + # Modified from python2.7/string.py + def substitute(self, mapping): + # Helper function for .sub() + def convert(mo): + # Check the most common path first. + named = mo.group('named') or mo.group('braced') + if named is not None: + if ':-' in named: + var, _, default = named.partition(':-') + return mapping.get(var) or default + if '-' in named: + var, _, default = named.partition('-') + return mapping.get(var, default) + val = mapping[named] + return '%s' % (val,) + if mo.group('escaped') is not None: + return self.delimiter + if mo.group('invalid') is not None: + self._invalid(mo) + raise ValueError('Unrecognized named group in pattern', + self.pattern) + return self.pattern.sub(convert, self.template) class InvalidInterpolation(Exception): diff --git a/compose/config/serialize.py b/compose/config/serialize.py index 06e0a027..46d283f0 100644 --- a/compose/config/serialize.py +++ b/compose/config/serialize.py @@ -5,6 +5,8 @@ import six import yaml from compose.config import types +from compose.config.config import V1 +from compose.config.config import V2_1 def serialize_config_type(dumper, data): @@ -16,15 +18,91 @@ yaml.SafeDumper.add_representer(types.VolumeFromSpec, serialize_config_type) yaml.SafeDumper.add_representer(types.VolumeSpec, serialize_config_type) -def serialize_config(config): - output = { - 'version': config.version, - 'services': {service.pop('name'): service for service in config.services}, - 'networks': config.networks, - 'volumes': config.volumes, +def denormalize_config(config): + denormalized_services = [ + denormalize_service_dict(service_dict, config.version) + for service_dict in config.services + ] + services = { + service_dict.pop('name'): service_dict + for service_dict in denormalized_services } + networks = config.networks.copy() + for net_name, net_conf in networks.items(): + if 'external_name' in net_conf: + del net_conf['external_name'] + + volumes = config.volumes.copy() + for vol_name, vol_conf in volumes.items(): + if 'external_name' in vol_conf: + del vol_conf['external_name'] + + version = config.version + if version == V1: + version = V2_1 + + return { + 'version': version, + 'services': services, + 'networks': networks, + 'volumes': volumes, + } + + +def serialize_config(config): return yaml.safe_dump( - output, + denormalize_config(config), default_flow_style=False, indent=2, width=80) + + +def serialize_ns_time_value(value): + result = (value, 'ns') + table = [ + (1000., 'us'), + (1000., 'ms'), + (1000., 's'), + (60., 'm'), + (60., 'h') + ] + for stage in table: + tmp = value / stage[0] + if tmp == int(value / stage[0]): + value = tmp + result = (int(value), stage[1]) + else: + break + return '{0}{1}'.format(*result) + + +def denormalize_service_dict(service_dict, version): + service_dict = service_dict.copy() + + if 'restart' in service_dict: + service_dict['restart'] = types.serialize_restart_spec( + service_dict['restart'] + ) + + if version == V1 and 'network_mode' not in service_dict: + service_dict['network_mode'] = 'bridge' + + if 'depends_on' in service_dict and version != V2_1: + service_dict['depends_on'] = sorted([ + svc for svc in service_dict['depends_on'].keys() + ]) + + if 'healthcheck' in service_dict: + if 'interval' in service_dict['healthcheck']: + service_dict['healthcheck']['interval'] = serialize_ns_time_value( + service_dict['healthcheck']['interval'] + ) + if 'timeout' in service_dict['healthcheck']: + service_dict['healthcheck']['timeout'] = serialize_ns_time_value( + service_dict['healthcheck']['timeout'] + ) + + if 'secrets' in service_dict: + service_dict['secrets'] = map(lambda s: s.repr(), service_dict['secrets']) + + return service_dict diff --git a/compose/config/sort_services.py b/compose/config/sort_services.py index 05552122..20ac4461 100644 --- a/compose/config/sort_services.py +++ b/compose/config/sort_services.py @@ -4,46 +4,60 @@ from __future__ import unicode_literals from compose.config.errors import DependencyError -def get_service_name_from_net(net_config): - if not net_config: +def get_service_name_from_network_mode(network_mode): + return get_source_name_from_network_mode(network_mode, 'service') + + +def get_container_name_from_network_mode(network_mode): + return get_source_name_from_network_mode(network_mode, 'container') + + +def get_source_name_from_network_mode(network_mode, source_type): + if not network_mode: return - if not net_config.startswith('container:'): + if not network_mode.startswith(source_type+':'): return - _, net_name = net_config.split(':', 1) + _, net_name = network_mode.split(':', 1) return net_name +def get_service_names(links): + return [link.split(':')[0] for link in links] + + +def get_service_names_from_volumes_from(volumes_from): + return [volume_from.source for volume_from in volumes_from] + + +def get_service_dependents(service_dict, services): + name = service_dict['name'] + return [ + service for service in services + if (name in get_service_names(service.get('links', [])) or + name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or + name == get_service_name_from_network_mode(service.get('network_mode')) or + name in service.get('depends_on', [])) + ] + + def sort_service_dicts(services): # Topological sort (Cormen/Tarjan algorithm). unmarked = services[:] temporary_marked = set() sorted_services = [] - def get_service_names(links): - return [link.split(':')[0] for link in links] - - def get_service_names_from_volumes_from(volumes_from): - return [volume_from.source for volume_from in volumes_from] - - def get_service_dependents(service_dict, services): - name = service_dict['name'] - return [ - service for service in services - if (name in get_service_names(service.get('links', [])) or - name in get_service_names_from_volumes_from(service.get('volumes_from', [])) or - name == get_service_name_from_net(service.get('net'))) - ] - def visit(n): if n['name'] in temporary_marked: if n['name'] in get_service_names(n.get('links', [])): raise DependencyError('A service can not link to itself: %s' % n['name']) if n['name'] in n.get('volumes_from', []): raise DependencyError('A service can not mount itself as volume: %s' % n['name']) - else: - raise DependencyError('Circular import between %s' % ' and '.join(temporary_marked)) + if n['name'] in n.get('depends_on', []): + raise DependencyError('A service can not depend on itself: %s' % n['name']) + raise DependencyError('Circular dependency between %s' % ' and '.join(temporary_marked)) + if n in unmarked: temporary_marked.add(n['name']) for m in get_service_dependents(n, services): diff --git a/compose/config/types.py b/compose/config/types.py index b872cba9..811e6c1f 100644 --- a/compose/config/types.py +++ b/compose/config/types.py @@ -5,10 +5,17 @@ from __future__ import absolute_import from __future__ import unicode_literals import os +import re from collections import namedtuple -from compose.config.errors import ConfigurationError +import six + +from ..const import COMPOSEFILE_V1 as V1 +from .errors import ConfigurationError from compose.const import IS_WINDOWS_PLATFORM +from compose.utils import splitdrive + +win32_root_path_pattern = re.compile(r'^[A-Za-z]\:\\.*') class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode type')): @@ -16,7 +23,7 @@ class VolumeFromSpec(namedtuple('_VolumeFromSpec', 'source mode type')): # TODO: drop service_names arg when v1 is removed @classmethod def parse(cls, volume_from_config, service_names, version): - func = cls.parse_v1 if version == 1 else cls.parse_v2 + func = cls.parse_v1 if version == V1 else cls.parse_v2 return func(service_names, volume_from_config) @classmethod @@ -88,6 +95,15 @@ def parse_restart_spec(restart_config): return {'Name': name, 'MaximumRetryCount': int(max_retry_count)} +def serialize_restart_spec(restart_spec): + if not restart_spec: + return '' + parts = [restart_spec['Name']] + if restart_spec['MaximumRetryCount']: + parts.append(six.text_type(restart_spec['MaximumRetryCount'])) + return ':'.join(parts) + + def parse_extra_hosts(extra_hosts_config): if not extra_hosts_config: return {} @@ -104,41 +120,23 @@ def parse_extra_hosts(extra_hosts_config): return extra_hosts_dict -def normalize_paths_for_engine(external_path, internal_path): +def normalize_path_for_engine(path): """Windows paths, c:\my\path\shiny, need to be changed to be compatible with the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ """ - if not IS_WINDOWS_PLATFORM: - return external_path, internal_path + drive, tail = splitdrive(path) - if external_path: - drive, tail = os.path.splitdrive(external_path) + if drive: + path = '/' + drive.lower().rstrip(':') + tail - if drive: - external_path = '/' + drive.lower().rstrip(':') + tail - - external_path = external_path.replace('\\', '/') - - return external_path, internal_path.replace('\\', '/') + return path.replace('\\', '/') class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): @classmethod - def parse(cls, volume_config): - """Parse a volume_config path and split it into external:internal[:mode] - parts to be returned as a valid VolumeSpec. - """ - if IS_WINDOWS_PLATFORM: - # relative paths in windows expand to include the drive, eg C:\ - # so we join the first 2 parts back together to count as one - drive, tail = os.path.splitdrive(volume_config) - parts = tail.split(":") - - if drive: - parts[0] = drive + parts[0] - else: - parts = volume_config.split(':') + def _parse_unix(cls, volume_config): + parts = volume_config.split(':') if len(parts) > 3: raise ConfigurationError( @@ -146,13 +144,11 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): "external:internal[:mode]" % volume_config) if len(parts) == 1: - external, internal = normalize_paths_for_engine( - None, - os.path.normpath(parts[0])) + external = None + internal = os.path.normpath(parts[0]) else: - external, internal = normalize_paths_for_engine( - os.path.normpath(parts[0]), - os.path.normpath(parts[1])) + external = os.path.normpath(parts[0]) + internal = os.path.normpath(parts[1]) mode = 'rw' if len(parts) == 3: @@ -160,6 +156,105 @@ class VolumeSpec(namedtuple('_VolumeSpec', 'external internal mode')): return cls(external, internal, mode) + @classmethod + def _parse_win32(cls, volume_config, normalize): + # relative paths in windows expand to include the drive, eg C:\ + # so we join the first 2 parts back together to count as one + mode = 'rw' + + def separate_next_section(volume_config): + drive, tail = splitdrive(volume_config) + parts = tail.split(':', 1) + if drive: + parts[0] = drive + parts[0] + return parts + + parts = separate_next_section(volume_config) + if len(parts) == 1: + internal = parts[0] + external = None + else: + external = parts[0] + parts = separate_next_section(parts[1]) + external = os.path.normpath(external) + internal = parts[0] + if len(parts) > 1: + if ':' in parts[1]: + raise ConfigurationError( + "Volume %s has incorrect format, should be " + "external:internal[:mode]" % volume_config + ) + mode = parts[1] + + if normalize: + external = normalize_path_for_engine(external) if external else None + + return cls(external, internal, mode) + + @classmethod + def parse(cls, volume_config, normalize=False): + """Parse a volume_config path and split it into external:internal[:mode] + parts to be returned as a valid VolumeSpec. + """ + if IS_WINDOWS_PLATFORM: + return cls._parse_win32(volume_config, normalize) + else: + return cls._parse_unix(volume_config) + def repr(self): external = self.external + ':' if self.external else '' return '{ext}{v.internal}:{v.mode}'.format(ext=external, v=self) + + @property + def is_named_volume(self): + res = self.external and not self.external.startswith(('.', '/', '~')) + if not IS_WINDOWS_PLATFORM: + return res + + return ( + res and not self.external.startswith('\\') and + not win32_root_path_pattern.match(self.external) + ) + + +class ServiceLink(namedtuple('_ServiceLink', 'target alias')): + + @classmethod + def parse(cls, link_spec): + target, _, alias = link_spec.partition(':') + if not alias: + alias = target + return cls(target, alias) + + def repr(self): + if self.target == self.alias: + return self.target + return '{s.target}:{s.alias}'.format(s=self) + + @property + def merge_field(self): + return self.alias + + +class ServiceSecret(namedtuple('_ServiceSecret', 'source target uid gid mode')): + + @classmethod + def parse(cls, spec): + if isinstance(spec, six.string_types): + return cls(spec, None, None, None, None) + return cls( + spec.get('source'), + spec.get('target'), + spec.get('uid'), + spec.get('gid'), + spec.get('mode'), + ) + + @property + def merge_field(self): + return self.source + + def repr(self): + return dict( + [(k, v) for k, v in self._asdict().items() if v is not None] + ) diff --git a/compose/config/validation.py b/compose/config/validation.py index 639e8bed..3f23f0a7 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -14,7 +14,10 @@ from jsonschema import FormatChecker from jsonschema import RefResolver from jsonschema import ValidationError +from ..const import COMPOSEFILE_V1 as V1 from .errors import ConfigurationError +from .errors import VERSION_EXPLANATION +from .sort_services import get_service_name_from_network_mode log = logging.getLogger(__name__) @@ -60,48 +63,63 @@ def format_expose(instance): return True -@FormatChecker.cls_checks(format="bool-value-in-mapping") -def format_boolean_in_environment(instance): - """ - Check if there is a boolean in the environment and display a warning. - Always return True here so the validation won't raise an error. - """ - if isinstance(instance, bool): - log.warn( - "There is a boolean value in the 'environment' key.\n" - "Environment variables can only be strings.\n" - "Please add quotes to any boolean values to make them string " - "(eg, 'True', 'yes', 'N').\n" - "This warning will become an error in a future release. \r\n" - ) - return True - - -def validate_top_level_service_objects(filename, service_dicts): - """Perform some high level validation of the service name and value. - - This validation must happen before interpolation, which must happen - before the rest of validation, which is why it's separate from the - rest of the service validation. - """ - for service_name, service_dict in service_dicts.items(): - if not isinstance(service_name, six.string_types): +def match_named_volumes(service_dict, project_volumes): + service_volumes = service_dict.get('volumes', []) + for volume_spec in service_volumes: + if volume_spec.is_named_volume and volume_spec.external not in project_volumes: raise ConfigurationError( - "In file '{}' service name: {} needs to be a string, eg '{}'".format( - filename, - service_name, - service_name)) - - if not isinstance(service_dict, dict): - raise ConfigurationError( - "In file '{}' service '{}' doesn\'t have any configuration options. " - "All top level keys in your docker-compose.yml must map " - "to a dictionary of configuration options.".format( - filename, service_name + 'Named volume "{0}" is used in service "{1}" but no' + ' declaration was found in the volumes section.'.format( + volume_spec.repr(), service_dict.get('name') ) ) +def python_type_to_yaml_type(type_): + type_name = type(type_).__name__ + return { + 'dict': 'mapping', + 'list': 'array', + 'int': 'number', + 'float': 'number', + 'bool': 'boolean', + 'unicode': 'string', + 'str': 'string', + 'bytes': 'string', + }.get(type_name, type_name) + + +def validate_config_section(filename, config, section): + """Validate the structure of a configuration section. This must be done + before interpolation so it's separate from schema validation. + """ + if not isinstance(config, dict): + raise ConfigurationError( + "In file '{filename}', {section} must be a mapping, not " + "{type}.".format( + filename=filename, + section=section, + type=anglicize_json_type(python_type_to_yaml_type(config)))) + + for key, value in config.items(): + if not isinstance(key, six.string_types): + raise ConfigurationError( + "In file '{filename}', the {section} name {name} must be a " + "quoted string, i.e. '{name}'.".format( + filename=filename, + section=section, + name=key)) + + if not isinstance(value, (dict, type(None))): + raise ConfigurationError( + "In file '{filename}', {section} '{name}' must be a mapping not " + "{type}.".format( + filename=filename, + section=section, + name=key, + type=anglicize_json_type(python_type_to_yaml_type(value)))) + + def validate_top_level_object(config_file): if not isinstance(config_file.config, dict): raise ConfigurationError( @@ -110,6 +128,18 @@ def validate_top_level_object(config_file): type(config_file.config))) +def validate_ulimits(service_config): + ulimit_config = service_config.config.get('ulimits', {}) + for limit_name, soft_hard_values in six.iteritems(ulimit_config): + if isinstance(soft_hard_values, dict): + if not soft_hard_values['soft'] <= soft_hard_values['hard']: + raise ConfigurationError( + "Service '{s.name}' has invalid ulimit '{ulimit}'. " + "'soft' value can not be greater than 'hard' value ".format( + s=service_config, + ulimit=ulimit_config)) + + def validate_extends_file_path(service_name, extends_options, filename): """ The service to be extended must either be defined in the config key 'file', @@ -123,24 +153,60 @@ def validate_extends_file_path(service_name, extends_options, filename): ) -def get_unsupported_config_msg(service_name, error_key): - msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key) +def validate_network_mode(service_config, service_names): + network_mode = service_config.config.get('network_mode') + if not network_mode: + return + + if 'networks' in service_config.config: + raise ConfigurationError("'network_mode' and 'networks' cannot be combined") + + dependency = get_service_name_from_network_mode(network_mode) + if not dependency: + return + + if dependency not in service_names: + raise ConfigurationError( + "Service '{s.name}' uses the network stack of service '{dep}' which " + "is undefined.".format(s=service_config, dep=dependency)) + + +def validate_links(service_config, service_names): + for link in service_config.config.get('links', []): + if link.split(':')[0] not in service_names: + raise ConfigurationError( + "Service '{s.name}' has a link to service '{link}' which is " + "undefined.".format(s=service_config, link=link)) + + +def validate_depends_on(service_config, service_names): + deps = service_config.config.get('depends_on', {}) + for dependency in deps.keys(): + if dependency not in service_names: + raise ConfigurationError( + "Service '{s.name}' depends on service '{dep}' which is " + "undefined.".format(s=service_config, dep=dependency) + ) + + +def get_unsupported_config_msg(path, error_key): + msg = "Unsupported config option for {}: '{}'".format(path_string(path), error_key) if error_key in DOCKER_CONFIG_HINTS: msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key]) return msg -def anglicize_validator(validator): - if validator in ["array", "object"]: - return 'an ' + validator - return 'a ' + validator +def anglicize_json_type(json_type): + if json_type.startswith(('a', 'e', 'i', 'o', 'u')): + return 'an ' + json_type + return 'a ' + json_type def is_service_dict_schema(schema_id): - return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services' + return schema_id in ('config_schema_v1.json', '#/properties/services') -def handle_error_for_schema_with_id(error, service_name): +def handle_error_for_schema_with_id(error, path): schema_id = error.schema['id'] if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': @@ -149,77 +215,52 @@ def handle_error_for_schema_with_id(error, service_name): list(error.instance)[0], VALID_NAME_CHARS) - if schema_id == '#/definitions/constraints': - # Build context could in 'build' or 'build.context' and dockerfile could be - # in 'dockerfile' or 'build.dockerfile' - context = False - dockerfile = 'dockerfile' in error.instance - if 'build' in error.instance: - if isinstance(error.instance['build'], six.string_types): - context = True - else: - context = 'context' in error.instance['build'] - dockerfile = dockerfile or 'dockerfile' in error.instance['build'] - - # TODO: only applies to v1 - if 'image' in error.instance and context: - return ( - "Service '{}' has both an image and build path specified. " - "A service can either be built to image or use an existing " - "image, not both.".format(service_name)) - if 'image' not in error.instance and not context: - return ( - "Service '{}' has neither an image nor a build path " - "specified. At least one must be provided.".format(service_name)) - # TODO: only applies to v1 - if 'image' in error.instance and dockerfile: - return ( - "Service '{}' has both an image and alternate Dockerfile. " - "A service can either be built to image or use an existing " - "image, not both.".format(service_name)) - - if schema_id == '#/definitions/service': - if error.validator == 'additionalProperties': + if error.validator == 'additionalProperties': + if schema_id == '#/definitions/service': invalid_config_key = parse_key_from_error_msg(error) - return get_unsupported_config_msg(service_name, invalid_config_key) + return get_unsupported_config_msg(path, invalid_config_key) + + if not error.path: + return '{}\n\n{}'.format(error.message, VERSION_EXPLANATION) -def handle_generic_service_error(error, service_name): - config_key = " ".join("'%s'" % k for k in error.path) +def handle_generic_error(error, path): msg_format = None error_msg = error.message if error.validator == 'oneOf': - msg_format = "Service '{}' configuration key {} {}" - error_msg = _parse_oneof_validator(error) + msg_format = "{path} {msg}" + config_key, error_msg = _parse_oneof_validator(error) + if config_key: + path.append(config_key) elif error.validator == 'type': - msg_format = ("Service '{}' configuration key {} contains an invalid " - "type, it should be {}") + msg_format = "{path} contains an invalid type, it should be {msg}" error_msg = _parse_valid_types_from_validator(error.validator_value) - # TODO: no test case for this branch, there are no config options - # which exercise this branch elif error.validator == 'required': - msg_format = "Service '{}' configuration key '{}' is invalid, {}" + error_msg = ", ".join(error.validator_value) + msg_format = "{path} is invalid, {msg} is required." elif error.validator == 'dependencies': - msg_format = "Service '{}' configuration key '{}' is invalid: {}" config_key = list(error.validator_value.keys())[0] required_keys = ",".join(error.validator_value[config_key]) + + msg_format = "{path} is invalid: {msg}" + path.append(config_key) error_msg = "when defining '{}' you must set '{}' as well".format( config_key, required_keys) elif error.cause: error_msg = six.text_type(error.cause) - msg_format = "Service '{}' configuration key {} is invalid: {}" + msg_format = "{path} is invalid: {msg}" elif error.path: - msg_format = "Service '{}' configuration key {} value {}" + msg_format = "{path} value {msg}" if msg_format: - return msg_format.format(service_name, config_key, error_msg) + return msg_format.format(path=path_string(path), msg=error_msg) return error.message @@ -228,19 +269,23 @@ def parse_key_from_error_msg(error): return error.message.split("'")[1] +def path_string(path): + return ".".join(c for c in path if isinstance(c, six.string_types)) + + def _parse_valid_types_from_validator(validator): """A validator value can be either an array of valid types or a string of a valid type. Parse the valid types and prefix with the correct article. """ if not isinstance(validator, list): - return anglicize_validator(validator) + return anglicize_json_type(validator) if len(validator) == 1: - return anglicize_validator(validator[0]) + return anglicize_json_type(validator[0]) return "{}, or {}".format( - ", ".join([anglicize_validator(validator[0])] + validator[1:-1]), - anglicize_validator(validator[-1])) + ", ".join([anglicize_json_type(validator[0])] + validator[1:-1]), + anglicize_json_type(validator[-1])) def _parse_oneof_validator(error): @@ -252,104 +297,127 @@ def _parse_oneof_validator(error): types = [] for context in error.context: + if context.validator == 'oneOf': + _, error_msg = _parse_oneof_validator(context) + return path_string(context.path), error_msg + if context.validator == 'required': - return context.message + return (None, context.message) if context.validator == 'additionalProperties': invalid_config_key = parse_key_from_error_msg(context) - return "contains unsupported option: '{}'".format(invalid_config_key) + return (None, "contains unsupported option: '{}'".format(invalid_config_key)) if context.path: - invalid_config_key = " ".join( - "'{}' ".format(fragment) for fragment in context.path - if isinstance(fragment, six.string_types) + return ( + path_string(context.path), + "contains {}, which is an invalid type, it should be {}".format( + json.dumps(context.instance), + _parse_valid_types_from_validator(context.validator_value)), ) - return "{}contains {}, which is an invalid type, it should be {}".format( - invalid_config_key, - # Always print the json repr of the invalid value - json.dumps(context.instance), - _parse_valid_types_from_validator(context.validator_value)) if context.validator == 'uniqueItems': - return "contains non unique items, please remove duplicates from {}".format( - context.instance) + return ( + None, + "contains non unique items, please remove duplicates from {}".format( + context.instance), + ) if context.validator == 'type': types.append(context.validator_value) valid_types = _parse_valid_types_from_validator(types) - return "contains an invalid type, it should be {}".format(valid_types) + return (None, "contains an invalid type, it should be {}".format(valid_types)) -def process_errors(errors, service_name=None): - """jsonschema gives us an error tree full of information to explain what has +def process_service_constraint_errors(error, service_name, version): + if version == V1: + if 'image' in error.instance and 'build' in error.instance: + return ( + "Service {} has both an image and build path specified. " + "A service can either be built to image or use an existing " + "image, not both.".format(service_name)) + + if 'image' in error.instance and 'dockerfile' in error.instance: + return ( + "Service {} has both an image and alternate Dockerfile. " + "A service can either be built to image or use an existing " + "image, not both.".format(service_name)) + + if 'image' not in error.instance and 'build' not in error.instance: + return ( + "Service {} has neither an image nor a build context specified. " + "At least one must be provided.".format(service_name)) + + +def process_config_schema_errors(error): + path = list(error.path) + + if 'id' in error.schema: + error_msg = handle_error_for_schema_with_id(error, path) + if error_msg: + return error_msg + + return handle_generic_error(error, path) + + +def validate_against_config_schema(config_file): + schema = load_jsonschema(config_file.version) + format_checker = FormatChecker(["ports", "expose"]) + validator = Draft4Validator( + schema, + resolver=RefResolver(get_resolver_path(), schema), + format_checker=format_checker) + handle_errors( + validator.iter_errors(config_file.config), + process_config_schema_errors, + config_file.filename) + + +def validate_service_constraints(config, service_name, version): + def handler(errors): + return process_service_constraint_errors(errors, service_name, version) + + schema = load_jsonschema(version) + validator = Draft4Validator(schema['definitions']['constraints']['service']) + handle_errors(validator.iter_errors(config), handler, None) + + +def get_schema_path(): + return os.path.dirname(os.path.abspath(__file__)) + + +def load_jsonschema(version): + filename = os.path.join( + get_schema_path(), + "config_schema_v{0}.json".format(version)) + + with open(filename, "r") as fh: + return json.load(fh) + + +def get_resolver_path(): + schema_path = get_schema_path() + if sys.platform == "win32": + scheme = "///" + # TODO: why is this necessary? + schema_path = schema_path.replace('\\', '/') + else: + scheme = "//" + return "file:{}{}/".format(scheme, schema_path) + + +def handle_errors(errors, format_error_func, filename): + """jsonschema returns an error tree full of information to explain what has gone wrong. Process each error and pull out relevant information and re-write helpful error messages that are relevant. """ - def format_error_message(error, service_name): - if not service_name and error.path: - # field_schema errors will have service name on the path - service_name = error.path.popleft() - - if 'id' in error.schema: - error_msg = handle_error_for_schema_with_id(error, service_name) - if error_msg: - return error_msg - - return handle_generic_service_error(error, service_name) - - return '\n'.join(format_error_message(error, service_name) for error in errors) - - -def validate_against_fields_schema(config_file): - schema_filename = "fields_schema_v{0}.json".format(config_file.version) - _validate_against_schema( - config_file.config, - schema_filename, - format_checker=["ports", "expose", "bool-value-in-mapping"], - filename=config_file.filename) - - -def validate_against_service_schema(config, service_name, version): - _validate_against_schema( - config, - "service_schema_v{0}.json".format(version), - format_checker=["ports"], - service_name=service_name) - - -def _validate_against_schema( - config, - schema_filename, - format_checker=(), - service_name=None, - filename=None): - config_source_dir = os.path.dirname(os.path.abspath(__file__)) - - if sys.platform == "win32": - file_pre_fix = "///" - config_source_dir = config_source_dir.replace('\\', '/') - else: - file_pre_fix = "//" - - resolver_full_path = "file:{}{}/".format(file_pre_fix, config_source_dir) - schema_file = os.path.join(config_source_dir, schema_filename) - - with open(schema_file, "r") as schema_fh: - schema = json.load(schema_fh) - - resolver = RefResolver(resolver_full_path, schema) - validation_output = Draft4Validator( - schema, - resolver=resolver, - format_checker=FormatChecker(format_checker)) - - errors = [error for error in sorted(validation_output.iter_errors(config), key=str)] + errors = list(sorted(errors, key=str)) if not errors: return - error_msg = process_errors(errors, service_name) - file_msg = " in file '{}'".format(filename) if filename else '' - raise ConfigurationError("Validation failed{}, reason(s):\n{}".format( - file_msg, - error_msg)) + error_msg = '\n'.join(format_error_func(error) for error in errors) + raise ConfigurationError( + "The Compose file{file_msg} is invalid because:\n{error_msg}".format( + file_msg=" '{}'".format(filename) if filename else "", + error_msg=error_msg)) diff --git a/compose/const.py b/compose/const.py index 6ff108fb..e694dbda 100644 --- a/compose/const.py +++ b/compose/const.py @@ -1,22 +1,41 @@ from __future__ import absolute_import from __future__ import unicode_literals -import os import sys DEFAULT_TIMEOUT = 10 -HTTP_TIMEOUT = int(os.environ.get('COMPOSE_HTTP_TIMEOUT', os.environ.get('DOCKER_CLIENT_TIMEOUT', 60))) -IMAGE_EVENTS = ['delete', 'import', 'pull', 'push', 'tag', 'untag'] +HTTP_TIMEOUT = 60 +IMAGE_EVENTS = ['delete', 'import', 'load', 'pull', 'push', 'save', 'tag', 'untag'] IS_WINDOWS_PLATFORM = (sys.platform == "win32") LABEL_CONTAINER_NUMBER = 'com.docker.compose.container-number' LABEL_ONE_OFF = 'com.docker.compose.oneoff' LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' +LABEL_NETWORK = 'com.docker.compose.network' LABEL_VERSION = 'com.docker.compose.version' +LABEL_VOLUME = 'com.docker.compose.volume' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' -COMPOSEFILE_VERSIONS = (1, 2) + +SECRETS_PATH = '/run/secrets' + +COMPOSEFILE_V1 = '1' +COMPOSEFILE_V2_0 = '2.0' +COMPOSEFILE_V2_1 = '2.1' +COMPOSEFILE_V3_0 = '3.0' +COMPOSEFILE_V3_1 = '3.1' API_VERSIONS = { - 1: '1.21', - 2: '1.22', + COMPOSEFILE_V1: '1.21', + COMPOSEFILE_V2_0: '1.22', + COMPOSEFILE_V2_1: '1.24', + COMPOSEFILE_V3_0: '1.25', + COMPOSEFILE_V3_1: '1.25', +} + +API_VERSION_TO_ENGINE_VERSION = { + API_VERSIONS[COMPOSEFILE_V1]: '1.9.0', + API_VERSIONS[COMPOSEFILE_V2_0]: '1.10.0', + API_VERSIONS[COMPOSEFILE_V2_1]: '1.12.0', + API_VERSIONS[COMPOSEFILE_V3_0]: '1.13.0', + API_VERSIONS[COMPOSEFILE_V3_1]: '1.13.0', } diff --git a/compose/container.py b/compose/container.py index 2565c8ff..bda4e659 100644 --- a/compose/container.py +++ b/compose/container.py @@ -39,7 +39,7 @@ class Container(object): @classmethod def from_id(cls, client, id): - return cls(client, client.inspect_container(id)) + return cls(client, client.inspect_container(id), has_been_inspected=True) @classmethod def create(cls, client, **options): @@ -60,7 +60,7 @@ class Container(object): @property def short_id(self): - return self.id[:10] + return self.id[:12] @property def name(self): @@ -134,7 +134,11 @@ class Container(object): @property def environment(self): - return dict(var.split("=", 1) for var in self.get('Config.Env') or []) + def parse_env(var): + if '=' in var: + return var.split("=", 1) + return var, None + return dict(parse_env(var) for var in self.get('Config.Env') or []) @property def exit_code(self): @@ -159,7 +163,7 @@ class Container(object): @property def has_api_logs(self): log_type = self.log_driver - return not log_type or log_type != 'none' + return not log_type or log_type in ('json-file', 'journald') def attach_log_stream(self): """A log stream can only be attached if the container uses a json-file @@ -212,6 +216,12 @@ class Container(object): def remove(self, **options): return self.client.remove_container(self.id, **options) + def create_exec(self, command, **options): + return self.client.exec_create(self.id, command, **options) + + def start_exec(self, exec_id, **options): + return self.client.exec_start(exec_id, **options) + def rename_to_tmp_name(self): """Rename the container to a hopefully unique temporary container name by prepending the short id. diff --git a/compose/errors.py b/compose/errors.py new file mode 100644 index 00000000..415b41e7 --- /dev/null +++ b/compose/errors.py @@ -0,0 +1,33 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + + +class OperationFailedError(Exception): + def __init__(self, reason): + self.msg = reason + + +class StreamParseError(RuntimeError): + def __init__(self, reason): + self.msg = reason + + +class HealthCheckException(Exception): + def __init__(self, reason): + self.msg = reason + + +class HealthCheckFailed(HealthCheckException): + def __init__(self, container_id): + super(HealthCheckFailed, self).__init__( + 'Container "{}" is unhealthy.'.format(container_id) + ) + + +class NoHealthCheckConfigured(HealthCheckException): + def __init__(self, service_name): + super(NoHealthCheckConfigured, self).__init__( + 'Service "{}" is missing a healthcheck configuration'.format( + service_name + ) + ) diff --git a/compose/network.py b/compose/network.py index eaad770c..d98f68d2 100644 --- a/compose/network.py +++ b/compose/network.py @@ -4,22 +4,37 @@ from __future__ import unicode_literals import logging from docker.errors import NotFound +from docker.types import IPAMConfig +from docker.types import IPAMPool +from docker.utils import version_gte +from docker.utils import version_lt from .config import ConfigurationError +from .const import LABEL_NETWORK +from .const import LABEL_PROJECT log = logging.getLogger(__name__) +OPTS_EXCEPTIONS = [ + 'com.docker.network.driver.overlay.vxlanid_list', +] + class Network(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - external_name=None): + ipam=None, external_name=None, internal=False, enable_ipv6=False, + labels=None): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts + self.ipam = create_ipam_config_from_dict(ipam) self.external_name = external_name + self.internal = internal + self.enable_ipv6 = enable_ipv6 + self.labels = labels def ensure(self): if self.external_name: @@ -42,14 +57,7 @@ class Network(object): try: data = self.inspect() - if self.driver and data['Driver'] != self.driver: - raise ConfigurationError( - 'Network {} needs to be recreated - driver has changed' - .format(self.full_name)) - if data['Options'] != (self.driver_opts or {}): - raise ConfigurationError( - 'Network {} needs to be recreated - options have changed' - .format(self.full_name)) + check_remote_network_config(data, self) except NotFound: driver_name = 'the default driver' if self.driver: @@ -61,7 +69,14 @@ class Network(object): ) self.client.create_network( - self.full_name, self.driver, self.driver_opts + name=self.full_name, + driver=self.driver, + options=self.driver_opts, + ipam=self.ipam, + internal=self.internal, + enable_ipv6=self.enable_ipv6, + labels=self._labels, + attachable=version_gte(self.client._version, '1.24') or None, ) def remove(self): @@ -80,3 +95,137 @@ class Network(object): if self.external_name: return self.external_name return '{0}_{1}'.format(self.project, self.name) + + @property + def _labels(self): + if version_lt(self.client._version, '1.23'): + return None + labels = self.labels.copy() if self.labels else {} + labels.update({ + LABEL_PROJECT: self.project, + LABEL_NETWORK: self.name, + }) + return labels + + +def create_ipam_config_from_dict(ipam_dict): + if not ipam_dict: + return None + + return IPAMConfig( + driver=ipam_dict.get('driver'), + pool_configs=[ + IPAMPool( + subnet=config.get('subnet'), + iprange=config.get('ip_range'), + gateway=config.get('gateway'), + aux_addresses=config.get('aux_addresses'), + ) + for config in ipam_dict.get('config', []) + ], + ) + + +def check_remote_network_config(remote, local): + if local.driver and remote.get('Driver') != local.driver: + raise ConfigurationError( + 'Network "{}" needs to be recreated - driver has changed' + .format(local.full_name) + ) + local_opts = local.driver_opts or {} + remote_opts = remote.get('Options') or {} + for k in set.union(set(remote_opts.keys()), set(local_opts.keys())): + if k in OPTS_EXCEPTIONS: + continue + if remote_opts.get(k) != local_opts.get(k): + raise ConfigurationError( + 'Network "{}" needs to be recreated - options have changed' + .format(local.full_name) + ) + + +def build_networks(name, config_data, client): + network_config = config_data.networks or {} + networks = { + network_name: Network( + client=client, project=name, name=network_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + ipam=data.get('ipam'), + external_name=data.get('external_name'), + internal=data.get('internal'), + enable_ipv6=data.get('enable_ipv6'), + labels=data.get('labels'), + ) + for network_name, data in network_config.items() + } + + if 'default' not in networks: + networks['default'] = Network(client, name, 'default') + + return networks + + +class ProjectNetworks(object): + + def __init__(self, networks, use_networking): + self.networks = networks or {} + self.use_networking = use_networking + + @classmethod + def from_services(cls, services, networks, use_networking): + service_networks = { + network: networks.get(network) + for service in services + for network in get_network_names_for_service(service) + } + unused = set(networks) - set(service_networks) - {'default'} + if unused: + log.warn( + "Some networks were defined but are not used by any service: " + "{}".format(", ".join(unused))) + return cls(service_networks, use_networking) + + def remove(self): + if not self.use_networking: + return + for network in self.networks.values(): + try: + network.remove() + except NotFound: + log.warn("Network %s not found.", network.full_name) + + def initialize(self): + if not self.use_networking: + return + + for network in self.networks.values(): + network.ensure() + + +def get_network_defs_for_service(service_dict): + if 'network_mode' in service_dict: + return {} + networks = service_dict.get('networks', {'default': None}) + return dict( + (net, (config or {})) + for net, config in networks.items() + ) + + +def get_network_names_for_service(service_dict): + return get_network_defs_for_service(service_dict).keys() + + +def get_networks(service_dict, network_definitions): + networks = {} + for name, netdef in get_network_defs_for_service(service_dict).items(): + network = network_definitions.get(name) + if network: + networks[network.full_name] = netdef + else: + raise ConfigurationError( + 'Service "{}" uses an undefined network "{}"' + .format(service_dict['name'], name)) + + return networks diff --git a/compose/parallel.py b/compose/parallel.py index b8415e5e..e495410c 100644 --- a/compose/parallel.py +++ b/compose/parallel.py @@ -1,71 +1,199 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging import operator import sys from threading import Thread from docker.errors import APIError +from six.moves import _thread as thread from six.moves.queue import Empty from six.moves.queue import Queue +from compose.cli.signals import ShutdownException +from compose.errors import HealthCheckFailed +from compose.errors import NoHealthCheckConfigured +from compose.errors import OperationFailedError from compose.utils import get_output_stream -def perform_operation(func, arg, callback, index): - try: - callback((index, func(arg))) - except Exception as e: - callback((index, e)) +log = logging.getLogger(__name__) + +STOP = object() -def parallel_execute(objects, func, index_func, msg): - """For a given list of objects, call the callable passing in the first - object we give it. +def parallel_execute(objects, func, get_name, msg, get_deps=None): + """Runs func on objects in parallel while ensuring that func is + ran on object only after it is ran on all its dependencies. + + get_deps called on object must return a collection with its dependencies. + get_name called on object must return its name. """ objects = list(objects) stream = get_output_stream(sys.stderr) + writer = ParallelStreamWriter(stream, msg) - for obj in objects: - writer.initialize(index_func(obj)) + writer.initialize(get_name(obj)) - q = Queue() + events = parallel_execute_iter(objects, func, get_deps) - # TODO: limit the number of threads #1828 - for obj in objects: - t = Thread( - target=perform_operation, - args=(func, obj, q.put, index_func(obj))) - t.daemon = True - t.start() - - done = 0 errors = {} + results = [] + error_to_reraise = None + + for obj, result, exception in events: + if exception is None: + writer.write(get_name(obj), 'done') + results.append(result) + elif isinstance(exception, APIError): + errors[get_name(obj)] = exception.explanation + writer.write(get_name(obj), 'error') + elif isinstance(exception, (OperationFailedError, HealthCheckFailed, NoHealthCheckConfigured)): + errors[get_name(obj)] = exception.msg + writer.write(get_name(obj), 'error') + elif isinstance(exception, UpstreamError): + writer.write(get_name(obj), 'error') + else: + errors[get_name(obj)] = exception + error_to_reraise = exception + + for obj_name, error in errors.items(): + stream.write("\nERROR: for {} {}\n".format(obj_name, error)) + + if error_to_reraise: + raise error_to_reraise + + return results, errors + + +def _no_deps(x): + return [] + + +class State(object): + """ + Holds the state of a partially-complete parallel operation. + + state.started: objects being processed + state.finished: objects which have been processed + state.failed: objects which either failed or whose dependencies failed + """ + def __init__(self, objects): + self.objects = objects + + self.started = set() + self.finished = set() + self.failed = set() + + def is_done(self): + return len(self.finished) + len(self.failed) >= len(self.objects) + + def pending(self): + return set(self.objects) - self.started - self.finished - self.failed + + +def parallel_execute_iter(objects, func, get_deps): + """ + Runs func on objects in parallel while ensuring that func is + ran on object only after it is ran on all its dependencies. + + Returns an iterator of tuples which look like: + + # if func returned normally when run on object + (object, result, None) + + # if func raised an exception when run on object + (object, None, exception) + + # if func raised an exception when run on one of object's dependencies + (object, None, UpstreamError()) + """ + if get_deps is None: + get_deps = _no_deps + + results = Queue() + state = State(objects) + + while True: + feed_queue(objects, func, get_deps, results, state) - while done < len(objects): try: - msg_index, result = q.get(timeout=1) + event = results.get(timeout=0.1) except Empty: continue + # See https://github.com/docker/compose/issues/189 + except thread.error: + raise ShutdownException() - if isinstance(result, APIError): - errors[msg_index] = "error", result.explanation - writer.write(msg_index, 'error') - elif isinstance(result, Exception): - errors[msg_index] = "unexpected_exception", result + if event is STOP: + break + + obj, _, exception = event + if exception is None: + log.debug('Finished processing: {}'.format(obj)) + state.finished.add(obj) else: - writer.write(msg_index, 'done') - done += 1 + log.debug('Failed: {}'.format(obj)) + state.failed.add(obj) - if not errors: - return + yield event - stream.write("\n") - for msg_index, (result, error) in errors.items(): - stream.write("ERROR: for {} {} \n".format(msg_index, error)) - if result == 'unexpected_exception': - raise error + +def producer(obj, func, results): + """ + The entry point for a producer thread which runs func on a single object. + Places a tuple on the results queue once func has either returned or raised. + """ + try: + result = func(obj) + results.put((obj, result, None)) + except Exception as e: + results.put((obj, None, e)) + + +def feed_queue(objects, func, get_deps, results, state): + """ + Starts producer threads for any objects which are ready to be processed + (i.e. they have no dependencies which haven't been successfully processed). + + Shortcuts any objects whose dependencies have failed and places an + (object, None, UpstreamError()) tuple on the results queue. + """ + pending = state.pending() + log.debug('Pending: {}'.format(pending)) + + for obj in pending: + deps = get_deps(obj) + try: + if any(dep[0] in state.failed for dep in deps): + log.debug('{} has upstream errors - not processing'.format(obj)) + results.put((obj, None, UpstreamError())) + state.failed.add(obj) + elif all( + dep not in objects or ( + dep in state.finished and (not ready_check or ready_check(dep)) + ) for dep, ready_check in deps + ): + log.debug('Starting producer thread for {}'.format(obj)) + t = Thread(target=producer, args=(obj, func, results)) + t.daemon = True + t.start() + state.started.add(obj) + except (HealthCheckFailed, NoHealthCheckConfigured) as e: + log.debug( + 'Healthcheck for service(s) upstream of {} failed - ' + 'not processing'.format(obj) + ) + results.put((obj, None, e)) + + if state.is_done(): + results.put(STOP) + + +class UpstreamError(Exception): + pass class ParallelStreamWriter(object): @@ -81,11 +209,15 @@ class ParallelStreamWriter(object): self.lines = [] def initialize(self, obj_index): + if self.msg is None: + return self.lines.append(obj_index) self.stream.write("{} {} ... \r\n".format(self.msg, obj_index)) self.stream.flush() def write(self, obj_index, status): + if self.msg is None: + return position = self.lines.index(obj_index) diff = len(self.lines) - position # move up @@ -111,10 +243,6 @@ def parallel_remove(containers, options): parallel_operation(stopped_containers, 'remove', options, 'Removing') -def parallel_stop(containers, options): - parallel_operation(containers, 'stop', options, 'Stopping') - - def parallel_start(containers, options): parallel_operation(containers, 'start', options, 'Starting') @@ -129,7 +257,3 @@ def parallel_unpause(containers, options): def parallel_kill(containers, options): parallel_operation(containers, 'kill', options, 'Killing') - - -def parallel_restart(containers, options): - parallel_operation(containers, 'restart', options, 'Restarting') diff --git a/compose/progress_stream.py b/compose/progress_stream.py index 1f873d1d..5314f89f 100644 --- a/compose/progress_stream.py +++ b/compose/progress_stream.py @@ -32,12 +32,11 @@ def stream_output(output, stream): if not image_id: continue - if image_id in lines: - diff = len(lines) - lines[image_id] - else: + if image_id not in lines: lines[image_id] = len(lines) stream.write("\n") - diff = 0 + + diff = len(lines) - lines[image_id] # move cursor up `diff` rows stream.write("%c[%dA" % (27, diff)) @@ -91,3 +90,22 @@ def print_output_event(event, stream, is_terminal): stream.write("%s%s" % (event['stream'], terminator)) else: stream.write("%s%s\n" % (status, terminator)) + + +def get_digest_from_pull(events): + for event in events: + status = event.get('status') + if not status or 'Digest' not in status: + continue + + _, digest = status.split(':', 1) + return digest.strip() + return None + + +def get_digest_from_push(events): + for event in events: + digest = event.get('aux', {}).get('Digest') + if digest: + return digest + return None diff --git a/compose/project.py b/compose/project.py index 1322c990..133071e7 100644 --- a/compose/project.py +++ b/compose/project.py @@ -3,112 +3,125 @@ from __future__ import unicode_literals import datetime import logging +import operator from functools import reduce +import enum from docker.errors import APIError -from docker.errors import NotFound from . import parallel from .config import ConfigurationError -from .config.sort_services import get_service_name_from_net -from .const import DEFAULT_TIMEOUT +from .config.config import V1 +from .config.sort_services import get_container_name_from_network_mode +from .config.sort_services import get_service_name_from_network_mode from .const import IMAGE_EVENTS from .const import LABEL_ONE_OFF from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .container import Container -from .network import Network -from .service import ContainerNet +from .network import build_networks +from .network import get_networks +from .network import ProjectNetworks +from .service import BuildAction +from .service import ContainerNetworkMode from .service import ConvergenceStrategy -from .service import Net +from .service import NetworkMode from .service import Service -from .service import ServiceNet +from .service import ServiceNetworkMode from .utils import microseconds_from_time_nano -from .volume import Volume +from .volume import ProjectVolumes log = logging.getLogger(__name__) +@enum.unique +class OneOffFilter(enum.Enum): + include = 0 + exclude = 1 + only = 2 + + @classmethod + def update_labels(cls, value, labels): + if value == cls.only: + labels.append('{0}={1}'.format(LABEL_ONE_OFF, "True")) + elif value == cls.exclude: + labels.append('{0}={1}'.format(LABEL_ONE_OFF, "False")) + elif value == cls.include: + pass + else: + raise ValueError("Invalid value for one_off: {}".format(repr(value))) + + class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, networks=None, volumes=None, - use_networking=False, network_driver=None): + def __init__(self, name, services, client, networks=None, volumes=None): self.name = name self.services = services self.client = client - self.use_networking = use_networking - self.network_driver = network_driver - self.networks = networks or [] - self.volumes = volumes or [] + self.volumes = volumes or ProjectVolumes({}) + self.networks = networks or ProjectNetworks({}, False) - def labels(self, one_off=False): - return [ - '{0}={1}'.format(LABEL_PROJECT, self.name), - '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False"), - ] + def labels(self, one_off=OneOffFilter.exclude): + labels = ['{0}={1}'.format(LABEL_PROJECT, self.name)] + + OneOffFilter.update_labels(one_off, labels) + return labels @classmethod def from_config(cls, name, config_data, client): """ Construct a Project from a config.Config object. """ - use_networking = (config_data.version and config_data.version >= 2) - project = cls(name, [], client, use_networking=use_networking) - - custom_networks = [] - if config_data.networks: - for network_name, data in config_data.networks.items(): - custom_networks.append( - Network( - client=client, project=name, name=network_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - external_name=data.get('external_name'), - ) - ) + use_networking = (config_data.version and config_data.version != V1) + networks = build_networks(name, config_data, client) + project_networks = ProjectNetworks.from_services( + config_data.services, + networks, + use_networking) + volumes = ProjectVolumes.from_config(name, config_data, client) + project = cls(name, [], client, project_networks, volumes) for service_dict in config_data.services: + service_dict = dict(service_dict) if use_networking: - networks = get_networks( - service_dict, - custom_networks + [project.default_network]) - net = Net(networks[0]) if networks else Net("none") - links = [] + service_networks = get_networks(service_dict, networks) else: - networks = [] - net = project.get_net(service_dict) - links = project.get_links(service_dict) + service_networks = {} + service_dict.pop('networks', None) + links = project.get_links(service_dict) + network_mode = project.get_network_mode( + service_dict, list(service_networks.keys()) + ) volumes_from = get_volumes_from(project, service_dict) + if config_data.version != V1: + service_dict['volumes'] = [ + volumes.namespace_spec(volume_spec) + for volume_spec in service_dict.get('volumes', []) + ] + + secrets = get_secrets( + service_dict['name'], + service_dict.pop('secrets', None) or [], + config_data.secrets) + project.services.append( Service( + service_dict.pop('name'), client=client, project=name, use_networking=use_networking, - networks=networks, + networks=service_networks, links=links, - net=net, + network_mode=network_mode, volumes_from=volumes_from, - **service_dict)) - - project.networks += custom_networks - if project.uses_default_network(): - project.networks.append(project.default_network) - - if config_data.volumes: - for vol_name, data in config_data.volumes.items(): - project.volumes.append( - Volume( - client=client, project=name, name=vol_name, - driver=data.get('driver'), - driver_opts=data.get('driver_opts'), - external_name=data.get('external_name') - ) - ) + secrets=secrets, + **service_dict) + ) return project @@ -188,37 +201,67 @@ class Project(object): del service_dict['links'] return links - def get_net(self, service_dict): - net = service_dict.pop('net', None) - if not net: - return Net(None) + def get_network_mode(self, service_dict, networks): + network_mode = service_dict.pop('network_mode', None) + if not network_mode: + if self.networks.use_networking: + return NetworkMode(networks[0]) if networks else NetworkMode('none') + return NetworkMode(None) - net_name = get_service_name_from_net(net) - if not net_name: - return Net(net) + service_name = get_service_name_from_network_mode(network_mode) + if service_name: + return ServiceNetworkMode(self.get_service(service_name)) - try: - return ServiceNet(self.get_service(net_name)) - except NoSuchService: - pass - try: - return ContainerNet(Container.from_id(self.client, net_name)) - except APIError: - raise ConfigurationError( - 'Service "%s" is trying to use the network of "%s", ' - 'which is not the name of a service or container.' % ( - service_dict['name'], - net_name)) + container_name = get_container_name_from_network_mode(network_mode) + if container_name: + try: + return ContainerNetworkMode(Container.from_id(self.client, container_name)) + except APIError: + raise ConfigurationError( + "Service '{name}' uses the network stack of container '{dep}' which " + "does not exist.".format(name=service_dict['name'], dep=container_name)) + + return NetworkMode(network_mode) def start(self, service_names=None, **options): containers = [] - for service in self.get_services(service_names): - service_containers = service.start(**options) + + def start_service(service): + service_containers = service.start(quiet=True, **options) containers.extend(service_containers) + + services = self.get_services(service_names) + + def get_deps(service): + return { + (self.get_service(dep), config) + for dep, config in service.get_dependency_configs().items() + } + + parallel.parallel_execute( + services, + start_service, + operator.attrgetter('name'), + 'Starting', + get_deps) + return containers - def stop(self, service_names=None, **options): - parallel.parallel_stop(self.containers(service_names), options) + def stop(self, service_names=None, one_off=OneOffFilter.exclude, **options): + containers = self.containers(service_names, one_off=one_off) + + def get_deps(container): + # actually returning inversed dependencies + return {(other, None) for other in containers + if container.service in + self.get_service(other.service).get_dependency_names()} + + parallel.parallel_execute( + containers, + self.build_container_operation_with_timeout_func('stop', options), + operator.attrgetter('name'), + 'Stopping', + get_deps) def pause(self, service_names=None, **options): containers = self.containers(service_names) @@ -233,52 +276,20 @@ class Project(object): def kill(self, service_names=None, **options): parallel.parallel_kill(self.containers(service_names), options) - def remove_stopped(self, service_names=None, **options): - parallel.parallel_remove(self.containers(service_names, stopped=True), options) + def remove_stopped(self, service_names=None, one_off=OneOffFilter.exclude, **options): + parallel.parallel_remove(self.containers( + service_names, stopped=True, one_off=one_off + ), options) - def initialize_volumes(self): - try: - for volume in self.volumes: - if volume.external: - log.debug( - 'Volume {0} declared as external. No new ' - 'volume will be created.'.format(volume.name) - ) - if not volume.exists(): - raise ConfigurationError( - 'Volume {name} declared as external, but could' - ' not be found. Please create the volume manually' - ' using `{command}{name}` and try again.'.format( - name=volume.full_name, - command='docker volume create --name=' - ) - ) - continue - volume.create() - except NotFound: - raise ConfigurationError( - 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) - ) - except APIError as e: - if 'Choose a different volume name' in str(e): - raise ConfigurationError( - 'Configuration for volume {0} specifies driver {1}, but ' - 'a volume with the same name uses a different driver ' - '({3}). If you wish to use the new configuration, please ' - 'remove the existing volume "{2}" first:\n' - '$ docker volume rm {2}'.format( - volume.name, volume.driver, volume.full_name, - volume.inspect()['Driver'] - ) - ) + def down(self, remove_image_type, include_volumes, remove_orphans=False): + self.stop(one_off=OneOffFilter.include) + self.find_orphan_containers(remove_orphans) + self.remove_stopped(v=include_volumes, one_off=OneOffFilter.include) - def down(self, remove_image_type, include_volumes): - self.stop() - self.remove_stopped(v=include_volumes) - self.remove_networks() + self.networks.remove() if include_volumes: - self.remove_volumes() + self.volumes.remove() self.remove_images(remove_image_type) @@ -286,36 +297,14 @@ class Project(object): for service in self.get_services(): service.remove_image(remove_image_type) - def remove_networks(self): - if not self.use_networking: - return - for network in self.networks: - network.remove() - - def remove_volumes(self): - for volume in self.volumes: - volume.remove() - - def initialize_networks(self): - if not self.use_networking: - return - - for network in self.networks: - network.ensure() - - def uses_default_network(self): - return any( - self.default_network.full_name in service.networks - for service in self.services - ) - - @property - def default_network(self): - return Network(client=self.client, project=self.name, name='default') - def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) - parallel.parallel_restart(containers, options) + + parallel.parallel_execute( + containers, + self.build_container_operation_with_timeout_func('restart', options), + operator.attrgetter('name'), + 'Restarting') return containers def build(self, service_names=None, no_cache=False, pull=False, force_rm=False): @@ -325,19 +314,25 @@ class Project(object): else: log.info('%s uses an image, skipping' % service.name) - def create(self, service_names=None, strategy=ConvergenceStrategy.changed, do_build=True): + def create( + self, + service_names=None, + strategy=ConvergenceStrategy.changed, + do_build=BuildAction.none, + ): services = self.get_services_without_duplicate(service_names, include_deps=True) + for svc in services: + svc.ensure_image_exists(do_build=do_build) plans = self._get_convergence_plans(services, strategy) for service in services: service.execute_convergence_plan( plans[service.name], - do_build, detached=True, start=False) - def events(self): + def events(self, service_names=None): def build_container_event(event, container): time = datetime.datetime.fromtimestamp(event['time']) time = time.replace( @@ -351,21 +346,29 @@ class Project(object): 'attributes': { 'name': container.name, 'image': event['from'], - } + }, + 'container': container, } - service_names = set(self.service_names) + service_names = set(service_names or self.service_names) for event in self.client.events( filters={'label': self.labels()}, decode=True ): - if event['status'] in IMAGE_EVENTS: + # The first part of this condition is a guard against some events + # broadcasted by swarm that don't have a status field. + # See https://github.com/docker/compose/issues/3316 + if 'status' not in event or event['status'] in IMAGE_EVENTS: # We don't receive any image events because labels aren't applied # to images continue # TODO: get labels from the API v1.22 , see github issue 2618 - container = Container.from_id(self.client, event['id']) + try: + # this can fail if the container has been removed + container = Container.from_id(self.client, event['id']) + except APIError: + continue if container.service not in service_names: continue yield build_container_event(event, container) @@ -374,28 +377,60 @@ class Project(object): service_names=None, start_deps=True, strategy=ConvergenceStrategy.changed, - do_build=True, - timeout=DEFAULT_TIMEOUT, - detached=False): + do_build=BuildAction.none, + timeout=None, + detached=False, + remove_orphans=False): - services = self.get_services_without_duplicate(service_names, include_deps=start_deps) + warn_for_swarm_mode(self.client) + self.initialize() + self.find_orphan_containers(remove_orphans) + + services = self.get_services_without_duplicate( + service_names, + include_deps=start_deps) + + for svc in services: + svc.ensure_image_exists(do_build=do_build) plans = self._get_convergence_plans(services, strategy) - self.initialize_networks() - self.initialize_volumes() - - return [ - container - for service in services - for container in service.execute_convergence_plan( + def do(service): + return service.execute_convergence_plan( plans[service.name], - do_build=do_build, timeout=timeout, detached=detached ) + + def get_deps(service): + return { + (self.get_service(dep), config) + for dep, config in service.get_dependency_configs().items() + } + + results, errors = parallel.parallel_execute( + services, + do, + operator.attrgetter('name'), + None, + get_deps + ) + if errors: + raise ProjectError( + 'Encountered errors while bringing up the project.' + ) + + return [ + container + for svc_containers in results + if svc_containers is not None + for container in svc_containers ] + def initialize(self): + self.networks.initialize() + self.volumes.initialize() + def _get_convergence_plans(self, services, strategy): plans = {} @@ -423,23 +458,56 @@ class Project(object): for service in self.get_services(service_names, include_deps=False): service.pull(ignore_pull_failures) - def containers(self, service_names=None, stopped=False, one_off=False): + def push(self, service_names=None, ignore_push_failures=False): + for service in self.get_services(service_names, include_deps=False): + service.push(ignore_push_failures) + + def _labeled_containers(self, stopped=False, one_off=OneOffFilter.exclude): + return list(filter(None, [ + Container.from_ps(self.client, container) + for container in self.client.containers( + all=stopped, + filters={'label': self.labels(one_off=one_off)})]) + ) + + def containers(self, service_names=None, stopped=False, one_off=OneOffFilter.exclude): if service_names: self.validate_service_names(service_names) else: service_names = self.service_names - containers = list(filter(None, [ - Container.from_ps(self.client, container) - for container in self.client.containers( - all=stopped, - filters={'label': self.labels(one_off=one_off)})])) + containers = self._labeled_containers(stopped, one_off) def matches_service_names(container): return container.labels.get(LABEL_SERVICE) in service_names return [c for c in containers if matches_service_names(c)] + def find_orphan_containers(self, remove_orphans): + def _find(): + containers = self._labeled_containers() + for ctnr in containers: + service_name = ctnr.labels.get(LABEL_SERVICE) + if service_name not in self.service_names: + yield ctnr + orphans = list(_find()) + if not orphans: + return + if remove_orphans: + for ctnr in orphans: + log.info('Removing orphan container "{0}"'.format(ctnr.name)) + ctnr.kill() + ctnr.remove(force=True) + else: + log.warning( + 'Found orphan containers ({0}) for this project. If ' + 'you removed or renamed this service in your compose ' + 'file, you can run this command with the ' + '--remove-orphans flag to clean it up.'.format( + ', '.join(["{}".format(ctnr.name) for ctnr in orphans]) + ) + ) + def _inject_deps(self, acc, service): dep_names = service.get_dependency_names() @@ -454,21 +522,13 @@ class Project(object): dep_services.append(service) return acc + dep_services - -def get_networks(service_dict, network_definitions): - networks = [] - for name in service_dict.pop('networks', ['default']): - if name in ['bridge', 'host']: - networks.append(name) - else: - matches = [n for n in network_definitions if n.name == name] - if matches: - networks.append(matches[0].full_name) - else: - raise ConfigurationError( - 'Service "{}" uses an undefined network "{}"' - .format(service_dict['name'], name)) - return networks + def build_container_operation_with_timeout_func(self, operation, options): + def container_operation_with_timeout(container): + if options.get('timeout') is None: + service = self.get_service(container.service) + options['timeout'] = service.stop_timeout(None) + return getattr(container, operation)(**options) + return container_operation_with_timeout def get_volumes_from(project, service_dict): @@ -499,6 +559,49 @@ def get_volumes_from(project, service_dict): return [build_volume_from(vf) for vf in volumes_from] +def get_secrets(service, service_secrets, secret_defs): + secrets = [] + + for secret in service_secrets: + secret_def = secret_defs.get(secret.source) + if not secret_def: + raise ConfigurationError( + "Service \"{service}\" uses an undefined secret \"{secret}\" " + .format(service=service, secret=secret.source)) + + if secret_def.get('external_name'): + log.warn("Service \"{service}\" uses secret \"{secret}\" which is external. " + "External secrets are not available to containers created by " + "docker-compose.".format(service=service, secret=secret.source)) + continue + + if secret.uid or secret.gid or secret.mode: + log.warn("Service \"{service}\" uses secret \"{secret}\" with uid, " + "gid, or mode. These fields are not supported by this " + "implementation of the Compose file".format( + service=service, secret=secret.source)) + + secrets.append({'secret': secret, 'file': secret_def.get('file')}) + + return secrets + + +def warn_for_swarm_mode(client): + info = client.info() + if info.get('Swarm', {}).get('LocalNodeState') == 'active': + if info.get('ServerVersion', '').startswith('ucp'): + # UCP does multi-node scheduling with traditional Compose files. + return + + log.warn( + "The Docker Engine you're using is running in swarm mode.\n\n" + "Compose does not use swarm mode to deploy services to multiple nodes in a swarm. " + "All containers will be scheduled on the current node.\n\n" + "To deploy your application across the swarm, " + "use `docker stack deploy`.\n" + ) + + class NoSuchService(Exception): def __init__(self, name): self.name = name @@ -506,3 +609,8 @@ class NoSuchService(Exception): def __str__(self): return self.msg + + +class ProjectError(Exception): + def __init__(self, msg): + self.msg = msg diff --git a/compose/service.py b/compose/service.py index f5db07fb..9f2fc68b 100644 --- a/compose/service.py +++ b/compose/service.py @@ -10,15 +10,20 @@ from operator import attrgetter import enum import six from docker.errors import APIError -from docker.utils import LogConfig +from docker.errors import ImageNotFound +from docker.errors import NotFound +from docker.types import LogConfig from docker.utils.ports import build_port_bindings from docker.utils.ports import split_port from . import __version__ +from . import const +from . import progress_stream from .config import DOCKER_CONFIG_KEYS from .config import merge_environment from .config.types import VolumeSpec from .const import DEFAULT_TIMEOUT +from .const import IS_WINDOWS_PLATFORM from .const import LABEL_CONFIG_HASH from .const import LABEL_CONTAINER_NUMBER from .const import LABEL_ONE_OFF @@ -26,13 +31,15 @@ from .const import LABEL_PROJECT from .const import LABEL_SERVICE from .const import LABEL_VERSION from .container import Container +from .errors import HealthCheckFailed +from .errors import NoHealthCheckConfigured +from .errors import OperationFailedError from .parallel import parallel_execute -from .parallel import parallel_remove from .parallel import parallel_start -from .parallel import parallel_stop from .progress_stream import stream_output from .progress_stream import StreamOutputError from .utils import json_hash +from .utils import parse_seconds_float log = logging.getLogger(__name__) @@ -42,26 +49,34 @@ DOCKER_START_KEYS = [ 'cap_add', 'cap_drop', 'cgroup_parent', + 'cpu_quota', 'devices', 'dns', 'dns_search', 'env_file', 'extra_hosts', + 'group_add', 'ipc', 'read_only', - 'net', 'log_driver', 'log_opt', 'mem_limit', 'memswap_limit', + 'oom_score_adj', + 'mem_swappiness', 'pid', 'privileged', 'restart', - 'volumes_from', 'security_opt', - 'cpu_quota', + 'shm_size', + 'sysctls', + 'userns_mode', + 'volumes_from', ] +CONDITION_STARTED = 'service_started' +CONDITION_HEALTHY = 'service_healthy' + class BuildError(Exception): def __init__(self, service, reason): @@ -106,6 +121,14 @@ class ImageType(enum.Enum): all = 2 +@enum.unique +class BuildAction(enum.Enum): + """Enumeration for the possible build actions.""" + none = 0 + force = 1 + skip = 2 + + class Service(object): def __init__( self, @@ -115,8 +138,9 @@ class Service(object): use_networking=False, links=None, volumes_from=None, - net=None, + network_mode=None, networks=None, + secrets=None, **options ): self.name = name @@ -125,10 +149,14 @@ class Service(object): self.use_networking = use_networking self.links = links or [] self.volumes_from = volumes_from or [] - self.net = net or Net(None) - self.networks = networks or [] + self.network_mode = network_mode or NetworkMode(None) + self.networks = networks or {} + self.secrets = secrets or [] self.options = options + def __repr__(self): + return ''.format(self.name) + def containers(self, stopped=False, one_off=False, filters={}): filters.update({'label': self.labels(one_off=one_off)}) @@ -154,7 +182,7 @@ class Service(object): self.start_container_if_stopped(c, **options) return containers - def scale(self, desired_num, timeout=DEFAULT_TIMEOUT): + def scale(self, desired_num, timeout=None): """ Adjusts the number of containers to the specified number and ensures they are running. @@ -164,13 +192,13 @@ class Service(object): - starts containers until there are at least `desired_num` running - removes all stopped containers """ - if self.custom_container_name() and desired_num > 1: + if self.custom_container_name and desired_num > 1: log.warn('The "%s" service is using the custom container name "%s". ' 'Docker requires each container to have a unique name. ' 'Remove the custom name to scale the service.' - % (self.name, self.custom_container_name())) + % (self.name, self.custom_container_name)) - if self.specifies_host_port(): + if self.specifies_host_port() and desired_num > 1: log.warn('The "%s" service specifies a port on the host. If multiple containers ' 'for this service are created on a single host, the port will clash.' % self.name) @@ -180,6 +208,10 @@ class Service(object): service.start_container(container) return container + def stop_and_remove(container): + container.stop(timeout=self.stop_timeout(timeout)) + container.remove() + running_containers = self.containers(stopped=False) num_running = len(running_containers) @@ -194,7 +226,9 @@ class Service(object): if num_running != len(all_containers): # we have some stopped containers, let's start them up again - stopped_containers = sorted([c for c in all_containers if not c.is_running], key=attrgetter('number')) + stopped_containers = sorted( + (c for c in all_containers if not c.is_running), + key=attrgetter('number')) num_stopped = len(stopped_containers) @@ -219,24 +253,26 @@ class Service(object): parallel_execute( container_numbers, lambda n: create_and_start(service=self, number=n), - lambda n: n, + lambda n: self.get_container_name(n), "Creating and starting" ) if desired_num < num_running: num_to_stop = num_running - desired_num + sorted_running_containers = sorted( running_containers, key=attrgetter('number')) - parallel_stop( - sorted_running_containers[-num_to_stop:], - dict(timeout=timeout)) - parallel_remove(self.containers(stopped=True), {}) + parallel_execute( + sorted_running_containers[-num_to_stop:], + stop_and_remove, + lambda c: c.name, + "Stopping and removing", + ) def create_container(self, one_off=False, - do_build=True, previous_container=None, number=None, quiet=False, @@ -245,7 +281,9 @@ class Service(object): Create a container for this service. If the image doesn't exist, attempt to pull it. """ - self.ensure_image_exists(do_build=do_build) + # This is only necessary for `scale` and `volumes_from` + # auto-creating containers to satisfy the dependency. + self.ensure_image_exists() container_options = self._get_container_create_options( override_options, @@ -257,31 +295,41 @@ class Service(object): if 'name' in container_options and not quiet: log.info("Creating %s" % container_options['name']) - return Container.create(self.client, **container_options) + try: + return Container.create(self.client, **container_options) + except APIError as ex: + raise OperationFailedError("Cannot create container for service %s: %s" % + (self.name, ex.explanation)) + + def ensure_image_exists(self, do_build=BuildAction.none): + if self.can_be_built() and do_build == BuildAction.force: + self.build() + return - def ensure_image_exists(self, do_build=True): try: self.image() return except NoSuchImageError: pass - if self.can_be_built(): - if do_build: - self.build() - else: - raise NeedsBuildError(self) - else: + if not self.can_be_built(): self.pull() + return + + if do_build == BuildAction.skip: + raise NeedsBuildError(self) + + self.build() + log.warn( + "Image for service {} was built because it did not already exist. To " + "rebuild this image you must use `docker-compose build` or " + "`docker-compose up --build`.".format(self.name)) def image(self): try: return self.client.inspect_image(self.image_name) - except APIError as e: - if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation): - raise NoSuchImageError("Image '{}' not found".format(self.image_name)) - else: - raise + except ImageNotFound: + raise NoSuchImageError("Image '{}' not found".format(self.image_name)) @property def image_name(self): @@ -336,15 +384,14 @@ class Service(object): def execute_convergence_plan(self, plan, - do_build=True, - timeout=DEFAULT_TIMEOUT, + timeout=None, detached=False, start=True): (action, containers) = plan should_attach_logs = not detached if action == 'create': - container = self.create_container(do_build=do_build) + container = self.create_container() if should_attach_logs: container.attach_log_stream() @@ -358,7 +405,6 @@ class Service(object): return [ self.recreate_container( container, - do_build=do_build, timeout=timeout, attach_logs=should_attach_logs, start_new_container=start @@ -385,8 +431,7 @@ class Service(object): def recreate_container( self, container, - do_build=False, - timeout=DEFAULT_TIMEOUT, + timeout=None, attach_logs=False, start_new_container=True): """Recreate a container. @@ -397,10 +442,9 @@ class Service(object): """ log.info("Recreating %s" % container.name) - container.stop(timeout=timeout) + container.stop(timeout=self.stop_timeout(timeout)) container.rename_to_tmp_name() new_container = self.create_container( - do_build=do_build, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), quiet=True, @@ -412,29 +456,55 @@ class Service(object): container.remove() return new_container - def start_container_if_stopped(self, container, attach_logs=False): + def stop_timeout(self, timeout): + if timeout is not None: + return timeout + timeout = parse_seconds_float(self.options.get('stop_grace_period')) + if timeout is not None: + return timeout + return DEFAULT_TIMEOUT + + def start_container_if_stopped(self, container, attach_logs=False, quiet=False): if not container.is_running: - log.info("Starting %s" % container.name) + if not quiet: + log.info("Starting %s" % container.name) if attach_logs: container.attach_log_stream() return self.start_container(container) def start_container(self, container): - container.start() self.connect_container_to_networks(container) + try: + container.start() + except APIError as ex: + raise OperationFailedError("Cannot start service %s: %s" % (self.name, ex.explanation)) return container def connect_container_to_networks(self, container): - for network in self.networks: - log.debug('Connecting "{}" to "{}"'.format(container.name, network)) + connected_networks = container.get('NetworkSettings.Networks') + + for network, netdefs in self.networks.items(): + if network in connected_networks: + if short_id_alias_exists(container, network): + continue + + self.client.disconnect_container_from_network( + container.id, + network) + self.client.connect_container_to_network( container.id, network, - aliases=[self.name]) + aliases=self._get_aliases(netdefs, container), + ipv4_address=netdefs.get('ipv4_address', None), + ipv6_address=netdefs.get('ipv6_address', None), + links=self._get_links(False), + link_local_ips=netdefs.get('link_local_ips', None), + ) - def remove_duplicate_containers(self, timeout=DEFAULT_TIMEOUT): + def remove_duplicate_containers(self, timeout=None): for c in self.duplicate_containers(): log.info('Removing %s' % c.name) - c.stop(timeout=timeout) + c.stop(timeout=self.stop_timeout(timeout)) c.remove() def duplicate_containers(self): @@ -460,7 +530,8 @@ class Service(object): 'options': self.options, 'image_id': self.image()['Id'], 'links': self.get_link_names(), - 'net': self.net.id, + 'net': self.network_mode.id, + 'networks': self.networks, 'volumes_from': [ (v.source.name, v.mode) for v in self.volumes_from if isinstance(v.source, Service) @@ -468,10 +539,39 @@ class Service(object): } def get_dependency_names(self): - net_name = self.net.service_name - return (self.get_linked_service_names() + - self.get_volumes_from_names() + - ([net_name] if net_name else [])) + net_name = self.network_mode.service_name + return ( + self.get_linked_service_names() + + self.get_volumes_from_names() + + ([net_name] if net_name else []) + + list(self.options.get('depends_on', {}).keys()) + ) + + def get_dependency_configs(self): + net_name = self.network_mode.service_name + configs = dict( + [(name, None) for name in self.get_linked_service_names()] + ) + configs.update(dict( + [(name, None) for name in self.get_volumes_from_names()] + )) + configs.update({net_name: None} if net_name else {}) + configs.update(self.options.get('depends_on', {})) + for svc, config in self.options.get('depends_on', {}).items(): + if config['condition'] == CONDITION_STARTED: + configs[svc] = lambda s: True + elif config['condition'] == CONDITION_HEALTHY: + configs[svc] = lambda s: s.is_healthy() + else: + # The config schema already prevents this, but it might be + # bypassed if Compose is called programmatically. + raise ValueError( + 'depends_on condition "{}" is invalid.'.format( + config['condition'] + ) + ) + + return configs def get_linked_service_names(self): return [service.name for (service, _) in self.links] @@ -482,10 +582,6 @@ class Service(object): def get_volumes_from_names(self): return [s.source.name for s in self.volumes_from if isinstance(s.source, Service)] - def get_container_name(self, number, one_off=False): - # TODO: Implement issue #652 here - return build_container_name(self.project, self.name, number, one_off) - # TODO: this would benefit from github.com/docker/docker/pull/14699 # to remove the need to inspect every container def _next_container_number(self, one_off=False): @@ -498,28 +594,59 @@ class Service(object): numbers = [c.number for c in containers] return 1 if not numbers else max(numbers) + 1 - def _get_links(self, link_to_self): - if self.use_networking: + def _get_aliases(self, network, container=None): + if container and container.labels.get(LABEL_ONE_OFF) == "True": return [] - links = [] + return list( + {self.name} | + ({container.short_id} if container else set()) | + set(network.get('aliases', ())) + ) + + def build_default_networking_config(self): + if not self.networks: + return {} + + network = self.networks[self.network_mode.id] + endpoint = { + 'Aliases': self._get_aliases(network), + 'IPAMConfig': {}, + } + + if network.get('ipv4_address'): + endpoint['IPAMConfig']['IPv4Address'] = network.get('ipv4_address') + if network.get('ipv6_address'): + endpoint['IPAMConfig']['IPv6Address'] = network.get('ipv6_address') + + return {"EndpointsConfig": {self.network_mode.id: endpoint}} + + def _get_links(self, link_to_self): + links = {} + for service, link_name in self.links: for container in service.containers(): - links.append((container.name, link_name or service.name)) - links.append((container.name, container.name)) - links.append((container.name, container.name_without_project)) + links[link_name or service.name] = container.name + links[container.name] = container.name + links[container.name_without_project] = container.name + if link_to_self: for container in self.containers(): - links.append((container.name, self.name)) - links.append((container.name, container.name)) - links.append((container.name, container.name_without_project)) + links[self.name] = container.name + links[container.name] = container.name + links[container.name_without_project] = container.name + for external_link in self.options.get('external_links') or []: if ':' not in external_link: link_name = external_link else: external_link, link_name = external_link.split(':') - links.append((external_link, link_name)) - return links + links[link_name] = external_link + + return [ + (alias, container_name) + for (container_name, alias) in links.items() + ] def _get_volumes_from(self): return [build_volume_from(spec) for spec in self.volumes_from] @@ -537,13 +664,10 @@ class Service(object): for k in DOCKER_CONFIG_KEYS if k in self.options) container_options.update(override_options) - if self.custom_container_name() and not one_off: - container_options['name'] = self.custom_container_name() - elif not container_options.get('name'): + if not container_options.get('name'): container_options['name'] = self.get_container_name(number, one_off) - if 'detach' not in container_options: - container_options['detach'] = True + container_options.setdefault('detach', True) # If a qualified hostname was given, split it into an # unqualified hostname and a domainname unless domainname @@ -557,31 +681,28 @@ class Service(object): container_options['domainname'] = parts[2] if 'ports' in container_options or 'expose' in self.options: - ports = [] - all_ports = container_options.get('ports', []) + self.options.get('expose', []) - for port_range in all_ports: - internal_range, _ = split_port(port_range) - for port in internal_range: - port = str(port) - if '/' in port: - port = tuple(port.split('/')) - ports.append(port) - container_options['ports'] = ports - - override_options['binds'] = merge_volume_bindings( - container_options.get('volumes') or [], - previous_container) - - if 'volumes' in container_options: - container_options['volumes'] = dict( - (v.internal, {}) for v in container_options['volumes']) + container_options['ports'] = build_container_ports( + container_options, + self.options) container_options['environment'] = merge_environment( self.options.get('environment'), override_options.get('environment')) - if previous_container: - container_options['environment']['affinity:container'] = ('=' + previous_container.id) + binds, affinity = merge_volume_bindings( + container_options.get('volumes') or [], + previous_container) + override_options['binds'] = binds + container_options['environment'].update(affinity) + + container_options['volumes'] = dict( + (v.internal, {}) for v in container_options.get('volumes') or {}) + + secret_volumes = self.get_secret_volumes() + if secret_volumes: + override_options['binds'].extend(v.repr() for v in secret_volumes) + container_options['volumes'].update( + (v.internal, {}) for v in secret_volumes) container_options['image'] = self.image_name @@ -599,8 +720,12 @@ class Service(object): override_options, one_off=one_off) - container_options['networking_config'] = self._get_container_networking_config() + networking_config = self.build_default_networking_config() + if networking_config: + container_options['networking_config'] = networking_config + container_options['environment'] = format_environment( + container_options['environment']) return container_options def _get_container_host_config(self, override_options, one_off=False): @@ -609,13 +734,13 @@ class Service(object): logging_dict = options.get('logging', None) log_config = get_log_config(logging_dict) - return self.client.create_host_config( + host_config = self.client.create_host_config( links=self._get_links(link_to_self=one_off), port_bindings=build_port_bindings(options.get('ports') or []), binds=options.get('binds'), volumes_from=self._get_volumes_from(), privileged=options.get('privileged', False), - network_mode=self.net.mode, + network_mode=self.network_mode.mode, devices=options.get('devices'), dns=options.get('dns'), dns_search=options.get('dns_search'), @@ -633,23 +758,38 @@ class Service(object): ipc_mode=options.get('ipc'), cgroup_parent=options.get('cgroup_parent'), cpu_quota=options.get('cpu_quota'), + shm_size=options.get('shm_size'), + sysctls=options.get('sysctls'), + tmpfs=options.get('tmpfs'), + oom_score_adj=options.get('oom_score_adj'), + mem_swappiness=options.get('mem_swappiness'), + group_add=options.get('group_add'), + userns_mode=options.get('userns_mode') ) - def _get_container_networking_config(self): - return self.client.create_networking_config({ - network_name: self.client.create_endpoint_config(aliases=[self.name]) - for network_name in self.networks - if network_name not in ['host', 'bridge'] - }) + # TODO: Add as an argument to create_host_config once it's supported + # in docker-py + host_config['Isolation'] = options.get('isolation') + + return host_config + + def get_secret_volumes(self): + def build_spec(secret): + target = '{}/{}'.format( + const.SECRETS_PATH, + secret['secret'].target or secret['secret'].source) + return VolumeSpec(secret['file'], target, 'ro') + + return [build_spec(secret) for secret in self.secrets] def build(self, no_cache=False, pull=False, force_rm=False): log.info('Building %s' % self.name) build_opts = self.options.get('build', {}) path = build_opts.get('context') - # python2 os.path() doesn't support unicode, so we need to encode it to - # a byte string - if not six.PY3: + # python2 os.stat() doesn't support unicode on some UNIX, so we + # encode it to a bytestring to be safe + if not six.PY3 and not IS_WINDOWS_PLATFORM: path = path.encode('utf8') build_output = self.client.build( @@ -697,9 +837,16 @@ class Service(object): '{0}={1}'.format(LABEL_ONE_OFF, "True" if one_off else "False") ] + @property def custom_container_name(self): return self.options.get('container_name') + def get_container_name(self, number, one_off=False): + if self.custom_container_name and not one_off: + return self.custom_container_name + + return build_container_name(self.project, self.name, number, one_off) + def remove_image(self, image_type): if not image_type or image_type == ImageType.none: return False @@ -745,37 +892,75 @@ class Service(object): repo, tag, separator = parse_repository_tag(self.options['image']) tag = tag or 'latest' log.info('Pulling %s (%s%s%s)...' % (self.name, repo, separator, tag)) - output = self.client.pull( - repo, - tag=tag, - stream=True, - ) - try: - stream_output(output, sys.stdout) - except StreamOutputError as e: + output = self.client.pull(repo, tag=tag, stream=True) + return progress_stream.get_digest_from_pull( + stream_output(output, sys.stdout)) + except (StreamOutputError, NotFound) as e: if not ignore_pull_failures: raise else: log.error(six.text_type(e)) + def push(self, ignore_push_failures=False): + if 'image' not in self.options or 'build' not in self.options: + return -class Net(object): + repo, tag, separator = parse_repository_tag(self.options['image']) + tag = tag or 'latest' + log.info('Pushing %s (%s%s%s)...' % (self.name, repo, separator, tag)) + output = self.client.push(repo, tag=tag, stream=True) + + try: + return progress_stream.get_digest_from_push( + stream_output(output, sys.stdout)) + except StreamOutputError as e: + if not ignore_push_failures: + raise + else: + log.error(six.text_type(e)) + + def is_healthy(self): + """ Check that all containers for this service report healthy. + Returns false if at least one healthcheck is pending. + If an unhealthy container is detected, raise a HealthCheckFailed + exception. + """ + result = True + for ctnr in self.containers(): + ctnr.inspect() + status = ctnr.get('State.Health.Status') + if status is None: + raise NoHealthCheckConfigured(self.name) + elif status == 'starting': + result = False + elif status == 'unhealthy': + raise HealthCheckFailed(ctnr.short_id) + return result + + +def short_id_alias_exists(container, network): + aliases = container.get( + 'NetworkSettings.Networks.{net}.Aliases'.format(net=network)) or () + return container.short_id in aliases + + +class NetworkMode(object): """A `standard` network mode (ex: host, bridge)""" service_name = None - def __init__(self, net): - self.net = net + def __init__(self, network_mode): + self.network_mode = network_mode @property def id(self): - return self.net + return self.network_mode mode = id -class ContainerNet(object): +class ContainerNetworkMode(object): """A network mode that uses a container's network stack.""" service_name = None @@ -792,7 +977,7 @@ class ContainerNet(object): return 'container:' + self.container.id -class ServiceNet(object): +class ServiceNetworkMode(object): """A network mode that uses a service's network stack.""" def __init__(self, service): @@ -861,18 +1046,23 @@ def merge_volume_bindings(volumes, previous_container): """Return a list of volume bindings for a container. Container data volumes are replaced by those from the previous container. """ + affinity = {} + volume_bindings = dict( build_volume_binding(volume) for volume in volumes if volume.external) if previous_container: - data_volumes = get_container_data_volumes(previous_container, volumes) - warn_on_masked_volume(volumes, data_volumes, previous_container.service) + old_volumes = get_container_data_volumes(previous_container, volumes) + warn_on_masked_volume(volumes, old_volumes, previous_container.service) volume_bindings.update( - build_volume_binding(volume) for volume in data_volumes) + build_volume_binding(volume) for volume in old_volumes) - return list(volume_bindings.values()) + if old_volumes: + affinity = {'affinity:container': '=' + previous_container.id} + + return list(volume_bindings.values()), affinity def get_container_data_volumes(container, volumes_option): @@ -904,8 +1094,12 @@ def get_container_data_volumes(container, volumes_option): if not mount: continue + # Volume was previously a host volume, now it's a container volume + if not mount.get('Name'): + continue + # Copy existing volume from old container - volume = volume._replace(external=mount['Source']) + volume = volume._replace(external=mount['Name']) volumes.append(volume) return volumes @@ -996,3 +1190,29 @@ def get_log_config(logging_dict): type=log_driver, config=log_options ) + + +# TODO: remove once fix is available in docker-py +def format_environment(environment): + def format_env(key, value): + if value is None: + return key + if isinstance(value, six.binary_type): + value = value.decode('utf-8') + return '{key}={value}'.format(key=key, value=value) + return [format_env(*item) for item in environment.items()] + +# Ports + + +def build_container_ports(container_options, options): + ports = [] + all_ports = container_options.get('ports', []) + options.get('expose', []) + for port_range in all_ports: + internal_range, _ = split_port(port_range) + for port in internal_range: + port = str(port) + if '/' in port: + port = tuple(port.split('/')) + ports.append(port) + return ports diff --git a/compose/timeparse.py b/compose/timeparse.py new file mode 100644 index 00000000..16ef8a6d --- /dev/null +++ b/compose/timeparse.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +''' +timeparse.py +(c) Will Roberts 1 February, 2014 + +This is a vendored and modified copy of: +github.com/wroberts/pytimeparse @ cc0550d + +It has been modified to mimic the behaviour of +https://golang.org/pkg/time/#ParseDuration +''' +# MIT LICENSE +# +# Permission is hereby granted, free of charge, to any person +# obtaining a copy of this software and associated documentation files +# (the "Software"), to deal in the Software without restriction, +# including without limitation the rights to use, copy, modify, merge, +# publish, distribute, sublicense, and/or sell copies of the Software, +# and to permit persons to whom the Software is furnished to do so, +# subject to the following conditions: +# +# The above copyright notice and this permission notice shall be +# included in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS +# BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN +# ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN +# CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. +from __future__ import absolute_import +from __future__ import unicode_literals + +import re + +HOURS = r'(?P[\d.]+)h' +MINS = r'(?P[\d.]+)m' +SECS = r'(?P[\d.]+)s' +MILLI = r'(?P[\d.]+)ms' +MICRO = r'(?P[\d.]+)(?:us|µs)' +NANO = r'(?P[\d.]+)ns' + + +def opt(x): + return r'(?:{x})?'.format(x=x) + + +TIMEFORMAT = r'{HOURS}{MINS}{SECS}{MILLI}{MICRO}{NANO}'.format( + HOURS=opt(HOURS), + MINS=opt(MINS), + SECS=opt(SECS), + MILLI=opt(MILLI), + MICRO=opt(MICRO), + NANO=opt(NANO), +) + +MULTIPLIERS = dict([ + ('hours', 60 * 60), + ('mins', 60), + ('secs', 1), + ('milli', 1.0 / 1000), + ('micro', 1.0 / 1000.0 / 1000), + ('nano', 1.0 / 1000.0 / 1000.0 / 1000.0), +]) + + +def timeparse(sval): + """Parse a time expression, returning it as a number of seconds. If + possible, the return value will be an `int`; if this is not + possible, the return will be a `float`. Returns `None` if a time + expression cannot be parsed from the given string. + + Arguments: + - `sval`: the string value to parse + + >>> timeparse('1m24s') + 84 + >>> timeparse('1.2 minutes') + 72 + >>> timeparse('1.2 seconds') + 1.2 + """ + match = re.match(r'\s*' + TIMEFORMAT + r'\s*$', sval, re.I) + if not match or not match.group(0).strip(): + return + + mdict = match.groupdict() + return sum( + MULTIPLIERS[k] * cast(v) for (k, v) in mdict.items() if v is not None) + + +def cast(value): + return int(value, 10) if value.isdigit() else float(value) diff --git a/compose/utils.py b/compose/utils.py index 29d8a695..b8bdf732 100644 --- a/compose/utils.py +++ b/compose/utils.py @@ -5,11 +5,17 @@ import codecs import hashlib import json import json.decoder +import logging +import ntpath import six +from .errors import StreamParseError +from .timeparse import timeparse + json_decoder = json.JSONDecoder() +log = logging.getLogger(__name__) def get_output_stream(stream): @@ -60,13 +66,21 @@ def split_buffer(stream, splitter=None, decoder=lambda a: a): yield item if buffered: - yield decoder(buffered) + try: + yield decoder(buffered) + except Exception as e: + log.error( + 'Compose tried decoding the following data chunk, but failed:' + '\n%s' % repr(buffered) + ) + raise StreamParseError(e) def json_splitter(buffer): """Attempt to parse a json object from a buffer. If there is at least one object, return it and the rest of the buffer, otherwise return None. """ + buffer = buffer.strip() try: obj, index = json_decoder.raw_decode(buffer) rest = buffer[json.decoder.WHITESPACE.match(buffer, index).end():] @@ -92,3 +106,30 @@ def json_hash(obj): def microseconds_from_time_nano(time_nano): return int(time_nano % 1000000000 / 1000) + + +def nanoseconds_from_time_seconds(time_seconds): + return time_seconds * 1000000000 + + +def parse_seconds_float(value): + return timeparse(value or '') + + +def parse_nanoseconds_int(value): + parsed = timeparse(value or '') + if parsed is None: + return None + return int(parsed * 1000000000) + + +def build_string_dict(source_dict): + return dict((k, str(v if v is not None else '')) for k, v in source_dict.items()) + + +def splitdrive(path): + if len(path) == 0: + return ('', '') + if path[0] in ['.', '\\', '/', '~']: + return ('', path) + return ntpath.splitdrive(path) diff --git a/compose/volume.py b/compose/volume.py index 469e406a..ab6a88fa 100644 --- a/compose/volume.py +++ b/compose/volume.py @@ -4,24 +4,29 @@ from __future__ import unicode_literals import logging from docker.errors import NotFound +from docker.utils import version_lt +from .config import ConfigurationError +from .const import LABEL_PROJECT +from .const import LABEL_VOLUME log = logging.getLogger(__name__) class Volume(object): def __init__(self, client, project, name, driver=None, driver_opts=None, - external_name=None): + external_name=None, labels=None): self.client = client self.project = project self.name = name self.driver = driver self.driver_opts = driver_opts self.external_name = external_name + self.labels = labels def create(self): return self.client.create_volume( - self.full_name, self.driver, self.driver_opts + self.full_name, self.driver, self.driver_opts, labels=self._labels ) def remove(self): @@ -50,3 +55,97 @@ class Volume(object): if self.external_name: return self.external_name return '{0}_{1}'.format(self.project, self.name) + + @property + def _labels(self): + if version_lt(self.client._version, '1.23'): + return None + labels = self.labels.copy() if self.labels else {} + labels.update({ + LABEL_PROJECT: self.project, + LABEL_VOLUME: self.name, + }) + return labels + + +class ProjectVolumes(object): + + def __init__(self, volumes): + self.volumes = volumes + + @classmethod + def from_config(cls, name, config_data, client): + config_volumes = config_data.volumes or {} + volumes = { + vol_name: Volume( + client=client, + project=name, + name=vol_name, + driver=data.get('driver'), + driver_opts=data.get('driver_opts'), + external_name=data.get('external_name'), + labels=data.get('labels') + ) + for vol_name, data in config_volumes.items() + } + return cls(volumes) + + def remove(self): + for volume in self.volumes.values(): + try: + volume.remove() + except NotFound: + log.warn("Volume %s not found.", volume.full_name) + + def initialize(self): + try: + for volume in self.volumes.values(): + volume_exists = volume.exists() + if volume.external: + log.debug( + 'Volume {0} declared as external. No new ' + 'volume will be created.'.format(volume.name) + ) + if not volume_exists: + raise ConfigurationError( + 'Volume {name} declared as external, but could' + ' not be found. Please create the volume manually' + ' using `{command}{name}` and try again.'.format( + name=volume.full_name, + command='docker volume create --name=' + ) + ) + continue + + if not volume_exists: + log.info( + 'Creating volume "{0}" with {1} driver'.format( + volume.full_name, volume.driver or 'default' + ) + ) + volume.create() + else: + driver = volume.inspect()['Driver'] + if volume.driver is not None and driver != volume.driver: + raise ConfigurationError( + 'Configuration for volume {0} specifies driver ' + '{1}, but a volume with the same name uses a ' + 'different driver ({3}). If you wish to use the ' + 'new configuration, please remove the existing ' + 'volume "{2}" first:\n' + '$ docker volume rm {2}'.format( + volume.name, volume.driver, volume.full_name, + volume.inspect()['Driver'] + ) + ) + except NotFound: + raise ConfigurationError( + 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) + ) + + def namespace_spec(self, volume_spec): + if not volume_spec.is_named_volume: + return volume_spec + + volume = self.volumes[volume_spec.external] + return volume_spec._replace(external=volume.full_name) diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index 2b020c37..77d02b42 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -17,29 +17,34 @@ # . ~/.docker-compose-completion.sh +__docker_compose_q() { + docker-compose 2>/dev/null $daemon_options "$@" +} + +# Transforms a multiline list of strings into a single line string +# with the words separated by "|". +__docker_compose_to_alternatives() { + local parts=( $1 ) + local IFS='|' + echo "${parts[*]}" +} + +# Transforms a multiline list of options into an extglob pattern +# suitable for use in case statements. +__docker_compose_to_extglob() { + local extglob=$( __docker_compose_to_alternatives "$1" ) + echo "@($extglob)" +} + # suppress trailing whitespace __docker_compose_nospace() { # compopt is not available in ancient bash versions type compopt &>/dev/null && compopt -o nospace } -# For compatibility reasons, Compose and therefore its completion supports several -# stack compositon files as listed here, in descending priority. -# Support for these filenames might be dropped in some future version. -__docker_compose_compose_file() { - local file - for file in docker-compose.y{,a}ml ; do - [ -e $file ] && { - echo $file - return - } - done - echo docker-compose.yml -} - # Extracts all service names from the compose file. ___docker_compose_all_services_in_compose_file() { - awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker_compose_compose_file)}" 2>/dev/null + __docker_compose_q config --services } # All services, even those without an existing container @@ -49,8 +54,12 @@ __docker_compose_services_all() { # All services that have an entry with the given key in their compose_file section ___docker_compose_services_with_key() { - # flatten sections to one line, then filter lines containing the key and return section name. - awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker_compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' + # flatten sections under "services" to one line, then filter lines containing the key and return section name + __docker_compose_q config \ + | sed -n -e '/^services:/,/^[^ ]/p' \ + | sed -n 's/^ //p' \ + | awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' \ + | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' } # All services that are defined by a Dockerfile reference @@ -67,11 +76,9 @@ __docker_compose_services_from_image() { # by a boolean expression passed in as argument. __docker_compose_services_with() { local containers names - containers="$(docker-compose 2>/dev/null ${compose_file:+-f $compose_file} ${compose_project:+-p $compose_project} ps -q)" - names=( $(docker 2>/dev/null inspect --format "{{if ${1:-true}}} {{ .Name }} {{end}}" $containers) ) - names=( ${names[@]%_*} ) # strip trailing numbers - names=( ${names[@]#*_} ) # strip project name - COMPREPLY=( $(compgen -W "${names[*]}" -- "$cur") ) + containers="$(__docker_compose_q ps -q)" + names=$(docker 2>/dev/null inspect -f "{{if ${1:-true}}}{{range \$k, \$v := .Config.Labels}}{{if eq \$k \"com.docker.compose.service\"}}{{\$v}}{{end}}{{end}}{{end}}" $containers) + COMPREPLY=( $(compgen -W "$names" -- "$cur") ) } # The services for which at least one paused container exists @@ -102,25 +109,53 @@ _docker_compose_build() { } +_docker_compose_bundle() { + case "$prev" in + --output|-o) + _filedir + return + ;; + esac + + COMPREPLY=( $( compgen -W "--push-images --help --output -o" -- "$cur" ) ) +} + + _docker_compose_config() { COMPREPLY=( $( compgen -W "--help --quiet -q --services" -- "$cur" ) ) } +_docker_compose_create() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--force-recreate --help --no-build --no-recreate" -- "$cur" ) ) + ;; + *) + __docker_compose_services_all + ;; + esac +} + + _docker_compose_docker_compose() { case "$prev" in + --tlscacert|--tlscert|--tlskey) + _filedir + return + ;; --file|-f) _filedir "y?(a)ml" return ;; - --project-name|-p) + $(__docker_compose_to_extglob "$daemon_options_with_args") ) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "$daemon_boolean_options $daemon_options_with_args --help -h --verbose --version -v" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -139,7 +174,7 @@ _docker_compose_down() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --rmi --volumes -v" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--help --rmi --volumes -v --remove-orphans" -- "$cur" ) ) ;; esac } @@ -163,6 +198,24 @@ _docker_compose_events() { } +_docker_compose_exec() { + case "$prev" in + --index|--user) + return + ;; + esac + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "-d --help --index --privileged -T --user" -- "$cur" ) ) + ;; + *) + __docker_compose_services_running + ;; + esac +} + + _docker_compose_help() { COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) } @@ -188,9 +241,15 @@ _docker_compose_kill() { _docker_compose_logs() { + case "$prev" in + --tail) + return + ;; + esac + case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help --no-color" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--follow -f --help --no-color --tail --timestamps -t" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -257,6 +316,18 @@ _docker_compose_pull() { } +_docker_compose_push() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help --ignore-push-failures" -- "$cur" ) ) + ;; + *) + __docker_compose_services_all + ;; + esac +} + + _docker_compose_restart() { case "$prev" in --timeout|-t) @@ -294,14 +365,14 @@ _docker_compose_run() { __docker_compose_nospace return ;; - --entrypoint|--name|--user|-u) + --entrypoint|--name|--user|-u|--workdir|-w) return ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "-d --entrypoint -e --help --name --no-deps --publish -p --rm --service-ports -T --user -u --workdir -w" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -363,6 +434,18 @@ _docker_compose_stop() { } +_docker_compose_top() { + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + __docker_compose_services_running + ;; + esac +} + + _docker_compose_unpause() { case "$cur" in -*) @@ -384,7 +467,7 @@ _docker_compose_up() { case "$cur" in -*) - COMPREPLY=( $( compgen -W "--abort-on-container-exit -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--abort-on-container-exit --build -d --force-recreate --help --no-build --no-color --no-deps --no-recreate --timeout -t --remove-orphans" -- "$cur" ) ) ;; *) __docker_compose_services_all @@ -408,9 +491,12 @@ _docker_compose() { local commands=( build + bundle config + create down events + exec help kill logs @@ -418,17 +504,35 @@ _docker_compose() { port ps pull + push restart rm run scale start stop + top unpause up version ) + # options for the docker daemon that have to be passed to secondary calls to + # docker-compose executed by this script + local daemon_boolean_options=" + --skip-hostname-check + --tls + --tlsverify + " + local daemon_options_with_args=" + --file -f + --host -H + --project-name -p + --tlscacert + --tlscert + --tlskey + " + COMPREPLY=() local cur prev words cword _get_comp_words_by_ref -n : cur prev words cword @@ -436,17 +540,19 @@ _docker_compose() { # search subcommand and invoke its handler. # special treatment of some top-level options local command='docker_compose' + local daemon_options=() local counter=1 - local compose_file compose_project + while [ $counter -lt $cword ]; do case "${words[$counter]}" in - --file|-f) - (( counter++ )) - compose_file="${words[$counter]}" + $(__docker_compose_to_extglob "$daemon_boolean_options") ) + local opt=${words[counter]} + daemon_options+=($opt) ;; - --project-name|p) - (( counter++ )) - compose_project="${words[$counter]}" + $(__docker_compose_to_extglob "$daemon_options_with_args") ) + local opt=${words[counter]} + local arg=${words[++counter]} + daemon_options+=($opt $arg) ;; -*) ;; diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index 01e5f896..66d924f7 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -19,52 +19,49 @@ # * @felixr docker zsh completion script : https://github.com/felixr/docker-zsh-completion # ------------------------------------------------------------------------- -# For compatibility reasons, Compose and therefore its completion supports several -# stack compositon files as listed here, in descending priority. -# Support for these filenames might be dropped in some future version. -__docker-compose_compose_file() { - local file - for file in docker-compose.y{,a}ml ; do - [ -e $file ] && { - echo $file - return - } - done - echo docker-compose.yml +__docker-compose_q() { + docker-compose 2>/dev/null $compose_options "$@" } -# Extracts all service names from docker-compose.yml. -___docker-compose_all_services_in_compose_file() { +# All services defined in docker-compose.yml +__docker-compose_all_services_in_compose_file() { local already_selected local -a services already_selected=$(echo $words | tr " " "|") - awk -F: '/^[a-zA-Z0-9]/{print $1}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | grep -Ev "$already_selected" + __docker-compose_q config --services \ + | grep -Ev "^(${already_selected})$" } # All services, even those without an existing container __docker-compose_services_all() { [[ $PREFIX = -* ]] && return 1 integer ret=1 - services=$(___docker-compose_all_services_in_compose_file) + services=$(__docker-compose_all_services_in_compose_file) _alternative "args:services:($services)" && ret=0 return ret } # All services that have an entry with the given key in their docker-compose.yml section -___docker-compose_services_with_key() { +__docker-compose_services_with_key() { local already_selected local -a buildable already_selected=$(echo $words | tr " " "|") # flatten sections to one line, then filter lines containing the key and return section name. - awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' "${compose_file:-$(__docker-compose_compose_file)}" 2>/dev/null | awk -F: -v key=": +$1:" '$0 ~ key {print $1}' 2>/dev/null | grep -Ev "$already_selected" + __docker-compose_q config \ + | sed -n -e '/^services:/,/^[^ ]/p' \ + | sed -n 's/^ //p' \ + | awk '/^[a-zA-Z0-9]/{printf "\n"};{printf $0;next;}' \ + | grep " \+$1:" \ + | cut -d: -f1 \ + | grep -Ev "^(${already_selected})$" } # All services that are defined by a Dockerfile reference __docker-compose_services_from_build() { [[ $PREFIX = -* ]] && return 1 integer ret=1 - buildable=$(___docker-compose_services_with_key build) + buildable=$(__docker-compose_services_with_key build) _alternative "args:buildable services:($buildable)" && ret=0 return ret @@ -74,7 +71,7 @@ __docker-compose_services_from_build() { __docker-compose_services_from_image() { [[ $PREFIX = -* ]] && return 1 integer ret=1 - pullable=$(___docker-compose_services_with_key image) + pullable=$(__docker-compose_services_with_key image) _alternative "args:pullable services:($pullable)" && ret=0 return ret @@ -96,7 +93,7 @@ __docker-compose_get_services() { shift [[ $kind =~ (stopped|all) ]] && args=($args -a) - lines=(${(f)"$(_call_program commands docker ps $args)"}) + lines=(${(f)"$(_call_program commands docker $docker_options ps $args)"}) services=(${(f)"$(_call_program commands docker-compose 2>/dev/null $compose_options ps -q)"}) # Parse header line to find columns @@ -179,13 +176,23 @@ __docker-compose_commands() { local -a lines lines=(${(f)"$(_call_program commands docker-compose 2>&1)"}) _docker_compose_subcommands=(${${${lines[$((${lines[(i)Commands:]} + 1)),${lines[(I) *]}]}## #}/ ##/:}) - _store_cache docker_compose_subcommands _docker_compose_subcommands + (( $#_docker_compose_subcommands > 0 )) && _store_cache docker_compose_subcommands _docker_compose_subcommands fi _describe -t docker-compose-commands "docker-compose command" _docker_compose_subcommands } __docker-compose_subcommand() { - local opts_help='(: -)--help[Print usage]' + local opts_help opts_force_recreate opts_no_recreate opts_no_build opts_remove_orphans opts_timeout opts_no_color opts_no_deps + + opts_help='(: -)--help[Print usage]' + opts_force_recreate="(--no-recreate)--force-recreate[Recreate containers even if their configuration and image haven't changed. Incompatible with --no-recreate.]" + opts_no_recreate="(--force-recreate)--no-recreate[If containers already exist, don't recreate them. Incompatible with --force-recreate.]" + opts_no_build="(--build)--no-build[Don't build an image, even if it's missing.]" + opts_remove_orphans="--remove-orphans[Remove containers for services not defined in the Compose file]" + opts_timeout=('(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: ") + opts_no_color='--no-color[Produce monochrome output.]' + opts_no_deps="--no-deps[Don't start linked services.]" + integer ret=1 case "$words[1]" in @@ -193,22 +200,56 @@ __docker-compose_subcommand() { _arguments \ $opts_help \ '--force-rm[Always remove intermediate containers.]' \ - '--no-cache[Do not use cache when building the image]' \ + '--no-cache[Do not use cache when building the image.]' \ '--pull[Always attempt to pull a newer version of the image.]' \ '*:services:__docker-compose_services_from_build' && ret=0 ;; + (bundle) + _arguments \ + $opts_help \ + '--push-images[Automatically push images for any services which have a `build` option specified.]' \ + '(--output -o)'{--output,-o}'[Path to write the bundle file to. Defaults to ".dab".]:file:_files' && ret=0 + ;; (config) _arguments \ $opts_help \ '(--quiet -q)'{--quiet,-q}"[Only validate the configuration, don't print anything.]" \ '--services[Print the service names, one per line.]' && ret=0 ;; + (create) + _arguments \ + $opts_help \ + $opts_force_recreate \ + $opts_no_recreate \ + $opts_no_build \ + "(--no-build)--build[Build images before creating containers.]" \ + '*:services:__docker-compose_services_all' && ret=0 + ;; + (down) + _arguments \ + $opts_help \ + "--rmi[Remove images. Type must be one of: 'all': Remove all images used by any service. 'local': Remove only images that don't have a custom tag set by the \`image\` field.]:type:(all local)" \ + '(-v --volumes)'{-v,--volumes}"[Remove named volumes declared in the \`volumes\` section of the Compose file and anonymous volumes attached to containers.]" \ + $opts_remove_orphans && ret=0 + ;; (events) _arguments \ $opts_help \ - '--json[Output events as a stream of json objects.]' \ + '--json[Output events as a stream of json objects]' \ '*:services:__docker-compose_services_all' && ret=0 ;; + (exec) + _arguments \ + $opts_help \ + '-d[Detached mode: Run command in the background.]' \ + '--privileged[Give extended privileges to the process.]' \ + '--user=[Run the command as this user.]:username:_users' \ + '-T[Disable pseudo-tty allocation. By default `docker-compose exec` allocates a TTY.]' \ + '--index=[Index of the container if there are multiple instances of a service \[default: 1\]]:index: ' \ + '(-):running services:__docker-compose_runningservices' \ + '(-):command: _command_names -e' \ + '*::arguments: _normal' && ret=0 + ;; (help) _arguments ':subcommand:__docker-compose_commands' && ret=0 ;; @@ -221,7 +262,10 @@ __docker-compose_subcommand() { (logs) _arguments \ $opts_help \ - '--no-color[Produce monochrome output.]' \ + '(-f --follow)'{-f,--follow}'[Follow log output]' \ + $opts_no_color \ + '--tail=[Number of lines to show from the end of the logs for each container.]:number of lines: ' \ + '(-t --timestamps)'{-t,--timestamps}'[Show timestamps]' \ '*:services:__docker-compose_services_all' && ret=0 ;; (pause) @@ -232,8 +276,8 @@ __docker-compose_subcommand() { (port) _arguments \ $opts_help \ - '--protocol=-[tcp or udap (defaults to tcp)]:protocol:(tcp udp)' \ - '--index=-[index of the container if there are mutiple instances of a service (defaults to 1)]:index: ' \ + '--protocol=[tcp or udp \[default: tcp\]]:protocol:(tcp udp)' \ + '--index=[index of the container if there are multiple instances of a service \[default: 1\]]:index: ' \ '1:running services:__docker-compose_runningservices' \ '2:port:_ports' && ret=0 ;; @@ -249,26 +293,33 @@ __docker-compose_subcommand() { '--ignore-pull-failures[Pull what it can and ignores images with pull failures.]' \ '*:services:__docker-compose_services_from_image' && ret=0 ;; + (push) + _arguments \ + $opts_help \ + '--ignore-push-failures[Push what it can and ignores images with push failures.]' \ + '*:services:__docker-compose_services' && ret=0 + ;; (rm) _arguments \ $opts_help \ '(-f --force)'{-f,--force}"[Don't ask to confirm removal]" \ - '-v[Remove volumes associated with containers]' \ + '-v[Remove any anonymous volumes attached to containers]' \ '*:stopped services:__docker-compose_stoppedservices' && ret=0 ;; (run) _arguments \ $opts_help \ '-d[Detached mode: Run container in the background, print new container name.]' \ - '--name[Assign a name to the container]:name: ' \ - '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ '*-e[KEY=VAL Set an environment variable (can be used multiple times)]:environment variable KEY=VAL: ' \ - '(-u --user)'{-u,--user=-}'[Run as specified username or uid]:username or uid:_users' \ - "--no-deps[Don't start linked services.]" \ + '--entrypoint[Overwrite the entrypoint of the image.]:entry point: ' \ + '--name=[Assign a name to the container]:name: ' \ + $opts_no_deps \ + '(-p --publish)'{-p,--publish=}"[Publish a container's port(s) to the host]" \ '--rm[Remove container after run. Ignored in detached mode.]' \ "--service-ports[Run command with the service's ports enabled and mapped to the host.]" \ - '(-p --publish)'{-p,--publish=-}"[Run command with manually mapped container's port(s) to the host.]" \ '-T[Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY.]' \ + '(-u --user)'{-u,--user=}'[Run as specified username or uid]:username or uid:_users' \ + '(-w --workdir)'{-w,--workdir=}'[Working directory inside the container]:workdir: ' \ '(-):services:__docker-compose_services' \ '(-):command: _command_names -e' \ '*::arguments: _normal' && ret=0 @@ -276,7 +327,7 @@ __docker-compose_subcommand() { (scale) _arguments \ $opts_help \ - '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + $opts_timeout \ '*:running services:__docker-compose_runningservices' && ret=0 ;; (start) @@ -287,7 +338,12 @@ __docker-compose_subcommand() { (stop|restart) _arguments \ $opts_help \ - '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + $opts_timeout \ + '*:running services:__docker-compose_runningservices' && ret=0 + ;; + (top) + _arguments \ + $opts_help \ '*:running services:__docker-compose_runningservices' && ret=0 ;; (unpause) @@ -298,13 +354,16 @@ __docker-compose_subcommand() { (up) _arguments \ $opts_help \ - '-d[Detached mode: Run containers in the background, print new container names.]' \ - '--no-color[Produce monochrome output.]' \ - "--no-deps[Don't start linked services.]" \ - "--force-recreate[Recreate containers even if their configuration and image haven't changed. Incompatible with --no-recreate.]" \ - "--no-recreate[If containers already exist, don't recreate them.]" \ - "--no-build[Don't build an image, even if it's missing]" \ - '(-t --timeout)'{-t,--timeout}"[Specify a shutdown timeout in seconds. (default: 10)]:seconds: " \ + '(--abort-on-container-exit)-d[Detached mode: Run containers in the background, print new container names. Incompatible with --abort-on-container-exit.]' \ + $opts_no_color \ + $opts_no_deps \ + $opts_force_recreate \ + $opts_no_recreate \ + $opts_no_build \ + "(--no-build)--build[Build images before starting containers.]" \ + "(-d)--abort-on-container-exit[Stops all containers if any container was stopped. Incompatible with -d.]" \ + '(-t --timeout)'{-t,--timeout}"[Use this timeout in seconds for container shutdown when attached or when containers are already running. (default: 10)]:seconds: " \ + $opts_remove_orphans \ '*:services:__docker-compose_services_all' && ret=0 ;; (version) @@ -332,18 +391,67 @@ _docker-compose() { integer ret=1 typeset -A opt_args + local file_description + + if [[ -n ${words[(r)-f]} || -n ${words[(r)--file]} ]] ; then + file_description="Specify an override docker-compose file (default: docker-compose.override.yml)" + else + file_description="Specify an alternate docker-compose file (default: docker-compose.yml)" + fi + _arguments -C \ '(- :)'{-h,--help}'[Get help]' \ + '*'{-f,--file}"[${file_description}]:file:_files -g '*.yml'" \ + '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ '--verbose[Show more output]' \ '(- :)'{-v,--version}'[Print version and exit]' \ - '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ - '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ + '(-H --host)'{-H,--host}'[Daemon socket to connect to]:host:' \ + '--tls[Use TLS; implied by --tlsverify]' \ + '--tlscacert=[Trust certs signed only by this CA]:ca path:' \ + '--tlscert=[Path to TLS certificate file]:client cert path:' \ + '--tlskey=[Path to TLS key file]:tls key path:' \ + '--tlsverify[Use TLS and verify the remote]' \ + "--skip-hostname-check[Don't check the daemon's hostname against the name specified in the client certificate (for example if your docker host is an IP address)]" \ '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 - local compose_file=${opt_args[-f]}${opt_args[--file]} - local compose_project=${opt_args[-p]}${opt_args[--project-name]} - local compose_options="${compose_file:+--file $compose_file} ${compose_project:+--project-name $compose_project}" + local -a relevant_compose_flags relevant_docker_flags compose_options docker_options + + relevant_compose_flags=( + "--file" "-f" + "--host" "-H" + "--project-name" "-p" + "--tls" + "--tlscacert" + "--tlscert" + "--tlskey" + "--tlsverify" + "--skip-hostname-check" + ) + + relevant_docker_flags=( + "--host" "-H" + "--tls" + "--tlscacert" + "--tlscert" + "--tlskey" + "--tlsverify" + ) + + for k in "${(@k)opt_args}"; do + if [[ -n "${relevant_docker_flags[(r)$k]}" ]]; then + docker_options+=$k + if [[ -n "$opt_args[$k]" ]]; then + docker_options+=$opt_args[$k] + fi + fi + if [[ -n "${relevant_compose_flags[(r)$k]}" ]]; then + compose_options+=$k + if [[ -n "$opt_args[$k]" ]]; then + compose_options+=$opt_args[$k] + fi + fi + done case $state in (command) diff --git a/contrib/migration/migrate-compose-file-v1-to-v2.py b/contrib/migration/migrate-compose-file-v1-to-v2.py new file mode 100755 index 00000000..c1785b0d --- /dev/null +++ b/contrib/migration/migrate-compose-file-v1-to-v2.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python +""" +Migrate a Compose file from the V1 format in Compose 1.5 to the V2 format +supported by Compose 1.6+ +""" +from __future__ import absolute_import +from __future__ import unicode_literals + +import argparse +import logging +import sys + +import ruamel.yaml + +from compose.config.types import VolumeSpec + + +log = logging.getLogger('migrate') + + +def migrate(content): + data = ruamel.yaml.load(content, ruamel.yaml.RoundTripLoader) + + service_names = data.keys() + + for name, service in data.items(): + warn_for_links(name, service) + warn_for_external_links(name, service) + rewrite_net(service, service_names) + rewrite_build(service) + rewrite_logging(service) + rewrite_volumes_from(service, service_names) + + services = {name: data.pop(name) for name in data.keys()} + + data['version'] = "2" + data['services'] = services + create_volumes_section(data) + + return data + + +def warn_for_links(name, service): + links = service.get('links') + if links: + example_service = links[0].partition(':')[0] + log.warn( + "Service {name} has links, which no longer create environment " + "variables such as {example_service_upper}_PORT. " + "If you are using those in your application code, you should " + "instead connect directly to the hostname, e.g. " + "'{example_service}'." + .format(name=name, example_service=example_service, + example_service_upper=example_service.upper())) + + +def warn_for_external_links(name, service): + external_links = service.get('external_links') + if external_links: + log.warn( + "Service {name} has external_links: {ext}, which now work " + "slightly differently. In particular, two containers must be " + "connected to at least one network in common in order to " + "communicate, even if explicitly linked together.\n\n" + "Either connect the external container to your app's default " + "network, or connect both the external container and your " + "service's containers to a pre-existing network. See " + "https://docs.docker.com/compose/networking/ " + "for more on how to do this." + .format(name=name, ext=external_links)) + + +def rewrite_net(service, service_names): + if 'net' in service: + network_mode = service.pop('net') + + # "container:" is now "service:" + if network_mode.startswith('container:'): + name = network_mode.partition(':')[2] + if name in service_names: + network_mode = 'service:{}'.format(name) + + service['network_mode'] = network_mode + + +def rewrite_build(service): + if 'dockerfile' in service: + service['build'] = { + 'context': service.pop('build'), + 'dockerfile': service.pop('dockerfile'), + } + + +def rewrite_logging(service): + if 'log_driver' in service: + service['logging'] = {'driver': service.pop('log_driver')} + if 'log_opt' in service: + service['logging']['options'] = service.pop('log_opt') + + +def rewrite_volumes_from(service, service_names): + for idx, volume_from in enumerate(service.get('volumes_from', [])): + if volume_from.split(':', 1)[0] not in service_names: + service['volumes_from'][idx] = 'container:%s' % volume_from + + +def create_volumes_section(data): + named_volumes = get_named_volumes(data['services']) + if named_volumes: + log.warn( + "Named volumes ({names}) must be explicitly declared. Creating a " + "'volumes' section with declarations.\n\n" + "For backwards-compatibility, they've been declared as external. " + "If you don't mind the volume names being prefixed with the " + "project name, you can remove the 'external' option from each one." + .format(names=', '.join(list(named_volumes)))) + + data['volumes'] = named_volumes + + +def get_named_volumes(services): + volume_specs = [ + VolumeSpec.parse(volume) + for service in services.values() + for volume in service.get('volumes', []) + ] + names = { + spec.external + for spec in volume_specs + if spec.is_named_volume + } + return {name: {'external': True} for name in names} + + +def write(stream, new_format, indent, width): + ruamel.yaml.dump( + new_format, + stream, + Dumper=ruamel.yaml.RoundTripDumper, + indent=indent, + width=width) + + +def parse_opts(args): + parser = argparse.ArgumentParser() + parser.add_argument("filename", help="Compose file filename.") + parser.add_argument("-i", "--in-place", action='store_true') + parser.add_argument( + "--indent", type=int, default=2, + help="Number of spaces used to indent the output yaml.") + parser.add_argument( + "--width", type=int, default=80, + help="Number of spaces used as the output width.") + return parser.parse_args() + + +def main(args): + logging.basicConfig(format='\033[33m%(levelname)s:\033[37m %(message)s\033[0m\n') + + opts = parse_opts(args) + + with open(opts.filename, 'r') as fh: + new_format = migrate(fh.read()) + + if opts.in_place: + output = open(opts.filename, 'w') + else: + output = sys.stdout + write(output, new_format, opts.indent, opts.width) + + +if __name__ == "__main__": + main(sys.argv) diff --git a/docker-compose.spec b/docker-compose.spec index f7f2059f..ef0e2593 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -18,23 +18,28 @@ exe = EXE(pyz, a.datas, [ ( - 'compose/config/fields_schema_v1.json', - 'compose/config/fields_schema_v1.json', + 'compose/config/config_schema_v1.json', + 'compose/config/config_schema_v1.json', 'DATA' ), ( - 'compose/config/fields_schema_v2.json', - 'compose/config/fields_schema_v2.json', + 'compose/config/config_schema_v2.0.json', + 'compose/config/config_schema_v2.0.json', 'DATA' ), ( - 'compose/config/service_schema_v1.json', - 'compose/config/service_schema_v1.json', + 'compose/config/config_schema_v2.1.json', + 'compose/config/config_schema_v2.1.json', 'DATA' ), ( - 'compose/config/service_schema_v2.json', - 'compose/config/service_schema_v2.json', + 'compose/config/config_schema_v3.0.json', + 'compose/config/config_schema_v3.0.json', + 'DATA' + ), + ( + 'compose/config/config_schema_v3.1.json', + 'compose/config/config_schema_v3.1.json', 'DATA' ), ( diff --git a/docs/Dockerfile b/docs/Dockerfile deleted file mode 100644 index 83b65633..00000000 --- a/docs/Dockerfile +++ /dev/null @@ -1,16 +0,0 @@ -FROM docs/base:latest -MAINTAINER Mary Anthony (@moxiegirl) - -RUN svn checkout https://github.com/docker/docker/trunk/docs /docs/content/engine -RUN svn checkout https://github.com/docker/swarm/trunk/docs /docs/content/swarm -RUN svn checkout https://github.com/docker/machine/trunk/docs /docs/content/machine -RUN svn checkout https://github.com/docker/distribution/trunk/docs /docs/content/registry -RUN svn checkout https://github.com/kitematic/kitematic/trunk/docs /docs/content/kitematic -RUN svn checkout https://github.com/docker/tutorials/trunk/docs /docs/content/tutorials -RUN svn checkout https://github.com/docker/opensource/trunk/docs /docs/content - -ENV PROJECT=compose -# To get the git info for this repo -COPY . /src - -COPY . /docs/content/$PROJECT/ diff --git a/docs/Makefile b/docs/Makefile deleted file mode 100644 index b9ef0548..00000000 --- a/docs/Makefile +++ /dev/null @@ -1,55 +0,0 @@ -.PHONY: all binary build cross default docs docs-build docs-shell shell test test-unit test-integration test-integration-cli test-docker-py validate - -# env vars passed through directly to Docker's build scripts -# to allow things like `make DOCKER_CLIENTONLY=1 binary` easily -# `docs/sources/contributing/devenvironment.md ` and `project/PACKAGERS.md` have some limited documentation of some of these -DOCKER_ENVS := \ - -e BUILDFLAGS \ - -e DOCKER_CLIENTONLY \ - -e DOCKER_EXECDRIVER \ - -e DOCKER_GRAPHDRIVER \ - -e TESTDIRS \ - -e TESTFLAGS \ - -e TIMEOUT -# note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds - -# to allow `make DOCSDIR=1 docs-shell` (to create a bind mount in docs) -DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR):/docs/content/compose) - -# to allow `make DOCSPORT=9000 docs` -DOCSPORT := 8000 - -# Get the IP ADDRESS -DOCKER_IP=$(shell python -c "import urlparse ; print urlparse.urlparse('$(DOCKER_HOST)').hostname or ''") -HUGO_BASE_URL=$(shell test -z "$(DOCKER_IP)" && echo localhost || echo "$(DOCKER_IP)") -HUGO_BIND_IP=0.0.0.0 - -GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) -DOCKER_IMAGE := docker$(if $(GIT_BRANCH),:$(GIT_BRANCH)) -DOCKER_DOCS_IMAGE := docs-base$(if $(GIT_BRANCH),:$(GIT_BRANCH)) - - -DOCKER_RUN_DOCS := docker run --rm -it $(DOCS_MOUNT) -e AWS_S3_BUCKET -e NOCACHE - -# for some docs workarounds (see below in "docs-build" target) -GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) - -default: docs - -docs: docs-build - $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) --watch - -docs-draft: docs-build - $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) - - -docs-shell: docs-build - $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 "$(DOCKER_DOCS_IMAGE)" bash - - -docs-build: -# ( git remote | grep -v upstream ) || git diff --name-status upstream/release..upstream/docs ./ > ./changed-files -# echo "$(GIT_BRANCH)" > GIT_BRANCH -# echo "$(AWS_S3_BUCKET)" > AWS_S3_BUCKET -# echo "$(GITCOMMIT)" > GITCOMMIT - docker build -t "$(DOCKER_DOCS_IMAGE)" . diff --git a/docs/README.md b/docs/README.md index d8ab7c3e..50c91d20 100644 --- a/docs/README.md +++ b/docs/README.md @@ -1,86 +1,16 @@ - +# The docs have been moved! -# Contributing to the Docker Compose documentation +The documentation for Compose has been merged into +[the general documentation repo](https://github.com/docker/docker.github.io). -The documentation in this directory is part of the [https://docs.docker.com](https://docs.docker.com) website. Docker uses [the Hugo static generator](http://gohugo.io/overview/introduction/) to convert project Markdown files to a static HTML site. +The docs for Compose are now here: +https://github.com/docker/docker.github.io/tree/master/compose -You don't need to be a Hugo expert to contribute to the compose documentation. If you are familiar with Markdown, you can modify the content in the `docs` files. +Please submit pull requests for unpublished features on the `vnext-compose` branch (https://github.com/docker/docker.github.io/tree/vnext-compose). -If you want to add a new file or change the location of the document in the menu, you do need to know a little more. +If you submit a PR to this codebase that has a docs impact, create a second docs PR on `docker.github.io`. Use the docs PR template provided (coming soon - watch this space). -## Documentation contributing workflow +PRs for typos, additional information, etc. for already-published features should be labeled as `okay-to-publish` (we are still settling on a naming convention, will provide a label soon). You can submit these PRs either to `vnext-compose` or directly to `master` on `docker.github.io` -1. Edit a Markdown file in the tree. - -2. Save your changes. - -3. Make sure you are in the `docs` subdirectory. - -4. Build the documentation. - - $ make docs - ---> ffcf3f6c4e97 - Removing intermediate container a676414185e8 - Successfully built ffcf3f6c4e97 - docker run --rm -it -e AWS_S3_BUCKET -e NOCACHE -p 8000:8000 -e DOCKERHOST "docs-base:test-tooling" hugo server --port=8000 --baseUrl=192.168.59.103 --bind=0.0.0.0 - ERROR: 2015/06/13 MenuEntry's .Url is deprecated and will be removed in Hugo 0.15. Use .URL instead. - 0 of 4 drafts rendered - 0 future content - 12 pages created - 0 paginator pages created - 0 tags created - 0 categories created - in 55 ms - Serving pages from /docs/public - Web Server is available at http://0.0.0.0:8000/ - Press Ctrl+C to stop - -5. Open the available server in your browser. - - The documentation server has the complete menu but only the Docker Compose - documentation resolves. You can't access the other project docs from this - localized build. - -## Tips on Hugo metadata and menu positioning - -The top of each Docker Compose documentation file contains TOML metadata. The metadata is commented out to prevent it from appearing in GitHub. - - - -The metadata alone has this structure: - - +++ - title = "Extending services in Compose" - description = "How to use Docker Compose's extends keyword to share configuration between files and projects" - keywords = ["fig, composition, compose, docker, orchestration, documentation, docs"] - [menu.main] - parent="smn_workw_compose" - weight=2 - +++ - -The `[menu.main]` section refers to navigation defined [in the main Docker menu](https://github.com/docker/docs-base/blob/hugo/config.toml). This metadata says *add a menu item called* Extending services in Compose *to the menu with the* `smn_workdw_compose` *identifier*. If you locate the menu in the configuration, you'll find *Create multi-container applications* is the menu title. - -You can move an article in the tree by specifying a new parent. You can shift the location of the item by changing its weight. Higher numbers are heavier and shift the item to the bottom of menu. Low or no numbers shift it up. - - -## Other key documentation repositories - -The `docker/docs-base` repository contains [the Hugo theme and menu configuration](https://github.com/docker/docs-base). If you open the `Dockerfile` you'll see the `make docs` relies on this as a base image for building the Compose documentation. - -The `docker/docs.docker.com` repository contains [build system for building the Docker documentation site](https://github.com/docker/docs.docker.com). Fork this repository to build the entire documentation site. +As always, the docs remain open-source and we appreciate your feedback and +pull requests! diff --git a/docs/completion.md b/docs/completion.md deleted file mode 100644 index cac0d1a6..00000000 --- a/docs/completion.md +++ /dev/null @@ -1,68 +0,0 @@ - - -# Command-line Completion - -Compose comes with [command completion](http://en.wikipedia.org/wiki/Command-line_completion) -for the bash and zsh shell. - -## Installing Command Completion - -### Bash - -Make sure bash completion is installed. If you use a current Linux in a non-minimal installation, bash completion should be available. -On a Mac, install with `brew install bash-completion` - -Place the completion script in `/etc/bash_completion.d/` (`/usr/local/etc/bash_completion.d/` on a Mac), using e.g. - - curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose version --short)/contrib/completion/bash/docker-compose > /etc/bash_completion.d/docker-compose - -Completion will be available upon next login. - -### Zsh - -Place the completion script in your `/path/to/zsh/completion`, using e.g. `~/.zsh/completion/` - - mkdir -p ~/.zsh/completion - curl -L https://raw.githubusercontent.com/docker/compose/$(docker-compose version --short)/contrib/completion/zsh/_docker-compose > ~/.zsh/completion/_docker-compose - -Include the directory in your `$fpath`, e.g. by adding in `~/.zshrc` - - fpath=(~/.zsh/completion $fpath) - -Make sure `compinit` is loaded or do it by adding in `~/.zshrc` - - autoload -Uz compinit && compinit -i - -Then reload your shell - - exec $SHELL -l - -## Available completions - -Depending on what you typed on the command line so far, it will complete - - - available docker-compose commands - - options that are available for a particular command - - service names that make sense in a given context (e.g. services with running or stopped instances or services based on images vs. services based on Dockerfiles). For `docker-compose scale`, completed service names will automatically have "=" appended. - - arguments for selected options, e.g. `docker-compose kill -s` will complete some signals like SIGHUP and SIGUSR1. - -Enjoy working with Compose faster and with less typos! - -## Compose documentation - -- [User guide](index.md) -- [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/compose-file.md b/docs/compose-file.md deleted file mode 100644 index ecd135f1..00000000 --- a/docs/compose-file.md +++ /dev/null @@ -1,623 +0,0 @@ - - - -# Compose file reference - -The compose file is a [YAML](http://yaml.org/) file where all the top level -keys are the name of a service, and the values are the service definition. -The default path for a compose file is `./docker-compose.yml`. - -Each service defined in `docker-compose.yml` must specify exactly one of -`image` or `build`. Other keys are optional, and are analogous to their -`docker run` command-line counterparts. - -As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, -`EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to -specify them again in `docker-compose.yml`. - -## Versioning - -It is possible to use different versions of the `compose.yml` format. -Below are the formats currently supported by compose. - - -### Version 1 - -Compose files that do not declare a version are considered "version 1". In -those files, all the [services](#service-configuration-reference) are declared -at the root of the document. - -Version 1 files do not support the declaration of -named [volumes](#volume-configuration-reference) or -[build arguments](#args). - -Example: - - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - - logvolume01:/var/log - links: - - redis - redis: - image: redis - - -### Version 2 - -Compose files using the version 2 syntax must indicate the version number at -the root of the document. All [services](#service-configuration-reference) -must be declared under the `services` key. -Named [volumes](#volume-configuration-reference) must be declared under the -`volumes` key. - -Example: - - version: 2 - services: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - - logvolume01:/var/log - links: - - redis - redis: - image: redis - volumes: - logvolume01: - driver: default - - -## Service configuration reference - -This section contains a list of all configuration options supported by a service -definition. - -### build - -Configuration options that are applied at build time. - -In version 1 this must be given as a string representing the context. - - build: . - -In version 2 this can alternatively be given as an object with extra options. - - version: 2 - services: - web: - build: . - - version: 2 - services: - web: - build: - context: . - dockerfile: Dockerfile-alternate - args: - buildno: 1 - -#### context - -Either a path to a directory containing a Dockerfile, or a url to a git repository. - -When the value supplied is a relative path, it is interpreted as relative to the -location of the Compose file. This directory is also the build context that is -sent to the Docker daemon. - -Compose will build and tag it with a generated name, and use that image thereafter. - - build: /path/to/build/dir - - build: - context: /path/to/build/dir - -Using `context` together with `image` is not allowed. Attempting to do so results in -an error. - -#### dockerfile - -Alternate Dockerfile. - -Compose will use an alternate file to build with. A build path must also be -specified using the `build` key. - - build: - context: /path/to/build/dir - dockerfile: Dockerfile-alternate - -Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. - -#### args - -Add build arguments. You can use either an array or a dictionary. Any -boolean values; true, false, yes, no, need to be enclosed in quotes to ensure -they are not converted to True or False by the YML parser. - -Build arguments with only a key are resolved to their environment value on the -machine Compose is running on. - -> **Note:** Introduced in version 2 of the compose file format. - - build: - args: - buildno: 1 - user: someuser - - build: - args: - - buildno=1 - - user=someuser - -### cap_add, cap_drop - -Add or drop container capabilities. -See `man 7 capabilities` for a full list. - - cap_add: - - ALL - - cap_drop: - - NET_ADMIN - - SYS_ADMIN - -### command - -Override the default command. - - command: bundle exec thin -p 3000 - -The command can also be a list, in a manner similar to [dockerfile](https://docs.docker.com/engine/reference/builder/#cmd): - - command: [bundle, exec, thin, -p, 3000] - -### cgroup_parent - -Specify an optional parent cgroup for the container. - - cgroup_parent: m-executor-abcd - -### container_name - -Specify a custom container name, rather than a generated default name. - - container_name: my-web-container - -Because Docker container names must be unique, you cannot scale a service -beyond 1 container if you have specified a custom name. Attempting to do so -results in an error. - -### devices - -List of device mappings. Uses the same format as the `--device` docker -client create option. - - devices: - - "/dev/ttyUSB0:/dev/ttyUSB0" - -### dns - -Custom DNS servers. Can be a single value or a list. - - dns: 8.8.8.8 - dns: - - 8.8.8.8 - - 9.9.9.9 - -### dns_search - -Custom DNS search domains. Can be a single value or a list. - - dns_search: example.com - dns_search: - - dc1.example.com - - dc2.example.com - -### entrypoint - -Override the default entrypoint. - - entrypoint: /code/entrypoint.sh - -The entrypoint can also be a list, in a manner similar to [dockerfile](https://docs.docker.com/engine/reference/builder/#entrypoint): - - entrypoint: - - php - - -d - - zend_extension=/usr/local/lib/php/extensions/no-debug-non-zts-20100525/xdebug.so - - -d - - memory_limit=-1 - - vendor/bin/phpunit - - -### env_file - -Add environment variables from a file. Can be a single value or a list. - -If you have specified a Compose file with `docker-compose -f FILE`, paths in -`env_file` are relative to the directory that file is in. - -Environment variables specified in `environment` override these values. - - env_file: .env - - env_file: - - ./common.env - - ./apps/web.env - - /opt/secrets.env - -Compose expects each line in an env file to be in `VAR=VAL` format. Lines -beginning with `#` (i.e. comments) are ignored, as are blank lines. - - # Set Rails/Rack environment - RACK_ENV=development - -### environment - -Add environment variables. You can use either an array or a dictionary. Any -boolean values; true, false, yes no, need to be enclosed in quotes to ensure -they are not converted to True or False by the YML parser. - -Environment variables with only a key are resolved to their values on the -machine Compose is running on, which can be helpful for secret or host-specific values. - - environment: - RACK_ENV: development - SHOW: 'true' - SESSION_SECRET: - - environment: - - RACK_ENV=development - - SHOW=true - - SESSION_SECRET - -### expose - -Expose ports without publishing them to the host machine - they'll only be -accessible to linked services. Only the internal port can be specified. - - expose: - - "3000" - - "8000" - -### extends - -Extend another service, in the current file or another, optionally overriding -configuration. - -You can use `extends` on any service together with other configuration keys. -The `extends` value must be a dictionary defined with a required `service` -and an optional `file` key. - - extends: - file: common.yml - service: webapp - -The `service` the name of the service being extended, for example -`web` or `database`. The `file` is the location of a Compose configuration -file defining that service. - -If you omit the `file` Compose looks for the service configuration in the -current file. The `file` value can be an absolute or relative path. If you -specify a relative path, Compose treats it as relative to the location of the -current file. - -You can extend a service that itself extends another. You can extend -indefinitely. Compose does not support circular references and `docker-compose` -returns an error if it encounters one. - -For more on `extends`, see the -[the extends documentation](extends.md#extending-services). - -### external_links - -Link to containers started outside this `docker-compose.yml` or even outside -of Compose, especially for containers that provide shared or common services. -`external_links` follow semantics similar to `links` when specifying both the -container name and the link alias (`CONTAINER:ALIAS`). - - external_links: - - redis_1 - - project_db_1:mysql - - project_db_1:postgresql - -### extra_hosts - -Add hostname mappings. Use the same values as the docker client `--add-host` parameter. - - extra_hosts: - - "somehost:162.242.195.82" - - "otherhost:50.31.209.229" - -An entry with the ip address and hostname will be created in `/etc/hosts` inside containers for this service, e.g: - - 162.242.195.82 somehost - 50.31.209.229 otherhost - -### image - -Tag or partial image ID. Can be local or remote - Compose will attempt to -pull if it doesn't exist locally. - - image: ubuntu - image: orchardup/postgresql - image: a4bc65fd - -### labels - -Add metadata to containers using [Docker labels](https://docs.docker.com/engine/userguide/labels-custom-metadata/). You can use either an array or a dictionary. - -It's recommended that you use reverse-DNS notation to prevent your labels from conflicting with those used by other software. - - labels: - com.example.description: "Accounting webapp" - com.example.department: "Finance" - com.example.label-with-empty-value: "" - - labels: - - "com.example.description=Accounting webapp" - - "com.example.department=Finance" - - "com.example.label-with-empty-value" - -### links - -Link to containers in another service. Either specify both the service name and -the link alias (`SERVICE:ALIAS`), or just the service name (which will also be -used for the alias). - - links: - - db - - db:database - - redis - -An entry with the alias' name will be created in `/etc/hosts` inside containers -for this service, e.g: - - 172.17.2.186 db - 172.17.2.186 database - 172.17.2.187 redis - -Environment variables will also be created - see the [environment variable -reference](env.md) for details. - -### logging - -Logging configuration for the service. This configuration replaces the previous -`log_driver` and `log_opt` keys. - - logging: - driver: log_driver - options: - syslog-address: "tcp://192.168.0.42:123" - -The `driver` name specifies a logging driver for the service's -containers, as with the ``--log-driver`` option for docker run -([documented here](https://docs.docker.com/engine/reference/logging/overview/)). - -The default value is json-file. - - driver: "json-file" - driver: "syslog" - driver: "none" - -> **Note:** Only the `json-file` driver makes the logs available directly from -> `docker-compose up` and `docker-compose logs`. Using any other driver will not -> print any logs. - -Specify logging options for the logging driver with the ``options`` key, as with the ``--log-opt`` option for `docker run`. - - -Logging options are key value pairs. An example of `syslog` options: - - driver: "syslog" - options: - syslog-address: "tcp://192.168.0.42:123" - -### net - -Networking mode. Use the same values as the docker client `--net` parameter. - - net: "bridge" - net: "none" - net: "container:[name or id]" - net: "host" - -### pid - - pid: "host" - -Sets the PID mode to the host PID mode. This turns on sharing between -container and the host operating system the PID address space. Containers -launched with this flag will be able to access and manipulate other -containers in the bare-metal machine's namespace and vise-versa. - -### ports - -Expose ports. Either specify both ports (`HOST:CONTAINER`), or just the container -port (a random host port will be chosen). - -> **Note:** When mapping ports in the `HOST:CONTAINER` format, you may experience -> erroneous results when using a container port lower than 60, because YAML will -> parse numbers in the format `xx:yy` as sexagesimal (base 60). For this reason, -> we recommend always explicitly specifying your port mappings as strings. - - ports: - - "3000" - - "3000-3005" - - "8000:8000" - - "9090-9091:8080-8081" - - "49100:22" - - "127.0.0.1:8001:8001" - - "127.0.0.1:5000-5010:5000-5010" - -### security_opt - -Override the default labeling scheme for each container. - - security_opt: - - label:user:USER - - label:role:ROLE - -### ulimits - -Override the default ulimits for a container. You can either specify a single -limit as an integer or soft/hard limits as a mapping. - - - ulimits: - nproc: 65535 - nofile: - soft: 20000 - hard: 40000 - -### volumes, volume\_driver - -Mount paths as volumes, optionally specifying a path on the host machine -(`HOST:CONTAINER`), or an access mode (`HOST:CONTAINER:ro`). - - volumes: - - /var/lib/mysql - - ./cache:/tmp/cache - - ~/configs:/etc/configs/:ro - -You can mount a relative path on the host, which will expand relative to -the directory of the Compose configuration file being used. Relative paths -should always begin with `.` or `..`. - -If you use a volume name (instead of a volume path), you may also specify -a `volume_driver`. - - volume_driver: mydriver - - -> Note: No path expansion will be done if you have also specified a -> `volume_driver`. - -See [Docker Volumes](https://docs.docker.com/engine/userguide/dockervolumes/) and -[Volume Plugins](https://docs.docker.com/engine/extend/plugins_volume/) for more -information. - -### volumes_from - -Mount all of the volumes from another service or container, optionally -specifying read-only access(``ro``) or read-write(``rw``). - - volumes_from: - - service_name - - container_name - - service_name:rw - -### cpu\_shares, cpu\_quota, cpuset, domainname, hostname, ipc, mac\_address, mem\_limit, memswap\_limit, privileged, read\_only, restart, stdin\_open, tty, user, working\_dir - -Each of these is a single value, analogous to its -[docker run](https://docs.docker.com/engine/reference/run/) counterpart. - - cpu_shares: 73 - cpu_quota: 50000 - cpuset: 0,1 - - user: postgresql - working_dir: /code - - domainname: foo.com - hostname: foo - ipc: host - mac_address: 02:42:ac:11:65:43 - - mem_limit: 1000000000 - memswap_limit: 2000000000 - privileged: true - - restart: always - - read_only: true - stdin_open: true - tty: true - - -## Volume configuration reference - -While it is possible to declare volumes on the fly as part of the service -declaration, this section allows you to create named volumes that can be -reused across multiple services (without relying on `volumes_from`), and are -easily retrieved and inspected using the docker command line or API. -See the [docker volume](http://docs.docker.com/reference/commandline/volume/) -subcommand documentation for more information. - -### driver - -Specify which volume driver should be used for this volume. Defaults to -`local`. An exception will be raised if the driver is not available. - - driver: foobar - -### driver_opts - -Specify a list of options as key-value pairs to pass to the driver for this -volume. Those options are driver dependent - consult the driver's -documentation for more information. Optional. - - driver_opts: - foo: "bar" - baz: 1 - - -## Variable substitution - -Your configuration options can contain environment variables. Compose uses the -variable values from the shell environment in which `docker-compose` is run. For -example, suppose the shell contains `POSTGRES_VERSION=9.3` and you supply this -configuration: - - db: - image: "postgres:${POSTGRES_VERSION}" - -When you run `docker-compose up` with this configuration, Compose looks for the -`POSTGRES_VERSION` environment variable in the shell and substitutes its value -in. For this example, Compose resolves the `image` to `postgres:9.3` before -running the configuration. - -If an environment variable is not set, Compose substitutes with an empty -string. In the example above, if `POSTGRES_VERSION` is not set, the value for -the `image` option is `postgres:`. - -Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style -features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not -supported. - -You can use a `$$` (double-dollar sign) when your configuration needs a literal -dollar sign. This also prevents Compose from interpolating a value, so a `$$` -allows you to refer to environment variables that you don't want processed by -Compose. - - web: - build: . - command: "$$VAR_NOT_INTERPOLATED_BY_COMPOSE" - -If you forget and use a single dollar sign (`$`), Compose interprets the value as an environment variable and will warn you: - - The VAR_NOT_INTERPOLATED_BY_COMPOSE is not set. Substituting an empty string. - -## Compose documentation - -- [User guide](index.md) -- [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) diff --git a/docs/django.md b/docs/django.md deleted file mode 100644 index 2d4fdaf9..00000000 --- a/docs/django.md +++ /dev/null @@ -1,180 +0,0 @@ - - - -# Quickstart Guide: Compose and Django - -This quick-start guide demonstrates how to use Compose to set up and run a -simple Django/PostgreSQL app. Before starting, you'll need to have -[Compose installed](install.md). - -## Define the project components - -For this project, you need to create a Dockerfile, a Python dependencies file, -and a `docker-compose.yml` file. - -1. Create an empty project directory. - - You can name the directory something easy for you to remember. This directory is the context for your application image. The directory should only contain resources to build that image. - -2. Create a new file called `Dockerfile` in your project directory. - - The Dockerfile defines an application's image content via one or more build - commands that configure that image. Once built, you can run the image in a - container. For more information on `Dockerfiles`, see the [Docker user - guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) - and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). - -3. Add the following content to the `Dockerfile`. - - FROM python:2.7 - ENV PYTHONUNBUFFERED 1 - RUN mkdir /code - WORKDIR /code - ADD requirements.txt /code/ - RUN pip install -r requirements.txt - ADD . /code/ - - This `Dockerfile` starts with a Python 2.7 base image. The base image is - modified by adding a new `code` directory. The base image is further modified - by installing the Python requirements defined in the `requirements.txt` file. - -4. Save and close the `Dockerfile`. - -5. Create a `requirements.txt` in your project directory. - - This file is used by the `RUN pip install -r requirements.txt` command in your `Dockerfile`. - -6. Add the required software in the file. - - Django - psycopg2 - -7. Save and close the `requirements.txt` file. - -8. Create a file called `docker-compose.yml` in your project directory. - - The `docker-compose.yml` file describes the services that make your app. In - this example those services are a web server and database. The compose file - also describes which Docker images these services use, how they link - together, any volumes they might need mounted inside the containers. - Finally, the `docker-compose.yml` file describes which ports these services - expose. See the [`docker-compose.yml` reference](compose-file.md) for more - information on how this file works. - -9. Add the following configuration to the file. - - db: - image: postgres - web: - build: . - command: python manage.py runserver 0.0.0.0:8000 - volumes: - - .:/code - ports: - - "8000:8000" - links: - - db - - This file defines two services: The `db` service and the `web` service. - -10. Save and close the `docker-compose.yml` file. - -## Create a Django project - -In this step, you create a Django started project by building the image from the build context defined in the previous procedure. - -1. Change to the root of your project directory. - -2. Create the Django project using the `docker-compose` command. - - $ docker-compose run web django-admin.py startproject composeexample . - - This instructs Compose to run `django-admin.py startproject composeeexample` - in a container, using the `web` service's image and configuration. Because - the `web` image doesn't exist yet, Compose builds it from the current - directory, as specified by the `build: .` line in `docker-compose.yml`. - - Once the `web` service image is built, Compose runs it and executes the - `django-admin.py startproject` command in the container. This command - instructs Django to create a set of files and directories representing a - Django project. - -3. After the `docker-compose` command completes, list the contents of your project. - - $ ls -l - drwxr-xr-x 2 root root composeexample - -rw-rw-r-- 1 user user docker-compose.yml - -rw-rw-r-- 1 user user Dockerfile - -rwxr-xr-x 1 root root manage.py - -rw-rw-r-- 1 user user requirements.txt - - The files `django-admin` created are owned by root. This happens because - the container runs as the `root` user. - -4. Change the ownership of the new files. - - sudo chown -R $USER:$USER . - - -## Connect the database - -In this section, you set up the database connection for Django. - -1. In your project dirctory, edit the `composeexample/settings.py` file. - -2. Replace the `DATABASES = ...` with the following: - - DATABASES = { - 'default': { - 'ENGINE': 'django.db.backends.postgresql_psycopg2', - 'NAME': 'postgres', - 'USER': 'postgres', - 'HOST': 'db', - 'PORT': 5432, - } - } - - These settings are determined by the - [postgres](https://hub.docker.com/_/postgres/) Docker image - specified in `docker-compose.yml`. - -3. Save and close the file. - -4. Run the `docker-compose up` command. - - $ docker-compose up - Starting composepractice_db_1... - Starting composepractice_web_1... - Attaching to composepractice_db_1, composepractice_web_1 - ... - db_1 | PostgreSQL init process complete; ready for start up. - ... - db_1 | LOG: database system is ready to accept connections - db_1 | LOG: autovacuum launcher started - .. - web_1 | Django version 1.8.4, using settings 'composeexample.settings' - web_1 | Starting development server at http://0.0.0.0:8000/ - web_1 | Quit the server with CONTROL-C. - - At this point, your Django app should be running at port `8000` on your - Docker host. If you are using a Docker Machine VM, you can use the - `docker-machine ip MACHINE_NAME` to get the IP address. - -## More Compose documentation - -- [User guide](index.md) -- [Installing Compose](install.md) -- [Getting Started](gettingstarted.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/env.md b/docs/env.md deleted file mode 100644 index c0e03a4e..00000000 --- a/docs/env.md +++ /dev/null @@ -1,45 +0,0 @@ - - -# Compose environment variables reference - -**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. - -Compose uses [Docker links] to expose services' containers to one another. Each linked container injects a set of environment variables, each of which begins with the uppercase name of the container. - -To see what environment variables are available to a service, run `docker-compose run SERVICE env`. - -name\_PORT
-Full URL, e.g. `DB_PORT=tcp://172.17.0.5:5432` - -name\_PORT\_num\_protocol
-Full URL, e.g. `DB_PORT_5432_TCP=tcp://172.17.0.5:5432` - -name\_PORT\_num\_protocol\_ADDR
-Container's IP address, e.g. `DB_PORT_5432_TCP_ADDR=172.17.0.5` - -name\_PORT\_num\_protocol\_PORT
-Exposed port number, e.g. `DB_PORT_5432_TCP_PORT=5432` - -name\_PORT\_num\_protocol\_PROTO
-Protocol (tcp or udp), e.g. `DB_PORT_5432_TCP_PROTO=tcp` - -name\_NAME
-Fully qualified container name, e.g. `DB_1_NAME=/myapp_web_1/myapp_db_1` - -[Docker links]: https://docs.docker.com/engine/userguide/networking/default_network/dockerlinks/ - -## Related Information - -- [User guide](index.md) -- [Installing Compose](install.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/extends.md b/docs/extends.md deleted file mode 100644 index 011a7350..00000000 --- a/docs/extends.md +++ /dev/null @@ -1,375 +0,0 @@ - - - -# Extending services and Compose files - -Compose supports two methods of sharing common configuration: - -1. Extending an entire Compose file by - [using multiple Compose files](#multiple-compose-files) -2. Extending individual services with [the `extends` field](#extending-services) - - -## Multiple Compose files - -Using multiple Compose files enables you to customize a Compose application -for different environments or different workflows. - -### Understanding multiple Compose files - -By default, Compose reads two files, a `docker-compose.yml` and an optional -`docker-compose.override.yml` file. By convention, the `docker-compose.yml` -contains your base configuration. The override file, as its name implies, can -contain configuration overrides for existing services or entirely new -services. - -If a service is defined in both files, Compose merges the configurations using -the same rules as the `extends` field (see [Adding and overriding -configuration](#adding-and-overriding-configuration)), with one exception. If a -service contains `links` or `volumes_from` those fields are copied over and -replace any values in the original service, in the same way single-valued fields -are copied. - -To use multiple override files, or an override file with a different name, you -can use the `-f` option to specify the list of files. Compose merges files in -the order they're specified on the command line. See the [`docker-compose` -command reference](./reference/docker-compose.md) for more information about -using `-f`. - -When you use multiple configuration files, you must make sure all paths in the -files are relative to the base Compose file (the first Compose file specified -with `-f`). This is required because override files need not be valid -Compose files. Override files can contain small fragments of configuration. -Tracking which fragment of a service is relative to which path is difficult and -confusing, so to keep paths easier to understand, all paths must be defined -relative to the base file. - -### Example use case - -In this section are two common use cases for multiple compose files: changing a -Compose app for different environments, and running administrative tasks -against a Compose app. - -#### Different environments - -A common use case for multiple files is changing a development Compose app -for a production-like environment (which may be production, staging or CI). -To support these differences, you can split your Compose configuration into -a few different files: - -Start with a base file that defines the canonical configuration for the -services. - -**docker-compose.yml** - - web: - image: example/my_web_app:latest - links: - - db - - cache - - db: - image: postgres:latest - - cache: - image: redis:latest - -In this example the development configuration exposes some ports to the -host, mounts our code as a volume, and builds the web image. - -**docker-compose.override.yml** - - - web: - build: . - volumes: - - '.:/code' - ports: - - 8883:80 - environment: - DEBUG: 'true' - - db: - command: '-d' - ports: - - 5432:5432 - - cache: - ports: - - 6379:6379 - -When you run `docker-compose up` it reads the overrides automatically. - -Now, it would be nice to use this Compose app in a production environment. So, -create another override file (which might be stored in a different git -repo or managed by a different team). - -**docker-compose.prod.yml** - - web: - ports: - - 80:80 - environment: - PRODUCTION: 'true' - - cache: - environment: - TTL: '500' - -To deploy with this production Compose file you can run - - docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d - -This deploys all three services using the configuration in -`docker-compose.yml` and `docker-compose.prod.yml` (but not the -dev configuration in `docker-compose.override.yml`). - - -See [production](production.md) for more information about Compose in -production. - -#### Administrative tasks - -Another common use case is running adhoc or administrative tasks against one -or more services in a Compose app. This example demonstrates running a -database backup. - -Start with a **docker-compose.yml**. - - web: - image: example/my_web_app:latest - links: - - db - - db: - image: postgres:latest - -In a **docker-compose.admin.yml** add a new service to run the database -export or backup. - - dbadmin: - build: database_admin/ - links: - - db - -To start a normal environment run `docker-compose up -d`. To run a database -backup, include the `docker-compose.admin.yml` as well. - - docker-compose -f docker-compose.yml -f docker-compose.admin.yml \ - run dbadmin db-backup - - -## Extending services - -Docker Compose's `extends` keyword enables sharing of common configurations -among different files, or even different projects entirely. Extending services -is useful if you have several services that reuse a common set of configuration -options. Using `extends` you can define a common set of service options in one -place and refer to it from anywhere. - -> **Note:** `links` and `volumes_from` are never shared between services using -> `extends`. See -> [Adding and overriding configuration](#adding-and-overriding-configuration) - > for more information. - -### Understand the extends configuration - -When defining any service in `docker-compose.yml`, you can declare that you are -extending another service like this: - - web: - extends: - file: common-services.yml - service: webapp - -This instructs Compose to re-use the configuration for the `webapp` service -defined in the `common-services.yml` file. Suppose that `common-services.yml` -looks like this: - - webapp: - build: . - ports: - - "8000:8000" - volumes: - - "/data" - -In this case, you'll get exactly the same result as if you wrote -`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration -values defined directly under `web`. - -You can go further and define (or re-define) configuration locally in -`docker-compose.yml`: - - web: - extends: - file: common-services.yml - service: webapp - environment: - - DEBUG=1 - cpu_shares: 5 - - important_web: - extends: web - cpu_shares: 10 - -You can also write other services and link your `web` service to them: - - web: - extends: - file: common-services.yml - service: webapp - environment: - - DEBUG=1 - cpu_shares: 5 - links: - - db - db: - image: postgres - -### Example use case - -Extending an individual service is useful when you have multiple services that -have a common configuration. The example below is a Compose app with -two services: a web application and a queue worker. Both services use the same -codebase and share many configuration options. - -In a **common.yml** we define the common configuration: - - app: - build: . - environment: - CONFIG_FILE_PATH: /code/config - API_KEY: xxxyyy - cpu_shares: 5 - -In a **docker-compose.yml** we define the concrete services which use the -common configuration: - - webapp: - extends: - file: common.yml - service: app - command: /code/run_web_app - ports: - - 8080:8080 - links: - - queue - - db - - queue_worker: - extends: - file: common.yml - service: app - command: /code/run_worker - links: - - queue - -## Adding and overriding configuration - -Compose copies configurations from the original service over to the local one, -**except** for `links` and `volumes_from`. These exceptions exist to avoid -implicit dependencies—you always define `links` and `volumes_from` -locally. This ensures dependencies between services are clearly visible when -reading the current file. Defining these locally also ensures changes to the -referenced file don't result in breakage. - -If a configuration option is defined in both the original service the local -service, the local value *replaces* or *extends* the original value. - -For single-value options like `image`, `command` or `mem_limit`, the new value -replaces the old value. - - # original service - command: python app.py - - # local service - command: python otherapp.py - - # result - command: python otherapp.py - -In the case of `build` and `image`, using one in the local service causes -Compose to discard the other, if it was defined in the original service. - -Example of image replacing build: - - # original service - build: . - - # local service - image: redis - - # result - image: redis - - -Example of build replacing image: - - # original service - image: redis - - # local service - build: . - - # result - build: . - -For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and -`dns_search`, Compose concatenates both sets of values: - - # original service - expose: - - "3000" - - # local service - expose: - - "4000" - - "5000" - - # result - expose: - - "3000" - - "4000" - - "5000" - -In the case of `environment`, `labels`, `volumes` and `devices`, Compose -"merges" entries together with locally-defined values taking precedence: - - # original service - environment: - - FOO=original - - BAR=original - - # local service - environment: - - BAR=local - - BAZ=local - - # result - environment: - - FOO=original - - BAR=local - - BAZ=local - - - - -## Compose documentation - -- [User guide](index.md) -- [Installing Compose](install.md) -- [Getting Started](gettingstarted.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/faq.md b/docs/faq.md deleted file mode 100644 index b36eb5ac..00000000 --- a/docs/faq.md +++ /dev/null @@ -1,139 +0,0 @@ - - -# Frequently asked questions - -If you don’t see your question here, feel free to drop by `#docker-compose` on -freenode IRC and ask the community. - -## Why do my services take 10 seconds to stop? - -Compose stop attempts to stop a container by sending a `SIGTERM`. It then waits -for a [default timeout of 10 seconds](./reference/stop.md). After the timeout, -a `SIGKILL` is sent to the container to forcefully kill it. If you -are waiting for this timeout, it means that your containers aren't shutting down -when they receive the `SIGTERM` signal. - -There has already been a lot written about this problem of -[processes handling signals](https://medium.com/@gchudnov/trapping-signals-in-docker-containers-7a57fdda7d86) -in containers. - -To fix this problem, try the following: - -* Make sure you're using the JSON form of `CMD` and `ENTRYPOINT` -in your Dockerfile. - - For example use `["program", "arg1", "arg2"]` not `"program arg1 arg2"`. - Using the string form causes Docker to run your process using `bash` which - doesn't handle signals properly. Compose always uses the JSON form, so don't - worry if you override the command or entrypoint in your Compose file. - -* If you are able, modify the application that you're running to -add an explicit signal handler for `SIGTERM`. - -* If you can't modify the application, wrap the application in a lightweight init -system (like [s6](http://skarnet.org/software/s6/)) or a signal proxy (like -[dumb-init](https://github.com/Yelp/dumb-init) or -[tini](https://github.com/krallin/tini)). Either of these wrappers take care of -handling `SIGTERM` properly. - -## How do I run multiple copies of a Compose file on the same host? - -Compose uses the project name to create unique identifiers for all of a -project's containers and other resources. To run multiple copies of a project, -set a custom project name using the [`-p` command line -option](./reference/docker-compose.md) or the [`COMPOSE_PROJECT_NAME` -environment variable](./reference/overview.md#compose-project-name). - -## What's the difference between `up`, `run`, and `start`? - -Typically, you want `docker-compose up`. Use `up` to start or restart all the -services defined in a `docker-compose.yml`. In the default "attached" -mode, you'll see all the logs from all the containers. In "detached" mode (`-d`), -Compose exits after starting the containers, but the containers continue to run -in the background. - -The `docker-compose run` command is for running "one-off" or "adhoc" tasks. It -requires the service name you want to run and only starts containers for services -that the running service depends on. Use `run` to run tests or perform -an administrative task such as removing or adding data to a data volume -container. The `run` command acts like `docker run -ti` in that it opens an -interactive terminal to the container and returns an exit status matching the -exit status of the process in the container. - -The `docker-compose start` command is useful only to restart containers -that were previously created, but were stopped. It never creates new -containers. - -## Can I use json instead of yaml for my Compose file? - -Yes. [Yaml is a superset of json](http://stackoverflow.com/a/1729545/444646) so -any JSON file should be valid Yaml. To use a JSON file with Compose, -specify the filename to use, for example: - -```bash -docker-compose -f docker-compose.json up -``` - -## How do I get Compose to wait for my database to be ready before starting my application? - -Unfortunately, Compose won't do that for you but for a good reason. - -The problem of waiting for a database to be ready is really just a subset of a -much larger problem of distributed systems. In production, your database could -become unavailable or move hosts at any time. The application needs to be -resilient to these types of failures. - -To handle this, the application would attempt to re-establish a connection to -the database after a failure. If the application retries the connection, -it should eventually be able to connect to the database. - -To wait for the application to be in a good state, you can implement a -healthcheck. A healthcheck makes a request to the application and checks -the response for a success status code. If it is not successful it waits -for a short period of time, and tries again. After some timeout value, the check -stops trying and report a failure. - -If you need to run tests against your application, you can start by running a -healthcheck. Once the healthcheck gets a successful response, you can start -running your tests. - - -## Should I include my code with `COPY`/`ADD` or a volume? - -You can add your code to the image using `COPY` or `ADD` directive in a -`Dockerfile`. This is useful if you need to relocate your code along with the -Docker image, for example when you're sending code to another environment -(production, CI, etc). - -You should use a `volume` if you want to make changes to your code and see them -reflected immediately, for example when you're developing code and your server -supports hot code reloading or live-reload. - -There may be cases where you'll want to use both. You can have the image -include the code using a `COPY`, and use a `volume` in your Compose file to -include the code from the host during development. The volume overrides -the directory contents of the image. - -## Where can I find example compose files? - -There are [many examples of Compose files on -github](https://github.com/search?q=in%3Apath+docker-compose.yml+extension%3Ayml&type=Code). - - -## Compose documentation - -- [Installing Compose](install.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/gettingstarted.md b/docs/gettingstarted.md deleted file mode 100644 index bb3d51e9..00000000 --- a/docs/gettingstarted.md +++ /dev/null @@ -1,188 +0,0 @@ - - - -# Getting Started - -On this page you build a simple Python web application running on Compose. The -application uses the Flask framework and increments a value in Redis. While the -sample uses Python, the concepts demonstrated here should be understandable even -if you're not familiar with it. - -## Prerequisites - -Make sure you have already -[installed both Docker Engine and Docker Compose](install.md). You -don't need to install Python, it is provided by a Docker image. - -## Step 1: Setup - -1. Create a directory for the project: - - $ mkdir composetest - $ cd composetest - -2. With your favorite text editor create a file called `app.py` in your project - directory. - - from flask import Flask - from redis import Redis - - app = Flask(__name__) - redis = Redis(host='redis', port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - -3. Create another file called `requirements.txt` in your project directory and - add the following: - - flask - redis - - These define the applications dependencies. - -## Step 2: Create a Docker image - -In this step, you build a new Docker image. The image contains all the -dependencies the Python application requires, including Python itself. - -1. In your project directory create a file named `Dockerfile` and add the - following: - - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py - - This tells Docker to: - - * Build an image starting with the Python 2.7 image. - * Add the current directory `.` into the path `/code` in the image. - * Set the working directory to `/code`. - * Install the Python dependencies. - * Set the default command for the container to `python app.py` - - For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](http://docs.docker.com/reference/builder/). - -2. Build the image. - - $ docker build -t web . - - This command builds an image named `web` from the contents of the current - directory. The command automatically locates the `Dockerfile`, `app.py`, and - `requirements.txt` files. - - -## Step 3: Define services - -Define a set of services using `docker-compose.yml`: - -1. Create a file called docker-compose.yml in your project directory and add - the following: - - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis - -This Compose file defines two services, `web` and `redis`. The web service: - -* Builds from the `Dockerfile` in the current directory. -* Forwards the exposed port 5000 on the container to port 5000 on the host machine. -* Mounts the project directory on the host to `/code` inside the container allowing you to modify the code without having to rebuild the image. -* Links the web service to the Redis service. - -The `redis` service uses the latest public [Redis](https://registry.hub.docker.com/_/redis/) image pulled from the Docker Hub registry. - -## Step 4: Build and run your app with Compose - -1. From your project directory, start up your application. - - $ docker-compose up - Pulling image redis... - Building web... - Starting composetest_redis_1... - Starting composetest_web_1... - redis_1 | [8] 02 Jan 18:43:35.576 # Server started, Redis version 2.8.3 - web_1 | * Running on http://0.0.0.0:5000/ - web_1 | * Restarting with stat - - Compose pulls a Redis image, builds an image for your code, and start the - services you defined. - -2. Enter `http://0.0.0.0:5000/` in a browser to see the application running. - - If you're using Docker on Linux natively, then the web app should now be - listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 - doesn't resolve, you can also try http://localhost:5000. - - If you're using Docker Machine on a Mac, use `docker-machine ip MACHINE_VM` to get - the IP address of your Docker host. Then, `open http://MACHINE_VM_IP:5000` in a - browser. - - You should see a message in your browser saying: - - `Hello World! I have been seen 1 times.` - -3. Refresh the page. - - The number should increment. - -## Step 5: Experiment with some other commands - -If you want to run your services in the background, you can pass the `-d` flag -(for "detached" mode) to `docker-compose up` and use `docker-compose ps` to -see what is currently running: - - $ docker-compose up -d - Starting composetest_redis_1... - Starting composetest_web_1... - $ docker-compose ps - Name Command State Ports - ------------------------------------------------------------------- - composetest_redis_1 /usr/local/bin/run Up - composetest_web_1 /bin/sh -c python app.py Up 5000->5000/tcp - -The `docker-compose run` command allows you to run one-off commands for your -services. For example, to see what environment variables are available to the -`web` service: - - $ docker-compose run web env - -See `docker-compose --help` to see other available commands. You can also install [command completion](completion.md) for the bash and zsh shell, which will also show you available commands. - -If you started Compose with `docker-compose up -d`, you'll probably want to stop -your services once you've finished with them: - - $ docker-compose stop - -At this point, you have seen the basics of how Compose works. - - -## Where to go next - -- Next, try the quick start guide for [Django](django.md), - [Rails](rails.md), or [WordPress](wordpress.md). -- [Explore the full list of Compose commands](./reference/index.md) -- [Compose configuration file reference](compose-file.md) diff --git a/docs/index.md b/docs/index.md deleted file mode 100644 index 887df99d..00000000 --- a/docs/index.md +++ /dev/null @@ -1,191 +0,0 @@ - - - -# Overview of Docker Compose - -Compose is a tool for defining and running multi-container Docker applications. -With Compose, you use a Compose file to configure your application's services. -Then, using a single command, you create and start all the services -from your configuration. To learn more about all the features of Compose -see [the list of features](#features). - -Compose is great for development, testing, and staging environments, as well as -CI workflows. You can learn more about each case in -[Common Use Cases](#common-use-cases). - -Using Compose is basically a three-step process. - -1. Define your app's environment with a `Dockerfile` so it can be -reproduced anywhere. -2. Define the services that make up your app in `docker-compose.yml` so -they can be run together in an isolated environment. -3. Lastly, run `docker-compose up` and Compose will start and run your entire app. - -A `docker-compose.yml` looks like this: - - version: 2 - services: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - - logvolume01:/var/log - links: - - redis - redis: - image: redis - volumes: - logvolume01: - driver: default - -For more information about the Compose file, see the -[Compose file reference](compose-file.md) - -Compose has commands for managing the whole lifecycle of your application: - - * Start, stop and rebuild services - * View the status of running services - * Stream the log output of running services - * Run a one-off command on a service - -## Compose documentation - -- [Installing Compose](install.md) -- [Getting Started](gettingstarted.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Frequently asked questions](faq.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) - -## Features - -The features of Compose that make it effective are: - -* [Multiple isolated environments on a single host](#Multiple-isolated-environments-on-a-single-host) -* [Preserve volume data when containers are created](#preserve-volume-data-when-containers-are-created) -* [Only recreate containers that have changed](#only-recreate-containers-that-have-changed) -* [Variables and moving a composition between environments](#variables-and-moving-a-composition-between-environments) - -#### Multiple isolated environments on a single host - -Compose uses a project name to isolate environments from each other. You can use -this project name to: - -* on a dev host, to create multiple copies of a single environment (ex: you want - to run a stable copy for each feature branch of a project) -* on a CI server, to keep builds from interfering with each other, you can set - the project name to a unique build number -* on a shared host or dev host, to prevent different projects which may use the - same service names, from interfering with each other - -The default project name is the basename of the project directory. You can set -a custom project name by using the -[`-p` command line option](./reference/docker-compose.md) or the -[`COMPOSE_PROJECT_NAME` environment variable](./reference/overview.md#compose-project-name). - -#### Preserve volume data when containers are created - -Compose preserves all volumes used by your services. When `docker-compose up` -runs, if it finds any containers from previous runs, it copies the volumes from -the old container to the new container. This process ensures that any data -you've created in volumes isn't lost. - - -#### Only recreate containers that have changed - -Compose caches the configuration used to create a container. When you -restart a service that has not changed, Compose re-uses the existing -containers. Re-using containers means that you can make changes to your -environment very quickly. - - -#### Variables and moving a composition between environments - -Compose supports variables in the Compose file. You can use these variables -to customize your composition for different environments, or different users. -See [Variable substitution](compose-file.md#variable-substitution) for more -details. - -You can extend a Compose file using the `extends` field or by creating multiple -Compose files. See [extends](extends.md) for more details. - - -## Common Use Cases - -Compose can be used in many different ways. Some common use cases are outlined -below. - -### Development environments - -When you're developing software, the ability to run an application in an -isolated environment and interact with it is crucial. The Compose command -line tool can be used to create the environment and interact with it. - -The [Compose file](compose-file.md) provides a way to document and configure -all of the application's service dependencies (databases, queues, caches, -web service APIs, etc). Using the Compose command line tool you can create -and start one or more containers for each dependency with a single command -(`docker-compose up`). - -Together, these features provide a convenient way for developers to get -started on a project. Compose can reduce a multi-page "developer getting -started guide" to a single machine readable Compose file and a few commands. - -### Automated testing environments - -An important part of any Continuous Deployment or Continuous Integration process -is the automated test suite. Automated end-to-end testing requires an -environment in which to run tests. Compose provides a convenient way to create -and destroy isolated testing environments for your test suite. By defining the full -environment in a [Compose file](compose-file.md) you can create and destroy these -environments in just a few commands: - - $ docker-compose up -d - $ ./run_tests - $ docker-compose down - -### Single host deployments - -Compose has traditionally been focused on development and testing workflows, -but with each release we're making progress on more production-oriented features. -You can use Compose to deploy to a remote Docker Engine. The Docker Engine may -be a single instance provisioned with -[Docker Machine](https://docs.docker.com/machine/) or an entire -[Docker Swarm](https://docs.docker.com/swarm/) cluster. - -For details on using production-oriented features, see -[compose in production](production.md) in this documentation. - - -## Release Notes - -To see a detailed list of changes for past and current releases of Docker -Compose, please refer to the -[CHANGELOG](https://github.com/docker/compose/blob/master/CHANGELOG.md). - -## Getting help - -Docker Compose is under active development. If you need help, would like to -contribute, or simply want to talk about the project with like-minded -individuals, we have a number of open channels for communication. - -* To report bugs or file feature requests: please use the [issue tracker on Github](https://github.com/docker/compose/issues). - -* To talk about the project with people in real time: please join the - `#docker-compose` channel on freenode IRC. - -* To contribute code or documentation changes: please submit a [pull request on Github](https://github.com/docker/compose/pulls). - -For more information and resources, please visit the [Getting Help project page](https://docs.docker.com/opensource/get-help/). diff --git a/docs/install.md b/docs/install.md deleted file mode 100644 index 417a48c1..00000000 --- a/docs/install.md +++ /dev/null @@ -1,136 +0,0 @@ - - - -# Install Docker Compose - -You can run Compose on OS X and 64-bit Linux. It is currently not supported on -the Windows operating system. To install Compose, you'll need to install Docker -first. - -To install Compose, do the following: - -1. Install Docker Engine version 1.7.1 or greater: - - * Mac OS X installation (Toolbox installation includes both Engine and Compose) - - * Ubuntu installation - - * other system installations - -2. Mac OS X users are done installing. Others should continue to the next step. - -3. Go to the Compose repository release page on GitHub. - -4. Follow the instructions from the release page and run the `curl` command, -which the release page specifies, in your terminal. - - > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory - probably isn't writable and you'll need to install Compose as the superuser. Run - `sudo -i`, then the two commands below, then `exit`. - - The following is an example command illustrating the format: - - curl -L https://github.com/docker/compose/releases/download/1.5.2/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - - If you have problems installing with `curl`, see - [Alternative Install Options](#alternative-install-options). - -5. Apply executable permissions to the binary: - - $ chmod +x /usr/local/bin/docker-compose - -6. Optionally, install [command completion](completion.md) for the -`bash` and `zsh` shell. - -7. Test the installation. - - $ docker-compose --version - docker-compose version: 1.5.2 - - -## Alternative install options - -### Install using pip - -Compose can be installed from [pypi](https://pypi.python.org/pypi/docker-compose) -using `pip`. If you install using `pip` it is highly recommended that you use a -[virtualenv](https://virtualenv.pypa.io/en/latest/) because many operating systems -have python system packages that conflict with docker-compose dependencies. See -the [virtualenv tutorial](http://docs.python-guide.org/en/latest/dev/virtualenvs/) -to get started. - - $ pip install docker-compose - -> **Note:** pip version 6.0 or greater is required - -### Install as a container - -Compose can also be run inside a container, from a small bash script wrapper. -To install compose as a container run: - - $ curl -L https://github.com/docker/compose/releases/download/1.5.2/run.sh > /usr/local/bin/docker-compose - $ chmod +x /usr/local/bin/docker-compose - -## Master builds - -If you're interested in trying out a pre-release build you can download a -binary from https://dl.bintray.com/docker-compose/master/. Pre-release -builds allow you to try out new features before they are released, but may -be less stable. - - -## Upgrading - -If you're upgrading from Compose 1.2 or earlier, you'll need to remove or migrate -your existing containers after upgrading Compose. This is because, as of version -1.3, Compose uses Docker labels to keep track of containers, and so they need to -be recreated with labels added. - -If Compose detects containers that were created without labels, it will refuse -to run so that you don't end up with two sets of them. If you want to keep using -your existing containers (for example, because they have data volumes you want -to preserve) you can use compose 1.5.x to migrate them with the following command: - - $ docker-compose migrate-to-labels - -Alternatively, if you're not worried about keeping them, you can remove them. -Compose will just create new ones. - - $ docker rm -f -v myapp_web_1 myapp_db_1 ... - - -## Uninstallation - -To uninstall Docker Compose if you installed using `curl`: - - $ rm /usr/local/bin/docker-compose - - -To uninstall Docker Compose if you installed using `pip`: - - $ pip uninstall docker-compose - ->**Note**: If you get a "Permission denied" error using either of the above ->methods, you probably do not have the proper permissions to remove ->`docker-compose`. To force the removal, prepend `sudo` to either of the above ->commands and run again. - - -## Where to go next - -- [User guide](index.md) -- [Getting Started](gettingstarted.md) -- [Get started with Django](django.md) -- [Get started with Rails](rails.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/networking.md b/docs/networking.md deleted file mode 100644 index 91ac0b73..00000000 --- a/docs/networking.md +++ /dev/null @@ -1,94 +0,0 @@ - - - -# Networking in Compose - -> **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. - -Compose sets up a single default -[network](/engine/reference/commandline/network_create.md) for your app. Each -container for a service joins the default network and is both *reachable* by -other containers on that network, and *discoverable* by them at a hostname -identical to the container name. - -> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. - -For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: - - web: - build: . - ports: - - "8000:8000" - db: - image: postgres - -When you run `docker-compose --x-networking up`, the following happens: - -1. A network called `myapp` is created. -2. A container is created using `web`'s configuration. It joins the network -`myapp` under the name `myapp_web_1`. -3. A container is created using `db`'s configuration. It joins the network -`myapp` under the name `myapp_db_1`. - -Each container can now look up the hostname `myapp_web_1` or `myapp_db_1` and -get back the appropriate container's IP address. For example, `web`'s -application code could connect to the URL `postgres://myapp_db_1:5432` and start -using the Postgres database. - -Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. - -> **Note:** in the next release there will be additional aliases for the -> container, including a short name without the project name and container -> index. The full container name will remain as one of the alias for backwards -> compatibility. - -## Updating containers - -If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. - -If any containers have connections open to the old container, they will be closed. It is a container's responsibility to detect this condition, look up the name again and reconnect. - -## Configure how services are published - -By default, containers for each service are published on the network with the -container name. If you want to change the name, or stop containers from being -discoverable at all, you can use the `container_name` option: - - web: - build: . - container_name: "my-web-application" - -## Links - -Docker links are a one-way, single-host communication system. They should now be considered deprecated, and you should update your app to use networking instead. In the majority of cases, this will simply involve removing the `links` sections from your `docker-compose.yml`. - -## Specifying the network driver - -By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](/engine/extend/plugins_network.md). - -You can specify which one to use with the `--x-network-driver` flag: - - $ docker-compose --x-networking --x-network-driver=overlay up - - - -## Custom container network modes - -Compose allows you to specify a custom network mode for a service with the `net` option - for example, `net: "host"` specifies that its containers should use the same network namespace as the Docker host, and `net: "none"` specifies that they should have no networking capabilities. - -If a service specifies the `net` option, its containers will *not* join the app’s network and will not be able to communicate with other services in the app. - -If *all* services in an app specify the `net` option, a network will not be created at all. diff --git a/docs/production.md b/docs/production.md deleted file mode 100644 index 46e221bb..00000000 --- a/docs/production.md +++ /dev/null @@ -1,85 +0,0 @@ - - - -## Using Compose in production - -> Compose is still primarily aimed at development and testing environments. -> Compose may be used for smaller production deployments, but is probably -> not yet suitable for larger deployments. - -When deploying to production, you'll almost certainly want to make changes to -your app configuration that are more appropriate to a live environment. These -changes may include: - -- Removing any volume bindings for application code, so that code stays inside - the container and can't be changed from outside -- Binding to different ports on the host -- Setting environment variables differently (e.g., to decrease the verbosity of - logging, or to enable email sending) -- Specifying a restart policy (e.g., `restart: always`) to avoid downtime -- Adding extra services (e.g., a log aggregator) - -For this reason, you'll probably want to define an additional Compose file, say -`production.yml`, which specifies production-appropriate -configuration. This configuration file only needs to include the changes you'd -like to make from the original Compose file. The additional Compose file -can be applied over the original `docker-compose.yml` to create a new configuration. - -Once you've got a second configuration file, tell Compose to use it with the -`-f` option: - - $ docker-compose -f docker-compose.yml -f production.yml up -d - -See [Using multiple compose files](extends.md#different-environments) for a more -complete example. - -### Deploying changes - -When you make changes to your app code, you'll need to rebuild your image and -recreate your app's containers. To redeploy a service called -`web`, you would use: - - $ docker-compose build web - $ docker-compose up --no-deps -d web - -This will first rebuild the image for `web` and then stop, destroy, and recreate -*just* the `web` service. The `--no-deps` flag prevents Compose from also -recreating any services which `web` depends on. - -### Running Compose on a single server - -You can use Compose to deploy an app to a remote Docker host by setting the -`DOCKER_HOST`, `DOCKER_TLS_VERIFY`, and `DOCKER_CERT_PATH` environment variables -appropriately. For tasks like this, -[Docker Machine](https://docs.docker.com/machine/) makes managing local and -remote Docker hosts very easy, and is recommended even if you're not deploying -remotely. - -Once you've set up your environment variables, all the normal `docker-compose` -commands will work with no further configuration. - -### Running Compose on a Swarm cluster - -[Docker Swarm](https://docs.docker.com/swarm/), a Docker-native clustering -system, exposes the same API as a single Docker host, which means you can use -Compose against a Swarm instance and run your apps across multiple hosts. - -Compose/Swarm integration is still in the experimental stage, and Swarm is still -in beta, but if you'd like to explore and experiment, check out the integration -guide. - -## Compose documentation - -- [Installing Compose](install.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/rails.md b/docs/rails.md deleted file mode 100644 index e3daff25..00000000 --- a/docs/rails.md +++ /dev/null @@ -1,142 +0,0 @@ - - -## Quickstart Guide: Compose and Rails - -This Quickstart guide will show you how to use Compose to set up and run a Rails/PostgreSQL app. Before starting, you'll need to have [Compose installed](install.md). - -### Define the project - -Start by setting up the three files you'll need to build the app. First, since -your app is going to run inside a Docker container containing all of its -dependencies, you'll need to define exactly what needs to be included in the -container. This is done using a file called `Dockerfile`. To begin with, the -Dockerfile consists of: - - FROM ruby:2.2.0 - RUN apt-get update -qq && apt-get install -y build-essential libpq-dev - RUN mkdir /myapp - WORKDIR /myapp - ADD Gemfile /myapp/Gemfile - ADD Gemfile.lock /myapp/Gemfile.lock - RUN bundle install - ADD . /myapp - -That'll put your application code inside an image that will build a container with Ruby, Bundler and all your dependencies inside it. For more information on how to write Dockerfiles, see the [Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the [Dockerfile reference](https://docs.docker.com/engine/reference/builder/). - -Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten in a moment by `rails new`. - - source 'https://rubygems.org' - gem 'rails', '4.2.0' - -You'll need an empty `Gemfile.lock` in order to build our `Dockerfile`. - - $ touch Gemfile.lock - -Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. - - db: - image: postgres - web: - build: . - command: bundle exec rails s -p 3000 -b '0.0.0.0' - volumes: - - .:/myapp - ports: - - "3000:3000" - links: - - db - -### Build the project - -With those three files in place, you can now generate the Rails skeleton app -using `docker-compose run`: - - $ docker-compose run web rails new . --force --database=postgresql --skip-bundle - -First, Compose will build the image for the `web` service using the -`Dockerfile`. Then it'll run `rails new` inside a new container, using that -image. Once it's done, you should have generated a fresh app: - - $ ls - Dockerfile app docker-compose.yml tmp - Gemfile bin lib vendor - Gemfile.lock config log - README.rdoc config.ru public - Rakefile db test - - -The files `rails new` created are owned by root. This happens because the -container runs as the `root` user. Change the ownership of the new files. - - sudo chown -R $USER:$USER . - -Uncomment the line in your new `Gemfile` which loads `therubyracer`, so you've -got a Javascript runtime: - - gem 'therubyracer', platforms: :ruby - -Now that you've got a new `Gemfile`, you need to build the image again. (This, -and changes to the Dockerfile itself, should be the only times you'll need to -rebuild.) - - $ docker-compose build - - -### Connect the database - -The app is now bootable, but you're not quite there yet. By default, Rails -expects a database to be running on `localhost` - so you need to point it at the -`db` container instead. You also need to change the database and username to -align with the defaults set by the `postgres` image. - -Replace the contents of `config/database.yml` with the following: - - development: &default - adapter: postgresql - encoding: unicode - database: postgres - pool: 5 - username: postgres - password: - host: db - - test: - <<: *default - database: myapp_test - -You can now boot the app with: - - $ docker-compose up - -If all's well, you should see some PostgreSQL output, and then—after a few -seconds—the familiar refrain: - - myapp_web_1 | [2014-01-17 17:16:29] INFO WEBrick 1.3.1 - myapp_web_1 | [2014-01-17 17:16:29] INFO ruby 2.2.0 (2014-12-25) [x86_64-linux-gnu] - myapp_web_1 | [2014-01-17 17:16:29] INFO WEBrick::HTTPServer#start: pid=1 port=3000 - -Finally, you need to create the database. In another terminal, run: - - $ docker-compose run web rake db:create - -That's it. Your app should now be running on port 3000 on your Docker daemon. If you're using [Docker Machine](https://docs.docker.com/machine/), then `docker-machine ip MACHINE_VM` returns the Docker host IP address. - - -## More Compose documentation - -- [User guide](index.md) -- [Installing Compose](install.md) -- [Getting Started](gettingstarted.md) -- [Get started with Django](django.md) -- [Get started with WordPress](wordpress.md) -- [Command line reference](./reference/index.md) -- [Compose file reference](compose-file.md) diff --git a/docs/reference/build.md b/docs/reference/build.md deleted file mode 100644 index 84aefc25..00000000 --- a/docs/reference/build.md +++ /dev/null @@ -1,25 +0,0 @@ - - -# build - -``` -Usage: build [options] [SERVICE...] - -Options: ---force-rm Always remove intermediate containers. ---no-cache Do not use cache when building the image. ---pull Always attempt to pull a newer version of the image. -``` - -Services are built once and then tagged as `project_service`, e.g., -`composetest_db`. If you change a service's Dockerfile or the contents of its -build directory, run `docker-compose build` to rebuild it. diff --git a/docs/reference/config.md b/docs/reference/config.md deleted file mode 100644 index 1a9706f4..00000000 --- a/docs/reference/config.md +++ /dev/null @@ -1,23 +0,0 @@ - - -# config - -```: -Usage: config [options] - -Options: --q, --quiet Only validate the configuration, don't print - anything. ---services Print the service names, one per line. -``` - -Validate and view the compose file. diff --git a/docs/reference/create.md b/docs/reference/create.md deleted file mode 100644 index a785e2c7..00000000 --- a/docs/reference/create.md +++ /dev/null @@ -1,25 +0,0 @@ - - -# create - -``` -Usage: create [options] [SERVICE...] - -Options: ---force-recreate Recreate containers even if their configuration and - image haven't changed. Incompatible with --no-recreate. ---no-recreate If containers already exist, don't recreate them. - Incompatible with --force-recreate. ---no-build Don't build an image, even if it's missing -``` - -Creates containers for a service. diff --git a/docs/reference/docker-compose.md b/docs/reference/docker-compose.md deleted file mode 100644 index c19e4284..00000000 --- a/docs/reference/docker-compose.md +++ /dev/null @@ -1,106 +0,0 @@ - - - -# docker-compose Command - -``` -Usage: - docker-compose [-f=...] [options] [COMMAND] [ARGS...] - docker-compose -h|--help - -Options: - -f, --file FILE Specify an alternate compose file (default: docker-compose.yml) - -p, --project-name NAME Specify an alternate project name (default: directory name) - --verbose Show more output - -v, --version Print version and exit - -Commands: - build Build or rebuild services - help Get help on a command - kill Kill containers - logs View output from containers - pause Pause services - port Print the public port for a port binding - ps List containers - pull Pulls service images - restart Restart services - rm Remove stopped containers - run Run a one-off command - scale Set number of containers for a service - start Start services - stop Stop services - unpause Unpause services - up Create and start containers - version Show the Docker-Compose version information -``` - -The Docker Compose binary. You use this command to build and manage multiple -services in Docker containers. - -Use the `-f` flag to specify the location of a Compose configuration file. You -can supply multiple `-f` configuration files. When you supply multiple files, -Compose combines them into a single configuration. Compose builds the -configuration in the order you supply the files. Subsequent files override and -add to their successors. - -For example, consider this command line: - -``` -$ docker-compose -f docker-compose.yml -f docker-compose.admin.yml run backup_db` -``` - -The `docker-compose.yml` file might specify a `webapp` service. - -``` -webapp: - image: examples/web - ports: - - "8000:8000" - volumes: - - "/data" -``` - -If the `docker-compose.admin.yml` also specifies this same service, any matching -fields will override the previous file. New values, add to the `webapp` service -configuration. - -``` -webapp: - build: . - environment: - - DEBUG=1 -``` - -Use a `-f` with `-` (dash) as the filename to read the configuration from -stdin. When stdin is used all paths in the configuration are -relative to the current working directory. - -The `-f` flag is optional. If you don't provide this flag on the command line, -Compose traverses the working directory and its subdirectories looking for a -`docker-compose.yml` and a `docker-compose.override.yml` file. You must -supply at least the `docker-compose.yml` file. If both files are present, -Compose combines the two files into a single configuration. The configuration -in the `docker-compose.override.yml` file is applied over and in addition to -the values in the `docker-compose.yml` file. - -See also the `COMPOSE_FILE` [environment variable](overview.md#compose-file). - -Each configuration has a project name. If you supply a `-p` flag, you can -specify a project name. If you don't specify the flag, Compose uses the current -directory name. See also the `COMPOSE_PROJECT_NAME` [environment variable]( -overview.md#compose-project-name) - - -## Where to go next - -* [CLI environment variables](overview.md) -* [Command line reference](index.md) diff --git a/docs/reference/down.md b/docs/reference/down.md deleted file mode 100644 index 428e4e58..00000000 --- a/docs/reference/down.md +++ /dev/null @@ -1,26 +0,0 @@ - - -# down - -``` -Stop containers and remove containers, networks, volumes, and images -created by `up`. Only containers and networks are removed by default. - -Usage: down [options] - -Options: - --rmi type Remove images, type may be one of: 'all' to remove - all images, or 'local' to remove only images that - don't have an custom name set by the `image` field - -v, --volumes Remove data volumes - -``` diff --git a/docs/reference/events.md b/docs/reference/events.md deleted file mode 100644 index 827258f2..00000000 --- a/docs/reference/events.md +++ /dev/null @@ -1,34 +0,0 @@ - - -# events - -``` -Usage: events [options] [SERVICE...] - -Options: - --json Output events as a stream of json objects -``` - -Stream container events for every container in the project. - -With the `--json` flag, a json object will be printed one per line with the -format: - -``` -{ - "service": "web", - "event": "create", - "container": "213cf75fc39a", - "image": "alpine:edge", - "time": "2015-11-20T18:01:03.615550", -} -``` diff --git a/docs/reference/help.md b/docs/reference/help.md deleted file mode 100644 index 613708ed..00000000 --- a/docs/reference/help.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# help - -``` -Usage: help COMMAND -``` - -Displays help and usage instructions for a command. diff --git a/docs/reference/index.md b/docs/reference/index.md deleted file mode 100644 index 5406b9c7..00000000 --- a/docs/reference/index.md +++ /dev/null @@ -1,40 +0,0 @@ - - -## Compose CLI reference - -The following pages describe the usage information for the [docker-compose](docker-compose.md) subcommands. You can also see this information by running `docker-compose [SUBCOMMAND] --help` from the command line. - -* [build](build.md) -* [config](config.md) -* [create](create.md) -* [down](down.md) -* [events](events.md) -* [help](help.md) -* [kill](kill.md) -* [logs](logs.md) -* [pause](pause.md) -* [port](port.md) -* [ps](ps.md) -* [pull](pull.md) -* [restart](restart.md) -* [rm](rm.md) -* [run](run.md) -* [scale](scale.md) -* [start](start.md) -* [stop](stop.md) -* [unpause](unpause.md) -* [up](up.md) - -## Where to go next - -* [CLI environment variables](overview.md) -* [docker-compose Command](docker-compose.md) diff --git a/docs/reference/kill.md b/docs/reference/kill.md deleted file mode 100644 index dc4bf23a..00000000 --- a/docs/reference/kill.md +++ /dev/null @@ -1,24 +0,0 @@ - - -# kill - -``` -Usage: kill [options] [SERVICE...] - -Options: --s SIGNAL SIGNAL to send to the container. Default signal is SIGKILL. -``` - -Forces running containers to stop by sending a `SIGKILL` signal. Optionally the -signal can be passed, for example: - - $ docker-compose kill -s SIGINT diff --git a/docs/reference/logs.md b/docs/reference/logs.md deleted file mode 100644 index 5b241ea7..00000000 --- a/docs/reference/logs.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# logs - -``` -Usage: logs [options] [SERVICE...] - -Options: ---no-color Produce monochrome output. -``` - -Displays log output from services. diff --git a/docs/reference/overview.md b/docs/reference/overview.md deleted file mode 100644 index 8e3967b2..00000000 --- a/docs/reference/overview.md +++ /dev/null @@ -1,89 +0,0 @@ - - - -# Introduction to the CLI - -This section describes the subcommands you can use with the `docker-compose` command. You can run subcommand against one or more services. To run against a specific service, you supply the service name from your compose configuration. If you do not specify the service name, the command runs against all the services in your configuration. - - -## Commands - -* [docker-compose Command](docker-compose.md) -* [CLI Reference](index.md) - - -## Environment Variables - -Several environment variables are available for you to configure the Docker Compose command-line behaviour. - -Variables starting with `DOCKER_` are the same as those used to configure the -Docker command-line client. If you're using `docker-machine`, then the `eval "$(docker-machine env my-docker-vm)"` command should set them to their correct values. (In this example, `my-docker-vm` is the name of a machine you created.) - -### COMPOSE\_PROJECT\_NAME - -Sets the project name. This value is prepended along with the service name to the container container on start up. For example, if you project name is `myapp` and it includes two services `db` and `web` then compose starts containers named `myapp_db_1` and `myapp_web_1` respectively. - -Setting this is optional. If you do not set this, the `COMPOSE_PROJECT_NAME` -defaults to the `basename` of the project directory. See also the `-p` -[command-line option](docker-compose.md). - -### COMPOSE\_FILE - -Specify the file containing the compose configuration. If not provided, -Compose looks for a file named `docker-compose.yml` in the current directory -and then each parent directory in succession until a file by that name is -found. See also the `-f` [command-line option](docker-compose.md). - -### COMPOSE\_API\_VERSION - -The Docker API only supports requests from clients which report a specific -version. If you receive a `client and server don't have same version error` using -`docker-compose`, you can workaround this error by setting this environment -variable. Set the version value to match the server version. - -Setting this variable is intended as a workaround for situations where you need -to run temporarily with a mismatch between the client and server version. For -example, if you can upgrade the client but need to wait to upgrade the server. - -Running with this variable set and a known mismatch does prevent some Docker -features from working properly. The exact features that fail would depend on the -Docker client and server versions. For this reason, running with this variable -set is only intended as a workaround and it is not officially supported. - -If you run into problems running with this set, resolve the mismatch through -upgrade and remove this setting to see if your problems resolve before notifying -support. - -### DOCKER\_HOST - -Sets the URL of the `docker` daemon. As with the Docker client, defaults to `unix:///var/run/docker.sock`. - -### DOCKER\_TLS\_VERIFY - -When set to anything other than an empty string, enables TLS communication with -the `docker` daemon. - -### DOCKER\_CERT\_PATH - -Configures the path to the `ca.pem`, `cert.pem`, and `key.pem` files used for TLS verification. Defaults to `~/.docker`. - -### COMPOSE\_HTTP\_TIMEOUT - -Configures the time (in seconds) a request to the Docker daemon is allowed to hang before Compose considers -it failed. Defaults to 60 seconds. - - -## Related Information - -- [User guide](../index.md) -- [Installing Compose](../install.md) -- [Compose file reference](../compose-file.md) diff --git a/docs/reference/pause.md b/docs/reference/pause.md deleted file mode 100644 index a0ffab03..00000000 --- a/docs/reference/pause.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# pause - -``` -Usage: pause [SERVICE...] -``` - -Pauses running containers of a service. They can be unpaused with `docker-compose unpause`. diff --git a/docs/reference/port.md b/docs/reference/port.md deleted file mode 100644 index c946a97d..00000000 --- a/docs/reference/port.md +++ /dev/null @@ -1,23 +0,0 @@ - - -# port - -``` -Usage: port [options] SERVICE PRIVATE_PORT - -Options: ---protocol=proto tcp or udp [default: tcp] ---index=index index of the container if there are multiple - instances of a service [default: 1] -``` - -Prints the public port for a port binding. diff --git a/docs/reference/ps.md b/docs/reference/ps.md deleted file mode 100644 index 546d68e7..00000000 --- a/docs/reference/ps.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# ps - -``` -Usage: ps [options] [SERVICE...] - -Options: --q Only display IDs -``` - -Lists containers. diff --git a/docs/reference/pull.md b/docs/reference/pull.md deleted file mode 100644 index 5ec184b7..00000000 --- a/docs/reference/pull.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# pull - -``` -Usage: pull [options] [SERVICE...] - -Options: ---ignore-pull-failures Pull what it can and ignores images with pull failures. -``` - -Pulls service images. diff --git a/docs/reference/restart.md b/docs/reference/restart.md deleted file mode 100644 index bbd4a68b..00000000 --- a/docs/reference/restart.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# restart - -``` -Usage: restart [options] [SERVICE...] - -Options: --t, --timeout TIMEOUT Specify a shutdown timeout in seconds. (default: 10) -``` - -Restarts services. diff --git a/docs/reference/rm.md b/docs/reference/rm.md deleted file mode 100644 index f8479224..00000000 --- a/docs/reference/rm.md +++ /dev/null @@ -1,27 +0,0 @@ - - -# rm - -``` -Usage: rm [options] [SERVICE...] - -Options: --f, --force Don't ask to confirm removal --v Remove volumes associated with containers -``` - -Removes stopped service containers. - -By default, volumes attached to containers will not be removed. You can see all -volumes with `docker volume ls`. - -Any data which is not in a volume will be lost. diff --git a/docs/reference/run.md b/docs/reference/run.md deleted file mode 100644 index 21890c60..00000000 --- a/docs/reference/run.md +++ /dev/null @@ -1,55 +0,0 @@ - - -# run - -``` -Usage: run [options] [-e KEY=VAL...] SERVICE [COMMAND] [ARGS...] - -Options: --d Detached mode: Run container in the background, print - new container name. ---name NAME Assign a name to the container ---entrypoint CMD Override the entrypoint of the image. --e KEY=VAL Set an environment variable (can be used multiple times) --u, --user="" Run as specified username or uid ---no-deps Don't start linked services. ---rm Remove container after run. Ignored in detached mode. --p, --publish=[] Publish a container's port(s) to the host ---service-ports Run command with the service's ports enabled and mapped to the host. --T Disable pseudo-tty allocation. By default `docker-compose run` allocates a TTY. -``` - -Runs a one-time command against a service. For example, the following command starts the `web` service and runs `bash` as its command. - - $ docker-compose run web bash - -Commands you use with `run` start in new containers with the same configuration as defined by the service' configuration. This means the container has the same volumes, links, as defined in the configuration file. There two differences though. - -First, the command passed by `run` overrides the command defined in the service configuration. For example, if the `web` service configuration is started with `bash`, then `docker-compose run web python app.py` overrides it with `python app.py`. - -The second difference is the `docker-compose run` command does not create any of the ports specified in the service configuration. This prevents the port collisions with already open ports. If you *do want* the service's ports created and mapped to the host, specify the `--service-ports` flag: - - $ docker-compose run --service-ports web python manage.py shell - -Alternatively manual port mapping can be specified. Same as when running Docker's `run` command - using `--publish` or `-p` options: - - $ docker-compose run --publish 8080:80 -p 2022:22 -p 127.0.0.1:2021:21 web python manage.py shell - -If you start a service configured with links, the `run` command first checks to see if the linked service is running and starts the service if it is stopped. Once all the linked services are running, the `run` executes the command you passed it. So, for example, you could run: - - $ docker-compose run db psql -h db -U docker - -This would open up an interactive PostgreSQL shell for the linked `db` container. - -If you do not want the `run` command to start linked containers, specify the `--no-deps` flag: - - $ docker-compose run --no-deps web python manage.py shell diff --git a/docs/reference/scale.md b/docs/reference/scale.md deleted file mode 100644 index 75140ee9..00000000 --- a/docs/reference/scale.md +++ /dev/null @@ -1,21 +0,0 @@ - - -# scale - -``` -Usage: scale [SERVICE=NUM...] -``` - -Sets the number of containers to run for a service. - -Numbers are specified as arguments in the form `service=num`. For example: - - $ docker-compose scale web=2 worker=3 diff --git a/docs/reference/start.md b/docs/reference/start.md deleted file mode 100644 index f0bdd5a9..00000000 --- a/docs/reference/start.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# start - -``` -Usage: start [SERVICE...] -``` - -Starts existing containers for a service. diff --git a/docs/reference/stop.md b/docs/reference/stop.md deleted file mode 100644 index ec7e6688..00000000 --- a/docs/reference/stop.md +++ /dev/null @@ -1,22 +0,0 @@ - - -# stop - -``` -Usage: stop [options] [SERVICE...] - -Options: --t, --timeout TIMEOUT Specify a shutdown timeout in seconds (default: 10). -``` - -Stops running containers without removing them. They can be started again with -`docker-compose start`. diff --git a/docs/reference/unpause.md b/docs/reference/unpause.md deleted file mode 100644 index 846b229e..00000000 --- a/docs/reference/unpause.md +++ /dev/null @@ -1,18 +0,0 @@ - - -# unpause - -``` -Usage: unpause [SERVICE...] -``` - -Unpauses paused containers of a service. diff --git a/docs/reference/up.md b/docs/reference/up.md deleted file mode 100644 index a02358ec..00000000 --- a/docs/reference/up.md +++ /dev/null @@ -1,51 +0,0 @@ - - -# up - -``` -Usage: up [options] [SERVICE...] - -Options: --d Detached mode: Run containers in the background, - print new container names. - Incompatible with --abort-on-container-exit. ---no-color Produce monochrome output. ---no-deps Don't start linked services. ---force-recreate Recreate containers even if their configuration - and image haven't changed. - Incompatible with --no-recreate. ---no-recreate If containers already exist, don't recreate them. - Incompatible with --force-recreate. ---no-build Don't build an image, even if it's missing ---abort-on-container-exit Stops all containers if any container was stopped. - Incompatible with -d. --t, --timeout TIMEOUT Use this timeout in seconds for container shutdown - when attached or when containers are already - running. (default: 10) -``` - -Builds, (re)creates, starts, and attaches to containers for a service. - -Unless they are already running, this command also starts any linked services. - -The `docker-compose up` command aggregates the output of each container. When -the command exits, all containers are stopped. Running `docker-compose up -d` -starts the containers in the background and leaves them running. - -If there are existing containers for a service, and the service's configuration -or image was changed after the container's creation, `docker-compose up` picks -up the changes by stopping and recreating the containers (preserving mounted -volumes). To prevent Compose from picking up changes, use the `--no-recreate` -flag. - -If you want to force Compose to stop and recreate all containers, use the -`--force-recreate` flag. diff --git a/docs/wordpress.md b/docs/wordpress.md deleted file mode 100644 index 84010491..00000000 --- a/docs/wordpress.md +++ /dev/null @@ -1,102 +0,0 @@ - - - -# Quickstart Guide: Compose and WordPress - -You can use Compose to easily run WordPress in an isolated environment built -with Docker containers. - -## Define the project - -First, [Install Compose](install.md) and then download WordPress into the -current directory: - - $ curl https://wordpress.org/latest.tar.gz | tar -xvzf - - -This will create a directory called `wordpress`. If you wish, you can rename it -to the name of your project. - -Next, inside that directory, create a `Dockerfile`, a file that defines what -environment your app is going to run in. For more information on how to write -Dockerfiles, see the -[Docker user guide](https://docs.docker.com/engine/userguide/dockerimages/#building-an-image-from-a-dockerfile) and the -[Dockerfile reference](https://docs.docker.com/engine/reference/builder/). In -this case, your Dockerfile should be: - - FROM orchardup/php5 - ADD . /code - -This tells Docker how to build an image defining a container that contains PHP -and WordPress. - -Next you'll create a `docker-compose.yml` file that will start your web service -and a separate MySQL instance: - - web: - build: . - command: php -S 0.0.0.0:8000 -t /code - ports: - - "8000:8000" - links: - - db - volumes: - - .:/code - db: - image: orchardup/mysql - environment: - MYSQL_DATABASE: wordpress - -A supporting file is needed to get this working. `wp-config.php` is -the standard WordPress config file with a single change to point the database -configuration at the `db` container: - - Note: This functionality is in the experimental stage, and contains some hacks and workarounds which will be removed as it matures. - -## Prerequisites - -Before you start, you’ll need to install the experimental build of Docker, and the latest versions of Machine and Compose. - -- To install the experimental Docker build on a Linux machine, follow the instructions [here](https://github.com/docker/docker/tree/master/experimental#install-docker-experimental). - -- To install the experimental Docker build on a Mac, run these commands: - - $ curl -L https://experimental.docker.com/builds/Darwin/x86_64/docker-latest > /usr/local/bin/docker - $ chmod +x /usr/local/bin/docker - -- To install Machine, follow the instructions [here](https://docs.docker.com/machine/install-machine/). - -- To install Compose, follow the instructions [here](https://docs.docker.com/compose/install/). - -You’ll also need a [Docker Hub](https://hub.docker.com/account/signup/) account and a [Digital Ocean](https://www.digitalocean.com/) account. - -## Set up a swarm with multi-host networking - -Set the `DIGITALOCEAN_ACCESS_TOKEN` environment variable to a valid Digital Ocean API token, which you can generate in the [API panel](https://cloud.digitalocean.com/settings/applications). - - DIGITALOCEAN_ACCESS_TOKEN=abc12345 - -Start a consul server: - - docker-machine create -d digitalocean --engine-install-url https://experimental.docker.com consul - docker $(docker-machine config consul) run -d -p 8500:8500 -h consul progrium/consul -server -bootstrap - -(In a real world setting you’d set up a distributed consul, but that’s beyond the scope of this guide!) - -Create a Swarm token: - - SWARM_TOKEN=$(docker run swarm create) - -Create a Swarm master: - - docker-machine create -d digitalocean --swarm --swarm-master --swarm-discovery=token://$SWARM_TOKEN --engine-install-url="https://experimental.docker.com" --digitalocean-image "ubuntu-14-10-x64" --engine-opt=default-network=overlay:multihost --engine-label=com.docker.network.driver.overlay.bind_interface=eth0 --engine-opt=kv-store=consul:$(docker-machine ip consul):8500 swarm-0 - -Create a Swarm node: - - docker-machine create -d digitalocean --swarm --swarm-discovery=token://$SWARM_TOKEN --engine-install-url="https://experimental.docker.com" --digitalocean-image "ubuntu-14-10-x64" --engine-opt=default-network=overlay:multihost --engine-label=com.docker.network.driver.overlay.bind_interface=eth0 --engine-opt=kv-store=consul:$(docker-machine ip consul):8500 --engine-label com.docker.network.driver.overlay.neighbor_ip=$(docker-machine ip swarm-0) swarm-1 - -You can create more Swarm nodes if you want - it’s best to give them sensible names (swarm-2, swarm-3, etc). - -Finally, point Docker at your swarm: - - eval "$(docker-machine env --swarm swarm-0)" - -## Run containers and get them communicating - -Now that you’ve got a swarm up and running, you can create containers on it just like a single Docker instance: - - $ docker run busybox echo hello world - hello world - -If you run `docker ps -a`, you can see what node that container was started on by looking at its name (here it’s swarm-3): - - $ docker ps -a - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 41f59749737b busybox "echo hello world" 15 seconds ago Exited (0) 13 seconds ago swarm-3/trusting_leakey - -As you start more containers, they’ll be placed on different nodes across the cluster, thanks to Swarm’s default “spread” scheduling strategy. - -Every container started on this swarm will use the “overlay:multihost” network by default, meaning they can all intercommunicate. Each container gets an IP address on that network, and an `/etc/hosts` file which will be updated on-the-fly with every other container’s IP address and name. That means that if you have a running container named ‘foo’, other containers can access it at the hostname ‘foo’. - -Let’s verify that multi-host networking is functioning. Start a long-running container: - - $ docker run -d --name long-running busybox top - - -If you start a new container and inspect its /etc/hosts file, you’ll see the long-running container in there: - - $ docker run busybox cat /etc/hosts - ... - 172.21.0.6 long-running - -Verify that connectivity works between containers: - - $ docker run busybox ping long-running - PING long-running (172.21.0.6): 56 data bytes - 64 bytes from 172.21.0.6: seq=0 ttl=64 time=7.975 ms - 64 bytes from 172.21.0.6: seq=1 ttl=64 time=1.378 ms - 64 bytes from 172.21.0.6: seq=2 ttl=64 time=1.348 ms - ^C - --- long-running ping statistics --- - 3 packets transmitted, 3 packets received, 0% packet loss - round-trip min/avg/max = 1.140/2.099/7.975 ms - -## Run a Compose application - -Here’s an example of a simple Python + Redis app using multi-host networking on a swarm. - -Create a directory for the app: - - $ mkdir composetest - $ cd composetest - -Inside this directory, create 2 files. - -First, create `app.py` - a simple web app that uses the Flask framework and increments a value in Redis: - - from flask import Flask - from redis import Redis - import os - app = Flask(__name__) - redis = Redis(host='composetest_redis_1', port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - -Note that we’re connecting to a host called `composetest_redis_1` - this is the name of the Redis container that Compose will start. - -Second, create a Dockerfile for the app container: - - FROM python:2.7 - RUN pip install flask redis - ADD . /code - WORKDIR /code - CMD ["python", "app.py"] - -Build the Docker image and push it to the Hub (you’ll need a Hub account). Replace `` with your Docker Hub username: - - $ docker build -t /counter . - $ docker push /counter - -Next, create a `docker-compose.yml`, which defines the configuration for the web and redis containers. Once again, replace `` with your Hub username: - - web: - image: /counter - ports: - - "80:5000" - redis: - image: redis - -Now start the app: - - $ docker-compose up -d - Pulling web (username/counter:latest)... - swarm-0: Pulling username/counter:latest... : downloaded - swarm-2: Pulling username/counter:latest... : downloaded - swarm-1: Pulling username/counter:latest... : downloaded - swarm-3: Pulling username/counter:latest... : downloaded - swarm-4: Pulling username/counter:latest... : downloaded - Creating composetest_web_1... - Pulling redis (redis:latest)... - swarm-2: Pulling redis:latest... : downloaded - swarm-1: Pulling redis:latest... : downloaded - swarm-3: Pulling redis:latest... : downloaded - swarm-4: Pulling redis:latest... : downloaded - swarm-0: Pulling redis:latest... : downloaded - Creating composetest_redis_1... - -Swarm has created containers for both web and redis, and placed them on different nodes, which you can check with `docker ps`: - - $ docker ps - CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES - 92faad2135c9 redis "/entrypoint.sh redi 43 seconds ago Up 42 seconds swarm-2/composetest_redis_1 - adb809e5cdac username/counter "/bin/sh -c 'python 55 seconds ago Up 54 seconds 45.67.8.9:80->5000/tcp swarm-1/composetest_web_1 - -You can also see that the web container has exposed port 80 on its swarm node. If you curl that IP, you’ll get a response from the container: - - $ curl http://45.67.8.9 - Hello World! I have been seen 1 times. - -If you hit it repeatedly, the counter will increment, demonstrating that the web and redis container are communicating: - - $ curl http://45.67.8.9 - Hello World! I have been seen 2 times. - $ curl http://45.67.8.9 - Hello World! I have been seen 3 times. - $ curl http://45.67.8.9 - Hello World! I have been seen 4 times. +https://docs.docker.com/compose/networking diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 040a2602..c1834f2f 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -20,18 +20,30 @@ release. As part of this script you'll be asked to: -1. Update the version in `docs/install.md` and `compose/__init__.py`. +1. Update the version in `compose/__init__.py` and `script/run/run.sh`. - If the next release will be an RC, append `rcN`, e.g. `1.4.0rc1`. + If the next release will be an RC, append `-rcN`, e.g. `1.4.0-rc1`. 2. Write release notes in `CHANGES.md`. - Almost every feature enhancement should be mentioned, with the most visible/exciting ones first. Use descriptive sentences and give context where appropriate. + Almost every feature enhancement should be mentioned, with the most + visible/exciting ones first. Use descriptive sentences and give context + where appropriate. - Bug fixes are worth mentioning if it's likely that they've affected lots of people, or if they were regressions in the previous version. + Bug fixes are worth mentioning if it's likely that they've affected lots + of people, or if they were regressions in the previous version. Improvements to the code are not worth mentioning. +3. Create a new repository on [bintray](https://bintray.com/docker-compose). + The name has to match the name of the branch (e.g. `bump-1.9.0`) and the + type should be "Generic". Other fields can be left blank. + +4. Check that the `vnext-compose` branch on + [the docs repo](https://github.com/docker/docker.github.io/) has + documentation for all the new additions in the upcoming release, and create + a PR there for what needs to be amended. + ## When a PR is merged into master that we want in the release @@ -55,10 +67,10 @@ Check out the bump branch and run the `build-binaries` script When prompted build the non-linux binaries and test them. -1. Build the Mac binary in a Mountain Lion VM: +1. Download the osx binary from Bintray. Make sure that the latest Travis + build has finished, otherwise you'll be downloading an old binary. - script/prepare-osx - script/build-osx + https://dl.bintray.com/docker-compose/$BRANCH_NAME/ 2. Download the windows binary from AppVeyor @@ -67,28 +79,30 @@ When prompted build the non-linux binaries and test them. 3. Draft a release from the tag on GitHub (the script will open the window for you) - In the "Tag version" dropdown, select the tag you just pushed. + The tag will only be present on Github when you run the `push-release` + script in step 7, but you can pre-fill it at that point. -4. Paste in installation instructions and release notes. Here's an example - change the Compose version and Docker version as appropriate: +4. Paste in installation instructions and release notes. Here's an example - + change the Compose version and Docker version as appropriate: - Firstly, note that Compose 1.5.0 requires Docker 1.8.0 or later. + If you're a Mac or Windows user, the best way to install Compose and keep it up-to-date is **[Docker for Mac and Windows](https://www.docker.com/products/docker)**. - Secondly, if you're a Mac user, the **[Docker Toolbox](https://www.docker.com/toolbox)** will install Compose 1.5.0 for you, alongside the latest versions of the Docker Engine, Machine and Kitematic. + Note that Compose 1.9.0 requires Docker Engine 1.10.0 or later for version 2 of the Compose File format, and Docker Engine 1.9.1 or later for version 1. Docker for Mac and Windows will automatically install the latest version of Docker Engine for you. - Otherwise, you can use the usual commands to install/upgrade. Either download the binary: + Alternatively, you can use the usual commands to install or upgrade Compose: - curl -L https://github.com/docker/compose/releases/download/1.5.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose - chmod +x /usr/local/bin/docker-compose + ``` + curl -L https://github.com/docker/compose/releases/download/1.9.0/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose + chmod +x /usr/local/bin/docker-compose + ``` - Or install the PyPi package: - - pip install -U docker-compose==1.5.0 + See the [install docs](https://docs.docker.com/compose/install/) for more install options and instructions. Here's what's new: ...release notes go here... -5. Attach the binaries and `script/run.sh` +5. Attach the binaries and `script/run/run.sh` 6. Add "Thanks" with a list of contributors. The contributor list can be generated by running `./script/release/contributors`. @@ -99,6 +113,8 @@ When prompted build the non-linux binaries and test them. ./script/release/push-release +8. Merge the bump PR. + 8. Publish the release on GitHub. 9. Check that all the binaries download (following the install instructions) and run. @@ -107,19 +123,7 @@ When prompted build the non-linux binaries and test them. ## If it’s a stable release (not an RC) -1. Merge the bump PR. - -2. Make sure `origin/release` is updated locally: - - git fetch origin - -3. Update the `docs` branch on the upstream repo: - - git push git@github.com:docker/compose.git origin/release:docs - -4. Let the docs team know that it’s been updated so they can publish it. - -5. Close the release’s milestone. +1. Close the release’s milestone. ## If it’s a minor release (1.x.0), rather than a patch release (1.x.y) diff --git a/requirements-build.txt b/requirements-build.txt index 20aad420..27f610ca 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -pyinstaller==3.0 +pyinstaller==3.2.1 diff --git a/requirements.txt b/requirements.txt index 563baa10..3b06bff4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,11 +1,16 @@ PyYAML==3.11 +backports.ssl-match-hostname==3.5.0.1; python_version < '3' cached-property==1.2.0 -dockerpty==0.3.4 +colorama==0.3.7 +docker==2.0.2 +dockerpty==0.4.1 docopt==0.6.1 -enum34==1.0.4 -git+https://github.com/docker/docker-py.git@master#egg=docker-py +enum34==1.0.4; python_version < '3.4' +functools32==3.2.3.post2; python_version < '3.2' +ipaddress==1.0.16 jsonschema==2.5.1 -requests==2.7.0 -six==1.7.3 +pypiwin32==219; sys_platform == 'win32' +requests==2.11.1 +six==1.10.0 texttable==0.8.4 websocket-client==0.32.0 diff --git a/script/build-image b/script/build-image deleted file mode 100755 index 89733505..00000000 --- a/script/build-image +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e - -if [ -z "$1" ]; then - >&2 echo "First argument must be image tag." - exit 1 -fi - -TAG=$1 -VERSION="$(python setup.py --version)" - -./script/write-git-sha -python setup.py sdist -cp dist/docker-compose-$VERSION.tar.gz dist/docker-compose-release.tar.gz -docker build -t docker/compose:$TAG -f Dockerfile.run . diff --git a/script/build/image b/script/build/image new file mode 100755 index 00000000..3590ce14 --- /dev/null +++ b/script/build/image @@ -0,0 +1,15 @@ +#!/bin/bash + +set -e + +if [ -z "$1" ]; then + >&2 echo "First argument must be image tag." + exit 1 +fi + +TAG=$1 +VERSION="$(python setup.py --version)" + +./script/build/write-git-sha +python setup.py sdist bdist_wheel +docker build --build-arg version=$VERSION -t docker/compose:$TAG -f Dockerfile.run . diff --git a/script/build-linux b/script/build/linux similarity index 76% rename from script/build-linux rename to script/build/linux index 47fb45e1..1a4cd4d9 100755 --- a/script/build-linux +++ b/script/build/linux @@ -7,7 +7,7 @@ set -ex TAG="docker-compose" docker build -t "$TAG" . | tail -n 200 docker run \ - --rm --entrypoint="script/build-linux-inner" \ + --rm --entrypoint="script/build/linux-entrypoint" \ -v $(pwd)/dist:/code/dist \ -v $(pwd)/.git:/code/.git \ "$TAG" diff --git a/script/build-linux-inner b/script/build/linux-entrypoint similarity index 90% rename from script/build-linux-inner rename to script/build/linux-entrypoint index 9bf7c95d..bf515060 100755 --- a/script/build-linux-inner +++ b/script/build/linux-entrypoint @@ -9,7 +9,7 @@ mkdir -p `pwd`/dist chmod 777 `pwd`/dist $VENV/bin/pip install -q -r requirements-build.txt -./script/write-git-sha +./script/build/write-git-sha su -c "$VENV/bin/pyinstaller docker-compose.spec" user mv dist/docker-compose $TARGET $TARGET version diff --git a/script/build-osx b/script/build/osx similarity index 92% rename from script/build-osx rename to script/build/osx index 168fd430..3de34576 100755 --- a/script/build-osx +++ b/script/build/osx @@ -9,7 +9,7 @@ virtualenv -p /usr/local/bin/python venv venv/bin/pip install -r requirements.txt venv/bin/pip install -r requirements-build.txt venv/bin/pip install --no-deps . -./script/write-git-sha +./script/build/write-git-sha venv/bin/pyinstaller docker-compose.spec mv dist/docker-compose dist/docker-compose-Darwin-x86_64 dist/docker-compose-Darwin-x86_64 version diff --git a/script/build-windows.ps1 b/script/build/windows.ps1 similarity index 93% rename from script/build-windows.ps1 rename to script/build/windows.ps1 index 28011b1d..db643274 100644 --- a/script/build-windows.ps1 +++ b/script/build/windows.ps1 @@ -26,7 +26,7 @@ # # 6. Build the binary: # -# .\script\build-windows.ps1 +# .\script\build\windows.ps1 $ErrorActionPreference = "Stop" @@ -41,6 +41,9 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } # Create virtualenv virtualenv .\venv +# pip and pyinstaller generate lots of warnings, so we need to ignore them +$ErrorActionPreference = "Continue" + # Install dependencies .\venv\Scripts\pip install pypiwin32==219 .\venv\Scripts\pip install -r requirements.txt @@ -50,8 +53,6 @@ virtualenv .\venv git rev-parse --short HEAD | out-file -encoding ASCII compose\GITSHA # Build binary -# pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue -$ErrorActionPreference = "Continue" .\venv\Scripts\pyinstaller .\docker-compose.spec $ErrorActionPreference = "Stop" diff --git a/script/write-git-sha b/script/build/write-git-sha similarity index 100% rename from script/write-git-sha rename to script/build/write-git-sha diff --git a/script/ci b/script/ci index f30265c0..7b3489a1 100755 --- a/script/ci +++ b/script/ci @@ -1,21 +1,8 @@ #!/bin/bash -# This should be run inside a container built from the Dockerfile -# at the root of the repo: # -# $ TAG="docker-compose:$(git rev-parse --short HEAD)" -# $ docker build -t "$TAG" . -# $ docker run --rm --volume="/var/run/docker.sock:/var/run/docker.sock" --volume="$(pwd)/.git:/code/.git" -e "TAG=$TAG" --entrypoint="script/ci" "$TAG" - -set -ex - -docker version - -export DOCKER_VERSIONS=all -STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} -export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" - -GIT_VOLUME="--volumes-from=$(hostname)" -. script/test-versions - ->&2 echo "Building Linux binary" -. script/build-linux-inner +# Backwards compatiblity for jenkins +# +# TODO: remove this script after all current PRs and jenkins are updated with +# the new script/test/ci change +set -e +exec script/test/ci diff --git a/script/dev b/script/dev deleted file mode 100755 index 80b3d013..00000000 --- a/script/dev +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -# This is a script for running Compose inside a Docker container. It's handy for -# development. -# -# $ ln -s `pwd`/script/dev /usr/local/bin/docker-compose -# $ cd /a/compose/project -# $ docker-compose up -# - -set -e - -# Follow symbolic links -if [ -h "$0" ]; then - DIR=$(readlink "$0") -else - DIR=$0 -fi -DIR="$(dirname "$DIR")"/.. - -docker build -t docker-compose $DIR -exec docker run -i -t -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`:`pwd` -w `pwd` docker-compose $@ diff --git a/script/release/build-binaries b/script/release/build-binaries index 083f8eb5..9d4a606e 100755 --- a/script/release/build-binaries +++ b/script/release/build-binaries @@ -22,16 +22,16 @@ REPO=docker/compose # Build the binaries script/clean -script/build-linux -# TODO: build osx binary -# script/prepare-osx -# script/build-osx -# TODO: build or fetch the windows binary -echo "You need to build the osx/windows binaries, that step is not automated yet." +script/build/linux echo "Building the container distribution" -script/build-image $VERSION +script/build/image $VERSION echo "Create a github release" # TODO: script more of this https://developer.github.com/v3/repos/releases/ browser https://github.com/$REPO/releases/new + +echo "Don't forget to download the osx and windows binaries from appveyor/bintray\!" +echo "https://dl.bintray.com/docker-compose/$BRANCH/" +echo "https://ci.appveyor.com/project/docker/compose" +echo diff --git a/script/release/contributors b/script/release/contributors index bb9fe871..4657dd80 100755 --- a/script/release/contributors +++ b/script/release/contributors @@ -15,13 +15,16 @@ EOM [[ -n "$1" ]] || usage PREV_RELEASE=$1 -VERSION=HEAD +BRANCH="$(git rev-parse --abbrev-ref HEAD)" URL="https://api.github.com/repos/docker/compose/compare" -curl -sf "$URL/$PREV_RELEASE...$VERSION" | \ +contribs=$(curl -sf "$URL/$PREV_RELEASE...$BRANCH" | \ jq -r '.commits[].author.login' | \ sort | \ uniq -c | \ - sort -nr | \ - awk '{print "@"$2","}' | \ - xargs echo + sort -nr) + +echo "Contributions by user: " +echo "$contribs" +echo +echo "$contribs" | awk '{print "@"$2","}' | xargs diff --git a/script/release/make-branch b/script/release/make-branch index 48fa771b..7ccf3f05 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -63,15 +63,17 @@ git merge --strategy=ours --no-edit $REMOTE/release git config "branch.${BRANCH}.release" $VERSION -echo "Update versions in docs/install.md, compose/__init__.py, script/run.sh" -$EDITOR docs/install.md -$EDITOR compose/__init__.py -$EDITOR script/run.sh +editor=${EDITOR:-vim} + +echo "Update versions in docs/install.md, compose/__init__.py, script/run/run.sh" +$editor docs/install.md +$editor compose/__init__.py +$editor script/run/run.sh echo "Write release notes in CHANGELOG.md" browser "https://github.com/docker/compose/issues?q=milestone%3A$VERSION+is%3Aclosed" -$EDITOR CHANGELOG.md +$editor CHANGELOG.md git diff @@ -80,20 +82,6 @@ $SHELL || true git commit -a -m "Bump $VERSION" --signoff --no-verify -echo "Push branch to user remote" -GITHUB_USER=$USER -USER_REMOTE="$(find_remote $GITHUB_USER/compose)" -if [ -z "$USER_REMOTE" ]; then - echo "No user remote found for $GITHUB_USER" - read -r -p "Enter the name of your github user: " GITHUB_USER - # assumes there is already a user remote somewhere - USER_REMOTE=$(find_remote $GITHUB_USER/compose) -fi -if [ -z "$USER_REMOTE" ]; then - >&2 echo "No user remote found. You need to 'git push' your branch." - exit 2 -fi - - -git push $USER_REMOTE -browser https://github.com/$REPO/compare/docker:release...$GITHUB_USER:$BRANCH?expand=1 +echo "Push branch to docker remote" +git push $REMOTE +browser https://github.com/$REPO/compare/docker:release...$BRANCH?expand=1 diff --git a/script/release/push-release b/script/release/push-release index b754d40f..9db6f689 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -54,18 +54,19 @@ git push $GITHUB_REPO $VERSION echo "Uploading the docker image" docker push docker/compose:$VERSION -echo "Uploading sdist to pypi" +echo "Uploading package to PyPI" pandoc -f markdown -t rst README.md -o README.rst sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst -./script/write-git-sha -python setup.py sdist +./script/build/write-git-sha +python setup.py sdist bdist_wheel if [ "$(command -v twine 2> /dev/null)" ]; then - twine upload ./dist/docker-compose-${VERSION}.tar.gz + twine upload ./dist/docker-compose-${VERSION/-/}.tar.gz ./dist/docker_compose-${VERSION/-/}-py2.py3-none-any.whl else python setup.py upload fi echo "Testing pip package" +deactivate || true virtualenv venv-test source venv-test/bin/activate pip install docker-compose==$VERSION diff --git a/script/release/rebase-bump-commit b/script/release/rebase-bump-commit index 23877bb5..3c2ae72b 100755 --- a/script/release/rebase-bump-commit +++ b/script/release/rebase-bump-commit @@ -32,7 +32,7 @@ if [[ "$sha" == "$(git rev-parse HEAD)" ]]; then exit 0 fi -commits=$(git log --format="%H" "$sha..HEAD" | wc -l) +commits=$(git log --format="%H" "$sha..HEAD" | wc -l | xargs echo) git rebase --onto $sha~1 HEAD~$commits $BRANCH git cherry-pick $sha diff --git a/script/release/utils.sh b/script/release/utils.sh index b4e5a2e6..321c1fb7 100644 --- a/script/release/utils.sh +++ b/script/release/utils.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Util functions for release scritps +# Util functions for release scripts # set -e diff --git a/script/run.ps1 b/script/run/run.ps1 similarity index 100% rename from script/run.ps1 rename to script/run/run.ps1 diff --git a/script/run.sh b/script/run/run.sh similarity index 80% rename from script/run.sh rename to script/run/run.sh index 087c2692..4e173894 100755 --- a/script/run.sh +++ b/script/run/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.2" +VERSION="1.12.0dev" IMAGE="docker/compose:$VERSION" @@ -31,9 +31,11 @@ fi # Setup volume mounts for compose config and context -VOLUMES="-v $(pwd):$(pwd)" +if [ "$(pwd)" != '/' ]; then + VOLUMES="-v $(pwd):$(pwd)" +fi if [ -n "$COMPOSE_FILE" ]; then - compose_dir=$(dirname $COMPOSE_FILE) + compose_dir=$(realpath $(dirname $COMPOSE_FILE)) fi # TODO: also check --file argument if [ -n "$compose_dir" ]; then @@ -45,9 +47,10 @@ fi # Only allocate tty if we detect one if [ -t 1 ]; then - DOCKER_RUN_OPTIONS="-ti" -else - DOCKER_RUN_OPTIONS="-i" + DOCKER_RUN_OPTIONS="-t" +fi +if [ -t 0 ]; then + DOCKER_RUN_OPTIONS="$DOCKER_RUN_OPTIONS -i" fi -exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w $(pwd) $IMAGE $@ +exec docker run --rm $DOCKER_RUN_OPTIONS $DOCKER_ADDR $COMPOSE_OPTIONS $VOLUMES -w "$(pwd)" $IMAGE "$@" diff --git a/script/prepare-osx b/script/setup/osx similarity index 77% rename from script/prepare-osx rename to script/setup/osx index 10bbbecc..e6ab62a8 100755 --- a/script/prepare-osx +++ b/script/setup/osx @@ -10,13 +10,13 @@ openssl_version() { python -c "import ssl; print ssl.OPENSSL_VERSION" } -desired_python_version="2.7.9" -desired_python_brew_version="2.7.9" -python_formula="https://raw.githubusercontent.com/Homebrew/homebrew/1681e193e4d91c9620c4901efd4458d9b6fcda8e/Library/Formula/python.rb" +desired_python_version="2.7.12" +desired_python_brew_version="2.7.12" +python_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/737a2e34a89b213c1f0a2a24fc1a3c06635eed04/Formula/python.rb" -desired_openssl_version="1.0.1j" -desired_openssl_brew_version="1.0.1j_1" -openssl_formula="https://raw.githubusercontent.com/Homebrew/homebrew/62fc2a1a65e83ba9dbb30b2e0a2b7355831c714b/Library/Formula/openssl.rb" +desired_openssl_version="1.0.2j" +desired_openssl_brew_version="1.0.2j" +openssl_formula="https://raw.githubusercontent.com/Homebrew/homebrew-core/30d3766453347f6e22b3ed6c74bb926d6def2eb5/Formula/openssl.rb" PATH="/usr/local/bin:$PATH" diff --git a/script/shell b/script/shell deleted file mode 100755 index 903be76f..00000000 --- a/script/shell +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -ex -docker build -t docker-compose . -exec docker run -v /var/run/docker.sock:/var/run/docker.sock -v `pwd`:/code -ti --rm --entrypoint bash docker-compose diff --git a/script/test-versions b/script/test/all similarity index 84% rename from script/test-versions rename to script/test/all index 24412b91..7151a75e 100755 --- a/script/test-versions +++ b/script/test/all @@ -6,6 +6,7 @@ set -e >&2 echo "Running lint checks" docker run --rm \ + --tty \ ${GIT_VOLUME} \ --entrypoint="tox" \ "$TAG" -e pre-commit @@ -13,16 +14,17 @@ docker run --rm \ get_versions="docker run --rm --entrypoint=/code/.tox/py27/bin/python $TAG - /code/script/versions.py docker/docker" + /code/script/test/versions.py docker/docker" if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="1.9.1 1.10.0-dev" + DOCKER_VERSIONS=$($get_versions -n 2 recent) fi BUILD_NUMBER=${BUILD_NUMBER-$USER} +PY_TEST_VERSIONS=${PY_TEST_VERSIONS:-py27,py34} for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" @@ -38,11 +40,7 @@ for version in $DOCKER_VERSIONS; do trap "on_exit" EXIT - if [[ $version == *"-dev" ]]; then - repo="dnephin/dind" - else - repo="dockerswarm/dind" - fi + repo="dockerswarm/dind" docker run \ -d \ @@ -55,11 +53,12 @@ for version in $DOCKER_VERSIONS; do docker run \ --rm \ + --tty \ --link="$daemon_container:docker" \ --env="DOCKER_HOST=tcp://docker:2375" \ --env="DOCKER_VERSION=$version" \ --entrypoint="tox" \ "$TAG" \ - -e py27,py34 -- "$@" + -e "$PY_TEST_VERSIONS" -- "$@" done diff --git a/script/test/ci b/script/test/ci new file mode 100755 index 00000000..c5927b2c --- /dev/null +++ b/script/test/ci @@ -0,0 +1,25 @@ +#!/bin/bash +# This should be run inside a container built from the Dockerfile +# at the root of the repo: +# +# $ TAG="docker-compose:$(git rev-parse --short HEAD)" +# $ docker build -t "$TAG" . +# $ docker run --rm \ +# --volume="/var/run/docker.sock:/var/run/docker.sock" \ +# --volume="$(pwd)/.git:/code/.git" \ +# -e "TAG=$TAG" \ +# --entrypoint="script/test/ci" "$TAG" + +set -ex + +docker version + +export DOCKER_VERSIONS=all +STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} +export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" + +GIT_VOLUME="--volumes-from=$(hostname)" +. script/test/all + +>&2 echo "Building Linux binary" +. script/build/linux-entrypoint diff --git a/script/test b/script/test/default similarity index 92% rename from script/test rename to script/test/default index bdb3579b..fa741a19 100755 --- a/script/test +++ b/script/test/default @@ -12,4 +12,4 @@ mkdir -p coverage-html docker build -t "$TAG" . GIT_VOLUME="--volume=$(pwd)/.git:/code/.git" -. script/test-versions +. script/test/all diff --git a/script/versions.py b/script/test/versions.py similarity index 91% rename from script/versions.py rename to script/test/versions.py index 98f97ef3..0c3b8162 100755 --- a/script/versions.py +++ b/script/test/versions.py @@ -5,7 +5,7 @@ version tags for recent releases, or the default release. The default release is the most recent non-RC version. -Recent is a list of unqiue major.minor versions, where each is the most +Recent is a list of unique major.minor versions, where each is the most recent version in the series. For example, if the list of versions is: @@ -28,6 +28,7 @@ from __future__ import unicode_literals import argparse import itertools import operator +import sys from collections import namedtuple import requests @@ -103,6 +104,14 @@ def get_default(versions): return version +def get_versions(tags): + for tag in tags: + try: + yield Version.parse(tag['name']) + except ValueError: + print("Skipping invalid tag: {name}".format(**tag), file=sys.stderr) + + def get_github_releases(project): """Query the Github API for a list of version tags and return them in sorted order. @@ -112,7 +121,7 @@ def get_github_releases(project): url = '{}/{}/tags'.format(GITHUB_API, project) response = requests.get(url) response.raise_for_status() - versions = [Version.parse(tag['name']) for tag in response.json()] + versions = get_versions(response.json()) return sorted(versions, reverse=True, key=operator.attrgetter('order')) diff --git a/script/travis/bintray.json.tmpl b/script/travis/bintray.json.tmpl index 7d0adbeb..f9728558 100644 --- a/script/travis/bintray.json.tmpl +++ b/script/travis/bintray.json.tmpl @@ -1,7 +1,7 @@ { "package": { "name": "${TRAVIS_OS_NAME}", - "repo": "master", + "repo": "${TRAVIS_BRANCH}", "subject": "docker-compose", "desc": "Automated build of master branch from travis ci.", "website_url": "https://github.com/docker/compose", @@ -11,8 +11,8 @@ }, "version": { - "name": "master", - "desc": "Automated build of the master branch.", + "name": "${TRAVIS_BRANCH}", + "desc": "Automated build of the ${TRAVIS_BRANCH} branch.", "released": "${DATE}", "vcs_tag": "master" }, diff --git a/script/travis/build-binary b/script/travis/build-binary index 7cc1092d..7707a1ee 100755 --- a/script/travis/build-binary +++ b/script/travis/build-binary @@ -3,11 +3,11 @@ set -ex if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then - script/build-linux + script/build/linux # TODO: requires auth to push, so disable for now - # script/build-image master + # script/build/image master # docker push docker/compose:master else - script/prepare-osx - script/build-osx + script/setup/osx + script/build/osx fi diff --git a/script/travis/ci b/script/travis/ci index 4cce1bc8..cd4fcc6d 100755 --- a/script/travis/ci +++ b/script/travis/ci @@ -6,5 +6,5 @@ if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then tox -e py27,py34 -- tests/unit else # TODO: we could also install py34 and test against it - python -m tox -e py27 -- tests/unit + tox -e py27 -- tests/unit fi diff --git a/script/travis/install b/script/travis/install index a23667bf..d4b34786 100755 --- a/script/travis/install +++ b/script/travis/install @@ -5,5 +5,6 @@ set -ex if [[ "$TRAVIS_OS_NAME" == "linux" ]]; then pip install tox==2.1.1 else - pip install --user tox==2.1.1 + sudo pip install --upgrade pip tox==2.1.1 virtualenv + pip --version fi diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 00000000..3c6e79cf --- /dev/null +++ b/setup.cfg @@ -0,0 +1,2 @@ +[bdist_wheel] +universal=1 diff --git a/setup.py b/setup.py index b365e05b..eafbc356 100644 --- a/setup.py +++ b/setup.py @@ -1,6 +1,7 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- from __future__ import absolute_import +from __future__ import print_function from __future__ import unicode_literals import codecs @@ -8,6 +9,7 @@ import os import re import sys +import pkg_resources from setuptools import find_packages from setuptools import setup @@ -29,13 +31,14 @@ def find_version(*file_paths): install_requires = [ 'cached-property >= 1.2.0, < 2', + 'colorama >= 0.3.7, < 0.4', 'docopt >= 0.6.1, < 0.7', 'PyYAML >= 3.10, < 4', - 'requests >= 2.6.1, < 2.8', + 'requests >= 2.6.1, != 2.11.0, < 2.12', 'texttable >= 0.8.1, < 0.9', 'websocket-client >= 0.32.0, < 1.0', - 'docker-py >= 1.5.0, < 2', - 'dockerpty >= 0.3.4, < 0.4', + 'docker >= 2.0.2, < 3.0', + 'dockerpty >= 0.4.1, < 0.5', 'six >= 1.3.0, < 2', 'jsonschema >= 2.5.1, < 3', ] @@ -48,7 +51,25 @@ tests_require = [ if sys.version_info[:2] < (3, 4): tests_require.append('mock >= 1.0.1') - install_requires.append('enum34 >= 1.0.4, < 2') + +extras_require = { + ':python_version < "3.4"': ['enum34 >= 1.0.4, < 2'], + ':python_version < "3.5"': ['backports.ssl_match_hostname >= 3.5'], + ':python_version < "3.3"': ['ipaddress >= 1.0.16'], +} + + +try: + if 'bdist_wheel' not in sys.argv: + for key, value in extras_require.items(): + if key.startswith(':') and pkg_resources.evaluate_marker(key[1:]): + install_requires.extend(value) +except Exception as e: + print("Failed to compute platform dependencies: {}. ".format(e) + + "All dependencies will be installed as a result.", file=sys.stderr) + for key, value in extras_require.items(): + if key.startswith(':'): + install_requires.extend(value) setup( @@ -62,6 +83,7 @@ setup( include_package_data=True, test_suite='nose.collector', install_requires=install_requires, + extras_require=extras_require, tests_require=tests_require, entry_points=""" [console_scripts] diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index 231b78db..8366ca75 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -1,26 +1,33 @@ +# -*- coding: utf-8 -*- from __future__ import absolute_import from __future__ import unicode_literals import datetime import json import os -import shlex import signal import subprocess import time +from collections import Counter from collections import namedtuple from operator import attrgetter +import py +import six import yaml from docker import errors from .. import mock from compose.cli.command import get_project from compose.container import Container +from compose.project import OneOffFilter +from compose.utils import nanoseconds_from_time_seconds from tests.integration.testcases import DockerClientTestCase from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox +from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only +from tests.integration.testcases import v3_only ProcessResult = namedtuple('ProcessResult', 'stdout stderr') @@ -43,12 +50,13 @@ def start_process(base_dir, options): def wait_on_process(proc, returncode=0): stdout, stderr = proc.communicate() if proc.returncode != returncode: - print(stderr.decode('utf-8')) + print("Stderr: {}".format(stderr)) + print("Stdout: {}".format(stdout)) assert proc.returncode == returncode return ProcessResult(stdout.decode('utf-8'), stderr.decode('utf-8')) -def wait_on_condition(condition, delay=0.1, timeout=20): +def wait_on_condition(condition, delay=0.1, timeout=40): start_time = time.time() while not condition(): if time.time() - start_time > timeout: @@ -76,22 +84,20 @@ class ContainerCountCondition(object): class ContainerStateCondition(object): - def __init__(self, client, name, running): + def __init__(self, client, name, status): self.client = client self.name = name - self.running = running + self.status = status - # State.Running == true def __call__(self): try: container = self.client.inspect_container(self.name) - return container['State']['Running'] == self.running + return container['State']['Status'] == self.status except errors.APIError: return False def __str__(self): - state = 'running' if self.running else 'stopped' - return "waiting for container to be %s" % state + return "waiting for container to be %s" % self.status class CLITestCase(DockerClientTestCase): @@ -105,13 +111,15 @@ class CLITestCase(DockerClientTestCase): self.project.kill() self.project.remove_stopped() - for container in self.project.containers(stopped=True, one_off=True): + for container in self.project.containers(stopped=True, one_off=OneOffFilter.only): container.remove(force=True) networks = self.client.networks() for n in networks: if n['Name'].startswith('{}_'.format(self.project.name)): self.client.remove_network(n['Name']) + if hasattr(self, '_project'): + del self._project super(CLITestCase, self).tearDown() @@ -139,36 +147,48 @@ class CLITestCase(DockerClientTestCase): def test_help(self): self.base_dir = 'tests/fixtures/no-composefile' - result = self.dispatch(['help', 'up'], returncode=1) - assert 'Usage: up [options] [SERVICE...]' in result.stderr + result = self.dispatch(['help', 'up'], returncode=0) + assert 'Usage: up [options] [SERVICE...]' in result.stdout # Prevent tearDown from trying to create a project self.base_dir = None - # TODO: this shouldn't be v2-dependent - @v2_only() + def test_shorthand_host_opt(self): + self.dispatch( + ['-H={0}'.format(os.environ.get('DOCKER_HOST', 'unix://')), + 'up', '-d'], + returncode=0 + ) + + def test_host_not_reachable(self): + result = self.dispatch(['-H=tcp://doesnotexist:8000', 'ps'], returncode=1) + assert "Couldn't connect to Docker daemon" in result.stderr + + def test_host_not_reachable_volumes_from_container(self): + self.base_dir = 'tests/fixtures/volumes-from-container' + + container = self.client.create_container('busybox', 'true', name='composetest_data_container') + self.addCleanup(self.client.remove_container, container) + + result = self.dispatch(['-H=tcp://doesnotexist:8000', 'ps'], returncode=1) + assert "Couldn't connect to Docker daemon" in result.stderr + def test_config_list_services(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config', '--services']) assert set(result.stdout.rstrip().split('\n')) == {'web', 'other'} - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_quiet_with_error(self): self.base_dir = None result = self.dispatch([ '-f', 'tests/fixtures/invalid-composefile/invalid.yml', 'config', '-q' ], returncode=1) - assert "'notaservice' doesn't have any configuration" in result.stderr + assert "'notaservice' must be a mapping" in result.stderr - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_quiet(self): self.base_dir = 'tests/fixtures/v2-full' assert self.dispatch(['config', '-q']).stdout == '' - # TODO: this shouldn't be v2-dependent - @v2_only() def test_config_default(self): self.base_dir = 'tests/fixtures/v2-full' result = self.dispatch(['config']) @@ -177,7 +197,7 @@ class CLITestCase(DockerClientTestCase): output = yaml.load(result.stdout) expected = { - 'version': 2, + 'version': '2.0', 'volumes': {'data': {'driver': 'local'}}, 'networks': {'front': {}}, 'services': { @@ -185,7 +205,7 @@ class CLITestCase(DockerClientTestCase): 'build': { 'context': os.path.abspath(self.base_dir), }, - 'networks': ['front', 'default'], + 'networks': {'front': None, 'default': None}, 'volumes_from': ['service:other:rw'], }, 'other': { @@ -197,6 +217,152 @@ class CLITestCase(DockerClientTestCase): } assert output == expected + def test_config_restart(self): + self.base_dir = 'tests/fixtures/restart' + result = self.dispatch(['config']) + assert yaml.load(result.stdout) == { + 'version': '2.0', + 'services': { + 'never': { + 'image': 'busybox', + 'restart': 'no', + }, + 'always': { + 'image': 'busybox', + 'restart': 'always', + }, + 'on-failure': { + 'image': 'busybox', + 'restart': 'on-failure', + }, + 'on-failure-5': { + 'image': 'busybox', + 'restart': 'on-failure:5', + }, + 'restart-null': { + 'image': 'busybox', + 'restart': '' + }, + }, + 'networks': {}, + 'volumes': {}, + } + + def test_config_external_network(self): + self.base_dir = 'tests/fixtures/networks' + result = self.dispatch(['-f', 'external-networks.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'networks' in json_result + assert json_result['networks'] == { + 'networks_foo': { + 'external': True # {'name': 'networks_foo'} + }, + 'bar': { + 'external': {'name': 'networks_bar'} + } + } + + def test_config_external_volume(self): + self.base_dir = 'tests/fixtures/volumes' + result = self.dispatch(['-f', 'external-volumes.yml', 'config']) + json_result = yaml.load(result.stdout) + assert 'volumes' in json_result + assert json_result['volumes'] == { + 'foo': { + 'external': True + }, + 'bar': { + 'external': {'name': 'some_bar'} + } + } + + def test_config_v1(self): + self.base_dir = 'tests/fixtures/v1-config' + result = self.dispatch(['config']) + assert yaml.load(result.stdout) == { + 'version': '2.1', + 'services': { + 'net': { + 'image': 'busybox', + 'network_mode': 'bridge', + }, + 'volume': { + 'image': 'busybox', + 'volumes': ['/data:rw'], + 'network_mode': 'bridge', + }, + 'app': { + 'image': 'busybox', + 'volumes_from': ['service:volume:rw'], + 'network_mode': 'service:net', + }, + }, + 'networks': {}, + 'volumes': {}, + } + + @v3_only() + def test_config_v3(self): + self.base_dir = 'tests/fixtures/v3-full' + result = self.dispatch(['config']) + + assert yaml.load(result.stdout) == { + 'version': '3.0', + 'networks': {}, + 'volumes': { + 'foobar': { + 'labels': { + 'com.docker.compose.test': 'true', + }, + }, + }, + 'services': { + 'web': { + 'image': 'busybox', + 'deploy': { + 'mode': 'replicated', + 'replicas': 6, + 'labels': ['FOO=BAR'], + 'update_config': { + 'parallelism': 3, + 'delay': '10s', + 'failure_action': 'continue', + 'monitor': '60s', + 'max_failure_ratio': 0.3, + }, + 'resources': { + 'limits': { + 'cpus': '0.001', + 'memory': '50M', + }, + 'reservations': { + 'cpus': '0.0001', + 'memory': '20M', + }, + }, + 'restart_policy': { + 'condition': 'on_failure', + 'delay': '5s', + 'max_attempts': 3, + 'window': '120s', + }, + 'placement': { + 'constraints': ['node=foo'], + }, + }, + + 'healthcheck': { + 'test': 'cat /etc/passwd', + 'interval': '10s', + 'timeout': '1s', + 'retries': 5, + }, + + 'stop_grace_period': '20s', + }, + }, + } + def test_ps(self): self.project.get_service('simple').create_container() result = self.dispatch(['ps']) @@ -242,12 +408,13 @@ class CLITestCase(DockerClientTestCase): def test_pull_with_ignore_pull_failures(self): result = self.dispatch([ '-f', 'ignore-pull-failures.yml', - 'pull', '--ignore-pull-failures']) + 'pull', '--ignore-pull-failures'] + ) assert 'Pulling simple (busybox:latest)...' in result.stderr assert 'Pulling another (nonexisting-image:latest)...' in result.stderr - assert 'Error: image library/nonexisting-image' in result.stderr - assert 'not found' in result.stderr + assert ('repository nonexisting-image not found' in result.stderr or + 'image library/nonexisting-image:latest not found' in result.stderr) def test_build_plain(self): self.base_dir = 'tests/fixtures/simple-dockerfile' @@ -312,6 +479,32 @@ class CLITestCase(DockerClientTestCase): ] assert not containers + def test_bundle_with_digests(self): + self.base_dir = 'tests/fixtures/bundle-with-digests/' + tmpdir = py.test.ensuretemp('cli_test_bundle') + self.addCleanup(tmpdir.remove) + filename = str(tmpdir.join('example.dab')) + + self.dispatch(['bundle', '--output', filename]) + with open(filename, 'r') as fh: + bundle = json.load(fh) + + assert bundle == { + 'Version': '0.1', + 'Services': { + 'web': { + 'Image': ('dockercloud/hello-world@sha256:fe79a2cfbd17eefc3' + '44fb8419420808df95a1e22d93b7f621a7399fd1e9dca1d'), + 'Networks': ['default'], + }, + 'redis': { + 'Image': ('redis@sha256:a84cb8f53a70e19f61ff2e1d5e73fb7ae62d' + '374b2b7392de1e7d77be26ef8f7b'), + 'Networks': ['default'], + } + }, + } + def test_create(self): self.dispatch(['create']) service = self.project.get_service('simple') @@ -365,14 +558,22 @@ class CLITestCase(DockerClientTestCase): @v2_only() def test_down(self): self.base_dir = 'tests/fixtures/v2-full' + self.dispatch(['up', '-d']) wait_on_condition(ContainerCountCondition(self.project, 2)) + self.dispatch(['run', 'web', 'true']) + self.dispatch(['run', '-d', 'web', 'tail', '-f', '/dev/null']) + assert len(self.project.containers(one_off=OneOffFilter.only, stopped=True)) == 2 + result = self.dispatch(['down', '--rmi=local', '--volumes']) assert 'Stopping v2full_web_1' in result.stderr assert 'Stopping v2full_other_1' in result.stderr + assert 'Stopping v2full_web_run_2' in result.stderr assert 'Removing v2full_web_1' in result.stderr assert 'Removing v2full_other_1' in result.stderr + assert 'Removing v2full_web_run_1' in result.stderr + assert 'Removing v2full_web_run_2' in result.stderr assert 'Removing volume v2full_data' in result.stderr assert 'Removing image v2full_web' in result.stderr assert 'Removing image busybox' not in result.stderr @@ -396,8 +597,10 @@ class CLITestCase(DockerClientTestCase): self.base_dir = 'tests/fixtures/echo-services' result = self.dispatch(['up', '--no-color']) - assert 'simple_1 | simple' in result.stdout - assert 'another_1 | another' in result.stdout + assert 'simple_1 | simple' in result.stdout + assert 'another_1 | another' in result.stdout + assert 'simple_1 exited with code 0' in result.stdout + assert 'another_1 exited with code 0' in result.stdout @v2_only() def test_up(self): @@ -406,9 +609,11 @@ class CLITestCase(DockerClientTestCase): services = self.project.get_services() - networks = self.client.networks(names=[self.project.default_network.full_name]) + network_name = self.project.networks.networks['default'].full_name + networks = self.client.networks(names=[network_name]) self.assertEqual(len(networks), 1) self.assertEqual(networks[0]['Driver'], 'bridge') + assert 'com.docker.network.bridge.enable_icc' not in networks[0]['Options'] network = self.client.inspect_network(networks[0]['Id']) @@ -419,12 +624,100 @@ class CLITestCase(DockerClientTestCase): container = containers[0] self.assertIn(container.id, network['Containers']) - networks = list(container.get('NetworkSettings.Networks')) - self.assertEqual(networks, [network['Name']]) + networks = container.get('NetworkSettings.Networks') + self.assertEqual(list(networks), [network['Name']]) + + self.assertEqual( + sorted(networks[network['Name']]['Aliases']), + sorted([service.name, container.short_id])) for service in services: assert self.lookup(container, service.name) + @v2_only() + def test_up_with_default_network_config(self): + filename = 'default-network-config.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + self.dispatch(['-f', filename, 'up', '-d'], None) + + network_name = self.project.networks.networks['default'].full_name + networks = self.client.networks(names=[network_name]) + + assert networks[0]['Options']['com.docker.network.bridge.enable_icc'] == 'false' + + @v2_only() + def test_up_with_network_aliases(self): + filename = 'network-aliases.yml' + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['-f', filename, 'up', '-d'], None) + back_name = '{}_back'.format(self.project.name) + front_name = '{}_front'.format(self.project.name) + + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + + # Two networks were created: back and front + assert sorted(n['Name'] for n in networks) == [back_name, front_name] + web_container = self.project.get_service('web').containers()[0] + + back_aliases = web_container.get( + 'NetworkSettings.Networks.{}.Aliases'.format(back_name) + ) + assert 'web' in back_aliases + front_aliases = web_container.get( + 'NetworkSettings.Networks.{}.Aliases'.format(front_name) + ) + assert 'web' in front_aliases + assert 'forward_facing' in front_aliases + assert 'ahead' in front_aliases + + @v2_only() + def test_up_with_network_internal(self): + self.require_api_version('1.23') + filename = 'network-internal.yml' + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['-f', filename, 'up', '-d'], None) + internal_net = '{}_internal'.format(self.project.name) + + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + + # One network was created: internal + assert sorted(n['Name'] for n in networks) == [internal_net] + + assert networks[0]['Internal'] is True + + @v2_only() + def test_up_with_network_static_addresses(self): + filename = 'network-static-addresses.yml' + ipv4_address = '172.16.100.100' + ipv6_address = 'fe80::1001:100' + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['-f', filename, 'up', '-d'], None) + static_net = '{}_static_test'.format(self.project.name) + + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + + # One networks was created: front + assert sorted(n['Name'] for n in networks) == [static_net] + web_container = self.project.get_service('web').containers()[0] + + ipam_config = web_container.get( + 'NetworkSettings.Networks.{}.IPAMConfig'.format(static_net) + ) + assert ipv4_address in ipam_config.values() + assert ipv6_address in ipam_config.values() + @v2_only() def test_up_with_networks(self): self.base_dir = 'tests/fixtures/networks' @@ -448,6 +741,10 @@ class CLITestCase(DockerClientTestCase): app_container = self.project.get_service('app').containers()[0] db_container = self.project.get_service('db').containers()[0] + for net_name in [front_name, back_name]: + links = app_container.get('NetworkSettings.Networks.{}.Links'.format(net_name)) + assert '{}:database'.format(db_container.name) in links + # db and app joined the back network assert sorted(back_network['Containers']) == sorted([db_container.id, app_container.id]) @@ -461,6 +758,9 @@ class CLITestCase(DockerClientTestCase): # app can see db assert self.lookup(app_container, "db") + # app has aliased db to "database" + assert self.lookup(app_container, "database") + @v2_only() def test_up_missing_network(self): self.base_dir = 'tests/fixtures/networks' @@ -472,8 +772,13 @@ class CLITestCase(DockerClientTestCase): assert 'Service "web" uses an undefined network "foo"' in result.stderr @v2_only() - def test_up_predefined_networks(self): - filename = 'predefined-networks.yml' + def test_up_with_network_mode(self): + c = self.client.create_container('busybox', 'top', name='composetest_network_mode_container') + self.addCleanup(self.client.remove_container, c, force=True) + self.client.start(c) + container_mode_source = 'container:{}'.format(c['Id']) + + filename = 'network-mode.yml' self.base_dir = 'tests/fixtures/networks' self._project = get_project(self.base_dir, [filename]) @@ -491,6 +796,16 @@ class CLITestCase(DockerClientTestCase): assert list(container.get('NetworkSettings.Networks')) == [name] assert container.get('HostConfig.NetworkMode') == name + service_mode_source = 'container:{}'.format( + self.project.get_service('bridge').containers()[0].id) + service_mode_container = self.project.get_service('service').containers()[0] + assert not service_mode_container.get('NetworkSettings.Networks') + assert service_mode_container.get('HostConfig.NetworkMode') == service_mode_source + + container_mode_container = self.project.get_service('container').containers()[0] + assert not container_mode_container.get('NetworkSettings.Networks') + assert container_mode_container.get('HostConfig.NetworkMode') == container_mode_source + @v2_only() def test_up_external_networks(self): filename = 'external-networks.yml' @@ -515,6 +830,69 @@ class CLITestCase(DockerClientTestCase): container = self.project.containers()[0] assert sorted(list(container.get('NetworkSettings.Networks'))) == sorted(network_names) + @v2_only() + def test_up_with_external_default_network(self): + filename = 'external-default.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + result = self.dispatch(['-f', filename, 'up', '-d'], returncode=1) + assert 'declared as external, but could not be found' in result.stderr + + networks = [ + n['Name'] for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + assert not networks + + network_name = 'composetest_external_network' + self.client.create_network(network_name) + + self.dispatch(['-f', filename, 'up', '-d']) + container = self.project.containers()[0] + assert list(container.get('NetworkSettings.Networks')) == [network_name] + + @v2_1_only() + def test_up_with_network_labels(self): + filename = 'network-label.yml' + + self.base_dir = 'tests/fixtures/networks' + self._project = get_project(self.base_dir, [filename]) + + self.dispatch(['-f', filename, 'up', '-d'], returncode=0) + + network_with_label = '{}_network_with_label'.format(self.project.name) + + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('{}_'.format(self.project.name)) + ] + + assert [n['Name'] for n in networks] == [network_with_label] + assert 'label_key' in networks[0]['Labels'] + assert networks[0]['Labels']['label_key'] == 'label_val' + + @v2_1_only() + def test_up_with_volume_labels(self): + filename = 'volume-label.yml' + + self.base_dir = 'tests/fixtures/volumes' + self._project = get_project(self.base_dir, [filename]) + + self.dispatch(['-f', filename, 'up', '-d'], returncode=0) + + volume_with_label = '{}_volume_with_label'.format(self.project.name) + + volumes = [ + v for v in self.client.volumes().get('Volumes', []) + if v['Name'].startswith('{}_'.format(self.project.name)) + ] + + assert [v['Name'] for v in volumes] == [volume_with_label] + assert 'label_key' in volumes[0]['Labels'] + assert volumes[0]['Labels']['label_key'] == 'label_val' + @v2_only() def test_up_no_services(self): self.base_dir = 'tests/fixtures/no-services' @@ -524,30 +902,15 @@ class CLITestCase(DockerClientTestCase): n['Name'] for n in self.client.networks() if n['Name'].startswith('{}_'.format(self.project.name)) ] - - assert sorted(network_names) == [ - '{}_{}'.format(self.project.name, name) - for name in ['bar', 'foo'] - ] - - @v2_only() - def test_up_with_links_is_invalid(self): - self.base_dir = 'tests/fixtures/v2-simple' - - result = self.dispatch( - ['-f', 'links-invalid.yml', 'up', '-d'], - returncode=1) - - # TODO: fix validation error messages for v2 files - # assert "Unsupported config option for service 'simple': 'links'" in result.stderr - assert "Unsupported config option" in result.stderr + assert network_names == [] def test_up_with_links_v1(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', 'web'], None) # No network was created - networks = self.client.networks(names=[self.project.default_network.full_name]) + network_name = self.project.networks.networks['default'].full_name + networks = self.client.networks(names=[network_name]) assert networks == [] web = self.project.get_service('web') @@ -570,9 +933,7 @@ class CLITestCase(DockerClientTestCase): ['-f', 'v2-invalid.yml', 'up', '-d'], returncode=1) - # TODO: fix validation error messages for v2 files - # assert "Unsupported config option for service 'web': 'net'" in exc.exconly() - assert "Unsupported config option" in result.stderr + assert "Unsupported config option for services.bar: 'net'" in result.stderr def test_up_with_net_v1(self): self.base_dir = 'tests/fixtures/net-container' @@ -587,6 +948,50 @@ class CLITestCase(DockerClientTestCase): assert foo_container.get('HostConfig.NetworkMode') == \ 'container:{}'.format(bar_container.id) + @v3_only() + def test_up_with_healthcheck(self): + def wait_on_health_status(container, status): + def condition(): + container.inspect() + return container.get('State.Health.Status') == status + + return wait_on_condition(condition, delay=0.5) + + self.base_dir = 'tests/fixtures/healthcheck' + self.dispatch(['up', '-d'], None) + + passes = self.project.get_service('passes') + passes_container = passes.containers()[0] + + assert passes_container.get('Config.Healthcheck') == { + "Test": ["CMD-SHELL", "/bin/true"], + "Interval": nanoseconds_from_time_seconds(1), + "Timeout": nanoseconds_from_time_seconds(30 * 60), + "Retries": 1, + } + + wait_on_health_status(passes_container, 'healthy') + + fails = self.project.get_service('fails') + fails_container = fails.containers()[0] + + assert fails_container.get('Config.Healthcheck') == { + "Test": ["CMD", "/bin/false"], + "Interval": nanoseconds_from_time_seconds(2.5), + "Retries": 2, + } + + wait_on_health_status(fails_container, 'unhealthy') + + disabled = self.project.get_service('disabled') + disabled_container = disabled.containers()[0] + + assert disabled_container.get('Config.Healthcheck') == { + "Test": ["NONE"], + } + + assert 'Health' not in disabled_container.get('State') + def test_up_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['up', '-d', '--no-deps', 'web'], None) @@ -648,14 +1053,49 @@ class CLITestCase(DockerClientTestCase): wait_on_condition(ContainerCountCondition(self.project, 2)) os.kill(proc.pid, signal.SIGINT) - wait_on_condition(ContainerCountCondition(self.project, 0), timeout=30) + wait_on_condition(ContainerCountCondition(self.project, 0)) def test_up_handles_sigterm(self): proc = start_process(self.base_dir, ['up', '-t', '2']) wait_on_condition(ContainerCountCondition(self.project, 2)) os.kill(proc.pid, signal.SIGTERM) - wait_on_condition(ContainerCountCondition(self.project, 0), timeout=30) + wait_on_condition(ContainerCountCondition(self.project, 0)) + + @v2_only() + def test_up_handles_force_shutdown(self): + self.base_dir = 'tests/fixtures/sleeps-composefile' + proc = start_process(self.base_dir, ['up', '-t', '200']) + wait_on_condition(ContainerCountCondition(self.project, 2)) + + os.kill(proc.pid, signal.SIGTERM) + time.sleep(0.1) + os.kill(proc.pid, signal.SIGTERM) + wait_on_condition(ContainerCountCondition(self.project, 0)) + + def test_up_handles_abort_on_container_exit(self): + start_process(self.base_dir, ['up', '--abort-on-container-exit']) + wait_on_condition(ContainerCountCondition(self.project, 2)) + self.project.stop(['simple']) + wait_on_condition(ContainerCountCondition(self.project, 0)) + + def test_exec_without_tty(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '-d', 'console']) + self.assertEqual(len(self.project.containers()), 1) + + stdout, stderr = self.dispatch(['exec', '-T', 'console', 'ls', '-1d', '/']) + self.assertEquals(stdout, "/\n") + self.assertEquals(stderr, "") + + def test_exec_custom_user(self): + self.base_dir = 'tests/fixtures/links-composefile' + self.dispatch(['up', '-d', 'console']) + self.assertEqual(len(self.project.containers()), 1) + + stdout, stderr = self.dispatch(['exec', '-T', '--user=operator', 'console', 'whoami']) + self.assertEquals(stdout, "operator\n") + self.assertEquals(stderr, "") def test_run_service_without_links(self): self.base_dir = 'tests/fixtures/links-composefile' @@ -663,7 +1103,7 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(self.project.containers()), 0) # Ensure stdin/out was open - container = self.project.containers(stopped=True, one_off=True)[0] + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] config = container.inspect()['Config'] self.assertTrue(config['AttachStderr']) self.assertTrue(config['AttachStdout']) @@ -677,6 +1117,15 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(db.containers()), 1) self.assertEqual(len(console.containers()), 0) + @v2_only() + def test_run_service_with_dependencies(self): + self.base_dir = 'tests/fixtures/v2-dependencies' + self.dispatch(['run', 'web', '/bin/true'], None) + db = self.project.get_service('db') + console = self.project.get_service('console') + self.assertEqual(len(db.containers()), 1) + self.assertEqual(len(console.containers()), 0) + def test_run_with_no_deps(self): self.base_dir = 'tests/fixtures/links-composefile' self.dispatch(['run', '--no-deps', 'web', '/bin/true']) @@ -704,7 +1153,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['run', 'implicit']) service = self.project.get_service('implicit') - containers = service.containers(stopped=True, one_off=True) + containers = service.containers(stopped=True, one_off=OneOffFilter.only) self.assertEqual( [c.human_readable_command for c in containers], [u'/bin/sh -c echo "success"'], @@ -712,22 +1161,60 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['run', 'explicit']) service = self.project.get_service('explicit') - containers = service.containers(stopped=True, one_off=True) + containers = service.containers(stopped=True, one_off=OneOffFilter.only) self.assertEqual( [c.human_readable_command for c in containers], [u'/bin/true'], ) - def test_run_service_with_entrypoint_overridden(self): - self.base_dir = 'tests/fixtures/dockerfile_with_entrypoint' - name = 'service' - self.dispatch(['run', '--entrypoint', '/bin/echo', name, 'helloworld']) - service = self.project.get_service(name) - container = service.containers(stopped=True, one_off=True)[0] - self.assertEqual( - shlex.split(container.human_readable_command), - [u'/bin/echo', u'helloworld'], - ) + def test_run_service_with_dockerfile_entrypoint(self): + self.base_dir = 'tests/fixtures/entrypoint-dockerfile' + self.dispatch(['run', 'test']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['printf'] + assert container.get('Config.Cmd') == ['default', 'args'] + + def test_run_service_with_dockerfile_entrypoint_overridden(self): + self.base_dir = 'tests/fixtures/entrypoint-dockerfile' + self.dispatch(['run', '--entrypoint', 'echo', 'test']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['echo'] + assert not container.get('Config.Cmd') + + def test_run_service_with_dockerfile_entrypoint_and_command_overridden(self): + self.base_dir = 'tests/fixtures/entrypoint-dockerfile' + self.dispatch(['run', '--entrypoint', 'echo', 'test', 'foo']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['echo'] + assert container.get('Config.Cmd') == ['foo'] + + def test_run_service_with_compose_file_entrypoint(self): + self.base_dir = 'tests/fixtures/entrypoint-composefile' + self.dispatch(['run', 'test']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['printf'] + assert container.get('Config.Cmd') == ['default', 'args'] + + def test_run_service_with_compose_file_entrypoint_overridden(self): + self.base_dir = 'tests/fixtures/entrypoint-composefile' + self.dispatch(['run', '--entrypoint', 'echo', 'test']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['echo'] + assert not container.get('Config.Cmd') + + def test_run_service_with_compose_file_entrypoint_and_command_overridden(self): + self.base_dir = 'tests/fixtures/entrypoint-composefile' + self.dispatch(['run', '--entrypoint', 'echo', 'test', 'foo']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['echo'] + assert container.get('Config.Cmd') == ['foo'] + + def test_run_service_with_compose_file_entrypoint_and_empty_string_command(self): + self.base_dir = 'tests/fixtures/entrypoint-composefile' + self.dispatch(['run', '--entrypoint', 'echo', 'test', '']) + container = self.project.containers(stopped=True, one_off=OneOffFilter.only)[0] + assert container.get('Config.Entrypoint') == ['echo'] + assert container.get('Config.Cmd') == [''] def test_run_service_with_user_overridden(self): self.base_dir = 'tests/fixtures/user-composefile' @@ -735,7 +1222,7 @@ class CLITestCase(DockerClientTestCase): user = 'sshd' self.dispatch(['run', '--user={user}'.format(user=user), name], returncode=1) service = self.project.get_service(name) - container = service.containers(stopped=True, one_off=True)[0] + container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] self.assertEqual(user, container.get('Config.User')) def test_run_service_with_user_overridden_short_form(self): @@ -744,10 +1231,10 @@ class CLITestCase(DockerClientTestCase): user = 'sshd' self.dispatch(['run', '-u', user, name], returncode=1) service = self.project.get_service(name) - container = service.containers(stopped=True, one_off=True)[0] + container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] self.assertEqual(user, container.get('Config.User')) - def test_run_service_with_environement_overridden(self): + def test_run_service_with_environment_overridden(self): name = 'service' self.base_dir = 'tests/fixtures/environment-composefile' self.dispatch([ @@ -758,10 +1245,10 @@ class CLITestCase(DockerClientTestCase): '/bin/true', ]) service = self.project.get_service(name) - container = service.containers(stopped=True, one_off=True)[0] - # env overriden + container = service.containers(stopped=True, one_off=OneOffFilter.only)[0] + # env overridden self.assertEqual('notbar', container.environment['foo']) - # keep environement from yaml + # keep environment from yaml self.assertEqual('world', container.environment['hello']) # added option from command line self.assertEqual('beta', container.environment['alpha']) @@ -772,7 +1259,7 @@ class CLITestCase(DockerClientTestCase): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch(['run', '-d', 'simple']) - container = self.project.get_service('simple').containers(one_off=True)[0] + container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0] # get port information port_random = container.get_local_port(3000) @@ -789,7 +1276,7 @@ class CLITestCase(DockerClientTestCase): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch(['run', '-d', '--service-ports', 'simple']) - container = self.project.get_service('simple').containers(one_off=True)[0] + container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0] # get port information port_random = container.get_local_port(3000) @@ -806,11 +1293,11 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_range[0], "0.0.0.0:49153") self.assertEqual(port_range[1], "0.0.0.0:49154") - def test_run_service_with_explicitly_maped_ports(self): + def test_run_service_with_explicitly_mapped_ports(self): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' self.dispatch(['run', '-d', '-p', '30000:3000', '--publish', '30001:3001', 'simple']) - container = self.project.get_service('simple').containers(one_off=True)[0] + container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0] # get port information port_short = container.get_local_port(3000) @@ -823,11 +1310,16 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(port_short, "0.0.0.0:30000") self.assertEqual(port_full, "0.0.0.0:30001") - def test_run_service_with_explicitly_maped_ip_ports(self): + def test_run_service_with_explicitly_mapped_ip_ports(self): # create one off container self.base_dir = 'tests/fixtures/ports-composefile' - self.dispatch(['run', '-d', '-p', '127.0.0.1:30000:3000', '--publish', '127.0.0.1:30001:3001', 'simple'], None) - container = self.project.get_service('simple').containers(one_off=True)[0] + self.dispatch([ + 'run', '-d', + '-p', '127.0.0.1:30000:3000', + '--publish', '127.0.0.1:30001:3001', + 'simple' + ]) + container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0] # get port information port_short = container.get_local_port(3000) @@ -844,7 +1336,7 @@ class CLITestCase(DockerClientTestCase): # create one off container self.base_dir = 'tests/fixtures/expose-composefile' self.dispatch(['run', '-d', '--service-ports', 'simple']) - container = self.project.get_service('simple').containers(one_off=True)[0] + container = self.project.get_service('simple').containers(one_off=OneOffFilter.only)[0] ports = container.ports self.assertEqual(len(ports), 9) @@ -868,44 +1360,130 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['run', '--name', name, 'service', '/bin/true']) service = self.project.get_service('service') - container, = service.containers(stopped=True, one_off=True) + container, = service.containers(stopped=True, one_off=OneOffFilter.only) self.assertEqual(container.name, name) + def test_run_service_with_workdir_overridden(self): + self.base_dir = 'tests/fixtures/run-workdir' + name = 'service' + workdir = '/var' + self.dispatch(['run', '--workdir={workdir}'.format(workdir=workdir), name]) + service = self.project.get_service(name) + container = service.containers(stopped=True, one_off=True)[0] + self.assertEqual(workdir, container.get('Config.WorkingDir')) + + def test_run_service_with_workdir_overridden_short_form(self): + self.base_dir = 'tests/fixtures/run-workdir' + name = 'service' + workdir = '/var' + self.dispatch(['run', '-w', workdir, name]) + service = self.project.get_service(name) + container = service.containers(stopped=True, one_off=True)[0] + self.assertEqual(workdir, container.get('Config.WorkingDir')) + @v2_only() - def test_run_with_networking(self): - self.base_dir = 'tests/fixtures/v2-simple' - self.dispatch(['run', 'simple', 'true'], None) - service = self.project.get_service('simple') - container, = service.containers(stopped=True, one_off=True) - networks = self.client.networks(names=[self.project.default_network.full_name]) - self.assertEqual(len(networks), 1) - self.assertEqual(container.human_readable_command, u'true') + def test_run_interactive_connects_to_network(self): + self.base_dir = 'tests/fixtures/networks' + + self.dispatch(['up', '-d']) + self.dispatch(['run', 'app', 'nslookup', 'app']) + self.dispatch(['run', 'app', 'nslookup', 'db']) + + containers = self.project.get_service('app').containers( + stopped=True, one_off=OneOffFilter.only) + assert len(containers) == 2 + + for container in containers: + networks = container.get('NetworkSettings.Networks') + + assert sorted(list(networks)) == [ + '{}_{}'.format(self.project.name, name) + for name in ['back', 'front'] + ] + + for _, config in networks.items(): + # TODO: once we drop support for API <1.24, this can be changed to: + # assert config['Aliases'] == [container.short_id] + aliases = set(config['Aliases'] or []) - set([container.short_id]) + assert not aliases + + @v2_only() + def test_run_detached_connects_to_network(self): + self.base_dir = 'tests/fixtures/networks' + self.dispatch(['up', '-d']) + self.dispatch(['run', '-d', 'app', 'top']) + + container = self.project.get_service('app').containers(one_off=OneOffFilter.only)[0] + networks = container.get('NetworkSettings.Networks') + + assert sorted(list(networks)) == [ + '{}_{}'.format(self.project.name, name) + for name in ['back', 'front'] + ] + + for _, config in networks.items(): + # TODO: once we drop support for API <1.24, this can be changed to: + # assert config['Aliases'] == [container.short_id] + aliases = set(config['Aliases'] or []) - set([container.short_id]) + assert not aliases + + assert self.lookup(container, 'app') + assert self.lookup(container, 'db') def test_run_handles_sigint(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, 'simplecomposefile_simple_run_1', - running=True)) + 'running')) os.kill(proc.pid, signal.SIGINT) wait_on_condition(ContainerStateCondition( self.project.client, 'simplecomposefile_simple_run_1', - running=False)) + 'exited')) def test_run_handles_sigterm(self): proc = start_process(self.base_dir, ['run', '-T', 'simple', 'top']) wait_on_condition(ContainerStateCondition( self.project.client, 'simplecomposefile_simple_run_1', - running=True)) + 'running')) os.kill(proc.pid, signal.SIGTERM) wait_on_condition(ContainerStateCondition( self.project.client, 'simplecomposefile_simple_run_1', - running=False)) + 'exited')) + + @mock.patch.dict(os.environ) + def test_run_unicode_env_values_from_system(self): + value = 'ą, ć, ę, ł, ń, ó, ś, ź, ż' + if six.PY2: # os.environ doesn't support unicode values in Py2 + os.environ['BAR'] = value.encode('utf-8') + else: # ... and doesn't support byte values in Py3 + os.environ['BAR'] = value + self.base_dir = 'tests/fixtures/unicode-environment' + result = self.dispatch(['run', 'simple']) + + if six.PY2: # Can't retrieve output on Py3. See issue #3670 + assert value == result.stdout.strip() + + container = self.project.containers(one_off=OneOffFilter.only, stopped=True)[0] + environment = container.get('Config.Env') + assert 'FOO={}'.format(value) in environment + + @mock.patch.dict(os.environ) + def test_run_env_values_from_system(self): + os.environ['FOO'] = 'bar' + os.environ['BAR'] = 'baz' + + self.dispatch(['run', '-e', 'FOO', 'simple', 'true'], None) + + container = self.project.containers(one_off=OneOffFilter.only, stopped=True)[0] + environment = container.get('Config.Env') + assert 'FOO=bar' in environment + assert 'BAR=baz' not in environment def test_rm(self): service = self.project.get_service('simple') @@ -921,6 +1499,26 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['rm', '-f'], None) self.assertEqual(len(service.containers(stopped=True)), 0) + def test_rm_all(self): + service = self.project.get_service('simple') + service.create_container(one_off=False) + service.create_container(one_off=True) + kill_service(service) + self.assertEqual(len(service.containers(stopped=True)), 1) + self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1) + self.dispatch(['rm', '-f'], None) + self.assertEqual(len(service.containers(stopped=True)), 0) + self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 0) + + service.create_container(one_off=False) + service.create_container(one_off=True) + kill_service(service) + self.assertEqual(len(service.containers(stopped=True)), 1) + self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 1) + self.dispatch(['rm', '-f', '--all'], None) + self.assertEqual(len(service.containers(stopped=True)), 0) + self.assertEqual(len(service.containers(stopped=True, one_off=OneOffFilter.only)), 0) + def test_stop(self): self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') @@ -999,6 +1597,70 @@ class CLITestCase(DockerClientTestCase): def test_logs_invalid_service_name(self): self.dispatch(['logs', 'madeupname'], returncode=1) + def test_logs_follow(self): + self.base_dir = 'tests/fixtures/echo-services' + self.dispatch(['up', '-d']) + + result = self.dispatch(['logs', '-f']) + + assert result.stdout.count('\n') == 5 + assert 'simple' in result.stdout + assert 'another' in result.stdout + assert 'exited with code 0' in result.stdout + + def test_logs_follow_logs_from_new_containers(self): + self.base_dir = 'tests/fixtures/logs-composefile' + self.dispatch(['up', '-d', 'simple']) + + proc = start_process(self.base_dir, ['logs', '-f']) + + self.dispatch(['up', '-d', 'another']) + wait_on_condition(ContainerStateCondition( + self.project.client, + 'logscomposefile_another_1', + 'exited')) + + self.dispatch(['kill', 'simple']) + + result = wait_on_process(proc) + + assert 'hello' in result.stdout + assert 'test' in result.stdout + assert 'logscomposefile_another_1 exited with code 0' in result.stdout + assert 'logscomposefile_simple_1 exited with code 137' in result.stdout + + def test_logs_default(self): + self.base_dir = 'tests/fixtures/logs-composefile' + self.dispatch(['up', '-d']) + + result = self.dispatch(['logs']) + assert 'hello' in result.stdout + assert 'test' in result.stdout + assert 'exited with' not in result.stdout + + def test_logs_on_stopped_containers_exits(self): + self.base_dir = 'tests/fixtures/echo-services' + self.dispatch(['up']) + + result = self.dispatch(['logs']) + assert 'simple' in result.stdout + assert 'another' in result.stdout + assert 'exited with' not in result.stdout + + def test_logs_timestamps(self): + self.base_dir = 'tests/fixtures/echo-services' + self.dispatch(['up', '-d']) + + result = self.dispatch(['logs', '-f', '-t']) + self.assertRegexpMatches(result.stdout, '(\d{4})-(\d{2})-(\d{2})T(\d{2})\:(\d{2})\:(\d{2})') + + def test_logs_tail(self): + self.base_dir = 'tests/fixtures/logs-tail-composefile' + self.dispatch(['up']) + + result = self.dispatch(['logs', '--tail', '2']) + assert result.stdout.count('\n') == 3 + def test_kill(self): self.dispatch(['up', '-d'], None) service = self.project.get_service('simple') @@ -1124,9 +1786,20 @@ class CLITestCase(DockerClientTestCase): os.kill(events_proc.pid, signal.SIGINT) result = wait_on_process(events_proc, returncode=1) lines = [json.loads(line) for line in result.stdout.rstrip().split('\n')] - assert [e['action'] for e in lines] == ['create', 'start', 'create', 'start'] + assert Counter(e['action'] for e in lines) == {'create': 2, 'start': 2} def test_events_human_readable(self): + + def has_timestamp(string): + str_iso_date, str_iso_time, container_info = string.split(' ', 2) + try: + return isinstance(datetime.datetime.strptime( + '%s %s' % (str_iso_date, str_iso_time), + '%Y-%m-%d %H:%M:%S.%f'), + datetime.datetime) + except ValueError: + return False + events_proc = start_process(self.base_dir, ['events']) self.dispatch(['up', '-d', 'simple']) wait_on_condition(ContainerCountCondition(self.project, 1)) @@ -1143,7 +1816,8 @@ class CLITestCase(DockerClientTestCase): assert expected_template.format('create', container.id) in lines[0] assert expected_template.format('start', container.id) in lines[1] - assert lines[0].startswith(datetime.date.today().isoformat()) + + assert has_timestamp(lines[0]) def test_env_file_relative_to_compose_file(self): config_path = os.path.abspath('tests/fixtures/env-file/docker-compose.yml') @@ -1233,3 +1907,23 @@ class CLITestCase(DockerClientTestCase): "BAZ=2", ]) self.assertTrue(expected_env <= set(web.get('Config.Env'))) + + def test_top_services_not_running(self): + self.base_dir = 'tests/fixtures/top' + result = self.dispatch(['top']) + assert len(result.stdout) == 0 + + def test_top_services_running(self): + self.base_dir = 'tests/fixtures/top' + self.dispatch(['up', '-d']) + result = self.dispatch(['top']) + + self.assertIn('top_service_a', result.stdout) + self.assertIn('top_service_b', result.stdout) + self.assertNotIn('top_not_a_service', result.stdout) + + def test_top_processes_running(self): + self.base_dir = 'tests/fixtures/top' + self.dispatch(['up', '-d']) + result = self.dispatch(['top']) + assert result.stdout.count("top") == 4 diff --git a/tests/fixtures/bundle-with-digests/docker-compose.yml b/tests/fixtures/bundle-with-digests/docker-compose.yml new file mode 100644 index 00000000..b7013512 --- /dev/null +++ b/tests/fixtures/bundle-with-digests/docker-compose.yml @@ -0,0 +1,9 @@ + +version: '2.0' + +services: + web: + image: dockercloud/hello-world@sha256:fe79a2cfbd17eefc344fb8419420808df95a1e22d93b7f621a7399fd1e9dca1d + + redis: + image: redis@sha256:a84cb8f53a70e19f61ff2e1d5e73fb7ae62d374b2b7392de1e7d77be26ef8f7b diff --git a/tests/fixtures/default-env-file/.env b/tests/fixtures/default-env-file/.env new file mode 100644 index 00000000..996c886c --- /dev/null +++ b/tests/fixtures/default-env-file/.env @@ -0,0 +1,4 @@ +IMAGE=alpine:latest +COMMAND=true +PORT1=5643 +PORT2=9999 \ No newline at end of file diff --git a/tests/fixtures/default-env-file/docker-compose.yml b/tests/fixtures/default-env-file/docker-compose.yml new file mode 100644 index 00000000..aa8e4409 --- /dev/null +++ b/tests/fixtures/default-env-file/docker-compose.yml @@ -0,0 +1,6 @@ +web: + image: ${IMAGE} + command: ${COMMAND} + ports: + - $PORT1 + - $PORT2 diff --git a/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml b/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml deleted file mode 100644 index 78631502..00000000 --- a/tests/fixtures/dockerfile_with_entrypoint/docker-compose.yml +++ /dev/null @@ -1,2 +0,0 @@ -service: - build: . diff --git a/tests/fixtures/entrypoint-composefile/docker-compose.yml b/tests/fixtures/entrypoint-composefile/docker-compose.yml new file mode 100644 index 00000000..e9880973 --- /dev/null +++ b/tests/fixtures/entrypoint-composefile/docker-compose.yml @@ -0,0 +1,6 @@ +version: "2" +services: + test: + image: busybox + entrypoint: printf + command: default args diff --git a/tests/fixtures/dockerfile_with_entrypoint/Dockerfile b/tests/fixtures/entrypoint-dockerfile/Dockerfile similarity index 57% rename from tests/fixtures/dockerfile_with_entrypoint/Dockerfile rename to tests/fixtures/entrypoint-dockerfile/Dockerfile index e7454e59..49f4416c 100644 --- a/tests/fixtures/dockerfile_with_entrypoint/Dockerfile +++ b/tests/fixtures/entrypoint-dockerfile/Dockerfile @@ -1,3 +1,4 @@ FROM busybox:latest LABEL com.docker.compose.test_image=true -ENTRYPOINT echo "From prebuilt entrypoint" +ENTRYPOINT ["printf"] +CMD ["default", "args"] diff --git a/tests/fixtures/entrypoint-dockerfile/docker-compose.yml b/tests/fixtures/entrypoint-dockerfile/docker-compose.yml new file mode 100644 index 00000000..8318e61f --- /dev/null +++ b/tests/fixtures/entrypoint-dockerfile/docker-compose.yml @@ -0,0 +1,4 @@ +version: "2" +services: + test: + build: . diff --git a/tests/fixtures/extends/common.yml b/tests/fixtures/extends/common.yml index 358ef5bc..b2d86aa4 100644 --- a/tests/fixtures/extends/common.yml +++ b/tests/fixtures/extends/common.yml @@ -1,6 +1,7 @@ web: image: busybox command: /bin/true + net: host environment: - FOO=1 - BAR=1 diff --git a/tests/fixtures/extends/docker-compose.yml b/tests/fixtures/extends/docker-compose.yml index c51be49e..8e37d404 100644 --- a/tests/fixtures/extends/docker-compose.yml +++ b/tests/fixtures/extends/docker-compose.yml @@ -11,6 +11,7 @@ myweb: BAR: "2" # add BAZ BAZ: "2" + net: bridge mydb: image: busybox command: top diff --git a/tests/fixtures/extends/healthcheck-1.yml b/tests/fixtures/extends/healthcheck-1.yml new file mode 100644 index 00000000..4c311e62 --- /dev/null +++ b/tests/fixtures/extends/healthcheck-1.yml @@ -0,0 +1,9 @@ +version: '2.1' +services: + demo: + image: foobar:latest + healthcheck: + test: ["CMD", "/health.sh"] + interval: 10s + timeout: 5s + retries: 36 diff --git a/tests/fixtures/extends/healthcheck-2.yml b/tests/fixtures/extends/healthcheck-2.yml new file mode 100644 index 00000000..11bc9f09 --- /dev/null +++ b/tests/fixtures/extends/healthcheck-2.yml @@ -0,0 +1,6 @@ +version: '2.1' +services: + demo: + extends: + file: healthcheck-1.yml + service: demo diff --git a/tests/fixtures/extends/invalid-links.yml b/tests/fixtures/extends/invalid-links.yml index edfeb8b2..cea740cb 100644 --- a/tests/fixtures/extends/invalid-links.yml +++ b/tests/fixtures/extends/invalid-links.yml @@ -1,3 +1,5 @@ +mydb: + build: '.' myweb: build: '.' extends: diff --git a/tests/fixtures/extends/invalid-net-v2.yml b/tests/fixtures/extends/invalid-net-v2.yml new file mode 100644 index 00000000..7ba714e8 --- /dev/null +++ b/tests/fixtures/extends/invalid-net-v2.yml @@ -0,0 +1,12 @@ +version: "2" +services: + myweb: + build: '.' + extends: + service: web + command: top + web: + build: '.' + network_mode: "service:net" + net: + build: '.' diff --git a/tests/fixtures/healthcheck/docker-compose.yml b/tests/fixtures/healthcheck/docker-compose.yml new file mode 100644 index 00000000..2c45b8d8 --- /dev/null +++ b/tests/fixtures/healthcheck/docker-compose.yml @@ -0,0 +1,24 @@ +version: "3" +services: + passes: + image: busybox + command: top + healthcheck: + test: "/bin/true" + interval: 1s + timeout: 30m + retries: 1 + + fails: + image: busybox + command: top + healthcheck: + test: ["CMD", "/bin/false"] + interval: 2.5s + retries: 2 + + disabled: + image: busybox + command: top + healthcheck: + disable: true diff --git a/tests/fixtures/logging-composefile/docker-compose.yml b/tests/fixtures/logging-composefile/docker-compose.yml index 0a73030a..466d13e5 100644 --- a/tests/fixtures/logging-composefile/docker-compose.yml +++ b/tests/fixtures/logging-composefile/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/logs-composefile/docker-compose.yml b/tests/fixtures/logs-composefile/docker-compose.yml new file mode 100644 index 00000000..b719c91e --- /dev/null +++ b/tests/fixtures/logs-composefile/docker-compose.yml @@ -0,0 +1,6 @@ +simple: + image: busybox:latest + command: sh -c "echo hello && tail -f /dev/null" +another: + image: busybox:latest + command: sh -c "echo test" diff --git a/tests/fixtures/logs-tail-composefile/docker-compose.yml b/tests/fixtures/logs-tail-composefile/docker-compose.yml new file mode 100644 index 00000000..80d8feae --- /dev/null +++ b/tests/fixtures/logs-tail-composefile/docker-compose.yml @@ -0,0 +1,3 @@ +simple: + image: busybox:latest + command: sh -c "echo a && echo b && echo c && echo d" diff --git a/tests/fixtures/net-container/v2-invalid.yml b/tests/fixtures/net-container/v2-invalid.yml index eac4b5f1..9b846295 100644 --- a/tests/fixtures/net-container/v2-invalid.yml +++ b/tests/fixtures/net-container/v2-invalid.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: foo: diff --git a/tests/fixtures/networks/bridge.yml b/tests/fixtures/networks/bridge.yml new file mode 100644 index 00000000..9fa7db82 --- /dev/null +++ b/tests/fixtures/networks/bridge.yml @@ -0,0 +1,9 @@ +version: "2" + +services: + web: + image: busybox + command: top + networks: + - bridge + - default diff --git a/tests/fixtures/networks/default-network-config.yml b/tests/fixtures/networks/default-network-config.yml new file mode 100644 index 00000000..4bd0989b --- /dev/null +++ b/tests/fixtures/networks/default-network-config.yml @@ -0,0 +1,13 @@ +version: "2" +services: + simple: + image: busybox:latest + command: top + another: + image: busybox:latest + command: top +networks: + default: + driver: bridge + driver_opts: + "com.docker.network.bridge.enable_icc": "false" diff --git a/tests/fixtures/networks/docker-compose.yml b/tests/fixtures/networks/docker-compose.yml index f1b79df0..c11fa682 100644 --- a/tests/fixtures/networks/docker-compose.yml +++ b/tests/fixtures/networks/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: @@ -9,6 +9,8 @@ services: image: busybox command: top networks: ["front", "back"] + links: + - "db:database" db: image: busybox command: top diff --git a/tests/fixtures/networks/external-default.yml b/tests/fixtures/networks/external-default.yml new file mode 100644 index 00000000..5c9426b8 --- /dev/null +++ b/tests/fixtures/networks/external-default.yml @@ -0,0 +1,12 @@ +version: "2" +services: + simple: + image: busybox:latest + command: top + another: + image: busybox:latest + command: top +networks: + default: + external: + name: composetest_external_network diff --git a/tests/fixtures/networks/external-networks.yml b/tests/fixtures/networks/external-networks.yml index 644e3dda..db75b780 100644 --- a/tests/fixtures/networks/external-networks.yml +++ b/tests/fixtures/networks/external-networks.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: diff --git a/tests/fixtures/networks/missing-network.yml b/tests/fixtures/networks/missing-network.yml index 666f7d34..41012535 100644 --- a/tests/fixtures/networks/missing-network.yml +++ b/tests/fixtures/networks/missing-network.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: web: diff --git a/tests/fixtures/networks/network-aliases.yml b/tests/fixtures/networks/network-aliases.yml new file mode 100644 index 00000000..8cf7d5af --- /dev/null +++ b/tests/fixtures/networks/network-aliases.yml @@ -0,0 +1,16 @@ +version: "2" + +services: + web: + image: busybox + command: top + networks: + front: + aliases: + - forward_facing + - ahead + back: + +networks: + front: {} + back: {} diff --git a/tests/fixtures/networks/network-internal.yml b/tests/fixtures/networks/network-internal.yml new file mode 100755 index 00000000..1fa339b1 --- /dev/null +++ b/tests/fixtures/networks/network-internal.yml @@ -0,0 +1,13 @@ +version: "2" + +services: + web: + image: busybox + command: top + networks: + - internal + +networks: + internal: + driver: bridge + internal: True diff --git a/tests/fixtures/networks/network-label.yml b/tests/fixtures/networks/network-label.yml new file mode 100644 index 00000000..fdb24f65 --- /dev/null +++ b/tests/fixtures/networks/network-label.yml @@ -0,0 +1,13 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + networks: + - network_with_label + +networks: + network_with_label: + labels: + - "label_key=label_val" diff --git a/tests/fixtures/networks/network-mode.yml b/tests/fixtures/networks/network-mode.yml new file mode 100644 index 00000000..e4d070b4 --- /dev/null +++ b/tests/fixtures/networks/network-mode.yml @@ -0,0 +1,27 @@ +version: "2" + +services: + bridge: + image: busybox + command: top + network_mode: bridge + + service: + image: busybox + command: top + network_mode: "service:bridge" + + container: + image: busybox + command: top + network_mode: "container:composetest_network_mode_container" + + host: + image: busybox + command: top + network_mode: host + + none: + image: busybox + command: top + network_mode: none diff --git a/tests/fixtures/networks/network-static-addresses.yml b/tests/fixtures/networks/network-static-addresses.yml new file mode 100755 index 00000000..f820ff6a --- /dev/null +++ b/tests/fixtures/networks/network-static-addresses.yml @@ -0,0 +1,23 @@ +version: "2" + +services: + web: + image: busybox + command: top + networks: + static_test: + ipv4_address: 172.16.100.100 + ipv6_address: fe80::1001:100 + +networks: + static_test: + driver: bridge + driver_opts: + com.docker.network.enable_ipv6: "true" + ipam: + driver: default + config: + - subnet: 172.16.100.0/24 + gateway: 172.16.100.1 + - subnet: fe80::/64 + gateway: fe80::1001:1 diff --git a/tests/fixtures/networks/predefined-networks.yml b/tests/fixtures/networks/predefined-networks.yml deleted file mode 100644 index d0fac377..00000000 --- a/tests/fixtures/networks/predefined-networks.yml +++ /dev/null @@ -1,17 +0,0 @@ -version: 2 - -services: - bridge: - image: busybox - command: top - networks: ["bridge"] - - host: - image: busybox - command: top - networks: ["host"] - - none: - image: busybox - command: top - networks: [] diff --git a/tests/fixtures/no-services/docker-compose.yml b/tests/fixtures/no-services/docker-compose.yml index fa498784..6e76ec0c 100644 --- a/tests/fixtures/no-services/docker-compose.yml +++ b/tests/fixtures/no-services/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" networks: foo: {} diff --git a/tests/fixtures/restart/docker-compose.yml b/tests/fixtures/restart/docker-compose.yml new file mode 100644 index 00000000..ecfdfbf5 --- /dev/null +++ b/tests/fixtures/restart/docker-compose.yml @@ -0,0 +1,17 @@ +version: "2" +services: + never: + image: busybox + restart: "no" + always: + image: busybox + restart: always + on-failure: + image: busybox + restart: on-failure + on-failure-5: + image: busybox + restart: "on-failure:5" + restart-null: + image: busybox + restart: "" diff --git a/tests/fixtures/run-workdir/docker-compose.yml b/tests/fixtures/run-workdir/docker-compose.yml new file mode 100644 index 00000000..dc3ea86a --- /dev/null +++ b/tests/fixtures/run-workdir/docker-compose.yml @@ -0,0 +1,4 @@ +service: + image: busybox:latest + working_dir: /etc + command: /bin/true diff --git a/tests/fixtures/secrets/default b/tests/fixtures/secrets/default new file mode 100644 index 00000000..f9dc2014 --- /dev/null +++ b/tests/fixtures/secrets/default @@ -0,0 +1 @@ +This is the secret diff --git a/tests/fixtures/sleeps-composefile/docker-compose.yml b/tests/fixtures/sleeps-composefile/docker-compose.yml new file mode 100644 index 00000000..7c8d84f8 --- /dev/null +++ b/tests/fixtures/sleeps-composefile/docker-compose.yml @@ -0,0 +1,10 @@ + +version: "2" + +services: + simple: + image: busybox:latest + command: sleep 200 + another: + image: busybox:latest + command: sleep 200 diff --git a/tests/fixtures/tls/ca.pem b/tests/fixtures/tls/ca.pem new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/tls/cert.pem b/tests/fixtures/tls/cert.pem new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/tls/key.key b/tests/fixtures/tls/key.key new file mode 100644 index 00000000..e69de29b diff --git a/tests/fixtures/top/docker-compose.yml b/tests/fixtures/top/docker-compose.yml new file mode 100644 index 00000000..d632a836 --- /dev/null +++ b/tests/fixtures/top/docker-compose.yml @@ -0,0 +1,6 @@ +service_a: + image: busybox:latest + command: top +service_b: + image: busybox:latest + command: top diff --git a/tests/fixtures/unicode-environment/docker-compose.yml b/tests/fixtures/unicode-environment/docker-compose.yml new file mode 100644 index 00000000..a41af4f0 --- /dev/null +++ b/tests/fixtures/unicode-environment/docker-compose.yml @@ -0,0 +1,7 @@ +version: '2' +services: + simple: + image: busybox:latest + command: sh -c 'echo $$FOO' + environment: + FOO: ${BAR} diff --git a/tests/fixtures/v1-config/docker-compose.yml b/tests/fixtures/v1-config/docker-compose.yml new file mode 100644 index 00000000..8646c4ed --- /dev/null +++ b/tests/fixtures/v1-config/docker-compose.yml @@ -0,0 +1,10 @@ +net: + image: busybox +volume: + image: busybox + volumes: + - /data +app: + image: busybox + net: "container:net" + volumes_from: ["volume"] diff --git a/tests/fixtures/v2-dependencies/docker-compose.yml b/tests/fixtures/v2-dependencies/docker-compose.yml new file mode 100644 index 00000000..2e14b94b --- /dev/null +++ b/tests/fixtures/v2-dependencies/docker-compose.yml @@ -0,0 +1,13 @@ +version: "2.0" +services: + db: + image: busybox:latest + command: top + web: + image: busybox:latest + command: top + depends_on: + - db + console: + image: busybox:latest + command: top diff --git a/tests/fixtures/v2-full/docker-compose.yml b/tests/fixtures/v2-full/docker-compose.yml index 725296c9..a973dd0c 100644 --- a/tests/fixtures/v2-full/docker-compose.yml +++ b/tests/fixtures/v2-full/docker-compose.yml @@ -1,5 +1,5 @@ -version: 2 +version: "2" volumes: data: diff --git a/tests/fixtures/v2-simple/docker-compose.yml b/tests/fixtures/v2-simple/docker-compose.yml index 12a9de72..c99ae02f 100644 --- a/tests/fixtures/v2-simple/docker-compose.yml +++ b/tests/fixtures/v2-simple/docker-compose.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/v2-simple/links-invalid.yml b/tests/fixtures/v2-simple/links-invalid.yml index 422f9314..481aa404 100644 --- a/tests/fixtures/v2-simple/links-invalid.yml +++ b/tests/fixtures/v2-simple/links-invalid.yml @@ -1,4 +1,4 @@ -version: 2 +version: "2" services: simple: image: busybox:latest diff --git a/tests/fixtures/v3-full/docker-compose.yml b/tests/fixtures/v3-full/docker-compose.yml new file mode 100644 index 00000000..a1661ab9 --- /dev/null +++ b/tests/fixtures/v3-full/docker-compose.yml @@ -0,0 +1,41 @@ +version: "3" +services: + web: + image: busybox + + deploy: + mode: replicated + replicas: 6 + labels: [FOO=BAR] + update_config: + parallelism: 3 + delay: 10s + failure_action: continue + monitor: 60s + max_failure_ratio: 0.3 + resources: + limits: + cpus: '0.001' + memory: 50M + reservations: + cpus: '0.0001' + memory: 20M + restart_policy: + condition: on_failure + delay: 5s + max_attempts: 3 + window: 120s + placement: + constraints: [node=foo] + + healthcheck: + test: cat /etc/passwd + interval: 10s + timeout: 1s + retries: 5 + + stop_grace_period: 20s +volumes: + foobar: + labels: + com.docker.compose.test: 'true' diff --git a/tests/fixtures/volumes-from-container/docker-compose.yml b/tests/fixtures/volumes-from-container/docker-compose.yml new file mode 100644 index 00000000..495fcaae --- /dev/null +++ b/tests/fixtures/volumes-from-container/docker-compose.yml @@ -0,0 +1,5 @@ +version: "2" +services: + test: + image: busybox + volumes_from: ["container:composetest_data_container"] diff --git a/tests/fixtures/volumes/docker-compose.yml b/tests/fixtures/volumes/docker-compose.yml new file mode 100644 index 00000000..da711ac4 --- /dev/null +++ b/tests/fixtures/volumes/docker-compose.yml @@ -0,0 +1,2 @@ +version: '2.1' +services: {} diff --git a/tests/fixtures/volumes/external-volumes.yml b/tests/fixtures/volumes/external-volumes.yml new file mode 100644 index 00000000..05c6c484 --- /dev/null +++ b/tests/fixtures/volumes/external-volumes.yml @@ -0,0 +1,16 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - foo:/var/lib/ + - bar:/etc/ + +volumes: + foo: + external: true + bar: + external: + name: some_bar diff --git a/tests/fixtures/volumes/volume-label.yml b/tests/fixtures/volumes/volume-label.yml new file mode 100644 index 00000000..a5f33a5a --- /dev/null +++ b/tests/fixtures/volumes/volume-label.yml @@ -0,0 +1,13 @@ +version: "2.1" + +services: + web: + image: busybox + command: top + volumes: + - volume_with_label:/data + +volumes: + volume_with_label: + labels: + - "label_key=label_val" diff --git a/tests/helpers.py b/tests/helpers.py new file mode 100644 index 00000000..4b422a6a --- /dev/null +++ b/tests/helpers.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from compose.config.config import ConfigDetails +from compose.config.config import ConfigFile +from compose.config.config import load + + +def build_config(contents, **kwargs): + return load(build_config_details(contents, **kwargs)) + + +def build_config_details(contents, working_dir='working_dir', filename='filename.yml'): + return ConfigDetails( + working_dir, + [ConfigFile(filename, contents)], + ) diff --git a/tests/integration/network_test.py b/tests/integration/network_test.py new file mode 100644 index 00000000..2ff610fb --- /dev/null +++ b/tests/integration/network_test.py @@ -0,0 +1,17 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from .testcases import DockerClientTestCase +from compose.const import LABEL_NETWORK +from compose.const import LABEL_PROJECT +from compose.network import Network + + +class NetworkTest(DockerClientTestCase): + def test_network_default_labels(self): + net = Network(self.client, 'composetest', 'foonet') + net.ensure() + net_data = net.inspect() + labels = net_data['Labels'] + assert labels[LABEL_NETWORK] == net.name + assert labels[LABEL_PROJECT] == net.project diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index d29d9f1e..28762cd2 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,27 +1,44 @@ from __future__ import absolute_import from __future__ import unicode_literals +import os.path import random import py +import pytest from docker.errors import NotFound +from .. import mock +from ..helpers import build_config as load_config from .testcases import DockerClientTestCase from compose.config import config +from compose.config import ConfigurationError +from compose.config import types +from compose.config.config import V2_0 +from compose.config.config import V2_1 +from compose.config.config import V3_1 from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec from compose.const import LABEL_PROJECT +from compose.const import LABEL_SERVICE from compose.container import Container +from compose.errors import HealthCheckFailed +from compose.errors import NoHealthCheckConfigured from compose.project import Project +from compose.project import ProjectError from compose.service import ConvergenceStrategy +from tests.integration.testcases import v2_1_only from tests.integration.testcases import v2_only +from tests.integration.testcases import v3_only -def build_service_dicts(service_config): - return config.load( - config.ConfigDetails( - 'working_dir', - [config.ConfigFile(None, service_config)])) +def build_config(**kwargs): + return config.Config( + version=kwargs.get('version'), + services=kwargs.get('services'), + volumes=kwargs.get('volumes'), + networks=kwargs.get('networks'), + secrets=kwargs.get('secrets')) class ProjectTest(DockerClientTestCase): @@ -64,19 +81,18 @@ class ProjectTest(DockerClientTestCase): ) def test_volumes_from_service(self): - service_dicts = build_service_dicts({ - 'data': { - 'image': 'busybox:latest', - 'volumes': ['/var/data'], - }, - 'db': { - 'image': 'busybox:latest', - 'volumes_from': ['data'], - }, - }) project = Project.from_config( name='composetest', - config_data=service_dicts, + config_data=load_config({ + 'data': { + 'image': 'busybox:latest', + 'volumes': ['/var/data'], + }, + 'db': { + 'image': 'busybox:latest', + 'volumes_from': ['data'], + }, + }), client=self.client, ) db = project.get_service('db') @@ -93,7 +109,7 @@ class ProjectTest(DockerClientTestCase): ) project = Project.from_config( name='composetest', - config_data=build_service_dicts({ + config_data=load_config({ 'db': { 'image': 'busybox:latest', 'volumes_from': ['composetest_data_container'], @@ -104,10 +120,74 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) - def test_net_from_service(self): + @v2_only() + def test_network_mode_from_service(self): project = Project.from_config( name='composetest', - config_data=build_service_dicts({ + client=self.client, + config_data=load_config({ + 'version': V2_0, + 'services': { + 'net': { + 'image': 'busybox:latest', + 'command': ["top"] + }, + 'web': { + 'image': 'busybox:latest', + 'network_mode': 'service:net', + 'command': ["top"] + }, + }, + }), + ) + + project.up() + + web = project.get_service('web') + net = project.get_service('net') + self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) + + @v2_only() + def test_network_mode_from_container(self): + def get_project(): + return Project.from_config( + name='composetest', + config_data=load_config({ + 'version': V2_0, + 'services': { + 'web': { + 'image': 'busybox:latest', + 'network_mode': 'container:composetest_net_container' + }, + }, + }), + client=self.client, + ) + + with pytest.raises(ConfigurationError) as excinfo: + get_project() + + assert "container 'composetest_net_container' which does not exist" in excinfo.exconly() + + net_container = Container.create( + self.client, + image='busybox:latest', + name='composetest_net_container', + command='top', + labels={LABEL_PROJECT: 'composetest'}, + ) + net_container.start() + + project = get_project() + project.up() + + web = project.get_service('web') + self.assertEqual(web.network_mode.mode, 'container:' + net_container.id) + + def test_net_from_service_v1(self): + project = Project.from_config( + name='composetest', + config_data=load_config({ 'net': { 'image': 'busybox:latest', 'command': ["top"] @@ -125,9 +205,26 @@ class ProjectTest(DockerClientTestCase): web = project.get_service('web') net = project.get_service('net') - self.assertEqual(web.net.mode, 'container:' + net.containers()[0].id) + self.assertEqual(web.network_mode.mode, 'container:' + net.containers()[0].id) + + def test_net_from_container_v1(self): + def get_project(): + return Project.from_config( + name='composetest', + config_data=load_config({ + 'web': { + 'image': 'busybox:latest', + 'net': 'container:composetest_net_container' + }, + }), + client=self.client, + ) + + with pytest.raises(ConfigurationError) as excinfo: + get_project() + + assert "container 'composetest_net_container' which does not exist" in excinfo.exconly() - def test_net_from_container(self): net_container = Container.create( self.client, image='busybox:latest', @@ -137,21 +234,11 @@ class ProjectTest(DockerClientTestCase): ) net_container.start() - project = Project.from_config( - name='composetest', - config_data=build_service_dicts({ - 'web': { - 'image': 'busybox:latest', - 'net': 'container:composetest_net_container' - }, - }), - client=self.client, - ) - + project = get_project() project.up() web = project.get_service('web') - self.assertEqual(web.net.mode, 'container:' + net_container.id) + self.assertEqual(web.network_mode.mode, 'container:' + net_container.id) def test_start_pause_unpause_stop_kill_remove(self): web = self.create_service('web') @@ -168,19 +255,24 @@ class ProjectTest(DockerClientTestCase): db_container = db.create_container() project.start(service_names=['web']) - self.assertEqual(set(c.name for c in project.containers()), set([web_container_1.name, web_container_2.name])) + self.assertEqual( + set(c.name for c in project.containers()), + set([web_container_1.name, web_container_2.name])) project.start() - self.assertEqual(set(c.name for c in project.containers()), - set([web_container_1.name, web_container_2.name, db_container.name])) + self.assertEqual( + set(c.name for c in project.containers()), + set([web_container_1.name, web_container_2.name, db_container.name])) project.pause(service_names=['web']) - self.assertEqual(set([c.name for c in project.containers() if c.is_paused]), - set([web_container_1.name, web_container_2.name])) + self.assertEqual( + set([c.name for c in project.containers() if c.is_paused]), + set([web_container_1.name, web_container_2.name])) project.pause() - self.assertEqual(set([c.name for c in project.containers() if c.is_paused]), - set([web_container_1.name, web_container_2.name, db_container.name])) + self.assertEqual( + set([c.name for c in project.containers() if c.is_paused]), + set([web_container_1.name, web_container_2.name, db_container.name])) project.unpause(service_names=['db']) self.assertEqual(len([c.name for c in project.containers() if c.is_paused]), 2) @@ -390,7 +482,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_starts_depends(self): project = Project.from_config( name='composetest', - config_data=build_service_dicts({ + config_data=load_config({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -425,7 +517,7 @@ class ProjectTest(DockerClientTestCase): def test_project_up_with_no_deps(self): project = Project.from_config( name='composetest', - config_data=build_service_dicts({ + config_data=load_config({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -485,14 +577,18 @@ class ProjectTest(DockerClientTestCase): @v2_only() def test_project_up_networks(self): - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top', + 'networks': { + 'foo': None, + 'bar': None, + 'baz': {'aliases': ['extra']}, + }, }], - volumes={}, networks={ 'foo': {'driver': 'bridge'}, 'bar': {'driver': None}, @@ -506,29 +602,364 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, ) project.up() - self.assertEqual(len(project.containers()), 1) + + containers = project.containers() + assert len(containers) == 1 + container, = containers for net_name in ['foo', 'bar', 'baz']: full_net_name = 'composetest_{}'.format(net_name) network_data = self.client.inspect_network(full_net_name) - self.assertEqual(network_data['Name'], full_net_name) + assert network_data['Name'] == full_net_name + + aliases_key = 'NetworkSettings.Networks.{net}.Aliases' + assert 'web' in container.get(aliases_key.format(net='composetest_foo')) + assert 'web' in container.get(aliases_key.format(net='composetest_baz')) + assert 'extra' in container.get(aliases_key.format(net='composetest_baz')) foo_data = self.client.inspect_network('composetest_foo') - self.assertEqual(foo_data['Driver'], 'bridge') + assert foo_data['Driver'] == 'bridge' + + @v2_only() + def test_up_with_ipam_config(self): + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': {'front': None}, + }], + networks={ + 'front': { + 'driver': 'bridge', + 'driver_opts': { + "com.docker.network.bridge.enable_icc": "false", + }, + 'ipam': { + 'driver': 'default', + 'config': [{ + "subnet": "172.28.0.0/16", + "ip_range": "172.28.5.0/24", + "gateway": "172.28.5.254", + "aux_addresses": { + "a": "172.28.1.5", + "b": "172.28.1.6", + "c": "172.28.1.7", + }, + }], + }, + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + + network = self.client.networks(names=['composetest_front'])[0] + + assert network['Options'] == { + "com.docker.network.bridge.enable_icc": "false" + } + + assert network['IPAM'] == { + 'Driver': 'default', + 'Options': None, + 'Config': [{ + 'Subnet': "172.28.0.0/16", + 'IPRange': "172.28.5.0/24", + 'Gateway': "172.28.5.254", + 'AuxiliaryAddresses': { + 'a': '172.28.1.5', + 'b': '172.28.1.6', + 'c': '172.28.1.7', + }, + }], + } + + @v2_only() + def test_up_with_network_static_addresses(self): + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top', + 'networks': { + 'static_test': { + 'ipv4_address': '172.16.100.100', + 'ipv6_address': 'fe80::1001:102' + } + }, + }], + networks={ + 'static_test': { + 'driver': 'bridge', + 'driver_opts': { + "com.docker.network.enable_ipv6": "true", + }, + 'ipam': { + 'driver': 'default', + 'config': [ + {"subnet": "172.16.100.0/24", + "gateway": "172.16.100.1"}, + {"subnet": "fe80::/64", + "gateway": "fe80::1001:1"} + ] + } + } + } + ) + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up(detached=True) + + network = self.client.networks(names=['static_test'])[0] + service_container = project.get_service('web').containers()[0] + + assert network['Options'] == { + "com.docker.network.enable_ipv6": "true" + } + + IPAMConfig = (service_container.inspect().get('NetworkSettings', {}). + get('Networks', {}).get('composetest_static_test', {}). + get('IPAMConfig', {})) + assert IPAMConfig.get('IPv4Address') == '172.16.100.100' + assert IPAMConfig.get('IPv6Address') == 'fe80::1001:102' + + @v2_1_only() + def test_up_with_enable_ipv6(self): + self.require_api_version('1.23') + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top', + 'networks': { + 'static_test': { + 'ipv6_address': 'fe80::1001:102' + } + }, + }], + networks={ + 'static_test': { + 'driver': 'bridge', + 'enable_ipv6': True, + 'ipam': { + 'driver': 'default', + 'config': [ + {"subnet": "fe80::/64", + "gateway": "fe80::1001:1"} + ] + } + } + } + ) + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up(detached=True) + network = self.client.networks(names=['static_test'])[0] + service_container = project.get_service('web').containers()[0] + + assert network['EnableIPv6'] is True + ipam_config = (service_container.inspect().get('NetworkSettings', {}). + get('Networks', {}).get('composetest_static_test', {}). + get('IPAMConfig', {})) + assert ipam_config.get('IPv6Address') == 'fe80::1001:102' + + @v2_only() + def test_up_with_network_static_addresses_missing_subnet(self): + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': { + 'static_test': { + 'ipv4_address': '172.16.100.100', + 'ipv6_address': 'fe80::1001:101' + } + }, + }], + networks={ + 'static_test': { + 'driver': 'bridge', + 'driver_opts': { + "com.docker.network.enable_ipv6": "true", + }, + 'ipam': { + 'driver': 'default', + }, + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + + with self.assertRaises(ProjectError): + project.up() + + @v2_1_only() + def test_up_with_network_link_local_ips(self): + config_data = build_config( + version=V2_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': { + 'linklocaltest': { + 'link_local_ips': ['169.254.8.8'] + } + } + }], + networks={ + 'linklocaltest': {'driver': 'bridge'} + } + ) + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + project.up(detached=True) + + service_container = project.get_service('web').containers(stopped=True)[0] + ipam_config = service_container.inspect().get( + 'NetworkSettings', {} + ).get( + 'Networks', {} + ).get( + 'composetest_linklocaltest', {} + ).get('IPAMConfig', {}) + assert 'LinkLocalIPs' in ipam_config + assert ipam_config['LinkLocalIPs'] == ['169.254.8.8'] + + @v2_1_only() + def test_up_with_isolation(self): + self.require_api_version('1.24') + config_data = build_config( + version=V2_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'isolation': 'default' + }], + ) + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + project.up(detached=True) + service_container = project.get_service('web').containers(stopped=True)[0] + assert service_container.inspect()['HostConfig']['Isolation'] == 'default' + + @v2_1_only() + def test_up_with_invalid_isolation(self): + self.require_api_version('1.24') + config_data = build_config( + version=V2_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'isolation': 'foobar' + }], + ) + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + with self.assertRaises(ProjectError): + project.up() + + @v2_only() + def test_project_up_with_network_internal(self): + self.require_api_version('1.23') + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': {'internal': None}, + }], + networks={ + 'internal': {'driver': 'bridge', 'internal': True}, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + + network = self.client.networks(names=['composetest_internal'])[0] + + assert network['Internal'] is True + + @v2_1_only() + def test_project_up_with_network_label(self): + self.require_api_version('1.23') + + network_name = 'network_with_label' + + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'networks': {network_name: None} + }], + networks={ + network_name: {'labels': {'label_key': 'label_val'}} + } + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data + ) + + project.up() + + networks = [ + n for n in self.client.networks() + if n['Name'].startswith('composetest_') + ] + + assert [n['Name'] for n in networks] == ['composetest_{}'.format(network_name)] + assert 'label_key' in networks[0]['Labels'] + assert networks[0]['Labels']['label_key'] == 'label_val' @v2_only() def test_project_up_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( @@ -542,12 +973,52 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v2_1_only() + def test_project_up_with_volume_labels(self): + self.require_api_version('1.23') + + volume_name = 'volume_with_label' + + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'volumes': [VolumeSpec.parse('{}:/data'.format(volume_name))] + }], + volumes={ + volume_name: { + 'labels': { + 'label_key': 'label_val' + } + } + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + + project.up() + + volumes = [ + v for v in self.client.volumes().get('Volumes', []) + if v['Name'].startswith('composetest_') + ] + + assert [v['Name'] for v in volumes] == ['composetest_{}'.format(volume_name)] + + assert 'label_key' in volumes[0]['Labels'] + assert volumes[0]['Labels']['label_key'] == 'label_val' + @v2_only() def test_project_up_logging_with_multiple_files(self): base_file = config.ConfigFile( 'base.yml', { - 'version': 2, + 'version': V2_0, 'services': { 'simple': {'image': 'busybox:latest', 'command': 'top'}, 'another': { @@ -566,7 +1037,7 @@ class ProjectTest(DockerClientTestCase): override_file = config.ConfigFile( 'override.yml', { - 'version': 2, + 'version': V2_0, 'services': { 'another': { 'logging': { @@ -594,44 +1065,78 @@ class ProjectTest(DockerClientTestCase): self.assertTrue(log_config) self.assertEqual(log_config.get('Type'), 'none') + @v2_only() + def test_project_up_port_mappings_with_multiple_files(self): + base_file = config.ConfigFile( + 'base.yml', + { + 'version': V2_0, + 'services': { + 'simple': { + 'image': 'busybox:latest', + 'command': 'top', + 'ports': ['1234:1234'] + }, + }, + + }) + override_file = config.ConfigFile( + 'override.yml', + { + 'version': V2_0, + 'services': { + 'simple': { + 'ports': ['1234:1234'] + } + } + + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + config_data = config.load(details) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + containers = project.containers() + self.assertEqual(len(containers), 1) + @v2_only() def test_initialize_volumes(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' }], volumes={vol_name: {}}, - networks={}, ) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - project.initialize_volumes() + project.volumes.initialize() volume_data = self.client.inspect_volume(full_vol_name) - self.assertEqual(volume_data['Name'], full_vol_name) - self.assertEqual(volume_data['Driver'], 'local') + assert volume_data['Name'] == full_vol_name + assert volume_data['Driver'] == 'local' @v2_only() def test_project_up_implicit_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' }], volumes={vol_name: {}}, - networks={}, ) project = Project.from_config( @@ -644,19 +1149,54 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(volume_data['Name'], full_vol_name) self.assertEqual(volume_data['Driver'], 'local') + @v3_only() + def test_project_up_with_secrets(self): + create_host_file(self.client, os.path.abspath('tests/fixtures/secrets/default')) + + config_data = build_config( + version=V3_1, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'cat /run/secrets/special', + 'secrets': [ + types.ServiceSecret.parse({'source': 'super', 'target': 'special'}), + ], + }], + secrets={ + 'super': { + 'file': os.path.abspath('tests/fixtures/secrets/default'), + }, + }, + ) + + project = Project.from_config( + client=self.client, + name='composetest', + config_data=config_data, + ) + project.up() + project.stop() + + containers = project.containers(stopped=True) + assert len(containers) == 1 + container, = containers + + output = container.logs() + assert output == b"This is the secret\n" + @v2_only() def test_initialize_volumes_invalid_volume_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' }], volumes={vol_name: {'driver': 'foobar'}}, - networks={}, ) project = Project.from_config( @@ -664,28 +1204,27 @@ class ProjectTest(DockerClientTestCase): config_data=config_data, client=self.client ) with self.assertRaises(config.ConfigurationError): - project.initialize_volumes() + project.volumes.initialize() @v2_only() def test_initialize_volumes_updated_driver(self): vol_name = '{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', 'command': 'top' }], volumes={vol_name: {'driver': 'local'}}, - networks={}, ) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - project.initialize_volumes() + project.volumes.initialize() volume_data = self.client.inspect_volume(full_vol_name) self.assertEqual(volume_data['Name'], full_vol_name) @@ -696,22 +1235,60 @@ class ProjectTest(DockerClientTestCase): ) project = Project.from_config( name='composetest', - config_data=config_data, client=self.client + config_data=config_data, + client=self.client ) with self.assertRaises(config.ConfigurationError) as e: - project.initialize_volumes() + project.volumes.initialize() assert 'Configuration for volume {0} specifies driver smb'.format( vol_name ) in str(e.exception) + @v2_only() + def test_initialize_volumes_updated_blank_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + + config_data = build_config( + version=V2_0, + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], + volumes={vol_name: {'driver': 'local'}}, + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.volumes.initialize() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + config_data = config_data._replace( + volumes={vol_name: {}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, + client=self.client + ) + project.volumes.initialize() + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + @v2_only() def test_initialize_volumes_external_volumes(self): # Use composetest_ prefix so it gets garbage-collected in tearDown() vol_name = 'composetest_{0:x}'.format(random.getrandbits(32)) full_vol_name = 'composetest_{0}'.format(vol_name) self.client.create_volume(vol_name) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -720,13 +1297,12 @@ class ProjectTest(DockerClientTestCase): volumes={ vol_name: {'external': True, 'external_name': vol_name} }, - networks=None, ) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) - project.initialize_volumes() + project.volumes.initialize() with self.assertRaises(NotFound): self.client.inspect_volume(full_vol_name) @@ -735,8 +1311,8 @@ class ProjectTest(DockerClientTestCase): def test_initialize_volumes_inexistent_external_volume(self): vol_name = '{0:x}'.format(random.getrandbits(32)) - config_data = config.Config( - version=2, + config_data = build_config( + version=V2_0, services=[{ 'name': 'web', 'image': 'busybox:latest', @@ -745,14 +1321,226 @@ class ProjectTest(DockerClientTestCase): volumes={ vol_name: {'external': True, 'external_name': vol_name} }, - networks=None, ) project = Project.from_config( name='composetest', config_data=config_data, client=self.client ) with self.assertRaises(config.ConfigurationError) as e: - project.initialize_volumes() + project.volumes.initialize() assert 'Volume {0} declared as external'.format( vol_name ) in str(e.exception) + + @v2_only() + def test_project_up_named_volumes_in_binds(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + + base_file = config.ConfigFile( + 'base.yml', + { + 'version': V2_0, + 'services': { + 'simple': { + 'image': 'busybox:latest', + 'command': 'top', + 'volumes': ['{0}:/data'.format(vol_name)] + }, + }, + 'volumes': { + vol_name: {'driver': 'local'} + } + + }) + config_details = config.ConfigDetails('.', [base_file]) + config_data = config.load(config_details) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + service = project.services[0] + self.assertEqual(service.name, 'simple') + volumes = service.options.get('volumes') + self.assertEqual(len(volumes), 1) + self.assertEqual(volumes[0].external, full_vol_name) + project.up() + engine_volumes = self.client.volumes()['Volumes'] + container = service.get_container() + assert [mount['Name'] for mount in container.get('Mounts')] == [full_vol_name] + assert next((v for v in engine_volumes if v['Name'] == vol_name), None) is None + + def test_project_up_orphans(self): + config_dict = { + 'service1': { + 'image': 'busybox:latest', + 'command': 'top', + } + } + + config_data = load_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + config_dict['service2'] = config_dict['service1'] + del config_dict['service1'] + + config_data = load_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + with mock.patch('compose.project.log') as mock_log: + project.up() + + mock_log.warning.assert_called_once_with(mock.ANY) + + assert len([ + ctnr for ctnr in project._labeled_containers() + if ctnr.labels.get(LABEL_SERVICE) == 'service1' + ]) == 1 + + project.up(remove_orphans=True) + + assert len([ + ctnr for ctnr in project._labeled_containers() + if ctnr.labels.get(LABEL_SERVICE) == 'service1' + ]) == 0 + + @v2_1_only() + def test_project_up_healthy_dependency(self): + config_dict = { + 'version': '2.1', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'healthcheck': { + 'test': 'exit 0', + 'retries': 1, + 'timeout': '10s', + 'interval': '0.1s' + }, + }, + 'svc2': { + 'image': 'busybox:latest', + 'command': 'top', + 'depends_on': { + 'svc1': {'condition': 'service_healthy'}, + } + } + } + } + config_data = load_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + project.up() + containers = project.containers() + assert len(containers) == 2 + + svc1 = project.get_service('svc1') + svc2 = project.get_service('svc2') + assert 'svc1' in svc2.get_dependency_names() + assert svc1.is_healthy() + + @v2_1_only() + def test_project_up_unhealthy_dependency(self): + config_dict = { + 'version': '2.1', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'healthcheck': { + 'test': 'exit 1', + 'retries': 1, + 'timeout': '10s', + 'interval': '0.1s' + }, + }, + 'svc2': { + 'image': 'busybox:latest', + 'command': 'top', + 'depends_on': { + 'svc1': {'condition': 'service_healthy'}, + } + } + } + } + config_data = load_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + with pytest.raises(ProjectError): + project.up() + containers = project.containers() + assert len(containers) == 1 + + svc1 = project.get_service('svc1') + svc2 = project.get_service('svc2') + assert 'svc1' in svc2.get_dependency_names() + with pytest.raises(HealthCheckFailed): + svc1.is_healthy() + + @v2_1_only() + def test_project_up_no_healthcheck_dependency(self): + config_dict = { + 'version': '2.1', + 'services': { + 'svc1': { + 'image': 'busybox:latest', + 'command': 'top', + 'healthcheck': { + 'disable': True + }, + }, + 'svc2': { + 'image': 'busybox:latest', + 'command': 'top', + 'depends_on': { + 'svc1': {'condition': 'service_healthy'}, + } + } + } + } + config_data = load_config(config_dict) + project = Project.from_config( + name='composetest', config_data=config_data, client=self.client + ) + with pytest.raises(ProjectError): + project.up() + containers = project.containers() + assert len(containers) == 1 + + svc1 = project.get_service('svc1') + svc2 = project.get_service('svc2') + assert 'svc1' in svc2.get_dependency_names() + with pytest.raises(NoHealthCheckConfigured): + svc1.is_healthy() + + +def create_host_file(client, filename): + dirname = os.path.dirname(filename) + + with open(filename, 'r') as fh: + content = fh.read() + + container = client.create_container( + 'busybox:latest', + ['sh', '-c', 'echo -n "{}" > {}'.format(content, filename)], + volumes={dirname: {}}, + host_config=client.create_host_config( + binds={dirname: {'bind': dirname, 'ro': False}}, + network_mode='none', + ), + ) + try: + client.start(container) + exitcode = client.wait(container) + + if exitcode != 0: + output = client.logs(container) + raise Exception( + "Container exited with code {}:\n{}".format(exitcode, output)) + finally: + client.remove_container(container, force=True) diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index b544783a..2a2d1b56 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -20,6 +20,11 @@ class ResilienceTest(DockerClientTestCase): self.db.start_container(container) self.host_path = container.get_mount('/var/db')['Source'] + def tearDown(self): + del self.project + del self.db + super(ResilienceTest, self).tearDown() + def test_successful_recreate(self): self.project.up(strategy=ConvergenceStrategy.always) container = self.db.containers()[0] diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index bce3999b..09758eee 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -6,8 +6,8 @@ import shutil import tempfile from os import path +import pytest from docker.errors import APIError -from pytest import mark from six import StringIO from six import text_type @@ -25,10 +25,13 @@ from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.const import LABEL_VERSION from compose.container import Container +from compose.project import OneOffFilter from compose.service import ConvergencePlan from compose.service import ConvergenceStrategy -from compose.service import Net +from compose.service import NetworkMode from compose.service import Service +from tests.integration.testcases import v2_1_only +from tests.integration.testcases import v2_only def create_and_start_container(service, **override_options): @@ -61,7 +64,7 @@ class ServiceTest(DockerClientTestCase): db = self.create_service('db') container = db.create_container(one_off=True) self.assertEqual(db.containers(stopped=True), []) - self.assertEqual(db.containers(one_off=True, stopped=True), [container]) + self.assertEqual(db.containers(one_off=OneOffFilter.only, stopped=True), [container]) def test_project_is_added_to_container_name(self): service = self.create_service('web') @@ -103,6 +106,13 @@ class ServiceTest(DockerClientTestCase): container.start() self.assertEqual(container.get('HostConfig.CpuQuota'), 40000) + def test_create_container_with_shm_size(self): + self.require_api_version('1.22') + service = self.create_service('db', shm_size=67108864) + container = service.create_container() + service.start_container(container) + self.assertEqual(container.get('HostConfig.ShmSize'), 67108864) + def test_create_container_with_extra_hosts_list(self): extra_hosts = ['somehost:162.242.195.82', 'otherhost:50.31.209.229'] service = self.create_service('db', extra_hosts=extra_hosts) @@ -129,7 +139,7 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', read_only=read_only) container = service.create_container() service.start_container(container) - self.assertEqual(container.get('HostConfig.ReadonlyRootfs'), read_only, container.get('HostConfig')) + assert container.get('HostConfig.ReadonlyRootfs') == read_only def test_create_container_with_security_opt(self): security_opt = ['label:disable'] @@ -267,6 +277,30 @@ class ServiceTest(DockerClientTestCase): self.client.inspect_container, old_container.id) + def test_execute_convergence_plan_recreate_twice(self): + service = self.create_service( + 'db', + volumes=[VolumeSpec.parse('/etc')], + entrypoint=['top'], + command=['-d', '1']) + + orig_container = service.create_container() + service.start_container(orig_container) + + orig_container.inspect() # reload volume data + volume_path = orig_container.get_mount('/etc')['Source'] + + # Do this twice to reproduce the bug + for _ in range(2): + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [orig_container])) + + assert new_container.get_mount('/etc')['Source'] == volume_path + assert ('affinity:container==%s' % orig_container.id in + new_container.get('Config.Env')) + + orig_container = new_container + def test_execute_convergence_plan_when_containers_are_stopped(self): service = self.create_service( 'db', @@ -344,6 +378,31 @@ class ServiceTest(DockerClientTestCase): ) self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) + def test_execute_convergence_plan_when_host_volume_is_removed(self): + host_path = '/tmp/host-path' + service = self.create_service( + 'db', + build={'context': 'tests/fixtures/dockerfile-with-volume'}, + volumes=[VolumeSpec(host_path, '/data', 'rw')]) + + old_container = create_and_start_container(service) + assert ( + [mount['Destination'] for mount in old_container.get('Mounts')] == + ['/data'] + ) + service.options['volumes'] = [] + + with mock.patch('compose.service.log', autospec=True) as mock_log: + new_container, = service.execute_convergence_plan( + ConvergencePlan('recreate', [old_container])) + + assert not mock_log.warn.called + assert ( + [mount['Destination'] for mount in new_container.get('Mounts')] == + ['/data'] + ) + assert new_container.get_mount('/data')['Source'] != host_path + def test_execute_convergence_plan_without_start(self): service = self.create_service( 'db', @@ -354,7 +413,9 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(len(service.containers()), 0) self.assertEqual(len(service.containers(stopped=True)), 1) - containers = service.execute_convergence_plan(ConvergencePlan('recreate', containers), start=False) + containers = service.execute_convergence_plan( + ConvergencePlan('recreate', containers), + start=False) self.assertEqual(len(service.containers()), 0) self.assertEqual(len(service.containers(stopped=True)), 1) @@ -372,7 +433,6 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) self.assertEqual(db.containers()[0].environment['FOO'], 'BAR') - @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_container_creates_links(self): db = self.create_service('db') web = self.create_service('web', links=[(db, None)]) @@ -389,7 +449,6 @@ class ServiceTest(DockerClientTestCase): 'db']) ) - @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_container_creates_links_with_names(self): db = self.create_service('db') web = self.create_service('web', links=[(db, 'custom_link_name')]) @@ -433,14 +492,13 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db) self.assertEqual(set(get_links(c)), set([])) - @mark.skipif(True, reason="Engine returns error - needs investigating") def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') create_and_start_container(db) create_and_start_container(db) - c = create_and_start_container(db, one_off=True) + c = create_and_start_container(db, one_off=OneOffFilter.only) self.assertEqual( set(get_links(c)), @@ -536,12 +594,12 @@ class ServiceTest(DockerClientTestCase): service.build() assert service.image() - def test_start_container_stays_unpriviliged(self): + def test_start_container_stays_unprivileged(self): service = self.create_service('web') container = create_and_start_container(service).inspect() self.assertEqual(container['HostConfig']['Privileged'], False) - def test_start_container_becomes_priviliged(self): + def test_start_container_becomes_privileged(self): service = self.create_service('web', privileged=True) container = create_and_start_container(service).inspect() self.assertEqual(container['HostConfig']['Privileged'], True) @@ -681,7 +739,10 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(len(service.containers()), 1) self.assertTrue(service.containers()[0].is_running) - self.assertIn("ERROR: for 2 Boom", mock_stderr.getvalue()) + self.assertIn( + "ERROR: for composetest_web_2 Cannot create container for service web: Boom", + mock_stderr.getvalue() + ) def test_scale_with_unexpected_exception(self): """Test that when scaling if the API returns an error, that is not of type @@ -712,17 +773,17 @@ class ServiceTest(DockerClientTestCase): container = service.create_container(number=next_number, quiet=True) container.start() - self.assertTrue(container.is_running) - self.assertEqual(len(service.containers()), 1) + container.inspect() + assert container.is_running + assert len(service.containers()) == 1 service.scale(1) - - self.assertEqual(len(service.containers()), 1) + assert len(service.containers()) == 1 container.inspect() - self.assertTrue(container.is_running) + assert container.is_running captured_output = mock_log.info.call_args[0] - self.assertIn('Desired container number already achieved', captured_output) + assert 'Desired container number already achieved' in captured_output @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): @@ -730,7 +791,7 @@ class ServiceTest(DockerClientTestCase): results in warning output. """ service = self.create_service('app', container_name='custom-container') - self.assertEqual(service.custom_container_name(), 'custom-container') + self.assertEqual(service.custom_container_name, 'custom-container') service.scale(3) @@ -748,20 +809,27 @@ class ServiceTest(DockerClientTestCase): containers = service.containers() self.assertEqual(len(containers), 2) for container in containers: - self.assertEqual(list(container.inspect()['HostConfig']['PortBindings'].keys()), ['8000/tcp']) + self.assertEqual( + list(container.get('HostConfig.PortBindings')), + ['8000/tcp']) + + def test_scale_with_immediate_exit(self): + service = self.create_service('web', image='busybox', command='true') + service.scale(2) + assert len(service.containers(stopped=True)) == 2 def test_network_mode_none(self): - service = self.create_service('web', net=Net('none')) + service = self.create_service('web', network_mode=NetworkMode('none')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'none') def test_network_mode_bridged(self): - service = self.create_service('web', net=Net('bridge')) + service = self.create_service('web', network_mode=NetworkMode('bridge')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'bridge') def test_network_mode_host(self): - service = self.create_service('web', net=Net('host')) + service = self.create_service('web', network_mode=NetworkMode('host')) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.NetworkMode'), 'host') @@ -775,6 +843,18 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.PidMode'), 'host') + @v2_1_only() + def test_userns_mode_none_defined(self): + service = self.create_service('web', userns_mode=None) + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.UsernsMode'), '') + + @v2_1_only() + def test_userns_mode_host(self): + service = self.create_service('web', userns_mode='host') + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.UsernsMode'), 'host') + def test_dns_no_value(self): service = self.create_service('web') container = create_and_start_container(service) @@ -785,11 +865,29 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.Dns'), ['8.8.8.8', '9.9.9.9']) + def test_mem_swappiness(self): + service = self.create_service('web', mem_swappiness=11) + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.MemorySwappiness'), 11) + def test_restart_always_value(self): service = self.create_service('web', restart={'Name': 'always'}) container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.RestartPolicy.Name'), 'always') + def test_oom_score_adj_value(self): + service = self.create_service('web', oom_score_adj=500) + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.OomScoreAdj'), 500) + + def test_group_add_value(self): + service = self.create_service('web', group_add=["root", "1"]) + container = create_and_start_container(service) + + host_container_groupadd = container.get('HostConfig.GroupAdd') + self.assertTrue("root" in host_container_groupadd) + self.assertTrue("1" in host_container_groupadd) + def test_restart_on_failure_value(self): service = self.create_service('web', restart={ 'Name': 'on-failure', @@ -814,13 +912,21 @@ class ServiceTest(DockerClientTestCase): container = create_and_start_container(service) self.assertEqual(container.get('HostConfig.DnsSearch'), ['dc1.example.com', 'dc2.example.com']) + @v2_only() + def test_tmpfs(self): + service = self.create_service('web', tmpfs=['/run']) + container = create_and_start_container(service) + self.assertEqual(container.get('HostConfig.Tmpfs'), {'/run': ''}) + def test_working_dir_param(self): service = self.create_service('container', working_dir='/working/dir/sample') container = service.create_container() self.assertEqual(container.get('Config.WorkingDir'), '/working/dir/sample') def test_split_env(self): - service = self.create_service('web', environment=['NORMAL=F1', 'CONTAINS_EQUALS=F=2', 'TRAILING_EQUALS=']) + service = self.create_service( + 'web', + environment=['NORMAL=F1', 'CONTAINS_EQUALS=F=2', 'TRAILING_EQUALS=']) env = create_and_start_container(service).environment for k, v in {'NORMAL': 'F1', 'CONTAINS_EQUALS': 'F=2', 'TRAILING_EQUALS': ''}.items(): self.assertEqual(env[k], v) @@ -859,7 +965,7 @@ class ServiceTest(DockerClientTestCase): 'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', - 'NO_DEF': '' + 'NO_DEF': None }.items(): self.assertEqual(env[k], v) @@ -906,7 +1012,7 @@ class ServiceTest(DockerClientTestCase): def test_custom_container_name(self): service = self.create_service('web', container_name='my-web-container') - self.assertEqual(service.custom_container_name(), 'my-web-container') + self.assertEqual(service.custom_container_name, 'my-web-container') container = create_and_start_container(service) self.assertEqual(container.name, 'my-web-container') @@ -914,6 +1020,7 @@ class ServiceTest(DockerClientTestCase): one_off_container = service.create_container(one_off=True) self.assertNotEqual(one_off_container.name, 'my-web-container') + @pytest.mark.skipif(True, reason="Broken on 1.11.0rc1") def test_log_drive_invalid(self): service = self.create_service('web', logging={'driver': 'xxx'}) expected_error_msg = "logger: no log driver named 'xxx' is registered" @@ -964,12 +1071,10 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(set(service.duplicate_containers()), set([duplicate])) -def converge(service, - strategy=ConvergenceStrategy.changed, - do_build=True): +def converge(service, strategy=ConvergenceStrategy.changed): """Create a converge plan from a strategy and execute the plan.""" plan = service.convergence_plan(strategy) - return service.execute_convergence_plan(plan, do_build=do_build, timeout=1) + return service.execute_convergence_plan(plan, timeout=1) class ConfigHashTest(DockerClientTestCase): diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 36099d2d..07b28e78 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -38,8 +38,8 @@ class BasicProjectTest(ProjectTestCase): super(BasicProjectTest, self).setUp() self.cfg = { - 'db': {'image': 'busybox:latest'}, - 'web': {'image': 'busybox:latest'}, + 'db': {'image': 'busybox:latest', 'command': 'top'}, + 'web': {'image': 'busybox:latest', 'command': 'top'}, } def test_no_change(self): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 5870946d..efc1551b 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -10,6 +10,11 @@ from pytest import skip from .. import unittest from compose.cli.docker_client import docker_client from compose.config.config import resolve_environment +from compose.config.config import V1 +from compose.config.config import V2_0 +from compose.config.config import V2_1 +from compose.config.config import V3_0 +from compose.config.environment import Environment from compose.const import API_VERSIONS from compose.const import LABEL_PROJECT from compose.progress_stream import stream_output @@ -30,19 +35,26 @@ def get_links(container): return [format_link(link) for link in links] -def engine_version_too_low_for_v2(): +def engine_max_version(): if 'DOCKER_VERSION' not in os.environ: - return False + return V3_0 version = os.environ['DOCKER_VERSION'].partition('-')[0] - return version_lt(version, '1.10') + if version_lt(version, '1.10'): + return V1 + if version_lt(version, '1.12'): + return V2_0 + if version_lt(version, '1.13'): + return V2_1 + return V3_0 -def v2_only(): +def build_version_required_decorator(ignored_versions): def decorator(f): @functools.wraps(f) def wrapper(self, *args, **kwargs): - if engine_version_too_low_for_v2(): - skip("Engine version is too low") + max_version = engine_max_version() + if max_version in ignored_versions: + skip("Engine version %s is too low" % max_version) return return f(self, *args, **kwargs) return wrapper @@ -50,15 +62,27 @@ def v2_only(): return decorator +def v2_only(): + return build_version_required_decorator((V1,)) + + +def v2_1_only(): + return build_version_required_decorator((V1, V2_0)) + + +def v3_only(): + return build_version_required_decorator((V1, V2_0, V2_1)) + + class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): - if engine_version_too_low_for_v2(): - version = API_VERSIONS[1] - else: - version = API_VERSIONS[2] + version = API_VERSIONS[engine_max_version()] + cls.client = docker_client(Environment(), version) - cls.client = docker_client(version) + @classmethod + def tearDownClass(cls): + del cls.client def tearDown(self): for c in self.client.containers( @@ -87,7 +111,9 @@ class DockerClientTestCase(unittest.TestCase): if 'command' not in kwargs: kwargs['command'] = ["top"] - kwargs['environment'] = resolve_environment(kwargs) + kwargs['environment'] = resolve_environment( + kwargs, Environment.from_env_file(None) + ) labels = dict(kwargs.setdefault('labels', {})) labels['com.docker.compose.test-name'] = self.id() diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py index 706179ed..add16962 100644 --- a/tests/integration/volume_test.py +++ b/tests/integration/volume_test.py @@ -4,6 +4,8 @@ from __future__ import unicode_literals from docker.errors import DockerException from .testcases import DockerClientTestCase +from compose.const import LABEL_PROJECT +from compose.const import LABEL_VOLUME from compose.volume import Volume @@ -17,6 +19,8 @@ class VolumeTest(DockerClientTestCase): self.client.remove_volume(volume.full_name) except DockerException: pass + del self.tmp_volumes + super(VolumeTest, self).tearDown() def create_volume(self, name, driver=None, opts=None, external=None): if external and isinstance(external, bool): @@ -92,3 +96,11 @@ class VolumeTest(DockerClientTestCase): assert vol.exists() is False vol.create() assert vol.exists() is True + + def test_volume_default_labels(self): + vol = self.create_volume('volume01') + vol.create() + vol_data = vol.inspect() + labels = vol_data['Labels'] + assert labels[LABEL_VOLUME] == vol.name + assert labels[LABEL_PROJECT] == vol.project diff --git a/tests/unit/bundle_test.py b/tests/unit/bundle_test.py new file mode 100644 index 00000000..21bdb31b --- /dev/null +++ b/tests/unit/bundle_test.py @@ -0,0 +1,219 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import docker +import mock +import pytest + +from compose import bundle +from compose import service +from compose.cli.errors import UserError +from compose.config.config import Config + + +@pytest.fixture +def mock_service(): + return mock.create_autospec( + service.Service, + client=mock.create_autospec(docker.APIClient), + options={}) + + +def test_get_image_digest_exists(mock_service): + mock_service.options['image'] = 'abcd' + mock_service.image.return_value = {'RepoDigests': ['digest1']} + digest = bundle.get_image_digest(mock_service) + assert digest == 'digest1' + + +def test_get_image_digest_image_uses_digest(mock_service): + mock_service.options['image'] = image_id = 'redis@sha256:digest' + + digest = bundle.get_image_digest(mock_service) + assert digest == image_id + assert not mock_service.image.called + + +def test_get_image_digest_no_image(mock_service): + with pytest.raises(UserError) as exc: + bundle.get_image_digest(service.Service(name='theservice')) + + assert "doesn't define an image tag" in exc.exconly() + + +def test_push_image_with_saved_digest(mock_service): + mock_service.options['build'] = '.' + mock_service.options['image'] = image_id = 'abcd' + mock_service.push.return_value = expected = 'sha256:thedigest' + mock_service.image.return_value = {'RepoDigests': ['digest1']} + + digest = bundle.push_image(mock_service) + assert digest == image_id + '@' + expected + + mock_service.push.assert_called_once_with() + assert not mock_service.client.push.called + + +def test_push_image(mock_service): + mock_service.options['build'] = '.' + mock_service.options['image'] = image_id = 'abcd' + mock_service.push.return_value = expected = 'sha256:thedigest' + mock_service.image.return_value = {'RepoDigests': []} + + digest = bundle.push_image(mock_service) + assert digest == image_id + '@' + expected + + mock_service.push.assert_called_once_with() + mock_service.client.pull.assert_called_once_with(digest) + + +def test_to_bundle(): + image_digests = {'a': 'aaaa', 'b': 'bbbb'} + services = [ + {'name': 'a', 'build': '.', }, + {'name': 'b', 'build': './b'}, + ] + config = Config( + version=2, + services=services, + volumes={'special': {}}, + networks={'extra': {}}, + secrets={}) + + with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + output = bundle.to_bundle(config, image_digests) + + assert mock_log.mock_calls == [ + mock.call("Unsupported top level key 'networks' - ignoring"), + mock.call("Unsupported top level key 'volumes' - ignoring"), + ] + + assert output == { + 'Version': '0.1', + 'Services': { + 'a': {'Image': 'aaaa', 'Networks': ['default']}, + 'b': {'Image': 'bbbb', 'Networks': ['default']}, + } + } + + +def test_convert_service_to_bundle(): + name = 'theservice' + image_digest = 'thedigest' + service_dict = { + 'ports': ['80'], + 'expose': ['1234'], + 'networks': {'extra': {}}, + 'command': 'foo', + 'entrypoint': 'entry', + 'environment': {'BAZ': 'ENV'}, + 'build': '.', + 'working_dir': '/tmp', + 'user': 'root', + 'labels': {'FOO': 'LABEL'}, + 'privileged': True, + } + + with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + config = bundle.convert_service_to_bundle(name, service_dict, image_digest) + + mock_log.assert_called_once_with( + "Unsupported key 'privileged' in services.theservice - ignoring") + + assert config == { + 'Image': image_digest, + 'Ports': [ + {'Protocol': 'tcp', 'Port': 80}, + {'Protocol': 'tcp', 'Port': 1234}, + ], + 'Networks': ['extra'], + 'Command': ['entry', 'foo'], + 'Env': ['BAZ=ENV'], + 'WorkingDir': '/tmp', + 'User': 'root', + 'Labels': {'FOO': 'LABEL'}, + } + + +def test_set_command_and_args_none(): + config = {} + bundle.set_command_and_args(config, [], []) + assert config == {} + + +def test_set_command_and_args_from_command(): + config = {} + bundle.set_command_and_args(config, [], "echo ok") + assert config == {'Args': ['echo', 'ok']} + + +def test_set_command_and_args_from_entrypoint(): + config = {} + bundle.set_command_and_args(config, "echo entry", []) + assert config == {'Command': ['echo', 'entry']} + + +def test_set_command_and_args_from_both(): + config = {} + bundle.set_command_and_args(config, "echo entry", ["extra", "arg"]) + assert config == {'Command': ['echo', 'entry', "extra", "arg"]} + + +def test_make_service_networks_default(): + name = 'theservice' + service_dict = {} + + with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + networks = bundle.make_service_networks(name, service_dict) + + assert not mock_log.called + assert networks == ['default'] + + +def test_make_service_networks(): + name = 'theservice' + service_dict = { + 'networks': { + 'foo': { + 'aliases': ['one', 'two'], + }, + 'bar': {} + }, + } + + with mock.patch('compose.bundle.log.warn', autospec=True) as mock_log: + networks = bundle.make_service_networks(name, service_dict) + + mock_log.assert_called_once_with( + "Unsupported key 'aliases' in services.theservice.networks.foo - ignoring") + assert sorted(networks) == sorted(service_dict['networks']) + + +def test_make_port_specs(): + service_dict = { + 'expose': ['80', '500/udp'], + 'ports': [ + '400:80', + '222', + '127.0.0.1:8001:8001', + '127.0.0.1:5000-5001:3000-3001'], + } + port_specs = bundle.make_port_specs(service_dict) + assert port_specs == [ + {'Protocol': 'tcp', 'Port': 80}, + {'Protocol': 'tcp', 'Port': 222}, + {'Protocol': 'tcp', 'Port': 8001}, + {'Protocol': 'tcp', 'Port': 3000}, + {'Protocol': 'tcp', 'Port': 3001}, + {'Protocol': 'udp', 'Port': 500}, + ] + + +def test_make_port_spec_with_protocol(): + port_spec = bundle.make_port_spec("5000/udp") + assert port_spec == {'Protocol': 'udp', 'Port': 5000} + + +def test_make_port_spec_default_protocol(): + port_spec = bundle.make_port_spec("50000") + assert port_spec == {'Protocol': 'tcp', 'Port': 50000} diff --git a/tests/unit/cli/command_test.py b/tests/unit/cli/command_test.py index 18044672..50fc84e1 100644 --- a/tests/unit/cli/command_test.py +++ b/tests/unit/cli/command_test.py @@ -1,23 +1,68 @@ from __future__ import absolute_import from __future__ import unicode_literals +import os +import ssl + import pytest -from requests.exceptions import ConnectionError -from compose.cli import errors -from compose.cli.command import friendly_error_message +from compose.cli.command import get_config_path_from_options +from compose.cli.command import get_tls_version +from compose.config.environment import Environment +from compose.const import IS_WINDOWS_PLATFORM from tests import mock -from tests import unittest -class FriendlyErrorMessageTestCase(unittest.TestCase): +class TestGetConfigPathFromOptions(object): - def test_dispatch_generic_connection_error(self): - with pytest.raises(errors.ConnectionErrorGeneric): - with mock.patch( - 'compose.cli.command.call_silently', - autospec=True, - side_effect=[0, 1] - ): - with friendly_error_message(): - raise ConnectionError() + def test_path_from_options(self): + paths = ['one.yml', 'two.yml'] + opts = {'--file': paths} + environment = Environment.from_env_file('.') + assert get_config_path_from_options('.', opts, environment) == paths + + def test_single_path_from_env(self): + with mock.patch.dict(os.environ): + os.environ['COMPOSE_FILE'] = 'one.yml' + environment = Environment.from_env_file('.') + assert get_config_path_from_options('.', {}, environment) == ['one.yml'] + + @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix separator') + def test_multiple_path_from_env(self): + with mock.patch.dict(os.environ): + os.environ['COMPOSE_FILE'] = 'one.yml:two.yml' + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', {}, environment + ) == ['one.yml', 'two.yml'] + + @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows separator') + def test_multiple_path_from_env_windows(self): + with mock.patch.dict(os.environ): + os.environ['COMPOSE_FILE'] = 'one.yml;two.yml' + environment = Environment.from_env_file('.') + assert get_config_path_from_options( + '.', {}, environment + ) == ['one.yml', 'two.yml'] + + def test_no_path(self): + environment = Environment.from_env_file('.') + assert not get_config_path_from_options('.', {}, environment) + + +class TestGetTlsVersion(object): + def test_get_tls_version_default(self): + environment = {} + assert get_tls_version(environment) is None + + @pytest.mark.skipif(not hasattr(ssl, 'PROTOCOL_TLSv1_2'), reason='TLS v1.2 unsupported') + def test_get_tls_version_upgrade(self): + environment = {'COMPOSE_TLS_VERSION': 'TLSv1_2'} + assert get_tls_version(environment) == ssl.PROTOCOL_TLSv1_2 + + def test_get_tls_version_unavailable(self): + environment = {'COMPOSE_TLS_VERSION': 'TLSv5_5'} + with mock.patch('compose.cli.command.log') as mock_log: + tls_version = get_tls_version(environment) + mock_log.warn.assert_called_once_with(mock.ANY) + assert tls_version is None diff --git a/tests/unit/cli/docker_client_test.py b/tests/unit/cli/docker_client_test.py index d497495b..aaa935af 100644 --- a/tests/unit/cli/docker_client_test.py +++ b/tests/unit/cli/docker_client_test.py @@ -2,8 +2,15 @@ from __future__ import absolute_import from __future__ import unicode_literals import os +import platform -from compose.cli import docker_client +import docker +import pytest + +import compose +from compose.cli import errors +from compose.cli.docker_client import docker_client +from compose.cli.docker_client import tls_config_from_options from tests import mock from tests import unittest @@ -13,10 +20,140 @@ class DockerClientTestCase(unittest.TestCase): def test_docker_client_no_home(self): with mock.patch.dict(os.environ): del os.environ['HOME'] - docker_client.docker_client() + docker_client(os.environ) + @mock.patch.dict(os.environ) def test_docker_client_with_custom_timeout(self): - timeout = 300 - with mock.patch('compose.cli.docker_client.HTTP_TIMEOUT', 300): - client = docker_client.docker_client() - self.assertEqual(client.timeout, int(timeout)) + os.environ['COMPOSE_HTTP_TIMEOUT'] = '123' + client = docker_client(os.environ) + assert client.timeout == 123 + + @mock.patch.dict(os.environ) + def test_custom_timeout_error(self): + os.environ['COMPOSE_HTTP_TIMEOUT'] = '123' + client = docker_client(os.environ) + + with mock.patch('compose.cli.errors.log') as fake_log: + with pytest.raises(errors.ConnectionError): + with errors.handle_connection_errors(client): + raise errors.RequestsConnectionError( + errors.ReadTimeoutError(None, None, None)) + + assert fake_log.error.call_count == 1 + assert '123' in fake_log.error.call_args[0][0] + + with mock.patch('compose.cli.errors.log') as fake_log: + with pytest.raises(errors.ConnectionError): + with errors.handle_connection_errors(client): + raise errors.ReadTimeout() + + assert fake_log.error.call_count == 1 + assert '123' in fake_log.error.call_args[0][0] + + def test_user_agent(self): + client = docker_client(os.environ) + expected = "docker-compose/{0} docker-py/{1} {2}/{3}".format( + compose.__version__, + docker.__version__, + platform.system(), + platform.release() + ) + self.assertEqual(client.headers['User-Agent'], expected) + + +class TLSConfigTestCase(unittest.TestCase): + ca_cert = 'tests/fixtures/tls/ca.pem' + client_cert = 'tests/fixtures/tls/cert.pem' + key = 'tests/fixtures/tls/key.key' + + def test_simple_tls(self): + options = {'--tls': True} + result = tls_config_from_options(options) + assert result is True + + def test_tls_ca_cert(self): + options = { + '--tlscacert': self.ca_cert, '--tlsverify': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_ca_cert_explicit(self): + options = { + '--tlscacert': self.ca_cert, '--tls': True, + '--tlsverify': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_client_cert(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + + def test_tls_client_cert_explicit(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key, + '--tls': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + + def test_tls_client_and_ca(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key, + '--tlsverify': True, '--tlscacert': self.ca_cert + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_client_and_ca_explicit(self): + options = { + '--tlscert': self.client_cert, '--tlskey': self.key, + '--tlsverify': True, '--tlscacert': self.ca_cert, + '--tls': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (options['--tlscert'], options['--tlskey']) + assert result.ca_cert == options['--tlscacert'] + assert result.verify is True + + def test_tls_client_missing_key(self): + options = {'--tlscert': self.client_cert} + with pytest.raises(docker.errors.TLSParameterError): + tls_config_from_options(options) + + options = {'--tlskey': self.key} + with pytest.raises(docker.errors.TLSParameterError): + tls_config_from_options(options) + + def test_assert_hostname_explicit_skip(self): + options = {'--tlscacert': self.ca_cert, '--skip-hostname-check': True} + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.assert_hostname is False + + def test_tls_client_and_ca_quoted_paths(self): + options = { + '--tlscacert': '"{0}"'.format(self.ca_cert), + '--tlscert': '"{0}"'.format(self.client_cert), + '--tlskey': '"{0}"'.format(self.key), + '--tlsverify': True + } + result = tls_config_from_options(options) + assert isinstance(result, docker.tls.TLSConfig) + assert result.cert == (self.client_cert, self.key) + assert result.ca_cert == self.ca_cert + assert result.verify is True diff --git a/tests/unit/cli/errors_test.py b/tests/unit/cli/errors_test.py new file mode 100644 index 00000000..a7b57562 --- /dev/null +++ b/tests/unit/cli/errors_test.py @@ -0,0 +1,51 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import pytest +from docker.errors import APIError +from requests.exceptions import ConnectionError + +from compose.cli import errors +from compose.cli.errors import handle_connection_errors +from tests import mock + + +@pytest.yield_fixture +def mock_logging(): + with mock.patch('compose.cli.errors.log', autospec=True) as mock_log: + yield mock_log + + +def patch_find_executable(side_effect): + return mock.patch( + 'compose.cli.errors.find_executable', + autospec=True, + side_effect=side_effect) + + +class TestHandleConnectionErrors(object): + + def test_generic_connection_error(self, mock_logging): + with pytest.raises(errors.ConnectionError): + with patch_find_executable(['/bin/docker', None]): + with handle_connection_errors(mock.Mock()): + raise ConnectionError() + + _, args, _ = mock_logging.error.mock_calls[0] + assert "Couldn't connect to Docker daemon" in args[0] + + def test_api_error_version_mismatch(self, mock_logging): + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise APIError(None, None, b"client is newer than server") + + _, args, _ = mock_logging.error.mock_calls[0] + assert "Docker Engine of version 1.10.0 or greater" in args[0] + + def test_api_error_version_other(self, mock_logging): + msg = b"Something broke!" + with pytest.raises(errors.ConnectionError): + with handle_connection_errors(mock.Mock(api_version='1.22')): + raise APIError(None, None, msg) + + mock_logging.error.assert_called_once_with(msg) diff --git a/tests/unit/cli/formatter_test.py b/tests/unit/cli/formatter_test.py index 1c3b6a68..4aa025e6 100644 --- a/tests/unit/cli/formatter_test.py +++ b/tests/unit/cli/formatter_test.py @@ -11,8 +11,8 @@ from tests import unittest MESSAGE = 'this is the message' -def makeLogRecord(level): - return logging.LogRecord('name', level, 'pathame', 0, MESSAGE, (), None) +def make_log_record(level, message=None): + return logging.LogRecord('name', level, 'pathame', 0, message or MESSAGE, (), None) class ConsoleWarningFormatterTestCase(unittest.TestCase): @@ -21,15 +21,33 @@ class ConsoleWarningFormatterTestCase(unittest.TestCase): self.formatter = ConsoleWarningFormatter() def test_format_warn(self): - output = self.formatter.format(makeLogRecord(logging.WARN)) + output = self.formatter.format(make_log_record(logging.WARN)) expected = colors.yellow('WARNING') + ': ' assert output == expected + MESSAGE def test_format_error(self): - output = self.formatter.format(makeLogRecord(logging.ERROR)) + output = self.formatter.format(make_log_record(logging.ERROR)) expected = colors.red('ERROR') + ': ' assert output == expected + MESSAGE def test_format_info(self): - output = self.formatter.format(makeLogRecord(logging.INFO)) + output = self.formatter.format(make_log_record(logging.INFO)) assert output == MESSAGE + + def test_format_unicode_info(self): + message = b'\xec\xa0\x95\xec\x88\x98\xec\xa0\x95' + output = self.formatter.format(make_log_record(logging.INFO, message)) + print(output) + assert output == message.decode('utf-8') + + def test_format_unicode_warn(self): + message = b'\xec\xa0\x95\xec\x88\x98\xec\xa0\x95' + output = self.formatter.format(make_log_record(logging.WARN, message)) + expected = colors.yellow('WARNING') + ': ' + assert output == '{0}{1}'.format(expected, message.decode('utf-8')) + + def test_format_unicode_error(self): + message = b'\xec\xa0\x95\xec\x88\x98\xec\xa0\x95' + output = self.formatter.format(make_log_record(logging.ERROR, message)) + expected = colors.red('ERROR') + ': ' + assert output == '{0}{1}'.format(expected, message.decode('utf-8')) diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 5b04226c..b908eb68 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -1,27 +1,25 @@ from __future__ import absolute_import from __future__ import unicode_literals -import pytest -import six +import itertools -from compose.cli.log_printer import LogPrinter +import pytest +import requests +import six +from docker.errors import APIError +from six.moves.queue import Queue + +from compose.cli.log_printer import build_log_generator +from compose.cli.log_printer import build_log_presenters +from compose.cli.log_printer import build_no_log_generator +from compose.cli.log_printer import consume_queue +from compose.cli.log_printer import QueueItem from compose.cli.log_printer import wait_on_exit +from compose.cli.log_printer import watch_events from compose.container import Container from tests import mock -def build_mock_container(reader): - return mock.Mock( - spec=Container, - name='myapp_web_1', - name_without_project='web_1', - has_api_logs=True, - log_stream=None, - attach=reader, - wait=mock.Mock(return_value=0), - ) - - @pytest.fixture def output_stream(): output = six.StringIO() @@ -31,58 +29,171 @@ def output_stream(): @pytest.fixture def mock_container(): - def reader(*args, **kwargs): - yield b"hello\nworld" - return build_mock_container(reader) + return mock.Mock(spec=Container, name_without_project='web_1') -class TestLogPrinter(object): +class TestLogPresenter(object): - def test_single_container(self, output_stream, mock_container): - LogPrinter([mock_container], output=output_stream).run() + def test_monochrome(self, mock_container): + presenters = build_log_presenters(['foo', 'bar'], True) + presenter = next(presenters) + actual = presenter.present(mock_container, "this line") + assert actual == "web_1 | this line" - output = output_stream.getvalue() - assert 'hello' in output - assert 'world' in output - # Call count is 2 lines + "container exited line" - assert output_stream.flush.call_count == 3 + def test_polychrome(self, mock_container): + presenters = build_log_presenters(['foo', 'bar'], False) + presenter = next(presenters) + actual = presenter.present(mock_container, "this line") + assert '\033[' in actual - def test_monochrome(self, output_stream, mock_container): - LogPrinter([mock_container], output=output_stream, monochrome=True).run() - assert '\033[' not in output_stream.getvalue() - def test_polychrome(self, output_stream, mock_container): - LogPrinter([mock_container], output=output_stream).run() - assert '\033[' in output_stream.getvalue() +def test_wait_on_exit(): + exit_status = 3 + mock_container = mock.Mock( + spec=Container, + name='cname', + wait=mock.Mock(return_value=exit_status)) + + expected = '{} exited with code {}\n'.format(mock_container.name, exit_status) + assert expected == wait_on_exit(mock_container) + + +def test_wait_on_exit_raises(): + status_code = 500 + + def mock_wait(): + resp = requests.Response() + resp.status_code = status_code + raise APIError('Bad server', resp) + + mock_container = mock.Mock( + spec=Container, + name='cname', + wait=mock_wait + ) + + expected = 'Unexpected API error for {} (HTTP code {})\n'.format( + mock_container.name, status_code, + ) + assert expected in wait_on_exit(mock_container) + + +def test_build_no_log_generator(mock_container): + mock_container.has_api_logs = False + mock_container.log_driver = 'none' + output, = build_no_log_generator(mock_container, None) + assert "WARNING: no logs are available with the 'none' log driver\n" in output + assert "exited with code" not in output + + +class TestBuildLogGenerator(object): + + def test_no_log_stream(self, mock_container): + mock_container.log_stream = None + mock_container.logs.return_value = iter([b"hello\nworld"]) + log_args = {'follow': True} + + generator = build_log_generator(mock_container, log_args) + assert next(generator) == "hello\n" + assert next(generator) == "world" + mock_container.logs.assert_called_once_with( + stdout=True, + stderr=True, + stream=True, + **log_args) + + def test_with_log_stream(self, mock_container): + mock_container.log_stream = iter([b"hello\nworld"]) + log_args = {'follow': True} + + generator = build_log_generator(mock_container, log_args) + assert next(generator) == "hello\n" + assert next(generator) == "world" def test_unicode(self, output_stream): - glyph = u'\u2022' + glyph = u'\u2022\n' + mock_container.log_stream = iter([glyph.encode('utf-8')]) - def reader(*args, **kwargs): - yield glyph.encode('utf-8') + b'\n' + generator = build_log_generator(mock_container, {}) + assert next(generator) == glyph - container = build_mock_container(reader) - LogPrinter([container], output=output_stream).run() - output = output_stream.getvalue() - if six.PY2: - output = output.decode('utf-8') - assert glyph in output +@pytest.fixture +def thread_map(): + return {'cid': mock.Mock()} - def test_wait_on_exit(self): - exit_status = 3 - mock_container = mock.Mock( - spec=Container, - name='cname', - wait=mock.Mock(return_value=exit_status)) - expected = '{} exited with code {}\n'.format(mock_container.name, exit_status) - assert expected == wait_on_exit(mock_container) +@pytest.fixture +def mock_presenters(): + return itertools.cycle([mock.Mock()]) - def test_generator_with_no_logs(self, mock_container, output_stream): - mock_container.has_api_logs = False - mock_container.log_driver = 'none' - LogPrinter([mock_container], output=output_stream).run() - output = output_stream.getvalue() - assert "WARNING: no logs are available with the 'none' log driver\n" in output +class TestWatchEvents(object): + + def test_stop_event(self, thread_map, mock_presenters): + event_stream = [{'action': 'stop', 'id': 'cid'}] + watch_events(thread_map, event_stream, mock_presenters, ()) + assert not thread_map + + def test_start_event(self, thread_map, mock_presenters): + container_id = 'abcd' + event = {'action': 'start', 'id': container_id, 'container': mock.Mock()} + event_stream = [event] + thread_args = 'foo', 'bar' + + with mock.patch( + 'compose.cli.log_printer.build_thread', + autospec=True + ) as mock_build_thread: + watch_events(thread_map, event_stream, mock_presenters, thread_args) + mock_build_thread.assert_called_once_with( + event['container'], + next(mock_presenters), + *thread_args) + assert container_id in thread_map + + def test_other_event(self, thread_map, mock_presenters): + container_id = 'abcd' + event_stream = [{'action': 'create', 'id': container_id}] + watch_events(thread_map, event_stream, mock_presenters, ()) + assert container_id not in thread_map + + +class TestConsumeQueue(object): + + def test_item_is_an_exception(self): + + class Problem(Exception): + pass + + queue = Queue() + error = Problem('oops') + for item in QueueItem.new('a'), QueueItem.new('b'), QueueItem.exception(error): + queue.put(item) + + generator = consume_queue(queue, False) + assert next(generator) == 'a' + assert next(generator) == 'b' + with pytest.raises(Problem): + next(generator) + + def test_item_is_stop_without_cascade_stop(self): + queue = Queue() + for item in QueueItem.stop(), QueueItem.new('a'), QueueItem.new('b'): + queue.put(item) + + generator = consume_queue(queue, False) + assert next(generator) == 'a' + assert next(generator) == 'b' + + def test_item_is_stop_with_cascade_stop(self): + queue = Queue() + for item in QueueItem.stop(), QueueItem.new('a'), QueueItem.new('b'): + queue.put(item) + + assert list(consume_queue(queue, True)) == [] + + def test_item_is_none_when_timeout_is_hit(self): + queue = Queue() + generator = consume_queue(queue, False) + assert next(generator) is None diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index 6f5dd3ca..dc527880 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -3,18 +3,16 @@ from __future__ import unicode_literals import logging +import pytest + from compose import container from compose.cli.errors import UserError from compose.cli.formatter import ConsoleWarningFormatter -from compose.cli.log_printer import LogPrinter -from compose.cli.main import attach_to_logs -from compose.cli.main import build_log_printer from compose.cli.main import convergence_strategy_from_opts +from compose.cli.main import filter_containers_to_service_names from compose.cli.main import setup_console_handler -from compose.project import Project from compose.service import ConvergenceStrategy from tests import mock -from tests import unittest def mock_container(service, number): @@ -25,9 +23,16 @@ def mock_container(service, number): name_without_project='{0}_{1}'.format(service, number)) -class CLIMainTestCase(unittest.TestCase): +@pytest.fixture +def logging_handler(): + stream = mock.Mock() + stream.isatty.return_value = True + return logging.StreamHandler(stream=stream) - def test_build_log_printer(self): + +class TestCLIMainTestCase(object): + + def test_filter_containers_to_service_names(self): containers = [ mock_container('web', 1), mock_container('web', 2), @@ -36,84 +41,64 @@ class CLIMainTestCase(unittest.TestCase): mock_container('another', 1), ] service_names = ['web', 'db'] - log_printer = build_log_printer(containers, service_names, True, False) - self.assertEqual(log_printer.containers, containers[:3]) + actual = filter_containers_to_service_names(containers, service_names) + assert actual == containers[:3] - def test_build_log_printer_all_services(self): + def test_filter_containers_to_service_names_all(self): containers = [ mock_container('web', 1), mock_container('db', 1), mock_container('other', 1), ] service_names = [] - log_printer = build_log_printer(containers, service_names, True, False) - self.assertEqual(log_printer.containers, containers) - - def test_attach_to_logs(self): - project = mock.create_autospec(Project) - log_printer = mock.create_autospec(LogPrinter, containers=[]) - service_names = ['web', 'db'] - timeout = 12 - - with mock.patch('compose.cli.main.signals.signal', autospec=True) as mock_signal: - attach_to_logs(project, log_printer, service_names, timeout) - - assert mock_signal.signal.mock_calls == [ - mock.call(mock_signal.SIGINT, mock.ANY), - mock.call(mock_signal.SIGTERM, mock.ANY), - ] - log_printer.run.assert_called_once_with() + actual = filter_containers_to_service_names(containers, service_names) + assert actual == containers -class SetupConsoleHandlerTestCase(unittest.TestCase): +class TestSetupConsoleHandlerTestCase(object): - def setUp(self): - self.stream = mock.Mock() - self.stream.isatty.return_value = True - self.handler = logging.StreamHandler(stream=self.stream) + def test_with_tty_verbose(self, logging_handler): + setup_console_handler(logging_handler, True) + assert type(logging_handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' in logging_handler.formatter._fmt + assert '%(funcName)s' in logging_handler.formatter._fmt - def test_with_tty_verbose(self): - setup_console_handler(self.handler, True) - assert type(self.handler.formatter) == ConsoleWarningFormatter - assert '%(name)s' in self.handler.formatter._fmt - assert '%(funcName)s' in self.handler.formatter._fmt + def test_with_tty_not_verbose(self, logging_handler): + setup_console_handler(logging_handler, False) + assert type(logging_handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' not in logging_handler.formatter._fmt + assert '%(funcName)s' not in logging_handler.formatter._fmt - def test_with_tty_not_verbose(self): - setup_console_handler(self.handler, False) - assert type(self.handler.formatter) == ConsoleWarningFormatter - assert '%(name)s' not in self.handler.formatter._fmt - assert '%(funcName)s' not in self.handler.formatter._fmt - - def test_with_not_a_tty(self): - self.stream.isatty.return_value = False - setup_console_handler(self.handler, False) - assert type(self.handler.formatter) == logging.Formatter + def test_with_not_a_tty(self, logging_handler): + logging_handler.stream.isatty.return_value = False + setup_console_handler(logging_handler, False) + assert type(logging_handler.formatter) == logging.Formatter -class ConvergeStrategyFromOptsTestCase(unittest.TestCase): +class TestConvergeStrategyFromOptsTestCase(object): def test_invalid_opts(self): options = {'--force-recreate': True, '--no-recreate': True} - with self.assertRaises(UserError): + with pytest.raises(UserError): convergence_strategy_from_opts(options) def test_always(self): options = {'--force-recreate': True, '--no-recreate': False} - self.assertEqual( - convergence_strategy_from_opts(options), + assert ( + convergence_strategy_from_opts(options) == ConvergenceStrategy.always ) def test_never(self): options = {'--force-recreate': False, '--no-recreate': True} - self.assertEqual( - convergence_strategy_from_opts(options), + assert ( + convergence_strategy_from_opts(options) == ConvergenceStrategy.never ) def test_changed(self): options = {'--force-recreate': False, '--no-recreate': False} - self.assertEqual( - convergence_strategy_from_opts(options), + assert ( + convergence_strategy_from_opts(options) == ConvergenceStrategy.changed ) diff --git a/tests/unit/cli/utils_test.py b/tests/unit/cli/utils_test.py new file mode 100644 index 00000000..066fb359 --- /dev/null +++ b/tests/unit/cli/utils_test.py @@ -0,0 +1,23 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import unittest + +from compose.cli.utils import unquote_path + + +class UnquotePathTest(unittest.TestCase): + def test_no_quotes(self): + assert unquote_path('hello') == 'hello' + + def test_simple_quotes(self): + assert unquote_path('"hello"') == 'hello' + + def test_uneven_quotes(self): + assert unquote_path('"hello') == '"hello' + assert unquote_path('hello"') == 'hello"' + + def test_nested_quotes(self): + assert unquote_path('""hello""') == '"hello"' + assert unquote_path('"hel"lo"') == 'hel"lo' + assert unquote_path('"hello""') == 'hello"' diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index a5767097..317650cb 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -3,6 +3,9 @@ from __future__ import absolute_import from __future__ import unicode_literals import os +import shutil +import tempfile +from io import StringIO import docker import py @@ -10,13 +13,14 @@ import pytest from .. import mock from .. import unittest +from ..helpers import build_config from compose.cli.command import get_project from compose.cli.command import get_project_name from compose.cli.docopt_command import NoSuchCommand from compose.cli.errors import UserError from compose.cli.main import TopLevelCommand from compose.const import IS_WINDOWS_PLATFORM -from compose.service import Service +from compose.project import Project class CLITestCase(unittest.TestCase): @@ -42,13 +46,36 @@ class CLITestCase(unittest.TestCase): project_name = get_project_name(None, project_name=name) self.assertEquals('explicitprojectname', project_name) + @mock.patch.dict(os.environ) def test_project_name_from_environment_new_var(self): name = 'namefromenv' - with mock.patch.dict(os.environ): - os.environ['COMPOSE_PROJECT_NAME'] = name - project_name = get_project_name(None) + os.environ['COMPOSE_PROJECT_NAME'] = name + project_name = get_project_name(None) self.assertEquals(project_name, name) + def test_project_name_with_empty_environment_var(self): + base_dir = 'tests/fixtures/simple-composefile' + with mock.patch.dict(os.environ): + os.environ['COMPOSE_PROJECT_NAME'] = '' + project_name = get_project_name(base_dir) + self.assertEquals('simplecomposefile', project_name) + + @mock.patch.dict(os.environ) + def test_project_name_with_environment_file(self): + base_dir = tempfile.mkdtemp() + try: + name = 'namefromenvfile' + with open(os.path.join(base_dir, '.env'), 'w') as f: + f.write('COMPOSE_PROJECT_NAME={}'.format(name)) + project_name = get_project_name(base_dir) + assert project_name == name + + # Environment has priority over .env file + os.environ['COMPOSE_PROJECT_NAME'] = 'namefromenv' + assert get_project_name(base_dir) == os.environ['COMPOSE_PROJECT_NAME'] + finally: + shutil.rmtree(base_dir) + def test_get_project(self): base_dir = 'tests/fixtures/longer-filename-composefile' project = get_project(base_dir) @@ -56,63 +83,66 @@ class CLITestCase(unittest.TestCase): self.assertTrue(project.client) self.assertTrue(project.services) - def test_help(self): - command = TopLevelCommand() - with self.assertRaises(SystemExit): - command.dispatch(['-h'], None) - def test_command_help(self): - with self.assertRaises(SystemExit) as ctx: - TopLevelCommand().dispatch(['help', 'up'], None) + with mock.patch('sys.stdout', new=StringIO()) as fake_stdout: + TopLevelCommand.help({'COMMAND': 'up'}) - self.assertIn('Usage: up', str(ctx.exception)) + assert "Usage: up" in fake_stdout.getvalue() def test_command_help_nonexistent(self): - with self.assertRaises(NoSuchCommand): - TopLevelCommand().dispatch(['help', 'nonexistent'], None) + with pytest.raises(NoSuchCommand): + TopLevelCommand.help({'COMMAND': 'nonexistent'}) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason="requires dockerpty") - @mock.patch('compose.cli.main.dockerpty', autospec=True) - def test_run_with_environment_merged_with_options_list(self, mock_dockerpty): - command = TopLevelCommand() - mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock(client=mock_client) - mock_project.get_service.return_value = Service( - 'service', + @mock.patch('compose.cli.main.RunOperation', autospec=True) + @mock.patch('compose.cli.main.PseudoTerminal', autospec=True) + def test_run_interactive_passes_logs_false(self, mock_pseudo_terminal, mock_run_operation): + mock_client = mock.create_autospec(docker.APIClient) + project = Project.from_config( + name='composetest', client=mock_client, - environment=['FOO=ONE', 'BAR=TWO'], - image='someimage') + config_data=build_config({ + 'service': {'image': 'busybox'} + }), + ) + command = TopLevelCommand(project) - command.run(mock_project, { - 'SERVICE': 'service', - 'COMMAND': None, - '-e': ['BAR=NEW', 'OTHER=bär'.encode('utf-8')], - '--user': None, - '--no-deps': None, - '-d': True, - '-T': None, - '--entrypoint': None, - '--service-ports': None, - '--publish': [], - '--rm': None, - '--name': None, - }) + with pytest.raises(SystemExit): + command.run({ + 'SERVICE': 'service', + 'COMMAND': None, + '-e': [], + '--user': None, + '--no-deps': None, + '-d': False, + '-T': None, + '--entrypoint': None, + '--service-ports': None, + '--publish': [], + '--rm': None, + '--name': None, + '--workdir': None, + }) - _, _, call_kwargs = mock_client.create_container.mock_calls[0] - self.assertEqual( - call_kwargs['environment'], - {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': u'bär'}) + _, _, call_kwargs = mock_run_operation.mock_calls[0] + assert call_kwargs['logs'] is False def test_run_service_with_restart_always(self): - command = TopLevelCommand() - mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock(client=mock_client) - mock_project.get_service.return_value = Service( - 'service', + mock_client = mock.create_autospec(docker.APIClient) + + project = Project.from_config( + name='composetest', client=mock_client, - restart={'Name': 'always', 'MaximumRetryCount': 0}, - image='someimage') - command.run(mock_project, { + config_data=build_config({ + 'service': { + 'image': 'busybox', + 'restart': 'always', + } + }), + ) + + command = TopLevelCommand(project) + command.run({ 'SERVICE': 'service', 'COMMAND': None, '-e': [], @@ -125,6 +155,7 @@ class CLITestCase(unittest.TestCase): '--publish': [], '--rm': None, '--name': None, + '--workdir': None, }) self.assertEquals( @@ -132,15 +163,8 @@ class CLITestCase(unittest.TestCase): 'always' ) - command = TopLevelCommand() - mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock(client=mock_client) - mock_project.get_service.return_value = Service( - 'service', - client=mock_client, - restart='always', - image='someimage') - command.run(mock_project, { + command = TopLevelCommand(project) + command.run({ 'SERVICE': 'service', 'COMMAND': None, '-e': [], @@ -153,25 +177,25 @@ class CLITestCase(unittest.TestCase): '--publish': [], '--rm': True, '--name': None, + '--workdir': None, }) self.assertFalse( mock_client.create_host_config.call_args[1].get('restart_policy') ) - def test_command_manula_and_service_ports_together(self): - command = TopLevelCommand() - mock_client = mock.create_autospec(docker.Client) - mock_project = mock.Mock(client=mock_client) - mock_project.get_service.return_value = Service( - 'service', - client=mock_client, - restart='always', - image='someimage', + def test_command_manual_and_service_ports_together(self): + project = Project.from_config( + name='composetest', + client=None, + config_data=build_config({ + 'service': {'image': 'busybox'}, + }), ) + command = TopLevelCommand(project) with self.assertRaises(UserError): - command.run(mock_project, { + command.run({ 'SERVICE': 'service', 'COMMAND': None, '-e': [], diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e24dc904..c26272d9 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -11,28 +11,42 @@ from operator import itemgetter import py import pytest +from ...helpers import build_config_details from compose.config import config +from compose.config import types from compose.config.config import resolve_build_args from compose.config.config import resolve_environment +from compose.config.config import V1 +from compose.config.config import V2_0 +from compose.config.config import V2_1 +from compose.config.config import V3_0 +from compose.config.config import V3_1 +from compose.config.environment import Environment from compose.config.errors import ConfigurationError +from compose.config.errors import VERSION_EXPLANATION +from compose.config.serialize import denormalize_service_dict +from compose.config.serialize import serialize_ns_time_value from compose.config.types import VolumeSpec from compose.const import IS_WINDOWS_PLATFORM +from compose.utils import nanoseconds_from_time_seconds from tests import mock from tests import unittest -DEFAULT_VERSION = V2 = 2 -V1 = 1 +DEFAULT_VERSION = V2_0 def make_service_dict(name, service_dict, working_dir, filename=None): + """Test helper function to construct a ServiceExtendsResolver """ - Test helper function to construct a ServiceExtendsResolver - """ - resolver = config.ServiceExtendsResolver(config.ServiceConfig( - working_dir=working_dir, - filename=filename, - name=name, - config=service_dict), version=1) + resolver = config.ServiceExtendsResolver( + config.ServiceConfig( + working_dir=working_dir, + filename=filename, + name=name, + config=service_dict), + config.ConfigFile(filename=filename, config={}), + environment=Environment.from_env_file(working_dir) + ) return config.process_service(resolver.run()) @@ -40,10 +54,8 @@ def service_sort(services): return sorted(services, key=itemgetter('name')) -def build_config_details(contents, working_dir='working_dir', filename='filename.yml'): - return config.ConfigDetails( - working_dir, - [config.ConfigFile(filename, contents)]) +def secret_sort(secrets): + return sorted(secrets, key=itemgetter('source')) class ConfigTest(unittest.TestCase): @@ -77,7 +89,7 @@ class ConfigTest(unittest.TestCase): def test_load_v2(self): config_data = config.load( build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'foo': {'image': 'busybox'}, 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, @@ -87,11 +99,30 @@ class ConfigTest(unittest.TestCase): 'driver': 'default', 'driver_opts': {'beep': 'boop'} } + }, + 'networks': { + 'default': { + 'driver': 'bridge', + 'driver_opts': {'beep': 'boop'} + }, + 'with_ipam': { + 'ipam': { + 'driver': 'default', + 'config': [ + {'subnet': '172.28.0.0/16'} + ] + } + }, + 'internal': { + 'driver': 'bridge', + 'internal': True + } } }, 'working_dir', 'filename.yml') ) service_dicts = config_data.services volume_dict = config_data.volumes + networks_dict = config_data.networks self.assertEqual( service_sort(service_dicts), service_sort([ @@ -112,10 +143,107 @@ class ConfigTest(unittest.TestCase): 'driver_opts': {'beep': 'boop'} } }) + self.assertEqual(networks_dict, { + 'default': { + 'driver': 'bridge', + 'driver_opts': {'beep': 'boop'} + }, + 'with_ipam': { + 'ipam': { + 'driver': 'default', + 'config': [ + {'subnet': '172.28.0.0/16'} + ] + } + }, + 'internal': { + 'driver': 'bridge', + 'internal': True + } + }) + + def test_valid_versions(self): + for version in ['2', '2.0']: + cfg = config.load(build_config_details({'version': version})) + assert cfg.version == V2_0 + + cfg = config.load(build_config_details({'version': '2.1'})) + assert cfg.version == V2_1 + + for version in ['3', '3.0']: + cfg = config.load(build_config_details({'version': version})) + assert cfg.version == V3_0 + + cfg = config.load(build_config_details({'version': '3.1'})) + assert cfg.version == V3_1 + + def test_v1_file_version(self): + cfg = config.load(build_config_details({'web': {'image': 'busybox'}})) + assert cfg.version == V1 + assert list(s['name'] for s in cfg.services) == ['web'] + + cfg = config.load(build_config_details({'version': {'image': 'busybox'}})) + assert cfg.version == V1 + assert list(s['name'] for s in cfg.services) == ['version'] + + def test_wrong_version_type(self): + for version in [None, 1, 2, 2.0]: + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + {'version': version}, + filename='filename.yml', + ) + ) + + assert 'Version in "filename.yml" is invalid - it should be a string.' \ + in excinfo.exconly() + + def test_unsupported_version(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + {'version': '2.18'}, + filename='filename.yml', + ) + ) + + assert 'Version in "filename.yml" is unsupported' in excinfo.exconly() + assert VERSION_EXPLANATION in excinfo.exconly() + + def test_version_1_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': '1', + 'web': {'image': 'busybox'}, + }, + filename='filename.yml', + ) + ) + + assert 'Version in "filename.yml" is invalid' in excinfo.exconly() + assert VERSION_EXPLANATION in excinfo.exconly() + + def test_v1_file_with_version_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': '2', + 'web': {'image': 'busybox'}, + }, + filename='filename.yml', + ) + ) + + assert 'Additional properties are not allowed' in excinfo.exconly() + assert VERSION_EXPLANATION in excinfo.exconly() def test_named_volume_config_empty(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'simple': {'image': 'busybox'} }, @@ -130,14 +258,70 @@ class ConfigTest(unittest.TestCase): assert volumes['simple'] == {} assert volumes['other'] == {} + def test_named_volume_numeric_driver_opt(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'simple': {'image': 'busybox'} + }, + 'volumes': { + 'simple': {'driver_opts': {'size': 42}}, + } + }) + cfg = config.load(config_details) + assert cfg.volumes['simple']['driver_opts']['size'] == '42' + + def test_volume_invalid_driver_opt(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'simple': {'image': 'busybox'} + }, + 'volumes': { + 'simple': {'driver_opts': {'size': True}}, + } + }) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert 'driver_opts.size contains an invalid type' in exc.exconly() + + def test_named_volume_invalid_type_list(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'simple': {'image': 'busybox'} + }, + 'volumes': [] + }) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert "volume must be a mapping, not an array" in exc.exconly() + + def test_networks_invalid_type_list(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'simple': {'image': 'busybox'} + }, + 'networks': [] + }) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert "network must be a mapping, not an array" in exc.exconly() + def test_load_service_with_name_version(self): - config_data = config.load( - build_config_details({ - 'version': { - 'image': 'busybox' - } - }, 'working_dir', 'filename.yml') - ) + with mock.patch('compose.config.config.log') as mock_logging: + config_data = config.load( + build_config_details({ + 'version': { + 'image': 'busybox' + } + }, 'working_dir', 'filename.yml') + ) + + assert 'Unexpected type for "version" key in "filename.yml"' \ + in mock_logging.warn.call_args[0][0] + service_dicts = config_data.services self.assertEqual( service_sort(service_dicts), @@ -149,27 +333,6 @@ class ConfigTest(unittest.TestCase): ]) ) - def test_load_invalid_version(self): - with self.assertRaises(ConfigurationError): - config.load( - build_config_details({ - 'version': 18, - 'services': { - 'foo': {'image': 'busybox'} - } - }, 'working_dir', 'filename.yml') - ) - - with self.assertRaises(ConfigurationError): - config.load( - build_config_details({ - 'version': 'two point oh', - 'services': { - 'foo': {'image': 'busybox'} - } - }, 'working_dir', 'filename.yml') - ) - def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( @@ -184,42 +347,149 @@ class ConfigTest(unittest.TestCase): with self.assertRaises(ConfigurationError): config.load( build_config_details( - {'version': 2, 'services': {'web': 'busybox:latest'}}, + {'version': '2', 'services': {'web': 'busybox:latest'}}, 'working_dir', 'filename.yml' ) ) + def test_load_throws_error_with_invalid_network_fields(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': '2', + 'services': {'web': 'busybox:latest'}, + 'networks': { + 'invalid': {'foo', 'bar'} + } + }, 'working_dir', 'filename.yml') + ) + + def test_load_config_link_local_ips_network(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': V2_1, + 'services': { + 'web': { + 'image': 'example/web', + 'networks': { + 'foobar': { + 'aliases': ['foo', 'bar'], + 'link_local_ips': ['169.254.8.8'] + } + } + } + }, + 'networks': {'foobar': {}} + } + ) + + details = config.ConfigDetails('.', [base_file]) + web_service = config.load(details).services[0] + assert web_service['networks'] == { + 'foobar': { + 'aliases': ['foo', 'bar'], + 'link_local_ips': ['169.254.8.8'] + } + } + + def test_load_config_volume_and_network_labels(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2.1', + 'services': { + 'web': { + 'image': 'example/web', + }, + }, + 'networks': { + 'with_label': { + 'labels': { + 'label_key': 'label_val' + } + } + }, + 'volumes': { + 'with_label': { + 'labels': { + 'label_key': 'label_val' + } + } + } + } + ) + + details = config.ConfigDetails('.', [base_file]) + network_dict = config.load(details).networks + volume_dict = config.load(details).volumes + + self.assertEqual( + network_dict, + { + 'with_label': { + 'labels': { + 'label_key': 'label_val' + } + } + } + ) + + self.assertEqual( + volume_dict, + { + 'with_label': { + 'labels': { + 'label_key': 'label_val' + } + } + } + ) + def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( - {invalid_name: {'image': 'busybox'}}, - 'working_dir', - 'filename.yml')) + {invalid_name: {'image': 'busybox'}})) assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() - def test_config_invalid_service_names_v2(self): + def test_load_config_invalid_service_names_v2(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: - config.load( - build_config_details({ - 'version': 2, - 'services': {invalid_name: {'image': 'busybox'}} - }, 'working_dir', 'filename.yml') - ) + config.load(build_config_details( + { + 'version': '2', + 'services': {invalid_name: {'image': 'busybox'}}, + })) assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() def test_load_with_invalid_field_name(self): - config_details = build_config_details( - {'web': {'image': 'busybox', 'name': 'bogus'}}, - 'working_dir', - 'filename.yml') with pytest.raises(ConfigurationError) as exc: - config.load(config_details) - error_msg = "Unsupported config option for 'web' service: 'name'" - assert error_msg in exc.exconly() - assert "Validation failed in file 'filename.yml'" in exc.exconly() + config.load(build_config_details( + { + 'version': '2', + 'services': { + 'web': {'image': 'busybox', 'name': 'bogus'}, + } + }, + 'working_dir', + 'filename.yml', + )) + + assert "Unsupported config option for services.web: 'name'" in exc.exconly() + + def test_load_with_invalid_field_name_v1(self): + with pytest.raises(ConfigurationError) as exc: + config.load(build_config_details( + { + 'web': {'image': 'busybox', 'name': 'bogus'}, + }, + 'working_dir', + 'filename.yml', + )) + + assert "Unsupported config option for web: 'name'" in exc.exconly() def test_load_invalid_service_definition(self): config_details = build_config_details( @@ -228,13 +498,31 @@ class ConfigTest(unittest.TestCase): 'filename.yml') with pytest.raises(ConfigurationError) as exc: config.load(config_details) - error_msg = "service 'web' doesn't have any configuration options" - assert error_msg in exc.exconly() + assert "service 'web' must be a mapping not a string." in exc.exconly() + + def test_load_with_empty_build_args(self): + config_details = build_config_details( + { + 'version': '2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'args': None, + }, + }, + }, + } + ) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert ( + "services.web.build.args contains an invalid type, it should be an " + "object, or an array" in exc.exconly() + ) def test_config_integer_service_name_raise_validation_error(self): - expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " - "be a string, eg '1'") - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {1: {'image': 'busybox'}}, @@ -243,15 +531,17 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_integer_service_name_raise_validation_error_v2(self): - expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " - "be a string, eg '1'") + assert ( + "In file 'filename.yml', the service name 1 must be a quoted string, i.e. '1'" in + excinfo.exconly() + ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_config_integer_service_name_raise_validation_error_v2(self): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': {1: {'image': 'busybox'}} }, 'working_dir', @@ -259,6 +549,11 @@ class ConfigTest(unittest.TestCase): ) ) + assert ( + "In file 'filename.yml', the service name 1 must be a quoted string, i.e. '1'." in + excinfo.exconly() + ) + def test_load_with_multiple_files_v1(self): base_file = config.ConfigFile( 'base.yaml', @@ -311,7 +606,7 @@ class ConfigTest(unittest.TestCase): def test_load_with_multiple_files_and_empty_override_v2(self): base_file = config.ConfigFile( 'base.yml', - {'version': 2, 'services': {'web': {'image': 'example/web'}}}) + {'version': '2', 'services': {'web': {'image': 'example/web'}}}) override_file = config.ConfigFile('override.yml', None) details = config.ConfigDetails('.', [base_file, override_file]) @@ -335,7 +630,7 @@ class ConfigTest(unittest.TestCase): base_file = config.ConfigFile('base.yml', None) override_file = config.ConfigFile( 'override.tml', - {'version': 2, 'services': {'web': {'image': 'example/web'}}} + {'version': '2', 'services': {'web': {'image': 'example/web'}}} ) details = config.ConfigDetails('.', [base_file, override_file]) with pytest.raises(ConfigurationError) as exc: @@ -391,7 +686,7 @@ class ConfigTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: config.load(details) - assert "service 'bogus' doesn't have any configuration" in exc.exconly() + assert "service 'bogus' must be a mapping not a string." in exc.exconly() assert "In file 'override.yaml'" in exc.exconly() def test_load_sorts_in_dependency_order(self): @@ -435,7 +730,7 @@ class ConfigTest(unittest.TestCase): config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': '.', @@ -450,7 +745,7 @@ class ConfigTest(unittest.TestCase): service = config.load( build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': '.' @@ -463,7 +758,7 @@ class ConfigTest(unittest.TestCase): service = config.load( build_config_details( { - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': { @@ -480,14 +775,132 @@ class ConfigTest(unittest.TestCase): self.assertTrue('context' in service[0]['build']) self.assertEqual(service[0]['build']['dockerfile'], 'Dockerfile-alt') + def test_load_with_buildargs(self): + service = config.load( + build_config_details( + { + 'version': '2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt', + 'args': { + 'opt1': 42, + 'opt2': 'foobar' + } + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services[0] + assert 'args' in service['build'] + assert 'opt1' in service['build']['args'] + assert isinstance(service['build']['args']['opt1'], str) + assert service['build']['args']['opt1'] == '42' + assert service['build']['args']['opt2'] == 'foobar' + + def test_build_args_allow_empty_properties(self): + service = config.load( + build_config_details( + { + 'version': '2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt', + 'args': { + 'foo': None + } + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services[0] + assert 'args' in service['build'] + assert 'foo' in service['build']['args'] + assert service['build']['args']['foo'] == '' + + # If build argument is None then it will be converted to the empty + # string. Make sure that int zero kept as it is, i.e. not converted to + # the empty string + def test_build_args_check_zero_preserved(self): + service = config.load( + build_config_details( + { + 'version': '2', + 'services': { + 'web': { + 'build': { + 'context': '.', + 'dockerfile': 'Dockerfile-alt', + 'args': { + 'foo': 0 + } + } + } + } + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ).services[0] + assert 'args' in service['build'] + assert 'foo' in service['build']['args'] + assert service['build']['args']['foo'] == '0' + + def test_load_with_multiple_files_mismatched_networks_format(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2', + 'services': { + 'web': { + 'image': 'example/web', + 'networks': { + 'foobar': {'aliases': ['foo', 'bar']} + } + } + }, + 'networks': {'foobar': {}, 'baz': {}} + } + ) + + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': '2', + 'services': { + 'web': { + 'networks': ['baz'] + } + } + } + ) + + details = config.ConfigDetails('.', [base_file, override_file]) + web_service = config.load(details).services[0] + assert web_service['networks'] == { + 'foobar': {'aliases': ['foo', 'bar']}, + 'baz': None + } + def test_load_with_multiple_files_v2(self): base_file = config.ConfigFile( 'base.yaml', { - 'version': 2, + 'version': '2', 'services': { 'web': { 'image': 'example/web', + 'depends_on': ['db'], }, 'db': { 'image': 'example/db', @@ -497,12 +910,16 @@ class ConfigTest(unittest.TestCase): override_file = config.ConfigFile( 'override.yaml', { - 'version': 2, + 'version': '2', 'services': { 'web': { 'build': '/', 'volumes': ['/home/user/project:/code'], + 'depends_on': ['other'], }, + 'other': { + 'image': 'example/other', + } } }) details = config.ConfigDetails('.', [base_file, override_file]) @@ -514,14 +931,72 @@ class ConfigTest(unittest.TestCase): 'build': {'context': os.path.abspath('/')}, 'image': 'example/web', 'volumes': [VolumeSpec.parse('/home/user/project:/code')], + 'depends_on': { + 'db': {'condition': 'service_started'}, + 'other': {'condition': 'service_started'}, + }, }, { 'name': 'db', 'image': 'example/db', }, + { + 'name': 'other', + 'image': 'example/other', + }, ] assert service_sort(service_dicts) == service_sort(expected) + def test_undeclared_volume_v2(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': ['data0028:/data:ro'], + }, + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + with self.assertRaises(ConfigurationError): + config.load(details) + + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox:latest', + 'volumes': ['./data0028:/data:ro'], + }, + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + volume = config_data.services[0].get('volumes')[0] + assert not volume.is_named_volume + + def test_undeclared_volume_v1(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': { + 'image': 'busybox:latest', + 'volumes': ['data0028:/data:ro'], + }, + } + ) + details = config.ConfigDetails('.', [base_file]) + config_data = config.load(details) + volume = config_data.services[0].get('volumes')[0] + assert volume.external == 'data0028' + assert volume.is_named_volume + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( @@ -532,8 +1007,7 @@ class ConfigTest(unittest.TestCase): assert services[0]['name'] == valid_name def test_config_hint(self): - expected_error_msg = "(did you mean 'privileged'?)" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -544,6 +1018,8 @@ class ConfigTest(unittest.TestCase): ) ) + assert "(did you mean 'privileged'?)" in excinfo.exconly() + def test_load_errors_on_uppercase_with_no_image(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details({ @@ -551,9 +1027,41 @@ class ConfigTest(unittest.TestCase): }, 'tests/fixtures/build-ctx')) assert "Service 'Foo' contains uppercase characters" in exc.exconly() - def test_invalid_config_build_and_image_specified(self): - expected_error_msg = "Service 'foo' has both an image and build path specified." - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_invalid_config_v1(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'foo': {'image': 1}, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + assert "foo.image contains an invalid type, it should be a string" \ + in excinfo.exconly() + + def test_invalid_config_v2(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load( + build_config_details( + { + 'version': '2', + 'services': { + 'foo': {'image': 1}, + }, + }, + 'tests/fixtures/extends', + 'filename.yml' + ) + ) + + assert "services.foo.image contains an invalid type, it should be a string" \ + in excinfo.exconly() + + def test_invalid_config_build_and_image_specified_v1(self): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -564,9 +1072,10 @@ class ConfigTest(unittest.TestCase): ) ) + assert "foo has both an image and build path specified." in excinfo.exconly() + def test_invalid_config_type_should_be_an_array(self): - expected_error_msg = "Service 'foo' configuration key 'links' contains an invalid type, it should be an array" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -577,10 +1086,11 @@ class ConfigTest(unittest.TestCase): ) ) + assert "foo.links contains an invalid type, it should be an array" \ + in excinfo.exconly() + def test_invalid_config_not_a_dictionary(self): - expected_error_msg = ("Top level object in 'filename.yml' needs to be " - "an object.") - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( ['foo', 'lol'], @@ -589,9 +1099,11 @@ class ConfigTest(unittest.TestCase): ) ) + assert "Top level object in 'filename.yml' needs to be an object" \ + in excinfo.exconly() + def test_invalid_config_not_unique_items(self): - expected_error_msg = "has non-unique elements" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -602,10 +1114,10 @@ class ConfigTest(unittest.TestCase): ) ) + assert "has non-unique elements" in excinfo.exconly() + def test_invalid_list_of_strings_format(self): - expected_error_msg = "Service 'web' configuration key 'command' contains 1" - expected_error_msg += ", which is an invalid type, it should be a string" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -616,7 +1128,10 @@ class ConfigTest(unittest.TestCase): ) ) - def test_load_config_dockerfile_without_build_raises_error(self): + assert "web.command contains 1, which is an invalid type, it should be a string" \ + in excinfo.exconly() + + def test_load_config_dockerfile_without_build_raises_error_v1(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details({ 'web': { @@ -624,12 +1139,11 @@ class ConfigTest(unittest.TestCase): 'dockerfile': 'Dockerfile.alt' } })) - assert "Service 'web' has both an image and alternate Dockerfile." in exc.exconly() + + assert "web has both an image and alternate Dockerfile." in exc.exconly() def test_config_extra_hosts_string_raises_validation_error(self): - expected_error_msg = "Service 'web' configuration key 'extra_hosts' contains an invalid type" - - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {'web': { @@ -641,12 +1155,11 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_extra_hosts_list_of_dicts_validation_error(self): - expected_error_msg = ( - "key 'extra_hosts' contains {\"somehost\": \"162.242.195.82\"}, " - "which is an invalid type, it should be a string") + assert "web.extra_hosts contains an invalid type" \ + in excinfo.exconly() - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_config_extra_hosts_list_of_dicts_validation_error(self): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( {'web': { @@ -661,10 +1174,11 @@ class ConfigTest(unittest.TestCase): ) ) - def test_config_ulimits_invalid_keys_validation_error(self): - expected = ("Service 'web' configuration key 'ulimits' 'nofile' contains " - "unsupported option: 'not_soft_or_hard'") + assert "web.extra_hosts contains {\"somehost\": \"162.242.195.82\"}, " \ + "which is an invalid type, it should be a string" \ + in excinfo.exconly() + def test_config_ulimits_invalid_keys_validation_error(self): with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( { @@ -681,10 +1195,11 @@ class ConfigTest(unittest.TestCase): }, 'working_dir', 'filename.yml')) - assert expected in exc.exconly() + + assert "web.ulimits.nofile contains unsupported option: 'not_soft_or_hard'" \ + in exc.exconly() def test_config_ulimits_required_keys_validation_error(self): - with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( { @@ -695,11 +1210,11 @@ class ConfigTest(unittest.TestCase): }, 'working_dir', 'filename.yml')) - assert "Service 'web' configuration key 'ulimits' 'nofile'" in exc.exconly() + assert "web.ulimits.nofile" in exc.exconly() assert "'hard' is a required property" in exc.exconly() def test_config_ulimits_soft_greater_than_hard_error(self): - expected = "cannot contain a 'soft' value higher than 'hard' value" + expected = "'soft' value can not be greater than 'hard' value" with pytest.raises(ConfigurationError) as exc: config.load(build_config_details( @@ -745,22 +1260,18 @@ class ConfigTest(unittest.TestCase): ).services self.assertEqual(service[0]['entrypoint'], entrypoint) - @mock.patch('compose.config.validation.log') - def test_logs_warning_for_boolean_in_environment(self, mock_logging): - expected_warning_msg = "There is a boolean value in the 'environment' key." - config.load( - build_config_details( - {'web': { - 'image': 'busybox', - 'environment': {'SHOW_STUFF': True} - }}, - 'working_dir', - 'filename.yml' - ) - ) + def test_logs_warning_for_boolean_in_environment(self): + config_details = build_config_details({ + 'web': { + 'image': 'busybox', + 'environment': {'SHOW_STUFF': True} + } + }) - assert mock_logging.warn.called - assert expected_warning_msg in mock_logging.warn.call_args[0][0] + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + + assert "contains true, which is an invalid type" in exc.exconly() def test_config_valid_environment_dict_key_contains_dashes(self): services = config.load( @@ -796,7 +1307,7 @@ class ConfigTest(unittest.TestCase): 'extra_hosts': "www.example.com: 192.168.0.17", } })) - assert "'extra_hosts' contains an invalid type" in exc.exconly() + assert "web.extra_hosts contains an invalid type" in exc.exconly() def test_validate_extra_hosts_invalid_list(self): with pytest.raises(ConfigurationError) as exc: @@ -828,6 +1339,101 @@ class ConfigTest(unittest.TestCase): } ] + def test_tmpfs_option(self): + actual = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'alpine', + 'tmpfs': '/run', + } + } + })) + assert actual.services == [ + { + 'name': 'web', + 'image': 'alpine', + 'tmpfs': ['/run'], + } + ] + + def test_oom_score_adj_option(self): + + actual = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'alpine', + 'oom_score_adj': 500 + } + } + })) + + assert actual.services == [ + { + 'name': 'web', + 'image': 'alpine', + 'oom_score_adj': 500 + } + ] + + def test_swappiness_option(self): + actual = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'alpine', + 'mem_swappiness': 10, + } + } + })) + assert actual.services == [ + { + 'name': 'web', + 'image': 'alpine', + 'mem_swappiness': 10, + } + ] + + def test_group_add_option(self): + + actual = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'alpine', + 'group_add': ["docker", 777] + } + } + })) + + assert actual.services == [ + { + 'name': 'web', + 'image': 'alpine', + 'group_add': ["docker", 777] + } + ] + + def test_isolation_option(self): + actual = config.load(build_config_details({ + 'version': V2_1, + 'services': { + 'web': { + 'image': 'win10', + 'isolation': 'hyperv' + } + } + })) + + assert actual.services == [ + { + 'name': 'web', + 'image': 'win10', + 'isolation': 'hyperv', + } + ] + def test_merge_service_dicts_from_files_with_extends_in_base(self): base = { 'volumes': ['.:/app'], @@ -865,9 +1471,345 @@ class ConfigTest(unittest.TestCase): 'extends': {'service': 'foo'} } + def test_merge_build_args(self): + base = { + 'build': { + 'context': '.', + 'args': { + 'ONE': '1', + 'TWO': '2', + }, + } + } + override = { + 'build': { + 'args': { + 'TWO': 'dos', + 'THREE': '3', + }, + } + } + actual = config.merge_service_dicts( + base, + override, + DEFAULT_VERSION) + assert actual == { + 'build': { + 'context': '.', + 'args': { + 'ONE': '1', + 'TWO': 'dos', + 'THREE': '3', + }, + } + } + + def test_merge_logging_v1(self): + base = { + 'image': 'alpine:edge', + 'log_driver': 'something', + 'log_opt': {'foo': 'three'}, + } + override = { + 'image': 'alpine:edge', + 'command': 'true', + } + actual = config.merge_service_dicts(base, override, V1) + assert actual == { + 'image': 'alpine:edge', + 'log_driver': 'something', + 'log_opt': {'foo': 'three'}, + 'command': 'true', + } + + def test_merge_logging_v2(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000', + 'timeout': '23' + } + } + } + override = { + 'logging': { + 'options': { + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000', + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + def test_merge_logging_v2_override_driver(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000', + 'timeout': '23' + } + } + } + override = { + 'logging': { + 'driver': 'syslog', + 'options': { + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + def test_merge_logging_v2_no_base_driver(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'options': { + 'frequency': '2000', + 'timeout': '23' + } + } + } + override = { + 'logging': { + 'driver': 'json-file', + 'options': { + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000', + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + def test_merge_logging_v2_no_drivers(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'options': { + 'frequency': '2000', + 'timeout': '23' + } + } + } + override = { + 'logging': { + 'options': { + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'options': { + 'frequency': '2000', + 'timeout': '360', + 'pretty-print': 'on' + } + } + } + + def test_merge_logging_v2_no_override_options(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000', + 'timeout': '23' + } + } + } + override = { + 'logging': { + 'driver': 'syslog' + } + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': None + } + } + + def test_merge_logging_v2_no_base(self): + base = { + 'image': 'alpine:edge' + } + override = { + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000' + } + } + } + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'json-file', + 'options': { + 'frequency': '2000' + } + } + } + + def test_merge_logging_v2_no_override(self): + base = { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'frequency': '2000' + } + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'alpine:edge', + 'logging': { + 'driver': 'syslog', + 'options': { + 'frequency': '2000' + } + } + } + + def test_merge_depends_on_no_override(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = {} + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == base + + def test_merge_depends_on_mixed_syntax(self): + base = { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'} + } + } + override = { + 'depends_on': ['app3'] + } + + actual = config.merge_service_dicts(base, override, V2_1) + assert actual == { + 'image': 'busybox', + 'depends_on': { + 'app1': {'condition': 'service_started'}, + 'app2': {'condition': 'service_healthy'}, + 'app3': {'condition': 'service_started'} + } + } + + def test_merge_pid(self): + # Regression: https://github.com/docker/compose/issues/4184 + base = { + 'image': 'busybox', + 'pid': 'host' + } + + override = { + 'labels': {'com.docker.compose.test': 'yes'} + } + + actual = config.merge_service_dicts(base, override, V2_0) + assert actual == { + 'image': 'busybox', + 'pid': 'host', + 'labels': {'com.docker.compose.test': 'yes'} + } + + def test_merge_different_secrets(self): + base = { + 'image': 'busybox', + 'secrets': [ + {'source': 'src.txt'} + ] + } + override = {'secrets': ['other-src.txt']} + + actual = config.merge_service_dicts(base, override, V3_1) + assert secret_sort(actual['secrets']) == secret_sort([ + {'source': 'src.txt'}, + {'source': 'other-src.txt'} + ]) + + def test_merge_secrets_override(self): + base = { + 'image': 'busybox', + 'secrets': ['src.txt'], + } + override = { + 'secrets': [ + { + 'source': 'src.txt', + 'target': 'data.txt', + 'mode': 0o400 + } + ] + } + actual = config.merge_service_dicts(base, override, V3_1) + assert actual['secrets'] == override['secrets'] + def test_external_volume_config(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'bogus': {'image': 'busybox'} }, @@ -885,7 +1827,7 @@ class ConfigTest(unittest.TestCase): def test_external_volume_invalid_config(self): config_details = build_config_details({ - 'version': 2, + 'version': '2', 'services': { 'bogus': {'image': 'busybox'} }, @@ -893,9 +1835,262 @@ class ConfigTest(unittest.TestCase): 'ext': {'external': True, 'driver': 'foo'} } }) - with self.assertRaises(ConfigurationError): + with pytest.raises(ConfigurationError): config.load(config_details) + def test_depends_on_orders_services(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'one': {'image': 'busybox', 'depends_on': ['three', 'two']}, + 'two': {'image': 'busybox', 'depends_on': ['three']}, + 'three': {'image': 'busybox'}, + }, + }) + actual = config.load(config_details) + assert ( + [service['name'] for service in actual.services] == + ['three', 'two', 'one'] + ) + + def test_depends_on_unknown_service_errors(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'one': {'image': 'busybox', 'depends_on': ['three']}, + }, + }) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert "Service 'one' depends on service 'three'" in exc.exconly() + + def test_linked_service_is_undefined(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': '2', + 'services': { + 'web': {'image': 'busybox', 'links': ['db:db']}, + }, + }) + ) + + def test_load_dockerfile_without_context(self): + config_details = build_config_details({ + 'version': '2', + 'services': { + 'one': {'build': {'dockerfile': 'Dockerfile.foo'}}, + }, + }) + with pytest.raises(ConfigurationError) as exc: + config.load(config_details) + assert 'has neither an image nor a build context' in exc.exconly() + + def test_load_secrets(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': [ + 'one', + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + details = config.ConfigDetails('.', [base_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + + def test_load_secrets_multi_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'image': 'example/web', + 'secrets': ['one'], + }, + }, + 'secrets': { + 'one': {'file': 'secret.txt'}, + }, + }) + override_file = config.ConfigFile( + 'base.yaml', + { + 'version': '3.1', + 'services': { + 'web': { + 'secrets': [ + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ], + }, + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + }, + ] + assert service_sort(service_dicts) == service_sort(expected) + + +class NetworkModeTest(unittest.TestCase): + def test_network_mode_standard(self): + config_data = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'bridge', + }, + }, + })) + + assert config_data.services[0]['network_mode'] == 'bridge' + + def test_network_mode_standard_v1(self): + config_data = config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'command': "top", + 'net': 'bridge', + }, + })) + + assert config_data.services[0]['network_mode'] == 'bridge' + assert 'net' not in config_data.services[0] + + def test_network_mode_container(self): + config_data = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'container:foo', + }, + }, + })) + + assert config_data.services[0]['network_mode'] == 'container:foo' + + def test_network_mode_container_v1(self): + config_data = config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'command': "top", + 'net': 'container:foo', + }, + })) + + assert config_data.services[0]['network_mode'] == 'container:foo' + + def test_network_mode_service(self): + config_data = config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'service:foo', + }, + 'foo': { + 'image': 'busybox', + 'command': "top", + }, + }, + })) + + assert config_data.services[1]['network_mode'] == 'service:foo' + + def test_network_mode_service_v1(self): + config_data = config.load(build_config_details({ + 'web': { + 'image': 'busybox', + 'command': "top", + 'net': 'container:foo', + }, + 'foo': { + 'image': 'busybox', + 'command': "top", + }, + })) + + assert config_data.services[1]['network_mode'] == 'service:foo' + + def test_network_mode_service_nonexistent(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'service:foo', + }, + }, + })) + + assert "service 'foo' which is undefined" in excinfo.exconly() + + def test_network_mode_plus_networks_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + config.load(build_config_details({ + 'version': '2', + 'services': { + 'web': { + 'image': 'busybox', + 'command': "top", + 'network_mode': 'bridge', + 'networks': ['front'], + }, + }, + 'networks': { + 'front': None, + } + })) + + assert "'network_mode' and 'networks' cannot be combined" in excinfo.exconly() + class PortsTest(unittest.TestCase): INVALID_PORTS_TYPES = [ @@ -989,8 +2184,25 @@ class PortsTest(unittest.TestCase): class InterpolationTest(unittest.TestCase): + @mock.patch.dict(os.environ) + def test_config_file_with_environment_file(self): + project_dir = 'tests/fixtures/default-env-file' + service_dicts = config.load( + config.find( + project_dir, None, Environment.from_env_file(project_dir) + ) + ).services + + self.assertEqual(service_dicts[0], { + 'name': 'web', + 'image': 'alpine:latest', + 'ports': ['5643', '9999'], + 'command': 'true' + }) + @mock.patch.dict(os.environ) def test_config_file_with_environment_variable(self): + project_dir = 'tests/fixtures/environment-interpolation' os.environ.update( IMAGE="busybox", HOST_PORT="80", @@ -998,7 +2210,9 @@ class InterpolationTest(unittest.TestCase): ) service_dicts = config.load( - config.find('tests/fixtures/environment-interpolation', None), + config.find( + project_dir, None, Environment.from_env_file(project_dir) + ) ).services self.assertEqual(service_dicts, [ @@ -1028,7 +2242,7 @@ class InterpolationTest(unittest.TestCase): None, ) - with mock.patch('compose.config.interpolation.log') as log: + with mock.patch('compose.config.environment.log') as log: config.load(config_details) self.assertEqual(2, log.warn.call_count) @@ -1109,24 +2323,42 @@ class VolumeConfigTest(unittest.TestCase): @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') def test_relative_path_does_expand_posix(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='/home/me/myproject') + d = make_service_dict( + 'foo', + {'build': '.', 'volumes': ['./data:/data']}, + working_dir='/home/me/myproject') self.assertEqual(d['volumes'], ['/home/me/myproject/data:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='/home/me/myproject') + d = make_service_dict( + 'foo', + {'build': '.', 'volumes': ['.:/data']}, + working_dir='/home/me/myproject') self.assertEqual(d['volumes'], ['/home/me/myproject:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='/home/me/myproject') + d = make_service_dict( + 'foo', + {'build': '.', 'volumes': ['../otherproject:/data']}, + working_dir='/home/me/myproject') self.assertEqual(d['volumes'], ['/home/me/otherproject:/data']) @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') def test_relative_path_does_expand_windows(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='c:\\Users\\me\\myproject') + d = make_service_dict( + 'foo', + {'build': '.', 'volumes': ['./data:/data']}, + working_dir='c:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject\\data:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='c:\\Users\\me\\myproject') + d = make_service_dict( + 'foo', + {'build': '.', 'volumes': ['.:/data']}, + working_dir='c:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='c:\\Users\\me\\myproject') + d = make_service_dict( + 'foo', + {'build': '.', 'volumes': ['../otherproject:/data']}, + working_dir='c:\\Users\\me\\myproject') self.assertEqual(d['volumes'], ['c:\\Users\\me\\otherproject:/data']) @mock.patch.dict(os.environ) @@ -1146,57 +2378,54 @@ class VolumeConfigTest(unittest.TestCase): class MergePathMappingTest(object): - def config_name(self): - return "" + config_name = "" def test_empty(self): service_dict = config.merge_service_dicts({}, {}, DEFAULT_VERSION) - assert self.config_name() not in service_dict + assert self.config_name not in service_dict def test_no_override(self): service_dict = config.merge_service_dicts( - {self.config_name(): ['/foo:/code', '/data']}, + {self.config_name: ['/foo:/code', '/data']}, {}, DEFAULT_VERSION) - assert set(service_dict[self.config_name()]) == set(['/foo:/code', '/data']) + assert set(service_dict[self.config_name]) == set(['/foo:/code', '/data']) def test_no_base(self): service_dict = config.merge_service_dicts( {}, - {self.config_name(): ['/bar:/code']}, + {self.config_name: ['/bar:/code']}, DEFAULT_VERSION) - assert set(service_dict[self.config_name()]) == set(['/bar:/code']) + assert set(service_dict[self.config_name]) == set(['/bar:/code']) def test_override_explicit_path(self): service_dict = config.merge_service_dicts( - {self.config_name(): ['/foo:/code', '/data']}, - {self.config_name(): ['/bar:/code']}, + {self.config_name: ['/foo:/code', '/data']}, + {self.config_name: ['/bar:/code']}, DEFAULT_VERSION) - assert set(service_dict[self.config_name()]) == set(['/bar:/code', '/data']) + assert set(service_dict[self.config_name]) == set(['/bar:/code', '/data']) def test_add_explicit_path(self): service_dict = config.merge_service_dicts( - {self.config_name(): ['/foo:/code', '/data']}, - {self.config_name(): ['/bar:/code', '/quux:/data']}, + {self.config_name: ['/foo:/code', '/data']}, + {self.config_name: ['/bar:/code', '/quux:/data']}, DEFAULT_VERSION) - assert set(service_dict[self.config_name()]) == set(['/bar:/code', '/quux:/data']) + assert set(service_dict[self.config_name]) == set(['/bar:/code', '/quux:/data']) def test_remove_explicit_path(self): service_dict = config.merge_service_dicts( - {self.config_name(): ['/foo:/code', '/quux:/data']}, - {self.config_name(): ['/bar:/code', '/data']}, + {self.config_name: ['/foo:/code', '/quux:/data']}, + {self.config_name: ['/bar:/code', '/data']}, DEFAULT_VERSION) - assert set(service_dict[self.config_name()]) == set(['/bar:/code', '/data']) + assert set(service_dict[self.config_name]) == set(['/bar:/code', '/data']) class MergeVolumesTest(unittest.TestCase, MergePathMappingTest): - def config_name(self): - return 'volumes' + config_name = 'volumes' class MergeDevicesTest(unittest.TestCase, MergePathMappingTest): - def config_name(self): - return 'devices' + config_name = 'devices' class BuildOrImageMergeTest(unittest.TestCase): @@ -1234,30 +2463,57 @@ class BuildOrImageMergeTest(unittest.TestCase): ) -class MergeListsTest(unittest.TestCase): +class MergeListsTest(object): + config_name = "" + base_config = [] + override_config = [] + + def merged_config(self): + return set(self.base_config) | set(self.override_config) + def test_empty(self): - assert 'ports' not in config.merge_service_dicts({}, {}, DEFAULT_VERSION) + assert self.config_name not in config.merge_service_dicts({}, {}, DEFAULT_VERSION) def test_no_override(self): service_dict = config.merge_service_dicts( - {'ports': ['10:8000', '9000']}, + {self.config_name: self.base_config}, {}, DEFAULT_VERSION) - assert set(service_dict['ports']) == set(['10:8000', '9000']) + assert set(service_dict[self.config_name]) == set(self.base_config) def test_no_base(self): service_dict = config.merge_service_dicts( {}, - {'ports': ['10:8000', '9000']}, + {self.config_name: self.base_config}, DEFAULT_VERSION) - assert set(service_dict['ports']) == set(['10:8000', '9000']) + assert set(service_dict[self.config_name]) == set(self.base_config) def test_add_item(self): service_dict = config.merge_service_dicts( - {'ports': ['10:8000', '9000']}, - {'ports': ['20:8000']}, + {self.config_name: self.base_config}, + {self.config_name: self.override_config}, DEFAULT_VERSION) - assert set(service_dict['ports']) == set(['10:8000', '9000', '20:8000']) + assert set(service_dict[self.config_name]) == set(self.merged_config()) + + +class MergePortsTest(unittest.TestCase, MergeListsTest): + config_name = 'ports' + base_config = ['10:8000', '9000'] + override_config = ['20:8000'] + + def test_duplicate_port_mappings(self): + service_dict = config.merge_service_dicts( + {self.config_name: self.base_config}, + {self.config_name: self.base_config}, + DEFAULT_VERSION + ) + assert set(service_dict[self.config_name]) == set(self.base_config) + + +class MergeNetworksTest(unittest.TestCase, MergeListsTest): + config_name = 'networks' + base_config = ['frontend', 'backend'] + override_config = ['monitoring'] class MergeStringsOrListsTest(unittest.TestCase): @@ -1336,11 +2592,7 @@ class MemoryOptionsTest(unittest.TestCase): When you set a 'memswap_limit' it is invalid config unless you also set a mem_limit """ - expected_error_msg = ( - "Service 'foo' configuration key 'memswap_limit' is invalid: when " - "defining 'memswap_limit' you must set 'mem_limit' as well" - ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1351,6 +2603,10 @@ class MemoryOptionsTest(unittest.TestCase): ) ) + assert "foo.memswap_limit is invalid: when defining " \ + "'memswap_limit' you must set 'mem_limit' as well" \ + in excinfo.exconly() + def test_validation_with_correct_memswap_values(self): service_dict = config.load( build_config_details( @@ -1415,8 +2671,10 @@ class EnvTest(unittest.TestCase): }, } self.assertEqual( - resolve_environment(service_dict), - {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, + resolve_environment( + service_dict, Environment.from_env_file(None) + ), + {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': None}, ) def test_resolve_environment_from_env_file(self): @@ -1425,6 +2683,15 @@ class EnvTest(unittest.TestCase): {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'bar'}, ) + def test_environment_overrides_env_file(self): + self.assertEqual( + resolve_environment({ + 'environment': {'FOO': 'baz'}, + 'env_file': ['tests/fixtures/env/one.env'], + }), + {'ONE': '2', 'TWO': '1', 'THREE': '3', 'FOO': 'baz'}, + ) + def test_resolve_environment_with_multiple_env_files(self): service_dict = { 'env_file': [ @@ -1452,12 +2719,15 @@ class EnvTest(unittest.TestCase): os.environ['FILE_DEF_EMPTY'] = 'E2' os.environ['ENV_DEF'] = 'E3' self.assertEqual( - resolve_environment({'env_file': ['tests/fixtures/env/resolve.env']}), + resolve_environment( + {'env_file': ['tests/fixtures/env/resolve.env']}, + Environment.from_env_file(None) + ), { 'FILE_DEF': u'bär', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', - 'NO_DEF': '' + 'NO_DEF': None }, ) @@ -1475,8 +2745,8 @@ class EnvTest(unittest.TestCase): } } self.assertEqual( - resolve_build_args(build), - {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': ''}, + resolve_build_args(build, Environment.from_env_file(build['context'])), + {'arg1': 'value1', 'empty_arg': '', 'env_arg': 'value2', 'no_env': None}, ) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @@ -1507,7 +2777,9 @@ class EnvTest(unittest.TestCase): def load_from_filename(filename): - return config.load(config.find('.', [filename])).services + return config.load( + config.find('.', [filename], Environment.from_env_file('.')) + ).services class ExtendsTest(unittest.TestCase): @@ -1524,6 +2796,7 @@ class ExtendsTest(unittest.TestCase): 'name': 'myweb', 'image': 'busybox', 'command': 'top', + 'network_mode': 'bridge', 'links': ['mydb:db'], 'environment': { "FOO": "1", @@ -1541,6 +2814,7 @@ class ExtendsTest(unittest.TestCase): 'name': 'web', 'image': 'busybox', 'command': '/bin/true', + 'network_mode': 'host', 'environment': { "FOO": "2", "BAR": "1", @@ -1559,6 +2833,7 @@ class ExtendsTest(unittest.TestCase): 'name': 'myweb', 'image': 'busybox', 'command': '/bin/true', + 'network_mode': 'host', 'environment': { "FOO": "2", "BAR": "2", @@ -1610,7 +2885,7 @@ class ExtendsTest(unittest.TestCase): self.assertEqual(path, expected) def test_extends_validation_empty_dictionary(self): - with self.assertRaisesRegexp(ConfigurationError, 'service'): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1621,8 +2896,10 @@ class ExtendsTest(unittest.TestCase): ) ) + assert 'service' in excinfo.exconly() + def test_extends_validation_missing_service_key(self): - with self.assertRaisesRegexp(ConfigurationError, "'service' is a required property"): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1633,12 +2910,10 @@ class ExtendsTest(unittest.TestCase): ) ) + assert "'service' is a required property" in excinfo.exconly() + def test_extends_validation_invalid_key(self): - expected_error_msg = ( - "Service 'web' configuration key 'extends' " - "contains unsupported option: 'rogue_key'" - ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1656,12 +2931,11 @@ class ExtendsTest(unittest.TestCase): ) ) + assert "web.extends contains unsupported option: 'rogue_key'" \ + in excinfo.exconly() + def test_extends_validation_sub_property_key(self): - expected_error_msg = ( - "Service 'web' configuration key 'extends' 'file' contains 1, " - "which is an invalid type, it should be a string" - ) - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: config.load( build_config_details( { @@ -1678,13 +2952,16 @@ class ExtendsTest(unittest.TestCase): ) ) + assert "web.extends.file contains 1, which is an invalid type, it should be a string" \ + in excinfo.exconly() + def test_extends_validation_no_file_key_no_filename_set(self): dictionary = {'extends': {'service': 'web'}} - def load_config(): - return make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') + with pytest.raises(ConfigurationError) as excinfo: + make_service_dict('myweb', dictionary, working_dir='tests/fixtures/extends') - self.assertRaisesRegexp(ConfigurationError, 'file', load_config) + assert 'file' in excinfo.exconly() def test_extends_validation_valid_config(self): service = config.load( @@ -1705,7 +2982,7 @@ class ExtendsTest(unittest.TestCase): with pytest.raises(ConfigurationError) as exc: load_from_filename('tests/fixtures/extends/service-with-invalid-schema.yml') assert ( - "Service 'myweb' has neither an image nor a build path specified" in + "myweb has neither an image nor a build context specified" in exc.exconly() ) @@ -1738,22 +3015,30 @@ class ExtendsTest(unittest.TestCase): ])) def test_invalid_links_in_extended_service(self): - expected_error_msg = "services with 'links' cannot be extended" - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-links.yml') - def test_invalid_volumes_from_in_extended_service(self): - expected_error_msg = "services with 'volumes_from' cannot be extended" + assert "services with 'links' cannot be extended" in excinfo.exconly() - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_invalid_volumes_from_in_extended_service(self): + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-volumes.yml') - def test_invalid_net_in_extended_service(self): - expected_error_msg = "services with 'net: container' cannot be extended" + assert "services with 'volumes_from' cannot be extended" in excinfo.exconly() - with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + def test_invalid_net_in_extended_service(self): + with pytest.raises(ConfigurationError) as excinfo: + load_from_filename('tests/fixtures/extends/invalid-net-v2.yml') + + assert 'network_mode: service' in excinfo.exconly() + assert 'cannot be extended' in excinfo.exconly() + + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/invalid-net.yml') + assert 'net: container' in excinfo.exconly() + assert 'cannot be extended' in excinfo.exconly() + @mock.patch.dict(os.environ) def test_load_config_runs_interpolation_in_extended_service(self): os.environ.update(HOSTNAME_VALUE="penguin") @@ -1796,10 +3081,12 @@ class ExtendsTest(unittest.TestCase): ]) def test_load_throws_error_when_base_service_does_not_exist(self): - err_msg = r'''Cannot extend service 'foo' in .*: Service not found''' - with self.assertRaisesRegexp(ConfigurationError, err_msg): + with pytest.raises(ConfigurationError) as excinfo: load_from_filename('tests/fixtures/extends/nonexistent-service.yml') + assert "Cannot extend service 'foo'" in excinfo.exconly() + assert "Service not found" in excinfo.exconly() + def test_partial_service_config_in_extends_is_still_valid(self): dicts = load_from_filename('tests/fixtures/extends/valid-common-config.yml') self.assertEqual(dicts[0]['environment'], {'FOO': '1'}) @@ -1824,6 +3111,7 @@ class ExtendsTest(unittest.TestCase): }, ])) + @mock.patch.dict(os.environ) def test_extends_with_environment_and_env_files(self): tmpdir = py.test.ensuretemp('test_extends_with_environment') self.addCleanup(tmpdir.remove) @@ -1879,15 +3167,94 @@ class ExtendsTest(unittest.TestCase): }, }, ] - with mock.patch.dict(os.environ): - os.environ['SECRET'] = 'secret' - os.environ['THING'] = 'thing' - os.environ['COMMON_ENV_FILE'] = 'secret' - os.environ['TOP_ENV_FILE'] = 'secret' - config = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + + os.environ['SECRET'] = 'secret' + os.environ['THING'] = 'thing' + os.environ['COMMON_ENV_FILE'] = 'secret' + os.environ['TOP_ENV_FILE'] = 'secret' + config = load_from_filename(str(tmpdir.join('docker-compose.yml'))) assert config == expected + def test_extends_with_mixed_versions_is_error(self): + tmpdir = py.test.ensuretemp('test_extends_with_mixed_version') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: "2" + services: + web: + extends: + file: base.yml + service: base + image: busybox + """) + tmpdir.join('base.yml').write(""" + base: + volumes: ['/foo'] + ports: ['3000:3000'] + """) + + with pytest.raises(ConfigurationError) as exc: + load_from_filename(str(tmpdir.join('docker-compose.yml'))) + assert 'Version mismatch' in exc.exconly() + + def test_extends_with_defined_version_passes(self): + tmpdir = py.test.ensuretemp('test_extends_with_defined_version') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: "2" + services: + web: + extends: + file: base.yml + service: base + image: busybox + """) + tmpdir.join('base.yml').write(""" + version: "2" + services: + base: + volumes: ['/foo'] + ports: ['3000:3000'] + command: top + """) + + service = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + self.assertEquals(service[0]['command'], "top") + + def test_extends_with_depends_on(self): + tmpdir = py.test.ensuretemp('test_extends_with_defined_version') + self.addCleanup(tmpdir.remove) + tmpdir.join('docker-compose.yml').write(""" + version: "2" + services: + base: + image: example + web: + extends: base + image: busybox + depends_on: ['other'] + other: + image: example + """) + services = load_from_filename(str(tmpdir.join('docker-compose.yml'))) + assert service_sort(services)[2]['depends_on'] == { + 'other': {'condition': 'service_started'} + } + + def test_extends_with_healthcheck(self): + service_dicts = load_from_filename('tests/fixtures/extends/healthcheck-2.yml') + assert service_sort(service_dicts) == [{ + 'name': 'demo', + 'image': 'foobar:latest', + 'healthcheck': { + 'test': ['CMD', '/health.sh'], + 'interval': 10000000000, + 'timeout': 5000000000, + 'retries': 36, + } + }] + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') class ExpandPathTest(unittest.TestCase): @@ -1912,18 +3279,28 @@ class ExpandPathTest(unittest.TestCase): class VolumePathTest(unittest.TestCase): - - @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') def test_split_path_mapping_with_windows_path(self): - windows_volume_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config:/opt/connect/config:ro" - expected_mapping = ( - "/opt/connect/config:ro", - "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" - ) + host_path = "c:\\Users\\msamblanet\\Documents\\anvil\\connect\\config" + windows_volume_path = host_path + ":/opt/connect/config:ro" + expected_mapping = ("/opt/connect/config:ro", host_path) mapping = config.split_path_mapping(windows_volume_path) + assert mapping == expected_mapping - self.assertEqual(mapping, expected_mapping) + def test_split_path_mapping_with_windows_path_in_container(self): + host_path = 'c:\\Users\\remilia\\data' + container_path = 'c:\\scarletdevil\\data' + expected_mapping = (container_path, host_path) + + mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) + assert mapping == expected_mapping + + def test_split_path_mapping_with_root_mount(self): + host_path = '/' + container_path = '/var/hostroot' + expected_mapping = (container_path, host_path) + mapping = config.split_path_mapping('{0}:{1}'.format(host_path, container_path)) + assert mapping == expected_mapping @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') @@ -1993,6 +3370,54 @@ class BuildPathTest(unittest.TestCase): assert 'build path' in exc.exconly() +class HealthcheckTest(unittest.TestCase): + def test_healthcheck(self): + service_dict = make_service_dict( + 'test', + {'healthcheck': { + 'test': ['CMD', 'true'], + 'interval': '1s', + 'timeout': '1m', + 'retries': 3, + }}, + '.', + ) + + assert service_dict['healthcheck'] == { + 'test': ['CMD', 'true'], + 'interval': nanoseconds_from_time_seconds(1), + 'timeout': nanoseconds_from_time_seconds(60), + 'retries': 3, + } + + def test_disable(self): + service_dict = make_service_dict( + 'test', + {'healthcheck': { + 'disable': True, + }}, + '.', + ) + + assert service_dict['healthcheck'] == { + 'test': ['NONE'], + } + + def test_disable_with_other_config_is_invalid(self): + with pytest.raises(ConfigurationError) as excinfo: + make_service_dict( + 'invalid-healthcheck', + {'healthcheck': { + 'disable': True, + 'interval': '1s', + }}, + '.', + ) + + assert 'invalid-healthcheck' in excinfo.exconly() + assert 'disable' in excinfo.exconly() + + class GetDefaultConfigFilesTestCase(unittest.TestCase): files = [ @@ -2037,3 +3462,89 @@ def get_config_filename_for_files(filenames, subdir=None): return os.path.basename(filename) finally: shutil.rmtree(project_dir) + + +class SerializeTest(unittest.TestCase): + def test_denormalize_depends_on_v3(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V3_0) == { + 'image': 'busybox', + 'command': 'true', + 'depends_on': ['service2', 'service3'] + } + + def test_denormalize_depends_on_v2_1(self): + service_dict = { + 'image': 'busybox', + 'command': 'true', + 'depends_on': { + 'service2': {'condition': 'service_started'}, + 'service3': {'condition': 'service_started'}, + } + } + + assert denormalize_service_dict(service_dict, V2_1) == service_dict + + def test_serialize_time(self): + data = { + 9: '9ns', + 9000: '9us', + 9000000: '9ms', + 90000000: '90ms', + 900000000: '900ms', + 999999999: '999999999ns', + 1000000000: '1s', + 60000000000: '1m', + 60000000001: '60000000001ns', + 9000000000000: '150m', + 90000000000000: '25h', + } + + for k, v in data.items(): + assert serialize_ns_time_value(k) == v + + def test_denormalize_healthcheck(self): + service_dict = { + 'image': 'test', + 'healthcheck': { + 'test': 'exit 1', + 'interval': '1m40s', + 'timeout': '30s', + 'retries': 5 + } + } + processed_service = config.process_service(config.ServiceConfig( + '.', 'test', 'test', service_dict + )) + denormalized_service = denormalize_service_dict(processed_service, V2_1) + assert denormalized_service['healthcheck']['interval'] == '100s' + assert denormalized_service['healthcheck']['timeout'] == '30s' + + def test_denormalize_secrets(self): + service_dict = { + 'name': 'web', + 'image': 'example/web', + 'secrets': [ + types.ServiceSecret('one', None, None, None, None), + types.ServiceSecret('source', 'target', '100', '200', 0o777), + ], + } + denormalized_service = denormalize_service_dict(service_dict, V3_1) + assert secret_sort(denormalized_service['secrets']) == secret_sort([ + {'source': 'one'}, + { + 'source': 'source', + 'target': 'target', + 'uid': '100', + 'gid': '200', + 'mode': 0o777, + }, + ]) diff --git a/tests/unit/config/environment_test.py b/tests/unit/config/environment_test.py new file mode 100644 index 00000000..20446d2b --- /dev/null +++ b/tests/unit/config/environment_test.py @@ -0,0 +1,40 @@ +# encoding: utf-8 +from __future__ import absolute_import +from __future__ import print_function +from __future__ import unicode_literals + +from compose.config.environment import Environment +from tests import unittest + + +class EnvironmentTest(unittest.TestCase): + def test_get_simple(self): + env = Environment({ + 'FOO': 'bar', + 'BAR': '1', + 'BAZ': '' + }) + + assert env.get('FOO') == 'bar' + assert env.get('BAR') == '1' + assert env.get('BAZ') == '' + + def test_get_undefined(self): + env = Environment({ + 'FOO': 'bar' + }) + assert env.get('FOOBAR') is None + + def test_get_boolean(self): + env = Environment({ + 'FOO': '', + 'BAR': '0', + 'BAZ': 'FALSE', + 'FOOBAR': 'true', + }) + + assert env.get_boolean('FOO') is False + assert env.get_boolean('BAR') is False + assert env.get_boolean('BAZ') is False + assert env.get_boolean('FOOBAR') is True + assert env.get_boolean('UNDEFINED') is False diff --git a/tests/unit/config/interpolation_test.py b/tests/unit/config/interpolation_test.py index 0691e886..fd40153d 100644 --- a/tests/unit/config/interpolation_test.py +++ b/tests/unit/config/interpolation_test.py @@ -1,25 +1,33 @@ from __future__ import absolute_import from __future__ import unicode_literals -import os - -import mock import pytest +from compose.config.environment import Environment from compose.config.interpolation import interpolate_environment_variables +from compose.config.interpolation import Interpolator +from compose.config.interpolation import InvalidInterpolation +from compose.config.interpolation import TemplateWithDefaults -@pytest.yield_fixture +@pytest.fixture def mock_env(): - with mock.patch.dict(os.environ): - os.environ['USER'] = 'jenny' - os.environ['FOO'] = 'bar' - yield + return Environment({'USER': 'jenny', 'FOO': 'bar'}) + + +@pytest.fixture +def variable_mapping(): + return Environment({'FOO': 'first', 'BAR': ''}) + + +@pytest.fixture +def defaults_interpolator(variable_mapping): + return Interpolator(TemplateWithDefaults, variable_mapping).interpolate def test_interpolate_environment_variables_in_services(mock_env): services = { - 'servivea': { + 'servicea': { 'image': 'example:${USER}', 'volumes': ['$FOO:/target'], 'logging': { @@ -31,7 +39,7 @@ def test_interpolate_environment_variables_in_services(mock_env): } } expected = { - 'servivea': { + 'servicea': { 'image': 'example:jenny', 'volumes': ['bar:/target'], 'logging': { @@ -42,7 +50,8 @@ def test_interpolate_environment_variables_in_services(mock_env): } } } - assert interpolate_environment_variables(services, 'service') == expected + value = interpolate_environment_variables("2.0", services, 'service', mock_env) + assert value == expected def test_interpolate_environment_variables_in_volumes(mock_env): @@ -66,4 +75,47 @@ def test_interpolate_environment_variables_in_volumes(mock_env): }, 'other': {}, } - assert interpolate_environment_variables(volumes, 'volume') == expected + value = interpolate_environment_variables("2.0", volumes, 'volume', mock_env) + assert value == expected + + +def test_escaped_interpolation(defaults_interpolator): + assert defaults_interpolator('$${foo}') == '${foo}' + + +def test_invalid_interpolation(defaults_interpolator): + with pytest.raises(InvalidInterpolation): + defaults_interpolator('${') + with pytest.raises(InvalidInterpolation): + defaults_interpolator('$}') + with pytest.raises(InvalidInterpolation): + defaults_interpolator('${}') + with pytest.raises(InvalidInterpolation): + defaults_interpolator('${ }') + with pytest.raises(InvalidInterpolation): + defaults_interpolator('${ foo}') + with pytest.raises(InvalidInterpolation): + defaults_interpolator('${foo }') + with pytest.raises(InvalidInterpolation): + defaults_interpolator('${foo!}') + + +def test_interpolate_missing_no_default(defaults_interpolator): + assert defaults_interpolator("This ${missing} var") == "This var" + assert defaults_interpolator("This ${BAR} var") == "This var" + + +def test_interpolate_with_value(defaults_interpolator): + assert defaults_interpolator("This $FOO var") == "This first var" + assert defaults_interpolator("This ${FOO} var") == "This first var" + + +def test_interpolate_missing_with_default(defaults_interpolator): + assert defaults_interpolator("ok ${missing:-def}") == "ok def" + assert defaults_interpolator("ok ${missing-def}") == "ok def" + assert defaults_interpolator("ok ${BAR:-/non:-alphanumeric}") == "ok /non:-alphanumeric" + + +def test_interpolate_with_empty_and_default_value(defaults_interpolator): + assert defaults_interpolator("ok ${BAR:-def}") == "ok def" + assert defaults_interpolator("ok ${BAR-def}") == "ok " diff --git a/tests/unit/config/sort_services_test.py b/tests/unit/config/sort_services_test.py index ebe444fe..c39ac022 100644 --- a/tests/unit/config/sort_services_test.py +++ b/tests/unit/config/sort_services_test.py @@ -1,13 +1,14 @@ from __future__ import absolute_import from __future__ import unicode_literals +import pytest + from compose.config.errors import DependencyError from compose.config.sort_services import sort_service_dicts from compose.config.types import VolumeFromSpec -from tests import unittest -class SortServiceTest(unittest.TestCase): +class TestSortService(object): def test_sort_service_dicts_1(self): services = [ { @@ -23,10 +24,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'grunt') - self.assertEqual(sorted_services[1]['name'], 'redis') - self.assertEqual(sorted_services[2]['name'], 'web') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'grunt' + assert sorted_services[1]['name'] == 'redis' + assert sorted_services[2]['name'] == 'web' def test_sort_service_dicts_2(self): services = [ @@ -44,10 +45,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'redis') - self.assertEqual(sorted_services[1]['name'], 'postgres') - self.assertEqual(sorted_services[2]['name'], 'web') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'redis' + assert sorted_services[1]['name'] == 'postgres' + assert sorted_services[2]['name'] == 'web' def test_sort_service_dicts_3(self): services = [ @@ -65,10 +66,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_4(self): services = [ @@ -86,10 +87,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_5(self): services = [ @@ -99,7 +100,7 @@ class SortServiceTest(unittest.TestCase): }, { 'name': 'parent', - 'net': 'container:child' + 'network_mode': 'service:child' }, { 'name': 'child' @@ -107,10 +108,10 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_6(self): services = [ @@ -128,15 +129,15 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 3) - self.assertEqual(sorted_services[0]['name'], 'child') - self.assertEqual(sorted_services[1]['name'], 'parent') - self.assertEqual(sorted_services[2]['name'], 'grandparent') + assert len(sorted_services) == 3 + assert sorted_services[0]['name'] == 'child' + assert sorted_services[1]['name'] == 'parent' + assert sorted_services[2]['name'] == 'grandparent' def test_sort_service_dicts_7(self): services = [ { - 'net': 'container:three', + 'network_mode': 'service:three', 'name': 'four' }, { @@ -153,11 +154,11 @@ class SortServiceTest(unittest.TestCase): ] sorted_services = sort_service_dicts(services) - self.assertEqual(len(sorted_services), 4) - self.assertEqual(sorted_services[0]['name'], 'one') - self.assertEqual(sorted_services[1]['name'], 'two') - self.assertEqual(sorted_services[2]['name'], 'three') - self.assertEqual(sorted_services[3]['name'], 'four') + assert len(sorted_services) == 4 + assert sorted_services[0]['name'] == 'one' + assert sorted_services[1]['name'] == 'two' + assert sorted_services[2]['name'] == 'three' + assert sorted_services[3]['name'] == 'four' def test_sort_service_dicts_circular_imports(self): services = [ @@ -171,13 +172,10 @@ class SortServiceTest(unittest.TestCase): }, ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('redis', e.msg) - self.assertIn('web', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'redis' in exc.exconly() + assert 'web' in exc.exconly() def test_sort_service_dicts_circular_imports_2(self): services = [ @@ -194,13 +192,10 @@ class SortServiceTest(unittest.TestCase): } ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('redis', e.msg) - self.assertIn('web', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'redis' in exc.exconly() + assert 'web' in exc.exconly() def test_sort_service_dicts_circular_imports_3(self): services = [ @@ -218,13 +213,10 @@ class SortServiceTest(unittest.TestCase): } ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('a', e.msg) - self.assertIn('b', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'a' in exc.exconly() + assert 'b' in exc.exconly() def test_sort_service_dicts_self_imports(self): services = [ @@ -234,9 +226,18 @@ class SortServiceTest(unittest.TestCase): }, ] - try: + with pytest.raises(DependencyError) as exc: sort_service_dicts(services) - except DependencyError as e: - self.assertIn('web', e.msg) - else: - self.fail('Should have thrown an DependencyError') + assert 'web' in exc.exconly() + + def test_sort_service_dicts_depends_on_self(self): + services = [ + { + 'depends_on': ['web'], + 'name': 'web' + }, + ] + + with pytest.raises(DependencyError) as exc: + sort_service_dicts(services) + assert 'A service can not depend on itself: web' in exc.exconly() diff --git a/tests/unit/config/types_test.py b/tests/unit/config/types_test.py index 50b7efcb..11427352 100644 --- a/tests/unit/config/types_test.py +++ b/tests/unit/config/types_test.py @@ -3,13 +3,12 @@ from __future__ import unicode_literals import pytest +from compose.config.config import V1 +from compose.config.config import V2_0 from compose.config.errors import ConfigurationError from compose.config.types import parse_extra_hosts from compose.config.types import VolumeFromSpec from compose.config.types import VolumeSpec -from compose.const import IS_WINDOWS_PLATFORM -from tests.unit.config.config_test import V1 -from tests.unit.config.config_test import V2 def test_parse_extra_hosts_list(): @@ -64,15 +63,70 @@ class TestVolumeSpec(object): VolumeSpec.parse('one:two:three:four') assert 'has incorrect format' in exc.exconly() - @pytest.mark.xfail((not IS_WINDOWS_PLATFORM), reason='does not have a drive') - def test_parse_volume_windows_absolute_path(self): - windows_path = "c:\\Users\\me\\Documents\\shiny\\config:\\opt\\shiny\\config:ro" - assert VolumeSpec.parse(windows_path) == ( + def test_parse_volume_windows_absolute_path_normalized(self): + windows_path = "c:\\Users\\me\\Documents\\shiny\\config:/opt/shiny/config:ro" + assert VolumeSpec._parse_win32(windows_path, True) == ( "/c/Users/me/Documents/shiny/config", "/opt/shiny/config", "ro" ) + def test_parse_volume_windows_absolute_path_native(self): + windows_path = "c:\\Users\\me\\Documents\\shiny\\config:/opt/shiny/config:ro" + assert VolumeSpec._parse_win32(windows_path, False) == ( + "c:\\Users\\me\\Documents\\shiny\\config", + "/opt/shiny/config", + "ro" + ) + + def test_parse_volume_windows_internal_path_normalized(self): + windows_path = 'C:\\Users\\reimu\\scarlet:C:\\scarlet\\app:ro' + assert VolumeSpec._parse_win32(windows_path, True) == ( + '/c/Users/reimu/scarlet', + 'C:\\scarlet\\app', + 'ro' + ) + + def test_parse_volume_windows_internal_path_native(self): + windows_path = 'C:\\Users\\reimu\\scarlet:C:\\scarlet\\app:ro' + assert VolumeSpec._parse_win32(windows_path, False) == ( + 'C:\\Users\\reimu\\scarlet', + 'C:\\scarlet\\app', + 'ro' + ) + + def test_parse_volume_windows_just_drives_normalized(self): + windows_path = 'E:\\:C:\\:ro' + assert VolumeSpec._parse_win32(windows_path, True) == ( + '/e/', + 'C:\\', + 'ro' + ) + + def test_parse_volume_windows_just_drives_native(self): + windows_path = 'E:\\:C:\\:ro' + assert VolumeSpec._parse_win32(windows_path, False) == ( + 'E:\\', + 'C:\\', + 'ro' + ) + + def test_parse_volume_windows_mixed_notations_normalized(self): + windows_path = 'C:\\Foo:/root/foo' + assert VolumeSpec._parse_win32(windows_path, True) == ( + '/c/Foo', + '/root/foo', + 'rw' + ) + + def test_parse_volume_windows_mixed_notations_native(self): + windows_path = 'C:\\Foo:/root/foo' + assert VolumeSpec._parse_win32(windows_path, False) == ( + 'C:\\Foo', + '/root/foo', + 'rw' + ) + class TestVolumesFromSpec(object): @@ -91,26 +145,26 @@ class TestVolumesFromSpec(object): VolumeFromSpec.parse('unknown:format:ro', self.services, V1) def test_parse_v2_from_service(self): - volume_from = VolumeFromSpec.parse('servicea', self.services, V2) + volume_from = VolumeFromSpec.parse('servicea', self.services, V2_0) assert volume_from == VolumeFromSpec('servicea', 'rw', 'service') def test_parse_v2_from_service_with_mode(self): - volume_from = VolumeFromSpec.parse('servicea:ro', self.services, V2) + volume_from = VolumeFromSpec.parse('servicea:ro', self.services, V2_0) assert volume_from == VolumeFromSpec('servicea', 'ro', 'service') def test_parse_v2_from_container(self): - volume_from = VolumeFromSpec.parse('container:foo', self.services, V2) + volume_from = VolumeFromSpec.parse('container:foo', self.services, V2_0) assert volume_from == VolumeFromSpec('foo', 'rw', 'container') def test_parse_v2_from_container_with_mode(self): - volume_from = VolumeFromSpec.parse('container:foo:ro', self.services, V2) + volume_from = VolumeFromSpec.parse('container:foo:ro', self.services, V2_0) assert volume_from == VolumeFromSpec('foo', 'ro', 'container') def test_parse_v2_invalid_type(self): with pytest.raises(ConfigurationError) as exc: - VolumeFromSpec.parse('bogus:foo:ro', self.services, V2) + VolumeFromSpec.parse('bogus:foo:ro', self.services, V2_0) assert "Unknown volumes_from type 'bogus'" in exc.exconly() def test_parse_v2_invalid(self): with pytest.raises(ConfigurationError): - VolumeFromSpec.parse('unknown:format:ro', self.services, V2) + VolumeFromSpec.parse('unknown:format:ro', self.services, V2_0) diff --git a/tests/unit/container_test.py b/tests/unit/container_test.py index 88691150..04f43016 100644 --- a/tests/unit/container_test.py +++ b/tests/unit/container_test.py @@ -12,8 +12,9 @@ from compose.container import get_container_name class ContainerTest(unittest.TestCase): def setUp(self): + self.container_id = "abcabcabcbabc12345" self.container_dict = { - "Id": "abc", + "Id": self.container_id, "Image": "busybox:latest", "Command": "top", "Created": 1387384730, @@ -41,19 +42,22 @@ class ContainerTest(unittest.TestCase): self.assertEqual( container.dictionary, { - "Id": "abc", + "Id": self.container_id, "Image": "busybox:latest", "Name": "/composetest_db_1", }) def test_from_ps_prefixed(self): - self.container_dict['Names'] = ['/swarm-host-1' + n for n in self.container_dict['Names']] + self.container_dict['Names'] = [ + '/swarm-host-1' + n for n in self.container_dict['Names'] + ] - container = Container.from_ps(None, - self.container_dict, - has_been_inspected=True) + container = Container.from_ps( + None, + self.container_dict, + has_been_inspected=True) self.assertEqual(container.dictionary, { - "Id": "abc", + "Id": self.container_id, "Image": "busybox:latest", "Name": "/composetest_db_1", }) @@ -94,7 +98,7 @@ class ContainerTest(unittest.TestCase): self.assertEqual(container.name_without_project, "custom_name_of_container") def test_inspect_if_not_inspected(self): - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) container = Container(mock_client, dict(Id="the_id")) container.inspect_if_not_inspected() @@ -142,13 +146,47 @@ class ContainerTest(unittest.TestCase): self.assertEqual(container.get('HostConfig.VolumesFrom'), ["volume_id"]) self.assertEqual(container.get('Foo.Bar.DoesNotExist'), None) + def test_short_id(self): + container = Container(None, self.container_dict, has_been_inspected=True) + assert container.short_id == self.container_id[:12] + + def test_has_api_logs(self): + container_dict = { + 'HostConfig': { + 'LogConfig': { + 'Type': 'json-file' + } + } + } + + container = Container(None, container_dict, has_been_inspected=True) + assert container.has_api_logs is True + + container_dict['HostConfig']['LogConfig']['Type'] = 'none' + container = Container(None, container_dict, has_been_inspected=True) + assert container.has_api_logs is False + + container_dict['HostConfig']['LogConfig']['Type'] = 'syslog' + container = Container(None, container_dict, has_been_inspected=True) + assert container.has_api_logs is False + + container_dict['HostConfig']['LogConfig']['Type'] = 'journald' + container = Container(None, container_dict, has_been_inspected=True) + assert container.has_api_logs is True + + container_dict['HostConfig']['LogConfig']['Type'] = 'foobar' + container = Container(None, container_dict, has_been_inspected=True) + assert container.has_api_logs is False + class GetContainerNameTestCase(unittest.TestCase): def test_get_container_name(self): self.assertIsNone(get_container_name({})) self.assertEqual(get_container_name({'Name': 'myproject_db_1'}), 'myproject_db_1') - self.assertEqual(get_container_name({'Names': ['/myproject_db_1', '/myproject_web_1/db']}), 'myproject_db_1') + self.assertEqual( + get_container_name({'Names': ['/myproject_db_1', '/myproject_web_1/db']}), + 'myproject_db_1') self.assertEqual( get_container_name({ 'Names': [ diff --git a/tests/unit/interpolation_test.py b/tests/unit/interpolation_test.py deleted file mode 100644 index 317982a9..00000000 --- a/tests/unit/interpolation_test.py +++ /dev/null @@ -1,36 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import unittest - -from compose.config.interpolation import BlankDefaultDict as bddict -from compose.config.interpolation import interpolate -from compose.config.interpolation import InvalidInterpolation - - -class InterpolationTest(unittest.TestCase): - def test_valid_interpolations(self): - self.assertEqual(interpolate('$foo', bddict(foo='hi')), 'hi') - self.assertEqual(interpolate('${foo}', bddict(foo='hi')), 'hi') - - self.assertEqual(interpolate('${subject} love you', bddict(subject='i')), 'i love you') - self.assertEqual(interpolate('i ${verb} you', bddict(verb='love')), 'i love you') - self.assertEqual(interpolate('i love ${object}', bddict(object='you')), 'i love you') - - def test_empty_value(self): - self.assertEqual(interpolate('${foo}', bddict(foo='')), '') - - def test_unset_value(self): - self.assertEqual(interpolate('${foo}', bddict()), '') - - def test_escaped_interpolation(self): - self.assertEqual(interpolate('$${foo}', bddict(foo='hi')), '${foo}') - - def test_invalid_strings(self): - self.assertRaises(InvalidInterpolation, lambda: interpolate('${', bddict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('$}', bddict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${}', bddict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${ }', bddict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${ foo}', bddict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo }', bddict())) - self.assertRaises(InvalidInterpolation, lambda: interpolate('${foo!}', bddict())) diff --git a/tests/unit/multiplexer_test.py b/tests/unit/multiplexer_test.py deleted file mode 100644 index 737ba25d..00000000 --- a/tests/unit/multiplexer_test.py +++ /dev/null @@ -1,61 +0,0 @@ -from __future__ import absolute_import -from __future__ import unicode_literals - -import unittest -from time import sleep - -from compose.cli.multiplexer import Multiplexer - - -class MultiplexerTest(unittest.TestCase): - def test_no_iterators(self): - mux = Multiplexer([]) - self.assertEqual([], list(mux.loop())) - - def test_empty_iterators(self): - mux = Multiplexer([ - (x for x in []), - (x for x in []), - ]) - - self.assertEqual([], list(mux.loop())) - - def test_aggregates_output(self): - mux = Multiplexer([ - (x for x in [0, 2, 4]), - (x for x in [1, 3, 5]), - ]) - - self.assertEqual( - [0, 1, 2, 3, 4, 5], - sorted(list(mux.loop())), - ) - - def test_exception(self): - class Problem(Exception): - pass - - def problematic_iterator(): - yield 0 - yield 2 - raise Problem(":(") - - mux = Multiplexer([ - problematic_iterator(), - (x for x in [1, 3, 5]), - ]) - - with self.assertRaises(Problem): - list(mux.loop()) - - def test_cascade_stop(self): - def fast_stream(): - for num in range(3): - yield "stream1 %s" % num - - def slow_stream(): - sleep(5) - yield "stream2 FAIL" - - mux = Multiplexer([fast_stream(), slow_stream()], cascade_stop=True) - assert "stream2 FAIL" not in set(mux.loop()) diff --git a/tests/unit/network_test.py b/tests/unit/network_test.py new file mode 100644 index 00000000..12d06f41 --- /dev/null +++ b/tests/unit/network_test.py @@ -0,0 +1,55 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import pytest + +from .. import unittest +from compose.config import ConfigurationError +from compose.network import check_remote_network_config +from compose.network import Network + + +class NetworkTest(unittest.TestCase): + def test_check_remote_network_config_success(self): + options = {'com.docker.network.driver.foo': 'bar'} + net = Network( + None, 'compose_test', 'net1', 'bridge', + options + ) + check_remote_network_config( + {'Driver': 'bridge', 'Options': options}, net + ) + + def test_check_remote_network_config_whitelist(self): + options = {'com.docker.network.driver.foo': 'bar'} + remote_options = { + 'com.docker.network.driver.overlay.vxlanid_list': '257', + 'com.docker.network.driver.foo': 'bar' + } + net = Network( + None, 'compose_test', 'net1', 'overlay', + options + ) + check_remote_network_config( + {'Driver': 'overlay', 'Options': remote_options}, net + ) + + def test_check_remote_network_config_driver_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + with pytest.raises(ConfigurationError): + check_remote_network_config( + {'Driver': 'bridge', 'Options': {}}, net + ) + + def test_check_remote_network_config_options_mismatch(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + with pytest.raises(ConfigurationError): + check_remote_network_config({'Driver': 'overlay', 'Options': { + 'com.docker.network.driver.foo': 'baz' + }}, net) + + def test_check_remote_network_config_null_remote(self): + net = Network(None, 'compose_test', 'net1', 'overlay') + check_remote_network_config( + {'Driver': 'overlay', 'Options': None}, net + ) diff --git a/tests/unit/parallel_test.py b/tests/unit/parallel_test.py new file mode 100644 index 00000000..2a50b718 --- /dev/null +++ b/tests/unit/parallel_test.py @@ -0,0 +1,91 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import six +from docker.errors import APIError + +from compose.parallel import parallel_execute +from compose.parallel import parallel_execute_iter +from compose.parallel import UpstreamError + + +web = 'web' +db = 'db' +data_volume = 'data_volume' +cache = 'cache' + +objects = [web, db, data_volume, cache] + +deps = { + web: [db, cache], + db: [data_volume], + data_volume: [], + cache: [], +} + + +def get_deps(obj): + return [(dep, None) for dep in deps[obj]] + + +def test_parallel_execute(): + results, errors = parallel_execute( + objects=[1, 2, 3, 4, 5], + func=lambda x: x * 2, + get_name=six.text_type, + msg="Doubling", + ) + + assert sorted(results) == [2, 4, 6, 8, 10] + assert errors == {} + + +def test_parallel_execute_with_deps(): + log = [] + + def process(x): + log.append(x) + + parallel_execute( + objects=objects, + func=process, + get_name=lambda obj: obj, + msg="Processing", + get_deps=get_deps, + ) + + assert sorted(log) == sorted(objects) + + assert log.index(data_volume) < log.index(db) + assert log.index(db) < log.index(web) + assert log.index(cache) < log.index(web) + + +def test_parallel_execute_with_upstream_errors(): + log = [] + + def process(x): + if x is data_volume: + raise APIError(None, None, "Something went wrong") + log.append(x) + + parallel_execute( + objects=objects, + func=process, + get_name=lambda obj: obj, + msg="Processing", + get_deps=get_deps, + ) + + assert log == [cache] + + events = [ + (obj, result, type(exception)) + for obj, result, exception + in parallel_execute_iter(objects, process, get_deps) + ] + + assert (cache, None, type(None)) in events + assert (data_volume, None, APIError) in events + assert (db, None, UpstreamError) in events + assert (web, None, UpstreamError) in events diff --git a/tests/unit/progress_stream_test.py b/tests/unit/progress_stream_test.py index b01be11a..c0cb906d 100644 --- a/tests/unit/progress_stream_test.py +++ b/tests/unit/progress_stream_test.py @@ -65,3 +65,23 @@ class ProgressStreamTestCase(unittest.TestCase): events = progress_stream.stream_output(events, output) self.assertTrue(len(output.getvalue()) > 0) + + +def test_get_digest_from_push(): + digest = "sha256:abcd" + events = [ + {"status": "..."}, + {"status": "..."}, + {"progressDetail": {}, "aux": {"Digest": digest}}, + ] + assert progress_stream.get_digest_from_push(events) == digest + + +def test_get_digest_from_pull(): + digest = "sha256:abcd" + events = [ + {"status": "..."}, + {"status": "..."}, + {"status": "Digest: %s" % digest}, + ] + assert progress_stream.get_digest_from_pull(events) == digest diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index ffd4455f..32d0adfa 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -4,6 +4,7 @@ from __future__ import unicode_literals import datetime import docker +from docker.errors import NotFound from .. import mock from .. import unittest @@ -12,12 +13,13 @@ from compose.config.types import VolumeFromSpec from compose.const import LABEL_SERVICE from compose.container import Container from compose.project import Project +from compose.service import ImageType from compose.service import Service class ProjectTest(unittest.TestCase): def setUp(self): - self.mock_client = mock.create_autospec(docker.Client) + self.mock_client = mock.create_autospec(docker.APIClient) def test_from_config(self): config = Config( @@ -34,6 +36,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ) project = Project.from_config( name='composetest', @@ -45,7 +48,7 @@ class ProjectTest(unittest.TestCase): self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') self.assertEqual(project.get_service('db').name, 'db') self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') - self.assertFalse(project.use_networking) + self.assertFalse(project.networks.use_networking) def test_from_config_v2(self): config = Config( @@ -62,10 +65,11 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ) project = Project.from_config('composetest', config, None) self.assertEqual(len(project.services), 2) - self.assertTrue(project.use_networking) + self.assertTrue(project.networks.use_networking) def test_get_service(self): web = Service( @@ -168,6 +172,7 @@ class ProjectTest(unittest.TestCase): }], networks=None, volumes=None, + secrets=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_id + ":rw"] @@ -200,6 +205,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) assert project.get_service('test')._get_volumes_from() == [container_name + ":rw"] @@ -225,6 +231,7 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) with mock.patch.object(Service, 'containers') as mock_return: @@ -268,12 +275,21 @@ class ProjectTest(unittest.TestCase): 'time': 1420092061, 'timeNano': 14200920610000004000, }, + { + 'status': 'destroy', + 'from': 'example/db', + 'id': 'eeeee', + 'time': 1420092061, + 'timeNano': 14200920610000004000, + }, ]) def dt_with_microseconds(dt, us): return datetime.datetime.fromtimestamp(dt).replace(microsecond=us) def get_container(cid): + if cid == 'eeeee': + raise NotFound(None, None, "oops") if cid == 'abcde': name = 'web' labels = {LABEL_SERVICE: name} @@ -307,6 +323,7 @@ class ProjectTest(unittest.TestCase): 'image': 'example/image', }, 'time': dt_with_microseconds(1420092061, 2), + 'container': Container(None, {'Id': 'abcde'}), }, { 'type': 'container', @@ -318,6 +335,7 @@ class ProjectTest(unittest.TestCase): 'image': 'example/image', }, 'time': dt_with_microseconds(1420092061, 3), + 'container': Container(None, {'Id': 'abcde'}), }, { 'type': 'container', @@ -329,6 +347,7 @@ class ProjectTest(unittest.TestCase): 'image': 'example/db', }, 'time': dt_with_microseconds(1420092061, 4), + 'container': Container(None, {'Id': 'ababa'}), }, ] @@ -346,10 +365,11 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') - self.assertEqual(service.net.id, None) + self.assertEqual(service.network_mode.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) def test_use_net_from_container(self): @@ -365,15 +385,16 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'net': 'container:aaa' + 'network_mode': 'container:aaa' }, ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') - self.assertEqual(service.net.mode, 'container:' + container_id) + self.assertEqual(service.network_mode.mode, 'container:' + container_id) def test_use_net_from_service(self): container_name = 'test_aaa_1' @@ -398,16 +419,17 @@ class ProjectTest(unittest.TestCase): { 'name': 'test', 'image': 'busybox:latest', - 'net': 'container:aaa' + 'network_mode': 'service:aaa' }, ], networks=None, volumes=None, + secrets=None, ), ) service = project.get_service('test') - self.assertEqual(service.net.mode, 'container:' + container_name) + self.assertEqual(service.network_mode.mode, 'container:' + container_name) def test_uses_default_network_true(self): project = Project.from_config( @@ -423,10 +445,11 @@ class ProjectTest(unittest.TestCase): ], networks=None, volumes=None, + secrets=None, ), ) - assert project.uses_default_network() + assert 'default' in project.networks.networks def test_uses_default_network_false(self): project = Project.from_config( @@ -438,15 +461,16 @@ class ProjectTest(unittest.TestCase): { 'name': 'foo', 'image': 'busybox:latest', - 'networks': ['custom'] + 'networks': {'custom': None} }, ], networks={'custom': {}}, volumes=None, + secrets=None, ), ) - assert not project.uses_default_network() + assert 'default' not in project.networks.networks def test_container_without_name(self): self.mock_client.containers.return_value = [ @@ -473,6 +497,60 @@ class ProjectTest(unittest.TestCase): }], networks=None, volumes=None, + secrets=None, ), ) self.assertEqual([c.id for c in project.containers()], ['1']) + + def test_down_with_no_resources(self): + project = Project.from_config( + name='test', + client=self.mock_client, + config_data=Config( + version='2', + services=[{ + 'name': 'web', + 'image': 'busybox:latest', + }], + networks={'default': {}}, + volumes={'data': {}}, + secrets=None, + ), + ) + self.mock_client.remove_network.side_effect = NotFound(None, None, 'oops') + self.mock_client.remove_volume.side_effect = NotFound(None, None, 'oops') + + project.down(ImageType.all, True) + self.mock_client.remove_image.assert_called_once_with("busybox:latest") + + def test_warning_in_swarm_mode(self): + self.mock_client.info.return_value = {'Swarm': {'LocalNodeState': 'active'}} + project = Project('composetest', [], self.mock_client) + + with mock.patch('compose.project.log') as fake_log: + project.up() + assert fake_log.warn.call_count == 1 + + def test_no_warning_on_stop(self): + self.mock_client.info.return_value = {'Swarm': {'LocalNodeState': 'active'}} + project = Project('composetest', [], self.mock_client) + + with mock.patch('compose.project.log') as fake_log: + project.stop() + assert fake_log.warn.call_count == 0 + + def test_no_warning_in_normal_mode(self): + self.mock_client.info.return_value = {'Swarm': {'LocalNodeState': 'inactive'}} + project = Project('composetest', [], self.mock_client) + + with mock.patch('compose.project.log') as fake_log: + project.up() + assert fake_log.warn.call_count == 0 + + def test_no_warning_with_no_swarm_info(self): + self.mock_client.info.return_value = {} + project = Project('composetest', [], self.mock_client) + + with mock.patch('compose.project.log') as fake_log: + project.up() + assert fake_log.warn.call_count == 0 diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c9244a47..2d5b1761 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -2,6 +2,7 @@ from __future__ import absolute_import from __future__ import unicode_literals import docker +import pytest from docker.errors import APIError from .. import mock @@ -13,25 +14,27 @@ from compose.const import LABEL_ONE_OFF from compose.const import LABEL_PROJECT from compose.const import LABEL_SERVICE from compose.container import Container +from compose.project import OneOffFilter from compose.service import build_ulimits from compose.service import build_volume_binding -from compose.service import ContainerNet +from compose.service import BuildAction +from compose.service import ContainerNetworkMode from compose.service import get_container_data_volumes from compose.service import ImageType from compose.service import merge_volume_bindings from compose.service import NeedsBuildError -from compose.service import Net +from compose.service import NetworkMode from compose.service import NoSuchImageError from compose.service import parse_repository_tag from compose.service import Service -from compose.service import ServiceNet +from compose.service import ServiceNetworkMode from compose.service import warn_on_masked_volume class ServiceTest(unittest.TestCase): def setUp(self): - self.mock_client = mock.create_autospec(docker.Client) + self.mock_client = mock.create_autospec(docker.APIClient) def test_containers(self): service = Service('db', self.mock_client, 'myproject', image='foo') @@ -146,7 +149,13 @@ class ServiceTest(unittest.TestCase): def test_memory_swap_limit(self): self.mock_client.create_host_config.return_value = {} - service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, mem_limit=1000000000, memswap_limit=2000000000) + service = Service( + name='foo', + image='foo', + hostname='name', + client=self.mock_client, + mem_limit=1000000000, + memswap_limit=2000000000) service._get_container_create_options({'some': 'overrides'}, 1) self.assertTrue(self.mock_client.create_host_config.called) @@ -162,7 +171,12 @@ class ServiceTest(unittest.TestCase): def test_cgroup_parent(self): self.mock_client.create_host_config.return_value = {} - service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, cgroup_parent='test') + service = Service( + name='foo', + image='foo', + hostname='name', + client=self.mock_client, + cgroup_parent='test') service._get_container_create_options({'some': 'overrides'}, 1) self.assertTrue(self.mock_client.create_host_config.called) @@ -176,7 +190,13 @@ class ServiceTest(unittest.TestCase): log_opt = {'syslog-address': 'tcp://192.168.0.42:123'} logging = {'driver': 'syslog', 'options': log_opt} - service = Service(name='foo', image='foo', hostname='name', client=self.mock_client, logging=logging) + service = Service( + name='foo', + image='foo', + hostname='name', + client=self.mock_client, + log_driver='syslog', + logging=logging) service._get_container_create_options({'some': 'overrides'}, 1) self.assertTrue(self.mock_client.create_host_config.called) @@ -237,7 +257,7 @@ class ServiceTest(unittest.TestCase): opts = service._get_container_create_options( {'name': name}, 1, - one_off=True) + one_off=OneOffFilter.only) self.assertEqual(opts['name'], name) def test_get_container_create_options_does_not_mutate_options(self): @@ -266,14 +286,53 @@ class ServiceTest(unittest.TestCase): self.assertEqual( opts['labels'][LABEL_CONFIG_HASH], - '3c85881a8903b9d73a06c41860c8be08acce1494ab4cf8408375966dccd714de') - self.assertEqual( - opts['environment'], - { - 'affinity:container': '=ababab', - 'also': 'real', - } + '2524a06fcb3d781aa2c981fc40bcfa08013bb318e4273bfa388df22023e6f2aa') + assert opts['environment'] == ['also=real'] + + def test_get_container_create_options_sets_affinity_with_binds(self): + service = Service( + 'foo', + image='foo', + client=self.mock_client, ) + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + prev_container = mock.Mock( + id='ababab', + image_config={'ContainerConfig': {'Volumes': ['/data']}}) + + def container_get(key): + return { + 'Mounts': [ + { + 'Destination': '/data', + 'Source': '/some/path', + 'Name': 'abab1234', + }, + ] + }.get(key, None) + + prev_container.get.side_effect = container_get + + opts = service._get_container_create_options( + {}, + 1, + previous_container=prev_container) + + assert opts['environment'] == ['affinity:container==ababab'] + + def test_get_container_create_options_no_affinity_without_binds(self): + service = Service('foo', image='foo', client=self.mock_client) + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + prev_container = mock.Mock( + id='ababab', + image_config={'ContainerConfig': {}}) + prev_container.get.return_value = None + + opts = service._get_container_create_options( + {}, + 1, + previous_container=prev_container) + assert opts['environment'] == [] def test_get_container_not_found(self): self.mock_client.containers.return_value = [] @@ -348,13 +407,20 @@ class ServiceTest(unittest.TestCase): self.assertEqual(parse_repository_tag("user/repo"), ("user/repo", "", ":")) self.assertEqual(parse_repository_tag("user/repo:tag"), ("user/repo", "tag", ":")) self.assertEqual(parse_repository_tag("url:5000/repo"), ("url:5000/repo", "", ":")) - self.assertEqual(parse_repository_tag("url:5000/repo:tag"), ("url:5000/repo", "tag", ":")) + self.assertEqual( + parse_repository_tag("url:5000/repo:tag"), + ("url:5000/repo", "tag", ":")) + self.assertEqual( + parse_repository_tag("root@sha256:digest"), + ("root", "sha256:digest", "@")) + self.assertEqual( + parse_repository_tag("user/repo@sha256:digest"), + ("user/repo", "sha256:digest", "@")) + self.assertEqual( + parse_repository_tag("url:5000/repo@sha256:digest"), + ("url:5000/repo", "sha256:digest", "@")) - self.assertEqual(parse_repository_tag("root@sha256:digest"), ("root", "sha256:digest", "@")) - self.assertEqual(parse_repository_tag("user/repo@sha256:digest"), ("user/repo", "sha256:digest", "@")) - self.assertEqual(parse_repository_tag("url:5000/repo@sha256:digest"), ("url:5000/repo", "sha256:digest", "@")) - - def test_create_container_with_build(self): + def test_create_container(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = [ NoSuchImageError, @@ -364,7 +430,12 @@ class ServiceTest(unittest.TestCase): '{"stream": "Successfully built abcd"}', ] - service.create_container(do_build=True) + with mock.patch('compose.service.log', autospec=True) as mock_log: + service.create_container() + assert mock_log.warn.called + _, args, _ = mock_log.warn.mock_calls[0] + assert 'was built because it did not already exist' in args[0] + self.mock_client.build.assert_called_once_with( tag='default_foo', dockerfile=None, @@ -377,18 +448,41 @@ class ServiceTest(unittest.TestCase): buildargs=None, ) - def test_create_container_no_build(self): + def test_ensure_image_exists_no_build(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.return_value = {'Id': 'abc123'} - service.create_container(do_build=False) - self.assertFalse(self.mock_client.build.called) + service.ensure_image_exists(do_build=BuildAction.skip) + assert not self.mock_client.build.called - def test_create_container_no_build_but_needs_build(self): + def test_ensure_image_exists_no_build_but_needs_build(self): service = Service('foo', client=self.mock_client, build={'context': '.'}) self.mock_client.inspect_image.side_effect = NoSuchImageError - with self.assertRaises(NeedsBuildError): - service.create_container(do_build=False) + with pytest.raises(NeedsBuildError): + service.ensure_image_exists(do_build=BuildAction.skip) + + def test_ensure_image_exists_force_build(self): + service = Service('foo', client=self.mock_client, build={'context': '.'}) + self.mock_client.inspect_image.return_value = {'Id': 'abc123'} + self.mock_client.build.return_value = [ + '{"stream": "Successfully built abcd"}', + ] + + with mock.patch('compose.service.log', autospec=True) as mock_log: + service.ensure_image_exists(do_build=BuildAction.force) + + assert not mock_log.warn.called + self.mock_client.build.assert_called_once_with( + tag='default_foo', + dockerfile=None, + stream=True, + path='.', + pull=False, + forcerm=False, + nocache=False, + rm=True, + buildargs=None, + ) def test_build_does_not_pull(self): self.mock_client.build.return_value = [ @@ -407,7 +501,8 @@ class ServiceTest(unittest.TestCase): 'foo', image='example.com/foo', client=self.mock_client, - net=ServiceNet(Service('other')), + network_mode=ServiceNetworkMode(Service('other')), + networks={'default': None}, links=[(Service('one'), 'one')], volumes_from=[VolumeFromSpec(Service('two'), 'rw', 'service')]) @@ -417,11 +512,12 @@ class ServiceTest(unittest.TestCase): 'options': {'image': 'example.com/foo'}, 'links': [('one', 'one')], 'net': 'other', + 'networks': {'default': None}, 'volumes_from': [('two', 'rw')], } - self.assertEqual(config_dict, expected) + assert config_dict == expected - def test_config_dict_with_net_from_container(self): + def test_config_dict_with_network_mode_from_container(self): self.mock_client.inspect_image.return_value = {'Id': 'abcd'} container = Container( self.mock_client, @@ -430,17 +526,18 @@ class ServiceTest(unittest.TestCase): 'foo', image='example.com/foo', client=self.mock_client, - net=container) + network_mode=ContainerNetworkMode(container)) config_dict = service.config_dict() expected = { 'image_id': 'abcd', 'options': {'image': 'example.com/foo'}, 'links': [], + 'networks': {}, 'net': 'aaabbb', 'volumes_from': [], } - self.assertEqual(config_dict, expected) + assert config_dict == expected def test_remove_image_none(self): web = Service('web', image='example', client=self.mock_client) @@ -536,14 +633,6 @@ class ServiceTest(unittest.TestCase): ports=["127.0.0.1:1000-2000:2000-3000"]) self.assertEqual(service.specifies_host_port(), True) - def test_get_links_with_networking(self): - service = Service( - 'foo', - image='foo', - links=[(Service('one'), 'one')], - use_networking=True) - self.assertEqual(service._get_links(link_to_self=True), []) - def test_image_name_from_config(self): image_name = 'example/web:latest' service = Service('foo', image=image_name) @@ -553,6 +642,55 @@ class ServiceTest(unittest.TestCase): service = Service('foo', project='testing') assert service.image_name == 'testing_foo' + @mock.patch('compose.service.log', autospec=True) + def test_only_log_warning_when_host_ports_clash(self, mock_log): + self.mock_client.inspect_image.return_value = {'Id': 'abcd'} + name = 'foo' + service = Service( + name, + client=self.mock_client, + ports=["8080:80"]) + + service.scale(0) + self.assertFalse(mock_log.warn.called) + + service.scale(1) + self.assertFalse(mock_log.warn.called) + + service.scale(2) + mock_log.warn.assert_called_once_with( + 'The "{}" service specifies a port on the host. If multiple containers ' + 'for this service are created on a single host, the port will clash.'.format(name)) + + +class TestServiceNetwork(object): + + def test_connect_container_to_networks_short_aliase_exists(self): + mock_client = mock.create_autospec(docker.APIClient) + service = Service( + 'db', + mock_client, + 'myproject', + image='foo', + networks={'project_default': {}}) + container = Container( + None, + { + 'Id': 'abcdef', + 'NetworkSettings': { + 'Networks': { + 'project_default': { + 'Aliases': ['analias', 'abcdef'], + }, + }, + }, + }, + True) + service.connect_container_to_networks(container) + + assert not mock_client.disconnect_container_from_network.call_count + assert not mock_client.connect_container_to_network.call_count + def sort_by_name(dictionary_list): return sorted(dictionary_list, key=lambda k: k['name']) @@ -597,45 +735,45 @@ class BuildUlimitsTestCase(unittest.TestCase): class NetTestCase(unittest.TestCase): - def test_net(self): - net = Net('host') - self.assertEqual(net.id, 'host') - self.assertEqual(net.mode, 'host') - self.assertEqual(net.service_name, None) + def test_network_mode(self): + network_mode = NetworkMode('host') + self.assertEqual(network_mode.id, 'host') + self.assertEqual(network_mode.mode, 'host') + self.assertEqual(network_mode.service_name, None) - def test_net_container(self): + def test_network_mode_container(self): container_id = 'abcd' - net = ContainerNet(Container(None, {'Id': container_id})) - self.assertEqual(net.id, container_id) - self.assertEqual(net.mode, 'container:' + container_id) - self.assertEqual(net.service_name, None) + network_mode = ContainerNetworkMode(Container(None, {'Id': container_id})) + self.assertEqual(network_mode.id, container_id) + self.assertEqual(network_mode.mode, 'container:' + container_id) + self.assertEqual(network_mode.service_name, None) - def test_net_service(self): + def test_network_mode_service(self): container_id = 'bbbb' service_name = 'web' - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) mock_client.containers.return_value = [ {'Id': container_id, 'Name': container_id, 'Image': 'abcd'}, ] service = Service(name=service_name, client=mock_client) - net = ServiceNet(service) + network_mode = ServiceNetworkMode(service) - self.assertEqual(net.id, service_name) - self.assertEqual(net.mode, 'container:' + container_id) - self.assertEqual(net.service_name, service_name) + self.assertEqual(network_mode.id, service_name) + self.assertEqual(network_mode.mode, 'container:' + container_id) + self.assertEqual(network_mode.service_name, service_name) - def test_net_service_no_containers(self): + def test_network_mode_service_no_containers(self): service_name = 'web' - mock_client = mock.create_autospec(docker.Client) + mock_client = mock.create_autospec(docker.APIClient) mock_client.containers.return_value = [] service = Service(name=service_name, client=mock_client) - net = ServiceNet(service) + network_mode = ServiceNetworkMode(service) - self.assertEqual(net.id, service_name) - self.assertEqual(net.mode, None) - self.assertEqual(net.service_name, service_name) + self.assertEqual(network_mode.id, service_name) + self.assertEqual(network_mode.mode, None) + self.assertEqual(network_mode.service_name, service_name) def build_mount(destination, source, mode='rw'): @@ -645,10 +783,10 @@ def build_mount(destination, source, mode='rw'): class ServiceVolumesTest(unittest.TestCase): def setUp(self): - self.mock_client = mock.create_autospec(docker.Client) + self.mock_client = mock.create_autospec(docker.APIClient) def test_build_volume_binding(self): - binding = build_volume_binding(VolumeSpec.parse('/outside:/inside')) + binding = build_volume_binding(VolumeSpec.parse('/outside:/inside', True)) assert binding == ('/inside', '/outside:/inside:rw') def test_get_container_data_volumes(self): @@ -656,6 +794,7 @@ class ServiceVolumesTest(unittest.TestCase): '/host/volume:/host/volume:ro', '/new/volume', '/existing/volume', + 'named:/named/vol', ]] self.mock_client.inspect_image.return_value = { @@ -697,8 +836,8 @@ class ServiceVolumesTest(unittest.TestCase): }, has_been_inspected=True) expected = [ - VolumeSpec.parse('/var/lib/docker/aaaaaaaa:/existing/volume:rw'), - VolumeSpec.parse('/var/lib/docker/cccccccc:/mnt/image/data:rw'), + VolumeSpec.parse('existingvolume:/existing/volume:rw'), + VolumeSpec.parse('imagedata:/mnt/image/data:rw'), ] volumes = get_container_data_volumes(container, options) @@ -706,17 +845,18 @@ class ServiceVolumesTest(unittest.TestCase): def test_merge_volume_bindings(self): options = [ - VolumeSpec.parse('/host/volume:/host/volume:ro'), - VolumeSpec.parse('/host/rw/volume:/host/rw/volume'), - VolumeSpec.parse('/new/volume'), - VolumeSpec.parse('/existing/volume'), + VolumeSpec.parse('/host/volume:/host/volume:ro', True), + VolumeSpec.parse('/host/rw/volume:/host/rw/volume', True), + VolumeSpec.parse('/new/volume', True), + VolumeSpec.parse('/existing/volume', True), ] self.mock_client.inspect_image.return_value = { 'ContainerConfig': {'Volumes': {}} } - intermediate_container = Container(self.mock_client, { + previous_container = Container(self.mock_client, { + 'Id': 'cdefab', 'Image': 'ababab', 'Mounts': [{ 'Source': '/var/lib/docker/aaaaaaaa', @@ -730,19 +870,20 @@ class ServiceVolumesTest(unittest.TestCase): expected = [ '/host/volume:/host/volume:ro', '/host/rw/volume:/host/rw/volume:rw', - '/var/lib/docker/aaaaaaaa:/existing/volume:rw', + 'existingvolume:/existing/volume:rw', ] - binds = merge_volume_bindings(options, intermediate_container) - self.assertEqual(set(binds), set(expected)) + binds, affinity = merge_volume_bindings(options, previous_container) + assert sorted(binds) == sorted(expected) + assert affinity == {'affinity:container': '=cdefab'} def test_mount_same_host_path_to_two_volumes(self): service = Service( 'web', image='busybox', volumes=[ - VolumeSpec.parse('/host/path:/data1'), - VolumeSpec.parse('/host/path:/data2'), + VolumeSpec.parse('/host/path:/data1', True), + VolumeSpec.parse('/host/path:/data2', True), ], client=self.mock_client, ) @@ -767,13 +908,14 @@ class ServiceVolumesTest(unittest.TestCase): ]), ) - def test_different_host_path_in_container_json(self): + def test_get_container_create_options_with_different_host_path_in_container_json(self): service = Service( 'web', image='busybox', volumes=[VolumeSpec.parse('/host/path:/data')], client=self.mock_client, ) + volume_name = 'abcdefff1234' self.mock_client.inspect_image.return_value = { 'Id': 'ababab', @@ -794,7 +936,7 @@ class ServiceVolumesTest(unittest.TestCase): 'Mode': '', 'RW': True, 'Driver': 'local', - 'Name': 'abcdefff1234' + 'Name': volume_name, }, ] } @@ -805,9 +947,9 @@ class ServiceVolumesTest(unittest.TestCase): previous_container=Container(self.mock_client, {'Id': '123123123'}), ) - self.assertEqual( - self.mock_client.create_host_config.call_args[1]['binds'], - ['/mnt/sda1/host/path:/data:rw'], + assert ( + self.mock_client.create_host_config.call_args[1]['binds'] == + ['{}:/data:rw'.format(volume_name)] ) def test_warn_on_masked_volume_no_warning_when_no_container_volumes(self): @@ -865,7 +1007,7 @@ class ServiceVolumesTest(unittest.TestCase): 'web', client=self.mock_client, image='busybox', - volumes=[VolumeSpec.parse(volume)], + volumes=[VolumeSpec.parse(volume, True)], ).create_container() assert self.mock_client.create_container.call_count == 1 diff --git a/tests/unit/timeparse_test.py b/tests/unit/timeparse_test.py new file mode 100644 index 00000000..9915932c --- /dev/null +++ b/tests/unit/timeparse_test.py @@ -0,0 +1,56 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +from compose import timeparse + + +def test_milli(): + assert timeparse.timeparse('5ms') == 0.005 + + +def test_milli_float(): + assert timeparse.timeparse('50.5ms') == 0.0505 + + +def test_second_milli(): + assert timeparse.timeparse('200s5ms') == 200.005 + + +def test_second_milli_micro(): + assert timeparse.timeparse('200s5ms10us') == 200.00501 + + +def test_second(): + assert timeparse.timeparse('200s') == 200 + + +def test_second_as_float(): + assert timeparse.timeparse('20.5s') == 20.5 + + +def test_minute(): + assert timeparse.timeparse('32m') == 1920 + + +def test_hour_minute(): + assert timeparse.timeparse('2h32m') == 9120 + + +def test_minute_as_float(): + assert timeparse.timeparse('1.5m') == 90 + + +def test_hour_minute_second(): + assert timeparse.timeparse('5h34m56s') == 20096 + + +def test_invalid_with_space(): + assert timeparse.timeparse('5h 34m 56s') is None + + +def test_invalid_with_comma(): + assert timeparse.timeparse('5h,34m,56s') is None + + +def test_invalid_with_empty_string(): + assert timeparse.timeparse('') is None diff --git a/tests/unit/utils_test.py b/tests/unit/utils_test.py index 8ee37b07..85231957 100644 --- a/tests/unit/utils_test.py +++ b/tests/unit/utils_test.py @@ -15,6 +15,10 @@ class TestJsonSplitter(object): data = '{"foo": "bar"}\n \n{"next": "obj"}' assert utils.json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}') + def test_json_splitter_leading_whitespace(self): + data = '\n \r{"foo": "bar"}\n\n {"next": "obj"}' + assert utils.json_splitter(data) == ({'foo': 'bar'}, '{"next": "obj"}') + class TestStreamAsText(object): @@ -43,3 +47,16 @@ class TestJsonStream(object): [1, 2, 3], [], ] + + def test_with_leading_whitespace(self): + stream = [ + '\n \r\n {"one": "two"}{"x": 1}', + ' {"three": "four"}\t\t{"x": 2}' + ] + output = list(utils.json_stream(stream)) + assert output == [ + {'one': 'two'}, + {'x': 1}, + {'three': 'four'}, + {'x': 2} + ] diff --git a/tests/unit/volume_test.py b/tests/unit/volume_test.py index d7ad0792..24829192 100644 --- a/tests/unit/volume_test.py +++ b/tests/unit/volume_test.py @@ -10,7 +10,7 @@ from tests import mock @pytest.fixture def mock_client(): - return mock.create_autospec(docker.Client) + return mock.create_autospec(docker.APIClient) class TestVolume(object): diff --git a/tox.ini b/tox.ini index dc85bc6d..61bc0574 100644 --- a/tox.ini +++ b/tox.ini @@ -15,7 +15,7 @@ deps = -rrequirements.txt -rrequirements-dev.txt commands = - py.test -v -rxs \ + py.test -v \ --cov=compose \ --cov-report html \ --cov-report term \ @@ -42,8 +42,10 @@ directory = coverage-html # end coverage configuration [flake8] -# Allow really long lines for now -max-line-length = 140 +max-line-length = 105 # Set this high for now -max-complexity = 12 +max-complexity = 11 exclude = compose/packages + +[pytest] +addopts = --tb=short -rxs