diff --git a/docs/Makefile b/docs/Makefile index 021e8f6e..b9ef0548 100644 --- a/docs/Makefile +++ b/docs/Makefile @@ -13,8 +13,8 @@ DOCKER_ENVS := \ -e TIMEOUT # note: we _cannot_ add "-e DOCKER_BUILDTAGS" here because even if it's unset in the shell, that would shadow the "ENV DOCKER_BUILDTAGS" set in our Dockerfile, which is very important for our official builds -# to allow `make DOCSDIR=docs docs-shell` (to create a bind mount in docs) -DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR)/$(DOCSDIR):/$(DOCSDIR)) +# to allow `make DOCSDIR=1 docs-shell` (to create a bind mount in docs) +DOCS_MOUNT := $(if $(DOCSDIR),-v $(CURDIR):/docs/content/compose) # to allow `make DOCSPORT=9000 docs` DOCSPORT := 8000 @@ -37,7 +37,7 @@ GITCOMMIT := $(shell git rev-parse --short HEAD 2>/dev/null) default: docs docs: docs-build - $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) + $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) --watch docs-draft: docs-build $(DOCKER_RUN_DOCS) -p $(if $(DOCSPORT),$(DOCSPORT):)8000 -e DOCKERHOST "$(DOCKER_DOCS_IMAGE)" hugo server --buildDrafts="true" --port=$(DOCSPORT) --baseUrl=$(HUGO_BASE_URL) --bind=$(HUGO_BIND_IP) diff --git a/docs/compose-file.md b/docs/compose-file.md index 7723a784..4f8fc9e0 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -168,44 +168,29 @@ accessible to linked services. Only the internal port can be specified. Extend another service, in the current file or another, optionally overriding configuration. -Here's a simple example. Suppose we have 2 files - **common.yml** and -**development.yml**. We can use `extends` to define a service in -**development.yml** which uses configuration defined in **common.yml**: +You can use `extends` on any service together with other configuration keys. +The `extends` value must be a dictionary defined with a required `service` +and an optional `file` key. -**common.yml** + extends: + file: common.yml + service: webapp - webapp: - build: ./webapp - environment: - - DEBUG=false - - SEND_EMAILS=false +The `service` the name of the service being extended, for example +`web` or `database`. The `file` is the location of a Compose configuration +file defining that service. -**development.yml** +If you omit the `file` Compose looks for the service configuration in the +current file. The `file` value can be an absolute or relative path. If you +specify a relative path, Compose treats it as relative to the location of the +current file. - web: - extends: - file: common.yml - service: webapp - ports: - - "8000:8000" - links: - - db - environment: - - DEBUG=true - db: - image: postgres +You can extend a service that itself extends another. You can extend +indefinitely. Compose does not support circular references and `docker-compose` +returns an error if it encounters one. -Here, the `web` service in **development.yml** inherits the configuration of -the `webapp` service in **common.yml** - the `build` and `environment` keys - -and adds `ports` and `links` configuration. It overrides one of the defined -environment variables (DEBUG) with a new value, and the other one -(SEND_EMAILS) is left untouched. - -The `file` key is optional, if it is not set then Compose will look for the -service within the current file. - -For more on `extends`, see the [tutorial](extends.md#example) and -[reference](extends.md#reference). +For more on `extends`, see the +[the extends documentation](extends.md#extending-services). ### external_links diff --git a/docs/extends.md b/docs/extends.md index e63cf466..b21b6d76 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -10,18 +10,176 @@ weight=2 -## Extending services in Compose +# Extending services and Compose files + +Compose supports two methods of sharing common configuration: + +1. Extending an entire Compose file by + [using multiple Compose files](#multiple-compose-files) +2. Extending individual services with [the `extends` field](#extending-services) + + +## Multiple Compose files + +Using multiple Compose files enables you to customize a Compose application +for different environments or different workflows. + +### Understanding multiple Compose files + +By default, Compose reads two files, a `docker-compose.yml` and an optional +`docker-compose.override.yml` file. By convention, the `docker-compose.yml` +contains your base configuration. The override file, as its name implies, can +contain configuration overrides for existing services or entirely new +services. + +If a service is defined in both files, Compose merges the configurations using +the same rules as the `extends` field (see [Adding and overriding +configuration](#adding-and-overriding-configuration)), with one exception. If a +service contains `links` or `volumes_from` those fields are copied over and +replace any values in the original service, in the same way single-valued fields +are copied. + +To use multiple override files, or an override file with a different name, you +can use the `-f` option to specify the list of files. Compose merges files in +the order they're specified on the command line. See the [`docker-compose` +command reference](./reference/docker-compose.md) for more information about +using `-f`. + +When you use multiple configuration files, you must make sure all paths in the +files are relative to the base Compose file (the first Compose file specified +with `-f`). This is required because override files need not be valid +Compose files. Override files can contain small fragments of configuration. +Tracking which fragment of a service is relative to which path is difficult and +confusing, so to keep paths easier to understand, all paths must be defined +relative to the base file. + +### Example use case + +In this section are two common use cases for multiple compose files: changing a +Compose app for different environments, and running administrative tasks +against a Compose app. + +#### Different environments + +A common use case for multiple files is changing a development Compose app +for a production-like environment (which may be production, staging or CI). +To support these differences, you can split your Compose configuration into +a few different files: + +Start with a base file that defines the canonical configuration for the +services. + +**docker-compose.yml** + + web: + image: example/my_web_app:latest + links: + - db + - cache + + db: + image: postgres:latest + + cache: + image: redis:latest + +In this example the development configuration exposes some ports to the +host, mounts our code as a volume, and builds the web image. + +**docker-compose.override.yml** + + + web: + build: . + volumes: + - '.:/code' + ports: + - 8883:80 + environment: + DEBUG: 'true' + + db: + command: '-d' + ports: + - 5432:5432 + + cache: + ports: + - 6379:6379 + +When you run `docker-compose up` it reads the overrides automatically. + +Now, it would be nice to use this Compose app in a production environment. So, +create another override file (which might be stored in a different git +repo or managed by a different team). + +**docker-compose.prod.yml** + + web: + ports: + - 80:80 + environment: + PRODUCTION: 'true' + + cache: + environment: + TTL: '500' + +To deploy with this production Compose file you can run + + docker-compose -f docker-compose.yml -f docker-compose.prod.yml up -d + +This deploys all three services using the configuration in +`docker-compose.yml` and `docker-compose.prod.yml` (but not the +dev configuration in `docker-compose.override.yml`). + + +See [production](production.md) for more information about Compose in +production. + +#### Administrative tasks + +Another common use case is running adhoc or administrative tasks against one +or more services in a Compose app. This example demonstrates running a +database backup. + +Start with a **docker-compose.yml**. + + web: + image: example/my_web_app:latest + links: + - db + + db: + image: postgres:latest + +In a **docker-compose.admin.yml** add a new service to run the database +export or backup. + + dbadmin: + build: database_admin/ + links: + - db + +To start a normal environment run `docker-compose up -d`. To run a database +backup, include the `docker-compose.admin.yml` as well. + + docker-compose -f docker-compose.yml -f docker-compose.admin.yml \ + run dbadmin db-backup + + +## Extending services Docker Compose's `extends` keyword enables sharing of common configurations among different files, or even different projects entirely. Extending services -is useful if you have several applications that reuse commonly-defined services. -Using `extends` you can define a service in one place and refer to it from -anywhere. +is useful if you have several services that reuse a common set of configuration +options. Using `extends` you can define a common set of service options in one +place and refer to it from anywhere. -Alternatively, you can deploy the same application to multiple environments with -a slightly different set of services in each case (or with changes to the -configuration of some services). Moreover, you can do so without copy-pasting -the configuration around. +> **Note:** `links` and `volumes_from` are never shared between services using +> `extends`. See +> [Adding and overriding configuration](#adding-and-overriding-configuration) + > for more information. ### Understand the extends configuration @@ -45,8 +203,8 @@ looks like this: - "/data" In this case, you'll get exactly the same result as if you wrote -`docker-compose.yml` with that `build`, `ports` and `volumes` configuration -defined directly under `web`. +`docker-compose.yml` with the same `build`, `ports` and `volumes` configuration +values defined directly under `web`. You can go further and define (or re-define) configuration locally in `docker-compose.yml`: @@ -77,183 +235,45 @@ You can also write other services and link your `web` service to them: db: image: postgres -For full details on how to use `extends`, refer to the [reference](#reference). - ### Example use case -In this example, you’ll repurpose the example app from the [quick start -guide](/). (If you're not familiar with Compose, it's recommended that -you go through the quick start first.) This example assumes you want to use -Compose both to develop an application locally and then deploy it to a -production environment. +Extending an individual service is useful when you have multiple services that +have a common configuration. The example below is a Compose app with +two services: a web application and a queue worker. Both services use the same +codebase and share many configuration options. -The local and production environments are similar, but there are some -differences. In development, you mount the application code as a volume so that -it can pick up changes; in production, the code should be immutable from the -outside. This ensures it’s not accidentally changed. The development environment -uses a local Redis container, but in production another team manages the Redis -service, which is listening at `redis-production.example.com`. +In a **common.yml** we define the common configuration: -To configure with `extends` for this sample, you must: + app: + build: . + environment: + CONFIG_FILE_PATH: /code/config + API_KEY: xxxyyy + cpu_shares: 5 -1. Define the web application as a Docker image in `Dockerfile` and a Compose - service in `common.yml`. +In a **docker-compose.yml** we define the concrete services which use the +common configuration: -2. Define the development environment in the standard Compose file, - `docker-compose.yml`. + webapp: + extends: + file: common.yml + service: app + command: /code/run_web_app + ports: + - 8080:8080 + links: + - queue + - db - - Use `extends` to pull in the web service. - - Configure a volume to enable code reloading. - - Create an additional Redis service for the application to use locally. + queue_worker: + extends: + file: common.yml + service: app + command: /code/run_worker + links: + - queue -3. Define the production environment in a third Compose file, `production.yml`. - - - Use `extends` to pull in the web service. - - Configure the web service to talk to the external, production Redis service. - -#### Define the web app - -Defining the web application requires the following: - -1. Create an `app.py` file. - - This file contains a simple Python application that uses Flask to serve HTTP - and increments a counter in Redis: - - from flask import Flask - from redis import Redis - import os - - app = Flask(__name__) - redis = Redis(host=os.environ['REDIS_HOST'], port=6379) - - @app.route('/') - def hello(): - redis.incr('hits') - return 'Hello World! I have been seen %s times.\n' % redis.get('hits') - - if __name__ == "__main__": - app.run(host="0.0.0.0", debug=True) - - This code uses a `REDIS_HOST` environment variable to determine where to - find Redis. - -2. Define the Python dependencies in a `requirements.txt` file: - - flask - redis - -3. Create a `Dockerfile` to build an image containing the app: - - FROM python:2.7 - ADD . /code - WORKDIR /code - RUN pip install -r requirements.txt - CMD python app.py - -4. Create a Compose configuration file called `common.yml`: - - This configuration defines how to run the app. - - web: - build: . - ports: - - "5000:5000" - - Typically, you would have dropped this configuration into - `docker-compose.yml` file, but in order to pull it into multiple files with - `extends`, it needs to be in a separate file. - -#### Define the development environment - -1. Create a `docker-compose.yml` file. - - The `extends` option pulls in the `web` service from the `common.yml` file - you created in the previous section. - - web: - extends: - file: common.yml - service: web - volumes: - - .:/code - links: - - redis - environment: - - REDIS_HOST=redis - redis: - image: redis - - The new addition defines a `web` service that: - - - Fetches the base configuration for `web` out of `common.yml`. - - Adds `volumes` and `links` configuration to the base (`common.yml`) - configuration. - - Sets the `REDIS_HOST` environment variable to point to the linked redis - container. This environment uses a stock `redis` image from the Docker Hub. - -2. Run `docker-compose up`. - - Compose creates, links, and starts a web and redis container linked together. - It mounts your application code inside the web container. - -3. Verify that the code is mounted by changing the message in - `app.py`—say, from `Hello world!` to `Hello from Compose!`. - - Don't forget to refresh your browser to see the change! - -#### Define the production environment - -You are almost done. Now, define your production environment: - -1. Create a `production.yml` file. - - As with `docker-compose.yml`, the `extends` option pulls in the `web` service - from `common.yml`. - - web: - extends: - file: common.yml - service: web - environment: - - REDIS_HOST=redis-production.example.com - -2. Run `docker-compose -f production.yml up`. - - Compose creates *just* a web container and configures the Redis connection via - the `REDIS_HOST` environment variable. This variable points to the production - Redis instance. - - > **Note**: If you try to load up the webapp in your browser you'll get an - > error—`redis-production.example.com` isn't actually a Redis server. - -You've now done a basic `extends` configuration. As your application develops, -you can make any necessary changes to the web service in `common.yml`. Compose -picks up both the development and production environments when you next run -`docker-compose`. You don't have to do any copy-and-paste, and you don't have to -manually keep both environments in sync. - - -### Reference - -You can use `extends` on any service together with other configuration keys. It -expects a dictionary that contains a `service` key and optionally a `file` key. -The `extends` key can also take a string, whose value is the name of a `service` defined in the same file. - -The `file` key specifies the location of a Compose configuration file defining -the extension. The `file` value can be an absolute or relative path. If you -specify a relative path, Docker Compose treats it as relative to the location -of the current file. If you don't specify a `file`, Compose looks in the -current configuration file. - -The `service` key specifies the name of the service to extend, for example `web` -or `database`. - -You can extend a service that itself extends another. You can extend -indefinitely. Compose does not support circular references and `docker-compose` -returns an error if it encounters them. - -#### Adding and overriding configuration +## Adding and overriding configuration Compose copies configurations from the original service over to the local one, **except** for `links` and `volumes_from`. These exceptions exist to avoid @@ -262,13 +282,11 @@ locally. This ensures dependencies between services are clearly visible when reading the current file. Defining these locally also ensures changes to the referenced file don't result in breakage. -If a configuration option is defined in both the original service and the local -service, the local value either *override*s or *extend*s the definition of the -original service. This works differently for other configuration options. +If a configuration option is defined in both the original service the local +service, the local value *replaces* or *extends* the original value. For single-value options like `image`, `command` or `mem_limit`, the new value -replaces the old value. **This is the default behaviour - all exceptions are -listed below.** +replaces the old value. # original service command: python app.py @@ -282,6 +300,8 @@ listed below.** In the case of `build` and `image`, using one in the local service causes Compose to discard the other, if it was defined in the original service. +Example of image replacing build: + # original service build: . @@ -291,6 +311,9 @@ Compose to discard the other, if it was defined in the original service. # result image: redis + +Example of build replacing image: + # original service image: redis @@ -318,8 +341,8 @@ For the **multi-value options** `ports`, `expose`, `external_links`, `dns` and - "4000" - "5000" -In the case of `environment` and `labels`, Compose "merges" entries together -with locally-defined values taking precedence: +In the case of `environment`, `labels`, `volumes` and `devices`, Compose +"merges" entries together with locally-defined values taking precedence: # original service environment: @@ -337,24 +360,8 @@ with locally-defined values taking precedence: - BAR=local - BAZ=local -Finally, for `volumes` and `devices`, Compose "merges" entries together with -locally-defined bindings taking precedence: - # original service - volumes: - - /original-dir/foo:/foo - - /original-dir/bar:/bar - # local service - volumes: - - /local-dir/bar:/bar - - /local-dir/baz/:baz - - # result - volumes: - - /original-dir/foo:/foo - - /local-dir/bar:/bar - - /local-dir/baz/:baz ## Compose documentation diff --git a/docs/production.md b/docs/production.md index 0b0e46c3..0a5e77b5 100644 --- a/docs/production.md +++ b/docs/production.md @@ -12,11 +12,9 @@ weight=1 ## Using Compose in production -While **Compose is not yet considered production-ready**, if you'd like to experiment and learn more about using it in production deployments, this guide -can help. -The project is actively working towards becoming -production-ready; to learn more about the progress being made, check out the roadmap for details -on how it's coming along and what still needs to be done. +> Compose is still primarily aimed at development and testing environments. +> Compose may be used for smaller production deployments, but is probably +> not yet suitable for larger deployments. When deploying to production, you'll almost certainly want to make changes to your app configuration that are more appropriate to a live environment. These @@ -30,22 +28,19 @@ changes may include: - Specifying a restart policy (e.g., `restart: always`) to avoid downtime - Adding extra services (e.g., a log aggregator) -For this reason, you'll probably want to define a separate Compose file, say -`production.yml`, which specifies production-appropriate configuration. +For this reason, you'll probably want to define an additional Compose file, say +`production.yml`, which specifies production-appropriate +configuration. This configuration file only needs to include the changes you'd +like to make from the original Compose file. The additional Compose file +can be applied over the original `docker-compose.yml` to create a new configuration. -> **Note:** The [extends](extends.md) keyword is useful for maintaining multiple -> Compose files which re-use common services without having to manually copy and -> paste. +Once you've got a second configuration file, tell Compose to use it with the +`-f` option: -Once you've got an alternate configuration file, make Compose use it -by setting the `COMPOSE_FILE` environment variable: + $ docker-compose -f docker-compose.yml -f production.yml up -d - $ export COMPOSE_FILE=production.yml - $ docker-compose up -d - -> **Note:** You can also use the file for a one-off command without setting -> an environment variable. You do this by passing the `-f` flag, e.g., -> `docker-compose -f production.yml up -d`. +See [Using multiple compose files](extends.md#different-environments) for a more +complete example. ### Deploying changes