diff --git a/.gitignore b/.gitignore index 1b0c5011..83a08a0e 100644 --- a/.gitignore +++ b/.gitignore @@ -7,3 +7,4 @@ /dist /docs/_site /venv +README.rst diff --git a/CHANGELOG.md b/CHANGELOG.md index 598f5e57..b0474ae2 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,108 @@ Change log ========== +1.5.0 (2015-11-02) +------------------ + +**Breaking changes:** + +With the introduction of variable substitution support in the Compose file, any +Compose file that uses an environment variable (`$VAR` or `${VAR}`) in the `command:` +or `entrypoint:` field will break. + +Previously these values were interpolated inside the container, with a value +from the container environment. In Compose 1.5.0, the values will be +interpolated on the host, with a value from the host environment. + +To migrate a Compose file to 1.5.0, escape the variables with an extra `$` +(ex: `$$VAR` or `$${VAR}`). See +https://github.com/docker/compose/blob/8cc8e61/docs/compose-file.md#variable-substitution + +Major features: + +- Compose is now available for Windows. + +- Environment variables can be used in the Compose file. See + https://github.com/docker/compose/blob/8cc8e61/docs/compose-file.md#variable-substitution + +- Multiple compose files can be specified, allowing you to override + settings in the default Compose file. See + https://github.com/docker/compose/blob/8cc8e61/docs/reference/docker-compose.md + for more details. + +- Compose now produces better error messages when a file contains + invalid configuration. + +- `up` now waits for all services to exit before shutting down, + rather than shutting down as soon as one container exits. + +- Experimental support for the new docker networking system can be + enabled with the `--x-networking` flag. Read more here: + https://github.com/docker/docker/blob/8fee1c20/docs/userguide/dockernetworks.md + +New features: + +- You can now optionally pass a mode to `volumes_from`, e.g. + `volumes_from: ["servicename:ro"]`. + +- Since Docker now lets you create volumes with names, you can refer to those + volumes by name in `docker-compose.yml`. For example, + `volumes: ["mydatavolume:/data"]` will mount the volume named + `mydatavolume` at the path `/data` inside the container. + + If the first component of an entry in `volumes` starts with a `.`, `/` or + `~`, it is treated as a path and expansion of relative paths is performed as + necessary. Otherwise, it is treated as a volume name and passed straight + through to Docker. + + Read more on named volumes and volume drivers here: + https://github.com/docker/docker/blob/244d9c33/docs/userguide/dockervolumes.md + +- `docker-compose build --pull` instructs Compose to pull the base image for + each Dockerfile before building. + +- `docker-compose pull --ignore-pull-failures` instructs Compose to continue + if it fails to pull a single service's image, rather than aborting. + +- You can now specify an IPC namespace in `docker-compose.yml` with the `ipc` + option. + +- Containers created by `docker-compose run` can now be named with the + `--name` flag. + +- If you install Compose with pip or use it as a library, it now works with + Python 3. + +- `image` now supports image digests (in addition to ids and tags), e.g. + `image: "busybox@sha256:38a203e1986cf79639cfb9b2e1d6e773de84002feea2d4eb006b52004ee8502d"` + +- `ports` now supports ranges of ports, e.g. + + ports: + - "3000-3005" + - "9000-9001:8000-8001" + +- `docker-compose run` now supports a `-p|--publish` parameter, much like + `docker run -p`, for publishing specific ports to the host. + +- `docker-compose pause` and `docker-compose unpause` have been implemented, + analogous to `docker pause` and `docker unpause`. + +- When using `extends` to copy configuration from another service in the same + Compose file, you can omit the `file` option. + +- Compose can be installed and run as a Docker image. This is an experimental + feature. + +Bug fixes: + +- All values for the `log_driver` option which are supported by the Docker + daemon are now supported by Compose. + +- `docker-compose build` can now be run successfully against a Swarm cluster. + + + 1.4.2 (2015-09-22) ------------------ diff --git a/MANIFEST.in b/MANIFEST.in index 43ae06d3..0342e35b 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -4,6 +4,8 @@ include requirements.txt include requirements-dev.txt include tox.ini include *.md +exclude README.md +include README.rst include compose/config/*.json recursive-include contrib/completion * recursive-include tests * diff --git a/README.md b/README.md index 3c776a71..d779d607 100644 --- a/README.md +++ b/README.md @@ -56,4 +56,4 @@ Want to help build Compose? Check out our [contributing documentation](https://g Releasing --------- -Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/RELEASE_PROCESS.md). +Releases are built by maintainers, following an outline of the [release process](https://github.com/docker/compose/blob/master/project/RELEASE-PROCESS.md). diff --git a/compose/__init__.py b/compose/__init__.py index e3ace983..7199babb 100644 --- a/compose/__init__.py +++ b/compose/__init__.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '1.5.0dev' +__version__ = '1.5.0rc3' diff --git a/compose/cli/formatter.py b/compose/cli/formatter.py index 9ed52c4a..d0ed0f87 100644 --- a/compose/cli/formatter.py +++ b/compose/cli/formatter.py @@ -1,10 +1,13 @@ from __future__ import absolute_import from __future__ import unicode_literals +import logging import os import texttable +from compose.cli import colors + def get_tty_width(): tty_size = os.popen('stty size', 'r').read().split() @@ -15,6 +18,7 @@ def get_tty_width(): class Formatter(object): + """Format tabular data for printing.""" def table(self, headers, rows): table = texttable.Texttable(max_width=get_tty_width()) table.set_cols_dtype(['t' for h in headers]) @@ -23,3 +27,22 @@ class Formatter(object): table.set_chars(['-', '|', '+', '-']) return table.draw() + + +class ConsoleWarningFormatter(logging.Formatter): + """A logging.Formatter which prints WARNING and ERROR messages with + a prefix of the log level colored appropriate for the log level. + """ + + def get_level_message(self, record): + separator = ': ' + if record.levelno == logging.WARNING: + return colors.yellow(record.levelname) + separator + if record.levelno == logging.ERROR: + return colors.red(record.levelname) + separator + + return '' + + def format(self, record): + message = super(ConsoleWarningFormatter, self).format(record) + return self.get_level_message(record) + message diff --git a/compose/cli/log_printer.py b/compose/cli/log_printer.py index 6e1499e1..66920726 100644 --- a/compose/cli/log_printer.py +++ b/compose/cli/log_printer.py @@ -73,9 +73,13 @@ def build_no_log_generator(container, prefix, color_func): def build_log_generator(container, prefix, color_func): - # Attach to container before log printer starts running - stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) - line_generator = split_buffer(stream) + # if the container doesn't have a log_stream we need to attach to container + # before log printer starts running + if container.log_stream is None: + stream = container.attach(stdout=True, stderr=True, stream=True, logs=True) + line_generator = split_buffer(stream) + else: + line_generator = split_buffer(container.log_stream) for line in line_generator: yield prefix + line diff --git a/compose/cli/main.py b/compose/cli/main.py index c800d95f..b54b307e 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -28,6 +28,7 @@ from .command import project_from_options from .docopt_command import DocoptCommand from .docopt_command import NoSuchCommand from .errors import UserError +from .formatter import ConsoleWarningFormatter from .formatter import Formatter from .log_printer import LogPrinter from .utils import get_version_info @@ -41,7 +42,7 @@ log = logging.getLogger(__name__) console_handler = logging.StreamHandler(sys.stderr) INSECURE_SSL_WARNING = """ -Warning: --allow-insecure-ssl is deprecated and has no effect. +--allow-insecure-ssl is deprecated and has no effect. It will be removed in a future version of Compose. """ @@ -58,9 +59,8 @@ def main(): log.error(e.msg) sys.exit(1) except NoSuchCommand as e: - log.error("No such command: %s", e.command) - log.error("") - log.error("\n".join(parse_doc_section("commands:", getdoc(e.supercommand)))) + commands = "\n".join(parse_doc_section("commands:", getdoc(e.supercommand))) + log.error("No such command: %s\n\n%s", e.command, commands) sys.exit(1) except APIError as e: log.error(e.explanation) @@ -91,13 +91,18 @@ def setup_logging(): logging.getLogger("requests").propagate = False -def setup_console_handler(verbose): - if verbose: - console_handler.setFormatter(logging.Formatter('%(name)s.%(funcName)s: %(message)s')) - console_handler.setLevel(logging.DEBUG) +def setup_console_handler(handler, verbose): + if handler.stream.isatty(): + format_class = ConsoleWarningFormatter else: - console_handler.setFormatter(logging.Formatter()) - console_handler.setLevel(logging.INFO) + format_class = logging.Formatter + + if verbose: + handler.setFormatter(format_class('%(name)s.%(funcName)s: %(message)s')) + handler.setLevel(logging.DEBUG) + else: + handler.setFormatter(format_class()) + handler.setLevel(logging.INFO) # stolen from docopt master @@ -153,7 +158,7 @@ class TopLevelCommand(DocoptCommand): return options def perform_command(self, options, handler, command_options): - setup_console_handler(options.get('--verbose')) + setup_console_handler(console_handler, options.get('--verbose')) if options['COMMAND'] in ('help', 'version'): # Skip looking up the compose file. @@ -380,6 +385,8 @@ class TopLevelCommand(DocoptCommand): start_deps=True, strategy=ConvergenceStrategy.never, ) + elif project.use_networking: + project.ensure_network_exists() tty = True if detach or options['-T'] or not sys.stdin.isatty(): @@ -565,16 +572,18 @@ class TopLevelCommand(DocoptCommand): start_deps = not options['--no-deps'] service_names = options['SERVICE'] timeout = int(options.get('--timeout') or DEFAULT_TIMEOUT) + detached = options.get('-d') to_attach = project.up( service_names=service_names, start_deps=start_deps, strategy=convergence_strategy_from_opts(options), do_build=not options['--no-build'], - timeout=timeout + timeout=timeout, + detached=detached ) - if not options['-d']: + if not detached: log_printer = build_log_printer(to_attach, service_names, monochrome) attach_to_logs(project, log_printer, service_names, timeout) @@ -636,7 +645,10 @@ def convergence_strategy_from_opts(options): def build_log_printer(containers, service_names, monochrome): if service_names: - containers = [c for c in containers if c.service in service_names] + containers = [ + container + for container in containers if container.service in service_names + ] return LogPrinter(containers, monochrome=monochrome) diff --git a/compose/config/config.py b/compose/config/config.py index 3bcd769a..21549e9b 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -1,3 +1,4 @@ +import codecs import logging import os import sys @@ -14,7 +15,6 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extended_service_exists from .validation import validate_extends_file_path -from .validation import validate_service_names from .validation import validate_top_level_object @@ -165,16 +165,6 @@ def find_candidates_in_parent_dirs(filenames, path): return (candidates, path) -@validate_top_level_object -@validate_service_names -def pre_process_config(config): - """ - Pre validation checks and processing of the config file to interpolate env - vars returning a config dict ready to be tested against the schema. - """ - return interpolate_environment_variables(config) - - def load(config_details): """Load the configuration from a working directory and a list of configuration files. Files are loaded in order, and merged on top @@ -194,7 +184,7 @@ def load(config_details): return service_dict def load_file(filename, config): - processed_config = pre_process_config(config) + processed_config = interpolate_environment_variables(config) validate_against_fields_schema(processed_config) return [ build_service(filename, name, service_config) @@ -204,12 +194,17 @@ def load(config_details): def merge_services(base, override): all_service_names = set(base) | set(override) return { - name: merge_service_dicts(base.get(name, {}), override.get(name, {})) + name: merge_service_dicts_from_files( + base.get(name, {}), + override.get(name, {})) for name in all_service_names } config_file = config_details.config_files[0] + validate_top_level_object(config_file.config) for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file.config) + config_file = ConfigFile( config_file.filename, merge_services(config_file.config, next_file.config)) @@ -278,14 +273,12 @@ class ServiceLoader(object): extends, self.filename ) - self.extended_config_path = self.get_extended_config_path( - extends - ) + self.extended_config_path = self.get_extended_config_path(extends) self.extended_service_name = extends['service'] - full_extended_config = pre_process_config( - load_yaml(self.extended_config_path) - ) + config = load_yaml(self.extended_config_path) + validate_top_level_object(config) + full_extended_config = interpolate_environment_variables(config) validate_extended_service_exists( self.extended_service_name, @@ -363,6 +356,17 @@ def process_container_options(service_dict, working_dir=None): return service_dict +def merge_service_dicts_from_files(base, override): + """When merging services from multiple files we need to merge the `extends` + field. This is not handled by `merge_service_dicts()` which is used to + perform the `extends`. + """ + new_service = merge_service_dicts(base, override) + if 'extends' in override: + new_service['extends'] = override['extends'] + return new_service + + def merge_service_dicts(base, override): d = base.copy() @@ -448,6 +452,8 @@ def parse_environment(environment): def split_env(env): + if isinstance(env, six.binary_type): + env = env.decode('utf-8') if '=' in env: return env.split('=', 1) else: @@ -470,7 +476,7 @@ def env_vars_from_file(filename): if not os.path.exists(filename): raise ConfigurationError("Couldn't find env file: %s" % filename) env = {} - for line in open(filename, 'r'): + for line in codecs.open(filename, 'r', 'utf-8'): line = line.strip() if line and not line.startswith('#'): k, v = split_env(line) @@ -610,5 +616,6 @@ def load_yaml(filename): try: with open(filename, 'r') as fh: return yaml.safe_load(fh) - except IOError as e: - raise ConfigurationError(six.text_type(e)) + except (IOError, yaml.YAMLError) as e: + error_name = getattr(e, '__module__', '') + '.' + e.__class__.__name__ + raise ConfigurationError(u"{}: {}".format(error_name, e)) diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema.json index cc37f444..e254e353 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema.json @@ -41,7 +41,7 @@ "type": "object", "patternProperties": { "^[^-]+$": { - "type": ["string", "number", "boolean"], + "type": ["string", "number", "boolean", "null"], "format": "environment" } }, diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index f870ab4b..f8e1da61 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -78,7 +78,7 @@ class BlankDefaultDict(dict): except KeyError: if key not in self.missing_keys: log.warn( - "The {} variable is not set. Substituting a blank string." + "The {} variable is not set. Defaulting to a blank string." .format(key) ) self.missing_keys.append(key) diff --git a/compose/config/validation.py b/compose/config/validation.py index 46b891b7..542081d5 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -2,7 +2,6 @@ import json import logging import os import sys -from functools import wraps import six from docker.utils.ports import split_port @@ -58,34 +57,30 @@ def format_boolean_in_environment(instance): """ if isinstance(instance, bool): log.warn( - "Warning: There is a boolean value in the 'environment' key.\n" - "Environment variables can only be strings.\nPlease add quotes to any boolean values to make them string " - "(eg, 'True', 'yes', 'N').\nThis warning will become an error in a future release. \r\n" + "There is a boolean value in the 'environment' key.\n" + "Environment variables can only be strings.\n" + "Please add quotes to any boolean values to make them string " + "(eg, 'True', 'yes', 'N').\n" + "This warning will become an error in a future release. \r\n" ) return True -def validate_service_names(func): - @wraps(func) - def func_wrapper(config): - for service_name in config.keys(): - if type(service_name) is int: - raise ConfigurationError( - "Service name: {} needs to be a string, eg '{}'".format(service_name, service_name) - ) - return func(config) - return func_wrapper - - -def validate_top_level_object(func): - @wraps(func) - def func_wrapper(config): - if not isinstance(config, dict): +def validate_service_names(config): + for service_name in config.keys(): + if not isinstance(service_name, six.string_types): raise ConfigurationError( - "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level." - ) - return func(config) - return func_wrapper + "Service name: {} needs to be a string, eg '{}'".format( + service_name, + service_name)) + + +def validate_top_level_object(config): + if not isinstance(config, dict): + raise ConfigurationError( + "Top level object needs to be a dictionary. Check your .yml file " + "that you have defined a service at the top level.") + validate_service_names(config) def validate_extends_file_path(service_name, extends_options, filename): @@ -143,7 +138,7 @@ def process_errors(errors, service_name=None): if len(validator) >= 2: first_type = anglicize_validator(validator[0]) last_type = anglicize_validator(validator[-1]) - types_from_validator = "{}{}".format(first_type, ", ".join(validator[1:-1])) + types_from_validator = ", ".join([first_type] + validator[1:-1]) msg = "{} or {}".format( types_from_validator, @@ -163,7 +158,6 @@ def process_errors(errors, service_name=None): Inspecting the context value of a ValidationError gives us information about which sub schema failed and which kind of error it is. """ - required = [context for context in error.context if context.validator == 'required'] if required: return required[0].message diff --git a/compose/container.py b/compose/container.py index a03acf56..1ca48380 100644 --- a/compose/container.py +++ b/compose/container.py @@ -19,6 +19,7 @@ class Container(object): self.client = client self.dictionary = dictionary self.has_been_inspected = has_been_inspected + self.log_stream = None @classmethod def from_ps(cls, client, dictionary, **kwargs): @@ -144,7 +145,14 @@ class Container(object): @property def has_api_logs(self): log_type = self.log_driver - return not log_type or log_type == 'json-file' + return not log_type or log_type != 'none' + + def attach_log_stream(self): + """A log stream can only be attached if the container uses a json-file + log driver. + """ + if self.has_api_logs: + self.log_stream = self.attach(stdout=True, stderr=True, stream=True) def get(self, key): """Return a value from the container or None if the value is not set. @@ -184,6 +192,15 @@ class Container(object): def remove(self, **options): return self.client.remove_container(self.id, **options) + def rename_to_tmp_name(self): + """Rename the container to a hopefully unique temporary container name + by prepending the short id. + """ + self.client.rename( + self.id, + '%s_%s' % (self.short_id, self.name) + ) + def inspect_if_not_inspected(self): if not self.has_been_inspected: self.inspect() diff --git a/compose/project.py b/compose/project.py index 0e20a4ce..1e01eaf6 100644 --- a/compose/project.py +++ b/compose/project.py @@ -5,6 +5,7 @@ import logging from functools import reduce from docker.errors import APIError +from docker.errors import NotFound from .config import ConfigurationError from .config import get_service_name_from_net @@ -82,7 +83,7 @@ class Project(object): self.services = services self.client = client self.use_networking = use_networking - self.network_driver = network_driver or 'bridge' + self.network_driver = network_driver def labels(self, one_off=False): return [ @@ -289,7 +290,8 @@ class Project(object): start_deps=True, strategy=ConvergenceStrategy.changed, do_build=True, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + detached=False): services = self.get_services(service_names, include_deps=start_deps) @@ -307,7 +309,8 @@ class Project(object): for container in service.execute_convergence_plan( plans[service.name], do_build=do_build, - timeout=timeout + timeout=timeout, + detached=detached ) ] @@ -335,7 +338,7 @@ class Project(object): return plans def pull(self, service_names=None, ignore_pull_failures=False): - for service in self.get_services(service_names, include_deps=True): + for service in self.get_services(service_names, include_deps=False): service.pull(ignore_pull_failures) def containers(self, service_names=None, stopped=False, one_off=False): @@ -363,10 +366,10 @@ class Project(object): return [c for c in containers if matches_service_names(c)] def get_network(self): - networks = self.client.networks(names=[self.name]) - if networks: - return networks[0] - return None + try: + return self.client.inspect_network(self.name) + except NotFound: + return None def ensure_network_exists(self): # TODO: recreate network if driver has changed? diff --git a/compose/service.py b/compose/service.py index 5f1d5946..66c90b0e 100644 --- a/compose/service.py +++ b/compose/service.py @@ -399,13 +399,17 @@ class Service(object): def execute_convergence_plan(self, plan, do_build=True, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + detached=False): (action, containers) = plan + should_attach_logs = not detached if action == 'create': - container = self.create_container( - do_build=do_build, - ) + container = self.create_container(do_build=do_build) + + if should_attach_logs: + container.attach_log_stream() + self.start_container(container) return [container] @@ -413,15 +417,16 @@ class Service(object): elif action == 'recreate': return [ self.recreate_container( - c, - timeout=timeout + container, + timeout=timeout, + attach_logs=should_attach_logs ) - for c in containers + for container in containers ] elif action == 'start': - for c in containers: - self.start_container_if_stopped(c) + for container in containers: + self.start_container_if_stopped(container, attach_logs=should_attach_logs) return containers @@ -436,7 +441,8 @@ class Service(object): def recreate_container(self, container, - timeout=DEFAULT_TIMEOUT): + timeout=DEFAULT_TIMEOUT, + attach_logs=False): """Recreate a container. The original container is renamed to a temporary name so that data @@ -444,36 +450,28 @@ class Service(object): container is removed. """ log.info("Recreating %s" % container.name) - try: - container.stop(timeout=timeout) - except APIError as e: - if (e.response.status_code == 500 - and e.explanation - and 'no such process' in str(e.explanation)): - pass - else: - raise - - # Use a hopefully unique container name by prepending the short id - self.client.rename( - container.id, - '%s_%s' % (container.short_id, container.name)) + container.stop(timeout=timeout) + container.rename_to_tmp_name() new_container = self.create_container( do_build=False, previous_container=container, number=container.labels.get(LABEL_CONTAINER_NUMBER), quiet=True, ) + if attach_logs: + new_container.attach_log_stream() self.start_container(new_container) container.remove() return new_container - def start_container_if_stopped(self, container): + def start_container_if_stopped(self, container, attach_logs=False): if container.is_running: return container else: log.info("Starting %s" % container.name) + if attach_logs: + container.attach_log_stream() return self.start_container(container) def start_container(self, container): @@ -545,6 +543,9 @@ class Service(object): return 1 if not numbers else max(numbers) + 1 def _get_links(self, link_to_self): + if self.use_networking: + return [] + links = [] for service, link_name in self.links: for container in service.containers(): @@ -847,7 +848,7 @@ class ServiceNet(object): if containers: return 'container:' + containers[0].id - log.warn("Warning: Service %s is trying to use reuse the network stack " + log.warn("Service %s is trying to use reuse the network stack " "of another service that is not running." % (self.id)) return None @@ -943,24 +944,22 @@ def build_volume_binding(volume_spec): def normalize_paths_for_engine(external_path, internal_path): - """ - Windows paths, c:\my\path\shiny, need to be changed to be compatible with + """Windows paths, c:\my\path\shiny, need to be changed to be compatible with the Engine. Volume paths are expected to be linux style /c/my/path/shiny/ """ - if IS_WINDOWS_PLATFORM: - if external_path: - drive, tail = os.path.splitdrive(external_path) - - if drive: - reformatted_drive = "/{}".format(drive.replace(":", "")) - external_path = reformatted_drive + tail - - external_path = "/".join(external_path.split("\\")) - - return external_path, "/".join(internal_path.split("\\")) - else: + if not IS_WINDOWS_PLATFORM: return external_path, internal_path + if external_path: + drive, tail = os.path.splitdrive(external_path) + + if drive: + external_path = '/' + drive.lower().rstrip(':') + tail + + external_path = external_path.replace('\\', '/') + + return external_path, internal_path.replace('\\', '/') + def parse_volume_spec(volume_config): """ diff --git a/contrib/completion/bash/docker-compose b/contrib/completion/bash/docker-compose index e64b24a0..0eed1f18 100644 --- a/contrib/completion/bash/docker-compose +++ b/contrib/completion/bash/docker-compose @@ -105,11 +105,15 @@ _docker_compose_docker_compose() { --project-name|-p) return ;; + --x-network-driver) + COMPREPLY=( $( compgen -W "bridge host none overlay" -- "$cur" ) ) + return + ;; esac case "$cur" in -*) - COMPREPLY=( $( compgen -W "--help -h --verbose --version -v --file -f --project-name -p" -- "$cur" ) ) + COMPREPLY=( $( compgen -W "--file -f --help -h --project-name -p --verbose --version -v --x-networking --x-network-driver" -- "$cur" ) ) ;; *) COMPREPLY=( $( compgen -W "${commands[*]}" -- "$cur" ) ) @@ -410,6 +414,9 @@ _docker_compose() { (( counter++ )) compose_project="${words[$counter]}" ;; + --x-network-driver) + (( counter++ )) + ;; -*) ;; *) diff --git a/contrib/completion/zsh/_docker-compose b/contrib/completion/zsh/_docker-compose index cefcb109..d79b25d1 100644 --- a/contrib/completion/zsh/_docker-compose +++ b/contrib/completion/zsh/_docker-compose @@ -330,6 +330,8 @@ _docker-compose() { '(- :)'{-v,--version}'[Print version and exit]' \ '(-f --file)'{-f,--file}'[Specify an alternate docker-compose file (default: docker-compose.yml)]:file:_files -g "*.yml"' \ '(-p --project-name)'{-p,--project-name}'[Specify an alternate project name (default: directory name)]:project name:' \ + '--x-networking[(EXPERIMENTAL) Use new Docker networking functionality. Requires Docker 1.9 or later.]' \ + '--x-network-driver[(EXPERIMENTAL) Specify a network driver (default: "bridge"). Requires Docker 1.9 or later.]:Network Driver:(bridge host none overlay)' \ '(-): :->command' \ '(-)*:: :->option-or-argument' && ret=0 diff --git a/docs/compose-file.md b/docs/compose-file.md index 67322335..b72a7cc4 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -48,8 +48,6 @@ See `man 7 capabilities` for a full list. - NET_ADMIN - SYS_ADMIN -Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. - ### command Override the default command. @@ -106,6 +104,8 @@ Compose will use an alternate file to build with. dockerfile: Dockerfile-alternate +Using `dockerfile` together with `image` is not allowed. Attempting to do so results in an error. + ### env_file Add environment variables from a file. Can be a single value or a list. @@ -428,9 +428,18 @@ Both `$VARIABLE` and `${VARIABLE}` syntax are supported. Extended shell-style features, such as `${VARIABLE-default}` and `${VARIABLE/foo/bar}`, are not supported. -If you need to put a literal dollar sign in a configuration value, use a double -dollar sign (`$$`). +You can use a `$$` (double-dollar sign) when your configuration needs a literal +dollar sign. This also prevents Compose from interpolating a value, so a `$$` +allows you to refer to environment variables that you don't want processed by +Compose. + web: + build: . + command: "$$VAR_NOT_INTERPOLATED_BY_COMPOSE" + +If you forget and use a single dollar sign (`$`), Compose interprets the value as an environment variable and will warn you: + + The VAR_NOT_INTERPOLATED_BY_COMPOSE is not set. Substituting an empty string. ## Compose documentation diff --git a/docs/django.md b/docs/django.md index 2ebf4b4b..2bb67399 100644 --- a/docs/django.md +++ b/docs/django.md @@ -64,9 +64,10 @@ and a `docker-compose.yml` file. The `docker-compose.yml` file describes the services that make your app. In this example those services are a web server and database. The compose file - also describes which Docker images these services use, any volumes they might - need mounted inside the containers, and any ports they might - expose. See the [`docker-compose.yml` reference](yml.md) for more + also describes which Docker images these services use, how they link + together, any volumes they might need mounted inside the containers. + Finally, the `docker-compose.yml` file describes which ports these services + expose. See the [`docker-compose.yml` reference](compose-file.md) for more information on how this file works. 9. Add the following configuration to the file. @@ -80,6 +81,8 @@ and a `docker-compose.yml` file. - .:/code ports: - "8000:8000" + links: + - db This file defines two services: The `db` service and the `web` service. @@ -107,8 +110,20 @@ In this step, you create a Django started project by building the image from the 3. After the `docker-compose` command completes, list the contents of your project. - $ ls - Dockerfile docker-compose.yml composeexample manage.py requirements.txt + $ ls -l + drwxr-xr-x 2 root root composeexample + -rw-rw-r-- 1 user user docker-compose.yml + -rw-rw-r-- 1 user user Dockerfile + -rwxr-xr-x 1 root root manage.py + -rw-rw-r-- 1 user user requirements.txt + + The files `django-admin` created are owned by root. This happens because + the container runs as the `root` user. + +4. Change the ownership of the new files. + + sudo chown -R $USER:$USER . + ## Connect the database diff --git a/docs/env.md b/docs/env.md index 8f3cc3cc..d7b51ba2 100644 --- a/docs/env.md +++ b/docs/env.md @@ -11,7 +11,7 @@ weight=3 # Compose environment variables reference -**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](yml.md#links) for details. +**Note:** Environment variables are no longer the recommended method for connecting to linked services. Instead, you should use the link name (by default, the name of the linked service) as the hostname to connect to. See the [docker-compose.yml documentation](compose-file.md#links) for details. Compose uses [Docker links] to expose services' containers to one another. Each linked container injects a set of environment variables, each of which begins with the uppercase name of the container. diff --git a/docs/extends.md b/docs/extends.md index d88ce61c..f0b9e9ea 100644 --- a/docs/extends.md +++ b/docs/extends.md @@ -59,6 +59,10 @@ You can go further and define (or re-define) configuration locally in - DEBUG=1 cpu_shares: 5 + important_web: + extends: web + cpu_shares: 10 + You can also write other services and link your `web` service to them: web: @@ -233,7 +237,8 @@ manually keep both environments in sync. ### Reference You can use `extends` on any service together with other configuration keys. It -always expects a dictionary that should always contain the key: `service` and optionally the `file` key. +expects a dictionary that contains a `service` key and optionally a `file` key. +The `extends` key can also take a string, whose value is the name of a `service` defined in the same file. The `file` key specifies the location of a Compose configuration file defining the extension. The `file` value can be an absolute or relative path. If you diff --git a/docs/index.md b/docs/index.md index e19e7d7f..62c78d68 100644 --- a/docs/index.md +++ b/docs/index.md @@ -154,7 +154,7 @@ Now, when you run `docker-compose up`, Compose will pull a Redis image, build an If you're using [Docker Machine](https://docs.docker.com/machine), then `docker-machine ip MACHINE_VM` will tell you its address and you can open `http://MACHINE_VM_IP:5000` in a browser. -If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If http://0.0.0.0:5000 doesn't resolve, you can also try http://localhost:5000. +If you're using Docker on Linux natively, then the web app should now be listening on port 5000 on your Docker daemon host. If `http://0.0.0.0:5000` doesn't resolve, you can also try `http://localhost:5000`. You should get a message in your browser saying: diff --git a/docs/install.md b/docs/install.md index 2d4d6cad..711902c7 100644 --- a/docs/install.md +++ b/docs/install.md @@ -30,13 +30,14 @@ To install Compose, do the following: 3. Go to the Compose repository release page on GitHub. -4. Follow the instructions from the release page and run the `curl` command in your terminal. +4. Follow the instructions from the release page and run the `curl` command, +which the release page specifies, in your terminal. > Note: If you get a "Permission denied" error, your `/usr/local/bin` directory probably isn't writable and you'll need to install Compose as the superuser. Run `sudo -i`, then the two commands below, then `exit`. - The command has the following format: + The following is an example command illustrating the format: curl -L https://github.com/docker/compose/releases/download/VERSION_NUM/docker-compose-`uname -s`-`uname -m` > /usr/local/bin/docker-compose @@ -53,7 +54,7 @@ To install Compose, do the following: 7. Test the installation. $ docker-compose --version - docker-compose version: 1.4.2 + docker-compose version: 1.5.0rc3 ## Alternative install options @@ -75,7 +76,7 @@ to get started. Compose can also be run inside a container, from a small bash script wrapper. To install compose as a container run: - $ curl -L https://github.com/docker/compose/releases/download/1.5.0/run.sh > /usr/local/bin/docker-compose + $ curl -L https://github.com/docker/compose/releases/download/1.5.0rc3/run.sh > /usr/local/bin/docker-compose $ chmod +x /usr/local/bin/docker-compose ## Master builds diff --git a/docs/networking.md b/docs/networking.md index f4227917..718d56c7 100644 --- a/docs/networking.md +++ b/docs/networking.md @@ -12,11 +12,15 @@ weight=6 # Networking in Compose -> **Note:** Compose’s networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. +> **Note:** Compose's networking support is experimental, and must be explicitly enabled with the `docker-compose --x-networking` flag. -Compose sets up a single default [network](http://TODO/docker-networking-docs) for your app. Each container for a service joins the default network and is both *reachable* by other containers on that network, and *discoverable* by them at a hostname identical to the service's name. +Compose sets up a single default +[network](/engine/reference/commandline/network_create.md) for your app. Each +container for a service joins the default network and is both *reachable* by +other containers on that network, and *discoverable* by them at a hostname +identical to the container name. -> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [CLI docs](cli.md#p-project-name-name) for how to override it. +> **Note:** Your app's network is given the same name as the "project name", which is based on the name of the directory it lives in. See the [Command line overview](reference/docker-compose.md) for how to override it. For example, suppose your app is in a directory called `myapp`, and your `docker-compose.yml` looks like this: @@ -30,13 +34,23 @@ For example, suppose your app is in a directory called `myapp`, and your `docker When you run `docker-compose --x-networking up`, the following happens: 1. A network called `myapp` is created. -2. A container is created using `web`'s configuration. It joins the network `myapp` under the name `web`. -3. A container is created using `db`'s configuration. It joins the network `myapp` under the name `db`. +2. A container is created using `web`'s configuration. It joins the network +`myapp` under the name `myapp_web_1`. +3. A container is created using `db`'s configuration. It joins the network +`myapp` under the name `myapp_db_1`. -Each container can now look up the hostname `web` or `db` and get back the appropriate container's IP address. For example, `web`'s application code could connect to the URL `postgres://db:5432` and start using the Postgres database. +Each container can now look up the hostname `myapp_web_1` or `myapp_db_1` and +get back the appropriate container's IP address. For example, `web`'s +application code could connect to the URL `postgres://myapp_db_1:5432` and start +using the Postgres database. Because `web` explicitly maps a port, it's also accessible from the outside world via port 8000 on your Docker host's network interface. +> **Note:** in the next release there will be additional aliases for the +> container, including a short name without the project name and container +> index. The full container name will remain as one of the alias for backwards +> compatibility. + ## Updating containers If you make a configuration change to a service and run `docker-compose up` to update it, the old container will be removed and the new one will join the network under a different IP address but the same name. Running containers will be able to look up that name and connect to the new address, but the old address will stop working. @@ -45,19 +59,13 @@ If any containers have connections open to the old container, they will be close ## Configure how services are published -By default, containers for each service are published on the network with the same name as the service. If you want to change the name, or stop containers from being discoverable at all, you can use the `hostname` option: +By default, containers for each service are published on the network with the +container name. If you want to change the name, or stop containers from being +discoverable at all, you can use the `container_name` option: web: build: . - hostname: "my-web-application" - -This will also change the hostname inside the container, so that the `hostname` command will return `my-web-application`. - -## Scaling services - -If you create multiple containers for a service with `docker-compose scale`, each container will join the network with the same name. For example, if you run `docker-compose scale web=3`, then 3 containers will join the network under the name `web`. Inside any container on the network, looking up the name `web` will return the IP address of one of them, but Docker and Compose do not provide any guarantees about which one. - -This limitation will be addressed in a future version of Compose, where a load balancer will join under the service name and balance traffic between the service's containers in a configurable manner. + container_name: "my-web-application" ## Links @@ -65,7 +73,7 @@ Docker links are a one-way, single-host communication system. They should now be ## Specifying the network driver -By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](http://TODO/custom-driver-docs). +By default, Compose uses the `bridge` driver when creating the app’s network. The Docker Engine provides one other driver out-of-the-box: `overlay`, which implements secure communication between containers on different hosts (see the next section for how to set up and use the `overlay` driver). Docker also allows you to install [custom network drivers](/engine/extend/plugins_network.md). You can specify which one to use with the `--x-network-driver` flag: diff --git a/docs/rails.md b/docs/rails.md index 9801ef74..e81675c5 100644 --- a/docs/rails.md +++ b/docs/rails.md @@ -37,7 +37,11 @@ Next, create a bootstrap `Gemfile` which just loads Rails. It'll be overwritten source 'https://rubygems.org' gem 'rails', '4.2.0' -Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to expose the web app's port. +You'll need an empty `Gemfile.lock` in order to build our `Dockerfile`. + + $ touch Gemfile.lock + +Finally, `docker-compose.yml` is where the magic happens. This file describes the services that comprise your app (a database and a web app), how to get each one's Docker image (the database just runs on a pre-made PostgreSQL image, and the web app is built from the current directory), and the configuration needed to link them together and expose the web app's port. db: image: postgres @@ -48,6 +52,8 @@ Finally, `docker-compose.yml` is where the magic happens. This file describes th - .:/myapp ports: - "3000:3000" + links: + - db ### Build the project @@ -67,6 +73,12 @@ image. Once it's done, you should have generated a fresh app: README.rdoc config.ru public Rakefile db test + +The files `rails new` created are owned by root. This happens because the +container runs as the `root` user. Change the ownership of the new files. + + sudo chown -R $USER:$USER . + Uncomment the line in your new `Gemfile` which loads `therubyracer`, so you've got a Javascript runtime: @@ -78,6 +90,7 @@ rebuild.) $ docker-compose build + ### Connect the database The app is now bootable, but you're not quite there yet. By default, Rails @@ -85,8 +98,7 @@ expects a database to be running on `localhost` - so you need to point it at the `db` container instead. You also need to change the database and username to align with the defaults set by the `postgres` image. -Open up your newly-generated `database.yml` file. Replace its contents with the -following: +Replace the contents of `config/database.yml` with the following: development: &default adapter: postgresql diff --git a/docs/wordpress.md b/docs/wordpress.md index 5c9bcdbd..8c1f5b0a 100644 --- a/docs/wordpress.md +++ b/docs/wordpress.md @@ -46,6 +46,8 @@ and a separate MySQL instance: command: php -S 0.0.0.0:8000 -t /code ports: - "8000:8000" + links: + - db volumes: - .:/code db: diff --git a/project/ISSUE-TRIAGE.md b/project/ISSUE-TRIAGE.md index 58312a60..b89cdc24 100644 --- a/project/ISSUE-TRIAGE.md +++ b/project/ISSUE-TRIAGE.md @@ -20,15 +20,16 @@ The following labels are provided in additional to the standard labels: Most issues should fit into one of the following functional areas: -| Area | -|----------------| -| area/build | -| area/cli | -| area/config | -| area/logs | -| area/packaging | -| area/run | -| area/scale | -| area/tests | -| area/up | -| area/volumes | +| Area | +|-----------------| +| area/build | +| area/cli | +| area/config | +| area/logs | +| area/networking | +| area/packaging | +| area/run | +| area/scale | +| area/tests | +| area/up | +| area/volumes | diff --git a/project/RELEASE-PROCESS.md b/project/RELEASE-PROCESS.md index 85bbaf29..040a2602 100644 --- a/project/RELEASE-PROCESS.md +++ b/project/RELEASE-PROCESS.md @@ -1,6 +1,14 @@ Building a Compose release ========================== +## Prerequisites + +The release scripts require the following tools installed on the host: + +* https://hub.github.com/ +* https://stedolan.github.io/jq/ +* http://pandoc.org/ + ## To get started with a new release Create a branch, update version, and add release notes by running `make-branch` @@ -40,10 +48,10 @@ As part of this script you'll be asked to: ## To release a version (whether RC or stable) -Check out the bump branch and run the `build-binary` script +Check out the bump branch and run the `build-binaries` script git checkout bump-$VERSION - ./script/release/build-binary + ./script/release/build-binaries When prompted build the non-linux binaries and test them. @@ -54,7 +62,7 @@ When prompted build the non-linux binaries and test them. 2. Download the windows binary from AppVeyor - https://ci.appveyor.com/project/docker/compose/build//artifacts + https://ci.appveyor.com/project/docker/compose 3. Draft a release from the tag on GitHub (the script will open the window for you) @@ -82,17 +90,20 @@ When prompted build the non-linux binaries and test them. 5. Attach the binaries and `script/run.sh` -6. If everything looks good, it's time to push the release. +6. Add "Thanks" with a list of contributors. The contributor list can be generated + by running `./script/release/contributors`. + +7. If everything looks good, it's time to push the release. ./script/release/push-release -7. Publish the release on GitHub. +8. Publish the release on GitHub. -8. Check that both binaries download (following the install instructions) and run. +9. Check that all the binaries download (following the install instructions) and run. -9. Email maintainers@dockerproject.org and engineering@docker.com about the new release. +10. Email maintainers@dockerproject.org and engineering@docker.com about the new release. ## If it’s a stable release (not an RC) diff --git a/requirements-build.txt b/requirements-build.txt index 5da6fa49..20aad420 100644 --- a/requirements-build.txt +++ b/requirements-build.txt @@ -1 +1 @@ -git+https://github.com/pyinstaller/pyinstaller.git@12e40471c77f588ea5be352f7219c873ddaae056#egg=pyinstaller +pyinstaller==3.0 diff --git a/script/build-windows.ps1 b/script/build-windows.ps1 index 6e8a7c5a..42a4a501 100644 --- a/script/build-windows.ps1 +++ b/script/build-windows.ps1 @@ -42,11 +42,6 @@ Get-ChildItem -Recurse -Include *.pyc | foreach ($_) { Remove-Item $_.FullName } virtualenv .\venv # Install dependencies -# TODO: pip warns when installing from a git sha, so we need to set ErrorAction to -# 'Continue'. See -# https://github.com/pypa/pip/blob/fbc4b7ae5fee00f95bce9ba4b887b22681327bb1/pip/vcs/git.py#L77 -# This can be removed once pyinstaller 3.x is released and we upgrade -$ErrorActionPreference = "Continue" .\venv\Scripts\pip install pypiwin32==219 .\venv\Scripts\pip install -r requirements.txt .\venv\Scripts\pip install --no-deps . @@ -54,8 +49,9 @@ $ErrorActionPreference = "Continue" # Build binary # pyinstaller has lots of warnings, so we need to run with ErrorAction = Continue +$ErrorActionPreference = "Continue" .\venv\Scripts\pyinstaller .\docker-compose.spec $ErrorActionPreference = "Stop" -Move-Item -Force .\dist\docker-compose .\dist\docker-compose-Windows-x86_64.exe +Move-Item -Force .\dist\docker-compose.exe .\dist\docker-compose-Windows-x86_64.exe .\dist\docker-compose-Windows-x86_64.exe --version diff --git a/script/ci b/script/ci index 12dc3c47..f30265c0 100755 --- a/script/ci +++ b/script/ci @@ -11,7 +11,9 @@ set -ex docker version export DOCKER_VERSIONS=all -export DOCKER_DAEMON_ARGS="--storage-driver=overlay" +STORAGE_DRIVER=${STORAGE_DRIVER:-overlay} +export DOCKER_DAEMON_ARGS="--storage-driver=$STORAGE_DRIVER" + GIT_VOLUME="--volumes-from=$(hostname)" . script/test-versions diff --git a/script/release/cherry-pick-pr b/script/release/cherry-pick-pr index 60460087..f4a5a740 100755 --- a/script/release/cherry-pick-pr +++ b/script/release/cherry-pick-pr @@ -22,7 +22,7 @@ EOM if [ -z "$(command -v hub 2> /dev/null)" ]; then >&2 echo "$0 requires https://hub.github.com/." - >&2 echo "Please install it and ake sure it is available on your \$PATH." + >&2 echo "Please install it and make sure it is available on your \$PATH." exit 2 fi diff --git a/script/release/contributors b/script/release/contributors new file mode 100755 index 00000000..bb9fe871 --- /dev/null +++ b/script/release/contributors @@ -0,0 +1,27 @@ +#!/bin/bash +set -e + + +function usage() { + >&2 cat << EOM +Print the list of github contributors for the release + +Usage: + + $0 +EOM + exit 1 +} + +[[ -n "$1" ]] || usage +PREV_RELEASE=$1 +VERSION=HEAD +URL="https://api.github.com/repos/docker/compose/compare" + +curl -sf "$URL/$PREV_RELEASE...$VERSION" | \ + jq -r '.commits[].author.login' | \ + sort | \ + uniq -c | \ + sort -nr | \ + awk '{print "@"$2","}' | \ + xargs echo diff --git a/script/release/make-branch b/script/release/make-branch index dde1fb65..e2eae4d5 100755 --- a/script/release/make-branch +++ b/script/release/make-branch @@ -39,7 +39,7 @@ fi DEFAULT_REMOTE=release REMOTE="$(find_remote "$GITHUB_REPO")" -# If we don't have a docker origin add one +# If we don't have a docker remote add one if [ -z "$REMOTE" ]; then echo "Creating $DEFAULT_REMOTE remote" git remote add ${DEFAULT_REMOTE} ${GITHUB_REPO} @@ -55,6 +55,8 @@ read -n1 -r -p "Continue? (ctrl+c to cancel)" git fetch $REMOTE -p git checkout -b $BRANCH $BASE_VERSION +echo "Merging remote release branch into new release branch" +git merge --strategy=ours --no-edit $REMOTE/release # Store the release version for this branch in git, so that other release # scripts can use it diff --git a/script/release/push-release b/script/release/push-release index 039436da..ccdf2496 100755 --- a/script/release/push-release +++ b/script/release/push-release @@ -21,11 +21,17 @@ VERSION="$(git config "branch.${BRANCH}.release")" || usage if [ -z "$(command -v jq 2> /dev/null)" ]; then >&2 echo "$0 requires https://stedolan.github.io/jq/" - >&2 echo "Please install it and ake sure it is available on your \$PATH." + >&2 echo "Please install it and make sure it is available on your \$PATH." exit 2 fi +if [ -z "$(command -v pandoc 2> /dev/null)" ]; then + >&2 echo "$0 requires http://pandoc.org/" + >&2 echo "Please install it and make sure it is available on your \$PATH." + exit 2 +fi + API=https://api.github.com/repos REPO=docker/compose GITHUB_REPO=git@github.com:$REPO @@ -34,7 +40,9 @@ GITHUB_REPO=git@github.com:$REPO sha=$(git rev-parse HEAD) url=$API/$REPO/statuses/$sha build_status=$(curl -s $url | jq -r '.[0].state') -if [[ "$build_status" != "success" ]]; then +if [ -n "$SKIP_BUILD_CHECK" ]; then + echo "Skipping build status check..." +elif [[ "$build_status" != "success" ]]; then >&2 echo "Build status is $build_status, but it should be success." exit -1 fi @@ -43,12 +51,13 @@ echo "Tagging the release as $VERSION" git tag $VERSION git push $GITHUB_REPO $VERSION -echo "Uploading sdist to pypi" -python setup.py sdist - echo "Uploading the docker image" docker push docker/compose:$VERSION +echo "Uploading sdist to pypi" +pandoc -f markdown -t rst README.md -o README.rst +sed -i -e 's/logo.png?raw=true/https:\/\/github.com\/docker\/compose\/raw\/master\/logo.png?raw=true/' README.rst +python setup.py sdist if [ "$(command -v twine 2> /dev/null)" ]; then twine upload ./dist/docker-compose-${VERSION}.tar.gz else @@ -61,6 +70,7 @@ source venv-test/bin/activate pip install docker-compose==$VERSION docker-compose version deactivate +rm -rf venv-test echo "Now publish the github release, and test the downloads." echo "Email maintainers@dockerproject.org and engineering@docker.com about the new release." diff --git a/script/run.sh b/script/run.sh index cf46c143..9ed1ea74 100755 --- a/script/run.sh +++ b/script/run.sh @@ -15,7 +15,7 @@ set -e -VERSION="1.5.0" +VERSION="1.5.0rc3" IMAGE="docker/compose:$VERSION" diff --git a/script/test-versions b/script/test-versions index 89793359..43326ccb 100755 --- a/script/test-versions +++ b/script/test-versions @@ -28,10 +28,15 @@ for version in $DOCKER_VERSIONS; do >&2 echo "Running tests against Docker $version" daemon_container="compose-dind-$version-$BUILD_NUMBER" - trap "docker rm -vf $daemon_container" EXIT - # TODO: remove when we stop testing against 1.7.x - daemon=$([[ "$version" == "1.7"* ]] && echo "-d" || echo "daemon") + function on_exit() { + if [[ "$?" != "0" ]]; then + docker logs "$daemon_container" + fi + docker rm -vf "$daemon_container" + } + + trap "on_exit" EXIT docker run \ -d \ @@ -39,7 +44,7 @@ for version in $DOCKER_VERSIONS; do --privileged \ --volume="/var/lib/docker" \ dockerswarm/dind:$version \ - docker $daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ + docker daemon -H tcp://0.0.0.0:2375 $DOCKER_DAEMON_ARGS \ docker run \ --rm \ diff --git a/tests/fixtures/env/resolve.env b/tests/fixtures/env/resolve.env index 720520d2..b4f76b29 100644 --- a/tests/fixtures/env/resolve.env +++ b/tests/fixtures/env/resolve.env @@ -1,4 +1,4 @@ -FILE_DEF=F1 +FILE_DEF=bär FILE_DEF_EMPTY= ENV_DEF NO_DEF diff --git a/tests/integration/cli_test.py b/tests/integration/cli_test.py index 78519d14..45f45645 100644 --- a/tests/integration/cli_test.py +++ b/tests/integration/cli_test.py @@ -187,7 +187,7 @@ class CLITestCase(DockerClientTestCase): ) def test_up_without_networking(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['up', '-d'], None) @@ -205,7 +205,7 @@ class CLITestCase(DockerClientTestCase): self.assertTrue(web_container.get('HostConfig.Links')) def test_up_with_networking(self): - self.require_engine_version("1.9") + self.require_api_version('1.21') self.command.base_dir = 'tests/fixtures/links-composefile' self.command.dispatch(['--x-networking', 'up', '-d'], None) @@ -508,6 +508,20 @@ class CLITestCase(DockerClientTestCase): container, = service.containers(stopped=True, one_off=True) self.assertEqual(container.name, name) + @mock.patch('dockerpty.start') + def test_run_with_networking(self, _): + self.require_api_version('1.21') + client = docker_client(version='1.21') + self.command.base_dir = 'tests/fixtures/simple-dockerfile' + self.command.dispatch(['--x-networking', 'run', 'simple', 'true'], None) + service = self.project.get_service('simple') + container, = service.containers(stopped=True, one_off=True) + networks = client.networks(names=[self.project.name]) + for n in networks: + self.addCleanup(client.remove_network, n['id']) + self.assertEqual(len(networks), 1) + self.assertEqual(container.human_readable_command, u'true') + def test_rm(self): service = self.project.get_service('simple') service.create_container() diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index ff50c80b..fd45b939 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,6 +1,7 @@ from __future__ import unicode_literals from .testcases import DockerClientTestCase +from compose.cli.docker_client import docker_client from compose.config import config from compose.const import LABEL_PROJECT from compose.container import Container @@ -96,6 +97,22 @@ class ProjectTest(DockerClientTestCase): db = project.get_service('db') self.assertEqual(db._get_volumes_from(), [data_container.id + ':rw']) + def test_get_network_does_not_exist(self): + self.require_api_version('1.21') + client = docker_client(version='1.21') + + project = Project('composetest', [], client) + assert project.get_network() is None + + def test_get_network(self): + self.require_api_version('1.21') + client = docker_client(version='1.21') + + network_name = 'network_does_exist' + project = Project(network_name, [], client) + client.create_network(network_name) + assert project.get_network()['name'] == network_name + def test_net_from_service(self): project = Project.from_dicts( name='composetest', diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index 8a8e4d54..4ac04545 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -362,6 +362,7 @@ class ServiceTest(DockerClientTestCase): new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) + self.assertEqual(list(new_container.get('Volumes')), ['/data']) self.assertEqual(new_container.get('Volumes')['/data'], volume_path) @@ -692,10 +693,11 @@ class ServiceTest(DockerClientTestCase): @mock.patch('compose.service.log') def test_scale_with_custom_container_name_outputs_warning(self, mock_log): - """ - Test that calling scale on a service that has a custom container name + """Test that calling scale on a service that has a custom container name results in warning output. """ + # Disable this test against earlier versions because it is flaky + self.require_api_version('1.21') service = self.create_service('app', container_name='custom-container') self.assertEqual(service.custom_container_name(), 'custom-container') diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index ef7276bd..3230aefc 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -18,6 +18,7 @@ from compose.service import ConvergenceStrategy class ProjectTestCase(DockerClientTestCase): def run_up(self, cfg, **kwargs): kwargs.setdefault('timeout', 1) + kwargs.setdefault('detached', True) project = self.make_project(cfg) project.up(**kwargs) diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index a412fb04..686a2b69 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -76,11 +76,7 @@ class DockerClientTestCase(unittest.TestCase): build_output = self.client.build(*args, **kwargs) stream_output(build_output, open('/dev/null', 'w')) - def require_engine_version(self, minimum): - # Drop '-dev' or '-rcN' suffix - engine = self.client.version()['Version'].split('-', 1)[0] - if version_lt(engine, minimum): - skip( - "Engine version is too low ({} < {})" - .format(engine, minimum) - ) + def require_api_version(self, minimum): + api_version = self.client.version()['ApiVersion'] + if version_lt(api_version, minimum): + skip("API version is too low ({} < {})".format(api_version, minimum)) diff --git a/tests/unit/cli/formatter_test.py b/tests/unit/cli/formatter_test.py new file mode 100644 index 00000000..1c3b6a68 --- /dev/null +++ b/tests/unit/cli/formatter_test.py @@ -0,0 +1,35 @@ +from __future__ import absolute_import +from __future__ import unicode_literals + +import logging + +from compose.cli import colors +from compose.cli.formatter import ConsoleWarningFormatter +from tests import unittest + + +MESSAGE = 'this is the message' + + +def makeLogRecord(level): + return logging.LogRecord('name', level, 'pathame', 0, MESSAGE, (), None) + + +class ConsoleWarningFormatterTestCase(unittest.TestCase): + + def setUp(self): + self.formatter = ConsoleWarningFormatter() + + def test_format_warn(self): + output = self.formatter.format(makeLogRecord(logging.WARN)) + expected = colors.yellow('WARNING') + ': ' + assert output == expected + MESSAGE + + def test_format_error(self): + output = self.formatter.format(makeLogRecord(logging.ERROR)) + expected = colors.red('ERROR') + ': ' + assert output == expected + MESSAGE + + def test_format_info(self): + output = self.formatter.format(makeLogRecord(logging.INFO)) + assert output == MESSAGE diff --git a/tests/unit/cli/log_printer_test.py b/tests/unit/cli/log_printer_test.py index 2c916898..575fcaf7 100644 --- a/tests/unit/cli/log_printer_test.py +++ b/tests/unit/cli/log_printer_test.py @@ -16,6 +16,7 @@ def build_mock_container(reader): name='myapp_web_1', name_without_project='web_1', has_api_logs=True, + log_stream=None, attach=reader, wait=mock.Mock(return_value=0), ) diff --git a/tests/unit/cli/main_test.py b/tests/unit/cli/main_test.py index a5b36980..ee837fcd 100644 --- a/tests/unit/cli/main_test.py +++ b/tests/unit/cli/main_test.py @@ -1,11 +1,15 @@ from __future__ import absolute_import +import logging + from compose import container from compose.cli.errors import UserError +from compose.cli.formatter import ConsoleWarningFormatter from compose.cli.log_printer import LogPrinter from compose.cli.main import attach_to_logs from compose.cli.main import build_log_printer from compose.cli.main import convergence_strategy_from_opts +from compose.cli.main import setup_console_handler from compose.project import Project from compose.service import ConvergenceStrategy from tests import mock @@ -60,6 +64,31 @@ class CLIMainTestCase(unittest.TestCase): timeout=timeout) +class SetupConsoleHandlerTestCase(unittest.TestCase): + + def setUp(self): + self.stream = mock.Mock() + self.stream.isatty.return_value = True + self.handler = logging.StreamHandler(stream=self.stream) + + def test_with_tty_verbose(self): + setup_console_handler(self.handler, True) + assert type(self.handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' in self.handler.formatter._fmt + assert '%(funcName)s' in self.handler.formatter._fmt + + def test_with_tty_not_verbose(self): + setup_console_handler(self.handler, False) + assert type(self.handler.formatter) == ConsoleWarningFormatter + assert '%(name)s' not in self.handler.formatter._fmt + assert '%(funcName)s' not in self.handler.formatter._fmt + + def test_with_not_a_tty(self): + self.stream.isatty.return_value = False + setup_console_handler(self.handler, False) + assert type(self.handler.formatter) == logging.Formatter + + class ConvergeStrategyFromOptsTestCase(unittest.TestCase): def test_invalid_opts(self): diff --git a/tests/unit/cli_test.py b/tests/unit/cli_test.py index 0c78e6bb..5b63d2e8 100644 --- a/tests/unit/cli_test.py +++ b/tests/unit/cli_test.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import absolute_import from __future__ import unicode_literals @@ -98,7 +99,7 @@ class CLITestCase(unittest.TestCase): command.run(mock_project, { 'SERVICE': 'service', 'COMMAND': None, - '-e': ['BAR=NEW', 'OTHER=THREE'], + '-e': ['BAR=NEW', 'OTHER=bär'.encode('utf-8')], '--user': None, '--no-deps': None, '--allow-insecure-ssl': None, @@ -114,7 +115,7 @@ class CLITestCase(unittest.TestCase): _, _, call_kwargs = mock_client.create_container.mock_calls[0] self.assertEqual( call_kwargs['environment'], - {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': 'THREE'}) + {'FOO': 'ONE', 'BAR': 'NEW', 'OTHER': u'bär'}) def test_run_service_with_restart_always(self): command = TopLevelCommand() diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index 20ae7fa3..2835e9c8 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -1,3 +1,4 @@ +# encoding: utf-8 from __future__ import print_function import os @@ -5,6 +6,7 @@ import shutil import tempfile from operator import itemgetter +import py import pytest from compose.config import config @@ -133,6 +135,65 @@ class ConfigTest(unittest.TestCase): ] self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_load_with_multiple_files_and_empty_override(self): + base_file = config.ConfigFile( + 'base.yaml', + {'web': {'image': 'example/web'}}) + override_file = config.ConfigFile('override.yaml', None) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Top level object needs to be a dictionary' in exc.exconly() + + def test_load_with_multiple_files_and_empty_base(self): + base_file = config.ConfigFile('base.yaml', None) + override_file = config.ConfigFile( + 'override.yaml', + {'web': {'image': 'example/web'}}) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert 'Top level object needs to be a dictionary' in exc.exconly() + + def test_load_with_multiple_files_and_extends_in_override_file(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'web': {'image': 'example/web'}, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'web': { + 'extends': { + 'file': 'common.yml', + 'service': 'base', + }, + 'volumes': ['/home/user/project:/code'], + }, + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + tmpdir = py.test.ensuretemp('config_test') + tmpdir.join('common.yml').write(""" + base: + labels: ['label=one'] + """) + with tmpdir.as_cwd(): + service_dicts = config.load(details) + + expected = [ + { + 'name': 'web', + 'image': 'example/web', + 'volumes': ['/home/user/project:/code'], + 'labels': {'label': 'one'}, + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: config.load( @@ -319,7 +380,7 @@ class ConfigTest(unittest.TestCase): @mock.patch('compose.config.validation.log') def test_logs_warning_for_boolean_in_environment(self, mock_logging): - expected_warning_msg = "Warning: There is a boolean value in the 'environment' key." + expected_warning_msg = "There is a boolean value in the 'environment' key." config.load( build_config_details( {'web': { @@ -349,6 +410,18 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_yaml_with_yaml_error(self): + tmpdir = py.test.ensuretemp('invalid_yaml_test') + invalid_yaml_file = tmpdir.join('docker-compose.yml') + invalid_yaml_file.write(""" + web: + this is bogus: ok: what + """) + with pytest.raises(ConfigurationError) as exc: + config.load_yaml(str(invalid_yaml_file)) + + assert 'line 3, column 32' in exc.exconly() + class InterpolationTest(unittest.TestCase): @mock.patch.dict(os.environ) @@ -414,6 +487,23 @@ class InterpolationTest(unittest.TestCase): self.assertIn('in service "web"', cm.exception.msg) self.assertIn('"${"', cm.exception.msg) + def test_empty_environment_key_allowed(self): + service_dict = config.load( + build_config_details( + { + 'web': { + 'build': '.', + 'environment': { + 'POSTGRES_PASSWORD': '' + }, + }, + }, + '.', + None, + ) + )[0] + self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') + class VolumeConfigTest(unittest.TestCase): def test_no_binding(self): @@ -448,8 +538,8 @@ class VolumeConfigTest(unittest.TestCase): self.assertEqual(d['volumes'], ['/var/lib/data:/data']) def test_absolute_windows_path_does_not_expand(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['C:\\data:/data']}, working_dir='.') - self.assertEqual(d['volumes'], ['C:\\data:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['c:\\data:/data']}, working_dir='.') + self.assertEqual(d['volumes'], ['c:\\data:/data']) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') def test_relative_path_does_expand_posix(self): @@ -464,14 +554,14 @@ class VolumeConfigTest(unittest.TestCase): @pytest.mark.skipif(not IS_WINDOWS_PLATFORM, reason='windows paths') def test_relative_path_does_expand_windows(self): - d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject\\data:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['./data:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject\\data:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\myproject:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['.:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\myproject:/data']) - d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='C:\\Users\\me\\myproject') - self.assertEqual(d['volumes'], ['C:\\Users\\me\\otherproject:/data']) + d = make_service_dict('foo', {'build': '.', 'volumes': ['../otherproject:/data']}, working_dir='c:\\Users\\me\\myproject') + self.assertEqual(d['volumes'], ['c:\\Users\\me\\otherproject:/data']) @mock.patch.dict(os.environ) def test_home_directory_with_driver_does_not_expand(self): @@ -805,7 +895,12 @@ class EnvTest(unittest.TestCase): ) self.assertEqual( service_dict['environment'], - {'FILE_DEF': 'F1', 'FILE_DEF_EMPTY': '', 'ENV_DEF': 'E3', 'NO_DEF': ''}, + { + 'FILE_DEF': u'bär', + 'FILE_DEF_EMPTY': '', + 'ENV_DEF': 'E3', + 'NO_DEF': '' + }, ) @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index c5e1a9fb..d86f80f7 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -323,9 +323,7 @@ class ServiceTest(unittest.TestCase): new_container = service.recreate_container(mock_container) mock_container.stop.assert_called_once_with(timeout=10) - self.mock_client.rename.assert_called_once_with( - mock_container.id, - '%s_%s' % (mock_container.short_id, mock_container.name)) + mock_container.rename_to_tmp_name.assert_called_once_with() new_container.start.assert_called_once_with() mock_container.remove.assert_called_once_with() @@ -444,6 +442,14 @@ class ServiceTest(unittest.TestCase): } self.assertEqual(config_dict, expected) + def test_get_links_with_networking(self): + service = Service( + 'foo', + image='foo', + links=[(Service('one'), 'one')], + use_networking=True) + self.assertEqual(service._get_links(link_to_self=True), []) + class NetTestCase(unittest.TestCase):