diff --git a/compose/cli/command.py b/compose/cli/command.py index 21d6ff0d..b278af3a 100644 --- a/compose/cli/command.py +++ b/compose/cli/command.py @@ -80,12 +80,13 @@ def get_project(base_dir, config_path=None, project_name=None, verbose=False, config_details = config.find(base_dir, config_path) api_version = '1.21' if use_networking else None - return Project.from_dicts( + return Project.from_config( get_project_name(config_details.working_dir, project_name), config.load(config_details), get_client(verbose=verbose, version=api_version), use_networking=use_networking, - network_driver=network_driver) + network_driver=network_driver + ) def get_project_name(working_dir, project_name=None): diff --git a/compose/cli/docker_client.py b/compose/cli/docker_client.py index 734f4237..177d5d6c 100644 --- a/compose/cli/docker_client.py +++ b/compose/cli/docker_client.py @@ -8,8 +8,7 @@ from ..const import HTTP_TIMEOUT log = logging.getLogger(__name__) - -DEFAULT_API_VERSION = '1.19' +DEFAULT_API_VERSION = '1.21' def docker_client(version=None): diff --git a/compose/cli/main.py b/compose/cli/main.py index 4a766133..006d33ec 100644 --- a/compose/cli/main.py +++ b/compose/cli/main.py @@ -211,11 +211,11 @@ class TopLevelCommand(DocoptCommand): return if options['--services']: - print('\n'.join(service['name'] for service in compose_config)) + print('\n'.join(service['name'] for service in compose_config.services)) return compose_config = dict( - (service.pop('name'), service) for service in compose_config) + (service.pop('name'), service) for service in compose_config.services) print(yaml.dump( compose_config, default_flow_style=False, diff --git a/compose/config/config.py b/compose/config/config.py index 195665b5..c77e6100 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -10,6 +10,7 @@ from collections import namedtuple import six import yaml +from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError @@ -24,6 +25,7 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extends_file_path from .validation import validate_top_level_object +from .validation import validate_top_level_service_objects DOCKER_CONFIG_KEYS = [ @@ -116,6 +118,20 @@ class ConfigFile(namedtuple('_ConfigFile', 'filename config')): def from_filename(cls, filename): return cls(filename, load_yaml(filename)) + def get_service_dicts(self, version): + return self.config if version == 1 else self.config.get('services', {}) + + +class Config(namedtuple('_Config', 'version services volumes')): + """ + :param version: configuration version + :type version: int + :param services: List of service description dictionaries + :type services: :class:`list` + :param volumes: List of volume description dictionaries + :type volumes: :class:`list` + """ + class ServiceConfig(namedtuple('_ServiceConfig', 'working_dir filename name config')): @@ -148,6 +164,34 @@ def find(base_dir, filenames): [ConfigFile.from_filename(f) for f in filenames]) +def get_config_version(config_details): + def get_version(config): + if config.config is None: + return 1 + version = config.config.get('version', 1) + if isinstance(version, dict): + # in that case 'version' is probably a service name, so assume + # this is a legacy (version=1) file + version = 1 + return version + + main_file = config_details.config_files[0] + validate_top_level_object(main_file) + version = get_version(main_file) + for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file) + next_file_version = get_version(next_file) + + if version != next_file_version and next_file_version is not None: + raise ConfigurationError( + "Version mismatch: main file {0} specifies version {1} but " + "extension file {2} uses version {3}".format( + main_file.filename, version, next_file.filename, next_file_version + ) + ) + return version + + def get_default_config_files(base_dir): (candidates, path) = find_candidates_in_parent_dirs(SUPPORTED_FILENAMES, base_dir) @@ -194,14 +238,52 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ + version = get_config_version(config_details) + if version not in COMPOSEFILE_VERSIONS: + raise ConfigurationError('Invalid config version provided: {0}'.format(version)) + processed_files = [] + for config_file in config_details.config_files: + processed_files.append( + process_config_file(config_file, version=version) + ) + config_details = config_details._replace(config_files=processed_files) + + if version == 1: + service_dicts = load_services( + config_details.working_dir, config_details.config_files, + version + ) + volumes = {} + elif version == 2: + config_files = [ + ConfigFile(f.filename, f.config.get('services', {})) + for f in config_details.config_files + ] + service_dicts = load_services( + config_details.working_dir, config_files, version + ) + volumes = load_volumes(config_details.config_files) + + return Config(version, service_dicts, volumes) + + +def load_volumes(config_files): + volumes = {} + for config_file in config_files: + for name, volume_config in config_file.config.get('volumes', {}).items(): + volumes.update({name: volume_config}) + return volumes + + +def load_services(working_dir, config_files, version): def build_service(filename, service_name, service_dict): service_config = ServiceConfig.with_abs_paths( - config_details.working_dir, + working_dir, filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config) + resolver = ServiceExtendsResolver(service_config, version) service_dict = process_service(resolver.run()) # TODO: move to validate_service() @@ -227,20 +309,28 @@ def load(config_details): for name in all_service_names } - config_file = process_config_file(config_details.config_files[0]) - for next_file in config_details.config_files[1:]: - next_file = process_config_file(next_file) - + config_file = config_files[0] + for next_file in config_files[1:]: config = merge_services(config_file.config, next_file.config) config_file = config_file._replace(config=config) return build_services(config_file) -def process_config_file(config_file, service_name=None): - validate_top_level_object(config_file) - processed_config = interpolate_environment_variables(config_file.config) - validate_against_fields_schema(processed_config, config_file.filename) +def process_config_file(config_file, version, service_name=None): + service_dicts = config_file.get_service_dicts(version) + validate_top_level_service_objects( + config_file.filename, service_dicts + ) + interpolated_config = interpolate_environment_variables(service_dicts) + if version == 2: + processed_config = dict(config_file.config) + processed_config.update({'services': interpolated_config}) + if version == 1: + processed_config = interpolated_config + validate_against_fields_schema( + processed_config, config_file.filename, version + ) if service_name and service_name not in processed_config: raise ConfigurationError( @@ -251,10 +341,11 @@ def process_config_file(config_file, service_name=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, already_seen=None): + def __init__(self, service_config, version, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] + self.version = version @property def signature(self): @@ -283,7 +374,8 @@ class ServiceExtendsResolver(object): extended_file = process_config_file( ConfigFile.from_filename(config_path), - service_name=service_name) + version=self.version, service_name=service_name + ) service_config = extended_file.config[service_name] return config_path, service_config, service_name @@ -294,6 +386,7 @@ class ServiceExtendsResolver(object): extended_config_path, service_name, service_dict), + self.version, already_seen=self.already_seen + [self.signature]) service_config = resolver.run() diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema_v1.json similarity index 99% rename from compose/config/fields_schema.json rename to compose/config/fields_schema_v1.json index fdf56fd9..6f0a3631 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema_v1.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "id": "fields_schema.json", + "id": "fields_schema_v1.json", "patternProperties": { "^[a-zA-Z0-9._-]+$": { diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json new file mode 100644 index 00000000..49cab367 --- /dev/null +++ b/compose/config/fields_schema_v2.json @@ -0,0 +1,49 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "id": "fields_schema_v2.json", + + "properties": { + "version": { + "enum": [2] + }, + "services": { + "id": "#/properties/services", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "fields_schema_v1.json#/definitions/service" + } + }, + "additionalProperties": false + }, + "volumes": { + "id": "#/properties/volumes", + "type": "object", + "patternProperties": { + "^[a-zA-Z0-9._-]+$": { + "$ref": "#/definitions/volume" + } + }, + "additionalProperties": false + } + }, + + "definitions": { + "volume": { + "id": "#/definitions/volume", + "type": "object", + "properties": { + "driver": {"type": "string"}, + "driver_opts": { + "type": "object", + "patternProperties": { + "^.+$": {"type": ["string", "number"]} + }, + "additionalProperties": false + } + } + } + }, + "additionalProperties": false +} diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index ba7e35c1..12eb497b 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -8,12 +8,12 @@ from .errors import ConfigurationError log = logging.getLogger(__name__) -def interpolate_environment_variables(config): +def interpolate_environment_variables(service_dicts): mapping = BlankDefaultDict(os.environ) return dict( (service_name, process_service(service_name, service_dict, mapping)) - for (service_name, service_dict) in config.items() + for (service_name, service_dict) in service_dicts.items() ) diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json index 05774efd..91a1e005 100644 --- a/compose/config/service_schema.json +++ b/compose/config/service_schema.json @@ -5,7 +5,7 @@ "type": "object", "allOf": [ - {"$ref": "fields_schema.json#/definitions/service"}, + {"$ref": "fields_schema_v1.json#/definitions/service"}, {"$ref": "#/definitions/constraints"} ], diff --git a/compose/config/validation.py b/compose/config/validation.py index d16bdb9d..091014f6 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -74,18 +74,18 @@ def format_boolean_in_environment(instance): return True -def validate_top_level_service_objects(config_file): +def validate_top_level_service_objects(filename, service_dicts): """Perform some high level validation of the service name and value. This validation must happen before interpolation, which must happen before the rest of validation, which is why it's separate from the rest of the service validation. """ - for service_name, service_dict in config_file.config.items(): + for service_name, service_dict in service_dicts.items(): if not isinstance(service_name, six.string_types): raise ConfigurationError( "In file '{}' service name: {} needs to be a string, eg '{}'".format( - config_file.filename, + filename, service_name, service_name)) @@ -94,8 +94,9 @@ def validate_top_level_service_objects(config_file): "In file '{}' service '{}' doesn\'t have any configuration options. " "All top level keys in your docker-compose.yml must map " "to a dictionary of configuration options.".format( - config_file.filename, - service_name)) + filename, service_name + ) + ) def validate_top_level_object(config_file): @@ -105,7 +106,6 @@ def validate_top_level_object(config_file): "that you have defined a service at the top level.".format( config_file.filename, type(config_file.config))) - validate_top_level_service_objects(config_file) def validate_extends_file_path(service_name, extends_options, filename): @@ -134,10 +134,14 @@ def anglicize_validator(validator): return 'a ' + validator +def is_service_dict_schema(schema_id): + return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services' + + def handle_error_for_schema_with_id(error, service_name): schema_id = error.schema['id'] - if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties': + if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': return "Invalid service name '{}' - only {} characters are allowed".format( # The service_name is the key to the json object list(error.instance)[0], @@ -281,10 +285,11 @@ def process_errors(errors, service_name=None): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config, filename): +def validate_against_fields_schema(config, filename, version): + schema_filename = "fields_schema_v{0}.json".format(version) _validate_against_schema( config, - "fields_schema.json", + schema_filename, format_checker=["ports", "expose", "bool-value-in-mapping"], filename=filename) diff --git a/compose/const.py b/compose/const.py index 1b689418..9c607ca2 100644 --- a/compose/const.py +++ b/compose/const.py @@ -10,3 +10,4 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +COMPOSEFILE_VERSIONS = (1, 2) diff --git a/compose/container.py b/compose/container.py index 8f96a944..5730f224 100644 --- a/compose/container.py +++ b/compose/container.py @@ -177,6 +177,12 @@ class Container(object): port = self.ports.get("%s/%s" % (port, protocol)) return "{HostIp}:{HostPort}".format(**port[0]) if port else None + def get_mount(self, mount_dest): + for mount in self.get('Mounts'): + if mount['Destination'] == mount_dest: + return mount + return None + def start(self, **options): return self.client.start(self.id, **options) @@ -222,16 +228,6 @@ class Container(object): self.has_been_inspected = True return self.dictionary - # TODO: only used by tests, move to test module - def links(self): - links = [] - for container in self.client.containers(): - for name in container['Names']: - bits = name.split('/') - if len(bits) > 2 and bits[1] == self.name: - links.append(bits[2]) - return links - def attach(self, *args, **kwargs): return self.client.attach(self.id, *args, **kwargs) diff --git a/compose/project.py b/compose/project.py index 76dccfe2..3801bbb9 100644 --- a/compose/project.py +++ b/compose/project.py @@ -20,6 +20,7 @@ from .service import ConvergenceStrategy from .service import Net from .service import Service from .service import ServiceNet +from .volume import Volume log = logging.getLogger(__name__) @@ -29,12 +30,13 @@ class Project(object): """ A collection of services. """ - def __init__(self, name, services, client, use_networking=False, network_driver=None): + def __init__(self, name, services, client, volumes=None, use_networking=False, network_driver=None): self.name = name self.services = services self.client = client self.use_networking = use_networking self.network_driver = network_driver + self.volumes = volumes or [] def labels(self, one_off=False): return [ @@ -43,16 +45,16 @@ class Project(object): ] @classmethod - def from_dicts(cls, name, service_dicts, client, use_networking=False, network_driver=None): + def from_config(cls, name, config_data, client, use_networking=False, network_driver=None): """ - Construct a ServiceCollection from a list of dicts representing services. + Construct a Project from a config.Config object. """ project = cls(name, [], client, use_networking=use_networking, network_driver=network_driver) if use_networking: - remove_links(service_dicts) + remove_links(config_data.services) - for service_dict in service_dicts: + for service_dict in config_data.services: links = project.get_links(service_dict) volumes_from = project.get_volumes_from(service_dict) net = project.get_net(service_dict) @@ -66,6 +68,14 @@ class Project(object): net=net, volumes_from=volumes_from, **service_dict)) + if config_data.volumes: + for vol_name, data in config_data.volumes.items(): + project.volumes.append( + Volume( + client=client, project=name, name=vol_name, + driver=data.get('driver'), driver_opts=data.get('driver_opts') + ) + ) return project @property @@ -218,6 +228,27 @@ class Project(object): def remove_stopped(self, service_names=None, **options): parallel.parallel_remove(self.containers(service_names, stopped=True), options) + def initialize_volumes(self): + try: + for volume in self.volumes: + volume.create() + except NotFound: + raise ConfigurationError( + 'Volume %s specifies nonexistent driver %s' % (volume.name, volume.driver) + ) + except APIError as e: + if 'Choose a different volume name' in str(e): + raise ConfigurationError( + 'Configuration for volume {0} specifies driver {1}, but ' + 'a volume with the same name uses a different driver ' + '({3}). If you wish to use the new configuration, please ' + 'remove the existing volume "{2}" first:\n' + '$ docker volume rm {2}'.format( + volume.name, volume.driver, volume.full_name, + volume.inspect()['Driver'] + ) + ) + def restart(self, service_names=None, **options): containers = self.containers(service_names, stopped=True) parallel.parallel_restart(containers, options) @@ -253,6 +284,8 @@ class Project(object): if self.use_networking and self.uses_default_network(): self.ensure_network_exists() + self.initialize_volumes() + return [ container for service in services diff --git a/compose/service.py b/compose/service.py index d1509871..24fa6394 100644 --- a/compose/service.py +++ b/compose/service.py @@ -849,7 +849,13 @@ def get_container_data_volumes(container, volumes_option): a mapping of volume bindings for those volumes. """ volumes = [] - container_volumes = container.get('Volumes') or {} + volumes_option = volumes_option or [] + + container_mounts = dict( + (mount['Destination'], mount) + for mount in container.get('Mounts') or {} + ) + image_volumes = [ VolumeSpec.parse(volume) for volume in @@ -861,13 +867,14 @@ def get_container_data_volumes(container, volumes_option): if volume.external: continue - volume_path = container_volumes.get(volume.internal) + mount = container_mounts.get(volume.internal) + # New volume, doesn't exist in the old container - if not volume_path: + if not mount: continue # Copy existing volume from old container - volume = volume._replace(external=volume_path) + volume = volume._replace(external=mount['Source']) volumes.append(volume) return volumes diff --git a/compose/volume.py b/compose/volume.py new file mode 100644 index 00000000..055bd6ab --- /dev/null +++ b/compose/volume.py @@ -0,0 +1,25 @@ +from __future__ import unicode_literals + + +class Volume(object): + def __init__(self, client, project, name, driver=None, driver_opts=None): + self.client = client + self.project = project + self.name = name + self.driver = driver + self.driver_opts = driver_opts + + def create(self): + return self.client.create_volume( + self.full_name, self.driver, self.driver_opts + ) + + def remove(self): + return self.client.remove_volume(self.full_name) + + def inspect(self): + return self.client.inspect_volume(self.full_name) + + @property + def full_name(self): + return '{0}_{1}'.format(self.project, self.name) diff --git a/docker-compose.spec b/docker-compose.spec index 24d03e05..c760d7b4 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -18,8 +18,13 @@ exe = EXE(pyz, a.datas, [ ( - 'compose/config/fields_schema.json', - 'compose/config/fields_schema.json', + 'compose/config/fields_schema_v1.json', + 'compose/config/fields_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/fields_schema_v2.json', + 'compose/config/fields_schema_v2.json', 'DATA' ), ( @@ -33,6 +38,7 @@ exe = EXE(pyz, 'DATA' ) ], + name='docker-compose', debug=False, strip=None, diff --git a/docs/compose-file.md b/docs/compose-file.md index 2a6028b8..29e0c647 100644 --- a/docs/compose-file.md +++ b/docs/compose-file.md @@ -24,6 +24,64 @@ As with `docker run`, options specified in the Dockerfile (e.g., `CMD`, `EXPOSE`, `VOLUME`, `ENV`) are respected by default - you don't need to specify them again in `docker-compose.yml`. +## Versioning + +It is possible to use different versions of the `compose.yml` format. +Below are the formats currently supported by compose. + + +### Version 1 + +Compose files that do not declare a version are considered "version 1". In +those files, all the [services](#service-configuration-reference) are declared +at the root of the document. + +Version 1 files do not support the declaration of +named [volumes](#volume-configuration-reference) + +Example: + + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + + +### Version 2 + +Compose files using the version 2 syntax must indicate the version number at +the root of the document. All [services](#service-configuration-reference) +must be declared under the `services` key. +Named [volumes](#volume-configuration-reference) must be declared under the +`volumes` key. + +Example: + + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: + driver: default + + ## Service configuration reference This section contains a list of all configuration options supported by a service @@ -413,6 +471,34 @@ Each of these is a single value, analogous to its stdin_open: true tty: true + +## Volume configuration reference + +While it is possible to declare volumes on the fly as part of the service +declaration, this section allows you to create named volumes that can be +reused across multiple services (without relying on `volumes_from`), and are +easily retrieved and inspected using the docker command line or API. +See the [docker volume](http://docs.docker.com/reference/commandline/volume/) +subcommand documentation for more information. + +### driver + +Specify which volume driver should be used for this volume. Defaults to +`local`. An exception will be raised if the driver is not available. + + driver: foobar + +### driver_opts + +Specify a list of options as key-value pairs to pass to the driver for this +volume. Those options are driver dependent - consult the driver's +documentation for more information. Optional. + + driver_opts: + foo: "bar" + baz: 1 + + ## Variable substitution Your configuration options can contain environment variables. Compose uses the diff --git a/docs/index.md b/docs/index.md index 36b93a39..6e8f2090 100644 --- a/docs/index.md +++ b/docs/index.md @@ -31,16 +31,22 @@ they can be run together in an isolated environment. A `docker-compose.yml` looks like this: - web: - build: . - ports: - - "5000:5000" - volumes: - - .:/code - links: - - redis - redis: - image: redis + version: 2 + services: + web: + build: . + ports: + - "5000:5000" + volumes: + - .:/code + - logvolume01:/var/log + links: + - redis + redis: + image: redis + volumes: + logvolume01: + driver: default For more information about the Compose file, see the [Compose file reference](compose-file.md) diff --git a/requirements.txt b/requirements.txt index 659cb57f..8c6d5f3a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ PyYAML==3.11 -docker-py==1.5.0 +docker-py==1.6.0 dockerpty==0.3.4 docopt==0.6.1 enum34==1.0.4 diff --git a/script/test-versions b/script/test-versions index 623b107b..76e55e11 100755 --- a/script/test-versions +++ b/script/test-versions @@ -18,7 +18,8 @@ get_versions="docker run --rm if [ "$DOCKER_VERSIONS" == "" ]; then DOCKER_VERSIONS="$($get_versions default)" elif [ "$DOCKER_VERSIONS" == "all" ]; then - DOCKER_VERSIONS="$($get_versions recent -n 2)" + # TODO: `-n 2` when engine 1.10 releases + DOCKER_VERSIONS="$($get_versions recent -n 1)" fi diff --git a/tests/acceptance/cli_test.py b/tests/acceptance/cli_test.py index e6fa38a8..1885727a 100644 --- a/tests/acceptance/cli_test.py +++ b/tests/acceptance/cli_test.py @@ -16,6 +16,7 @@ from compose.cli.command import get_project from compose.cli.docker_client import docker_client from compose.container import Container from tests.integration.testcases import DockerClientTestCase +from tests.integration.testcases import get_links from tests.integration.testcases import pull_busybox @@ -871,7 +872,7 @@ class CLITestCase(DockerClientTestCase): self.dispatch(['up', '-d'], None) container = self.project.containers(stopped=True)[0] - actual_host_path = container.get('Volumes')['/container-path'] + actual_host_path = container.get_mount('/container-path')['Source'] components = actual_host_path.split('/') assert components[-2:] == ['home-dir', 'my-volume'] @@ -909,7 +910,7 @@ class CLITestCase(DockerClientTestCase): web, other, db = containers self.assertEqual(web.human_readable_command, 'top') - self.assertTrue({'db', 'other'} <= set(web.links())) + self.assertTrue({'db', 'other'} <= set(get_links(web))) self.assertEqual(db.human_readable_command, 'top') self.assertEqual(other.human_readable_command, 'top') @@ -931,7 +932,9 @@ class CLITestCase(DockerClientTestCase): self.assertEqual(len(containers), 2) web = containers[1] - self.assertEqual(set(web.links()), set(['db', 'mydb_1', 'extends_mydb_1'])) + self.assertEqual( + set(get_links(web)), + set(['db', 'mydb_1', 'extends_mydb_1'])) expected_env = set([ "FOO=1", diff --git a/tests/integration/project_test.py b/tests/integration/project_test.py index 229f653a..d51830bb 100644 --- a/tests/integration/project_test.py +++ b/tests/integration/project_test.py @@ -1,5 +1,7 @@ from __future__ import unicode_literals +import random + from .testcases import DockerClientTestCase from compose.cli.docker_client import docker_client from compose.config import config @@ -69,9 +71,9 @@ class ProjectTest(DockerClientTestCase): 'volumes_from': ['data'], }, }) - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=service_dicts, + config_data=service_dicts, client=self.client, ) db = project.get_service('db') @@ -86,9 +88,9 @@ class ProjectTest(DockerClientTestCase): name='composetest_data_container', labels={LABEL_PROJECT: 'composetest'}, ) - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'db': { 'image': 'busybox:latest', 'volumes_from': ['composetest_data_container'], @@ -117,9 +119,9 @@ class ProjectTest(DockerClientTestCase): assert project.get_network()['Name'] == network_name def test_net_from_service(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'net': { 'image': 'busybox:latest', 'command': ["top"] @@ -149,9 +151,9 @@ class ProjectTest(DockerClientTestCase): ) net_container.start() - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'web': { 'image': 'busybox:latest', 'net': 'container:composetest_net_container' @@ -331,15 +333,17 @@ class ProjectTest(DockerClientTestCase): project.up(['db']) self.assertEqual(len(project.containers()), 1) old_db_id = project.containers()[0].id - db_volume_path = project.containers()[0].inspect()['Volumes']['/var/db'] + container, = project.containers() + db_volume_path = container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) self.assertEqual(len(project.containers()), 2) db_container = [c for c in project.containers() if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - self.assertEqual(db_container.inspect()['Volumes']['/var/db'], - db_volume_path) + self.assertEqual( + db_container.get_mount('/var/db')['Source'], + db_volume_path) def test_project_up_with_no_recreate_stopped(self): web = self.create_service('web') @@ -354,8 +358,9 @@ class ProjectTest(DockerClientTestCase): old_containers = project.containers(stopped=True) self.assertEqual(len(old_containers), 1) - old_db_id = old_containers[0].id - db_volume_path = old_containers[0].inspect()['Volumes']['/var/db'] + old_container, = old_containers + old_db_id = old_container.id + db_volume_path = old_container.get_mount('/var/db')['Source'] project.up(strategy=ConvergenceStrategy.never) @@ -365,8 +370,9 @@ class ProjectTest(DockerClientTestCase): db_container = [c for c in new_containers if 'db' in c.name][0] self.assertEqual(db_container.id, old_db_id) - self.assertEqual(db_container.inspect()['Volumes']['/var/db'], - db_volume_path) + self.assertEqual( + db_container.get_mount('/var/db')['Source'], + db_volume_path) def test_project_up_without_all_services(self): console = self.create_service('console') @@ -396,9 +402,9 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(console.containers()), 0) def test_project_up_starts_depends(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -431,9 +437,9 @@ class ProjectTest(DockerClientTestCase): self.assertEqual(len(project.get_service('console').containers()), 0) def test_project_up_with_no_deps(self): - project = Project.from_dicts( + project = Project.from_config( name='composetest', - service_dicts=build_service_dicts({ + config_data=build_service_dicts({ 'console': { 'image': 'busybox:latest', 'command': ["top"], @@ -504,3 +510,119 @@ class ProjectTest(DockerClientTestCase): project.up() service = project.get_service('web') self.assertEqual(len(service.containers()), 1) + + def test_project_up_volumes(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'local'}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.up() + self.assertEqual(len(project.containers()), 1) + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_initialize_volumes(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_project_up_implicit_volume_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.up() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + def test_project_up_invalid_volume_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'foobar'}} + ) + + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError): + project.initialize_volumes() + + def test_project_up_updated_driver(self): + vol_name = '{0:x}'.format(random.getrandbits(32)) + full_vol_name = 'composetest_{0}'.format(vol_name) + + config_data = config.Config( + version=2, services=[{ + 'name': 'web', + 'image': 'busybox:latest', + 'command': 'top' + }], volumes={vol_name: {'driver': 'local'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + project.initialize_volumes() + + volume_data = self.client.inspect_volume(full_vol_name) + self.assertEqual(volume_data['Name'], full_vol_name) + self.assertEqual(volume_data['Driver'], 'local') + + config_data = config_data._replace( + volumes={vol_name: {'driver': 'smb'}} + ) + project = Project.from_config( + name='composetest', + config_data=config_data, client=self.client + ) + with self.assertRaises(config.ConfigurationError) as e: + project.initialize_volumes() + assert 'Configuration for volume {0} specifies driver smb'.format( + vol_name + ) in str(e.exception) diff --git a/tests/integration/resilience_test.py b/tests/integration/resilience_test.py index 7f75356d..5df751c7 100644 --- a/tests/integration/resilience_test.py +++ b/tests/integration/resilience_test.py @@ -18,12 +18,12 @@ class ResilienceTest(DockerClientTestCase): container = self.db.create_container() container.start() - self.host_path = container.get('Volumes')['/var/db'] + self.host_path = container.get_mount('/var/db')['Source'] def test_successful_recreate(self): self.project.up(strategy=ConvergenceStrategy.always) container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_create_failure(self): with mock.patch('compose.service.Service.create_container', crash): @@ -32,7 +32,7 @@ class ResilienceTest(DockerClientTestCase): self.project.up() container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) def test_start_failure(self): with mock.patch('compose.container.Container.start', crash): @@ -41,7 +41,7 @@ class ResilienceTest(DockerClientTestCase): self.project.up() container = self.db.containers()[0] - self.assertEqual(container.get('Volumes')['/var/db'], self.host_path) + self.assertEqual(container.get_mount('/var/db')['Source'], self.host_path) class Crash(Exception): diff --git a/tests/integration/service_test.py b/tests/integration/service_test.py index ff571656..4a0eaacb 100644 --- a/tests/integration/service_test.py +++ b/tests/integration/service_test.py @@ -12,6 +12,7 @@ from six import text_type from .. import mock from .testcases import DockerClientTestCase +from .testcases import get_links from .testcases import pull_busybox from compose import __version__ from compose.config.types import VolumeFromSpec @@ -88,13 +89,13 @@ class ServiceTest(DockerClientTestCase): service = self.create_service('db', volumes=[VolumeSpec.parse('/var/db')]) container = service.create_container() container.start() - self.assertIn('/var/db', container.get('Volumes')) + assert container.get_mount('/var/db') def test_create_container_with_volume_driver(self): service = self.create_service('db', volume_driver='foodriver') container = service.create_container() container.start() - self.assertEqual('foodriver', container.get('Config.VolumeDriver')) + self.assertEqual('foodriver', container.get('HostConfig.VolumeDriver')) def test_create_container_with_cpu_shares(self): service = self.create_service('db', cpu_shares=73) @@ -158,12 +159,11 @@ class ServiceTest(DockerClientTestCase): volumes=[VolumeSpec(host_path, container_path, 'rw')]) container = service.create_container() container.start() - - volumes = container.inspect()['Volumes'] - self.assertIn(container_path, volumes) + assert container.get_mount(container_path) # Match the last component ("host-path"), because boot2docker symlinks /tmp - actual_host_path = volumes[container_path] + actual_host_path = container.get_mount(container_path)['Source'] + self.assertTrue(path.basename(actual_host_path) == path.basename(host_path), msg=("Last component differs: %s, %s" % (actual_host_path, host_path))) @@ -173,10 +173,10 @@ class ServiceTest(DockerClientTestCase): """ service = self.create_service('data', volumes=[VolumeSpec.parse('/data/')]) old_container = create_and_start_container(service) - volume_path = old_container.get('Volumes')['/data'] + volume_path = old_container.get_mount('/data')['Source'] new_container = service.recreate_container(old_container) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_duplicate_volume_trailing_slash(self): """ @@ -250,7 +250,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(old_container.name, 'composetest_db_1') old_container.start() old_container.inspect() # reload volume data - volume_path = old_container.get('Volumes')['/etc'] + volume_path = old_container.get_mount('/etc')['Source'] num_containers_before = len(self.client.containers(all=True)) @@ -262,7 +262,7 @@ class ServiceTest(DockerClientTestCase): self.assertEqual(new_container.get('Config.Cmd'), ['-d', '1']) self.assertIn('FOO=2', new_container.get('Config.Env')) self.assertEqual(new_container.name, 'composetest_db_1') - self.assertEqual(new_container.get('Volumes')['/etc'], volume_path) + self.assertEqual(new_container.get_mount('/etc')['Source'], volume_path) self.assertIn( 'affinity:container==%s' % old_container.id, new_container.get('Config.Env')) @@ -305,14 +305,19 @@ class ServiceTest(DockerClientTestCase): ) old_container = create_and_start_container(service) - self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) - volume_path = old_container.get('Volumes')['/data'] + self.assertEqual( + [mount['Destination'] for mount in old_container.get('Mounts')], ['/data'] + ) + volume_path = old_container.get_mount('/data')['Source'] new_container, = service.execute_convergence_plan( ConvergencePlan('recreate', [old_container])) - self.assertEqual(list(new_container.get('Volumes')), ['/data']) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual( + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] + ) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_execute_convergence_plan_when_image_volume_masks_config(self): service = self.create_service( @@ -321,8 +326,11 @@ class ServiceTest(DockerClientTestCase): ) old_container = create_and_start_container(service) - self.assertEqual(list(old_container.get('Volumes').keys()), ['/data']) - volume_path = old_container.get('Volumes')['/data'] + self.assertEqual( + [mount['Destination'] for mount in old_container.get('Mounts')], + ['/data'] + ) + volume_path = old_container.get_mount('/data')['Source'] service.options['volumes'] = [VolumeSpec.parse('/tmp:/data')] @@ -336,8 +344,11 @@ class ServiceTest(DockerClientTestCase): "Service \"db\" is using volume \"/data\" from the previous container", args[0]) - self.assertEqual(list(new_container.get('Volumes')), ['/data']) - self.assertEqual(new_container.get('Volumes')['/data'], volume_path) + self.assertEqual( + [mount['Destination'] for mount in new_container.get('Mounts')], + ['/data'] + ) + self.assertEqual(new_container.get_mount('/data')['Source'], volume_path) def test_execute_convergence_plan_without_start(self): service = self.create_service( @@ -376,7 +387,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', @@ -392,7 +403,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', @@ -410,7 +421,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(web) self.assertEqual( - set(web.containers()[0].links()), + set(get_links(web.containers()[0])), set([ 'composetest_db_1', 'composetest_db_2', @@ -424,7 +435,7 @@ class ServiceTest(DockerClientTestCase): create_and_start_container(db) c = create_and_start_container(db) - self.assertEqual(set(c.links()), set([])) + self.assertEqual(set(get_links(c)), set([])) def test_start_one_off_container_creates_links_to_its_own_service(self): db = self.create_service('db') @@ -435,7 +446,7 @@ class ServiceTest(DockerClientTestCase): c = create_and_start_container(db, one_off=True) self.assertEqual( - set(c.links()), + set(get_links(c)), set([ 'composetest_db_1', 'db_1', 'composetest_db_2', 'db_2', diff --git a/tests/integration/state_test.py b/tests/integration/state_test.py index 1fecce87..d07dfa82 100644 --- a/tests/integration/state_test.py +++ b/tests/integration/state_test.py @@ -7,6 +7,7 @@ from __future__ import unicode_literals import py from .testcases import DockerClientTestCase +from .testcases import get_links from compose.config import config from compose.project import Project from compose.service import ConvergenceStrategy @@ -25,10 +26,10 @@ class ProjectTestCase(DockerClientTestCase): details = config.ConfigDetails( 'working_dir', [config.ConfigFile(None, cfg)]) - return Project.from_dicts( + return Project.from_config( name='composetest', client=self.client, - service_dicts=config.load(details)) + config_data=config.load(details)) class BasicProjectTest(ProjectTestCase): @@ -186,8 +187,8 @@ class ProjectWithDependenciesTest(ProjectTestCase): web, = [c for c in containers if c.service == 'web'] nginx, = [c for c in containers if c.service == 'nginx'] - self.assertEqual(web.links(), ['composetest_db_1', 'db', 'db_1']) - self.assertEqual(nginx.links(), ['composetest_web_1', 'web', 'web_1']) + self.assertEqual(set(get_links(web)), {'composetest_db_1', 'db', 'db_1'}) + self.assertEqual(set(get_links(nginx)), {'composetest_web_1', 'web', 'web_1'}) class ServiceStateTest(DockerClientTestCase): diff --git a/tests/integration/testcases.py b/tests/integration/testcases.py index 9ea68e39..8e0525ee 100644 --- a/tests/integration/testcases.py +++ b/tests/integration/testcases.py @@ -16,6 +16,16 @@ def pull_busybox(client): client.pull('busybox:latest', stream=False) +def get_links(container): + links = container.get('HostConfig.Links') or [] + + def format_link(link): + _, alias = link.split(':') + return alias.split('/')[-1] + + return [format_link(link) for link in links] + + class DockerClientTestCase(unittest.TestCase): @classmethod def setUpClass(cls): @@ -25,11 +35,14 @@ class DockerClientTestCase(unittest.TestCase): for c in self.client.containers( all=True, filters={'label': '%s=composetest' % LABEL_PROJECT}): - self.client.kill(c['Id']) - self.client.remove_container(c['Id']) + self.client.remove_container(c['Id'], force=True) for i in self.client.images( filters={'label': 'com.docker.compose.test_image'}): self.client.remove_image(i) + volumes = self.client.volumes().get('Volumes') or [] + for v in volumes: + if 'composetest_' in v['Name']: + self.client.remove_volume(v['Name']) def create_service(self, name, **kwargs): if 'image' not in kwargs and 'build' not in kwargs: diff --git a/tests/integration/volume_test.py b/tests/integration/volume_test.py new file mode 100644 index 00000000..b6086040 --- /dev/null +++ b/tests/integration/volume_test.py @@ -0,0 +1,55 @@ +from __future__ import unicode_literals + +from docker.errors import DockerException + +from .testcases import DockerClientTestCase +from compose.volume import Volume + + +class VolumeTest(DockerClientTestCase): + def setUp(self): + self.tmp_volumes = [] + + def tearDown(self): + for volume in self.tmp_volumes: + try: + self.client.remove_volume(volume.full_name) + except DockerException: + pass + + def create_volume(self, name, driver=None, opts=None): + vol = Volume( + self.client, 'composetest', name, driver=driver, driver_opts=opts + ) + self.tmp_volumes.append(vol) + return vol + + def test_create_volume(self): + vol = self.create_volume('volume01') + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + def test_recreate_existing_volume(self): + vol = self.create_volume('volume01') + + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + vol.create() + info = self.client.inspect_volume(vol.full_name) + assert info['Name'] == vol.full_name + + def test_inspect_volume(self): + vol = self.create_volume('volume01') + vol.create() + info = vol.inspect() + assert info['Name'] == vol.full_name + + def test_remove_volume(self): + vol = Volume(self.client, 'composetest', 'volume01') + vol.create() + vol.remove() + volumes = self.client.volumes()['Volumes'] + assert len([v for v in volumes if v['Name'] == vol.full_name]) == 0 diff --git a/tests/unit/config/config_test.py b/tests/unit/config/config_test.py index e975cb9d..281e81d1 100644 --- a/tests/unit/config/config_test.py +++ b/tests/unit/config/config_test.py @@ -26,7 +26,7 @@ def make_service_dict(name, service_dict, working_dir, filename=None): working_dir=working_dir, filename=filename, name=name, - config=service_dict)) + config=service_dict), version=1) return config.process_service(resolver.run()) @@ -51,7 +51,7 @@ class ConfigTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual( service_sort(service_dicts), @@ -68,6 +68,85 @@ class ConfigTest(unittest.TestCase): ]) ) + def test_load_v2(self): + config_data = config.load( + build_config_details({ + 'version': 2, + 'services': { + 'foo': {'image': 'busybox'}, + 'bar': {'image': 'busybox', 'environment': ['FOO=1']}, + }, + 'volumes': { + 'hello': { + 'driver': 'default', + 'driver_opts': {'beep': 'boop'} + } + } + }, 'working_dir', 'filename.yml') + ) + service_dicts = config_data.services + volume_dict = config_data.volumes + self.assertEqual( + service_sort(service_dicts), + service_sort([ + { + 'name': 'bar', + 'image': 'busybox', + 'environment': {'FOO': '1'}, + }, + { + 'name': 'foo', + 'image': 'busybox', + } + ]) + ) + self.assertEqual(volume_dict, { + 'hello': { + 'driver': 'default', + 'driver_opts': {'beep': 'boop'} + } + }) + + def test_load_service_with_name_version(self): + config_data = config.load( + build_config_details({ + 'version': { + 'image': 'busybox' + } + }, 'working_dir', 'filename.yml') + ) + service_dicts = config_data.services + self.assertEqual( + service_sort(service_dicts), + service_sort([ + { + 'name': 'version', + 'image': 'busybox', + } + ]) + ) + + def test_load_invalid_version(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 18, + 'services': { + 'foo': {'image': 'busybox'} + } + }, 'working_dir', 'filename.yml') + ) + + with self.assertRaises(ConfigurationError): + config.load( + build_config_details({ + 'version': 'two point oh', + 'services': { + 'foo': {'image': 'busybox'} + } + }, 'working_dir', 'filename.yml') + ) + def test_load_throws_error_when_not_dict(self): with self.assertRaises(ConfigurationError): config.load( @@ -78,6 +157,16 @@ class ConfigTest(unittest.TestCase): ) ) + def test_load_throws_error_when_not_dict_v2(self): + with self.assertRaises(ConfigurationError): + config.load( + build_config_details( + {'version': 2, 'services': {'web': 'busybox:latest'}}, + 'working_dir', + 'filename.yml' + ) + ) + def test_load_config_invalid_service_names(self): for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: with pytest.raises(ConfigurationError) as exc: @@ -87,6 +176,17 @@ class ConfigTest(unittest.TestCase): 'filename.yml')) assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() + def test_config_invalid_service_names_v2(self): + for invalid_name in ['?not?allowed', ' ', '', '!', '/', '\xe2']: + with pytest.raises(ConfigurationError) as exc: + config.load( + build_config_details({ + 'version': 2, + 'services': {invalid_name: {'image': 'busybox'}} + }, 'working_dir', 'filename.yml') + ) + assert 'Invalid service name \'%s\'' % invalid_name in exc.exconly() + def test_load_with_invalid_field_name(self): config_details = build_config_details( {'web': {'image': 'busybox', 'name': 'bogus'}}, @@ -120,6 +220,22 @@ class ConfigTest(unittest.TestCase): ) ) + def test_config_integer_service_name_raise_validation_error_v2(self): + expected_error_msg = ("In file 'filename.yml' service name: 1 needs to " + "be a string, eg '1'") + + with self.assertRaisesRegexp(ConfigurationError, expected_error_msg): + config.load( + build_config_details( + { + 'version': 2, + 'services': {1: {'image': 'busybox'}} + }, + 'working_dir', + 'filename.yml' + ) + ) + @pytest.mark.xfail(IS_WINDOWS_PLATFORM, reason='paths use slash') def test_load_with_multiple_files(self): base_file = config.ConfigFile( @@ -143,7 +259,7 @@ class ConfigTest(unittest.TestCase): }) details = config.ConfigDetails('.', [base_file, override_file]) - service_dicts = config.load(details) + service_dicts = config.load(details).services expected = [ { 'name': 'web', @@ -170,6 +286,18 @@ class ConfigTest(unittest.TestCase): error_msg = "Top level object in 'override.yml' needs to be an object" assert error_msg in exc.exconly() + def test_load_with_multiple_files_and_empty_override_v2(self): + base_file = config.ConfigFile( + 'base.yml', + {'version': 2, 'services': {'web': {'image': 'example/web'}}}) + override_file = config.ConfigFile('override.yml', None) + details = config.ConfigDetails('.', [base_file, override_file]) + + with pytest.raises(ConfigurationError) as exc: + config.load(details) + error_msg = "Top level object in 'override.yml' needs to be an object" + assert error_msg in exc.exconly() + def test_load_with_multiple_files_and_empty_base(self): base_file = config.ConfigFile('base.yml', None) override_file = config.ConfigFile( @@ -181,6 +309,17 @@ class ConfigTest(unittest.TestCase): config.load(details) assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() + def test_load_with_multiple_files_and_empty_base_v2(self): + base_file = config.ConfigFile('base.yml', None) + override_file = config.ConfigFile( + 'override.tml', + {'version': 2, 'services': {'web': {'image': 'example/web'}}} + ) + details = config.ConfigDetails('.', [base_file, override_file]) + with pytest.raises(ConfigurationError) as exc: + config.load(details) + assert "Top level object in 'base.yml' needs to be an object" in exc.exconly() + def test_load_with_multiple_files_and_extends_in_override_file(self): base_file = config.ConfigFile( 'base.yaml', @@ -207,7 +346,7 @@ class ConfigTest(unittest.TestCase): labels: ['label=one'] """) with tmpdir.as_cwd(): - service_dicts = config.load(details) + service_dicts = config.load(details).services expected = [ { @@ -248,19 +387,62 @@ class ConfigTest(unittest.TestCase): 'volumes': ['/tmp'], } }) - services = config.load(config_details) + services = config.load(config_details).services assert services[0]['name'] == 'volume' assert services[1]['name'] == 'db' assert services[2]['name'] == 'web' + def test_load_with_multiple_files_v2(self): + base_file = config.ConfigFile( + 'base.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'image': 'example/web', + 'links': ['db'], + }, + 'db': { + 'image': 'example/db', + } + }, + }) + override_file = config.ConfigFile( + 'override.yaml', + { + 'version': 2, + 'services': { + 'web': { + 'build': '/', + 'volumes': ['/home/user/project:/code'], + }, + } + }) + details = config.ConfigDetails('.', [base_file, override_file]) + + service_dicts = config.load(details).services + expected = [ + { + 'name': 'web', + 'build': os.path.abspath('/'), + 'links': ['db'], + 'volumes': [VolumeSpec.parse('/home/user/project:/code')], + }, + { + 'name': 'db', + 'image': 'example/db', + }, + ] + self.assertEqual(service_sort(service_dicts), service_sort(expected)) + def test_config_valid_service_names(self): for valid_name in ['_', '-', '.__.', '_what-up.', 'what_.up----', 'whatup']: services = config.load( build_config_details( {valid_name: {'image': 'busybox'}}, 'tests/fixtures/extends', - 'common.yml')) + 'common.yml')).services assert services[0]['name'] == valid_name def test_config_hint(self): @@ -451,7 +633,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(service[0]['expose'], expose) def test_valid_config_oneof_string_or_list(self): @@ -466,7 +648,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(service[0]['entrypoint'], entrypoint) @mock.patch('compose.config.validation.log') @@ -496,7 +678,7 @@ class ConfigTest(unittest.TestCase): 'working_dir', 'filename.yml' ) - ) + ).services self.assertEqual(services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'], 'none') def test_load_yaml_with_yaml_error(self): @@ -543,7 +725,7 @@ class ConfigTest(unittest.TestCase): 'dns_search': 'domain.local', } })) - assert actual == [ + assert actual.services == [ { 'name': 'web', 'image': 'alpine', @@ -655,7 +837,7 @@ class InterpolationTest(unittest.TestCase): service_dicts = config.load( config.find('tests/fixtures/environment-interpolation', None), - ) + ).services self.assertEqual(service_dicts, [ { @@ -722,7 +904,7 @@ class InterpolationTest(unittest.TestCase): '.', None, ) - )[0] + ).services[0] self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '') @@ -734,10 +916,14 @@ class VolumeConfigTest(unittest.TestCase): @mock.patch.dict(os.environ) def test_volume_binding_with_environment_variable(self): os.environ['VOLUME_PATH'] = '/host/path' - d = config.load(build_config_details( - {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, - '.', - ))[0] + + d = config.load( + build_config_details( + {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}}, + '.', + None, + ) + ).services[0] self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')]) @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths') @@ -1012,7 +1198,7 @@ class MemoryOptionsTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual(service_dict[0]['memswap_limit'], 2000000) def test_memswap_can_be_a_string(self): @@ -1022,7 +1208,7 @@ class MemoryOptionsTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEqual(service_dict[0]['memswap_limit'], "512M") @@ -1126,7 +1312,7 @@ class EnvTest(unittest.TestCase): {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}}, "tests/fixtures/env", ) - )[0] + ).services[0] self.assertEqual( set(service_dict['volumes']), set([VolumeSpec.parse('/tmp:/host/tmp')])) @@ -1136,14 +1322,14 @@ class EnvTest(unittest.TestCase): {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}}, "tests/fixtures/env", ) - )[0] + ).services[0] self.assertEqual( set(service_dict['volumes']), set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')])) def load_from_filename(filename): - return config.load(config.find('.', [filename])) + return config.load(config.find('.', [filename])).services class ExtendsTest(unittest.TestCase): @@ -1313,7 +1499,7 @@ class ExtendsTest(unittest.TestCase): 'tests/fixtures/extends', 'common.yml' ) - ) + ).services self.assertEquals(len(service), 1) self.assertIsInstance(service[0], dict) @@ -1594,7 +1780,7 @@ class BuildPathTest(unittest.TestCase): for valid_url in valid_urls: service_dict = config.load(build_config_details({ 'validurl': {'build': valid_url}, - }, '.', None)) + }, '.', None)).services assert service_dict[0]['build'] == valid_url def test_invalid_url_in_build_path(self): diff --git a/tests/unit/project_test.py b/tests/unit/project_test.py index f4c6f8ca..4bf5f463 100644 --- a/tests/unit/project_test.py +++ b/tests/unit/project_test.py @@ -4,6 +4,7 @@ import docker from .. import mock from .. import unittest +from compose.config.config import Config from compose.config.types import VolumeFromSpec from compose.const import LABEL_SERVICE from compose.container import Container @@ -18,7 +19,7 @@ class ProjectTest(unittest.TestCase): self.mock_client = mock.create_autospec(docker.Client) def test_from_dict(self): - project = Project.from_dicts('composetest', [ + project = Project.from_config('composetest', Config(None, [ { 'name': 'web', 'image': 'busybox:latest' @@ -27,7 +28,7 @@ class ProjectTest(unittest.TestCase): 'name': 'db', 'image': 'busybox:latest' }, - ], None) + ], None), None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -35,7 +36,7 @@ class ProjectTest(unittest.TestCase): self.assertEqual(project.get_service('db').options['image'], 'busybox:latest') def test_from_config(self): - dicts = [ + dicts = Config(None, [ { 'name': 'web', 'image': 'busybox:latest', @@ -44,8 +45,8 @@ class ProjectTest(unittest.TestCase): 'name': 'db', 'image': 'busybox:latest', }, - ] - project = Project.from_dicts('composetest', dicts, None) + ], None) + project = Project.from_config('composetest', dicts, None) self.assertEqual(len(project.services), 2) self.assertEqual(project.get_service('web').name, 'web') self.assertEqual(project.get_service('web').options['image'], 'busybox:latest') @@ -141,13 +142,13 @@ class ProjectTest(unittest.TestCase): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('aaa', 'rw')] } - ], self.mock_client) + ], None), self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"]) def test_use_volumes_from_service_no_container(self): @@ -160,7 +161,7 @@ class ProjectTest(unittest.TestCase): "Image": 'busybox:latest' } ] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'vol', 'image': 'busybox:latest' @@ -170,13 +171,13 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('vol', 'rw')] } - ], self.mock_client) + ], None), self.mock_client) self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"]) def test_use_volumes_from_service_container(self): container_ids = ['aabbccddee', '12345'] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'vol', 'image': 'busybox:latest' @@ -186,7 +187,7 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', 'volumes_from': [VolumeFromSpec('vol', 'rw')] } - ], None) + ], None), None) with mock.patch.object(Service, 'containers') as mock_return: mock_return.return_value = [ mock.Mock(id=container_id, spec=Container) @@ -196,12 +197,12 @@ class ProjectTest(unittest.TestCase): [container_ids[0] + ':rw']) def test_net_unset(self): - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.id, None) self.assertNotIn('NetworkMode', service._get_container_host_config({})) @@ -210,13 +211,13 @@ class ProjectTest(unittest.TestCase): container_id = 'aabbccddee' container_dict = dict(Name='aaa', Id=container_id) self.mock_client.inspect_container.return_value = container_dict - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'test', 'image': 'busybox:latest', 'net': 'container:aaa' } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_id) @@ -230,7 +231,7 @@ class ProjectTest(unittest.TestCase): "Image": 'busybox:latest' } ] - project = Project.from_dicts('test', [ + project = Project.from_config('test', Config(None, [ { 'name': 'aaa', 'image': 'busybox:latest' @@ -240,7 +241,7 @@ class ProjectTest(unittest.TestCase): 'image': 'busybox:latest', 'net': 'container:aaa' } - ], self.mock_client) + ], None), self.mock_client) service = project.get_service('test') self.assertEqual(service.net.mode, 'container:' + container_name) @@ -285,12 +286,12 @@ class ProjectTest(unittest.TestCase): }, }, } - project = Project.from_dicts( + project = Project.from_config( 'test', - [{ + Config(None, [{ 'name': 'web', 'image': 'busybox:latest', - }], + }], None), self.mock_client, ) self.assertEqual([c.id for c in project.containers()], ['1']) diff --git a/tests/unit/service_test.py b/tests/unit/service_test.py index 1c8b441f..87d6af59 100644 --- a/tests/unit/service_test.py +++ b/tests/unit/service_test.py @@ -234,6 +234,7 @@ class ServiceTest(unittest.TestCase): prev_container = mock.Mock( id='ababab', image_config={'ContainerConfig': {}}) + prev_container.get.return_value = None opts = service._get_container_create_options( {}, @@ -575,6 +576,10 @@ class NetTestCase(unittest.TestCase): self.assertEqual(net.service_name, service_name) +def build_mount(destination, source, mode='rw'): + return {'Source': source, 'Destination': destination, 'Mode': mode} + + class ServiceVolumesTest(unittest.TestCase): def setUp(self): @@ -600,12 +605,33 @@ class ServiceVolumesTest(unittest.TestCase): } container = Container(self.mock_client, { 'Image': 'ababab', - 'Volumes': { - '/host/volume': '/host/volume', - '/existing/volume': '/var/lib/docker/aaaaaaaa', - '/removed/volume': '/var/lib/docker/bbbbbbbb', - '/mnt/image/data': '/var/lib/docker/cccccccc', - }, + 'Mounts': [ + { + 'Source': '/host/volume', + 'Destination': '/host/volume', + 'Mode': '', + 'RW': True, + 'Name': 'hostvolume', + }, { + 'Source': '/var/lib/docker/aaaaaaaa', + 'Destination': '/existing/volume', + 'Mode': '', + 'RW': True, + 'Name': 'existingvolume', + }, { + 'Source': '/var/lib/docker/bbbbbbbb', + 'Destination': '/removed/volume', + 'Mode': '', + 'RW': True, + 'Name': 'removedvolume', + }, { + 'Source': '/var/lib/docker/cccccccc', + 'Destination': '/mnt/image/data', + 'Mode': '', + 'RW': True, + 'Name': 'imagedata', + }, + ] }, has_been_inspected=True) expected = [ @@ -630,7 +656,13 @@ class ServiceVolumesTest(unittest.TestCase): intermediate_container = Container(self.mock_client, { 'Image': 'ababab', - 'Volumes': {'/existing/volume': '/var/lib/docker/aaaaaaaa'}, + 'Mounts': [{ + 'Source': '/var/lib/docker/aaaaaaaa', + 'Destination': '/existing/volume', + 'Mode': '', + 'RW': True, + 'Name': 'existingvolume', + }], }, has_been_inspected=True) expected = [ @@ -693,9 +725,16 @@ class ServiceVolumesTest(unittest.TestCase): self.mock_client.inspect_container.return_value = { 'Id': '123123123', 'Image': 'ababab', - 'Volumes': { - '/data': '/mnt/sda1/host/path', - }, + 'Mounts': [ + { + 'Destination': '/data', + 'Source': '/mnt/sda1/host/path', + 'Mode': '', + 'RW': True, + 'Driver': 'local', + 'Name': 'abcdefff1234' + }, + ] } service._get_container_create_options(