From abe145bbe77d09a5b31ee1453a37938e132604e1 Mon Sep 17 00:00:00 2001 From: Joffrey F Date: Tue, 1 Dec 2015 17:26:32 -0800 Subject: [PATCH] Update config resolution to always use explicit version numbers Also includes several bugfixes for resolution and validation. Signed-off-by: Joffrey F --- compose/config/config.py | 46 +++++++++++++------ ...elds_schema.json => fields_schema_v1.json} | 2 +- compose/config/fields_schema_v2.json | 16 +++++-- compose/config/interpolation.py | 12 +++-- compose/config/service_schema.json | 2 +- compose/config/validation.py | 18 ++++---- compose/const.py | 1 + docker-compose.spec | 10 +++- 8 files changed, 73 insertions(+), 34 deletions(-) rename compose/config/{fields_schema.json => fields_schema_v1.json} (99%) diff --git a/compose/config/config.py b/compose/config/config.py index 1e82068f..295dc494 100644 --- a/compose/config/config.py +++ b/compose/config/config.py @@ -10,6 +10,7 @@ from collections import namedtuple import six import yaml +from ..const import COMPOSEFILE_VERSIONS from .errors import CircularReference from .errors import ComposeFileNotFound from .errors import ConfigurationError @@ -24,6 +25,7 @@ from .validation import validate_against_fields_schema from .validation import validate_against_service_schema from .validation import validate_extends_file_path from .validation import validate_top_level_object +from .validation import validate_top_level_service_objects DOCKER_CONFIG_KEYS = [ @@ -161,13 +163,24 @@ def find(base_dir, filenames): def get_config_version(config_details): def get_version(config): - validate_top_level_object(config) - return config.config.get('version') + if config.config is None: + return None + version = config.config.get('version', 1) + if isinstance(version, dict): + version = 1 + return version + main_file = config_details.config_files[0] + validate_top_level_object(main_file) version = get_version(main_file) for next_file in config_details.config_files[1:]: + validate_top_level_object(next_file) next_file_version = get_version(next_file) - if version != next_file_version: + if version is None: + version = next_file_version + continue + + if version != next_file_version and next_file_version is not None: raise ConfigurationError( "Version mismatch: main file {0} specifies version {1} but " "extension file {2} uses version {3}".format( @@ -224,6 +237,9 @@ def load(config_details): Return a fully interpolated, extended and validated configuration. """ version = get_config_version(config_details) + if version not in COMPOSEFILE_VERSIONS: + raise ConfigurationError('Invalid config version provided: {0}'.format(version)) + processed_files = [] for config_file in config_details.config_files: processed_files.append( @@ -231,9 +247,10 @@ def load(config_details): ) config_details = config_details._replace(config_files=processed_files) - if not version or isinstance(version, dict): + if version == 1: service_dicts = load_services( - config_details.working_dir, config_details.config_files + config_details.working_dir, config_details.config_files, + version ) volumes = {} elif version == 2: @@ -242,11 +259,9 @@ def load(config_details): for f in config_details.config_files ] service_dicts = load_services( - config_details.working_dir, config_files + config_details.working_dir, config_files, version ) volumes = load_volumes(config_details.config_files) - else: - raise ConfigurationError('Invalid config version provided: {0}'.format(version)) return Config(version, service_dicts, volumes) @@ -259,14 +274,14 @@ def load_volumes(config_files): return volumes -def load_services(working_dir, config_files): +def load_services(working_dir, config_files, version): def build_service(filename, service_name, service_dict): service_config = ServiceConfig.with_abs_paths( working_dir, filename, service_name, service_dict) - resolver = ServiceExtendsResolver(service_config) + resolver = ServiceExtendsResolver(service_config, version) service_dict = process_service(resolver.run()) # TODO: move to validate_service() @@ -301,8 +316,8 @@ def load_services(working_dir, config_files): def process_config_file(config_file, service_name=None, version=None): - validate_top_level_object(config_file) - processed_config = interpolate_environment_variables(config_file.config) + validate_top_level_service_objects(config_file, version) + processed_config = interpolate_environment_variables(config_file.config, version) validate_against_fields_schema( processed_config, config_file.filename, version ) @@ -316,10 +331,11 @@ def process_config_file(config_file, service_name=None, version=None): class ServiceExtendsResolver(object): - def __init__(self, service_config, already_seen=None): + def __init__(self, service_config, version, already_seen=None): self.service_config = service_config self.working_dir = service_config.working_dir self.already_seen = already_seen or [] + self.version = version @property def signature(self): @@ -348,7 +364,8 @@ class ServiceExtendsResolver(object): extended_file = process_config_file( ConfigFile.from_filename(config_path), - service_name=service_name) + service_name=service_name, version=self.version + ) service_config = extended_file.config[service_name] return config_path, service_config, service_name @@ -359,6 +376,7 @@ class ServiceExtendsResolver(object): extended_config_path, service_name, service_dict), + self.version, already_seen=self.already_seen + [self.signature]) service_config = resolver.run() diff --git a/compose/config/fields_schema.json b/compose/config/fields_schema_v1.json similarity index 99% rename from compose/config/fields_schema.json rename to compose/config/fields_schema_v1.json index fdf56fd9..6f0a3631 100644 --- a/compose/config/fields_schema.json +++ b/compose/config/fields_schema_v1.json @@ -2,7 +2,7 @@ "$schema": "http://json-schema.org/draft-04/schema#", "type": "object", - "id": "fields_schema.json", + "id": "fields_schema_v1.json", "patternProperties": { "^[a-zA-Z0-9._-]+$": { diff --git a/compose/config/fields_schema_v2.json b/compose/config/fields_schema_v2.json index 2ca41c47..49cab367 100644 --- a/compose/config/fields_schema_v2.json +++ b/compose/config/fields_schema_v2.json @@ -1,38 +1,44 @@ { "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", + "id": "fields_schema_v2.json", + "properties": { "version": { "enum": [2] }, "services": { + "id": "#/properties/services", "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { - "$ref": "fields_schema.json#/definitions/service" + "$ref": "fields_schema_v1.json#/definitions/service" } - } + }, + "additionalProperties": false }, "volumes": { + "id": "#/properties/volumes", "type": "object", "patternProperties": { "^[a-zA-Z0-9._-]+$": { "$ref": "#/definitions/volume" } - } + }, + "additionalProperties": false } }, "definitions": { "volume": { + "id": "#/definitions/volume", "type": "object", "properties": { "driver": {"type": "string"}, "driver_opts": { "type": "object", "patternProperties": { - "^.+$": {"type": ["boolean", "string", "number"]} + "^.+$": {"type": ["string", "number"]} }, "additionalProperties": false } diff --git a/compose/config/interpolation.py b/compose/config/interpolation.py index ba7e35c1..a8ff08d8 100644 --- a/compose/config/interpolation.py +++ b/compose/config/interpolation.py @@ -8,13 +8,19 @@ from .errors import ConfigurationError log = logging.getLogger(__name__) -def interpolate_environment_variables(config): +def interpolate_environment_variables(config, version): mapping = BlankDefaultDict(os.environ) + service_dicts = config if version == 1 else config.get('services', {}) - return dict( + interpolated = dict( (service_name, process_service(service_name, service_dict, mapping)) - for (service_name, service_dict) in config.items() + for (service_name, service_dict) in service_dicts.items() ) + if version == 1: + return interpolated + result = dict(config) + result.update({'services': interpolated}) + return result def process_service(service_name, service_dict, mapping): diff --git a/compose/config/service_schema.json b/compose/config/service_schema.json index 05774efd..91a1e005 100644 --- a/compose/config/service_schema.json +++ b/compose/config/service_schema.json @@ -5,7 +5,7 @@ "type": "object", "allOf": [ - {"$ref": "fields_schema.json#/definitions/service"}, + {"$ref": "fields_schema_v1.json#/definitions/service"}, {"$ref": "#/definitions/constraints"} ], diff --git a/compose/config/validation.py b/compose/config/validation.py index 861cb10f..617c95b6 100644 --- a/compose/config/validation.py +++ b/compose/config/validation.py @@ -74,14 +74,15 @@ def format_boolean_in_environment(instance): return True -def validate_top_level_service_objects(config_file): +def validate_top_level_service_objects(config_file, version): """Perform some high level validation of the service name and value. This validation must happen before interpolation, which must happen before the rest of validation, which is why it's separate from the rest of the service validation. """ - for service_name, service_dict in config_file.config.items(): + service_dicts = config_file.config if version == 1 else config_file.config.get('services', {}) + for service_name, service_dict in service_dicts.items(): if not isinstance(service_name, six.string_types): raise ConfigurationError( "In file '{}' service name: {} needs to be a string, eg '{}'".format( @@ -105,7 +106,6 @@ def validate_top_level_object(config_file): "that you have defined a service at the top level.".format( config_file.filename, type(config_file.config))) - validate_top_level_service_objects(config_file) def validate_extends_file_path(service_name, extends_options, filename): @@ -134,10 +134,14 @@ def anglicize_validator(validator): return 'a ' + validator +def is_service_dict_schema(schema_id): + return schema_id == 'fields_schema_v1.json' or schema_id == '#/properties/services' + + def handle_error_for_schema_with_id(error, service_name): schema_id = error.schema['id'] - if schema_id == 'fields_schema.json' and error.validator == 'additionalProperties': + if is_service_dict_schema(schema_id) and error.validator == 'additionalProperties': return "Invalid service name '{}' - only {} characters are allowed".format( # The service_name is the key to the json object list(error.instance)[0], @@ -281,10 +285,8 @@ def process_errors(errors, service_name=None): return '\n'.join(format_error_message(error, service_name) for error in errors) -def validate_against_fields_schema(config, filename, version=None): - schema_filename = "fields_schema.json" - if version: - schema_filename = "fields_schema_v{0}.json".format(version) +def validate_against_fields_schema(config, filename, version): + schema_filename = "fields_schema_v{0}.json".format(version) _validate_against_schema( config, schema_filename, diff --git a/compose/const.py b/compose/const.py index 1b689418..9c607ca2 100644 --- a/compose/const.py +++ b/compose/const.py @@ -10,3 +10,4 @@ LABEL_PROJECT = 'com.docker.compose.project' LABEL_SERVICE = 'com.docker.compose.service' LABEL_VERSION = 'com.docker.compose.version' LABEL_CONFIG_HASH = 'com.docker.compose.config-hash' +COMPOSEFILE_VERSIONS = (1, 2) diff --git a/docker-compose.spec b/docker-compose.spec index 24d03e05..c760d7b4 100644 --- a/docker-compose.spec +++ b/docker-compose.spec @@ -18,8 +18,13 @@ exe = EXE(pyz, a.datas, [ ( - 'compose/config/fields_schema.json', - 'compose/config/fields_schema.json', + 'compose/config/fields_schema_v1.json', + 'compose/config/fields_schema_v1.json', + 'DATA' + ), + ( + 'compose/config/fields_schema_v2.json', + 'compose/config/fields_schema_v2.json', 'DATA' ), ( @@ -33,6 +38,7 @@ exe = EXE(pyz, 'DATA' ) ], + name='docker-compose', debug=False, strip=None,