Refactor validation out
Move validation out into its own file without causing circular import errors. Fix some of the tests to import from the right place. Also fix tests that were not using valid test data, as the validation schema is now firing telling you that you couldn't "just" have this dict without a build/image config key. Signed-off-by: Mazz Mosley <mazz@houseofmnowster.com>
This commit is contained in:
Mazz Mosley
parent
0557b5dce6
commit
2e428f94ca
5 changed files with 165 additions and 167 deletions
|
|
@ -3,9 +3,6 @@ import os
|
|||
import sys
|
||||
import yaml
|
||||
from collections import namedtuple
|
||||
import json
|
||||
from jsonschema import Draft4Validator, FormatChecker, ValidationError
|
||||
|
||||
import six
|
||||
|
||||
from compose.cli.utils import find_candidates_in_parent_dirs
|
||||
|
|
@ -16,10 +13,9 @@ from .errors import (
|
|||
CircularReference,
|
||||
ComposeFileNotFound,
|
||||
)
|
||||
from .validation import validate_against_schema
|
||||
|
||||
|
||||
VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]'
|
||||
|
||||
DOCKER_CONFIG_KEYS = [
|
||||
'cap_add',
|
||||
'cap_drop',
|
||||
|
|
@ -69,22 +65,6 @@ ALLOWED_KEYS = DOCKER_CONFIG_KEYS + [
|
|||
'name',
|
||||
]
|
||||
|
||||
DOCKER_CONFIG_HINTS = {
|
||||
'cpu_share': 'cpu_shares',
|
||||
'add_host': 'extra_hosts',
|
||||
'hosts': 'extra_hosts',
|
||||
'extra_host': 'extra_hosts',
|
||||
'device': 'devices',
|
||||
'link': 'links',
|
||||
'memory_swap': 'memswap_limit',
|
||||
'port': 'ports',
|
||||
'privilege': 'privileged',
|
||||
'priviliged': 'privileged',
|
||||
'privilige': 'privileged',
|
||||
'volume': 'volumes',
|
||||
'workdir': 'working_dir',
|
||||
}
|
||||
|
||||
|
||||
SUPPORTED_FILENAMES = [
|
||||
'docker-compose.yml',
|
||||
|
|
@ -135,122 +115,18 @@ def get_config_path(base_dir):
|
|||
return os.path.join(path, winner)
|
||||
|
||||
|
||||
@FormatChecker.cls_checks(format="ports", raises=ValidationError("Ports is incorrectly formatted."))
|
||||
def format_ports(instance):
|
||||
def _is_valid(port):
|
||||
if ':' in port or '/' in port:
|
||||
return True
|
||||
try:
|
||||
int(port)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
return False
|
||||
|
||||
if isinstance(instance, list):
|
||||
for port in instance:
|
||||
if not _is_valid(port):
|
||||
return False
|
||||
return True
|
||||
elif isinstance(instance, str):
|
||||
return _is_valid(instance)
|
||||
return False
|
||||
|
||||
|
||||
def get_unsupported_config_msg(service_name, error_key):
|
||||
msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key)
|
||||
if error_key in DOCKER_CONFIG_HINTS:
|
||||
msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key])
|
||||
return msg
|
||||
|
||||
|
||||
def process_errors(errors):
|
||||
"""
|
||||
jsonschema gives us an error tree full of information to explain what has
|
||||
gone wrong. Process each error and pull out relevant information and re-write
|
||||
helpful error messages that are relevant.
|
||||
"""
|
||||
def _parse_key_from_error_msg(error):
|
||||
return error.message.split("'")[1]
|
||||
|
||||
root_msgs = []
|
||||
invalid_keys = []
|
||||
required = []
|
||||
type_errors = []
|
||||
|
||||
for error in errors:
|
||||
# handle root level errors
|
||||
if len(error.path) == 0:
|
||||
if error.validator == 'type':
|
||||
msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level."
|
||||
root_msgs.append(msg)
|
||||
elif error.validator == 'additionalProperties':
|
||||
invalid_service_name = _parse_key_from_error_msg(error)
|
||||
msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS)
|
||||
root_msgs.append(msg)
|
||||
else:
|
||||
root_msgs.append(error.message)
|
||||
|
||||
else:
|
||||
# handle service level errors
|
||||
service_name = error.path[0]
|
||||
|
||||
if error.validator == 'additionalProperties':
|
||||
invalid_config_key = _parse_key_from_error_msg(error)
|
||||
invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key))
|
||||
elif error.validator == 'anyOf':
|
||||
if 'image' in error.instance and 'build' in error.instance:
|
||||
required.append("Service '{}' has both an image and build path specified. A service can either be built to image or use an existing image, not both.".format(service_name))
|
||||
elif 'image' not in error.instance and 'build' not in error.instance:
|
||||
required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name))
|
||||
else:
|
||||
required.append(error.message)
|
||||
elif error.validator == 'type':
|
||||
msg = "a"
|
||||
if error.validator_value == "array":
|
||||
msg = "an"
|
||||
|
||||
try:
|
||||
config_key = error.path[1]
|
||||
type_errors.append("Service '{}' has an invalid value for '{}', it should be {} {}".format(service_name, config_key, msg, error.validator_value))
|
||||
except IndexError:
|
||||
config_key = error.path[0]
|
||||
root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key))
|
||||
elif error.validator == 'required':
|
||||
config_key = error.path[1]
|
||||
required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, error.message))
|
||||
elif error.validator == 'dependencies':
|
||||
dependency_key = error.validator_value.keys()[0]
|
||||
required_keys = ",".join(error.validator_value[dependency_key])
|
||||
required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format(
|
||||
dependency_key, service_name, dependency_key, required_keys))
|
||||
|
||||
return "\n".join(root_msgs + invalid_keys + required + type_errors)
|
||||
|
||||
|
||||
def validate_against_schema(config):
|
||||
config_source_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
schema_file = os.path.join(config_source_dir, "schema.json")
|
||||
|
||||
with open(schema_file, "r") as schema_fh:
|
||||
schema = json.load(schema_fh)
|
||||
|
||||
validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"]))
|
||||
|
||||
errors = [error for error in sorted(validation_output.iter_errors(config), key=str)]
|
||||
if errors:
|
||||
error_msg = process_errors(errors)
|
||||
raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg))
|
||||
|
||||
|
||||
def load(config_details):
|
||||
config, working_dir, filename = config_details
|
||||
if not isinstance(config, dict):
|
||||
raise ConfigurationError(
|
||||
"Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level."
|
||||
)
|
||||
|
||||
config = interpolate_environment_variables(config)
|
||||
validate_against_schema(config)
|
||||
|
||||
service_dicts = []
|
||||
|
||||
validate_against_schema(config)
|
||||
|
||||
for service_name, service_dict in list(config.items()):
|
||||
loader = ServiceLoader(working_dir=working_dir, filename=filename)
|
||||
service_dict = loader.make_service_dict(service_name, service_dict)
|
||||
|
|
@ -347,13 +223,6 @@ def validate_extended_service_dict(service_dict, filename, service):
|
|||
|
||||
|
||||
def process_container_options(service_dict, working_dir=None):
|
||||
for k in service_dict:
|
||||
if k not in ALLOWED_KEYS:
|
||||
msg = "Unsupported config option for %s service: '%s'" % (service_dict['name'], k)
|
||||
if k in DOCKER_CONFIG_HINTS:
|
||||
msg += " (did you mean '%s'?)" % DOCKER_CONFIG_HINTS[k]
|
||||
raise ConfigurationError(msg)
|
||||
|
||||
service_dict = service_dict.copy()
|
||||
|
||||
if 'volumes' in service_dict and service_dict.get('volume_driver') is None:
|
||||
|
|
|
|||
144
compose/config/schema.json
Normal file
144
compose/config/schema.json
Normal file
|
|
@ -0,0 +1,144 @@
|
|||
{
|
||||
"$schema": "http://json-schema.org/draft-04/schema#",
|
||||
|
||||
"type": "object",
|
||||
|
||||
"patternProperties": {
|
||||
"^[a-zA-Z0-9._-]+$": {
|
||||
"$ref": "#/definitions/service"
|
||||
}
|
||||
},
|
||||
|
||||
"definitions": {
|
||||
"service": {
|
||||
"type": "object",
|
||||
|
||||
"properties": {
|
||||
"build": {"type": "string"},
|
||||
"cap_add": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"cap_drop": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"command": {"$ref": "#/definitions/string_or_list"},
|
||||
"container_name": {"type": "string"},
|
||||
"cpu_shares": {"type": "string"},
|
||||
"cpuset": {"type": "string"},
|
||||
"detach": {"type": "boolean"},
|
||||
"devices": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"dns": {"$ref": "#/definitions/string_or_list"},
|
||||
"dns_search": {"$ref": "#/definitions/string_or_list"},
|
||||
"dockerfile": {"type": "string"},
|
||||
"domainname": {"type": "string"},
|
||||
"entrypoint": {"type": "string"},
|
||||
"env_file": {"$ref": "#/definitions/string_or_list"},
|
||||
|
||||
"environment": {
|
||||
"oneOf": [
|
||||
{"type": "object"},
|
||||
{"type": "array", "items": {"type": "string"}, "uniqueItems": true}
|
||||
]
|
||||
},
|
||||
|
||||
"expose": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
|
||||
"extends": {
|
||||
"type": "object",
|
||||
|
||||
"properties": {
|
||||
"service": {"type": "string"},
|
||||
"file": {"type": "string"}
|
||||
},
|
||||
"required": ["service"],
|
||||
"additionalProperties": false
|
||||
},
|
||||
|
||||
"extra_hosts": {"$ref": "#/definitions/list_or_dict"},
|
||||
"external_links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"hostname": {"type": "string"},
|
||||
"image": {"type": "string"},
|
||||
"labels": {"$ref": "#/definitions/list_or_dict"},
|
||||
"links": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"log_driver": {"type": "string"},
|
||||
|
||||
"log_opt": {
|
||||
"type": "object",
|
||||
|
||||
"properties": {
|
||||
"address": {"type": "string"}
|
||||
},
|
||||
"required": ["address"]
|
||||
},
|
||||
|
||||
"mac_address": {"type": "string"},
|
||||
"mem_limit": {"type": "number"},
|
||||
"memswap_limit": {"type": "number"},
|
||||
"name": {"type": "string"},
|
||||
"net": {"type": "string"},
|
||||
"pid": {"type": "string"},
|
||||
|
||||
"ports": {
|
||||
"oneOf": [
|
||||
{"type": "string", "format": "ports"},
|
||||
{
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"uniqueItems": true,
|
||||
"format": "ports"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
"privileged": {"type": "string"},
|
||||
"read_only": {"type": "boolean"},
|
||||
"restart": {"type": "string"},
|
||||
"security_opt": {"type": "string"},
|
||||
"stdin_open": {"type": "string"},
|
||||
"tty": {"type": "string"},
|
||||
"user": {"type": "string"},
|
||||
"volumes": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"volumes_from": {"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
"working_dir": {"type": "string"}
|
||||
},
|
||||
|
||||
"anyOf": [
|
||||
{
|
||||
"required": ["build"],
|
||||
"not": {"required": ["image"]}
|
||||
},
|
||||
{
|
||||
"required": ["image"],
|
||||
"not": {"required": ["build"]}
|
||||
},
|
||||
{
|
||||
"required": ["extends"],
|
||||
"not": {"required": ["build", "image"]}
|
||||
}
|
||||
],
|
||||
|
||||
"dependencies": {
|
||||
"memswap_limit": ["mem_limit"]
|
||||
},
|
||||
"additionalProperties": false
|
||||
},
|
||||
|
||||
"string_or_list": {
|
||||
"oneOf": [
|
||||
{"type": "string"},
|
||||
{"$ref": "#/definitions/list_of_strings"}
|
||||
]
|
||||
},
|
||||
|
||||
"list_of_strings": {
|
||||
"type": "array",
|
||||
"items": {"type": "string"},
|
||||
"uniqueItems": true
|
||||
},
|
||||
|
||||
"list_or_dict": {
|
||||
"oneOf": [
|
||||
{"type": "array", "items": {"type": "string"}, "uniqueItems": true},
|
||||
{"type": "object"}
|
||||
]
|
||||
}
|
||||
|
||||
},
|
||||
"additionalProperties": false
|
||||
}
|
||||
134
compose/config/validation.py
Normal file
134
compose/config/validation.py
Normal file
|
|
@ -0,0 +1,134 @@
|
|||
import os
|
||||
|
||||
import json
|
||||
from jsonschema import Draft4Validator, FormatChecker, ValidationError
|
||||
|
||||
from .errors import ConfigurationError
|
||||
|
||||
|
||||
DOCKER_CONFIG_HINTS = {
|
||||
'cpu_share': 'cpu_shares',
|
||||
'add_host': 'extra_hosts',
|
||||
'hosts': 'extra_hosts',
|
||||
'extra_host': 'extra_hosts',
|
||||
'device': 'devices',
|
||||
'link': 'links',
|
||||
'memory_swap': 'memswap_limit',
|
||||
'port': 'ports',
|
||||
'privilege': 'privileged',
|
||||
'priviliged': 'privileged',
|
||||
'privilige': 'privileged',
|
||||
'volume': 'volumes',
|
||||
'workdir': 'working_dir',
|
||||
}
|
||||
|
||||
|
||||
VALID_NAME_CHARS = '[a-zA-Z0-9\._\-]'
|
||||
|
||||
|
||||
@FormatChecker.cls_checks(format="ports", raises=ValidationError("Ports is incorrectly formatted."))
|
||||
def format_ports(instance):
|
||||
def _is_valid(port):
|
||||
if ':' in port or '/' in port:
|
||||
return True
|
||||
try:
|
||||
int(port)
|
||||
return True
|
||||
except ValueError:
|
||||
return False
|
||||
return False
|
||||
|
||||
if isinstance(instance, list):
|
||||
for port in instance:
|
||||
if not _is_valid(port):
|
||||
return False
|
||||
return True
|
||||
elif isinstance(instance, str):
|
||||
return _is_valid(instance)
|
||||
return False
|
||||
|
||||
|
||||
def get_unsupported_config_msg(service_name, error_key):
|
||||
msg = "Unsupported config option for '{}' service: '{}'".format(service_name, error_key)
|
||||
if error_key in DOCKER_CONFIG_HINTS:
|
||||
msg += " (did you mean '{}'?)".format(DOCKER_CONFIG_HINTS[error_key])
|
||||
return msg
|
||||
|
||||
|
||||
def process_errors(errors):
|
||||
"""
|
||||
jsonschema gives us an error tree full of information to explain what has
|
||||
gone wrong. Process each error and pull out relevant information and re-write
|
||||
helpful error messages that are relevant.
|
||||
"""
|
||||
def _parse_key_from_error_msg(error):
|
||||
return error.message.split("'")[1]
|
||||
|
||||
root_msgs = []
|
||||
invalid_keys = []
|
||||
required = []
|
||||
type_errors = []
|
||||
|
||||
for error in errors:
|
||||
# handle root level errors
|
||||
if len(error.path) == 0:
|
||||
if error.validator == 'type':
|
||||
msg = "Top level object needs to be a dictionary. Check your .yml file that you have defined a service at the top level."
|
||||
root_msgs.append(msg)
|
||||
elif error.validator == 'additionalProperties':
|
||||
invalid_service_name = _parse_key_from_error_msg(error)
|
||||
msg = "Invalid service name '{}' - only {} characters are allowed".format(invalid_service_name, VALID_NAME_CHARS)
|
||||
root_msgs.append(msg)
|
||||
else:
|
||||
root_msgs.append(error.message)
|
||||
|
||||
else:
|
||||
# handle service level errors
|
||||
service_name = error.path[0]
|
||||
|
||||
if error.validator == 'additionalProperties':
|
||||
invalid_config_key = _parse_key_from_error_msg(error)
|
||||
invalid_keys.append(get_unsupported_config_msg(service_name, invalid_config_key))
|
||||
elif error.validator == 'anyOf':
|
||||
if 'image' in error.instance and 'build' in error.instance:
|
||||
required.append("Service '{}' has both an image and build path specified. A service can either be built to image or use an existing image, not both.".format(service_name))
|
||||
elif 'image' not in error.instance and 'build' not in error.instance:
|
||||
required.append("Service '{}' has neither an image nor a build path specified. Exactly one must be provided.".format(service_name))
|
||||
else:
|
||||
required.append(error.message)
|
||||
elif error.validator == 'type':
|
||||
msg = "a"
|
||||
if error.validator_value == "array":
|
||||
msg = "an"
|
||||
|
||||
try:
|
||||
config_key = error.path[1]
|
||||
type_errors.append("Service '{}' has an invalid value for '{}', it should be {} {}".format(service_name, config_key, msg, error.validator_value))
|
||||
except IndexError:
|
||||
config_key = error.path[0]
|
||||
root_msgs.append("Service '{}' doesn\'t have any configuration options. All top level keys in your docker-compose.yml must map to a dictionary of configuration options.'".format(config_key))
|
||||
elif error.validator == 'required':
|
||||
config_key = error.path[1]
|
||||
required.append("Service '{}' option '{}' is invalid, {}".format(service_name, config_key, error.message))
|
||||
elif error.validator == 'dependencies':
|
||||
dependency_key = error.validator_value.keys()[0]
|
||||
required_keys = ",".join(error.validator_value[dependency_key])
|
||||
required.append("Invalid '{}' configuration for '{}' service: when defining '{}' you must set '{}' as well".format(
|
||||
dependency_key, service_name, dependency_key, required_keys))
|
||||
|
||||
return "\n".join(root_msgs + invalid_keys + required + type_errors)
|
||||
|
||||
|
||||
def validate_against_schema(config):
|
||||
config_source_dir = os.path.dirname(os.path.abspath(__file__))
|
||||
schema_file = os.path.join(config_source_dir, "schema.json")
|
||||
|
||||
with open(schema_file, "r") as schema_fh:
|
||||
schema = json.load(schema_fh)
|
||||
|
||||
validation_output = Draft4Validator(schema, format_checker=FormatChecker(["ports"]))
|
||||
|
||||
errors = [error for error in sorted(validation_output.iter_errors(config), key=str)]
|
||||
if errors:
|
||||
error_msg = process_errors(errors)
|
||||
raise ConfigurationError("Validation failed, reason(s):\n{}".format(error_msg))
|
||||
Loading…
Add table
Add a link
Reference in a new issue