jyapayne/compose
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implement smart recreate behind an experimental CLI flag

Browse source 
Signed-off-by: Aanand Prasad <aanand.prasad@gmail.com>
This commit is contained in:
Aanand Prasad 2015-05-12 11:11:36 +01:00
commit ef4eb66723
11 changed files with 563 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

15
compose/cli/main.py
View file

@ -13,7 +13,7 @@ import dockerpty
from .. import __version__
from .. import migration
from ..project import NoSuchService, ConfigurationError
from ..service import BuildError, CannotBeScaledError
from ..service import BuildError, CannotBeScaledError, NeedsBuildError
from ..config import parse_environment
from .command import Command
from .docopt_command import NoSuchCommand
@ -47,6 +47,9 @@ def main():
except BuildError as e:
log.error("Service '%s' failed to build: %s" % (e.service.name, e.reason))
sys.exit(1)
except NeedsBuildError as e:
log.error("Service '%s' needs to be built, but --no-build was passed." % e.service.name)
sys.exit(1)
def setup_logging():
@ -297,7 +300,7 @@ class TopLevelCommand(Command):
project.up(
service_names=deps,
start_deps=True,
recreate=False,
allow_recreate=False,
insecure_registry=insecure_registry,
)
@ -440,6 +443,8 @@ class TopLevelCommand(Command):
print new container names.
--no-color Produce monochrome output.
--no-deps Don't start linked services.
--x-smart-recreate Only recreate containers whose configuration or
image needs to be updated. (EXPERIMENTAL)
--no-recreate If containers already exist, don't recreate them.
--no-build Don't build an image, even if it's missing
-t, --timeout TIMEOUT When attached, use this timeout in seconds
@ -452,13 +457,15 @@ class TopLevelCommand(Command):
monochrome = options['--no-color']
start_deps = not options['--no-deps']
recreate = not options['--no-recreate']
allow_recreate = not options['--no-recreate']
smart_recreate = options['--x-smart-recreate']
service_names = options['SERVICE']
project.up(
service_names=service_names,
start_deps=start_deps,
recreate=recreate,
allow_recreate=allow_recreate,
smart_recreate=smart_recreate,
insecure_registry=insecure_registry,
do_build=not options['--no-build'],
)

1
compose/const.py
View file

@ -4,3 +4,4 @@ LABEL_ONE_OFF = 'com.docker.compose.oneoff'
LABEL_PROJECT = 'com.docker.compose.project'
LABEL_SERVICE = 'com.docker.compose.service'
LABEL_VERSION = 'com.docker.compose.version'
LABEL_CONFIG_HASH = 'com.docker.compose.config-hash'

5
compose/container.py
View file

@ -179,13 +179,16 @@ class Container(object):
return self.client.attach_socket(self.id, **kwargs)
def __repr__(self):
return '<Container: %s>' % self.name
return '<Container: %s (%s)>' % (self.name, self.id[:6])
def __eq__(self, other):
if type(self) != type(other):
return False
return self.id == other.id
def __hash__(self):
return self.id.__hash__()
def get_container_name(container):
if not container.get('Name') and not container.get('Names'):

64
compose/project.py
View file

@ -207,22 +207,59 @@ class Project(object):
def up(self,
service_names=None,
start_deps=True,
recreate=True,
allow_recreate=True,
smart_recreate=False,
insecure_registry=False,
do_build=True):
running_containers = []
for service in self.get_services(service_names, include_deps=start_deps):
if recreate:
create_func = service.recreate_containers
services = self.get_services(service_names, include_deps=start_deps)
plans = self._get_convergence_plans(
services,
allow_recreate=allow_recreate,
smart_recreate=smart_recreate,
)
return [
container
for service in services
for container in service.execute_convergence_plan(
plans[service.name],
insecure_registry=insecure_registry,
do_build=do_build,
)
]
def _get_convergence_plans(self,
services,
allow_recreate=True,
smart_recreate=False):
plans = {}
for service in services:
updated_dependencies = [
name
for name in service.get_dependency_names()
if name in plans
and plans[name].action == 'recreate'
]
if updated_dependencies:
log.debug(
'%s has not changed but its dependencies (%s) have, so recreating',
service.name, ", ".join(updated_dependencies),
)
plan = service.recreate_plan()
else:
create_func = service.start_or_create_containers
plan = service.convergence_plan(
allow_recreate=allow_recreate,
smart_recreate=smart_recreate,
)
for container in create_func(
insecure_registry=insecure_registry,
do_build=do_build):
running_containers.append(container)
plans[service.name] = plan
return running_containers
return plans
def pull(self, service_names=None, insecure_registry=False):
for service in self.get_services(service_names, include_deps=True):
@ -250,10 +287,7 @@ class Project(object):
return containers
def _inject_deps(self, acc, service):
net_name = service.get_net_name()
dep_names = (service.get_linked_names() +
service.get_volumes_from_names() +
([net_name] if net_name else []))
dep_names = service.get_dependency_names()
if len(dep_names) > 0:
dep_services = self.get_services(

176
compose/service.py
View file

@ -18,9 +18,11 @@ from .const import (
LABEL_PROJECT,
LABEL_SERVICE,
LABEL_VERSION,
LABEL_CONFIG_HASH,
)
from .container import Container, get_container_name
from .progress_stream import stream_output, StreamOutputError
from .utils import json_hash
log = logging.getLogger(__name__)
@ -59,12 +61,20 @@ class ConfigError(ValueError):
pass
class NeedsBuildError(Exception):
def __init__(self, service):
self.service = service
VolumeSpec = namedtuple('VolumeSpec', 'external internal mode')
ServiceName = namedtuple('ServiceName', 'project service number')
ConvergencePlan = namedtuple('ConvergencePlan', 'action containers')
class Service(object):
def __init__(self, name, client=None, project='default', links=None, external_links=None, volumes_from=None, net=None, **options):
if not re.match('^%s+$' % VALID_NAME_CHARS, name):
@ -192,6 +202,11 @@ class Service(object):
Create a container for this service. If the image doesn't exist, attempt to pull
it.
"""
self.ensure_image_exists(
do_build=do_build,
insecure_registry=insecure_registry,
)
container_options = self._get_container_create_options(
override_options,
number or self._next_container_number(one_off=one_off),
@ -199,38 +214,142 @@ class Service(object):
previous_container=previous_container,
)
if (do_build and
self.can_be_built() and
not self.client.images(name=self.full_name)):
self.build()
return Container.create(self.client, **container_options)
def ensure_image_exists(self,
do_build=True,
insecure_registry=False):
if self.image():
return
if self.can_be_built():
if do_build:
self.build()
else:
raise NeedsBuildError(self)
else:
self.pull(insecure_registry=insecure_registry)
def image(self):
try:
return Container.create(self.client, **container_options)
return self.client.inspect_image(self.image_name)
except APIError as e:
if e.response.status_code == 404 and e.explanation and 'No such image' in str(e.explanation):
self.pull(insecure_registry=insecure_registry)
return Container.create(self.client, **container_options)
raise
return None
else:
raise
def recreate_containers(self, insecure_registry=False, do_build=True):
@property
def image_name(self):
if self.can_be_built():
return self.full_name
else:
return self.options['image']
def converge(self,
allow_recreate=True,
smart_recreate=False,
insecure_registry=False,
do_build=True):
"""
If a container for this service doesn't exist, create and start one. If there are
any, stop them, create+start new ones, and remove the old containers.
"""
plan = self.convergence_plan(
allow_recreate=allow_recreate,
smart_recreate=smart_recreate,
)
return self.execute_convergence_plan(
plan,
insecure_registry=insecure_registry,
do_build=do_build,
)
def convergence_plan(self,
allow_recreate=True,
smart_recreate=False):
containers = self.containers(stopped=True)
if not containers:
return ConvergencePlan('create', [])
if smart_recreate and not self._containers_have_diverged(containers):
stopped = [c for c in containers if not c.is_running]
if stopped:
return ConvergencePlan('start', stopped)
return ConvergencePlan('noop', containers)
if not allow_recreate:
return ConvergencePlan('start', containers)
return ConvergencePlan('recreate', containers)
def recreate_plan(self):
containers = self.containers(stopped=True)
return ConvergencePlan('recreate', containers)
def _containers_have_diverged(self, containers):
config_hash = self.config_hash()
has_diverged = False
for c in containers:
container_config_hash = c.labels.get(LABEL_CONFIG_HASH, None)
if container_config_hash != config_hash:
log.debug(
'%s has diverged: %s != %s',
c.name, container_config_hash, config_hash,
)
has_diverged = True
return has_diverged
def execute_convergence_plan(self,
plan,
insecure_registry=False,
do_build=True):
(action, containers) = plan
if action == 'create':
container = self.create_container(
insecure_registry=insecure_registry,
do_build=do_build)
do_build=do_build,
)
self.start_container(container)
return [container]
return [
self.recreate_container(c, insecure_registry=insecure_registry)
for c in containers
]
elif action == 'recreate':
return [
self.recreate_container(
c,
insecure_registry=insecure_registry,
)
for c in containers
]
def recreate_container(self, container, insecure_registry=False):
elif action == 'start':
for c in containers:
self.start_container_if_stopped(c)
return containers
elif action == 'noop':
for c in containers:
log.info("%s is up-to-date" % c.name)
return containers
else:
raise Exception("Invalid action: {}".format(action))
def recreate_container(self,
container,
insecure_registry=False):
"""Recreate a container.
The original container is renamed to a temporary name so that data
@ -289,6 +408,21 @@ class Service(object):
else:
return [self.start_container_if_stopped(c) for c in containers]
def config_hash(self):
return json_hash(self.config_dict())
def config_dict(self):
return {
'options': self.options,
'image_id': self.image()['Id'],
}
def get_dependency_names(self):
net_name = self.get_net_name()
return (self.get_linked_names() +
self.get_volumes_from_names() +
([net_name] if net_name else []))
def get_linked_names(self):
return [s.name for (s, _) in self.links]
@ -376,6 +510,9 @@ class Service(object):
number,
one_off=False,
previous_container=None):
add_config_hash = (not one_off and not override_options)
container_options = dict(
(k, self.options[k])
for k in DOCKER_CONFIG_KEYS if k in self.options)
@ -383,6 +520,13 @@ class Service(object):
container_options['name'] = self.get_container_name(number, one_off)
if add_config_hash:
config_hash = self.config_hash()
if 'labels' not in container_options:
container_options['labels'] = {}
container_options['labels'][LABEL_CONFIG_HASH] = config_hash
log.debug("Added config hash: %s" % config_hash)
if 'detach' not in container_options:
container_options['detach'] = True
@ -493,7 +637,7 @@ class Service(object):
build_output = self.client.build(
path=path,
tag=self.full_name,
tag=self.image_name,
stream=True,
rm=True,
nocache=no_cache,

0
compose/state.py Normal file
View file

9
compose/utils.py Normal file
View file

@ -0,0 +1,9 @@
import json
import hashlib
def json_hash(obj):
dump = json.dumps(obj, sort_keys=True)
h = hashlib.sha256()
h.update(dump)
return h.hexdigest()