Add support for declaring named volumes in compose files

* Bump default API version to 1.21 (required for named volume management)
* Introduce new, versioned compose file format while maintaining support
  for current (legacy) format
* Test updates to reflect changes made to the internal API

Signed-off-by: Joffrey F <joffrey@docker.com>

tests/integration/project_test.py

@@ -69,9 +69,9 @@ class ProjectTest(DockerClientTestCase):
                 'volumes_from': ['data'],
             },
         })
-        project = Project.from_dicts(
+        project = Project.from_config(
             name='composetest',
-            service_dicts=service_dicts,
+            config_data=service_dicts,
             client=self.client,
         )
         db = project.get_service('db')
@@ -86,9 +86,9 @@ class ProjectTest(DockerClientTestCase):
             name='composetest_data_container',
             labels={LABEL_PROJECT: 'composetest'},
         )
-        project = Project.from_dicts(
+        project = Project.from_config(
             name='composetest',
-            service_dicts=build_service_dicts({
+            config_data=build_service_dicts({
                 'db': {
                     'image': 'busybox:latest',
                     'volumes_from': ['composetest_data_container'],
@@ -117,9 +117,9 @@ class ProjectTest(DockerClientTestCase):
         assert project.get_network()['Name'] == network_name
 
     def test_net_from_service(self):
-        project = Project.from_dicts(
+        project = Project.from_config(
             name='composetest',
-            service_dicts=build_service_dicts({
+            config_data=build_service_dicts({
                 'net': {
                     'image': 'busybox:latest',
                     'command': ["top"]
@@ -149,9 +149,9 @@ class ProjectTest(DockerClientTestCase):
         )
         net_container.start()
 
-        project = Project.from_dicts(
+        project = Project.from_config(
             name='composetest',
-            service_dicts=build_service_dicts({
+            config_data=build_service_dicts({
                 'web': {
                     'image': 'busybox:latest',
                     'net': 'container:composetest_net_container'
@@ -331,7 +331,6 @@ class ProjectTest(DockerClientTestCase):
         project.up(['db'])
         self.assertEqual(len(project.containers()), 1)
         old_db_id = project.containers()[0].id
-
         container, = project.containers()
         db_volume_path = container.get_mount('/var/db')['Source']
 
@@ -401,9 +400,9 @@ class ProjectTest(DockerClientTestCase):
         self.assertEqual(len(console.containers()), 0)
 
     def test_project_up_starts_depends(self):
-        project = Project.from_dicts(
+        project = Project.from_config(
             name='composetest',
-            service_dicts=build_service_dicts({
+            config_data=build_service_dicts({
                 'console': {
                     'image': 'busybox:latest',
                     'command': ["top"],
@@ -436,9 +435,9 @@ class ProjectTest(DockerClientTestCase):
         self.assertEqual(len(project.get_service('console').containers()), 0)
 
     def test_project_up_with_no_deps(self):
-        project = Project.from_dicts(
+        project = Project.from_config(
             name='composetest',
-            service_dicts=build_service_dicts({
+            config_data=build_service_dicts({
                 'console': {
                     'image': 'busybox:latest',
                     'command': ["top"],

tests/integration/service_test.py

@@ -163,6 +163,7 @@ class ServiceTest(DockerClientTestCase):
 
         # Match the last component ("host-path"), because boot2docker symlinks /tmp
         actual_host_path = container.get_mount(container_path)['Source']
+
         self.assertTrue(path.basename(actual_host_path) == path.basename(host_path),
                         msg=("Last component differs: %s, %s" % (actual_host_path, host_path)))
 

tests/integration/state_test.py

@@ -26,10 +26,10 @@ class ProjectTestCase(DockerClientTestCase):
         details = config.ConfigDetails(
             'working_dir',
             [config.ConfigFile(None, cfg)])
-        return Project.from_dicts(
+        return Project.from_config(
             name='composetest',
             client=self.client,
-            service_dicts=config.load(details))
+            config_data=config.load(details))
 
 
 class BasicProjectTest(ProjectTestCase):

tests/integration/testcases.py

@@ -39,6 +39,10 @@ class DockerClientTestCase(unittest.TestCase):
         for i in self.client.images(
                 filters={'label': 'com.docker.compose.test_image'}):
             self.client.remove_image(i)
+        volumes = self.client.volumes().get('Volumes') or []
+        for v in volumes:
+            if 'composetests_' in v['Name']:
+                self.client.remove_volume(v['Name'])
 
     def create_service(self, name, **kwargs):
         if 'image' not in kwargs and 'build' not in kwargs:

tests/unit/config/config_test.py

@@ -51,7 +51,7 @@ class ConfigTest(unittest.TestCase):
                 'tests/fixtures/extends',
                 'common.yml'
             )
-        )
+        ).services
 
         self.assertEqual(
             service_sort(service_dicts),
@@ -143,7 +143,7 @@ class ConfigTest(unittest.TestCase):
             })
         details = config.ConfigDetails('.', [base_file, override_file])
 
-        service_dicts = config.load(details)
+        service_dicts = config.load(details).services
         expected = [
             {
                 'name': 'web',
@@ -207,7 +207,7 @@ class ConfigTest(unittest.TestCase):
               labels: ['label=one']
         """)
         with tmpdir.as_cwd():
-            service_dicts = config.load(details)
+            service_dicts = config.load(details).services
 
         expected = [
             {
@@ -260,7 +260,7 @@ class ConfigTest(unittest.TestCase):
                 build_config_details(
                     {valid_name: {'image': 'busybox'}},
                     'tests/fixtures/extends',
-                    'common.yml'))
+                    'common.yml')).services
             assert services[0]['name'] == valid_name
 
     def test_config_hint(self):
@@ -451,7 +451,7 @@ class ConfigTest(unittest.TestCase):
                     'working_dir',
                     'filename.yml'
                 )
-            )
+            ).services
             self.assertEqual(service[0]['expose'], expose)
 
     def test_valid_config_oneof_string_or_list(self):
@@ -466,7 +466,7 @@ class ConfigTest(unittest.TestCase):
                     'working_dir',
                     'filename.yml'
                 )
-            )
+            ).services
             self.assertEqual(service[0]['entrypoint'], entrypoint)
 
     @mock.patch('compose.config.validation.log')
@@ -496,7 +496,7 @@ class ConfigTest(unittest.TestCase):
                 'working_dir',
                 'filename.yml'
             )
-        )
+        ).services
         self.assertEqual(services[0]['environment']['SPRING_JPA_HIBERNATE_DDL-AUTO'], 'none')
 
     def test_load_yaml_with_yaml_error(self):
@@ -655,7 +655,7 @@ class InterpolationTest(unittest.TestCase):
 
         service_dicts = config.load(
             config.find('tests/fixtures/environment-interpolation', None),
-        )
+        ).services
 
         self.assertEqual(service_dicts, [
             {
@@ -722,7 +722,7 @@ class InterpolationTest(unittest.TestCase):
                 '.',
                 None,
             )
-        )[0]
+        ).services[0]
         self.assertEquals(service_dict['environment']['POSTGRES_PASSWORD'], '')
 
 
@@ -734,11 +734,15 @@ class VolumeConfigTest(unittest.TestCase):
     @mock.patch.dict(os.environ)
     def test_volume_binding_with_environment_variable(self):
         os.environ['VOLUME_PATH'] = '/host/path'
-        d = config.load(build_config_details(
-            {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
-            '.',
-        ))[0]
-        self.assertEqual(d['volumes'], [VolumeSpec.parse('/host/path:/container/path')])
+
+        d = config.load(
+            build_config_details(
+                {'foo': {'build': '.', 'volumes': ['${VOLUME_PATH}:/container/path']}},
+                '.',
+                None,
+            )
+        ).services[0]
+        self.assertEqual(d['volumes'], ['/host/path:/container/path'])
 
     @pytest.mark.skipif(IS_WINDOWS_PLATFORM, reason='posix paths')
     @mock.patch.dict(os.environ)
@@ -1012,7 +1016,7 @@ class MemoryOptionsTest(unittest.TestCase):
                 'tests/fixtures/extends',
                 'common.yml'
             )
-        )
+        ).services
         self.assertEqual(service_dict[0]['memswap_limit'], 2000000)
 
     def test_memswap_can_be_a_string(self):
@@ -1022,7 +1026,7 @@ class MemoryOptionsTest(unittest.TestCase):
                 'tests/fixtures/extends',
                 'common.yml'
             )
-        )
+        ).services
         self.assertEqual(service_dict[0]['memswap_limit'], "512M")
 
 
@@ -1126,24 +1130,21 @@ class EnvTest(unittest.TestCase):
                 {'foo': {'build': '.', 'volumes': ['$HOSTENV:$CONTAINERENV']}},
                 "tests/fixtures/env",
             )
-        )[0]
-        self.assertEqual(
-            set(service_dict['volumes']),
-            set([VolumeSpec.parse('/tmp:/host/tmp')]))
+
+        ).services[0]
+        self.assertEqual(set(service_dict['volumes']), set(['/tmp:/host/tmp']))
 
         service_dict = config.load(
             build_config_details(
                 {'foo': {'build': '.', 'volumes': ['/opt${HOSTENV}:/opt${CONTAINERENV}']}},
                 "tests/fixtures/env",
             )
-        )[0]
-        self.assertEqual(
-            set(service_dict['volumes']),
-            set([VolumeSpec.parse('/opt/tmp:/opt/host/tmp')]))
+        ).services[0]
+        self.assertEqual(set(service_dict['volumes']), set(['/opt/tmp:/opt/host/tmp']))
 
 
 def load_from_filename(filename):
-    return config.load(config.find('.', [filename]))
+    return config.load(config.find('.', [filename])).services
 
 
 class ExtendsTest(unittest.TestCase):
@@ -1313,7 +1314,7 @@ class ExtendsTest(unittest.TestCase):
                 'tests/fixtures/extends',
                 'common.yml'
             )
-        )
+        ).services
 
         self.assertEquals(len(service), 1)
         self.assertIsInstance(service[0], dict)

tests/unit/project_test.py

@@ -4,6 +4,7 @@ import docker
 
 from .. import mock
 from .. import unittest
+from compose.config.config import Config
 from compose.config.types import VolumeFromSpec
 from compose.const import LABEL_SERVICE
 from compose.container import Container
@@ -18,7 +19,7 @@ class ProjectTest(unittest.TestCase):
         self.mock_client = mock.create_autospec(docker.Client)
 
     def test_from_dict(self):
-        project = Project.from_dicts('composetest', [
+        project = Project.from_config('composetest', Config(None, [
             {
                 'name': 'web',
                 'image': 'busybox:latest'
@@ -27,15 +28,38 @@ class ProjectTest(unittest.TestCase):
                 'name': 'db',
                 'image': 'busybox:latest'
             },
-        ], None)
+        ], None), None)
         self.assertEqual(len(project.services), 2)
         self.assertEqual(project.get_service('web').name, 'web')
         self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
         self.assertEqual(project.get_service('db').name, 'db')
         self.assertEqual(project.get_service('db').options['image'], 'busybox:latest')
 
+    def test_from_dict_sorts_in_dependency_order(self):
+        project = Project.from_config('composetest', Config(None, [
+            {
+                'name': 'web',
+                'image': 'busybox:latest',
+                'links': ['db'],
+            },
+            {
+                'name': 'db',
+                'image': 'busybox:latest',
+                'volumes_from': ['volume']
+            },
+            {
+                'name': 'volume',
+                'image': 'busybox:latest',
+                'volumes': ['/tmp'],
+            }
+        ], None), None)
+
+        self.assertEqual(project.services[0].name, 'volume')
+        self.assertEqual(project.services[1].name, 'db')
+        self.assertEqual(project.services[2].name, 'web')
+
     def test_from_config(self):
-        dicts = [
+        dicts = Config(None, [
             {
                 'name': 'web',
                 'image': 'busybox:latest',
@@ -44,8 +68,8 @@ class ProjectTest(unittest.TestCase):
                 'name': 'db',
                 'image': 'busybox:latest',
             },
-        ]
-        project = Project.from_dicts('composetest', dicts, None)
+        ], None)
+        project = Project.from_config('composetest', dicts, None)
         self.assertEqual(len(project.services), 2)
         self.assertEqual(project.get_service('web').name, 'web')
         self.assertEqual(project.get_service('web').options['image'], 'busybox:latest')
@@ -141,13 +165,13 @@ class ProjectTest(unittest.TestCase):
         container_id = 'aabbccddee'
         container_dict = dict(Name='aaa', Id=container_id)
         self.mock_client.inspect_container.return_value = container_dict
-        project = Project.from_dicts('test', [
+        project = Project.from_config('test', Config(None, [
             {
                 'name': 'test',
                 'image': 'busybox:latest',
                 'volumes_from': [VolumeFromSpec('aaa', 'rw')]
             }
-        ], self.mock_client)
+        ], None), self.mock_client)
         self.assertEqual(project.get_service('test')._get_volumes_from(), [container_id + ":rw"])
 
     def test_use_volumes_from_service_no_container(self):
@@ -160,7 +184,7 @@ class ProjectTest(unittest.TestCase):
                 "Image": 'busybox:latest'
             }
         ]
-        project = Project.from_dicts('test', [
+        project = Project.from_config('test', Config(None, [
             {
                 'name': 'vol',
                 'image': 'busybox:latest'
@@ -170,13 +194,13 @@ class ProjectTest(unittest.TestCase):
                 'image': 'busybox:latest',
                 'volumes_from': [VolumeFromSpec('vol', 'rw')]
             }
-        ], self.mock_client)
+        ], None), self.mock_client)
         self.assertEqual(project.get_service('test')._get_volumes_from(), [container_name + ":rw"])
 
     def test_use_volumes_from_service_container(self):
         container_ids = ['aabbccddee', '12345']
 
-        project = Project.from_dicts('test', [
+        project = Project.from_config('test', Config(None, [
             {
                 'name': 'vol',
                 'image': 'busybox:latest'
@@ -186,7 +210,7 @@ class ProjectTest(unittest.TestCase):
                 'image': 'busybox:latest',
                 'volumes_from': [VolumeFromSpec('vol', 'rw')]
             }
-        ], None)
+        ], None), None)
         with mock.patch.object(Service, 'containers') as mock_return:
             mock_return.return_value = [
                 mock.Mock(id=container_id, spec=Container)
@@ -196,12 +220,12 @@ class ProjectTest(unittest.TestCase):
                 [container_ids[0] + ':rw'])
 
     def test_net_unset(self):
-        project = Project.from_dicts('test', [
+        project = Project.from_config('test', Config(None, [
             {
                 'name': 'test',
                 'image': 'busybox:latest',
             }
-        ], self.mock_client)
+        ], None), self.mock_client)
         service = project.get_service('test')
         self.assertEqual(service.net.id, None)
         self.assertNotIn('NetworkMode', service._get_container_host_config({}))
@@ -210,13 +234,13 @@ class ProjectTest(unittest.TestCase):
         container_id = 'aabbccddee'
         container_dict = dict(Name='aaa', Id=container_id)
         self.mock_client.inspect_container.return_value = container_dict
-        project = Project.from_dicts('test', [
+        project = Project.from_config('test', Config(None, [
             {
                 'name': 'test',
                 'image': 'busybox:latest',
                 'net': 'container:aaa'
             }
-        ], self.mock_client)
+        ], None), self.mock_client)
         service = project.get_service('test')
         self.assertEqual(service.net.mode, 'container:' + container_id)
 
@@ -230,7 +254,7 @@ class ProjectTest(unittest.TestCase):
                 "Image": 'busybox:latest'
             }
         ]
-        project = Project.from_dicts('test', [
+        project = Project.from_config('test', Config(None, [
             {
                 'name': 'aaa',
                 'image': 'busybox:latest'
@@ -240,7 +264,7 @@ class ProjectTest(unittest.TestCase):
                 'image': 'busybox:latest',
                 'net': 'container:aaa'
             }
-        ], self.mock_client)
+        ], None), self.mock_client)
 
         service = project.get_service('test')
         self.assertEqual(service.net.mode, 'container:' + container_name)
@@ -285,12 +309,12 @@ class ProjectTest(unittest.TestCase):
                 },
             },
         }
-        project = Project.from_dicts(
+        project = Project.from_config(
             'test',
-            [{
+            Config(None, [{
                 'name': 'web',
                 'image': 'busybox:latest',
-            }],
+            }], None),
             self.mock_client,
         )
         self.assertEqual([c.id for c in project.containers()], ['1'])