CLOUD-2069 Adding support for multiple compose files. (#332)

Support for multiple compose files

Author: MarkW

README.md

@@ -554,7 +554,8 @@ compose_test:
 Specify the `file` resource to add a Compose file to the machine you have Puppet running on. To define a `docker_compose` resource pointing to the Compose file, add the following code to the manifest file:
 
 ```puppet
-docker_compose { '/tmp/docker-compose.yml':
+docker_compose { 'test':
+  compose_files => ['/tmp/docker-compose.yml'],
   ensure  => present,
 }
 ```
@@ -564,7 +565,8 @@ Puppet automatically runs Compose, because the relevant Compose services aren't
 In the example below, Puppet runs Compose when the number of containers specified for a service don't match the scale values.
 
 ```puppet
-docker_compose { '/tmp/docker-compose.yml':
+docker_compose { 'test':
+  compose_files => ['/tmp/docker-compose.yml'],
   ensure  => present,
   scale   => {
     'compose_test' => 2,
@@ -575,6 +577,16 @@ docker_compose { '/tmp/docker-compose.yml':
 
 Give options to the ```docker-compose up``` command, such as ```--remove-orphans```, by using the ```up_args``` option.
 
+To supply multiple overide compose files add the following to the manifest file:
+
+```puppet
+docker_compose {'test':
+  compose_files => ['master-docker-compose.yml', 'override-compose.yml],
+}
+```
+
+Please note you should  supply your master docker-compose file as the first element in the array. As per docker multi compose file support compose files will be merged in the order they are specified in the array.
+
 If you are using a v3.2 compose file or above on a Docker Swarm cluster, use the `docker::stack` class. Include the file resource before you run the stack command.
 
 To deploy the stack, add the following code to the manifest file:

lib/puppet/provider/docker_compose/ruby.rb

@@ -6,45 +6,47 @@
   commands docker: 'docker'
 
   def exists?
-    Puppet.info("Checking for compose project #{project}")
-    compose_file = YAML.safe_load(File.read(name))
+    Puppet.info("Checking for compose project #{name}")
     compose_services = {}
-    containers = docker([
-                          'ps',
-                          '--format',
-                          "{{.Label \"com.docker.compose.service\"}}-{{.Image}}",
-                          '--filter',
-                          "label=com.docker.compose.project=#{project}",
-                        ]).split("\n")
-    case compose_file['version']
-    when %r{^2(\.[0-3])?$}, %r{^3(\.[0-6])?$}
-      compose_services = compose_file['services']
-    # in compose v1 "version" parameter is not specified
-    when nil
-      compose_services = compose_file
-    else
-      raise(Puppet::Error, "Unsupported docker compose file syntax version \"#{compose_file['version']}\"!")
-    end
+    resource[:compose_files].each do |file|
+      compose_file = YAML.safe_load(File.read(file))
+      containers = docker([
+                            'ps',
+                            '--format',
+                            "{{.Label \"com.docker.compose.service\"}}-{{.Image}}",
+                            '--filter',
+                            "label=com.docker.compose.project=#{name}",
+                          ]).split("\n")
+      case compose_file['version']
+      when %r{^2(\.[0-3])?$}, %r{^3(\.[0-6])?$}
+        compose_services = compose_services.merge(compose_file['services'])
+      # in compose v1 "version" parameter is not specified
+      when nil
+        compose_services = compose_services.merge(compose_file)
+      else
+        raise(Puppet::Error, "Unsupported docker compose file syntax version \"#{compose_file['version']}\"!")
+      end
 
-    if compose_services.count != containers.count
-      return false
-    end
+      if compose_services.count != containers.count
+        return false
+      end
 
-    counts = Hash[*compose_services.each.map { |key, array|
-                    image = (array['image']) ? array['image'] : get_image(key, compose_services)
-                    Puppet.info("Checking for compose service #{key} #{image}")
-                    ["#{key}-#{image}", containers.count("#{key}-#{image}")]
-                  }.flatten]
+      counts = Hash[*compose_services.each.map { |key, array|
+                      image = (array['image']) ? array['image'] : get_image(key, compose_services)
+                      Puppet.info("Checking for compose service #{key} #{image}")
+                      ["#{key}-#{image}", containers.count("#{key}-#{image}")]
+                    }.flatten]
 
-    # No containers found for the project
-    if counts.empty? ||
-       # Containers described in the compose file are not running
-       counts.any? { |_k, v| v.zero? } ||
-       # The scaling factors in the resource do not match the number of running containers
-       resource[:scale] && counts.merge(resource[:scale]) != counts
-      false
-    else
-      true
+      # No containers found for the project
+      if counts.empty? ||
+         # Containers described in the compose file are not running
+         counts.any? { |_k, v| v.zero? } ||
+         # The scaling factors in the resource do not match the number of running containers
+         resource[:scale] && counts.merge(resource[:scale]) != counts
+        false
+      else
+        true
+      end
     end
   end
 
@@ -61,37 +63,37 @@ def get_image(service_name, compose_services)
   end
 
   def create
-    Puppet.info("Running compose project #{project}")
-    args = ['-f', name, 'up', '-d', '--remove-orphans'].insert(2, resource[:options]).insert(5, resource[:up_args]).compact
+    Puppet.info("Running compose project #{name}")
+    args = [compose_files, '-p', name, 'up', '-d', '--remove-orphans'].insert(2, resource[:options]).insert(5, resource[:up_args]).compact
     dockercompose(args)
     return unless resource[:scale]
     instructions = resource[:scale].map { |k, v| "#{k}=#{v}" }
     Puppet.info("Scaling compose project #{project}: #{instructions.join(' ')}")
-    args = ['-f', name, 'scale'].insert(2, resource[:options]).compact + instructions
+    args = [compose_files, '-p', name, 'scale'].insert(2, resource[:options]).compact + instructions
     dockercompose(args)
   end
 
   def destroy
-    Puppet.info("Removing all containers for compose project #{project}")
-    kill_args = ['-f', name, 'kill'].insert(2, resource[:options]).compact
+    Puppet.info("Removing all containers for compose project #{name}")
+    kill_args = [compose_files, '-p', name, 'kill'].insert(2, resource[:options]).compact
     dockercompose(kill_args)
-    rm_args = ['-f', name, 'rm', '--force', '-v'].insert(2, resource[:options]).compact
+    rm_args = [compose_files, '-p', name, 'rm', '--force', '-v'].insert(2, resource[:options]).compact
     dockercompose(rm_args)
   end
 
   def restart
     return unless exists?
-    Puppet.info("Rebuilding and Restarting all containers for compose project #{project}")
-    kill_args = ['-f', name, 'kill'].insert(2, resource[:options]).compact
+    Puppet.info("Rebuilding and Restarting all containers for compose project #{name}")
+    kill_args = [compose_files, '-p', name, 'kill'].insert(2, resource[:options]).compact
     dockercompose(kill_args)
-    build_args = ['-f', name, 'build'].insert(2, resource[:options]).compact
+    build_args = [compose_files, '-p', name, 'build'].insert(2, resource[:options]).compact
     dockercompose(build_args)
     create
   end
 
-  private
-
-  def project
-    File.basename(File.dirname(name)).downcase.gsub(%r{[^0-9a-z ]}i, '')
+  def compose_files
+    resource[:compose_files].map { |x| ['-f', x] }.flatten
   end
+
+  private
 end

lib/puppet/type/docker_compose.rb

@@ -7,10 +7,6 @@ def refresh
     provider.restart
   end
 
-  newparam(:name) do
-    desc 'Docker compose file path.'
-  end
-
   newparam(:scale) do
     desc 'A hash of compose services and number of containers.'
     validate do |value|
@@ -38,7 +34,15 @@ def refresh
     end
   end
 
-  autorequire(:file) do
-    self[:name]
+  newparam(:compose_files, :array_matching => :all) do
+    desc 'An array of Docker Compose Files paths.'
+    validate do |value|
+      raise _('compose files should be an array') unless value.is_a? Array
+    end
+  end
+
+  newparam(:name) do
+    isnamevar
+    desc 'The name of the project'
   end
 end

spec/acceptance/compose_spec.rb

@@ -25,7 +25,8 @@ class { 'docker::compose': }
   context 'Creating compose projects' do
     before(:all) do
       @install = <<-code
-docker_compose { '/tmp/docker-compose.yml':
+docker_compose { 'test3':
+  compose_files => ['#{tmp_path}/docker-compose.yml'],
   ensure => present,
 }
       code
@@ -36,21 +37,23 @@ class { 'docker::compose': }
       apply_manifest(@install, :catch_changes=>true)
     end
 
-    describe command("docker inspect tmp_compose_test_1"), :sudo => true do
+    describe command("docker inspect test3_compose_test_1"), :sudo => true do
       its(:exit_status) { should eq 0 }
     end
   end
 
   context 'Destroying compose projects' do
     before(:all) do
       install = <<-code
-docker_compose { '/tmp/docker-compose.yml':
+docker_compose { 'test4':
+  compose_files => ['#{tmp_path}/docker-compose.yml'],
   ensure => present,
 }
       code
       apply_manifest(install, :catch_failures=>true)
       @uninstall = <<-code
-docker_compose { '/tmp/docker-compose.yml':
+docker_compose { 'test4':
+  compose_files => ['#{tmp_path}/docker-compose.yml'],
   ensure => absent,
 }
       code
@@ -61,7 +64,7 @@ class { 'docker::compose': }
       apply_manifest(@uninstall, :catch_changes=>true)
     end
 
-    describe command("docker inspect tmp_compose_test_1"), :sudo => true do
+    describe command("docker inspect test4_compose_test_1"), :sudo => true do
       its(:exit_status) { should eq 1 }
     end
   end

spec/acceptance/compose_v3_spec.rb

@@ -5,11 +5,13 @@
   file_extension = '.exe'
   docker_args = 'docker_ee => true'
   tmp_path = 'C:/cygwin64/tmp'
+  test_container = 'nanoserver-sac2016'
 else
   install_dir = '/usr/local/bin'
   file_extension = ''
   docker_args = ''
   tmp_path = '/tmp'
+  test_container = 'debian'
 end
 
 describe 'docker compose' do 
@@ -29,10 +31,10 @@ class { 'docker::compose':
     it 'should have docker compose installed' do
       shell('docker-compose --help', :acceptable_exit_codes => [0])
     end
-
     before(:all) do
       @install = <<-code
-docker_compose { '#{tmp_path}/docker-compose-v3.yml':
+docker_compose { 'web':
+  compose_files => ['#{tmp_path}/docker-compose-v3.yml'],
   ensure => present,
 }
       code
@@ -44,42 +46,62 @@ class { 'docker::compose':
     end
 
     it 'should find a docker container' do
-      shell('docker inspect tmp_compose_test_1', :acceptable_exit_codes => [0])
+      shell('docker inspect web_compose_test_1', :acceptable_exit_codes => [0])
     end
   end
 
-  context 'Destroying compose v3 projects' do
+  context 'creating compose projects with multi compose files' do
     before(:all) do
-      install = <<-code
-docker_compose { '#{tmp_path}/docker-compose-v3.yml':
+      @install = <<-pp1
+docker_compose { 'web1':
+  compose_files => ['#{tmp_path}/docker-compose-v3.yml', '#{tmp_path}/docker-compose-override-v3.yml'],
   ensure => present,
 }
-      code
-      apply_manifest(install, :catch_failures=>true)
-      @uninstall = <<-code
-docker_compose { '#{tmp_path}/docker-compose-v3.yml':
+      pp1
+
+      apply_manifest(@install, :catch_failures=>true)
+    end
+
+    it "should find container with #{test_container} tag" do
+      shell("docker inspect web1_compose_test_1 | grep #{test_container}", :acceptable_exit_codes => [0])
+    end
+  end
+
+  context 'Destroying project with multiple compose files' do
+    before(:all) do
+    @install = <<-pp1
+docker_compose { 'web1':
+  compose_files => ['#{tmp_path}/docker-compose-v3.yml', '#{tmp_path}/docker-compose-override-v3.yml'],
+  ensure => present,
+}
+    pp1
+
+    @destroy = <<-pp2
+docker_compose { 'web1':
+  compose_files => ['#{tmp_path}/docker-compose-v3.yml', '#{tmp_path}/docker-compose-override-v3.yml'],
   ensure => absent,
 }
-      code
-      apply_manifest(@uninstall, :catch_failures=>true)
+    pp2
+      apply_manifest(@install, :catch_failures=>true)
+      apply_manifest(@destroy, :catch_failures=>true)
     end
 
     it 'should be idempotent' do
-      apply_manifest(@uninstall, :catch_changes=>true)
+      apply_manifest(@destroy, :catch_changes=>true)
     end
 
     it 'should not find a docker container' do
-      shell('docker inspect tmp_compose_test_1', :acceptable_exit_codes => [1])
+      shell('docker inspect web1_compose_test_1', :acceptable_exit_codes => [1])
     end
   end
 
   context 'Requesting a specific version of compose' do
     before(:all) do
       @version = '1.21.2'
       @pp = <<-code
-        class { 'docker::compose':
-          version => '#{@version}',
-        }
+class { 'docker::compose':
+  version => '#{@version}',
+}
       code
       apply_manifest(@pp, :catch_failures=>true)
     end
@@ -99,10 +121,10 @@ class { 'docker::compose':
     before(:all) do
       @version = '1.21.2'
       @pp = <<-code
-        class { 'docker::compose':
-          ensure  => absent,
-          version => '#{@version}',
-        }
+class { 'docker::compose':
+  ensure  => absent,
+  version => '#{@version}',
+}
       code
       apply_manifest(@pp, :catch_failures=>true)
     end
@@ -124,4 +146,4 @@ class { 'docker::compose': }
       apply_manifest(install_code, :catch_failures=>true)
     end
   end
-end
+ end