Merge pull request #229 from seanmil/add_packages_inventory

Add support for PE package inventory facts

Author: ahayworth

doc/configuration-puppetdb.md

@@ -67,3 +67,11 @@ SSL support is enabled via any of the `--puppetdb-ssl-...` command line options
 - The CA certificate should be the public certificate of the CA that signed your PuppetDB server's certificate. This file can be found in `/etc/puppetlabs/puppetdb/ssl/ca.pem` on a PuppetDB server. Since this is a public certificate, it is safe (and recommended) to distribute this file to any clients that may connect to this PuppetDB instance.
 
 - The client keypair (key, certificate, and optionally password) should be generated individually for each client. You should NOT copy SSL keypairs from your PuppetDB server (or anywhere else) to your clients. If you are using `octocatalog-diff` on a system that is managed by Puppet, you may wish to use the same SSL credentials that the system uses to authenticate to Puppet. With recent versions of the Puppet agent, those certificates are found in `/etc/puppetlabs/puppet/ssl`.
+
+# Puppet Enterprise PuppetDB Package Inventory
+
+Puppet Enterprise customers have an optional package inventory feature which can be enabled. When this feature is enabled an inventory of all system packages
+is performed and uploaded as a fact which is then processed and stored independently of the normal Facter data in PuppetDB. Most environments won't need
+to replicate the package inventory facts for testing with Octocatalog-Diff but if you want the package inventory data (if present) to be included
+in the facts retrieved from PuppetDB by Octocatalog-Diff you should specify the `--puppetdb-package-inventory` flag. When enabled, this flag will instruct
+Octocatalog-Diff to retrieve any package data found for a node from PuppetDB and include it in the facts used during the Octocatalog-Diff compile.

lib/octocatalog-diff/cli/options/puppetdb_package_inventory.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+# When pulling facts from PuppetDB in a Puppet Enterprise environment, also include
+# the Puppet Enterprise Package Inventory data in the fact results, if available.
+# Generally you should not need to specify this, but including the package inventory
+# data will produce a more accurate set of input facts for environments using
+# package inventory.
+# @param parser [OptionParser object] The OptionParser argument
+# @param options [Hash] Options hash being constructed; this is modified in this method.
+OctocatalogDiff::Cli::Options::Option.newoption(:puppetdb_package_inventory) do
+  has_weight 150
+
+  def parse(parser, options)
+    parser.on('--[no-]puppetdb-package-inventory', 'Include Puppet Enterprise package inventory data, if found') do |x|
+      options[:puppetdb_package_inventory] = x
+    end
+  end
+end

lib/octocatalog-diff/facts/puppetdb.rb

@@ -36,6 +36,7 @@ def self.fact_retriever(options = {}, node)
         exception_class = nil
         exception_message = nil
         obj_to_return = nil
+        packages = nil
         (retries + 1).times do
           begin
             result = puppetdb.get(uri)
@@ -61,8 +62,48 @@ def self.fact_retriever(options = {}, node)
             exception_message = "Fact retrieval failed for node #{node} from PuppetDB (#{exc.message})"
           end
         end
-        return obj_to_return unless obj_to_return.nil?
-        raise exception_class, exception_message
+
+        raise exception_class, exception_message if obj_to_return.nil?
+
+        return obj_to_return if puppetdb_api_version < 4 || (!options[:puppetdb_package_inventory])
+
+        (retries + 1).times do
+          begin
+            result = puppetdb.get("/pdb/query/v4/package-inventory/#{node}")
+            packages = {}
+            result.each do |pkg|
+              key = "#{pkg['package_name']}+#{pkg['provider']}"
+              # Need to handle the situation where a package has multiple versions installed.
+              # The _puppet_inventory_1 hash lists them separated by "; ".
+              if packages.key?(key)
+                packages[key]['version'] += "; #{pkg['version']}"
+              else
+                packages[key] = pkg
+              end
+            end
+            break
+          rescue OctocatalogDiff::Errors::PuppetDBConnectionError => exc
+            exception_class = OctocatalogDiff::Errors::FactSourceError
+            exception_message = "Package inventory retrieval failed (#{exc.class}) (#{exc.message})"
+          # This is not expected to occur, but we'll leave it just in case. A query to package-inventory
+          # for a non-existant node returns a 200 OK with an empty list of packages:
+          rescue OctocatalogDiff::Errors::PuppetDBNodeNotFoundError
+            packages = {}
+          rescue OctocatalogDiff::Errors::PuppetDBGenericError => exc
+            exception_class = OctocatalogDiff::Errors::FactRetrievalError
+            exception_message = "Package inventory retrieval failed for node #{node} from PuppetDB (#{exc.message})"
+          end
+        end
+
+        raise exception_class, exception_message if packages.nil?
+
+        unless packages.empty?
+          obj_to_return['values']['_puppet_inventory_1'] = {
+            'packages' => packages.values.map { |pkg| [pkg['package_name'], pkg['version'], pkg['provider']] }
+          }
+        end
+
+        obj_to_return
       end
     end
   end

spec/octocatalog-diff/fixtures/facts/valid-packages.yaml

@@ -0,0 +1,12 @@
+--- !ruby/object:Puppet::Node::Facts
+name: rspec-node.xyz.github.net
+values:
+  _timestamp: '2016-03-16 16:02:13 -0500'
+  apt_update_last_success: 1458162123
+  architecture: amd64
+  clientcert: rspec-node.github.net
+  datacenter: xyz
+  domain: xyz.github.net
+  fqdn: rspec-node.xyz.github.net
+  ipaddress: 10.20.30.40
+  kernel: Linux

spec/octocatalog-diff/fixtures/packages/valid-packages.json

@@ -0,0 +1,14 @@
+[
+  {
+    "certname": "valid-packages",
+    "package_name": "kernel",
+    "version": "3.2.1",
+    "provider": "yum"
+  },
+  {
+    "certname": "valid-packages",
+    "package_name": "bash",
+    "version": "4.0.0",
+    "provider": "yum"
+  }
+]

spec/octocatalog-diff/mocks/puppetdb.rb

@@ -24,6 +24,7 @@ def initialize(overrides = {})
       def get(uri)
         return facts(Regexp.last_match(1)) if uri =~ %r{^/pdb/query/v4/nodes/([^/]+)/facts$}
         return catalog(Regexp.last_match(1)) if uri =~ %r{^/pdb/query/v4/catalogs/(.+)$}
+        return packages(Regexp.last_match(1)) if uri =~ %r{^/pdb/query/v4/package-inventory/(.+)$}
         raise ArgumentError, "PuppetDB URL not mocked: #{uri}"
       end
 
@@ -63,6 +64,21 @@ def catalog(hostname)
         raise OctocatalogDiff::Errors::PuppetDBNodeNotFoundError, '404 - Not Found' unless File.file?(fixture_file)
         JSON.parse(File.read(fixture_file))
       end
+
+      # Mock packages from PuppetDB
+      # @param hostname [String] Host name
+      # @return [String] JSON catalog
+      def packages(hostname)
+        fixture_file = OctocatalogDiff::Spec.fixture_path(File.join('packages', "#{hostname}.json"))
+
+        # If packages are requested from PuppetDB for an invalid node name, it will return 200 OK
+        # with an empty list:
+        if File.file?(fixture_file)
+          JSON.parse(File.read(fixture_file))
+        else
+          []
+        end
+      end
     end
   end
 end