Merge pull request #62 from suckatrash/add_profiles

Add profiles, move puppetdb metric defaults (part 2)

Author: suckatrash

.fixtures.yml

@@ -15,4 +15,7 @@ fixtures:
       yumrepo:
         repo: https://github.com/puppetlabs/puppetlabs-yumrepo_core.git
         ref: 1.0.2
+      telegraf:
+        repo: https://github.com/voxpupuli/puppet-telegraf.git
+        ref: v2.1.0
     forge_modules:

Gemfile

@@ -23,6 +23,7 @@ group :development do
   gem "json", '= 1.8.1',                                         require: false if Gem::Version.new(RUBY_VERSION.dup) == Gem::Version.new('2.1.9')
   gem "json", '= 2.0.4',                                         require: false if Gem::Requirement.create('~> 2.4.2').satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
   gem "json", '= 2.1.0',                                         require: false if Gem::Requirement.create(['>= 2.5.0', '< 2.7.0']).satisfied_by?(Gem::Version.new(RUBY_VERSION.dup))
+  gem "toml-rb", '= 1.1.2',                                      require: false, platforms: [:ruby]
   gem "puppet-module-posix-default-r#{minor_version}", '~> 0.3', require: false, platforms: [:ruby]
   gem "puppet-module-posix-dev-r#{minor_version}", '~> 0.3',     require: false, platforms: [:ruby]
   gem "puppet-module-win-default-r#{minor_version}", '~> 0.3',   require: false, platforms: [:mswin, :mingw, :x64_mingw]

README.md

@@ -13,25 +13,44 @@
 
 ## Description
 
-This module is used to configure grafana, telegraf, and influxdb to consume metrics from Puppet service.
+This module is used to configure grafana and influxdb to consume metrics from Puppet services.
+By default the metrics collection is done by another service called telegraf.
 
-You have the option of getting metrics from any or all of three of these methods:
+These services can all run on a single server by applying the base class.  You also have the option
+to use the [included defined types](#profile-defined-types) to configure telegraf on each of your Puppet infrastructure components
+(master,compilers, puppetdb, postgres server) and the metrics will be stored on another server
+running grafana and influxdb.  In environments where there is an existing grafana / influxdb
+instance, the later option is recommended. See [Determining where telegraf runs](#determining-where-telegraf-runs) for further
+details.
 
+You have the option of collecting metrics using any or all of these methods:
+
+* Through telegraf, which polls several of Puppet's metrics endpoints (recommended)
 * Through Archive files from the [puppetlabs/puppet_metrics_collector](https://forge.puppet.com/puppetlabs/puppet_metrics_collector) module
 * Natively, via Puppetserver's [built-in graphite support](https://puppet.com/docs/pe/2019.0/getting_started_with_graphite.html#task-7933)
-* Through telegraf, which polls several of Puppet's metrics endpoints
 
 ## Setup
 
-### Upgrade note
+### Upgrade notes, breaking changes in v2
+
+The `puppet_metrics_dashboard::profile::postgres` class is now deprecated and you should use the `puppet_metrics_dashboard::profile::Master::postgres_access` class instead.
+
+Parameters `telegraf_agent_interval` and `http_response_timeout` were previously integers but are now strings.  The value should match a time interval, such as `5s`, `10m`, or `1h`.
+
+`influxdb_urls` was previously a string, it should now be an array.
 
 Previous versions of this module put several `[[inputs.httpjson]]` entries in
 `/etc/telegraf/telegraf.conf`. These entries should be removed now as all
-module-specific settings now reside in
-`/etc/telegraf/telegraf.d/puppet_metrics_dashboard.conf`. Telegraf will
-continue to work if you do not remove them, however, the old 
+module-specific settings now reside in individual files within
+`/etc/telegraf/telegraf.d/`. Telegraf will continue to work if you do not remove them, however, the old
 `[[inputs.httpjson]]` will not be updated going forward.
 
+### Determining where telegraf runs
+
+Telegraf can run on the grafana server or on each Puppet infrastructure node.  To configure telegraf to run on the same host that
+grafana runs on, use the `puppet_metrics_dashboard` class and the parameters: `master_list`, `puppetdb_list` and `postgres_host_list`.  These parameters determine which hosts that telegraf polls.
+
+To configure telegraf to run on each Puppet infrastructure node, use the corresponding profiles for those hosts.  See [Profile defined types](#profile-defined-types).  The `puppet_metrics_dashboard` class is still applied to a separate host to setup grafana and influxdb and the profile classes configure telegraf when applied to your Puppet infrastructure hosts.
 
 ### Beginning with puppet_metrics_dashboard
 
@@ -45,12 +64,11 @@ include puppet_metrics_dashboard
 
 ## Usage
 
-### To install example dashboards for all of the collection methods:
+### To install example dashboards and use the telegraf collection method (default):
 
 ```
 class { 'puppet_metrics_dashboard':
   add_dashboard_examples => true,
-  influxdb_database_name => ['puppet_metrics','telegraf','graphite'],
 }
 ```
 
@@ -59,7 +77,6 @@ class { 'puppet_metrics_dashboard':
 ```
 class { 'puppet_metrics_dashboard':
   add_dashboard_examples => true,
-  influxdb_database_name => ['puppet_metrics','telegraf','graphite'],
   overwrite_dashboards   => false,
 }
 ```
@@ -68,8 +85,6 @@ class { 'puppet_metrics_dashboard':
 
 ```
 class { 'puppet_metrics_dashboard':
-  configure_telegraf  => true,
-  enable_telegraf     => true,
   master_list         => ['master1.com',
                           # Alternate ports may be configured using
                           # a pair of: [hostname, port_number]
@@ -79,18 +94,6 @@ class { 'puppet_metrics_dashboard':
 }
 ```
 
-### Allow access to PE-managed postgres nodes with the following class:
-
-This is required for collection of postgres metrics.
-
-```
-class { 'puppet_metrics_dashboard::profile::postgres':
-  grafana_host => 'postgres01',
-}
-```
-
-`grafana_host` is optional.  If you do not specify it, the class will look for a node with the `puppet_metrics_dashboard` class applied in PuppetDB and use the `certname` of the first host returned.  If the PuppetDB lookup fails and you do not specify `grafana_host` then the class outputs a warning.
-
 ### Enable Graphite support
 
 ```
@@ -102,17 +105,15 @@ class { 'puppet_metrics_dashboard':
 }
 ```
 
-* This method requires enabling on the master side as described [here](https://puppet.com/docs/pe/2017.3/puppet_server_metrics/getting_started_with_graphite.html#enabling-puppet-server-graphite-support).  The hostname(s) that you use in `master_list` should match the value(s) that you used for `metrics_server_id` in the `puppet_enterprise::profile::master` class. 
+* This method requires enabling on the master side as described [here](https://puppet.com/docs/pe/latest/puppet_server_metrics/getting_started_with_graphite.html#enabling-puppet-server-graphite-support).  The hostname(s) that you use in `master_list` should match the value(s) that you used for `metrics_server_id` in the `puppet_enterprise::profile::master` class.
 
-### Enable Telegraf, Graphite, and Archive
+### Enable Telegraf, Graphite, and Archive (puppet_metrics)
 
 ```
 class { 'puppet_metrics_dashboard':
   add_dashboard_examples => true,
   influxdb_database_name => ['puppet_metrics','telegraf','graphite'],
   consume_graphite       => true,
-  configure_telegraf     => true,
-  enable_telegraf        => true,
 }
 ```
 
@@ -130,14 +131,63 @@ By default, this will create a set of certificates in `/etc/grafana` that are ba
 
 _Note:_ Enabling SSL on Grafana will not allow for running on privileged ports such as `443`. To enable this capability you can use the suggestions documented in [this Grafana documentation](http://docs.grafana.org/installation/configuration/#http-port)
 
+### Allow access to PE-managed postgres nodes with the following class:
+
+This is required for collection of postgres metrics.  The class should be applied to the master (or postgres server if using external postgres).
+
+```
+class { 'puppet_metrics_dashboard::profile::master::postgres_access':
+  telegraf_host => 'grafana-server.example.com',
+}
+```
+
+`telegraf_host` is optional.  If you do not specify it, the class will look for a node with the `puppet_metrics_dashboard` class applied in PuppetDB and use the `certname` of the first host returned.  If the PuppetDB lookup fails and you do not specify `telegraf_host` then the class outputs a warning.
+
+### Profile defined types
+
+The module includes defined types that you can use with an existing grafana implementation.  For example:
+
+#### Add telegraf to a master / compiler
+
+```
+puppet_metrics_dashboard::profile::compiler{ $facts['networking']['fqdn']:
+  timeout => '5s',
+}
+```
+
+#### Add telegraf to a puppetdb node (see params.pp for `puppetdb_metrics` examples)
+
+```
+puppet_metrics_dashboard::profile::puppetdb{ $facts['networking']['fqdn']:
+  timeout => '5s',
+  puppetdb_metrics => [
+  { 'name' => 'global_command-parse-time',
+    'url'  => 'puppetlabs.puppetdb.mq:name=global.command-parse-time' },
+  { 'name' => 'global_discarded',
+    'url'  => 'puppetlabs.puppetdb.mq:name=global.discarded' },
+  ]
+}
+```
+
+#### Add telegraf to a postgres server
+
+```
+puppet_metrics_dashboard::profile::master::postgres{ $facts['networking']['fqdn']:
+  query_interval => '10m',
+}
+```
+
+#### Note on using the defined types
+
+Because of the way that the telegraf module works, these examples will overwrite any configuration in telegraf.config if it is *not* already puppet-managed.  See the [puppet-telegraf documentation](https://forge.puppet.com/puppet/telegraf#usage) on how to manage this file and add important settings.
+
 ### Other possibilities
 
-Configure the passwords for the InfluxDB and Grafana administrator users and enable additional [TICK Stack](https://www.influxdata.com/time-series-platform/) components.
+Configure the passwords for the InfluxDB and enable additional [TICK Stack](https://www.influxdata.com/time-series-platform/) components.
 
 ```
 class { 'puppet_metrics_dashboard':
   influx_db_password  => 'secret',
-  grafana_password    => 'secret',
   grafana_http_port   => 8080,
   grafana_version     => '4.5.2',
   enable_chronograf   => true,
@@ -147,7 +197,7 @@ class { 'puppet_metrics_dashboard':
 
 ## Reference
 
-**Note** This section is no longer maintained. Please see the REFERENCE.MD file for current listings. 
+**Note** This section is no longer maintained. Please see the REFERENCE.MD file for current listings.
 
 ## Limitations
 
@@ -159,7 +209,7 @@ Error: Execution of '/usr/bin/yum -d 0 -e 0 -y install telegraf' returned 1: Err
 Error: /Stage[main]/Pe_metrics_dashboard::Telegraf/Package[telegraf]/ensure: change from purged to present failed: Execution of '/usr/bin/yum -d 0 -e 0 -y install telegraf' returned 1: Error: Cannot retrieve repository metadata (repomd.xml) for repository: influxdb. Please verify its path and try again
 ```
 
-To recify the issue, please update `nss` and `curl` on the affected system.
+To rectify the issue, please update `nss` and `curl` on the affected system.
 
 ```
 yum install curl nss --disablerepo influxdb

REFERENCE.md

@@ -23,7 +23,6 @@
 * `puppet_metrics_dashboard::service`: Manages services
 * `puppet_metrics_dashboard::telegraf`: Install and configure Telegraf
 * `puppet_metrics_dashboard::telegraf::config`: Configures Telegraf
-* `puppet_metrics_dashboard::telegraf::install`: Installs Telegraf
 * `puppet_metrics_dashboard::telegraf::service`: Manages the Telegraf service
 
 ## Defined types

manifests/init.pp

@@ -32,7 +32,7 @@
 # @param consume_graphite
 #   Whether to enable the InfluxDB Graphite plugin.
 #   Valid values are `true`, `false`. Defaults to `false`
-#   This parameter enables the Graphite plugin for InfluxDB to allow for injesting Graphite metrics. Ensure `influxdb_database_name` 
+#   This parameter enables the Graphite plugin for InfluxDB to allow for injesting Graphite metrics. Ensure `influxdb_database_name`
 #   contains `graphite` when using this parameter.
 #   _Note:_ If using Graphite metrics from the Puppet Master, this needs to be set to `true`.
 #
@@ -52,8 +52,8 @@
 # @param influxdb_database_name
 #   An array of databases that should be created in InfluxDB.
 #   Valid values are 'puppet_metrics','telegraf', 'graphite', and any other string. Defaults to `['puppet_metrics']`
-#   Each database in the array will be created in InfluxDB. 'puppet_metrics','telegraf', and 'graphite' are specially named and will 
-#   be used with their associated metric collection method. Any other database name will be created, but not utilized with 
+#   Each database in the array will be created in InfluxDB. 'puppet_metrics','telegraf', and 'graphite' are specially named and will
+#   be used with their associated metric collection method. Any other database name will be created, but not utilized with
 #   components in this module.
 #
 # @param influx_db_password
@@ -86,16 +86,19 @@
 #   Defaults to `[$trusted['certname']]`
 #
 # @param influxdb_urls
-#   The string for telegraf's config defining where influxdb is
+#   An array for telegraf's config defining where influxdb instances are
 #
 # @param telegraf_db_name
 #   The database in influxdb where telefraf metrics are stored
 #
 # @param telegraf_agent_interval
-#   How often the telefraf agent queries for metrics
+#   How often the telefraf agent queries for metrics.  Defaults to "5s"
 #
 # @param http_response_timeout
-#   How long to wait for the queries by telegraf to finish before giving up
+#   How long to wait for the queries by telegraf to finish before giving up. Defaults to "5s"
+#
+# @param pg_query_interval
+#   How often postgres queries will run when monitoring a postgres host. Defaults to "10m"
 #
 # @param overwrite_dashboards
 #   Whether to overwrite the example Grafana dashboards.
@@ -113,7 +116,7 @@
 #       integer that specifies the port number.
 #   Defaults to `[$trusted['certname']]`
 #
-# @param puppetdb_list
+# @param postgres_host_list
 #   A list of PostgreSQL servers that Telegraf will be configured to
 #   collect metrics from. Entries in the list may be:
 #     - A single string that contains a hostname or IP address.
@@ -203,13 +206,15 @@
   Boolean $enable_telegraf                =  $puppet_metrics_dashboard::params::enable_telegraf,
   Boolean $configure_telegraf             =  $puppet_metrics_dashboard::params::configure_telegraf,
   Boolean $consume_graphite               =  $puppet_metrics_dashboard::params::consume_graphite,
-  Puppet_metrics_dashboard::HostList $master_list        = $puppet_metrics_dashboard::params::master_list,
-  Puppet_metrics_dashboard::HostList $puppetdb_list      = $puppet_metrics_dashboard::params::puppetdb_list,
-  Puppet_metrics_dashboard::HostList $postgres_host_list = $puppet_metrics_dashboard::params::postgres_host_list,
-  String $influxdb_urls                   =  $puppet_metrics_dashboard::params::influxdb_urls,
+  Puppet_metrics_dashboard::HostList $master_list             = $puppet_metrics_dashboard::params::master_list,
+  Puppet_metrics_dashboard::HostList $puppetdb_list           = $puppet_metrics_dashboard::params::puppetdb_list,
+  Puppet_metrics_dashboard::HostList $postgres_host_list      = $puppet_metrics_dashboard::params::postgres_host_list,
+  Puppet_metrics_dashboard::Puppetdb_metric $puppetdb_metrics = $puppet_metrics_dashboard::params::puppetdb_metrics,
+  Array[String] $influxdb_urls            =  $puppet_metrics_dashboard::params::influxdb_urls,
   String $telegraf_db_name                =  $puppet_metrics_dashboard::params::telegraf_db_name,
-  Integer[1] $telegraf_agent_interval     =  $puppet_metrics_dashboard::params::telegraf_agent_interval,
-  Integer[1] $http_response_timeout       =  $puppet_metrics_dashboard::params::http_response_timeout,
+  String[2] $telegraf_agent_interval     =  $puppet_metrics_dashboard::params::telegraf_agent_interval,
+  String[2] $http_response_timeout       =  $puppet_metrics_dashboard::params::http_response_timeout,
+  String[2] $pg_query_interval           =  $puppet_metrics_dashboard::params::pg_query_interval,
   ) inherits puppet_metrics_dashboard::params {
   if $manage_repos {
     contain puppet_metrics_dashboard::repos