(PE-40163) automate recovery of failed postgres server (#537)

* (PE-40163) automate recovery of failed postgres server

* (PE-40163) alternate approach to running a task

* (PE-40163) deal with empty csr attributes

* (PE-40163) update reference.md

Author: davidmalloncares

REFERENCE.md

@@ -109,6 +109,7 @@ Supported use cases:
 * [`peadm::convert`](#peadm--convert): Convert an existing PE cluster to a PEAdm-managed cluster
 * [`peadm::install`](#peadm--install): Install a new PE cluster
 * [`peadm::modify_certificate`](#peadm--modify_certificate): Modify the certificate of one or more targets
+* [`peadm::replace_failed_postgresql`](#peadm--replace_failed_postgresql): Replaces a failed PostgreSQL host
 * [`peadm::restore`](#peadm--restore): Restore puppet primary configuration
 * [`peadm::restore_ca`](#peadm--restore_ca)
 * [`peadm::status`](#peadm--status): Return status information from one or more PE clusters in a table format
@@ -2370,6 +2371,50 @@ Data type: `Boolean`
 
 Default value: `false`
 
+### <a name="peadm--replace_failed_postgresql"></a>`peadm::replace_failed_postgresql`
+
+Replaces a failed PostgreSQL host
+
+#### Parameters
+
+The following parameters are available in the `peadm::replace_failed_postgresql` plan:
+
+* [`primary_host`](#-peadm--replace_failed_postgresql--primary_host)
+* [`replica_host`](#-peadm--replace_failed_postgresql--replica_host)
+* [`working_postgresql_host`](#-peadm--replace_failed_postgresql--working_postgresql_host)
+* [`failed_postgresql_host`](#-peadm--replace_failed_postgresql--failed_postgresql_host)
+* [`replacement_postgresql_host`](#-peadm--replace_failed_postgresql--replacement_postgresql_host)
+
+##### <a name="-peadm--replace_failed_postgresql--primary_host"></a>`primary_host`
+
+Data type: `Peadm::SingleTargetSpec`
+
+- The hostname and certname of the primary Puppet server
+
+##### <a name="-peadm--replace_failed_postgresql--replica_host"></a>`replica_host`
+
+Data type: `Peadm::SingleTargetSpec`
+
+- The hostname and certname of the replica VM
+
+##### <a name="-peadm--replace_failed_postgresql--working_postgresql_host"></a>`working_postgresql_host`
+
+Data type: `Peadm::SingleTargetSpec`
+
+- The hostname and certname of the still-working PE-PostgreSQL server
+
+##### <a name="-peadm--replace_failed_postgresql--failed_postgresql_host"></a>`failed_postgresql_host`
+
+Data type: `Peadm::SingleTargetSpec`
+
+- The hostname and certname of the failed PE-PostgreSQL server
+
+##### <a name="-peadm--replace_failed_postgresql--replacement_postgresql_host"></a>`replacement_postgresql_host`
+
+Data type: `Peadm::SingleTargetSpec`
+
+- The hostname and certname of the server being brought in to replace the failed PE-PostgreSQL server
+
 ### <a name="peadm--restore"></a>`peadm::restore`
 
 Restore puppet primary configuration

plans/replace_failed_postgresql.pp

@@ -0,0 +1,59 @@
+# @summary Replaces a failed PostgreSQL host
+# @param primary_host - The hostname and certname of the primary Puppet server
+# @param replica_host - The hostname and certname of the replica VM
+# @param working_postgresql_host - The hostname and certname of the still-working PE-PostgreSQL server
+# @param failed_postgresql_host - The hostname and certname of the failed PE-PostgreSQL server
+# @param replacement_postgresql_host - The hostname and certname of the server being brought in to replace the failed PE-PostgreSQL server
+#
+plan peadm::replace_failed_postgresql(
+  Peadm::SingleTargetSpec   $primary_host,
+  Peadm::SingleTargetSpec   $replica_host,
+  Peadm::SingleTargetSpec   $working_postgresql_host,
+  Peadm::SingleTargetSpec   $failed_postgresql_host,
+  Peadm::SingleTargetSpec   $replacement_postgresql_host,
+) {
+  $all_hosts = peadm::flatten_compact([
+      $primary_host,
+      $replica_host,
+      $working_postgresql_host,
+      $failed_postgresql_host,
+      $replacement_postgresql_host,
+  ])
+
+  # verify we can connect to targets proded before proceeding
+  run_command('hostname', $all_hosts)
+
+  # Get current peadm config before making modifications
+  $peadm_config = run_task('peadm::get_peadm_config', $primary_host).first.value
+  $compilers = $peadm_config['params']['compilers']
+
+  # Bail if this is trying to be ran against Standard
+  if $compilers.empty {
+    fail_plan('Plan peadm::replace_failed_postgresql is only applicable for L and XL deployments')
+  }
+
+  $pe_hosts = peadm::flatten_compact([
+      $primary_host,
+      $replica_host,
+  ])
+
+  # Stop puppet.service on Puppet server primary and replica
+  run_task('service', $pe_hosts, 'action' => 'stop', 'name' => 'puppet.service')
+
+  # Temporarily set both primary and replica server nodes so that they use the remaining healthy PE-PostgreSQL server
+  run_plan('peadm::util::update_db_setting', $pe_hosts,
+    postgresql_host => $working_postgresql_host,
+    override => true,
+  )
+
+  # Restart pe-puppetdb.service on Puppet server primary and replica
+  run_task('service', $pe_hosts, { action => 'restart', name => 'pe-puppetdb.service' })
+
+  # Purge failed PE-PostgreSQL node from PuppetDB
+  run_command("/opt/puppetlabs/bin/puppet node purge ${$failed_postgresql_host}", $primary_host)
+
+  # Run peadm::add_database plan to deploy replacement PE-PostgreSQL server
+  run_plan('peadm::add_database', targets => $replacement_postgresql_host,
+    primary_host => $primary_host,
+  )
+}

plans/util/insert_csr_extension_requests.pp

@@ -15,9 +15,10 @@
     # If we're merging extension requests, existing requests will be preserved.
     # If we're not merging, only ours will be used; existing requests will be
     # overwritten.
-    $csr_file_data = $merge ? {
-      true  => $csr_attributes_data.deep_merge({ 'extension_requests' => $extension_requests }),
-      false => ($csr_attributes_data + { 'extension_requests' => $extension_requests }),
+    if $merge and !$csr_attributes_data.empty {
+      $csr_file_data = $csr_attributes_data.deep_merge({ 'extension_requests' => $extension_requests })
+    } else {
+      $csr_file_data = $csr_attributes_data + { 'extension_requests' => $extension_requests }
     }
 
     run_task('peadm::mkdir_p_file', $target,