Fix installer exit handling

CHANGELOG.md

@@ -1,5 +1,15 @@
 # PEADM module
 
+## Unreleased
+### Summary
+
+Unreleased
+
+### Changes
+
+- Improve error handling during early installation of PE
+- Implement concurrency in peadm::action::install to increase speed of installation process
+
 ## 2.5.0
 ### Summary
 

README.md

@@ -50,7 +50,7 @@ The normal usage pattern for peadm is as follows.
 ### Requirements
 
 * Puppet Enterprise 2019.8.1 or newer (tested with PE 2021.0)
-* Bolt 2.27.0 or newer (tested with Bolt 3.5)
+* Bolt 2.42.0 or newer (tested with Bolt 3.5.0)
 * EL 7, EL 8, Ubuntu 18.04, or Ubuntu 20.04
 * Classifier Data enabled. This PE feature is enabled by default on new installs, but can be disabled by users if they remove the relevant configuration from their global hiera.yaml file. See the [PE docs](https://puppet.com/docs/pe/latest/config_console.html#task-5039) for more information.
 

plans/action/install.pp

@@ -32,7 +32,7 @@
 
   # Common Configuration
   String               $console_password,
-  String               $version       = '2019.7.0',
+  String               $version       = '2019.8.5',
   Array[String]        $dns_alt_names = [ ],
   Hash                 $pe_conf_data  = { },
 
@@ -235,21 +235,15 @@
     }
   }
 
-  # Get the master installation up and running. The installer will
-  # "fail" because PuppetDB can't start, if puppetdb_database_target
-  # is set. That's expected.
-  $shortcircuit_puppetdb = !($puppetdb_database_target.empty)
-  without_default_logging() || {
-    out::message("Starting: task peadm::pe_install on ${master_target[0].name}")
-    run_task('peadm::pe_install', $master_target,
-      _catch_errors         => $shortcircuit_puppetdb,
-      tarball               => $upload_tarball_path,
-      peconf                => '/tmp/pe.conf',
-      puppet_service_ensure => 'stopped',
-      shortcircuit_puppetdb => $shortcircuit_puppetdb,
-    )
-    out::message("Finished: task peadm::pe_install on ${master_target[0].name}")
-  }
+  # Get the master installation up and running. The installer will "fail"
+  # because PuppetDB can't start, if puppetdb_database_target is set. That's
+  # expected, and handled by the task's install_extra_large parameter.
+  run_task('peadm::pe_install', $master_target,
+    tarball               => $upload_tarball_path,
+    peconf                => '/tmp/pe.conf',
+    puppet_service_ensure => 'stopped',
+    install_extra_large   => ($arch['architecture'] == 'extra-large'),
+  )
 
   parallelize($master_targets) |$target| {
     if $r10k_private_key {
@@ -312,14 +306,19 @@
     action => 'file-sync commit',
   )
 
-  parallelize($agent_installer_targets) |$target| {
+  parallelize($agent_installer_targets + $database_targets) |$target| {
     $common_install_flags = [
       '--puppet-service-ensure', 'stopped',
       "main:dns_alt_names=${dns_alt_names_csv}",
       "main:certname=${target.peadm::target_name()}",
     ]
 
-    if ($target in $compiler_a_targets) {
+    # Database targets don't need agent installed, they just need to run Puppet
+    if ($target in $database_targets) {
+      run_task('peadm::puppet_runonce', $target)
+    }
+    # Everything else needs an agent installed and cert signed
+    elsif ($target in $compiler_a_targets) {
       run_task('peadm::agent_install', $target,
         server        => $master_target.peadm::target_name(),
         install_flags => $common_install_flags + [
@@ -347,18 +346,14 @@
       )
     }
 
-    # Ensure certificate requests have been submitted
-    run_task('peadm::submit_csr', $target)
-    # TODO: come up with an intelligent way to validate that the expected CSRs
-    # have been submitted and are available for signing, prior to signing them.
-    # For now, waiting a short period of time is necessary to avoid a small race.
-    ctrl::sleep(5)
-    run_task('peadm::sign_csr', $master_target, { 'certnames' => [$target.name] } )
-    run_task('peadm::puppet_runonce', $target)
+    # Ensure certificate requests have been submitted, then run Puppet
+    unless ($target in $database_targets) {
+      run_task('peadm::submit_csr', $target)
+      run_task('peadm::sign_csr', $master_target, { 'certnames' => [$target.name] } )
+      run_task('peadm::puppet_runonce', $target)
+    }
   }
 
-  run_task('peadm::puppet_runonce', $database_targets )
-
   # The puppetserver might be in the middle of a restart after the Puppet run,
   # so we check the status by calling the api and ensuring the puppetserver is
   # taking requests before proceeding. It takes two runs to fully finish

plans/provision.pp

@@ -28,7 +28,7 @@
 
   # Common Configuration
   String                            $console_password,
-  String                            $version                          = '2019.8.1',
+  String                            $version                          = '2019.8.5',
   Optional[Array[String]]           $dns_alt_names                    = undef,
   Optional[String]                  $compiler_pool_address            = undef,
   Optional[String]                  $internal_compiler_a_pool_address = undef,

spec/docker/extra-large-ha/params.json

@@ -6,6 +6,6 @@
   "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"],
   "console_password": "puppetlabs",
   "dns_alt_names": [ "puppet", "pe-xl-core-0.puppet.vm", "puppet-xl.vm" ],
-  "version": "2019.8.0",
+  "version": "2019.8.5",
   "compiler_pool_address": "puppet-xl.vm"
 }

spec/docker/extra-large-ha/upgrade_params.json

@@ -4,5 +4,5 @@
     "puppetdb_database_replica_host": "pe-xl-db-1.puppet.vm",
     "master_replica_host": "pe-xl-core-1.puppet.vm",
     "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"],
-    "version": "2019.8.0"
+    "version": "2019.8.5"
 }

spec/docker/extra-large/params.json

@@ -4,5 +4,5 @@
   "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"],
   "console_password": "puppetlabs",
   "dns_alt_names": [ "puppet", "pe-xl-core-0.puppet.vm" ],
-  "version": "2019.8.0"
-}
+  "version": "2019.8.5"
+}

spec/docker/extra-large/upgrade_params.json

@@ -2,5 +2,5 @@
     "master_host": "pe-xl-core-0.puppet.vm",
     "puppetdb_database_host": "pe-xl-db-0.puppet.vm",
     "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"],
-    "version": "2019.8.0"
+    "version": "2019.8.5"
 }

spec/docker/large-ha/params.json

@@ -4,5 +4,5 @@
   "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"],
   "console_password": "puppetlabs",
   "dns_alt_names": [ "puppet", "pe-lg.puppet.vm" ],
-  "version": "2019.8.0"
+  "version": "2019.8.5"
 }

spec/docker/large-ha/upgrade_params.json

@@ -2,6 +2,6 @@
     "master_host": "pe-lg.puppet.vm",
     "master_replica_host": "pe-lg-replica.puppet.vm",
     "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"],
-    "version": "2019.8.0"
+    "version": "2019.8.5"
 }
 

spec/docker/large/params.json

@@ -3,5 +3,5 @@
   "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"],
   "console_password": "puppetlabs",
   "dns_alt_names": [ "puppet", "pe-lg.puppet.vm" ],
-  "version": "2019.8.0"
+  "version": "2019.8.5"
 }

spec/docker/large/upgrade_params.json

@@ -1,6 +1,6 @@
 {
     "master_host": "pe-lg.puppet.vm",
     "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"],
-    "version": "2019.8.0"
+    "version": "2019.8.5"
 }
 

spec/docker/standard-ha/params.json

@@ -3,5 +3,5 @@
   "master_replica_host": "pe-std-replica.puppet.vm",
   "console_password": "puppetlabs",
   "dns_alt_names": [ "puppet", "pe-std.puppet.vm" ],
-  "version": "2019.8.0"
+  "version": "2019.8.5"
 }

spec/docker/standard-ha/upgrade_params.json

@@ -1,6 +1,6 @@
 {
     "master_host": "pe-std.puppet.vm",
     "master_replica_host": "pe-std-replica.puppet.vm",
-    "version": "2019.8.0"
+    "version": "2019.8.5"
 }
 

spec/docker/standard/params.json

@@ -2,6 +2,6 @@
   "master_host": "pe-std.puppet.vm",
   "console_password": "puppetlabs",
   "dns_alt_names": [ "puppet", "pe-std.puppet.vm" ],
-  "version": "2019.8.0",
+  "version": "2019.8.5",
   "r10k_remote": "https://gitlab.com/nwops/control-repo.git"
 }

spec/docker/standard/upgrade_params.json

@@ -1,5 +1,5 @@
 {
     "master_host": "pe-std.puppet.vm",
-    "version": "2019.8.0"
+    "version": "2019.8.5"
 }
 

tasks/pe_install.json

@@ -9,9 +9,9 @@
       "type": "Optional[String]",
       "description": "The path to the pe.conf file"
     },
-    "shortcircuit_puppetdb": {
+    "install_extra_large": {
       "type": "Optional[Boolean]",
-      "description": "If true, during install, configure PuppetDB to short-circuit its startup"
+      "description": "If true, optimize task for known manual issues with extra-large installs. Do not use for upgrades"
     },
     "puppet_service_ensure": {
       "type": "Optional[Enum['stopped']]",

tasks/pe_install.sh

@@ -4,7 +4,7 @@
 # in situations where PuppetDB WILL fail, such as when PostgreSQL is not yet
 # configured, and we don't want to let PuppetDB wait five minutes before
 # giving up on it.
-if [ "$PT_shortcircuit_puppetdb" = "true" ]; then
+if [ "$PT_install_extra_large" = "true" ]; then
 	mkdir /etc/systemd/system/pe-puppetdb.service.d
 	cat > /etc/systemd/system/pe-puppetdb.service.d/10-shortcircuit.conf <<-EOF
 		[Service]
@@ -33,7 +33,7 @@ fi
 # The exit code of the installer script will be the exit code of the task
 exit_code=$?
 
-if [ "$PT_shortcircuit_puppetdb" = "true" ]; then
+if [ "$PT_install_extra_large" = "true" ]; then
 	systemctl stop pe-puppetdb.service
 	rm /etc/systemd/system/pe-puppetdb.service.d/10-shortcircuit.conf
 	systemctl daemon-reload
@@ -43,5 +43,16 @@ if [ "$PT_puppet_service_ensure" = "stopped" ]; then
 	systemctl stop puppet.service
 fi
 
-# Exit with the installer script's exit code
-exit $exit_code
+# In an extra large install, the installer is known to exit with code 1, even
+# on an otherwise successful install, because PuppetDB cannot start yet. The
+# task should indicate successful completion even if the exit code is 1, as
+# long as some basic "did it install?" health checks pass.
+if [ "$PT_install_extra_large" = "true" ]; then
+	for svc in pe-puppetserver pe-orchestration-services pe-console-services; do
+		systemctl is-active --quiet $svc.service || exit $exit_code
+	done
+	exit 0
+else
+	# Exit with the installer script's exit code
+	exit $exit_code
+fi

tasks/sign_csr.rb

@@ -1,33 +1,54 @@
 #!/opt/puppetlabs/puppet/bin/ruby
 # frozen_string_literal: true
 
-#
 require 'json'
 require 'open3'
 require 'puppet'
 
-def csr_signed?(certname)
-  !File.exist?(File.join(Puppet.settings[:csrdir], "#{certname}.pem")) &&
-    File.exist?(File.join(Puppet.settings[:cadir], 'signed', "#{certname}.pem"))
-end
+# Class to run and execute the `puppetserver ca sign` command as a task.
+class SignCSR
+  class SigningError; end
 
-def main
-  Puppet.initialize_settings
-  params = JSON.parse(STDIN.read)
-  unsigned = params['certnames'].reject { |name| csr_signed?(name) }
+  def initialize(params)
+    Puppet.initialize_settings
+    @certnames = params['certnames']
+  end
 
-  exit 0 if unsigned.empty?
+  def execute!
+    attempts = 0
 
-  cmd = ['/opt/puppetlabs/bin/puppetserver', 'ca', 'sign',
-         '--certname', unsigned.join(',')]
+    begin
+      unsigned = @certnames.reject { |name| csr_signed?(name) }
+      exit 0 if unsigned.empty?
+      sign(unsigned)
+    rescue SigningError
+      exit 1 if attempts > 5
+      attempts += 1
+      puts "Signing attempt #{attempts} failed; waiting 1s and trying again"
+      sleep 1
+      retry
+    end
+  end
 
-  stdout, status = Open3.capture2(*cmd)
-  puts stdout
-  if status.success?
-    exit 0
-  else
-    exit 1
+  def csr_signed?(certname)
+    !File.exist?(File.join(Puppet.settings[:csrdir], "#{certname}.pem")) &&
+      File.exist?(File.join(Puppet.settings[:cadir], 'signed', "#{certname}.pem"))
+  end
+
+  def sign(certnames)
+    cmd = ['/opt/puppetlabs/bin/puppetserver', 'ca', 'sign',
+           '--certname', certnames.join(',')]
+
+    stdout, status = Open3.capture2(*cmd)
+    puts stdout
+    raise SigningError unless status.success?
   end
 end
 
-main
+# Run the task unless an environment flag has been set, signaling not to. The
+# environment flag is used to disable auto-execution and enable Ruby unit
+# testing of this task.
+unless ENV['RSPEC_UNIT_TEST_MODE']
+  task = SignCSR.new(JSON.parse(STDIN.read))
+  task.execute!
+end