diff --git a/CHANGELOG.md b/CHANGELOG.md index fc7860f0..cafe1e6a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,15 @@ # PEADM module +## Unreleased +### Summary + +Unreleased + +### Changes + +- Improve error handling during early installation of PE +- Implement concurrency in peadm::action::install to increase speed of installation process + ## 2.5.0 ### Summary diff --git a/README.md b/README.md index 0bbc3cc1..a6a22e54 100644 --- a/README.md +++ b/README.md @@ -50,7 +50,7 @@ The normal usage pattern for peadm is as follows. ### Requirements * Puppet Enterprise 2019.8.1 or newer (tested with PE 2021.0) -* Bolt 2.27.0 or newer (tested with Bolt 3.5) +* Bolt 2.42.0 or newer (tested with Bolt 3.5.0) * EL 7, EL 8, Ubuntu 18.04, or Ubuntu 20.04 * Classifier Data enabled. This PE feature is enabled by default on new installs, but can be disabled by users if they remove the relevant configuration from their global hiera.yaml file. See the [PE docs](https://puppet.com/docs/pe/latest/config_console.html#task-5039) for more information. diff --git a/plans/action/install.pp b/plans/action/install.pp index c4511d55..da24961a 100644 --- a/plans/action/install.pp +++ b/plans/action/install.pp @@ -32,7 +32,7 @@ # Common Configuration String $console_password, - String $version = '2019.7.0', + String $version = '2019.8.5', Array[String] $dns_alt_names = [ ], Hash $pe_conf_data = { }, @@ -235,21 +235,15 @@ } } - # Get the master installation up and running. The installer will - # "fail" because PuppetDB can't start, if puppetdb_database_target - # is set. That's expected. - $shortcircuit_puppetdb = !($puppetdb_database_target.empty) - without_default_logging() || { - out::message("Starting: task peadm::pe_install on ${master_target[0].name}") - run_task('peadm::pe_install', $master_target, - _catch_errors => $shortcircuit_puppetdb, - tarball => $upload_tarball_path, - peconf => '/tmp/pe.conf', - puppet_service_ensure => 'stopped', - shortcircuit_puppetdb => $shortcircuit_puppetdb, - ) - out::message("Finished: task peadm::pe_install on ${master_target[0].name}") - } + # Get the master installation up and running. The installer will "fail" + # because PuppetDB can't start, if puppetdb_database_target is set. That's + # expected, and handled by the task's install_extra_large parameter. + run_task('peadm::pe_install', $master_target, + tarball => $upload_tarball_path, + peconf => '/tmp/pe.conf', + puppet_service_ensure => 'stopped', + install_extra_large => ($arch['architecture'] == 'extra-large'), + ) parallelize($master_targets) |$target| { if $r10k_private_key { @@ -312,14 +306,19 @@ action => 'file-sync commit', ) - parallelize($agent_installer_targets) |$target| { + parallelize($agent_installer_targets + $database_targets) |$target| { $common_install_flags = [ '--puppet-service-ensure', 'stopped', "main:dns_alt_names=${dns_alt_names_csv}", "main:certname=${target.peadm::target_name()}", ] - if ($target in $compiler_a_targets) { + # Database targets don't need agent installed, they just need to run Puppet + if ($target in $database_targets) { + run_task('peadm::puppet_runonce', $target) + } + # Everything else needs an agent installed and cert signed + elsif ($target in $compiler_a_targets) { run_task('peadm::agent_install', $target, server => $master_target.peadm::target_name(), install_flags => $common_install_flags + [ @@ -347,18 +346,14 @@ ) } - # Ensure certificate requests have been submitted - run_task('peadm::submit_csr', $target) - # TODO: come up with an intelligent way to validate that the expected CSRs - # have been submitted and are available for signing, prior to signing them. - # For now, waiting a short period of time is necessary to avoid a small race. - ctrl::sleep(5) - run_task('peadm::sign_csr', $master_target, { 'certnames' => [$target.name] } ) - run_task('peadm::puppet_runonce', $target) + # Ensure certificate requests have been submitted, then run Puppet + unless ($target in $database_targets) { + run_task('peadm::submit_csr', $target) + run_task('peadm::sign_csr', $master_target, { 'certnames' => [$target.name] } ) + run_task('peadm::puppet_runonce', $target) + } } - run_task('peadm::puppet_runonce', $database_targets ) - # The puppetserver might be in the middle of a restart after the Puppet run, # so we check the status by calling the api and ensuring the puppetserver is # taking requests before proceeding. It takes two runs to fully finish diff --git a/plans/provision.pp b/plans/provision.pp index 8841b804..885d9a67 100644 --- a/plans/provision.pp +++ b/plans/provision.pp @@ -28,7 +28,7 @@ # Common Configuration String $console_password, - String $version = '2019.8.1', + String $version = '2019.8.5', Optional[Array[String]] $dns_alt_names = undef, Optional[String] $compiler_pool_address = undef, Optional[String] $internal_compiler_a_pool_address = undef, diff --git a/spec/docker/extra-large-ha/params.json b/spec/docker/extra-large-ha/params.json index 30af7396..ff6b6bb3 100644 --- a/spec/docker/extra-large-ha/params.json +++ b/spec/docker/extra-large-ha/params.json @@ -6,6 +6,6 @@ "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], "console_password": "puppetlabs", "dns_alt_names": [ "puppet", "pe-xl-core-0.puppet.vm", "puppet-xl.vm" ], - "version": "2019.8.0", + "version": "2019.8.5", "compiler_pool_address": "puppet-xl.vm" } diff --git a/spec/docker/extra-large-ha/upgrade_params.json b/spec/docker/extra-large-ha/upgrade_params.json index 3d4a7bde..5d1facf4 100644 --- a/spec/docker/extra-large-ha/upgrade_params.json +++ b/spec/docker/extra-large-ha/upgrade_params.json @@ -4,5 +4,5 @@ "puppetdb_database_replica_host": "pe-xl-db-1.puppet.vm", "master_replica_host": "pe-xl-core-1.puppet.vm", "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], - "version": "2019.8.0" + "version": "2019.8.5" } diff --git a/spec/docker/extra-large/params.json b/spec/docker/extra-large/params.json index ce47c03f..24663e48 100644 --- a/spec/docker/extra-large/params.json +++ b/spec/docker/extra-large/params.json @@ -4,5 +4,5 @@ "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], "console_password": "puppetlabs", "dns_alt_names": [ "puppet", "pe-xl-core-0.puppet.vm" ], - "version": "2019.8.0" -} \ No newline at end of file + "version": "2019.8.5" +} diff --git a/spec/docker/extra-large/upgrade_params.json b/spec/docker/extra-large/upgrade_params.json index c2db571f..e7c1e007 100644 --- a/spec/docker/extra-large/upgrade_params.json +++ b/spec/docker/extra-large/upgrade_params.json @@ -2,5 +2,5 @@ "master_host": "pe-xl-core-0.puppet.vm", "puppetdb_database_host": "pe-xl-db-0.puppet.vm", "compiler_hosts": ["pe-xl-compiler-0.puppet.vm"], - "version": "2019.8.0" + "version": "2019.8.5" } diff --git a/spec/docker/large-ha/params.json b/spec/docker/large-ha/params.json index f994f171..6de502f9 100644 --- a/spec/docker/large-ha/params.json +++ b/spec/docker/large-ha/params.json @@ -4,5 +4,5 @@ "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], "console_password": "puppetlabs", "dns_alt_names": [ "puppet", "pe-lg.puppet.vm" ], - "version": "2019.8.0" + "version": "2019.8.5" } diff --git a/spec/docker/large-ha/upgrade_params.json b/spec/docker/large-ha/upgrade_params.json index 0286259b..bc6ebe27 100644 --- a/spec/docker/large-ha/upgrade_params.json +++ b/spec/docker/large-ha/upgrade_params.json @@ -2,6 +2,6 @@ "master_host": "pe-lg.puppet.vm", "master_replica_host": "pe-lg-replica.puppet.vm", "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], - "version": "2019.8.0" + "version": "2019.8.5" } \ No newline at end of file diff --git a/spec/docker/large/params.json b/spec/docker/large/params.json index 1b1d8fca..0ca27476 100644 --- a/spec/docker/large/params.json +++ b/spec/docker/large/params.json @@ -3,5 +3,5 @@ "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], "console_password": "puppetlabs", "dns_alt_names": [ "puppet", "pe-lg.puppet.vm" ], - "version": "2019.8.0" + "version": "2019.8.5" } diff --git a/spec/docker/large/upgrade_params.json b/spec/docker/large/upgrade_params.json index 61490469..1be64d18 100644 --- a/spec/docker/large/upgrade_params.json +++ b/spec/docker/large/upgrade_params.json @@ -1,6 +1,6 @@ { "master_host": "pe-lg.puppet.vm", "compiler_hosts": ["pe-lg-compiler-0.puppet.vm"], - "version": "2019.8.0" + "version": "2019.8.5" } \ No newline at end of file diff --git a/spec/docker/standard-ha/params.json b/spec/docker/standard-ha/params.json index 75135779..cea1a60f 100644 --- a/spec/docker/standard-ha/params.json +++ b/spec/docker/standard-ha/params.json @@ -3,5 +3,5 @@ "master_replica_host": "pe-std-replica.puppet.vm", "console_password": "puppetlabs", "dns_alt_names": [ "puppet", "pe-std.puppet.vm" ], - "version": "2019.8.0" + "version": "2019.8.5" } diff --git a/spec/docker/standard-ha/upgrade_params.json b/spec/docker/standard-ha/upgrade_params.json index c6d3d59e..db99cedd 100644 --- a/spec/docker/standard-ha/upgrade_params.json +++ b/spec/docker/standard-ha/upgrade_params.json @@ -1,6 +1,6 @@ { "master_host": "pe-std.puppet.vm", "master_replica_host": "pe-std-replica.puppet.vm", - "version": "2019.8.0" + "version": "2019.8.5" } \ No newline at end of file diff --git a/spec/docker/standard/params.json b/spec/docker/standard/params.json index bbfe994e..371f05a0 100644 --- a/spec/docker/standard/params.json +++ b/spec/docker/standard/params.json @@ -2,6 +2,6 @@ "master_host": "pe-std.puppet.vm", "console_password": "puppetlabs", "dns_alt_names": [ "puppet", "pe-std.puppet.vm" ], - "version": "2019.8.0", + "version": "2019.8.5", "r10k_remote": "https://gitlab.com/nwops/control-repo.git" } diff --git a/spec/docker/standard/upgrade_params.json b/spec/docker/standard/upgrade_params.json index c5a4434d..9c78a3f3 100644 --- a/spec/docker/standard/upgrade_params.json +++ b/spec/docker/standard/upgrade_params.json @@ -1,5 +1,5 @@ { "master_host": "pe-std.puppet.vm", - "version": "2019.8.0" + "version": "2019.8.5" } \ No newline at end of file diff --git a/tasks/pe_install.json b/tasks/pe_install.json index 17f77974..454200ac 100644 --- a/tasks/pe_install.json +++ b/tasks/pe_install.json @@ -9,9 +9,9 @@ "type": "Optional[String]", "description": "The path to the pe.conf file" }, - "shortcircuit_puppetdb": { + "install_extra_large": { "type": "Optional[Boolean]", - "description": "If true, during install, configure PuppetDB to short-circuit its startup" + "description": "If true, optimize task for known manual issues with extra-large installs. Do not use for upgrades" }, "puppet_service_ensure": { "type": "Optional[Enum['stopped']]", diff --git a/tasks/pe_install.sh b/tasks/pe_install.sh index 0da24df8..6288cec7 100755 --- a/tasks/pe_install.sh +++ b/tasks/pe_install.sh @@ -4,7 +4,7 @@ # in situations where PuppetDB WILL fail, such as when PostgreSQL is not yet # configured, and we don't want to let PuppetDB wait five minutes before # giving up on it. -if [ "$PT_shortcircuit_puppetdb" = "true" ]; then +if [ "$PT_install_extra_large" = "true" ]; then mkdir /etc/systemd/system/pe-puppetdb.service.d cat > /etc/systemd/system/pe-puppetdb.service.d/10-shortcircuit.conf <<-EOF [Service] @@ -33,7 +33,7 @@ fi # The exit code of the installer script will be the exit code of the task exit_code=$? -if [ "$PT_shortcircuit_puppetdb" = "true" ]; then +if [ "$PT_install_extra_large" = "true" ]; then systemctl stop pe-puppetdb.service rm /etc/systemd/system/pe-puppetdb.service.d/10-shortcircuit.conf systemctl daemon-reload @@ -43,5 +43,16 @@ if [ "$PT_puppet_service_ensure" = "stopped" ]; then systemctl stop puppet.service fi -# Exit with the installer script's exit code -exit $exit_code +# In an extra large install, the installer is known to exit with code 1, even +# on an otherwise successful install, because PuppetDB cannot start yet. The +# task should indicate successful completion even if the exit code is 1, as +# long as some basic "did it install?" health checks pass. +if [ "$PT_install_extra_large" = "true" ]; then + for svc in pe-puppetserver pe-orchestration-services pe-console-services; do + systemctl is-active --quiet $svc.service || exit $exit_code + done + exit 0 +else + # Exit with the installer script's exit code + exit $exit_code +fi diff --git a/tasks/sign_csr.rb b/tasks/sign_csr.rb index 45357242..bb7dc818 100755 --- a/tasks/sign_csr.rb +++ b/tasks/sign_csr.rb @@ -1,33 +1,54 @@ #!/opt/puppetlabs/puppet/bin/ruby # frozen_string_literal: true -# require 'json' require 'open3' require 'puppet' -def csr_signed?(certname) - !File.exist?(File.join(Puppet.settings[:csrdir], "#{certname}.pem")) && - File.exist?(File.join(Puppet.settings[:cadir], 'signed', "#{certname}.pem")) -end +# Class to run and execute the `puppetserver ca sign` command as a task. +class SignCSR + class SigningError; end -def main - Puppet.initialize_settings - params = JSON.parse(STDIN.read) - unsigned = params['certnames'].reject { |name| csr_signed?(name) } + def initialize(params) + Puppet.initialize_settings + @certnames = params['certnames'] + end - exit 0 if unsigned.empty? + def execute! + attempts = 0 - cmd = ['/opt/puppetlabs/bin/puppetserver', 'ca', 'sign', - '--certname', unsigned.join(',')] + begin + unsigned = @certnames.reject { |name| csr_signed?(name) } + exit 0 if unsigned.empty? + sign(unsigned) + rescue SigningError + exit 1 if attempts > 5 + attempts += 1 + puts "Signing attempt #{attempts} failed; waiting 1s and trying again" + sleep 1 + retry + end + end - stdout, status = Open3.capture2(*cmd) - puts stdout - if status.success? - exit 0 - else - exit 1 + def csr_signed?(certname) + !File.exist?(File.join(Puppet.settings[:csrdir], "#{certname}.pem")) && + File.exist?(File.join(Puppet.settings[:cadir], 'signed', "#{certname}.pem")) + end + + def sign(certnames) + cmd = ['/opt/puppetlabs/bin/puppetserver', 'ca', 'sign', + '--certname', certnames.join(',')] + + stdout, status = Open3.capture2(*cmd) + puts stdout + raise SigningError unless status.success? end end -main +# Run the task unless an environment flag has been set, signaling not to. The +# environment flag is used to disable auto-execution and enable Ruby unit +# testing of this task. +unless ENV['RSPEC_UNIT_TEST_MODE'] + task = SignCSR.new(JSON.parse(STDIN.read)) + task.execute! +end