Merge pull request #101 from puppetlabs/upgrade-from-2018

Upgrade from 2018

Author: reidmv

documentation/upgrade.md

@@ -123,3 +123,10 @@ The following steps apply _only_ if upgrading from 2019.5 or older
 **Phase 4: resume puppet service**
 
 * Ensure the `puppet` service on all PE infrastructure nodes is running again
+
+## Upgrade from 2018.1
+
+To upgrade to PE 2019.7 or newer from PE 2018.1:
+
+1. Run the peadm::convert plan with `configure_node_groups = false`
+2. Run the peadm::upgrade plan

manifests/setup/convert_pe2018.pp

@@ -0,0 +1,26 @@
+# @summary Defines configuration needed for converting PE 2018
+#
+class peadm::setup::convert_pe2018 {
+
+  # This is needed so that compiler certs can be signed. It's included by
+  # default in 2019.7 and newer, but isn't present in 2018.1. It would be
+  # preferable to use the hocon_setting resource, but we can't because it
+  # requires a gem not present by default. It would be preferable to use the
+  # pe_hocon_setting resource, but we can't because there's no Forge module
+  # that provides it for Bolt to use. So this is what we are reduced to.
+  $caconf = @(EOF)
+    # CA-related settings
+    certificate-authority: {
+      allow-subject-alt-names: true
+      allow-authorization-extensions: true
+    }
+    | EOF
+
+  file { '/etc/puppetlabs/puppetserver/conf.d/ca.conf':
+    ensure  => file,
+    content => $caconf,
+    notify  => Service['pe-puppetserver'],
+  }
+
+  service { 'pe-puppetserver': }
+}

metadata.json

@@ -33,19 +33,22 @@
     {
       "operatingsystem": "RedHat",
       "operatingsystemrelease": [
-        "7"
+        "7",
+        "8"
       ]
     },
     {
       "operatingsystem": "CentOS",
       "operatingsystemrelease": [
-        "7"
+        "7",
+        "8"
       ]
     },
     {
       "operatingsystem": "Ubuntu",
       "operatingsystemrelease": [
-        "18.04"
+        "18.04",
+        "20.04"
       ]
     }
   ],

plans/convert.pp

@@ -13,6 +13,9 @@
   # Common Configuration
   String                            $compiler_pool_address = $master_host,
   Array[String]                     $dns_alt_names         = [ ],
+
+  # Options
+  Boolean                           $configure_node_groups = true,
 ) {
   # TODO: read and validate convertable PE version
 
@@ -40,6 +43,11 @@
     $compiler_hosts,
   )
 
+  # Know what version of PE the current targets are
+  $pe_version = run_task('peadm::read_file', $master_target,
+    path => '/opt/puppetlabs/server/pe_version',
+  )[0][content].chomp
+
   # Get trusted fact information for all compilers. Use peadm::target_name() as
   # the hash key because the apply block below will break trying to parse the
   # $compiler_extensions variable if it has Target-type hash keys.
@@ -71,6 +79,12 @@
     $compiler_b_targets = []
   }
 
+  if $pe_version =~ /^2018/ {
+    apply($master_target) {
+      include peadm::setup::convert_pe2018
+    }
+  }
+
   # Modify csr_attributes.yaml and insert the peadm-specific OIDs to identify
   # each server's role and availability group
 
@@ -82,6 +96,14 @@
     },
   )
 
+  # If the orchestrator is in use, get certs fully straightened up before
+  # proceeding
+  if $all_targets.any |$target| { $target.protocol == 'pcp' } {
+    run_task('peadm::puppet_runonce', $master_target)
+    peadm::wait_until_service_ready('orchestrator-service', $master_target)
+    wait_until_available($all_targets, wait_time => 120)
+  }
+
   run_plan('peadm::util::add_cert_extensions', $master_replica_target,
     master_host => $master_target,
     extensions  => {
@@ -126,18 +148,20 @@
 
   # Create the necessary node groups in the console
 
-  apply($master_target) {
-    class { 'peadm::setup::node_manager_yaml':
-      master_host => $master_target.peadm::target_name(),
-    }
+  if $configure_node_groups {
+    apply($master_target) {
+      class { 'peadm::setup::node_manager_yaml':
+        master_host => $master_target.peadm::target_name(),
+      }
 
-    class { 'peadm::setup::node_manager':
-      master_host                    => $master_target.peadm::target_name(),
-      master_replica_host            => $master_replica_target.peadm::target_name(),
-      puppetdb_database_host         => $puppetdb_database_target.peadm::target_name(),
-      puppetdb_database_replica_host => $puppetdb_database_replica_target.peadm::target_name(),
-      compiler_pool_address          => $compiler_pool_address,
-      require                        => Class['peadm::setup::node_manager_yaml'],
+      class { 'peadm::setup::node_manager':
+        master_host                    => $master_target.peadm::target_name(),
+        master_replica_host            => $master_replica_target.peadm::target_name(),
+        puppetdb_database_host         => $puppetdb_database_target.peadm::target_name(),
+        puppetdb_database_replica_host => $puppetdb_database_replica_target.peadm::target_name(),
+        compiler_pool_address          => $compiler_pool_address,
+        require                        => Class['peadm::setup::node_manager_yaml'],
+      }
     }
   }
 

plans/upgrade.pp

@@ -66,7 +66,9 @@
   ).first['content']
 
   # Ensure needed trusted facts are available
-  if $cert_extensions.any |$t,$ext| { $ext[peadm::oid('peadm_role')] == undef } {
+  if $cert_extensions.any |$_,$cert| {
+    [peadm::oid('peadm_role'), 'pp_auth_role'].all |$ext| { $cert[$ext] == undef }
+  } {
     fail_plan(@(HEREDOC/L))
       Required trusted facts are not present; upgrade cannot be completed. If \
       this infrastructure was provisioned with an old version of peadm, you may \
@@ -114,21 +116,18 @@
     )
   }
 
-  # Shut down Puppet on all infra targets
-  run_task('service', $all_targets,
-    action => 'stop',
-    name   => 'puppet',
-  )
+  # Shut down Puppet on all infra targets. Avoid using the built-in service
+  # task for idempotency reasons. When the orchestrator has been upgraded but
+  # not all pxp-agents have, the built-in service task does not work over pcp.
+  run_command('systemctl stop puppet', $all_targets)
 
   ###########################################################################
   # UPGRADE MASTER SIDE
   ###########################################################################
 
-  # Shut down PuppetDB on CMs that use the PM's PDB PG
-  run_task('service', $compiler_m1_targets,
-    action => 'stop',
-    name   => 'pe-puppetdb',
-  )
+  # Shut down PuppetDB on CMs that use the PM's PDB PG. Use run_command instead
+  # of run_task(service, ...) so that upgrading from 2018.1 works over PCP.
+  run_command('systemctl stop pe-puppetdb', $compiler_m1_targets)
 
   run_task('peadm::pe_install', $puppetdb_database_target,
     tarball               => $upload_tarball_path,
@@ -160,6 +159,31 @@
     wait_until_available($all_targets, wait_time => 120)
   }
 
+  # If necessary, add missing cert extensions to compilers
+  run_plan('peadm::util::add_cert_extensions', $convert_targets,
+    master_host => $master_target,
+    extensions  => {
+      'pp_auth_role' => 'pe_compiler',
+    },
+  )
+
+  # Update classification. This needs to be done now because if we don't, and
+  # the PE Compiler node groups are wrong, then the compilers won't be able to
+  # successfully classify and update
+  apply($master_target) {
+    class { 'peadm::setup::node_manager_yaml':
+      master_host => $master_target.peadm::target_name(),
+    }
+
+    class { 'peadm::setup::node_manager':
+      master_host                    => $master_target.peadm::target_name(),
+      master_replica_host            => $master_replica_target.peadm::target_name(),
+      puppetdb_database_host         => $puppetdb_database_target.peadm::target_name(),
+      puppetdb_database_replica_host => $puppetdb_database_replica_target.peadm::target_name(),
+      require                        => Class['peadm::setup::node_manager_yaml'],
+    }
+  }
+
   # Upgrade the compiler group A targets
   run_task('peadm::puppet_infra_upgrade', $master_target,
     type    => 'compiler',
@@ -170,11 +194,10 @@
   # UPGRADE REPLICA SIDE
   ###########################################################################
 
-  # Shut down PuppetDB on compilers that use the replica's PDB PG
-  run_task('service', $compiler_m2_targets,
-    action => 'stop',
-    name   => 'pe-puppetdb',
-  )
+  # Shut down PuppetDB on CMs that use the replica's PDB PG. Use run_command
+  # instead of run_task(service, ...) so that upgrading from 2018.1 works
+  # over PCP.
+  run_command('systemctl stop pe-puppetdb', $compiler_m2_targets)
 
   run_task('peadm::pe_install', $puppetdb_database_replica_target,
     tarball               => $upload_tarball_path,
@@ -208,27 +231,6 @@
   # FINALIZE UPGRADE
   ###########################################################################
 
-  run_plan('peadm::util::add_cert_extensions', $convert_targets,
-    master_host => $master_target,
-    extensions  => {
-      'pp_auth_role' => 'pe_compiler',
-    },
-  )
-
-  apply($master_target) {
-    class { 'peadm::setup::node_manager_yaml':
-      master_host => $master_target.peadm::target_name(),
-    }
-
-    class { 'peadm::setup::node_manager':
-      master_host                    => $master_target.peadm::target_name(),
-      master_replica_host            => $master_replica_target.peadm::target_name(),
-      puppetdb_database_host         => $puppetdb_database_target.peadm::target_name(),
-      puppetdb_database_replica_host => $puppetdb_database_replica_target.peadm::target_name(),
-      require                        => Class['peadm::setup::node_manager_yaml'],
-    }
-  }
-
   # Ensure Puppet running on all infrastructure targets
   run_task('service', $all_targets,
     action => 'start',

plans/util/add_cert_extensions.pp

@@ -39,6 +39,7 @@
   # Loop through and recert each target one at at time, because Bolt lacks
   # real parallelism
   $all_targets.map |$target| {
+    $certname = $certdata[$target]['certname']
 
     # This will be the new trusted fact data for this node
     $extension_requests = $certdata[$target]['extensions'] + $extensions
@@ -49,25 +50,34 @@
       merge              => false,
     )
 
-    # Everything starts the same; we always revoke the existing cert
-    run_command("${pserver} ca clean --certname ${certdata[$target]['certname']}", $master_target)
+    # Everything starts the same; we always stop the agent and revoke the
+    # existing cert. We use `run_command` in case the master is 2019.x but
+    # the agent is only 2018.x. In that scenario `run_task(service, ...)`
+    # doesn't work.
+    $was_running = run_command('systemctl is-active puppet.service', $target, _catch_errors => true)[0].ok
+    if ($was_running) { run_command('systemctl stop puppet.service', $target) }
+    run_command("${pserver} ca clean --certname ${certname}", $master_target)
 
     # Then things get crazy...
 
-    # The procedure for regenerating an agent's cert
-    if ($certdata[$target]['certname'] != $master_certname) {
-      run_command("${puppet} ssl clean --certname ${certdata[$target]['certname']}", $target)
-      run_command("${puppet} ssl submit_request --certname ${certdata[$target]['certname']}", $target)
+    if ($certname != $master_certname) {
+      # AGENT cert regeneration
+      run_task('peadm::ssl_clean', $target, certname => $certname)
+      run_task('peadm::submit_csr', $target)
       ctrl::sleep(2) # some lag sometimes before the cert is available to sign
-      run_command(@("HEREDOC"/L), $master_target)
-        ${pserver} ca sign --certname ${certdata[$target]['certname']} || \
-        ${pserver} ca list --certname ${certdata[$target]['certname']} \
+      run_task('peadm::sign_csr', $master_target, certnames => [$certname])
+
+      # Use a command instead of a task so that this works for Puppet 5 agents
+      # w/ PCP transport. If using a task, we run into problems downloading
+      # the task file at this point, because there is no longer a cert file
+      # present on the agent.
+      run_command(@("HEREDOC"/L), $target)
+        ${puppet} ssl download_cert --certname ${certname} || \
+        ${puppet} certificate find --ca-location remote ${certname}
         | HEREDOC
-      run_command("${puppet} ssl download_cert --certname ${certdata[$target]['certname']}", $target)
     }
-
-    # The procedure for regenerating the master's cert
     else {
+      # MASTER cert regeneration
       # Store the node's current dns-alt-names, for use as a flag restoring
       # them later
       $alt_names_flag = $certdata[$target]['dns-alt-names'] ? {
@@ -76,22 +86,25 @@
       }
 
       # The docs are broken, and the process is unclean. Sadface.
+      run_task('service', $target, {action => 'stop', name => 'pe-puppetserver'})
       run_command(@("HEREDOC"/L), $target)
         rm -f \
-          /etc/puppetlabs/puppet/ssl/certs/${certdata[$target]['certname']}.pem \
-          /etc/puppetlabs/puppet/ssl/private_keys/${certdata[$target]['certname']}.pem \
-          /etc/puppetlabs/puppet/ssl/public_keys/${certdata[$target]['certname']}.pem \
-          /etc/puppetlabs/puppet/ssl/certificate_requests/${certdata[$target]['certname']}.pem \
+          /etc/puppetlabs/puppet/ssl/certs/${certname}.pem \
+          /etc/puppetlabs/puppet/ssl/private_keys/${certname}.pem \
+          /etc/puppetlabs/puppet/ssl/public_keys/${certname}.pem \
+          /etc/puppetlabs/puppet/ssl/certificate_requests/${certname}.pem \
         | HEREDOC
-      run_task('service', $target, {action => 'stop', name => 'pe-puppetserver'})
       run_command(@("HEREDOC"/L), $target)
         ${pserver} ca generate \
-          --certname ${certdata[$target]['certname']} \
+          --certname ${certname} \
           ${alt_names_flag} \
           --ca-client \
         | HEREDOC
       run_task('service', $target, {action => 'start', name => 'pe-puppetserver'})
     }
+
+    # Fire puppet back up when done
+    if ($was_running) { run_command('systemctl start puppet.service', $target) }
   }
 
   run_command("${puppet} facts upload", $all_targets)

tasks/ssl_clean.json

@@ -0,0 +1,10 @@
+{
+  "description": "Clean an agent's certificate",
+  "parameters": {
+    "certname": {
+      "type": "String",
+      "description": "The certname to clean"
+    }
+  },
+  "input_method": "stdin"
+}

tasks/ssl_clean.rb

@@ -0,0 +1,29 @@
+#!/opt/puppetlabs/puppet/bin/ruby
+# frozen_string_literal: true
+
+require 'puppet'
+require 'open3'
+
+def main
+  certname = JSON.parse(STDIN.read)['certname']
+  majver = Gem::Version.new(Puppet.version).segments.first
+  if majver < 6
+    puts "Deleting #{certname}.pem files..."
+    Dir.glob("/etc/puppetlabs/puppet/ssl/**/#{certname}.pem").each do |file|
+      File.delete(file)
+    end
+    puts 'Done'
+    exit 0
+  else
+    cmd = ['/opt/puppetlabs/bin/puppet', 'ssl', 'clean', '--certname', certname]
+    stdout, status = Open3.capture2(*cmd)
+    puts stdout
+    if status.success?
+      exit 0
+    else
+      exit 1
+    end
+  end
+end
+
+main