Plan steps

documentation/convert.md

@@ -1,6 +1,6 @@
 # Convert infrastructure for use with the peadm module
 
-The peadm::convert plan can be used to adopt manually deployed infrastructure for use with peadm, or to adopt infrastructure deployed with a version of peadm older than 1.0.0.
+The peadm::convert plan can be used to adopt manually deployed infrastructure for use with peadm, or to adopt infrastructure deployed with an older version of peadm.
 
 ## Convert an Existing Deployment
 
@@ -24,3 +24,9 @@ See the [provision](provision.md#reference-architectures) documentation for a li
 ```
 bolt plan run peadm::convert --params @params.json 
 ```
+
+## Retry or resume plan
+
+This plan is broken down into steps. Normally, the plan runs through all the steps from start to finish. The name of each step is displayed during the plan run, as the step begins.
+
+The `begin_at_step` parameter can be used to facilitate re-running this plan after a failed attempt, skipping past any steps that already completed successfully on the first try and picking up again at the step specified. The step name to resume at can be read from the previous run logs. A full list of available values for this parameter can be viewed by running `bolt plan show peadm::convert`.

documentation/upgrade.md

@@ -81,6 +81,12 @@ Pre-staging the installation media and using an inventory definition such as the
 bolt plan run peadm::upgrade --params @params.json 
 ```
 
+## Retry or resume plan
+
+This plan is broken down into steps. Normally, the plan runs through all the steps from start to finish. The name of each step is displayed during the plan run, as the step begins.
+
+The `begin_at_step` parameter can be used to facilitate re-running this plan after a failed attempt, skipping past any steps that already completed successfully on the first try and picking up again at the step specified. The step name to resume at can be read from the previous run logs. A full list of available values for this parameter can be viewed by running `bolt plan show peadm::upgrade`.
+
 ## Manual Upgrades
 
 In the event a manual upgrade is required, the steps may be followed along by reading directly from [the upgrade plan](../plans/upgrade.pp), which is itself the most accurate technical description of the steps required. In general form, the upgrade process is as given below.
@@ -128,5 +134,5 @@ The following steps apply _only_ if upgrading from 2019.5 or older
 
 To upgrade to PE 2019.7 or newer from PE 2018.1:
 
-1. Run the peadm::convert plan with `configure_node_groups = false`
+1. Run the peadm::convert plan
 2. Run the peadm::upgrade plan

lib/puppet/functions/peadm/plan_step.rb

@@ -0,0 +1,29 @@
+# rubocop:disable Style/Documentation
+# frozen_string_literal: true
+
+#Puppet::Functions.create_function(:'peadm::plan_step', Puppet::Functions::InternalFunction) do
+Puppet::Functions.create_function(:'peadm::plan_step', Puppet::Functions::InternalFunction) do
+  dispatch :plan_step do
+    scope_param
+    param 'String', :step_name
+    block_param
+  end
+
+  def plan_step(scope, step_name)
+    first_step = scope.bound?('begin_at_step') ? scope['begin_at_step'] : nil
+    first_step_reached = if first_step.nil? or scope.bound?('__first_plan_step_reached__')
+                           true
+                         elsif (step_name == first_step)
+                           scope['__first_plan_step_reached__'] = true
+                         else
+                           false
+                         end
+
+    if first_step_reached
+      call_function('out::message', "# Plan Step: #{step_name}")
+      yield
+    else
+      call_function('out::message', "# Plan Step: #{step_name} - SKIPPING")
+    end
+  end
+end

plans/convert.pp

@@ -15,6 +15,16 @@
   Optional[String]                  $internal_compiler_a_pool_address = undef,
   Optional[String]                  $internal_compiler_b_pool_address = undef,
   Array[String]                     $dns_alt_names                    = [ ],
+
+  Optional[Enum[
+    'convert-primary',
+    'convert-replica',
+    'convert-database',
+    'convert-database-replica',
+    'convert-compilers-a',
+    'convert-compilers-b',
+    'convert-node-groups',
+    'finalize']] $begin_at_step = undef,
 ) {
   # TODO: read and validate convertable PE version
 
@@ -42,6 +52,8 @@
     $compiler_hosts,
   )
 
+  out::message('# Gathering information')
+
   # Get trusted fact information for all compilers. Use peadm::target_name() as
   # the hash key because the apply block below will break trying to parse the
   # $compiler_extensions variable if it has Target-type hash keys.
@@ -104,109 +116,125 @@
     $compiler_b_targets = []
   }
 
-  # If PE version is older than 2019.7
-  if (versioncmp($pe_version, '2019.7.0') < 0) {
-    apply($master_target) {
-      include peadm::setup::convert_pre20197
-    }
-  }
-
   # Modify csr_attributes.yaml and insert the peadm-specific OIDs to identify
   # each server's role and availability group
 
-  run_plan('peadm::util::add_cert_extensions', $master_target,
-    master_host => $master_target,
-    extensions  => {
-      peadm::oid('peadm_role')               => 'puppet/master',
-      peadm::oid('peadm_availability_group') => 'A',
-    },
-  )
+  peadm::plan_step('convert-primary') || {
+    # If PE version is older than 2019.7
+    if (versioncmp($pe_version, '2019.7.0') < 0) {
+      apply($master_target) {
+        include peadm::setup::convert_pre20197
+      }
+    }
 
-  # If the orchestrator is in use, get certs fully straightened up before
-  # proceeding
-  if $all_targets.any |$target| { $target.protocol == 'pcp' } {
-    run_task('peadm::puppet_runonce', $master_target)
-    peadm::wait_until_service_ready('orchestrator-service', $master_target)
-    wait_until_available($all_targets, wait_time => 120)
+    run_plan('peadm::util::add_cert_extensions', $master_target,
+      master_host => $master_target,
+      extensions  => {
+        peadm::oid('peadm_role')               => 'puppet/master',
+        peadm::oid('peadm_availability_group') => 'A',
+      },
+    )
   }
 
-  run_plan('peadm::util::add_cert_extensions', $master_replica_target,
-    master_host => $master_target,
-    extensions  => {
-      peadm::oid('peadm_role')               => 'puppet/master',
-      peadm::oid('peadm_availability_group') => 'B',
-    },
-  )
+  peadm::plan_step('convert-replica') || {
+    # If the orchestrator is in use, get certs fully straightened up before
+    # proceeding
+    if $all_targets.any |$target| { $target.protocol == 'pcp' } {
+      run_task('peadm::puppet_runonce', $master_target)
+      peadm::wait_until_service_ready('orchestrator-service', $master_target)
+      wait_until_available($all_targets, wait_time => 120)
+    }
 
-  run_plan('peadm::util::add_cert_extensions', $puppetdb_database_target,
-    master_host => $master_target,
-    extensions  => {
-      peadm::oid('peadm_role')               => 'puppet/puppetdb-database',
-      peadm::oid('peadm_availability_group') => 'A',
-    },
-  )
+    run_plan('peadm::util::add_cert_extensions', $master_replica_target,
+      master_host => $master_target,
+      extensions  => {
+        peadm::oid('peadm_role')               => 'puppet/master',
+        peadm::oid('peadm_availability_group') => 'B',
+      },
+    )
+  }
 
-  run_plan('peadm::util::add_cert_extensions', $puppetdb_database_replica_target,
-    master_host => $master_target,
-    extensions  => {
-      peadm::oid('peadm_role')               => 'puppet/puppetdb-database',
-      peadm::oid('peadm_availability_group') => 'B',
-    },
-  )
+  peadm::plan_step('convert-database') || {
+    run_plan('peadm::util::add_cert_extensions', $puppetdb_database_target,
+      master_host => $master_target,
+      extensions  => {
+        peadm::oid('peadm_role')               => 'puppet/puppetdb-database',
+        peadm::oid('peadm_availability_group') => 'A',
+      },
+    )
+  }
 
-  run_plan('peadm::util::add_cert_extensions', $compiler_a_targets,
-    master_host => $master_target,
-    remove      => ['1.3.6.1.4.1.34380.1.3.13'], # OID form of pp_auth_role
-    extensions  => {
-      'pp_auth_role'                         => 'pe_compiler',
-      peadm::oid('peadm_availability_group') => 'A',
-    },
-  )
+  peadm::plan_step('convert-database-replica') || {
+    run_plan('peadm::util::add_cert_extensions', $puppetdb_database_replica_target,
+      master_host => $master_target,
+      extensions  => {
+        peadm::oid('peadm_role')               => 'puppet/puppetdb-database',
+        peadm::oid('peadm_availability_group') => 'B',
+      },
+    )
+  }
 
-  run_plan('peadm::util::add_cert_extensions', $compiler_b_targets,
-    master_host => $master_target,
-    remove      => ['1.3.6.1.4.1.34380.1.3.13'], # OID form of pp_auth_role
-    extensions  => {
-      'pp_auth_role'                         => 'pe_compiler',
-      peadm::oid('peadm_availability_group') => 'B',
-    },
-  )
+  peadm::plan_step('convert-compilers-a') || {
+    run_plan('peadm::util::add_cert_extensions', $compiler_a_targets,
+      master_host => $master_target,
+      remove      => ['1.3.6.1.4.1.34380.1.3.13'], # OID form of pp_auth_role
+      extensions  => {
+        'pp_auth_role'                         => 'pe_compiler',
+        peadm::oid('peadm_availability_group') => 'A',
+      },
+    )
+  }
 
-  # Create the necessary node groups in the console, unless the PE version is
-  # too old to support it pre-upgrade. In that circumstance, we trust that
-  # the existing groups are correct enough to function until the upgrade is
-  # performed.
-  if (versioncmp($pe_version, '2019.7.0') >= 0) {
-    apply($master_target) {
-      class { 'peadm::setup::node_manager_yaml':
-        master_host => $master_target.peadm::target_name(),
-      }
+  peadm::plan_step('convert-compilers-b') || {
+    run_plan('peadm::util::add_cert_extensions', $compiler_b_targets,
+      master_host => $master_target,
+      remove      => ['1.3.6.1.4.1.34380.1.3.13'], # OID form of pp_auth_role
+      extensions  => {
+        'pp_auth_role'                         => 'pe_compiler',
+        peadm::oid('peadm_availability_group') => 'B',
+      },
+    )
+  }
 
-      class { 'peadm::setup::node_manager':
-        master_host                      => $master_target.peadm::target_name(),
-        master_replica_host              => $master_replica_target.peadm::target_name(),
-        puppetdb_database_host           => $puppetdb_database_target.peadm::target_name(),
-        puppetdb_database_replica_host   => $puppetdb_database_replica_target.peadm::target_name(),
-        compiler_pool_address            => $compiler_pool_address,
-        internal_compiler_a_pool_address => $internal_compiler_a_pool_address,
-        internal_compiler_b_pool_address => $internal_compiler_b_pool_address,
-        require                          => Class['peadm::setup::node_manager_yaml'],
+  peadm::plan_step('convert-node-groups') || {
+    # Create the necessary node groups in the console, unless the PE version is
+    # too old to support it pre-upgrade. In that circumstance, we trust that
+    # the existing groups are correct enough to function until the upgrade is
+    # performed.
+    if (versioncmp($pe_version, '2019.7.0') >= 0) {
+      apply($master_target) {
+        class { 'peadm::setup::node_manager_yaml':
+          master_host => $master_target.peadm::target_name(),
+        }
+
+        class { 'peadm::setup::node_manager':
+          master_host                      => $master_target.peadm::target_name(),
+          master_replica_host              => $master_replica_target.peadm::target_name(),
+          puppetdb_database_host           => $puppetdb_database_target.peadm::target_name(),
+          puppetdb_database_replica_host   => $puppetdb_database_replica_target.peadm::target_name(),
+          compiler_pool_address            => $compiler_pool_address,
+          internal_compiler_a_pool_address => $internal_compiler_a_pool_address,
+          internal_compiler_b_pool_address => $internal_compiler_b_pool_address,
+          require                          => Class['peadm::setup::node_manager_yaml'],
+        }
       }
     }
-  }
-  else {
-    out::message(@("EOL"/L))
-      NOTICE: Node groups not created/updated as part of convert because PE \
-      version is too old to support them. Node groups will be updated when \
-      the peadm::upgrade plan is run.
-      | EOL
+    else {
+      out::message(@("EOL"/L))
+        NOTICE: Node groups not created/updated as part of convert because PE \
+        version is too old to support them. Node groups will be updated when \
+        the peadm::upgrade plan is run.
+        | EOL
+    }
   }
 
-  # Run Puppet on all targets to ensure catalogs and exported resources fully
-  # up-to-date. Run on master first in case puppet server restarts, 'cause
-  # that would cause the runs to fail on all the rest.
-  run_task('peadm::puppet_runonce', $master_target)
-  run_task('peadm::puppet_runonce', $all_targets - $master_target)
+  peadm::plan_step('finalize') || {
+    # Run Puppet on all targets to ensure catalogs and exported resources fully
+    # up-to-date. Run on master first in case puppet server restarts, 'cause
+    # that would cause the runs to fail on all the rest.
+    run_task('peadm::puppet_runonce', $master_target)
+    run_task('peadm::puppet_runonce', $all_targets - $master_target)
+  }
 
   return("Conversion to peadm Puppet Enterprise ${arch['architecture']} succeeded.")
 }