From 24e21cf99226fc6f554eaa1b13a98462f7aa9852 Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Wed, 19 Feb 2020 08:06:59 -0800 Subject: [PATCH 1/8] Fixed condition preventing compiler group B from being created. --- manifests/setup/node_manager.pp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 2d0d6dc8..846f227a 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -152,7 +152,9 @@ }, }, } + } + if $pupppetdb_database_replica_host or $master_replica_host { node_group { 'PE Compiler Group B': ensure => 'present', parent => 'PE Master', From 8bc32ac2baf0018f8a031181378264012745de6d Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Wed, 19 Feb 2020 08:08:01 -0800 Subject: [PATCH 2/8] Correct typo. --- manifests/setup/node_manager.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 846f227a..645595c5 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -154,7 +154,7 @@ } } - if $pupppetdb_database_replica_host or $master_replica_host { + if ($puppetdb_database_replica_host or $master_replica_host) { node_group { 'PE Compiler Group B': ensure => 'present', parent => 'PE Master', From b2af8caa7329150342cea81aa50e1bba15b095f9 Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Wed, 19 Feb 2020 09:02:27 -0800 Subject: [PATCH 3/8] Conditional group B if there is a db replica or just a master replica. --- manifests/setup/node_manager.pp | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 645595c5..2255c7bd 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -154,7 +154,7 @@ } } - if ($puppetdb_database_replica_host or $master_replica_host) { + if ($puppetdb_database_replica_host) { node_group { 'PE Compiler Group B': ensure => 'present', parent => 'PE Master', @@ -173,6 +173,25 @@ }, data => $compiler_data, } + } elsif ($master_replica_host) { + node_group { 'PE Compiler Group B': + ensure => 'present', + parent => 'PE Master', + rule => ['and', + ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/compiler'], + ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], + ], + classes => { + 'puppet_enterprise::profile::puppetdb' => { + 'database_host' => $puppetdb_database_host, + }, + 'puppet_enterprise::profile::master' => { + 'puppetdb_host' => ['${clientcert}', $master_host], # lint:ignore:single_quote_string_with_variables + 'puppetdb_port' => [8081], + } + }, + data => $compiler_data, + } } } From 3ca4d71d5703c9a8524e02ef6c4edfa69d37e08f Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Thu, 5 Mar 2020 11:36:24 -0700 Subject: [PATCH 4/8] First pass add in varialbles for pp_application and pp_cluster --- manifests/setup/node_manager.pp | 27 ++++++++++++++++----------- plans/action/configure.pp | 10 ++++++++++ plans/action/install.pp | 29 +++++++++++++++++------------ plans/provision.pp | 15 +++++++++++++++ 4 files changed, 58 insertions(+), 23 deletions(-) diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index acb3aa29..a8033a9e 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -15,6 +15,11 @@ String[1] $master_host, String[1] $puppetdb_database_host, String[1] $compiler_pool_address, + String[1] $pp_application_compiler, + String[1] $pp_application_master, + String[1] $pp_application_puppetdb, + String[1] $pp_cluster_a, + String[1] $pp_cluster_b, Optional[String[1]] $master_replica_host = undef, Optional[String[1]] $puppetdb_database_replica_host = undef, @@ -39,7 +44,7 @@ # We modify this group's rule such that all PE infrastructure nodes will be # members. node_group { 'PE Infrastructure Agent': - rule => ['and', ['~', ['trusted', 'extensions', 'pp_application'], '^puppet/']], + rule => ['and', ['~', ['trusted', 'extensions', 'pp_application'], "^${pp_application_compiler}/"]], } # We modify this group to add, as data, the compiler_pool_address only. @@ -48,7 +53,7 @@ node_group { 'PE Master': parent => 'PE Infrastructure', rule => ['or', - ['and', ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/compiler']], + ['and', ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_compiler]], ['=', 'name', $master_host], ], data => { @@ -67,7 +72,7 @@ parent => 'PE Infrastructure', environment => 'production', override_environment => false, - rule => ['and', ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/puppetdb-database']], + rule => ['and', ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_puppetdb]], classes => { 'puppet_enterprise::profile::database' => { }, }, @@ -80,8 +85,8 @@ ensure => present, parent => 'PE Infrastructure', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/master'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'A'], + ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_master], + ['=', ['trusted', 'extensions', 'pp_cluster'], $pp_cluster_a], ], data => { 'puppet_enterprise::profile::primary_master_replica' => { @@ -99,8 +104,8 @@ ensure => 'present', parent => 'PE Master', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/compiler'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'A'], + ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_compiler], + ['=', ['trusted', 'extensions', 'pp_cluster'], $pp_cluster_a], ], classes => { 'puppet_enterprise::profile::puppetdb' => { @@ -136,8 +141,8 @@ ensure => present, parent => 'PE Infrastructure', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/master'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], + ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_master], + ['=', ['trusted', 'extensions', 'pp_cluster'], $pp_cluster_b], ], data => { 'puppet_enterprise::profile::primary_master_replica' => { @@ -153,8 +158,8 @@ ensure => 'present', parent => 'PE Master', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/compiler'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], + ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_compiler], + ['=', ['trusted', 'extensions', 'pp_cluster'], $pp_cluster_b], ], classes => { 'puppet_enterprise::profile::puppetdb' => { diff --git a/plans/action/configure.pp b/plans/action/configure.pp index 61c283d7..b8d10db2 100644 --- a/plans/action/configure.pp +++ b/plans/action/configure.pp @@ -19,6 +19,11 @@ # Other String $stagingdir = '/tmp', + String $pp_application_compiler = 'puppet/compiler', + String $pp_application_master = 'puppet/master', + String $pp_application_puppetdb = 'puppet/puppetdb-database', + String $pp_cluster_a = 'A', + String $pp_cluster_b = 'B', ) { # Convert inputs into targets. $master_target = peadm::get_targets($master_host, 1) @@ -89,6 +94,11 @@ puppetdb_database_host => $puppetdb_database_host_string, # $puppetdb_database_target.peadm::target_name(), puppetdb_database_replica_host => $puppetdb_database_replica_host_string, # $puppetdb_database_replica_target.peadm::target_name(), compiler_pool_address => $compiler_pool_address, + pp_application_compiler => $pp_application_compiler, + pp_application_master => $pp_application_master, + pp_application_puppetdb => $pp_application_puppetdb, + pp_cluster_a => $pp_cluster_a, + pp_cluster_b => $pp_cluster_b, require => File['node_manager.yaml'], } } diff --git a/plans/action/install.pp b/plans/action/install.pp index 59d0621c..16513dcb 100644 --- a/plans/action/install.pp +++ b/plans/action/install.pp @@ -47,6 +47,11 @@ # Other String $stagingdir = '/tmp', + String $pp_application_compiler = 'puppet/compiler', + String $pp_application_master = 'puppet/master', + String $pp_application_puppetdb = 'puppet/puppetdb-database', + String $pp_cluster_a = 'A', + String $pp_cluster_b = 'B', ) { # Convert inputs into targets. $master_target = peadm::get_targets($master_host, 1) @@ -175,8 +180,8 @@ content => @("HEREDOC"), --- extension_requests: - ${pp_application}: "puppet/master" - ${pp_cluster}: "A" + ${pp_application}: "${pp_application_master}" + ${pp_cluster}: "${pp_cluster_a}" | HEREDOC ) @@ -185,8 +190,8 @@ content => @("HEREDOC"), --- extension_requests: - ${pp_application}: "puppet/puppetdb-database" - ${pp_cluster}: "A" + ${pp_application}: "${pp_application_puppetdb}" + ${pp_cluster}: "${pp_cluster_a}" | HEREDOC ) @@ -195,8 +200,8 @@ content => @("HEREDOC"), --- extension_requests: - ${pp_application}: "puppet/puppetdb-database" - ${pp_cluster}: "B" + ${pp_application}: "${pp_application_puppetdb}" + ${pp_cluster}: "${pp_cluster_b}" | HEREDOC ) @@ -291,8 +296,8 @@ install_flags => [ '--puppet-service-ensure', 'stopped', "main:dns_alt_names=${dns_alt_names_csv}", - "extension_requests:${pp_application}=puppet/master", - "extension_requests:${pp_cluster}=B", + "extension_requests:${pp_application}=${pp_application_master}", + "extension_requests:${pp_cluster}=${pp_cluster_b}", ], ) @@ -301,8 +306,8 @@ install_flags => [ '--puppet-service-ensure', 'stopped', "main:dns_alt_names=${dns_alt_names_csv}", - "extension_requests:${pp_application}=puppet/compiler", - "extension_requests:${pp_cluster}=A", + "extension_requests:${pp_application}=${pp_application_compiler}", + "extension_requests:${pp_cluster}=${pp_cluster_a}", ], ) @@ -311,8 +316,8 @@ install_flags => [ '--puppet-service-ensure', 'stopped', "main:dns_alt_names=${dns_alt_names_csv}", - "extension_requests:${pp_application}=puppet/compiler", - "extension_requests:${pp_cluster}=B", + "extension_requests:${pp_application}=${pp_application_compiler}", + "extension_requests:${pp_cluster}=${pp_cluster_b}", ], ) diff --git a/plans/provision.pp b/plans/provision.pp index 3d7473a8..e2654d6e 100644 --- a/plans/provision.pp +++ b/plans/provision.pp @@ -33,6 +33,11 @@ # Other Optional[String] $stagingdir = undef, + Optional[String] $pp_application_compiler = 'puppet/compiler', + Optional[String] $pp_application_master = 'puppet/master', + Optional[String] $pp_application_puppetdb = 'puppet/puppetdb-database', + Optional[String] $pp_cluster_a = 'A', + Optional[String] $pp_cluster_b = 'B', ) { $install_result = run_plan('peadm::action::install', @@ -64,6 +69,11 @@ # Other stagingdir => $stagingdir, + pp_application_compiler => $pp_application_compiler, + pp_application_master => $pp_application_master, + pp_application_puppetdb => $pp_application_puppetdb, + pp_cluster_a => $pp_cluster_a, + pp_cluster_b => $pp_cluster_b, ) $configure_result = run_plan('peadm::action::configure', @@ -84,6 +94,11 @@ # Other stagingdir => $stagingdir, + pp_application_compiler => $pp_application_compiler, + pp_application_master => $pp_application_master, + pp_application_puppetdb => $pp_application_puppetdb, + pp_cluster_a => $pp_cluster_a, + pp_cluster_b => $pp_cluster_b, ) # Return a string banner reporting on what was done From 3d63873b3f28c6b017b8e8afa6480a192718e8e9 Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Fri, 6 Mar 2020 08:03:13 -0700 Subject: [PATCH 5/8] Modify agent install script for alternate flags. --- tasks/agent_install.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tasks/agent_install.sh b/tasks/agent_install.sh index d3ad0f39..5c2fa99f 100755 --- a/tasks/agent_install.sh +++ b/tasks/agent_install.sh @@ -4,4 +4,16 @@ set -e flags=$(echo $PT_install_flags | sed -e 's/^\["//' -e 's/\"]$//' -e 's/", *"/ /g') +if [ -f /etc/puppetlabs/puppet/csr_attributes.yaml ]; then + # csr_attributes is already installed on the server, exclude those + # options from the command line flags. + read -a flags_array <<< $flags + for member in "${flags_array[@]}"; do + if [[ $member =~ ^extension ]]; then + flags_array=("${flags_array[@]/$member}") + fi + done + flags="${flags_array[@]}" +fi + curl -k "https://${PT_server}:8140/packages/current/install.bash" | bash -s -- $flags From 2263c60a634efaa40887b560e332a3e1bab1a87f Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Fri, 6 Mar 2020 08:30:39 -0700 Subject: [PATCH 6/8] Task to check for existing csr_attributes file and changes to install and node manager. --- manifests/setup/node_manager.pp | 8 ++-- plans/action/install.pp | 66 +++++++++++++++++++++------------ tasks/existing_csr_check.json | 7 ++++ tasks/existing_csr_check.sh | 10 +++++ 4 files changed, 63 insertions(+), 28 deletions(-) create mode 100644 tasks/existing_csr_check.json create mode 100644 tasks/existing_csr_check.sh diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index a8033a9e..5b9c83eb 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -81,7 +81,7 @@ # Create data-only groups to store PuppetDB PostgreSQL database configuration # information specific to the master and master replica nodes. - node_group { 'PE Master A': + node_group { "PE Master ${pp_cluster_a}": ensure => present, parent => 'PE Infrastructure', rule => ['and', @@ -100,7 +100,7 @@ # Configure the A pool for compilers. There are up to two pools for HA, each # having an affinity for one "availability zone" or the other. - node_group { 'PE Compiler Group A': + node_group { "PE Compiler Group ${pp_cluster_a}": ensure => 'present', parent => 'PE Master', rule => ['and', @@ -137,7 +137,7 @@ } if $puppetdb_database_replica_host { - node_group { 'PE Master B': + node_group { "PE Master ${pp_cluster_b}": ensure => present, parent => 'PE Infrastructure', rule => ['and', @@ -154,7 +154,7 @@ }, } - node_group { 'PE Compiler Group B': + node_group { "PE Compiler Group ${pp_cluster_b}": ensure => 'present', parent => 'PE Master', rule => ['and', diff --git a/plans/action/install.pp b/plans/action/install.pp index 16513dcb..48e051cb 100644 --- a/plans/action/install.pp +++ b/plans/action/install.pp @@ -175,35 +175,53 @@ $pp_application = '1.3.6.1.4.1.34380.1.1.8' $pp_cluster = '1.3.6.1.4.1.34380.1.1.16' - run_task('peadm::mkdir_p_file', $master_target, - path => '/etc/puppetlabs/puppet/csr_attributes.yaml', - content => @("HEREDOC"), - --- - extension_requests: - ${pp_application}: "${pp_application_master}" - ${pp_cluster}: "${pp_cluster_a}" - | HEREDOC + $master_check = run_task('peadm::existing_csr_check', + $master_target, + '_catch_errors' => true ) + if $master_check.ok { + run_task('peadm::mkdir_p_file', $master_target, + path => '/etc/puppetlabs/puppet/csr_attributes.yaml', + content => @("HEREDOC"), + --- + extension_requests: + ${pp_application}: "${pp_application_master}" + ${pp_cluster}: "${pp_cluster_a}" + | HEREDOC + ) + } - run_task('peadm::mkdir_p_file', $puppetdb_database_target, - path => '/etc/puppetlabs/puppet/csr_attributes.yaml', - content => @("HEREDOC"), - --- - extension_requests: - ${pp_application}: "${pp_application_puppetdb}" - ${pp_cluster}: "${pp_cluster_a}" - | HEREDOC + $puppetdb_check = run_task('peadm::existing_csr_check', + $puppetdb_database_target, + '_catch_errors' => true ) + if $puppetdb_check.ok { + run_task('peadm::mkdir_p_file', $puppetdb_database_target, + path => '/etc/puppetlabs/puppet/csr_attributes.yaml', + content => @("HEREDOC"), + --- + extension_requests: + ${pp_application}: "${pp_application_puppetdb}" + ${pp_cluster}: "${pp_cluster_a}" + | HEREDOC + ) + } - run_task('peadm::mkdir_p_file', $puppetdb_database_replica_target, - path => '/etc/puppetlabs/puppet/csr_attributes.yaml', - content => @("HEREDOC"), - --- - extension_requests: - ${pp_application}: "${pp_application_puppetdb}" - ${pp_cluster}: "${pp_cluster_b}" - | HEREDOC + $puppetdb_replica_check = run_task('peadm::existing_csr_check', + $puppetdb_database_replica_target, + '_catch_errors' => true ) + if $puppetdb_replica_check.ok { + run_task('peadm::mkdir_p_file', $puppetdb_database_replica_target, + path => '/etc/puppetlabs/puppet/csr_attributes.yaml', + content => @("HEREDOC"), + --- + extension_requests: + ${pp_application}: "${pp_application_puppetdb}" + ${pp_cluster}: "${pp_cluster_b}" + | HEREDOC + ) + } # Get the master installation up and running. The installer will # "fail" because PuppetDB can't start, if puppetdb_database_target diff --git a/tasks/existing_csr_check.json b/tasks/existing_csr_check.json new file mode 100644 index 00000000..436beaec --- /dev/null +++ b/tasks/existing_csr_check.json @@ -0,0 +1,7 @@ +{ + "puppet_task_version": 1, + "supports_noop": false, + "description": "This script checks for the existence of a csr_attributes file.", + "parameters": { + } +} diff --git a/tasks/existing_csr_check.sh b/tasks/existing_csr_check.sh new file mode 100644 index 00000000..f08ba2b3 --- /dev/null +++ b/tasks/existing_csr_check.sh @@ -0,0 +1,10 @@ +#!/bin/sh + +# Puppet Task Name: existing_csr_check +# + +if [ -f /etc/puppetlabs/puppet/csr_attributes.yaml ]; then + exit 1 +else + exit 0 +fi \ No newline at end of file From 25bd5a82008283aab3d22945cc9cdff73e6b7501 Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Fri, 6 Mar 2020 11:11:13 -0700 Subject: [PATCH 7/8] Update node manager for variables. --- manifests/setup/node_manager.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index b3f56618..3aff6e98 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -175,12 +175,12 @@ data => $compiler_data, } } elsif ($master_replica_host) { - node_group { 'PE Compiler Group B': + node_group { "PE Compiler Group ${pp_cluster_b}": ensure => 'present', parent => 'PE Master', rule => ['and', - ['=', ['trusted', 'extensions', 'pp_application'], 'puppet/compiler'], - ['=', ['trusted', 'extensions', 'pp_cluster'], 'B'], + ['=', ['trusted', 'extensions', 'pp_application'], $pp_application_compiler], + ['=', ['trusted', 'extensions', 'pp_cluster'], $pp_cluster_b], ], classes => { 'puppet_enterprise::profile::puppetdb' => { From 5b06c7549cdc71a838d6d8a166c2df420f1fc127 Mon Sep 17 00:00:00 2001 From: Bill Wilcox Date: Fri, 6 Mar 2020 18:39:01 -0700 Subject: [PATCH 8/8] Updated node manager for extension variables. --- manifests/setup/node_manager.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/setup/node_manager.pp b/manifests/setup/node_manager.pp index 3aff6e98..d451261f 100644 --- a/manifests/setup/node_manager.pp +++ b/manifests/setup/node_manager.pp @@ -44,7 +44,7 @@ # We modify this group's rule such that all PE infrastructure nodes will be # members. node_group { 'PE Infrastructure Agent': - rule => ['and', ['~', ['trusted', 'extensions', 'pp_application'], "^${pp_application_compiler}/"]], + rule => ['and', ['~', ['trusted', 'extensions', 'pp_application'], "^${pp_application_compiler}|^${pp_application_master}|^${pp_application_puppetdb}"]], # lint:ignore:140chars } # We modify this group to add, as data, the compiler_pool_address only.