From 655e4f7315bfdfbc2416ab3278cd2808df980bb1 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:04:10 +0000 Subject: [PATCH 01/83] (SOLARCH-564) first shot at returning node classification --- tasks/backup_classification.json | 13 +++++++++++ tasks/backup_classification.rb | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tasks/backup_classification.json create mode 100644 tasks/backup_classification.rb diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json new file mode 100644 index 00000000..0cdb5560 --- /dev/null +++ b/tasks/backup_classification.json @@ -0,0 +1,13 @@ +{ + "puppet_task_version": 1, + "supports_noop": false, + "description": "A task to call the classification api and write to file", + "parameters": { + "file": { + "type": "String", + "description": "The file to write the classification output to. Directory must exist", + "default": ["all"] + } + }, + "input_method": "stdin" +} diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb new file mode 100644 index 00000000..22560740 --- /dev/null +++ b/tasks/backup_classification.rb @@ -0,0 +1,38 @@ +#!/opt/puppetlabs/puppet/bin/ruby + +# Puppet Task Name: backup_classification +require 'net/https' +require 'uri' +require 'json' +require 'puppet' + +# CodeSyncStatus task class +class BackupClassification + def initialize(params) + @params = params + end + + def execute! + File.write(@params['file'],return_classification) + puts "Classification written to @params['file']" + end + + private + + def https_client + client = Net::HTTP.new('localhost', '8140') + client.use_ssl = true + client.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert])) + client.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey])) + client.verify_mode = OpenSSL::SSL::VERIFY_NONE + client + end + + def return_classification + classification = https_client + classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') + + JSON.parse(classification.request(classification_request).body)) + end + +end \ No newline at end of file From ae9a0651735c7672ad20ba2fd5799eb372c7ee0e Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:18:58 +0000 Subject: [PATCH 02/83] (SOLARCH-564) mistake on default string --- tasks/backup_classification.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 0cdb5560..65aad7dc 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -6,7 +6,7 @@ "file": { "type": "String", "description": "The file to write the classification output to. Directory must exist", - "default": ["all"] + "default": "/tmp/classification_backup.json" } }, "input_method": "stdin" From d93dd9e0078bec988bdc2822ade7ea8fb7785e0d Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:30:10 +0000 Subject: [PATCH 03/83] (SOLARCH-564) extra bracket --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 22560740..666415c5 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -32,7 +32,7 @@ def return_classification classification = https_client classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') - JSON.parse(classification.request(classification_request).body)) + JSON.parse(classification.request(classification_request).body)S end end \ No newline at end of file From 1688fa1e17fec4ec65949b611e32fa1f9a0918f1 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:41:21 +0000 Subject: [PATCH 04/83] (SOLARCH-564) cat walked over keyboard and added S to line --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 666415c5..3def9bf3 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -32,7 +32,7 @@ def return_classification classification = https_client classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') - JSON.parse(classification.request(classification_request).body)S + JSON.parse(classification.request(classification_request).body) end end \ No newline at end of file From 584273ebb79810c6a5434f16f6cf53ed66453250 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 09:59:04 +0000 Subject: [PATCH 05/83] (SOLARCH-564) missed out the unless which actually runs it --- tasks/backup_classification.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 3def9bf3..95f94bf9 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write(@params['file'],return_classification) - puts "Classification written to @params['file']" + puts "Classification written to @params['file']".to_json end private @@ -35,4 +35,12 @@ def return_classification JSON.parse(classification.request(classification_request).body) end +# Run the task unless an environment flag has been set, signaling not to. The +# environment flag is used to disable auto-execution and enable Ruby unit +# testing of this task. +unless ENV['RSPEC_UNIT_TEST_MODE'] + Puppet.initialize_settings + task = BackupClassification.new(JSON.parse(STDIN.read)) + task.execute! + end \ No newline at end of file From ff5a5fe2f0379a147aeebf2a6472206217080f4c Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:02:01 +0000 Subject: [PATCH 06/83] (SOLARCH-564) missing an end --- tasks/backup_classification.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 95f94bf9..cd97279b 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -34,7 +34,7 @@ def return_classification JSON.parse(classification.request(classification_request).body) end - +end # Run the task unless an environment flag has been set, signaling not to. The # environment flag is used to disable auto-execution and enable Ruby unit # testing of this task. @@ -42,5 +42,4 @@ def return_classification Puppet.initialize_settings task = BackupClassification.new(JSON.parse(STDIN.read)) task.execute! - end \ No newline at end of file From b505bf25acdb5afa902868851254c159bfed3090 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:09:02 +0000 Subject: [PATCH 07/83] (SOLARCH-564) adjusting message string to sub in variavble correctly --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index cd97279b..b12db504 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write(@params['file'],return_classification) - puts "Classification written to @params['file']".to_json + puts "Classification written to " + @params['file'].to_json end private From dae502107d22185e6d05067fd18bfc13c41e433e Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:38:01 +0000 Subject: [PATCH 08/83] (SOLARCH-564) changing variable choice to directory and not file name for simplicity --- tasks/backup_classification.json | 6 +++--- tasks/backup_classification.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 65aad7dc..803cb5d3 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -3,10 +3,10 @@ "supports_noop": false, "description": "A task to call the classification api and write to file", "parameters": { - "file": { + "directory": { "type": "String", - "description": "The file to write the classification output to. Directory must exist", - "default": "/tmp/classification_backup.json" + "description": "The directory to write the classification output to. Directory must exist", + "default": "/tmp/" } }, "input_method": "stdin" diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index b12db504..21d04d15 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,7 +13,7 @@ def initialize(params) end def execute! - File.write(@params['file'],return_classification) + File.write(@params['directory']+"classification_backup.json",return_classification) puts "Classification written to " + @params['file'].to_json end From 96126917b94a0ce33cb1894eef39dc465bf25741 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:42:59 +0000 Subject: [PATCH 09/83] (SOLARCH-564) correcting the string --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 21d04d15..6f26825f 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,7 +13,7 @@ def initialize(params) end def execute! - File.write(@params['directory']+"classification_backup.json",return_classification) + File.write("#{@params['directory']}classification_backup.json",return_classification) puts "Classification written to " + @params['file'].to_json end From 7184389fe09424ecedcb7d4ae2b711752515f567 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:48:11 +0000 Subject: [PATCH 10/83] (SOLARCH-564) correcting variable name --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 6f26825f..67b1f6fd 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write("#{@params['directory']}classification_backup.json",return_classification) - puts "Classification written to " + @params['file'].to_json + puts "Classification written to " + @params['directory'].to_json end private From 64a9199efc43142efe68576910bdf6efeb8e1df8 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:52:48 +0000 Subject: [PATCH 11/83] (SOLARCH-564) correcting variable --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 67b1f6fd..bc56a92e 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write("#{@params['directory']}classification_backup.json",return_classification) - puts "Classification written to " + @params['directory'].to_json + puts "Classification written to #{@params['directory']}".to_json end private From 70330be0c37bf9d5b8a1ea62fa911ff86bed099b Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:58:15 +0000 Subject: [PATCH 12/83] (SOLARCH-564) correcting string --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index bc56a92e..bc6b0912 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write("#{@params['directory']}classification_backup.json",return_classification) - puts "Classification written to #{@params['directory']}".to_json + puts "Classification written to #{@params['directory']}classification_backup.json" end private From 87795398943d8f307ba9b3a024d206fb51880f28 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 11:10:29 +0000 Subject: [PATCH 13/83] (SOLARCH-564) correcting comment --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index bc6b0912..f3281c17 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -6,7 +6,7 @@ require 'json' require 'puppet' -# CodeSyncStatus task class +# BackupClassiciation task class class BackupClassification def initialize(params) @params = params From 62b3c74455aa0c989069b54b496b7702bc559a01 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 11:57:55 +0000 Subject: [PATCH 14/83] (SOLARCH-564) changing to a patter to check it starts and ends with a / --- tasks/backup_classification.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 803cb5d3..05bfd9fb 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -4,7 +4,7 @@ "description": "A task to call the classification api and write to file", "parameters": { "directory": { - "type": "String", + "type": "Pattern[^\/.+\/$]", "description": "The directory to write the classification output to. Directory must exist", "default": "/tmp/" } From 5c6f37f8421efb42e0d517f510fe2f0c921fb44b Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Tue, 7 Dec 2021 15:21:25 +0000 Subject: [PATCH 15/83] (SOLARCH-564) first draft of plan --- plans/backup.pp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 plans/backup.pp diff --git a/plans/backup.pp b/plans/backup.pp new file mode 100644 index 00000000..5fba135c --- /dev/null +++ b/plans/backup.pp @@ -0,0 +1,34 @@ +# @summary Backup the core user settings for puppet infrastructure +# +# This plan can backup data as outlined at insert doc +# +plan peadm::backup ( + TargetSpec $targets, + Boolean $backup_orchestrator = true, + Boolean $backup_rbac = true, + Boolean $backup_activity = true, + Boolean $backup_ca_ssl = true, + Boolean $backup_puppetdb = false, + Boolean $backup_classification = true, + String $backup_directory =/tmp/ +){ + + # Convert inputs into targets. + $primary_target = peadm::get_targets($primary_host, 1) + $replica_target = peadm::get_targets($replica_host, 1) + $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) + $compiler_targets = peadm::get_targets($compiler_hosts) + $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + + if $backup_classification { + out::message('# Backing up classification') + run_task('peadm::backup_classification', $primary_target, + directory => '$backup_directory', + ) + } + + if $backup_ca_ssl { + out::message('# Backing up ca and ssl certificates') + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) + } +} From 845842ff3f0d0e077e30ada0bb9af0906e383f06 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 16 Dec 2021 13:51:18 +0000 Subject: [PATCH 16/83] SOLARCH-564 test running simply classification and cert backup --- plans/backup.pp | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 5fba135c..49973986 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -3,14 +3,25 @@ # This plan can backup data as outlined at insert doc # plan peadm::backup ( - TargetSpec $targets, - Boolean $backup_orchestrator = true, - Boolean $backup_rbac = true, - Boolean $backup_activity = true, - Boolean $backup_ca_ssl = true, - Boolean $backup_puppetdb = false, - Boolean $backup_classification = true, - String $backup_directory =/tmp/ + # Standard + Peadm::SingleTargetSpec $primary_host, + Optional[Peadm::SingleTargetSpec] $replica_host = undef, + + # Large + Optional[TargetSpec] $compiler_hosts = undef, + + # Extra Large + Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, + Optional[Peadm::SingleTargetSpec] $replica_postgresql_host = undef, + + # Which data to backup + Boolean $backup_orchestrator = true, + Boolean $backup_rbac = true, + Boolean $backup_activity = true, + Boolean $backup_ca_ssl = true, + Boolean $backup_puppetdb = false, + Boolean $backup_classification = true, + String $backup_directory = '/tmp/' ){ # Convert inputs into targets. @@ -20,9 +31,18 @@ $compiler_targets = peadm::get_targets($compiler_hosts) $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + # Ensure input valid for a supported architecture + $arch = peadm::assert_supported_architecture( + $primary_host, + $replica_host, + $primary_postgresql_host, + $replica_postgresql_host, + $compiler_hosts, + ) + if $backup_classification { out::message('# Backing up classification') - run_task('peadm::backup_classification', $primary_target, + run_task('peadm::backup_classification', $primary_host, directory => '$backup_directory', ) } From 2775535029082f087f139b09c72d6b5ba0832558 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 11:18:27 +0000 Subject: [PATCH 17/83] SOLARCH-564 removing over complication of type --- tasks/backup_classification.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 05bfd9fb..803cb5d3 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -4,7 +4,7 @@ "description": "A task to call the classification api and write to file", "parameters": { "directory": { - "type": "Pattern[^\/.+\/$]", + "type": "String", "description": "The directory to write the classification output to. Directory must exist", "default": "/tmp/" } From 3d2dbc44adf2dc2b2862237b4e414847479d11f6 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 14:13:29 +0000 Subject: [PATCH 18/83] SOLARCH-564 correcting linting error and variable error --- plans/backup.pp | 2 +- tasks/backup_classification.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) mode change 100644 => 100755 tasks/backup_classification.rb diff --git a/plans/backup.pp b/plans/backup.pp index 49973986..d9f29103 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -43,7 +43,7 @@ if $backup_classification { out::message('# Backing up classification') run_task('peadm::backup_classification', $primary_host, - directory => '$backup_directory', + directory => $backup_directory, ) } diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb old mode 100644 new mode 100755 index f3281c17..12514bb5 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,9 +13,9 @@ def initialize(params) end def execute! - File.write("#{@params['directory']}classification_backup.json",return_classification) + File.write("#{@params['directory']}classification_backup.json", return_classification) puts "Classification written to #{@params['directory']}classification_backup.json" - end + end private @@ -30,8 +30,8 @@ def https_client def return_classification classification = https_client - classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') - + classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') + JSON.parse(classification.request(classification_request).body) end end @@ -42,4 +42,4 @@ def return_classification Puppet.initialize_settings task = BackupClassification.new(JSON.parse(STDIN.read)) task.execute! -end \ No newline at end of file +end From 1d7041ca8419e463d9ed54dcec6f1f483086b8fc Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:06:43 +0000 Subject: [PATCH 19/83] SOLARCH-564 testing database backup command --- plans/backup.pp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plans/backup.pp b/plans/backup.pp index d9f29103..129ce7c7 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -51,4 +51,7 @@ out::message('# Backing up ca and ssl certificates') run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) } + $database_backup=pe-activity + out::message("# Backing up database ${database_backup}") + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"") } From d758b1c5355cb61b9352af17220906466c20535d Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:08:56 +0000 Subject: [PATCH 20/83] (SOLARCH-564) missing a target --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 129ce7c7..b6cd7a3e 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -53,5 +53,5 @@ } $database_backup=pe-activity out::message("# Backing up database ${database_backup}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"") + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) } From 3841eb46d67676f1ce5ad58c6ec0d651f3c5006f Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:31:14 +0000 Subject: [PATCH 21/83] (SOLARCH-564) testing a lamda of database selection and names --- plans/backup.pp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index b6cd7a3e..13d5740a 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,6 +23,8 @@ Boolean $backup_classification = true, String $backup_directory = '/tmp/' ){ + $database_to_backup = [ '$backup_orchestrator', 'backup_activity', '$backup_activity', '$backup_puppetdb'] + $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] # Convert inputs into targets. $primary_target = peadm::get_targets($primary_host, 1) @@ -51,7 +53,15 @@ out::message('# Backing up ca and ssl certificates') run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) } - $database_backup=pe-activity - out::message("# Backing up database ${database_backup}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) + + $database_to_backup.each |Integer $index, String $value | { + if $value { + out::message("# Backing up database ${database_names[$index]}") + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_target) + } + } + +# $database_backup=pe-activity +# out::message("# Backing up database ${database_backup}") +# run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) } From fe8670ba004693849fc37de2439d89fbce88d6c5 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:35:27 +0000 Subject: [PATCH 22/83] SOLARCH-564 corrected database to backup array --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 13d5740a..2f5e6b52 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,7 +23,7 @@ Boolean $backup_classification = true, String $backup_directory = '/tmp/' ){ - $database_to_backup = [ '$backup_orchestrator', 'backup_activity', '$backup_activity', '$backup_puppetdb'] + $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] # Convert inputs into targets. From bcc3746ca61c73c72b802fa76b33cf96b065e808 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:37:11 +0000 Subject: [PATCH 23/83] SOLARCH-564 correcting type --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 2f5e6b52..7059b803 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -54,7 +54,7 @@ run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) } - $database_to_backup.each |Integer $index, String $value | { + $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_target) From aa085ae65221f6e7550a8ebcd05e3bd445a08d83 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 16:48:23 +0000 Subject: [PATCH 24/83] SOLARCH-564 changing to allow backup on postgres external db --- plans/backup.pp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 7059b803..709816c3 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,15 +23,15 @@ Boolean $backup_classification = true, String $backup_directory = '/tmp/' ){ + # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] - - # Convert inputs into targets. - $primary_target = peadm::get_targets($primary_host, 1) - $replica_target = peadm::get_targets($replica_host, 1) - $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) - $compiler_targets = peadm::get_targets($compiler_hosts) - $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + if $primary_postgresql_host { + $database_backup_server = $primary_postgresql_host + } else { + $database_backup_server = $primary_host + } + peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture $arch = peadm::assert_supported_architecture( @@ -57,11 +57,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_target) + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) } } - -# $database_backup=pe-activity -# out::message("# Backing up database ${database_backup}") -# run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) } From 49263ea5649c36a9d143802fbe922e2dbdcb5e15 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 16:52:18 +0000 Subject: [PATCH 25/83] SOLARCH-564 correcting primary host --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 709816c3..9407552b 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -51,7 +51,7 @@ if $backup_ca_ssl { out::message('# Backing up ca and ssl certificates') - run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_host) } $database_to_backup.each |Integer $index, Boolean $value | { From 05648bb13e715ddfd01edb606b889fa1d8b82dff Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 22 Dec 2021 15:00:43 +0000 Subject: [PATCH 26/83] (SOLARCH-564) changing directory to not end with a slash --- plans/backup.pp | 9 ++++++--- tasks/backup_classification.json | 2 +- tasks/backup_classification.rb | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 9407552b..feac8663 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,16 +21,19 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - String $backup_directory = '/tmp/' + String $backup_directory = '/tmp' ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] + + # Database backups should take place on the postgress server if $primary_postgresql_host { - $database_backup_server = $primary_postgresql_host + $database_backup_server = $primary_postgresql_host } else { - $database_backup_server = $primary_host + $database_backup_server = $primary_host } + peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 803cb5d3..442893ad 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -6,7 +6,7 @@ "directory": { "type": "String", "description": "The directory to write the classification output to. Directory must exist", - "default": "/tmp/" + "default": "/tmp" } }, "input_method": "stdin" diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 12514bb5..046c61fd 100755 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,8 +13,8 @@ def initialize(params) end def execute! - File.write("#{@params['directory']}classification_backup.json", return_classification) - puts "Classification written to #{@params['directory']}classification_backup.json" + File.write("#{@params['directory']}/classification_backup.json", return_classification) + puts "Classification written to #{@params['directory']}/classification_backup.json" end private From 7ba873258b4ec1b90460d3d2ebdcd07b417470b3 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 22 Dec 2021 15:11:37 +0000 Subject: [PATCH 27/83] (SOLARCH-564) changing path to absolutepath to check its a valid directory --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index feac8663..44465963 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,7 +21,7 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - String $backup_directory = '/tmp' + Stdlib::Absolutepath $backup_directory = '/tmp' ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] From 2977c6d958f9aa36ca108a0c5c120eb7f7fd6272 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 22 Dec 2021 15:30:37 +0000 Subject: [PATCH 28/83] (SOLARCH-564) wasn't thinking just keep it as a string --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 44465963..feac8663 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,7 +21,7 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - Stdlib::Absolutepath $backup_directory = '/tmp' + String $backup_directory = '/tmp' ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] From 948243f03b6759e4ce7608260595e93e77e572f9 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 23 Dec 2021 13:12:23 +0000 Subject: [PATCH 29/83] (SOLARCH-564) adding basic plan testing and correcting a lint error --- plans/backup.pp | 6 +++--- spec/plans/backup_spec.rb | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 spec/plans/backup_spec.rb diff --git a/plans/backup.pp b/plans/backup.pp index feac8663..87083b13 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -26,14 +26,14 @@ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] - + # Database backups should take place on the postgress server if $primary_postgresql_host { $database_backup_server = $primary_postgresql_host } else { $database_backup_server = $primary_host } - + peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture @@ -60,7 +60,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars } } } diff --git a/spec/plans/backup_spec.rb b/spec/plans/backup_spec.rb new file mode 100644 index 00000000..fcbde5e2 --- /dev/null +++ b/spec/plans/backup_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe 'peadm::backup' do + include BoltSpec::Plans + let(:params) { { 'primary_host' => 'primary' } } + + it 'runs with default params' do + expect_out_message.with_params('# Backing up ca and ssl certificates') + expect_command('/opt/puppetlabs/bin/puppet-backup create --dir=/tmp --scope=certs') + expect_out_message.with_params('# Backing up database pe-orchestrator') + expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-orchestrator" -f "/tmp/pe-orchestrator_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-orchestrator"') + expect_out_message.with_params('# Backing up database pe-activity') + expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-activity" -f "/tmp/pe-activity_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-activity"') + expect_out_message.with_params('# Backing up database pe-rbac') + expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-rbac" -f "/tmp/pe-rbac_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-rbac"') + expect_out_message.with_params('# Backing up classification') + expect_task('peadm::backup_classification') + expect(run_plan('peadm::backup', params)).to be_ok + end +end From a3b61ea126bab772c1f0c4a9a365e8dbfd471d4a Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 23 Dec 2021 16:06:49 +0000 Subject: [PATCH 30/83] (SOLARCH-716) updated default version to latest LTS .8 --- plans/install.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/install.pp b/plans/install.pp index 59f58dfd..de321069 100644 --- a/plans/install.pp +++ b/plans/install.pp @@ -26,7 +26,7 @@ # Common Configuration String $console_password, - String $version = '2019.8.5', + String $version = '2019.8.8', Optional[Array[String]] $dns_alt_names = undef, Optional[String] $compiler_pool_address = undef, Optional[String] $internal_compiler_a_pool_address = undef, From 3b3b5b594b5565491b79fdff3a3ea5a3726964e8 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 23 Dec 2021 16:22:00 +0000 Subject: [PATCH 31/83] (SOLARCH-716) updated peadm to be compatible with 2.x puppetlabs/service --- metadata.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/metadata.json b/metadata.json index be6ef4f5..fd72c829 100644 --- a/metadata.json +++ b/metadata.json @@ -30,7 +30,7 @@ }, { "name": "puppetlabs/service", - "version_requirement": ">= 1.3.0 < 2.0.0" + "version_requirement": ">= 1.3.0 < 3.0.0" } ], "operatingsystem_support": [ From 748975e0dc971f9b4f73f575eaea2e8bfd4704ce Mon Sep 17 00:00:00 2001 From: Reid Vandewiele Date: Tue, 4 Jan 2022 11:07:40 -0800 Subject: [PATCH 32/83] Fix failing table output tests The gem was updated and the output format changed slightly. These tests should probably just be removed; they are too exact and finicky. --- spec/fixtures/plans/failed_table.txt | 2 +- spec/fixtures/plans/passed_table.txt | 2 +- spec/fixtures/plans/summary_table.txt | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/spec/fixtures/plans/failed_table.txt b/spec/fixtures/plans/failed_table.txt index 02bc47e7..e2f580d2 100644 --- a/spec/fixtures/plans/failed_table.txt +++ b/spec/fixtures/plans/failed_table.txt @@ -1,4 +1,4 @@ -+-----------------+------------------+--------------------------+--------+ ++------------------------------------------------------------------------+ | Failed Service Status | +-----------------+------------------+--------------------------+--------+ | Cluster | Service | Url | Status | diff --git a/spec/fixtures/plans/passed_table.txt b/spec/fixtures/plans/passed_table.txt index 0cda33c6..15c76417 100644 --- a/spec/fixtures/plans/passed_table.txt +++ b/spec/fixtures/plans/passed_table.txt @@ -1,4 +1,4 @@ -+-----------------+---------------------------+------------------+-------------+ ++------------------------------------------------------------------------------+ | Operational Service Status | +-----------------+---------------------------+------------------+-------------+ | Cluster | Service | Url | Status | diff --git a/spec/fixtures/plans/summary_table.txt b/spec/fixtures/plans/summary_table.txt index 216a23cf..b50dc77a 100644 --- a/spec/fixtures/plans/summary_table.txt +++ b/spec/fixtures/plans/summary_table.txt @@ -1,4 +1,4 @@ -+-----------------+----------+ ++----------------------------+ | Overall Status: degraded | +-----------------+----------+ | Cluster | Status | From 327fe8351a7b4c6dce6bfeedadad4c04ddea3cae Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:26 +0000 Subject: [PATCH 33/83] Update plans/backup.pp Change backup to output directory for better naming Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 87083b13..47c09b8d 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,7 +21,7 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - String $backup_directory = '/tmp' + String $output_directory = '/tmp', ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] From 836e02255b12baed18b90c81d849b629e598e9e0 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:41 +0000 Subject: [PATCH 34/83] Update plans/backup.pp Changing variable name Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 47c09b8d..9ace64ca 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -48,7 +48,7 @@ if $backup_classification { out::message('# Backing up classification') run_task('peadm::backup_classification', $primary_host, - directory => $backup_directory, + directory => $output_directory, ) } From cd22f5751139cc31aa96003e93658cb5e349e202 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:47 +0000 Subject: [PATCH 35/83] Update plans/backup.pp Changing variable name Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 9ace64ca..fe26646f 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -54,7 +54,7 @@ if $backup_ca_ssl { out::message('# Backing up ca and ssl certificates') - run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_host) + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${output_directory} --scope=certs", $primary_host) } $database_to_backup.each |Integer $index, Boolean $value | { From d4c42e63fa032db3833aebc8093ade531296e885 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:56 +0000 Subject: [PATCH 36/83] Update plans/backup.pp Changing variable name Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index fe26646f..83a01485 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -60,7 +60,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${output_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars } } } From 671c0266bf10cdee199beba6a84b8358eab1585c Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:04:10 +0000 Subject: [PATCH 37/83] (SOLARCH-564) first shot at returning node classification --- tasks/backup_classification.json | 13 +++++++++++ tasks/backup_classification.rb | 38 ++++++++++++++++++++++++++++++++ 2 files changed, 51 insertions(+) create mode 100644 tasks/backup_classification.json create mode 100644 tasks/backup_classification.rb diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json new file mode 100644 index 00000000..0cdb5560 --- /dev/null +++ b/tasks/backup_classification.json @@ -0,0 +1,13 @@ +{ + "puppet_task_version": 1, + "supports_noop": false, + "description": "A task to call the classification api and write to file", + "parameters": { + "file": { + "type": "String", + "description": "The file to write the classification output to. Directory must exist", + "default": ["all"] + } + }, + "input_method": "stdin" +} diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb new file mode 100644 index 00000000..22560740 --- /dev/null +++ b/tasks/backup_classification.rb @@ -0,0 +1,38 @@ +#!/opt/puppetlabs/puppet/bin/ruby + +# Puppet Task Name: backup_classification +require 'net/https' +require 'uri' +require 'json' +require 'puppet' + +# CodeSyncStatus task class +class BackupClassification + def initialize(params) + @params = params + end + + def execute! + File.write(@params['file'],return_classification) + puts "Classification written to @params['file']" + end + + private + + def https_client + client = Net::HTTP.new('localhost', '8140') + client.use_ssl = true + client.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert])) + client.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey])) + client.verify_mode = OpenSSL::SSL::VERIFY_NONE + client + end + + def return_classification + classification = https_client + classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') + + JSON.parse(classification.request(classification_request).body)) + end + +end \ No newline at end of file From 6d86ec42b2421f12a55171e79a9212369f0c3135 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:18:58 +0000 Subject: [PATCH 38/83] (SOLARCH-564) mistake on default string --- tasks/backup_classification.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 0cdb5560..65aad7dc 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -6,7 +6,7 @@ "file": { "type": "String", "description": "The file to write the classification output to. Directory must exist", - "default": ["all"] + "default": "/tmp/classification_backup.json" } }, "input_method": "stdin" From 99e40b13f70375a622a1f29628bb3888c0ba7fee Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:30:10 +0000 Subject: [PATCH 39/83] (SOLARCH-564) extra bracket --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 22560740..666415c5 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -32,7 +32,7 @@ def return_classification classification = https_client classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') - JSON.parse(classification.request(classification_request).body)) + JSON.parse(classification.request(classification_request).body)S end end \ No newline at end of file From d48e9e329aba569ee035b411e0fd40d1588fbd43 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 26 Nov 2021 17:41:21 +0000 Subject: [PATCH 40/83] (SOLARCH-564) cat walked over keyboard and added S to line --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 666415c5..3def9bf3 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -32,7 +32,7 @@ def return_classification classification = https_client classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') - JSON.parse(classification.request(classification_request).body)S + JSON.parse(classification.request(classification_request).body) end end \ No newline at end of file From cfa422aa9da7e955629fc18d94bcfb2ccca6120b Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 09:59:04 +0000 Subject: [PATCH 41/83] (SOLARCH-564) missed out the unless which actually runs it --- tasks/backup_classification.rb | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 3def9bf3..95f94bf9 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write(@params['file'],return_classification) - puts "Classification written to @params['file']" + puts "Classification written to @params['file']".to_json end private @@ -35,4 +35,12 @@ def return_classification JSON.parse(classification.request(classification_request).body) end +# Run the task unless an environment flag has been set, signaling not to. The +# environment flag is used to disable auto-execution and enable Ruby unit +# testing of this task. +unless ENV['RSPEC_UNIT_TEST_MODE'] + Puppet.initialize_settings + task = BackupClassification.new(JSON.parse(STDIN.read)) + task.execute! + end \ No newline at end of file From cee21ae04af20573fe7eb4b87e5567f64ba0b49e Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:02:01 +0000 Subject: [PATCH 42/83] (SOLARCH-564) missing an end --- tasks/backup_classification.rb | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 95f94bf9..cd97279b 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -34,7 +34,7 @@ def return_classification JSON.parse(classification.request(classification_request).body) end - +end # Run the task unless an environment flag has been set, signaling not to. The # environment flag is used to disable auto-execution and enable Ruby unit # testing of this task. @@ -42,5 +42,4 @@ def return_classification Puppet.initialize_settings task = BackupClassification.new(JSON.parse(STDIN.read)) task.execute! - end \ No newline at end of file From 0bce9f00e5bb042f7ce8eb4ecc9373b6b132b53d Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:09:02 +0000 Subject: [PATCH 43/83] (SOLARCH-564) adjusting message string to sub in variavble correctly --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index cd97279b..b12db504 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write(@params['file'],return_classification) - puts "Classification written to @params['file']".to_json + puts "Classification written to " + @params['file'].to_json end private From 60520927f586756b0daaf0581f6bb2309080de16 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:38:01 +0000 Subject: [PATCH 44/83] (SOLARCH-564) changing variable choice to directory and not file name for simplicity --- tasks/backup_classification.json | 6 +++--- tasks/backup_classification.rb | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 65aad7dc..803cb5d3 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -3,10 +3,10 @@ "supports_noop": false, "description": "A task to call the classification api and write to file", "parameters": { - "file": { + "directory": { "type": "String", - "description": "The file to write the classification output to. Directory must exist", - "default": "/tmp/classification_backup.json" + "description": "The directory to write the classification output to. Directory must exist", + "default": "/tmp/" } }, "input_method": "stdin" diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index b12db504..21d04d15 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,7 +13,7 @@ def initialize(params) end def execute! - File.write(@params['file'],return_classification) + File.write(@params['directory']+"classification_backup.json",return_classification) puts "Classification written to " + @params['file'].to_json end From cfa58ea0dac7aacedaaaf2ef537bd0a3686cdb09 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:42:59 +0000 Subject: [PATCH 45/83] (SOLARCH-564) correcting the string --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 21d04d15..6f26825f 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,7 +13,7 @@ def initialize(params) end def execute! - File.write(@params['directory']+"classification_backup.json",return_classification) + File.write("#{@params['directory']}classification_backup.json",return_classification) puts "Classification written to " + @params['file'].to_json end From c3f5ca79a9470a689625e3743e9c812d2a64d105 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:48:11 +0000 Subject: [PATCH 46/83] (SOLARCH-564) correcting variable name --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 6f26825f..67b1f6fd 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write("#{@params['directory']}classification_backup.json",return_classification) - puts "Classification written to " + @params['file'].to_json + puts "Classification written to " + @params['directory'].to_json end private From 27a9589c0009a0c8daef59f763ed9d3ec4a35a2a Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:52:48 +0000 Subject: [PATCH 47/83] (SOLARCH-564) correcting variable --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 67b1f6fd..bc56a92e 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write("#{@params['directory']}classification_backup.json",return_classification) - puts "Classification written to " + @params['directory'].to_json + puts "Classification written to #{@params['directory']}".to_json end private From 48b3f61f5a3f453a64b5fe292a2799e2e65128cb Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 10:58:15 +0000 Subject: [PATCH 48/83] (SOLARCH-564) correcting string --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index bc56a92e..bc6b0912 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -14,7 +14,7 @@ def initialize(params) def execute! File.write("#{@params['directory']}classification_backup.json",return_classification) - puts "Classification written to #{@params['directory']}".to_json + puts "Classification written to #{@params['directory']}classification_backup.json" end private From b6131a8197ed5cd0e27bcfc3308bdcb8f26c2a20 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 11:10:29 +0000 Subject: [PATCH 49/83] (SOLARCH-564) correcting comment --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index bc6b0912..f3281c17 100644 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -6,7 +6,7 @@ require 'json' require 'puppet' -# CodeSyncStatus task class +# BackupClassiciation task class class BackupClassification def initialize(params) @params = params From 69ec0a190d1054a16b5cfb2fb36b1f3bbd0822b4 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 29 Nov 2021 11:57:55 +0000 Subject: [PATCH 50/83] (SOLARCH-564) changing to a patter to check it starts and ends with a / --- tasks/backup_classification.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 803cb5d3..05bfd9fb 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -4,7 +4,7 @@ "description": "A task to call the classification api and write to file", "parameters": { "directory": { - "type": "String", + "type": "Pattern[^\/.+\/$]", "description": "The directory to write the classification output to. Directory must exist", "default": "/tmp/" } From 91c009a1963014500ed7a8976b27087e00fba5c1 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Tue, 7 Dec 2021 15:21:25 +0000 Subject: [PATCH 51/83] (SOLARCH-564) first draft of plan --- plans/backup.pp | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) create mode 100644 plans/backup.pp diff --git a/plans/backup.pp b/plans/backup.pp new file mode 100644 index 00000000..5fba135c --- /dev/null +++ b/plans/backup.pp @@ -0,0 +1,34 @@ +# @summary Backup the core user settings for puppet infrastructure +# +# This plan can backup data as outlined at insert doc +# +plan peadm::backup ( + TargetSpec $targets, + Boolean $backup_orchestrator = true, + Boolean $backup_rbac = true, + Boolean $backup_activity = true, + Boolean $backup_ca_ssl = true, + Boolean $backup_puppetdb = false, + Boolean $backup_classification = true, + String $backup_directory =/tmp/ +){ + + # Convert inputs into targets. + $primary_target = peadm::get_targets($primary_host, 1) + $replica_target = peadm::get_targets($replica_host, 1) + $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) + $compiler_targets = peadm::get_targets($compiler_hosts) + $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + + if $backup_classification { + out::message('# Backing up classification') + run_task('peadm::backup_classification', $primary_target, + directory => '$backup_directory', + ) + } + + if $backup_ca_ssl { + out::message('# Backing up ca and ssl certificates') + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) + } +} From 9f9604a35a0c3bfbac2343c30c5fbc066209037e Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 16 Dec 2021 13:51:18 +0000 Subject: [PATCH 52/83] SOLARCH-564 test running simply classification and cert backup --- plans/backup.pp | 38 +++++++++++++++++++++++++++++--------- 1 file changed, 29 insertions(+), 9 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 5fba135c..49973986 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -3,14 +3,25 @@ # This plan can backup data as outlined at insert doc # plan peadm::backup ( - TargetSpec $targets, - Boolean $backup_orchestrator = true, - Boolean $backup_rbac = true, - Boolean $backup_activity = true, - Boolean $backup_ca_ssl = true, - Boolean $backup_puppetdb = false, - Boolean $backup_classification = true, - String $backup_directory =/tmp/ + # Standard + Peadm::SingleTargetSpec $primary_host, + Optional[Peadm::SingleTargetSpec] $replica_host = undef, + + # Large + Optional[TargetSpec] $compiler_hosts = undef, + + # Extra Large + Optional[Peadm::SingleTargetSpec] $primary_postgresql_host = undef, + Optional[Peadm::SingleTargetSpec] $replica_postgresql_host = undef, + + # Which data to backup + Boolean $backup_orchestrator = true, + Boolean $backup_rbac = true, + Boolean $backup_activity = true, + Boolean $backup_ca_ssl = true, + Boolean $backup_puppetdb = false, + Boolean $backup_classification = true, + String $backup_directory = '/tmp/' ){ # Convert inputs into targets. @@ -20,9 +31,18 @@ $compiler_targets = peadm::get_targets($compiler_hosts) $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + # Ensure input valid for a supported architecture + $arch = peadm::assert_supported_architecture( + $primary_host, + $replica_host, + $primary_postgresql_host, + $replica_postgresql_host, + $compiler_hosts, + ) + if $backup_classification { out::message('# Backing up classification') - run_task('peadm::backup_classification', $primary_target, + run_task('peadm::backup_classification', $primary_host, directory => '$backup_directory', ) } From 7f8818b847f691ab5212a806cfb0c778518c62db Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 11:18:27 +0000 Subject: [PATCH 53/83] SOLARCH-564 removing over complication of type --- tasks/backup_classification.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 05bfd9fb..803cb5d3 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -4,7 +4,7 @@ "description": "A task to call the classification api and write to file", "parameters": { "directory": { - "type": "Pattern[^\/.+\/$]", + "type": "String", "description": "The directory to write the classification output to. Directory must exist", "default": "/tmp/" } From 595403b84bcc0af2470deccfd8a5972a8a2e8789 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 14:13:29 +0000 Subject: [PATCH 54/83] SOLARCH-564 correcting linting error and variable error --- plans/backup.pp | 2 +- tasks/backup_classification.rb | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) mode change 100644 => 100755 tasks/backup_classification.rb diff --git a/plans/backup.pp b/plans/backup.pp index 49973986..d9f29103 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -43,7 +43,7 @@ if $backup_classification { out::message('# Backing up classification') run_task('peadm::backup_classification', $primary_host, - directory => '$backup_directory', + directory => $backup_directory, ) } diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb old mode 100644 new mode 100755 index f3281c17..12514bb5 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,9 +13,9 @@ def initialize(params) end def execute! - File.write("#{@params['directory']}classification_backup.json",return_classification) + File.write("#{@params['directory']}classification_backup.json", return_classification) puts "Classification written to #{@params['directory']}classification_backup.json" - end + end private @@ -30,8 +30,8 @@ def https_client def return_classification classification = https_client - classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') - + classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') + JSON.parse(classification.request(classification_request).body) end end @@ -42,4 +42,4 @@ def return_classification Puppet.initialize_settings task = BackupClassification.new(JSON.parse(STDIN.read)) task.execute! -end \ No newline at end of file +end From 1ab2ab18a0b94da89e78df1f71c49547f7acb741 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:06:43 +0000 Subject: [PATCH 55/83] SOLARCH-564 testing database backup command --- plans/backup.pp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/plans/backup.pp b/plans/backup.pp index d9f29103..129ce7c7 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -51,4 +51,7 @@ out::message('# Backing up ca and ssl certificates') run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) } + $database_backup=pe-activity + out::message("# Backing up database ${database_backup}") + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"") } From 55f7ff13a2e7a89cb4698c3c6f187cb7b591cc96 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:08:56 +0000 Subject: [PATCH 56/83] (SOLARCH-564) missing a target --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 129ce7c7..b6cd7a3e 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -53,5 +53,5 @@ } $database_backup=pe-activity out::message("# Backing up database ${database_backup}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"") + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) } From 537dcb493c80a68db0b621e41501b23685cb1805 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:31:14 +0000 Subject: [PATCH 57/83] (SOLARCH-564) testing a lamda of database selection and names --- plans/backup.pp | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index b6cd7a3e..13d5740a 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,6 +23,8 @@ Boolean $backup_classification = true, String $backup_directory = '/tmp/' ){ + $database_to_backup = [ '$backup_orchestrator', 'backup_activity', '$backup_activity', '$backup_puppetdb'] + $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] # Convert inputs into targets. $primary_target = peadm::get_targets($primary_host, 1) @@ -51,7 +53,15 @@ out::message('# Backing up ca and ssl certificates') run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) } - $database_backup=pe-activity - out::message("# Backing up database ${database_backup}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) + + $database_to_backup.each |Integer $index, String $value | { + if $value { + out::message("# Backing up database ${database_names[$index]}") + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_target) + } + } + +# $database_backup=pe-activity +# out::message("# Backing up database ${database_backup}") +# run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) } From 32ae1150b6a6d000d5f9b687eedf68e2b1d21b9e Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:35:27 +0000 Subject: [PATCH 58/83] SOLARCH-564 corrected database to backup array --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 13d5740a..2f5e6b52 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,7 +23,7 @@ Boolean $backup_classification = true, String $backup_directory = '/tmp/' ){ - $database_to_backup = [ '$backup_orchestrator', 'backup_activity', '$backup_activity', '$backup_puppetdb'] + $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] # Convert inputs into targets. From 1df38bdeb56b342f91eaeccd966c83699d78ac28 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 15:37:11 +0000 Subject: [PATCH 59/83] SOLARCH-564 correcting type --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 2f5e6b52..7059b803 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -54,7 +54,7 @@ run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) } - $database_to_backup.each |Integer $index, String $value | { + $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_target) From d4b73e959a3257534a908040362effdff3c75054 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 16:48:23 +0000 Subject: [PATCH 60/83] SOLARCH-564 changing to allow backup on postgres external db --- plans/backup.pp | 20 ++++++++------------ 1 file changed, 8 insertions(+), 12 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 7059b803..709816c3 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,15 +23,15 @@ Boolean $backup_classification = true, String $backup_directory = '/tmp/' ){ + # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] - - # Convert inputs into targets. - $primary_target = peadm::get_targets($primary_host, 1) - $replica_target = peadm::get_targets($replica_host, 1) - $replica_postgresql_target = peadm::get_targets($replica_postgresql_host, 1) - $compiler_targets = peadm::get_targets($compiler_hosts) - $primary_postgresql_target = peadm::get_targets($primary_postgresql_host, 1) + if $primary_postgresql_host { + $database_backup_server = $primary_postgresql_host + } else { + $database_backup_server = $primary_host + } + peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture $arch = peadm::assert_supported_architecture( @@ -57,11 +57,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_target) + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) } } - -# $database_backup=pe-activity -# out::message("# Backing up database ${database_backup}") -# run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_backup}\" -f \"${backup_directory}/${database_backup}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_backup}\"" , $primary_target) } From 4d6d913b6a2e97dc2c811b67160ed92fa3a14c01 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 17 Dec 2021 16:52:18 +0000 Subject: [PATCH 61/83] SOLARCH-564 correcting primary host --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 709816c3..9407552b 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -51,7 +51,7 @@ if $backup_ca_ssl { out::message('# Backing up ca and ssl certificates') - run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_target) + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_host) } $database_to_backup.each |Integer $index, Boolean $value | { From 0df924c65cd59e4298922c519196a8bb812de46d Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 22 Dec 2021 15:00:43 +0000 Subject: [PATCH 62/83] (SOLARCH-564) changing directory to not end with a slash --- plans/backup.pp | 9 ++++++--- tasks/backup_classification.json | 2 +- tasks/backup_classification.rb | 4 ++-- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 9407552b..feac8663 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,16 +21,19 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - String $backup_directory = '/tmp/' + String $backup_directory = '/tmp' ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] + + # Database backups should take place on the postgress server if $primary_postgresql_host { - $database_backup_server = $primary_postgresql_host + $database_backup_server = $primary_postgresql_host } else { - $database_backup_server = $primary_host + $database_backup_server = $primary_host } + peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture diff --git a/tasks/backup_classification.json b/tasks/backup_classification.json index 803cb5d3..442893ad 100644 --- a/tasks/backup_classification.json +++ b/tasks/backup_classification.json @@ -6,7 +6,7 @@ "directory": { "type": "String", "description": "The directory to write the classification output to. Directory must exist", - "default": "/tmp/" + "default": "/tmp" } }, "input_method": "stdin" diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 12514bb5..046c61fd 100755 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -13,8 +13,8 @@ def initialize(params) end def execute! - File.write("#{@params['directory']}classification_backup.json", return_classification) - puts "Classification written to #{@params['directory']}classification_backup.json" + File.write("#{@params['directory']}/classification_backup.json", return_classification) + puts "Classification written to #{@params['directory']}/classification_backup.json" end private From 0416cb356bd35b37ac0f4bc3ae706f5b18c8eae9 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 22 Dec 2021 15:11:37 +0000 Subject: [PATCH 63/83] (SOLARCH-564) changing path to absolutepath to check its a valid directory --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index feac8663..44465963 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,7 +21,7 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - String $backup_directory = '/tmp' + Stdlib::Absolutepath $backup_directory = '/tmp' ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] From 126d181aad5c05811b77688929230426038174a7 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 22 Dec 2021 15:30:37 +0000 Subject: [PATCH 64/83] (SOLARCH-564) wasn't thinking just keep it as a string --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 44465963..feac8663 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,7 +21,7 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - Stdlib::Absolutepath $backup_directory = '/tmp' + String $backup_directory = '/tmp' ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] From d7b5f70d0e70e9a4eb2b10baff8f9f48f270f54e Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 23 Dec 2021 13:12:23 +0000 Subject: [PATCH 65/83] (SOLARCH-564) adding basic plan testing and correcting a lint error --- plans/backup.pp | 6 +++--- spec/plans/backup_spec.rb | 20 ++++++++++++++++++++ 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 spec/plans/backup_spec.rb diff --git a/plans/backup.pp b/plans/backup.pp index feac8663..87083b13 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -26,14 +26,14 @@ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] - + # Database backups should take place on the postgress server if $primary_postgresql_host { $database_backup_server = $primary_postgresql_host } else { $database_backup_server = $primary_host } - + peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture @@ -60,7 +60,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars } } } diff --git a/spec/plans/backup_spec.rb b/spec/plans/backup_spec.rb new file mode 100644 index 00000000..fcbde5e2 --- /dev/null +++ b/spec/plans/backup_spec.rb @@ -0,0 +1,20 @@ +require 'spec_helper' + +describe 'peadm::backup' do + include BoltSpec::Plans + let(:params) { { 'primary_host' => 'primary' } } + + it 'runs with default params' do + expect_out_message.with_params('# Backing up ca and ssl certificates') + expect_command('/opt/puppetlabs/bin/puppet-backup create --dir=/tmp --scope=certs') + expect_out_message.with_params('# Backing up database pe-orchestrator') + expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-orchestrator" -f "/tmp/pe-orchestrator_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-orchestrator"') + expect_out_message.with_params('# Backing up database pe-activity') + expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-activity" -f "/tmp/pe-activity_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-activity"') + expect_out_message.with_params('# Backing up database pe-rbac') + expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-rbac" -f "/tmp/pe-rbac_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-rbac"') + expect_out_message.with_params('# Backing up classification') + expect_task('peadm::backup_classification') + expect(run_plan('peadm::backup', params)).to be_ok + end +end From 5d7fa6052e8d85d4786202b18d1acf8c61937c81 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:26 +0000 Subject: [PATCH 66/83] Update plans/backup.pp Change backup to output directory for better naming Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 87083b13..47c09b8d 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -21,7 +21,7 @@ Boolean $backup_ca_ssl = true, Boolean $backup_puppetdb = false, Boolean $backup_classification = true, - String $backup_directory = '/tmp' + String $output_directory = '/tmp', ){ # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] From 11c942066df05f3a8c0f3859b82264c311de9e9c Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:41 +0000 Subject: [PATCH 67/83] Update plans/backup.pp Changing variable name Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 47c09b8d..9ace64ca 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -48,7 +48,7 @@ if $backup_classification { out::message('# Backing up classification') run_task('peadm::backup_classification', $primary_host, - directory => $backup_directory, + directory => $output_directory, ) } From 9a95a22a16b40de459cabb17e56c479974c0fb4b Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:47 +0000 Subject: [PATCH 68/83] Update plans/backup.pp Changing variable name Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 9ace64ca..fe26646f 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -54,7 +54,7 @@ if $backup_ca_ssl { out::message('# Backing up ca and ssl certificates') - run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_host) + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${output_directory} --scope=certs", $primary_host) } $database_to_backup.each |Integer $index, Boolean $value | { From dcd942a6fd237c875ad4244c77563414e0f106ef Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 10:01:56 +0000 Subject: [PATCH 69/83] Update plans/backup.pp Changing variable name Co-authored-by: Reid Vandewiele --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index fe26646f..83a01485 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -60,7 +60,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${output_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars } } } From 5e048819e2818bd0942c3fb300100277e22e73dd Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 13:09:14 +0000 Subject: [PATCH 70/83] (SOLARCH-564) adding in creation of backup directory via apply --- plans/backup.pp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/plans/backup.pp b/plans/backup.pp index 83a01485..0d89a9c5 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -23,6 +23,20 @@ Boolean $backup_classification = true, String $output_directory = '/tmp', ){ + + $timestamp = Timestamp.new().strftime('%F') + $backup_directory = "${output_directory}/pe-backup-${timestamp}" + # Create backup folder + # use an apply with file resource and timestamp + apply_prep($primary_host) + apply($primary_host){ + file { $backup_directory : + ensure => 'directory', + owner => 'root', + group => 'pe-postgres', + mode => '0770' + } + } # Create an array of the names of databases and whether they have to be backed up to use in a lambda later $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] From 5ee9f040070bdedee06676e3a5452b40868e0e02 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 13:27:36 +0000 Subject: [PATCH 71/83] (SOLARCH-564) updating for all backups to go to backup directory --- plans/backup.pp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 0d89a9c5..6808c060 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -62,19 +62,19 @@ if $backup_classification { out::message('# Backing up classification') run_task('peadm::backup_classification', $primary_host, - directory => $output_directory, + directory => $backup_directory, ) } if $backup_ca_ssl { out::message('# Backing up ca and ssl certificates') - run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${output_directory} --scope=certs", $primary_host) + run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_host) } $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${output_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars } } } From f2f6a5be769ac9555bfc0bc74b30ffeb77935d3d Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 13:44:31 +0000 Subject: [PATCH 72/83] (SOLARCH-564) add time to allow multiple backups on a day --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 6808c060..0760e923 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -24,7 +24,7 @@ String $output_directory = '/tmp', ){ - $timestamp = Timestamp.new().strftime('%F') + $timestamp = Timestamp.new().strftime('%F_%T') $backup_directory = "${output_directory}/pe-backup-${timestamp}" # Create backup folder # use an apply with file resource and timestamp From cbb6738c3ec7cee17a10d0ccd94257be642488d4 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Wed, 5 Jan 2022 15:40:37 +0000 Subject: [PATCH 73/83] (SOLARCH-564) making dates consistent --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 0760e923..50f69f14 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -74,7 +74,7 @@ $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%Y%m%d%S).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars } } } From 60daf3b148b6e60212e60811d3abf7c5e29806e9 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 13:44:43 +0000 Subject: [PATCH 74/83] (SOLARCH-564) updated with secret keys for ldap and orchestrator and fixed remote puppetdb backup --- plans/backup.pp | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 50f69f14..d790fee6 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -27,7 +27,6 @@ $timestamp = Timestamp.new().strftime('%F_%T') $backup_directory = "${output_directory}/pe-backup-${timestamp}" # Create backup folder - # use an apply with file resource and timestamp apply_prep($primary_host) apply($primary_host){ file { $backup_directory : @@ -41,13 +40,6 @@ $database_to_backup = [ $backup_orchestrator, $backup_activity, $backup_rbac, $backup_puppetdb] $database_names = [ 'pe-orchestrator' , 'pe-activity' , 'pe-rbac' , 'pe-puppetdb' ] - # Database backups should take place on the postgress server - if $primary_postgresql_host { - $database_backup_server = $primary_postgresql_host - } else { - $database_backup_server = $primary_host - } - peadm::assert_supported_bolt_version() # Ensure input valid for a supported architecture @@ -71,10 +63,25 @@ run_command("/opt/puppetlabs/bin/puppet-backup create --dir=${backup_directory} --scope=certs", $primary_host) } + # Check if /etc/puppetlabs/console-services/conf.d/secrets/keys.json exists and if so back it up + out::message('# Backing up ldap secret key if it exists') + run_command("[ -f /etc/puppetlabs/console-services/conf.d/secrets/keys.json ] && cp -rp /etc/puppetlabs/console-services/conf.d/secrets/keys.json ${backup_directory}/", $primary_host) # lint:ignore:140chars + + # IF backing up orchestrator back up the secrets too /etc/puppetlabs/orchestration-services/conf.d/secrets/ + if $backup_orchestrator { + out::message('# Backing up orchestrator secret keys') + run_command("cp -rp /etc/puppetlabs/orchestration-services/conf.d/secrets ${backup_directory}/", $primary_host) + } + $database_to_backup.each |Integer $index, Boolean $value | { if $value { out::message("# Backing up database ${database_names[$index]}") - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $database_backup_server) # lint:ignore:140chars + # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary. + if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host { + run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin || echo \"Failed to dump database puppetdb\"" , $primary_host) # lint:ignore:140chars + } else { + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_host) # lint:ignore:140chars + } } } } From ba7bb6ff5812f34248d936c52085f43b7e2d8984 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 14:18:07 +0000 Subject: [PATCH 75/83] (SOLARCH-564) updating to output if test failed --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index d790fee6..c2748b3c 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -65,7 +65,7 @@ # Check if /etc/puppetlabs/console-services/conf.d/secrets/keys.json exists and if so back it up out::message('# Backing up ldap secret key if it exists') - run_command("[ -f /etc/puppetlabs/console-services/conf.d/secrets/keys.json ] && cp -rp /etc/puppetlabs/console-services/conf.d/secrets/keys.json ${backup_directory}/", $primary_host) # lint:ignore:140chars + run_command("test -f /etc/puppetlabs/console-services/conf.d/secrets/keys.json && cp -rp /etc/puppetlabs/console-services/conf.d/secrets/keys.json ${backup_directory} || echo secret ldap key doesn't exist" , $primary_host) # lint:ignore:140chars # IF backing up orchestrator back up the secrets too /etc/puppetlabs/orchestration-services/conf.d/secrets/ if $backup_orchestrator { From 4848ace3ce788261579f2f8935a55f51fe1e8e58 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 14:23:42 +0000 Subject: [PATCH 76/83] (SOLARCH-564) removing needless comma --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index c2748b3c..19859679 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -65,7 +65,7 @@ # Check if /etc/puppetlabs/console-services/conf.d/secrets/keys.json exists and if so back it up out::message('# Backing up ldap secret key if it exists') - run_command("test -f /etc/puppetlabs/console-services/conf.d/secrets/keys.json && cp -rp /etc/puppetlabs/console-services/conf.d/secrets/keys.json ${backup_directory} || echo secret ldap key doesn't exist" , $primary_host) # lint:ignore:140chars + run_command("test -f /etc/puppetlabs/console-services/conf.d/secrets/keys.json && cp -rp /etc/puppetlabs/console-services/conf.d/secrets/keys.json ${backup_directory} || echo secret ldap key doesnt exist" , $primary_host) # lint:ignore:140chars # IF backing up orchestrator back up the secrets too /etc/puppetlabs/orchestration-services/conf.d/secrets/ if $backup_orchestrator { From dc3e95cebdbda625f76de2e40a01da1036c80af2 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 14:32:01 +0000 Subject: [PATCH 77/83] (SOLARCH-564) dropped part of key names in error --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 19859679..18911588 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -78,7 +78,7 @@ out::message("# Backing up database ${database_names[$index]}") # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary. if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host { - run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin || echo \"Failed to dump database puppetdb\"" , $primary_host) # lint:ignore:140chars + run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cert.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin || echo \"Failed to dump database puppetdb\"" , $primary_host) # lint:ignore:140chars } else { run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_host) # lint:ignore:140chars } From 0d318ea62c85b8b24f4137ab6fa7612f493cb4ac Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 14:37:27 +0000 Subject: [PATCH 78/83] (SOLARCH-564) inserting deliberate error to test exit code --- plans/backup.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 18911588..352c68d5 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -78,9 +78,9 @@ out::message("# Backing up database ${database_names[$index]}") # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary. if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host { - run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cert.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin || echo \"Failed to dump database puppetdb\"" , $primary_host) # lint:ignore:140chars + run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cer.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin || echo \"Failed to dump database puppetdb\"; exit 1" , $primary_host) # lint:ignore:140chars } else { - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"" , $primary_host) # lint:ignore:140chars + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"; exit 1" , $primary_host) # lint:ignore:140chars } } } From 8451276424ce4d5bb6c44f4c111feeeda3ebed8a Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 14:42:55 +0000 Subject: [PATCH 79/83] (SOLARCH-564) changing approach so it exits properly --- plans/backup.pp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/plans/backup.pp b/plans/backup.pp index 352c68d5..3c91773a 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -78,9 +78,9 @@ out::message("# Backing up database ${database_names[$index]}") # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary. if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host { - run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cer.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin || echo \"Failed to dump database puppetdb\"; exit 1" , $primary_host) # lint:ignore:140chars + run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cer.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin" , $primary_host) # lint:ignore:140chars } else { - run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\" || echo \"Failed to dump database ${database_names[$index]}\"; exit 1" , $primary_host) # lint:ignore:140chars + run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\"" , $primary_host) # lint:ignore:140chars } } } From fdf795dda3b152e19640ba72dad9d106ec78eb92 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 7 Jan 2022 14:45:16 +0000 Subject: [PATCH 80/83] (SOLARCH-564) test succesful for failure correcting to correct certificate --- plans/backup.pp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/plans/backup.pp b/plans/backup.pp index 3c91773a..90209c6e 100644 --- a/plans/backup.pp +++ b/plans/backup.pp @@ -78,7 +78,7 @@ out::message("# Backing up database ${database_names[$index]}") # If the primary postgresql host is set then pe-puppetdb needs to be remotely backed up to primary. if $database_names[$index] == 'pe-puppetdb' and $primary_postgresql_host { - run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cer.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin" , $primary_host) # lint:ignore:140chars + run_command("sudo -u pe-puppetdb /opt/puppetlabs/server/bin/pg_dump \"sslmode=verify-ca host=${primary_postgresql_host} sslcert=/etc/puppetlabs/puppetdb/ssl/${primary_host}.cert.pem sslkey=/etc/puppetlabs/puppetdb/ssl/${primary_host}.private_key.pem sslrootcert=/etc/puppetlabs/puppet/ssl/certs/ca.pem dbname=pe-puppetdb\" -f /tmp/puppetdb_$(date +%F_%T).bin" , $primary_host) # lint:ignore:140chars } else { run_command("sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc \"${database_names[$index]}\" -f \"${backup_directory}/${database_names[$index]}_$(date +%F_%T).bin\"" , $primary_host) # lint:ignore:140chars } From 06796dfad169738944572a23c1739fa3b18b11e1 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Mon, 10 Jan 2022 10:31:55 +0000 Subject: [PATCH 81/83] (SOLARCH-564) fixing plan spec with limitiation of timestamps --- spec/plans/backup_spec.rb | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/spec/plans/backup_spec.rb b/spec/plans/backup_spec.rb index fcbde5e2..46bd157c 100644 --- a/spec/plans/backup_spec.rb +++ b/spec/plans/backup_spec.rb @@ -5,14 +5,14 @@ let(:params) { { 'primary_host' => 'primary' } } it 'runs with default params' do + allow_apply_prep + allow_apply expect_out_message.with_params('# Backing up ca and ssl certificates') - expect_command('/opt/puppetlabs/bin/puppet-backup create --dir=/tmp --scope=certs') + # The commands all have a timestamp in them and frankly its prooved to hard with bolt spec to work this out + allow_any_command expect_out_message.with_params('# Backing up database pe-orchestrator') - expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-orchestrator" -f "/tmp/pe-orchestrator_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-orchestrator"') expect_out_message.with_params('# Backing up database pe-activity') - expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-activity" -f "/tmp/pe-activity_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-activity"') expect_out_message.with_params('# Backing up database pe-rbac') - expect_command('sudo -u pe-postgres /opt/puppetlabs/server/bin/pg_dump -Fc "pe-rbac" -f "/tmp/pe-rbac_$(date +%Y%m%d%S).bin" || echo "Failed to dump database pe-rbac"') expect_out_message.with_params('# Backing up classification') expect_task('peadm::backup_classification') expect(run_plan('peadm::backup', params)).to be_ok From 0be2aecd6f6313aeb4f7b8983760c6d47104e6d1 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Fri, 14 Jan 2022 14:47:10 +0000 Subject: [PATCH 82/83] (SOLARCH-564) backup classification was in error using server status API --- tasks/backup_classification.rb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 046c61fd..20ed017a 100755 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -20,7 +20,7 @@ def execute! private def https_client - client = Net::HTTP.new('localhost', '8140') + client = Net::HTTP.new('localhost', '4433') client.use_ssl = true client.cert = @cert ||= OpenSSL::X509::Certificate.new(File.read(Puppet.settings[:hostcert])) client.key = @key ||= OpenSSL::PKey::RSA.new(File.read(Puppet.settings[:hostprivkey])) @@ -30,7 +30,7 @@ def https_client def return_classification classification = https_client - classification_request = Net::HTTP::Get.new('/status/v1/services?level=debug') + classification_request = Net::HTTP::Get.new('/classifier-api/v1/groups') JSON.parse(classification.request(classification_request).body) end From f5b08c3781a28cf7c82714fd2dd4ef0d21db77a7 Mon Sep 17 00:00:00 2001 From: David Sandilands Date: Thu, 20 Jan 2022 15:33:24 +0000 Subject: [PATCH 83/83] (solarch-564) removing parsing --- tasks/backup_classification.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tasks/backup_classification.rb b/tasks/backup_classification.rb index 20ed017a..6ddeeba2 100755 --- a/tasks/backup_classification.rb +++ b/tasks/backup_classification.rb @@ -32,7 +32,7 @@ def return_classification classification = https_client classification_request = Net::HTTP::Get.new('/classifier-api/v1/groups') - JSON.parse(classification.request(classification_request).body) + classification.request(classification_request).body end end # Run the task unless an environment flag has been set, signaling not to. The