Skip to content

Commit aa688d1

Browse files
jarretlavalleejarretlavallee
authored
Merge pull request #35 from tkishel/maint_update
(refactor) correct pdk validate warnings, update for version changes
2 parents 5ee1f38 + 85ae45b commit aa688d1

18 files changed

+361
-227
lines changed

‎CHANGELOG.md

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
## Major Release 1.0.0
22

33
- Move from the npwalker namespace to the puppetlabs namespace.
4+
- Remove support for unsupported versions of PE.
45

56
## Minor Release 0.15.0
67

‎README.md

Lines changed: 100 additions & 52 deletions
Large diffs are not rendered by default.

‎files/puppet_enterprise_database_backup.sh

Lines changed: 19 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
#!/bin/bash
22

3-
BACKUPDIR=/opt/puppetlabs/server/data/postgresql/9.4/backups
3+
PG_VERSION=$(/usr/local/bin/facter -p pe_postgresql_info.installed_server_version)
4+
BACKUPDIR=/opt/puppetlabs/server/data/postgresql/$PG_VERSION/backups
45
LOGDIR=/var/log/puppetlabs/pe_databases_backup
56
RETENTION=2
67

@@ -26,35 +27,39 @@ while [[ $# -gt 0 ]]; do
2627
esac
2728
done
2829

29-
let RETENTION_ENFORCE=$RETENTION-1
30-
3130
if [[ -z "${DATABASES}" ]]; then
32-
echo "Usage: $0 [-l LOG_DIRECTORY] [-t BACKUP_TARGET] [-r RETENTION] <DATABASE1> [DATABASE...]"
31+
echo "Usage: $0 [-t BACKUP_TARGET] [-l LOG_DIRECTORY] [-r RETENTION] <DATABASE> [DATABASE_N ...]"
3332
exit 1
3433
fi
3534

35+
RETENTION_ENFORCE=$((RETENTION-1))
36+
3637
for db in $DATABASES; do
37-
echo "Enforcing retention policy of storing only ${RETENTION_ENFORCE} backups for ${db}" >> ${LOGDIR}/${db}.log 2>&1
38+
echo "Enforcing retention policy of storing only ${RETENTION_ENFORCE} backups for ${db}" >> "${LOGDIR}/${db}.log" 2>&1
3839

3940
ls -1tr ${BACKUPDIR}/${db}_* | head -n -${RETENTION_ENFORCE} | xargs -d '\n' rm -f --
4041

41-
echo "Starting dump of database: ${db}" >> ${LOGDIR}/${db}.log 2>&1
42+
echo "Starting dump of database: ${db}" >> "${LOGDIR}/${db}.log" 2>&1
4243

43-
if [ ${db} == "pe-classifier" ]; then
44-
#Save space before backing up by clearing unused node_check_ins table
45-
/opt/puppetlabs/server/bin/psql -d pe-classifier -c 'TRUNCATE TABLE node_check_ins' >> ${LOGDIR}/${db}.log 2>&1
46-
if [ $? != 0 ]; then
47-
echo "Failed to truncate node_check_ins table." >> ${LOGDIR}/${db}.log 2>&1
44+
if [ "${db}" == "pe-classifier" ]; then
45+
# Save space before backing up by clearing unused node_check_ins table.
46+
/opt/puppetlabs/server/bin/psql -d pe-classifier -c 'TRUNCATE TABLE node_check_ins' >> "${LOGDIR}/${db}.log" 2>&1
47+
48+
result=$?
49+
if [ $result != 0 ]; then
50+
echo "Failed to truncate node_check_ins table" >> "${LOGDIR}/${db}.log" 2>&1
4851
fi
4952
fi
5053

51-
/opt/puppetlabs/server/bin/pg_dump -Fc ${db} -f ${BACKUPDIR}/${db}_`date +%m_%d_%y_%H_%M`.bin >> ${LOGDIR}/${db}.log 2>&1
54+
DATETIME=$(date +%m_%d_%y_%H_%M)
55+
56+
/opt/puppetlabs/server/bin/pg_dump -Fc "${db}" -f "${BACKUPDIR}/${db}_$DATETIME.bin" >> "${LOGDIR}/${db}.log" 2>&1
5257

5358
result=$?
5459
if [[ $result -eq 0 ]]; then
55-
echo "Completed dump of database: ${db}" >> ${LOGDIR}/${db}.log 2>&1
60+
echo "Completed dump of database: ${db}" >> "${LOGDIR}/${db}.log" 2>&1
5661
else
57-
echo "Failed to dump database ${db}. Exit code is: ${result}" >> ${LOGDIR}/${db}.log 2>&1
62+
echo "Failed to dump database ${db}. Exit code is: ${result}" >> "${LOGDIR}/${db}.log" 2>&1
5863
exit 1
5964
fi
6065
done

‎files/vacuum_full_tables.sh

Lines changed: 12 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/bin/bash
2+
13
if [ "$1" = "" ]; then
24
echo "Usage: $0 <(facts, catalogs, or other) tables to VACUUM FULL> "
35
exit
@@ -9,12 +11,14 @@ else
911
SLEEP=$2
1012
fi
1113

12-
if [ $1 = 'facts' ]; then
13-
WHERE="'facts' ,'factsets', 'fact_paths', 'fact_values'"
14-
elif [ $1 = 'catalogs' ]; then
15-
WHERE="'catalogs' ,'catalog_resources', 'edges', 'certnames'"
16-
elif [ $1 = 'other' ]; then
17-
WHERE="'producers' ,'resource_params', 'resource_params_cache'"
14+
# TODO: Is this used in PE 2018 and newer? RE: fact_values
15+
16+
if [ "$1" = 'facts' ]; then
17+
WHERE="'facts', 'factsets', 'fact_paths', 'fact_values'"
18+
elif [ "$1" = 'catalogs' ]; then
19+
WHERE="'catalogs', 'catalog_resources', 'edges', 'certnames'"
20+
elif [ "$1" = 'other' ]; then
21+
WHERE="'producers', 'resource_params', 'resource_params_cache'"
1822
else
1923
echo "Must pass facts, catalogs, or other as first argument"
2024
exit 1
@@ -29,7 +33,7 @@ SQL="SELECT t.relname::varchar AS table_name
2933

3034
for TABLE in $(su - pe-postgres -s /bin/bash -c "/opt/puppetlabs/server/bin/psql -d pe-puppetdb -c \"$SQL\" --tuples-only")
3135
do
32-
#echo $TABLE
36+
# echo "$TABLE"
3337
su - pe-postgres -s /bin/bash -c "/opt/puppetlabs/server/bin/vacuumdb -d pe-puppetdb -t $TABLE --full --analyze"
34-
sleep $SLEEP
38+
sleep "$SLEEP"
3539
done

‎functions/version_based_databases.pp

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
# Function: version_based_databases
2+
#
3+
# Results:
4+
#
5+
# $databases: a version-specific array of databases.
6+
7+
function pe_databases::version_based_databases() >> Array[String] {
8+
$databases = [
9+
'pe-activity',
10+
'pe-classifier',
11+
'pe-inventory',
12+
'pe-orchestrator',
13+
'pe-postgres',
14+
'pe-rbac',
15+
]
16+
17+
if (versioncmp($facts['pe_server_version'], '2019.0.0') < 0) {
18+
return $databases - ['pe-inventory']
19+
} else {
20+
return $databases
21+
}
22+
}

‎hieradata_examples/backup_schedule_alt_example_1.yaml

Lines changed: 12 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -2,29 +2,29 @@ pe_databases::backup::databases_and_backup_schedule:
22
-
33
'databases':
44
- 'pe-classifier'
5-
- 'pe-rbac'
65
- 'pe-orchestrator'
6+
- 'pe-rbac'
77
'schedule':
88
'minute': '30'
99
'hour': '22'
1010
-
1111
'databases':
12-
- 'pe-puppetdb'
12+
- 'pe-activity'
1313
'schedule':
14-
'minute': '0'
15-
'hour': '2'
16-
'weekday': '7'
14+
'minute': '0'
15+
'hour': '1'
16+
'weekday': '6'
1717
-
1818
'databases':
19-
- 'pe-activity'
19+
- 'pe-postgres'
2020
'schedule':
21-
'minute': '0'
22-
'hour': '4'
21+
'minute': '0'
22+
'hour': '2'
2323
'weekday': '6'
2424
-
2525
'databases':
26-
- 'pe-postgres'
26+
- 'pe-puppetdb'
2727
'schedule':
28-
'minute': '0'
29-
'hour': '3'
30-
'weekday': '5'
28+
'minute': '0'
29+
'hour': '2'
30+
'weekday': '7'

‎hieradata_examples/backup_schedule_alt_example_2.yaml

Lines changed: 27 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -1,42 +1,50 @@
11
pe_databases::backup::databases_and_backup_schedule:
2+
-
3+
'databases':
4+
- 'pe-activity'
5+
'schedule':
6+
'minute': '0'
7+
'hour': '1'
8+
'weekday': '6'
29
-
310
'databases':
411
- 'pe-classifier'
512
'schedule':
6-
'minute': '30'
7-
'hour': '22'
13+
'minute': '0'
14+
'hour': '2'
15+
'weekday': '6'
816
-
917
'databases':
10-
- 'pe-puppetdb'
18+
- 'pe-inventory'
1119
'schedule':
12-
'minute': '0'
13-
'hour': '2'
14-
'weekday': '7'
20+
'minute': '0'
21+
'hour': '3'
22+
'weekday': '6'
1523
-
1624
'databases':
17-
- 'pe-activity'
25+
- 'pe-orchestrator'
1826
'schedule':
19-
'minute': '0'
20-
'hour': '4'
27+
'minute': '0'
28+
'hour': '4'
2129
'weekday': '6'
2230
-
2331
'databases':
2432
- 'pe-postgres'
2533
'schedule':
26-
'minute': '0'
27-
'hour': '3'
28-
'weekday': '5'
34+
'minute': '0'
35+
'hour': '5'
36+
'weekday': '6'
2937
-
3038
'databases':
3139
- 'pe-rbac'
3240
'schedule':
33-
'minute': '0'
34-
'hour': '1'
35-
'weekday': '1'
41+
'minute': '0'
42+
'hour': '6'
43+
'weekday': '6'
3644
-
3745
'databases':
38-
- 'pe-orchestrator'
46+
- 'pe-puppetdb'
3947
'schedule':
40-
'minute': '0'
41-
'hour': '7'
42-
'weekday': '2'
48+
'minute': '0'
49+
'hour': '2'
50+
'weekday': '7'

‎hieradata_examples/backup_schedule_default.yaml

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,16 +3,17 @@ pe_databases::backup::databases_and_backup_schedule:
33
'databases':
44
- 'pe-activity'
55
- 'pe-classifier'
6+
- 'pe-inventory'
7+
- 'pe-orchestrator'
68
- 'pe-postgres'
79
- 'pe-rbac'
8-
- 'pe-orchestrator'
910
'schedule':
1011
'minute': '30'
1112
'hour': '22'
1213
-
1314
'databases':
1415
- 'pe-puppetdb'
1516
'schedule':
16-
'minute': '0'
17-
'hour': '2'
17+
'minute': '0'
18+
'hour': '2'
1819
'weekday': '7'

‎manifests/backup.pp

Lines changed: 37 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
1+
# Backup PostgreSQL
2+
#
3+
# @summary Backup PostgreSQL
4+
15
class pe_databases::backup (
2-
Array[Hash] $databases_and_backup_schedule = [
6+
Array[Hash] $databases_and_backup_schedule =
7+
[
38
{
4-
'databases' => ['pe-activity', 'pe-classifier', 'pe-postgres', 'pe-rbac', 'pe-orchestrator'],
9+
'databases' => pe_databases::version_based_databases(),
510
'schedule' =>
611
{
712
'minute' => '30',
@@ -21,6 +26,7 @@
2126
String $psql_version = $pe_databases::psql_version,
2227
String $backup_directory = "/opt/puppetlabs/server/data/postgresql/${psql_version}/backups",
2328
String $backup_script_path = "${pe_databases::scripts_dir}/puppet_enterprise_database_backup.sh",
29+
String $daily_databases_path = "${pe_databases::install_dir}/default_daily_databases.txt",
2430
String $backup_logging_directory = '/var/log/puppetlabs/pe_databases_backup',
2531
Integer $retention_policy = 2,
2632
) {
@@ -31,7 +37,7 @@
3137
group => 'pe-postgres',
3238
}
3339

34-
file { 'puppet_enterprise_db_backup_script' :
40+
file { 'pe_databases_backup_script' :
3541
ensure => file,
3642
owner => 'pe-postgres',
3743
group => 'pe-postgres',
@@ -40,19 +46,39 @@
4046
source => 'puppet:///modules/pe_databases/puppet_enterprise_database_backup.sh',
4147
}
4248

43-
$databases_and_backup_schedule.each | Hash $dbs_and_schedule | {
49+
# Track the (databases backed up by default every day).
50+
file { 'pe_databases_default_daily_databases' :
51+
ensure => 'file',
52+
path => $daily_databases_path,
53+
content => "${pe_databases::version_based_databases()}",
54+
notify => Exec['reset_pe-postgres_crontab'],
55+
}
56+
57+
# Reset the crontab for pe-postgres if the (databases backed up by default every day) change.
58+
exec { 'reset_pe-postgres_crontab':
59+
path => '/usr/local/bin/:/bin/',
60+
command => 'crontab -r -u pe-postgres',
61+
onlyif => 'crontab -l -u pe-postgres',
62+
refreshonly => true,
63+
}
64+
65+
# Since the cron job titles below include the array ('databases') of database names,
66+
# the crontab for pe-postgres needs to be reset if the array of database names changes,
67+
# otherwise the change create a new cron job and unmanage the old cron job.
4468

45-
$databases_to_backup = $dbs_and_schedule['databases']
46-
$db_string = join($databases_to_backup, ' ')
69+
# TODO: This takes two runs to become idempotent. Why?
4770

71+
$databases_and_backup_schedule.each | Hash $database_backup_set | {
72+
$databases_to_backup = $database_backup_set['databases']
73+
$databases = join($databases_to_backup, ' ')
4874
cron { "puppet_enterprise_database_backup_${databases_to_backup}":
4975
ensure => present,
50-
command => "${backup_script_path} -l ${backup_logging_directory} -t ${backup_directory} -r ${retention_policy} ${db_string}",
76+
command => "${backup_script_path} -l ${backup_logging_directory} -t ${backup_directory} -r ${retention_policy} ${databases}",
5177
user => 'pe-postgres',
52-
minute => $dbs_and_schedule['schedule']['minute'],
53-
hour => $dbs_and_schedule['schedule']['hour'],
54-
weekday => $dbs_and_schedule['schedule']['weekday'],
55-
require => File['puppet_enterprise_db_backup_script'],
78+
minute => $database_backup_set['schedule']['minute'],
79+
hour => $database_backup_set['schedule']['hour'],
80+
weekday => $database_backup_set['schedule']['weekday'],
81+
require => [File['pe_databases_backup_script'], File['pe_databases_default_daily_databases']],
5682
}
5783
}
5884
}

‎manifests/init.pp

Lines changed: 13 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,7 @@
1+
# Tuning, maintenance, and backups for PE PostgreSQL.
2+
#
3+
# @summary Tuning, maintenance, and backups for PE PostgreSQL.
4+
15
class pe_databases (
26
Boolean $manage_database_backups = true,
37
Boolean $manage_database_maintenance = true,
@@ -7,10 +11,9 @@
711
String $scripts_dir = "${install_dir}/scripts"
812
) {
913

10-
if ( versioncmp('2017.3.0', $facts['pe_server_version']) <= 0 ) {
11-
$psql_version = '9.6'
12-
} else {
13-
$psql_version = '9.4'
14+
$psql_version = $facts['pe_postgresql_info']['installed_server_version'] ? {
15+
undef => undef,
16+
default => String($facts['pe_postgresql_info']['installed_server_version'])
1417
}
1518

1619
file { [$install_dir, $scripts_dir] :
@@ -22,17 +25,19 @@
2225
include pe_databases::maintenance
2326
}
2427

25-
if $manage_postgresql_settings and ( versioncmp('2018.1.0', $facts['pe_server_version']) > 0 ) {
28+
# Do not manage postgresql_settings in 2018.1.0 or newer.
29+
if $manage_postgresql_settings and (versioncmp('2018.1.0', $facts['pe_server_version']) > 0) {
2630
include pe_databases::postgresql_settings
27-
2831
class { 'pe_databases::postgresql_settings::table_settings' :
29-
manage_fact_values_autovacuum_cost_delay => $pe_databases::postgresql_settings::manage_fact_values_autovacuum_cost_delay,
3032
manage_reports_autovacuum_cost_delay => $pe_databases::postgresql_settings::manage_reports_autovacuum_cost_delay,
3133
factsets_autovacuum_vacuum_scale_factor => $pe_databases::postgresql_settings::factsets_autovacuum_vacuum_scale_factor,
3234
reports_autovacuum_vacuum_scale_factor => $pe_databases::postgresql_settings::reports_autovacuum_vacuum_scale_factor,
3335
require => Class['pe_databases::postgresql_settings'],
3436
}
35-
} elsif $manage_table_settings { #do not manage postgreql_settings in 2018.1.0
37+
} elsif $manage_table_settings {
38+
# This is to provide for situations, like PE XL,
39+
# where the pe-puppetdb database does not exist on the PostgreSQL system being tuned.
40+
# In PE XL, the Master and Replica run PostgreSQL for all databases *except* for pe-puppetdb.
3641
include pe_databases::postgresql_settings::table_settings
3742
}
3843

0 commit comments

Comments
 (0)