Skip to content

Commit 33317df

Browse files
odyody
authored
Merge pull request #258 from ody/use_failed_primary
Fix classification when adding some components
2 parents 64f2c05 + 1de4526 commit 33317df

16 files changed

+322
-210
lines changed

Diff for: ‎.fixtures.yml

+1
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ fixtures:
33
forge_modules:
44
ruby_task_helper: "puppetlabs/ruby_task_helper"
55
service: "puppetlabs/service"
6+
package: "puppetlabs/package"
67
repositories:
78
facts: 'https://github.com/puppetlabs/puppetlabs-facts.git'
89
puppet_agent: 'https://github.com/puppetlabs/puppetlabs-puppet_agent.git'

Diff for: ‎.vscode/settings.json

+2-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
{
22
"python.linting.pylintEnabled": true,
3-
"python.linting.enabled": true
3+
"python.linting.enabled": true,
4+
"git.ignoreLimitWarning": true
45
}

Diff for: ‎Gemfile

+1-1
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ group :development do
2626
gem "puppet-module-win-dev-r#{minor_version}", '~> 1.0', require: false, platforms: [:mswin, :mingw, :x64_mingw]
2727
gem "puppet-debugger", '>= 0.18.0', require: false
2828
gem "bolt", '>= 3.17.0', require: false
29-
gem "github_changelog_generator", require: false
29+
gem "github_changelog_generator", '>= 1.16.4', require: false
3030
gem "octokit", '4.21.0', require: false
3131
end
3232
group :system_tests do

Diff for: ‎documentation/automated_recovery.md

+1-1
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ Procedure:
2222

2323
2. Temporarily set both primary and replica server nodes so that they use the remaining healthy PE-PostgreSQL server
2424

25-
bolt plan run peadm::util::update_db_setting --target <primary-server-fqdn>,<replica-server-fqdn> primary_postgresql_host=<working-postgres-server-fqdn>
25+
bolt plan run peadm::util::update_db_setting --target <primary-server-fqdn>,<replica-server-fqdn> primary_postgresql_host=<working-postgres-server-fqdn> override=true
2626

2727
3. Restart `pe-puppetdb.service` on Puppet server primary and replica
2828

Diff for: ‎metadata.json

+4
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,10 @@
3232
"name": "puppetlabs/service",
3333
"version_requirement": ">= 1.3.0 < 3.0.0"
3434
},
35+
{
36+
"name": "puppetlabs/package",
37+
"version_requirement": ">= 2.1.0 < 3.0.0"
38+
},
3539
{
3640
"name": "puppetlabs/inifile",
3741
"version_requirement": ">= 5.2.0 < 6.0.0"

Diff for: ‎plans/add_database.pp

+65-76
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,9 @@
1212
) {
1313

1414
$primary_target = peadm::get_targets($primary_host, 1)
15+
$postgresql_target = peadm::get_targets($targets, 1)
16+
17+
$postgresql_host = $postgresql_target.peadm::certname()
1518

1619
# Get current peadm config before making modifications and shutting down
1720
# PuppetDB
@@ -21,13 +24,13 @@
2124

2225
# Bail if this is trying to be ran against Standard
2326
if $compilers.empty {
24-
fail_plan('Plan Peadm::Add_database only applicable for L and XL deployments')
27+
fail_plan('Plan peadm::add_database is only applicable for L and XL deployments')
2528
}
2629

2730
# Existing nodes and their assignments
2831
$replica_host = $peadm_config['params']['replica_host']
29-
$primary_postgresql_host = $peadm_config['params']['primary_postgresql_host']
30-
$replica_postgresql_host = $peadm_config['params']['replica_postgresql_host']
32+
$postgresql_a_host = $peadm_config['role-letter']['postgresql']['A']
33+
$postgresql_b_host = $peadm_config['role-letter']['postgresql']['B']
3134

3235
$replica_target = peadm::get_targets($replica_host, 1)
3336

@@ -41,8 +44,8 @@
4144
} else {
4245
# If array is empty then no external databases were previously configured
4346
$no_external_db = peadm::flatten_compact([
44-
$primary_postgresql_host,
45-
$replica_postgresql_host
47+
$postgresql_a_host,
48+
$postgresql_b_host
4649
]).empty
4750

4851
# Pick operating mode based on array check
@@ -67,26 +70,26 @@
6770
# The letter which doesn't yet have a server assigned or in the event this
6871
# is a replacement operation, the letter this node was assigned to previously
6972
$avail_group_letter = peadm::flatten_compact($roles['postgresql'].map |$k,$v| {
70-
if (! $v) or ($v == $targets.peadm::certname()) {
73+
if (! $v) or ($v == $postgresql_host) {
7174
$k
7275
}
7376
})[0]
7477
# When in pair mode we assume the other PSQL node will serve as our source
7578
$source_db_host = peadm::flatten_compact([
76-
$primary_postgresql_host,
77-
$replica_postgresql_host
78-
]).reject($targets.peadm::certname())[0]
79+
$postgresql_a_host,
80+
$postgresql_b_host
81+
]).reject($postgresql_host)[0]
7982
}
8083

81-
out::message("Adding PostgreSQL server ${targets.peadm::certname()} to availability group ${avail_group_letter}")
82-
out::message("Using ${source_db_host} to populate ${targets.peadm::certname()}")
84+
out::message("Adding PostgreSQL server ${postgresql_host} to availability group ${avail_group_letter}")
85+
out::message("Using ${source_db_host} to populate ${postgresql_host}")
8386

8487
$source_db_target = peadm::get_targets($source_db_host, 1)
8588

8689
peadm::plan_step('init-db-node') || {
8790
# Install PSQL on new node to be used as external PuppetDB backend by using
8891
# puppet in lieu of installer
89-
run_plan('peadm::subplans::component_install', $targets,
92+
run_plan('peadm::subplans::component_install', $postgresql_target,
9093
primary_host => $primary_target,
9194
avail_group_letter => $avail_group_letter,
9295
role => 'puppet/puppetdb-database'
@@ -95,7 +98,7 @@
9598

9699
# Stop Puppet to ensure catalogs are not being compiled for PE infrastructure nodes
97100
run_command('systemctl stop puppet.service', peadm::flatten_compact([
98-
$targets,
101+
$postgresql_target,
99102
$compilers,
100103
$primary_target,
101104
$replica_target,
@@ -108,44 +111,60 @@
108111

109112
peadm::plan_step('replicate-db') || {
110113
# Replicate content from source to newly installed PSQL server
111-
run_plan('peadm::subplans::db_populate', $targets, source_host => $source_db_target.peadm::certname())
114+
run_plan('peadm::subplans::db_populate', $postgresql_target, source_host => $source_db_target.peadm::certname())
112115

113116
# Run Puppet on new PSQL node to fix up certificates and permissions
114-
run_task('peadm::puppet_runonce', $targets)
117+
run_task('peadm::puppet_runonce', $postgresql_target)
115118
}
116119

117-
if $operating_mode == 'init' {
118-
119-
# Update classification and database.ini settings, assume a replica PSQL
120-
# does not exist
121-
peadm::plan_step('update-classification') || {
120+
# Update classification and database.ini settings, assume a replica PSQL
121+
# does not exist
122+
peadm::plan_step('update-classification') || {
123+
124+
# To ensure everything is functional when a replica exists but only a single
125+
# PostgreSQL node has been deployed, configure alternate availability group
126+
# to connect to other group's new node
127+
if ($operating_mode == 'init' and $replica_host) {
128+
$a_host = $avail_group_letter ? { 'A' => $postgresql_host, default => undef }
129+
$b_host = $avail_group_letter ? { 'B' => $postgresql_host, default => undef }
130+
$host = pick($a_host, $b_host)
131+
out::verbose("In transitive state, setting classification to ${host}")
122132
run_plan('peadm::util::update_classification', $primary_target,
123-
primary_postgresql_host => pick($primary_postgresql_host, $targets),
124-
peadm_config => $peadm_config
133+
postgresql_a_host => $host,
134+
postgresql_b_host => $host,
135+
peadm_config => $peadm_config
136+
)
137+
} else {
138+
run_plan('peadm::util::update_classification', $primary_target,
139+
postgresql_a_host => $avail_group_letter ? { 'A' => $postgresql_host, default => undef },
140+
postgresql_b_host => $avail_group_letter ? { 'B' => $postgresql_host, default => undef },
141+
peadm_config => $peadm_config
125142
)
126143
}
144+
}
127145

128-
peadm::plan_step('update-db-settings') || {
129-
run_plan('peadm::util::update_db_setting', peadm::flatten_compact([
130-
$compilers,
131-
$primary_target,
132-
$replica_target
133-
]),
134-
primary_postgresql_host => $targets,
135-
peadm_config => $peadm_config
136-
)
146+
peadm::plan_step('update-db-settings') || {
147+
run_plan('peadm::util::update_db_setting', peadm::flatten_compact([
148+
$compilers,
149+
$primary_target,
150+
$replica_target
151+
]),
152+
postgresql_host => $postgresql_host,
153+
peadm_config => $peadm_config
154+
)
137155

138-
# (Re-)Start PuppetDB now that we are done making modifications
139-
run_command('systemctl restart pe-puppetdb.service', peadm::flatten_compact([
140-
$primary_target,
141-
$replica_target
142-
]))
143-
}
156+
# (Re-)Start PuppetDB now that we are done making modifications
157+
run_command('systemctl restart pe-puppetdb.service', peadm::flatten_compact([
158+
$primary_target,
159+
$replica_target
160+
]))
161+
}
144162

145-
# Clean up old puppetdb database on primary and those which were copied to
146-
# new host.
147-
peadm::plan_step('cleanup-db') || {
163+
peadm::plan_step('cleanup-db') || {
148164

165+
if $operating_mode == 'init' {
166+
# Clean up old puppetdb database on primary and those which were copied to
167+
# new host.
149168
$target_db_purge = [
150169
'pe-activity',
151170
'pe-classifier',
@@ -157,7 +176,7 @@
157176
# If a primary replica exists then pglogical is enabled and will prevent
158177
# the clean up of databases on our target because it opens a connection.
159178
if $replica_host {
160-
run_plan('peadm::util::db_disable_pglogical', $targets, databases => $target_db_purge)
179+
run_plan('peadm::util::db_disable_pglogical', $postgresql_target, databases => $target_db_purge)
161180
}
162181

163182
# Clean up old databases
@@ -167,39 +186,9 @@
167186
$replica_target
168187
])
169188

170-
run_plan('peadm::util::db_purge', $clean_source, databases => ['pe-puppetdb'])
171-
run_plan('peadm::util::db_purge', $targets, databases => $target_db_purge)
172-
}
173-
} else {
174-
peadm::plan_step('update-classification') || {
175-
run_plan('peadm::util::update_classification', $primary_target,
176-
primary_postgresql_host => pick($primary_postgresql_host, $targets),
177-
replica_postgresql_host => pick($replica_postgresql_host, $targets),
178-
peadm_config => $peadm_config
179-
)
180-
}
181-
182-
# Plan needs to know which node is being added as well as primary and
183-
# replica designation
184-
peadm::plan_step('update-db-settings') || {
185-
run_plan('peadm::util::update_db_setting', peadm::flatten_compact([
186-
$compilers,
187-
$primary_target,
188-
$replica_target
189-
]),
190-
new_postgresql_host => $targets,
191-
primary_postgresql_host => pick($primary_postgresql_host, $targets),
192-
replica_postgresql_host => pick($replica_postgresql_host, $targets),
193-
peadm_config => $peadm_config
194-
)
195-
196-
# (Re-)Start PuppetDB now that we are done making modifications
197-
run_command('systemctl restart pe-puppetdb.service', peadm::flatten_compact([
198-
$primary_target,
199-
$replica_target
200-
]))
201-
}
202-
peadm::plan_step('cleanup-db') || {
189+
run_plan('peadm::util::db_purge', $clean_source, databases => ['pe-puppetdb'])
190+
run_plan('peadm::util::db_purge', $postgresql_target, databases => $target_db_purge)
191+
} else {
203192
out::message("No databases to cleanup when in ${operating_mode}")
204193
}
205194
}
@@ -212,15 +201,15 @@
212201
peadm::plan_step('finalize') || {
213202
# Run Puppet to sweep up but no restarts should occur so do them in parallel
214203
run_task('peadm::puppet_runonce', peadm::flatten_compact([
215-
$targets,
204+
$postgresql_target,
216205
$primary_target,
217206
$compilers,
218207
$replica_target
219208
]))
220209

221210
# Start Puppet agent
222211
run_command('systemctl start puppet.service', peadm::flatten_compact([
223-
$targets,
212+
$postgresql_target,
224213
$compilers,
225214
$primary_target,
226215
$replica_target,

0 commit comments

Comments
 (0)