Skip to content

Commit d86958b

Browse files
davidmalloncaresdavidmalloncares
authored
(PE-40222) add CI test for replacing a failed postgres server on an XL deployment with DR enabled (#550)
1 parent e5f627c commit d86958b

File tree

4 files changed

+185
-5
lines changed

4 files changed

+185
-5
lines changed

‎.github/workflows/test-replace-failed-postgresql.yaml

+131
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,131 @@
1+
---
2+
name: Test replace failed PostgreSQL
3+
on:
4+
pull_request:
5+
paths:
6+
- .github/workflows/**/*
7+
- spec/**/*
8+
- lib/**/*
9+
- tasks/**/*
10+
- functions/**/*
11+
- types/**/*
12+
- plans/**/*
13+
- hiera/**/*
14+
- manifests/**/*
15+
- templates/**/*
16+
- files/**/*
17+
- metadata.json
18+
- Rakefile
19+
- Gemfile
20+
- provision.yaml
21+
- .rspec
22+
- .rubocop.yml
23+
- .puppet-lint.rc
24+
- .fixtures.yml
25+
branches: [main]
26+
workflow_dispatch: {}
27+
jobs:
28+
test-replace-failed-postgresql:
29+
name: PE ${{ matrix.version }} ${{ matrix.architecture }} on ${{ matrix.image }}
30+
runs-on: ubuntu-20.04
31+
env:
32+
BOLT_GEM: true
33+
BOLT_DISABLE_ANALYTICS: true
34+
LANG: en_US.UTF-8
35+
strategy:
36+
fail-fast: false
37+
matrix:
38+
architecture: [extra-large-with-dr-and-spare-replica]
39+
install_architecture: [extra-large-with-dr]
40+
version: [2021.7.9, 2023.8.1, 2025.0.0]
41+
image: [almalinux-cloud/almalinux-8]
42+
steps:
43+
- name: Checkout Source
44+
uses: actions/checkout@v4
45+
with:
46+
ref: ${{ github.head_ref }}
47+
- name: Activate Ruby 2.7
48+
uses: ruby/setup-ruby@v1
49+
with:
50+
ruby-version: '2.7'
51+
bundler-cache: true
52+
- name: Print bundle environment
53+
if: ${{ github.repository_owner == 'puppetlabs' }}
54+
run: |
55+
echo ::group::info:bundler
56+
bundle env
57+
echo ::endgroup::
58+
- name: Provision test cluster
59+
timeout-minutes: 15
60+
run: |
61+
echo ::group::prepare
62+
mkdir -p $HOME/.ssh
63+
echo 'Host *' > $HOME/.ssh/config
64+
echo ' ServerAliveInterval 150' >> $HOME/.ssh/config
65+
echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config
66+
bundle exec rake spec_prep
67+
echo ::endgroup::
68+
echo ::group::provision
69+
bundle exec bolt plan run peadm_spec::provision_test_cluster \
70+
--modulepath spec/fixtures/modules \
71+
provider=provision_service \
72+
image=${{ matrix.image }} \
73+
architecture=${{ matrix.architecture }} \
74+
--log-level trace
75+
echo ::endgroup::
76+
echo ::group::info:request
77+
cat request.json || true; echo
78+
echo ::endgroup::
79+
echo ::group::info:inventory
80+
sed -e 's/password: .*/password: "[redacted]"/' < spec/fixtures/litmus_inventory.yaml || true
81+
echo ::endgroup::
82+
echo ::group::certnames
83+
bundle exec bolt plan run peadm_spec::add_inventory_hostnames \
84+
--inventory spec/fixtures/litmus_inventory.yaml \
85+
--modulepath spec/fixtures/modules \
86+
--no-host-key-check \
87+
inventory_file=spec/fixtures/litmus_inventory.yaml
88+
echo ::endgroup::
89+
- name: Output contents of litmus_inventory.yaml
90+
run: |
91+
cat spec/fixtures/litmus_inventory.yaml
92+
- name: Install PE on test cluster
93+
timeout-minutes: 120
94+
run: |
95+
bundle exec bolt plan run peadm_spec::install_test_cluster \
96+
--inventoryfile spec/fixtures/litmus_inventory.yaml \
97+
--modulepath spec/fixtures/modules \
98+
architecture=${{ matrix.install_architecture }} \
99+
version=${{ matrix.version }} \
100+
console_password=${{ secrets.CONSOLE_PASSWORD }} \
101+
code_manager_auto_configure=true
102+
- name: Replace failed PostgreSQL
103+
run: |
104+
echo ::group::prepare
105+
mkdir -p $HOME/.ssh
106+
echo 'Host *' > $HOME/.ssh/config
107+
echo ' ServerAliveInterval 150' >> $HOME/.ssh/config
108+
echo ' ServerAliveCountMax 2' >> $HOME/.ssh/config
109+
bundle exec rake spec_prep
110+
echo ::endgroup::
111+
bundle exec bolt plan run peadm_spec::test_replace_failed_postgres \
112+
--inventoryfile spec/fixtures/litmus_inventory.yaml \
113+
--modulepath spec/fixtures/modules \
114+
primary_host=$(yq '.groups[].targets[] | select(.vars.role == "primary") | .name' spec/fixtures/litmus_inventory.yaml) \
115+
replica_host=$(yq '.groups[].targets[] | select(.vars.role == "replica") | .name' spec/fixtures/litmus_inventory.yaml) \
116+
working_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "replica-pdb-postgresql") | .name' spec/fixtures/litmus_inventory.yaml) \
117+
failed_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "primary-pdb-postgresql") | .name' spec/fixtures/litmus_inventory.yaml) \
118+
replacement_postgresql_host=$(yq '.groups[].targets[] | select(.vars.role == "spare-replica") | .name' spec/fixtures/litmus_inventory.yaml) \
119+
--no-host-key-check
120+
- name: Tear down PE XL DR test cluster
121+
if: ${{ always() }}
122+
continue-on-error: true
123+
run: |-
124+
if [ -f spec/fixtures/litmus_inventory.yaml ]; then
125+
echo ::group::tear_down
126+
bundle exec rake 'litmus:tear_down'
127+
echo ::endgroup::
128+
echo ::group::info:request
129+
cat request.json || true; echo
130+
echo ::endgroup::
131+
fi

‎plans/replace_failed_postgresql.pp

+2-2
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,9 @@
2727
$peadm_config = run_task('peadm::get_peadm_config', $primary_host).first.value
2828
$compilers = $peadm_config['params']['compilers']
2929

30-
# Bail if this is trying to be ran against Standard
30+
# Bail if we are not running this against an XL deployment with DR enabled - the parameters also enforce this to some extent
3131
if $compilers.empty {
32-
fail_plan('Plan peadm::replace_failed_postgresql is only applicable for L and XL deployments')
32+
fail_plan('Plan peadm::replace_failed_postgresql is only applicable for XL deployments with DR enabled')
3333
}
3434

3535
$pe_hosts = peadm::flatten_compact([

‎spec/acceptance/peadm_spec/plans/test_replace_failed_postgres.pp

+43
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,43 @@
1+
plan peadm_spec::test_replace_failed_postgres(
2+
Peadm::SingleTargetSpec $primary_host,
3+
Peadm::SingleTargetSpec $replica_host,
4+
Peadm::SingleTargetSpec $working_postgresql_host,
5+
Peadm::SingleTargetSpec $failed_postgresql_host,
6+
Peadm::SingleTargetSpec $replacement_postgresql_host,
7+
) {
8+
# run infra status on the primary
9+
out::message("Running peadm::status on primary host ${primary_host}")
10+
$primary_status = run_plan('peadm::status', $primary_host, { 'format' => 'json' })
11+
out::message($primary_status)
12+
13+
if empty($primary_status['failed']) {
14+
out::message('Cluster is healthy, continuing')
15+
} else {
16+
fail_plan('Cluster is not healthy, aborting')
17+
}
18+
19+
# replace the failed postgres server
20+
run_plan('peadm::replace_failed_postgresql',
21+
primary_host => $primary_host,
22+
replica_host => $replica_host,
23+
working_postgresql_host => $working_postgresql_host,
24+
failed_postgresql_host => $failed_postgresql_host,
25+
replacement_postgresql_host => $replacement_postgresql_host,
26+
)
27+
28+
# get the config from primary_host and verify failed_postgresql_host is removed and replacement was added
29+
$result = run_task('peadm::get_peadm_config', $primary_host, '_catch_errors' => true).first.to_data()
30+
$primary_postgres_host = $result['value']['params']['primary_postgresql_host']
31+
$replica_postgres_host = $result['value']['params']['replica_postgresql_host']
32+
33+
if $primary_postgres_host == $failed_postgresql_host or $replica_postgres_host == $failed_postgresql_host {
34+
fail_plan("Failed PostgreSQL host ${failed_postgresql_host} was not removed from the PE configuration")
35+
} else {
36+
out::message("Failed PostgreSQL host ${failed_postgresql_host} was removed from the PE configuration")
37+
}
38+
if $primary_postgres_host == $replacement_postgresql_host or $replica_postgres_host == $replacement_postgresql_host {
39+
out::message("Replacement PostgreSQL host ${replacement_postgresql_host} was added to the PE configuration")
40+
} else {
41+
fail_plan("Replacement PostgreSQL host ${replacement_postgresql_host} was not added the PE configuration")
42+
}
43+
}

‎spec/docker/bolt-project.yaml

+9-3
Original file line numberDiff line numberDiff line change
@@ -2,12 +2,18 @@
22
name: peadm_docker_examples
33
modules:
44
- name: nwops/container_inventory
5-
version_requirement: ">= 0.1.1"
5+
version_requirement: '>= 0.1.1'
66
- name: puppetlabs/stdlib
7-
version_requirement: ">= 6.5.0 < 8.0.0"
7+
version_requirement: '>= 9.0.0 < 10.0.0'
88
- puppetlabs/ruby_task_helper
99
- name: puppetlabs/node_manager
10-
version_requirement: ">= 1.0.1 < 2.0.0"
10+
version_requirement: '>= 1.0.1 < 2.0.0'
11+
- name: puppetlabs/service
12+
version_requirement: '>= 1.3.0 <= 4.0.0'
13+
- name: puppetlabs/package
14+
version_requirement: '>= 2.1.0 <= 4.0.0'
15+
- name: puppetlabs/inifile
16+
version_requirement: '>= 6.1.0 < 7.0.0'
1117
- puppetlabs/apply_helpers
1218
- puppetlabs/bolt_shim
1319
- puppet/format

0 commit comments

Comments
 (0)