Skip to content

Commit 9774773

Browse files
author
Andre Wanderley de Souza
committed
integrated barman into primary promotion, integrated barman into standby clone
1 parent 75d8cbb commit 9774773

File tree

8 files changed

+122
-35
lines changed

8 files changed

+122
-35
lines changed

‎Dockerfile

+3-1
Original file line numberDiff line numberDiff line change
@@ -15,11 +15,13 @@ ENV PRIMARY_NODE=localhost
1515
ENV REPMGR_USER=repmgr
1616
ENV REPMGR_DB=repmgr
1717

18-
RUN apt-get update; apt-get install -y git make postgresql-server-dev-10 libpq-dev postgresql-10-repmgr repmgr-common openssh-server
18+
RUN apt-get update; apt-get install -y git make postgresql-server-dev-10 libpq-dev postgresql-10-repmgr repmgr-common openssh-server python3-pip
1919

2020
RUN git clone https://github.com/mreithub/pg_recall.git /root/pg_recall/
2121
RUN cd /root/pg_recall/; make install
2222

23+
RUN pip3 install barman-cli
24+
2325
RUN mkdir -p /home/postgres/; chown postgres:postgres /home/postgres/
2426

2527
COPY postgresql.conf /etc/postgresql/

‎README.md

+35-25
Original file line numberDiff line numberDiff line change
@@ -36,46 +36,31 @@ docker build --tag postgres-pgbouncer pgbouncer
3636

3737
docker build --tag postgres-barman barman
3838

39-
### RUN IT
39+
### SETUP LOCAL ENV
4040
export REPMGR_PASSWORD=`nicepass --password-length 24`
4141

4242
export BARMAN_PASSWORD=`nicepass --password-length 24`
4343

4444
export STREAMING_PASSWORD=`nicepass --password-length 24`
4545

46-
docker run --name pg-repmgr-1 --network pg_stream -e REPMGR_PASSWORD=$REPMGR_PASSWORD -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -d postgres-repmgr
46+
### RUN PRIMARY
47+
docker run --name pg-repmgr-1 --hostname pg-repmgr-1 --network pg_stream -e REPMGR_PASSWORD=$REPMGR_PASSWORD -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -d postgres-repmgr
4748

48-
sleep 2
49+
sleep 15
4950

50-
docker run --name pg-repmgr-2 --network pg_stream -e REPMGR_PASSWORD=$REPMGR_PASSWORD -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -e PRIMARY_NODE=pg-repmgr-1 -d postgres-repmgr
51-
52-
sleep 2
53-
54-
docker run --name pg-repmgr-3 --network pg_stream -e REPMGR_PASSWORD=$REPMGR_PASSWORD -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -e PRIMARY_NODE=pg-repmgr-1 -d postgres-repmgr
55-
56-
sleep 8
57-
58-
docker exec -it pg-repmgr-2 su -c "repmgr cluster show" - postgres
59-
sleep 3
60-
61-
62-
#### PGBOUNCER
63-
docker run --name pg-pgbouncer-1 --network pg_stream -e PRIMARY_NODE=pg-repmgr-1 -d postgres-pgbouncer
64-
65-
sleep 1
66-
67-
docker exec -it pg-pgbouncer-1 psql -U postgres -c "select client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn from pg_stat_replication;"
68-
69-
#### BARMAN
70-
docker run --name pg-barman-1 --network pg_stream -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -e PRIMARY_NODE=pg-repmgr-1 -d postgres-barman
51+
### RUN BARMAN ( for standby clones setup )
52+
docker run --name pg-barman-1 --hostname pg-barman-1 --network pg_stream -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -e PRIMARY_NODE=pg-repmgr-1 -d postgres-barman
7153

7254
sleep 30
7355

7456
docker exec -it pg-barman-1 barman backup all
7557

58+
docker exec -it pg-barman-1 barman backup all
59+
7660
docker exec -it pg-barman-1 barman cron
7761

7862
docker exec -it pg-barman-1 barman check pg-repmgr-1
63+
7964
`Server pg-repmgr-1:
8065
PostgreSQL: OK
8166
is_superuser: OK
@@ -95,6 +80,28 @@ docker exec -it pg-barman-1 barman check pg-repmgr-1
9580
receive-wal running: OK
9681
archiver errors: OK`
9782

83+
### RUN SECONDARIES
84+
85+
docker run --name pg-repmgr-2 --hostname pg-repmgr-2 --network pg_stream -e REPMGR_PASSWORD=$REPMGR_PASSWORD -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -e PRIMARY_NODE=pg-repmgr-1 -d postgres-repmgr
86+
87+
sleep 20
88+
89+
docker run --name pg-repmgr-3 --hostname pg-repmgr-3 --network pg_stream -e REPMGR_PASSWORD=$REPMGR_PASSWORD -e BARMAN_PASSWORD=$BARMAN_PASSWORD -e STREAMING_PASSWORD=$STREAMING_PASSWORD -e PRIMARY_NODE=pg-repmgr-1 -d postgres-repmgr
90+
91+
sleep 8
92+
93+
docker exec -it pg-repmgr-2 su -c "repmgr cluster show" - postgres
94+
sleep 3
95+
96+
97+
#### PGBOUNCER
98+
docker run --name pg-pgbouncer-1 --network pg_stream -e PRIMARY_NODE=pg-repmgr-1 -d postgres-pgbouncer
99+
100+
sleep 1
101+
102+
docker exec -it pg-pgbouncer-1 psql -U postgres -c "select client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn from pg_stat_replication;"
103+
104+
98105

99106
### FORCE FAILOVER
100107
[ monitor from another shell ] docker logs -f pg-repmgr-2
@@ -108,10 +115,13 @@ docker exec -it pg-repmgr-2 su -c "repmgr cluster show" - postgres
108115
### TEST BOUNCER TO NEW MASTER
109116
docker exec -it pg-pgbouncer-1 psql -U postgres -c "select client_addr, state, sent_lsn, write_lsn, flush_lsn, replay_lsn from pg_stat_replication;"
110117

118+
### TEST BARMAN TO NEW MASTER
119+
docker exec -it pg-barman-1 barman check all
120+
111121
### REJOIN OLD MASTER
112122
docker unpause pg-repmgr-1
113123

114-
sleep 10
124+
sleep 100
115125

116126
docker exec -it -u postgres pg-repmgr-1 bash -c 'repmgr node service --action=stop --checkpoint'
117127

‎barman/Dockerfile

+4-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
FROM postgres:10
22

33
RUN apt-get update \
4-
&& apt-get install -y --no-install-recommends wget cron gcc libpq-dev libpython-dev logrotate openssh-client rsync python3-pip \
4+
&& apt-get install -y --no-install-recommends wget cron gcc libpq-dev libpython-dev logrotate openssh-client rsync python3-pip openssh-server \
55
&& rm -rf /var/lib/apt/lists/* \
66
&& rm -f /etc/crontab /etc/cron.*/* \
77
&& sed -i 's/\(.*pam_loginuid.so\)/#\1/' /etc/pam.d/cron
@@ -17,8 +17,10 @@ RUN install -d -m 0700 -o barman -g barman /home/barman/.ssh
1717

1818
COPY barman.pem /home/barman/.ssh/id_rsa
1919
RUN chmod 600 /home/barman/.ssh/id_rsa
20-
RUN chown barman:barman /home/barman/.ssh/id_rsa
20+
2121
COPY ssh_config /home/barman/.ssh/config
22+
COPY pgbouncer.pub /home/barman/.ssh/authorized_keys
23+
2224
RUN chown -R barman:barman /home/barman/.ssh
2325

2426
ENV \

‎barman/entrypoint.sh

+5-1
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ echo "${PRIMARY_NODE}:*:*:${BARMAN_USER}:${BARMAN_PASSWORD}" > /home/barman/.pgp
66
echo "${PRIMARY_NODE}:*:*:${STREAMING_USER}:${STREAMING_PASSWORD}" >> /home/barman/.pgpass
77

88
chmod 600 /home/barman/.pgpass
9-
chown barman:barman /home/barman/.pgpass
9+
chown -R barman:barman /home/barman
1010

1111
install -d -m 0755 -o barman -g barman "${BARMAN_LOG_DIR}"
1212
install -d -m 0700 -o barman -g barman "${BARMAN_DATA_DIR}"
@@ -18,6 +18,10 @@ sed -i "s#\$BARMAN_USER#$BARMAN_USER#g" /etc/barman/barman.d/pg.conf
1818
sed -i "s#\$BARMAN_SLOT_NAME#$BARMAN_SLOT_NAME#g" /etc/barman/barman.d/pg.conf
1919
sed -i "s#\$STREAMING_USER#$STREAMING_USER#g" /etc/barman/barman.d/pg.conf
2020

21+
chown -R barman:barman /etc/barman
22+
23+
/etc/init.d/ssh start
24+
2125
gosu barman barman -q cron
2226

2327
gosu barman barman switch-xlog --force --archive $PRIMARY_NODE

‎postgresql.conf

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,8 +32,8 @@ effective_cache_size = 938MB
3232

3333
# logging
3434
logging_collector = on
35-
log_destination = 'csvlog' # Valid values are combinations of
36-
log_filename = 'postgresql-%a.log' # log file name pattern,
35+
log_destination = 'stderr,csvlog' # Valid values are combinations of
36+
log_filename = 'postgresql-%a.log' # log file name pattern,
3737
log_rotation_age = 1440
3838
log_truncate_on_rotation = on
3939
log_min_duration_statement = 600ms # -1 is disabled, 0 logs all statements

‎promote.sh

+63-1
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ PGBOUNCER_DATABASE_INI_NEW="/tmp/pgbouncer.database.ini"
66
PGBOUNCER_HOSTS="pg-pgbouncer-1"
77
DATABASES="postgres"
88

9+
BARMAN_DATABASE_INI_NEW="/tmp/barman.database.ini"
10+
BARMAN_DATABASE_PASSWORD="/tmp/barman.password.ini"
11+
BARMAN_HOSTS="pg-barman-1"
12+
913
# Pause pgbouncer
1014
for h in ${PGBOUNCER_HOSTS}
1115
do
@@ -38,4 +42,62 @@ done
3842

3943
rm ${PGBOUNCER_DATABASE_INI_NEW}
4044

41-
# TODO Generate new config file for barman, reconfigure barman, reload barman
45+
# BARMAN
46+
47+
PRIMARY_NODE=$(hostname -f)
48+
49+
cat >$BARMAN_DATABASE_INI_NEW <<EOF
50+
[$PRIMARY_NODE]
51+
; active = true
52+
; archiver = off
53+
; archiver_batch_size = 0
54+
; backup_directory = %(barman_home)s/%(name)s
55+
; backup_method = rsync
56+
; backup_options =
57+
; basebackup_retry_sleep = 30
58+
; basebackup_retry_times = 0
59+
; basebackups_directory = %(backup_directory)s/base
60+
; check_timeout = 30
61+
conninfo = host=$PRIMARY_NODE user=$BARMAN_USER dbname=postgres
62+
description = 'Test database'
63+
; disabled = false
64+
; errors_directory = %(backup_directory)s/errors
65+
; immediate_checkpoint = false
66+
; incoming_wals_directory = %(backup_directory)s/incoming
67+
; minimum_redundancy = 0
68+
; network_compression = false
69+
path_prefix = /usr/lib/postgresql/10
70+
; recovery_options =
71+
; retention_policy_mode = auto
72+
ssh_command = 'ssh -l postgres $PRIMARY_NODE'
73+
slot_name = $BARMAN_SLOT_NAME
74+
; streaming_archiver = off
75+
; streaming_archiver_batch_size = 0
76+
; streaming_archiver_name = barman_receive_wal
77+
; streaming_backup_name = barman_streaming_backup
78+
streaming_conninfo = host=$PRIMARY_NODE user=$STREAMING_USER dbname=postgres
79+
; streaming_wals_directory = %(backup_directory)s/streaming
80+
; wal_retention_policy = main
81+
; wals_directory = %(backup_directory)s/wals
82+
EOF
83+
84+
echo "${PRIMARY_NODE}:*:*:${BARMAN_USER}:${BARMAN_PASSWORD}" > ${BARMAN_DATABASE_PASSWORD}
85+
echo "${PRIMARY_NODE}:*:*:${STREAMING_USER}:${STREAMING_PASSWORD}" >> ${BARMAN_DATABASE_PASSWORD}
86+
chmod 600 ${BARMAN_DATABASE_PASSWORD}
87+
88+
for h in ${BARMAN_HOSTS}
89+
do
90+
rsync -a ${BARMAN_DATABASE_INI_NEW} barman@${h}:/etc/barman/barman.d/pg.conf
91+
rsync -a ${BARMAN_DATABASE_PASSWORD} barman@${h}:/home/barman/.pgpass
92+
93+
ssh -l barman ${h} "barman -q cron"
94+
sleep 1
95+
96+
ssh -l barman ${h} "barman receive-wal --create-slot $PRIMARY_NODE"
97+
sleep 3
98+
99+
ssh -l barman ${h} "barman switch-xlog --force --archive $PRIMARY_NODE || true"
100+
done
101+
102+
rm -f ${BARMAN_DATABASE_INI_NEW}
103+
rm -f ${BARMAN_DATABASE_PASSWORD}

‎scripts/02-conf_repmgr.sh

+4
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,10 @@ node_name=$(hostname -s | sed 's/\W\{1,\}/_/g;')
3333
conninfo=host='$NODE_HOST' user='$REPMGR_USER' dbname='$REPMGR_DB' connect_timeout=5
3434
data_directory=${PGDATA}
3535
36+
barman_host=${BARMAN_NODE:-pg-barman-1}
37+
barman_server=${PRIMARY_NODE}
38+
restore_command=/usr/local/bin/barman-wal-restore ${BARMAN_NODE:-pg-barman-1} ${PRIMARY_NODE} %f %p
39+
3640
log_level=INFO
3741
log_facility=STDERR
3842
log_status_interval=300

‎ssh_config

+6-3
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11

22
Host *
3-
LogLevel=ERROR
4-
StrictHostKeyChecking no
5-
UserKnownHostsFile /dev/null
3+
LogLevel=ERROR
4+
StrictHostKeyChecking no
5+
UserKnownHostsFile /dev/null
6+
7+
Host pg-barman-1
8+
User barman

0 commit comments

Comments
 (0)