feat: postgres 11 replication setup docker compose file

Author: ravishankar

README.md

@@ -0,0 +1,18 @@
+version: '3.1'
+services:
+  ms:
+    command: "-c 'config_file=/etc/postgresql/postgresql.conf'"
+    image: postgres:12
+    volumes:
+      - /mnt/dsk/data-01/staging/postgres/master:/var/lib/postgresql/data
+      - ./postgres-master.conf:/etc/postgresql/postgresql.conf
+      - ./setup-node.sh:/docker-entrypoint-initdb.d/setup-node.sh
+    restart: always
+    ports:
+      - 4000:5432
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: rav
+      ROLE: master
+      PG_REP_USER: replicator
+      PG_REP_PASSWORD: password

master-stack.yml

@@ -0,0 +1,7 @@
+listen_addresses = '*'
+wal_level = hot_standby
+archive_mode = on
+archive_command = 'cd .'
+max_wal_senders = 8
+wal_keep_segments = 8
+hot_standby = on

postgres-master.conf

@@ -0,0 +1,8 @@
+listen_addresses = '*'
+wal_level = hot_standby
+archive_mode = on
+archive_command = 'cd .'
+max_wal_senders = 8
+wal_keep_segments = 8
+hot_standby = on
+promote_trigger_file='/tmp/failover.log'

postgres-slave.conf

@@ -0,0 +1,90 @@
+# Postgres Replication Setup
+
+Postgres supports multiple methods of replication. 
+The replication method we are going to setup is Streaming Replication or Log Replication. 
+Refer [this documentation](https://www.postgresql.org/docs/12/warm-standby.html#STREAMING-REPLICATION) to read more about Streaming Replication.
+
+
+### Steps to set up Streaming Replication in PostgreSQL 12
+ 
+#### MASTER SETUP
+
+##### Step 1
+```
+Initialize and start PostgreSQL, if not done already on the Master.
+```
+
+##### Step 2
+```
+Modify the parameter listen_addresses to allow a specific IP interface or all (using *). 
+Modifying this parameter requires a restart of the PostgreSQL instance to get the change into effect.
+```
+
+##### Step 3
+```
+Create a User for replication in the Master. 
+It is discouraged to use superuser postgres in order to setup replication, though it works.
+```
+
+##### Step 4
+```
+Allow replication connections from Standby to Master by appending a similar line as following to the pg_hba.conf file of the Master.
+
+# Config need to added 
+host replication all 0.0.0.0/0 md5
+```
+
+----
+
+#### SLAVE SETUP
+
+##### Step 1
+```
+Initialize and start PostgreSQL, if not done already on the Master.
+```
+
+##### Step 2
+```
+Modify the parameter listen_addresses to allow a specific IP interface or all (using *). 
+Modifying this parameter requires a restart of the PostgreSQL instance to get the change into effect.
+```
+
+#### Step 3
+```
+Delete everything in the data directory
+```
+
+##### Step 4
+```
+You may use `pg_basebackup` to backup the data directory of the Master from the Standby. 
+While creating the backup, you may also tell pg_basebackup  to create the replication specific files and entries in the data directory using "-R" .
+```
+
+----
+
+### If you think it's a lot of steps to setup the replication ???
+
+### Answer is here
+
+```
+All the steps above mentioned has been automated into two steps. 
+First step is to deploy the master stack.
+Another step is to deploy the slave stack with the replication configuration.
+
+Info: Master and slave runs on a two different machines.
+Note: Please modify credentials and port before deploying it
+```
+
+#### Deploy Master
+Please ensure the data volume path is correct in yml file
+```
+sudo docker-compose -f master-stack.yml start
+# ensure master stack is up 
+```
+
+#### Deploy slave
+Please ensure the data volume path is correct in yml file
+```
+sudo docker-compose -f slave-stack.yml start
+# ensure replication works by creating a table in master node and check if the table present in slave node
+```

readme.md

@@ -0,0 +1,47 @@
+#!/bin/bash
+
+# check if the role is specified
+if [ -z "$ROLE" ]; then
+        echo >&2 'Error:  You need to specify ROLE, values are master or slave'
+        exit 1
+fi
+
+# check if the replication user is specified
+if [ -z "$PG_REP_USER" ]; then
+        echo >&2 'Error:  You need to specify PG_REP_USER'
+        exit 1
+fi
+
+# check if the replication user password is specified
+if [ -z "$PG_REP_PASSWORD" ]; then
+        echo >&2 'Error:  You need to specify PG_REP_PASSWORD'
+        exit 1
+fi
+
+# create replication user and also update hba conf to allow remote access of replication users
+if [ "$ROLE" == "master" ]; then
+	echo "master is setting up"
+	echo "host replication all 0.0.0.0/0 md5" >> "$PGDATA/pg_hba.conf"
+	set -e
+	psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" <<-EOSQL
+	CREATE USER $PG_REP_USER WITH REPLICATION ENCRYPTED PASSWORD '$PG_REP_PASSWORD';
+	EOSQL
+fi
+
+
+# creates a pgpass file to save the password and perform the pg_basebackup of the master into slave data directory also automatically configures the slave
+if [ "$ROLE" == "slave" ]; then
+	echo "$PG_MASTER_HOST:$PG_MASTER_PORT:*:$PG_REP_USER:$PG_REP_PASSWORD" > "$HOME"/.pgpass
+	chmod 0600 "$HOME"/.pgpass
+	export PGPASSFILE="$HOME/.pgpass"
+	rm -rf $PGDATA/*
+	until pg_basebackup -h "${PG_MASTER_HOST}" -p "${PG_MASTER_PORT}" -D "${PGDATA}" -U "${PG_REP_USER}" -Fp -Xs -P -R
+    		do
+        	echo "Waiting for master to connect..."
+        	sleep 1s
+	done
+	echo "host replication all 0.0.0.0/0 md5" >> "$PGDATA/pg_hba.conf"
+fi
+
+echo "Awesome ${ROLE} done!"
+

setup-node.sh

@@ -0,0 +1,21 @@
+version: '3.1'
+services:
+  sl:
+    command: "-c 'config_file=/etc/postgresql/postgresql.conf'"
+    image: postgres:12
+    volumes:
+      - /mnt/dsk/data-01/staging/postgres/slave:/var/lib/postgresql/data
+      - ./postgres-slave.conf:/etc/postgresql/postgresql.conf
+      - ./setup-node.sh:/docker-entrypoint-initdb.d/setup-node.sh
+    restart: always
+    ports:
+      - 4001:5432
+    environment:
+      POSTGRES_PASSWORD: password
+      POSTGRES_USER: rav
+      PG_REP_USER: replicator
+      PG_REP_PASSWORD: password
+      PG_MASTER_HOST: 10.132.0.7
+      PG_MASTER_PORT: 4000
+      ROLE: slave
+