Skip to content

Commit 021cbc4

Browse files
authored
Merge branch 'dev' into Jmpi-disable-new-golden-record-button
2 parents 4da42e7 + 2943283 commit 021cbc4

21 files changed

+1103
-13
lines changed

JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/Routes.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -130,7 +130,7 @@ public static Route patchIidNewGidLink(
130130
return entity(Jackson.unmarshaller(NotificationResolution.class),
131131
obj -> onComplete(Ask.patchIidNewGidLink(actorSystem,
132132
backEnd,
133-
obj.newGoldenId(),
133+
obj.currentGoldenId(),
134134
obj.interactionId()),
135135
result -> result.isSuccess()
136136
? result.get()
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -u
5+
6+
echo
7+
echo "Down stack"
8+
source ./helper/scripts/d-stack-08-rm.sh
9+
source ./helper/scripts/d-stack-09-wait-removed.sh
10+
11+
echo
12+
echo "Up app containers"
13+
source ./helper/scripts/d-stack-01-create-dirs.sh
14+
sleep 2
15+
source ./helper/scripts/d-stack-02-deploy-0.sh
16+
sleep 2
17+
source ./helper/scripts/d-stack-03-up-hub-containers.sh
18+

devops/windows/base-docker-wsl/conf/env/conf-env-high-1-pc.template

+3
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,9 @@ export DATA_DGRAPH_ALPHA_01_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-01
3030
export DATA_DGRAPH_ALPHA_02_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-02
3131
export DATA_DGRAPH_ALPHA_03_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-03
3232
export DATA_POSTGRESQL_DIR=${PROJECT_DATA_DIR}/postgres
33+
export DATA_POSTGRESQL_DB_DIR=${PROJECT_DATA_DIR}/postgres_db
34+
export POSTGRES_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/postgres
35+
export DGRAPH_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/dgraph
3336

3437
export SCALE_KEYCLOAK_TEST_SERVER=${SCALE_KEYCLOAK_TEST_SERVER}
3538
export SCALE_NGINX=${SCALE_NGINX}

devops/windows/base-docker-wsl/conf/env/conf-env-low-1-pc.template

+3
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,9 @@ export DATA_KAFKA_01_DIR=${PROJECT_DATA_DIR}/kafka-01
2626
export DATA_DGRAPH_ZERO_01_DIR=${PROJECT_DATA_DIR}/dgraph/zero-01
2727
export DATA_DGRAPH_ALPHA_01_DIR=${PROJECT_DATA_DIR}/dgraph/alpha-01
2828
export DATA_POSTGRESQL_DIR=${PROJECT_DATA_DIR}/postgres
29+
export DATA_POSTGRESQL_DB_DIR=${PROJECT_DATA_DIR}/postgres_db
30+
export POSTGRES_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/postgres
31+
export DGRAPH_BACKUP_DIRECTORY=${PROJECT_DATA_DIR}/backups/dgraph
2932

3033
export SCALE_KEYCLOAK_TEST_SERVER=${SCALE_KEYCLOAK_TEST_SERVER}
3134
export SCALE_NGINX=${SCALE_NGINX}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,14 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -u
5+
6+
pushd .
7+
SCRIPT_DIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
8+
cd ${SCRIPT_DIR}/../..
9+
10+
java_args="${@:1}"
11+
source ./conf.env
12+
docker exec $(docker ps -q -f name=bootstrapper) /bootstrapper.sh $java_args
13+
14+
popd
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/bin/bash
2+
3+
pushd "$(dirname "$0")"
4+
# Creating conf.env file
5+
pushd ./docker/conf/env || exit
6+
source ./create-env-linux-high-1.sh
7+
popd || exit
8+
9+
source ./source/conf.env
10+
11+
java_args="${@:1}"
12+
13+
pushd ../../JeMPI_Apps/JeMPI_Bootstrapper
14+
mvn compile exec:java -Dexec.mainClass="org.jembi.jempi.bootstrapper.BootstrapperCLI" -Dexec.args="$java_args"
15+
popd
16+
popd

devops/windows/base-docker-wsl/helper/scripts/d-stack-01-create-dirs.sh

+5
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,11 @@ pushd .
2727
mkdir -p ${DATA_POSTGRESQL_DIR}
2828
cp conf/postgres/*.* ${DATA_POSTGRESQL_DIR}/.
2929

30+
mkdir -p ${DATA_POSTGRESQL_DB_DIR}
31+
sudo chown -R 1001:1001 ${DATA_POSTGRESQL_DB_DIR}
32+
sudo chmod -R 770 ${DATA_POSTGRESQL_DB_DIR}
33+
34+
3035
echo
3136

3237
popd

devops/windows/base-docker-wsl/helper/scripts/d-stack-03-up-hub-containers.sh

-7
Original file line numberDiff line numberDiff line change
@@ -23,11 +23,4 @@ pushd .
2323
docker service scale ${STACK_NAME}_ratel=${SCALE_RATEL}
2424

2525

26-
pushd helper/topics
27-
source ./topics-create.sh
28-
source ./topics-list.sh
29-
popd
30-
pushd helper/postgres
31-
source ./create-schema.sh
32-
popd
3326
popd
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
#!/bin/bash
2+
3+
set -e
4+
set -u
5+
6+
echo
7+
echo "Down stack"
8+
source ./helper/scripts/d-stack-08-rm.sh
9+
source ./helper/scripts/d-stack-09-wait-removed.sh
10+
11+
echo
12+
echo "Up app containers"
13+
source ./helper/scripts/d-stack-01-create-dirs.sh
14+
sleep 2
15+
source ./helper/scripts/d-stack-02-deploy-0.sh
16+
sleep 2
17+
source ./helper/scripts/d-stack-03-up-hub-containers.sh
18+
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,80 @@
1+
#!/bin/bash
2+
echo "$PWD"
3+
source ../../base-docker-wsl/conf.env
4+
# Load Environment Variables for DGraph Alpha and Zero Nodes
5+
DGRAPH_ALPHA_HOSTS="${DGRAPH_HOSTS:-localhost}"
6+
DGRAPH_ALPHA_PORTS="${DGRAPH_PORTS:-8080}"
7+
DGRAPH_ZERO_HOSTS="${DGRAPH_ZERO_HOSTS:-localhost}"
8+
DGRAPH_ZERO_PORTS="${DGRAPH_ZERO_PORTS:-5080}"
9+
10+
# Load Environment Variables for Data Directories
11+
DATA_DGRAPH_ZERO_01_DIR="${DATA_DGRAPH_ZERO_01_DIR}"
12+
DATA_DGRAPH_ALPHA_01_DIR="${DATA_DGRAPH_ALPHA_01_DIR}"
13+
DATA_DGRAPH_ALPHA_02_DIR="${DATA_DGRAPH_ALPHA_02_DIR:-}" # Optional
14+
DATA_DGRAPH_ALPHA_03_DIR="${DATA_DGRAPH_ALPHA_03_DIR:-}" # Optional
15+
BACKUP_PATH="${DGRAPH_BACKUP_DIRECTORY}/$(date +%Y%m%d_%H%M%S)"
16+
REMOTE_SERVER="${DGRAPH_BACKUP_REMOTE_SERVER}"
17+
REMOTE_PATH="${DGRAPH_BACKUP_REMOTE_PATH}"
18+
19+
# Create Backup Directory if it doesn't exist
20+
[ ! -d "$BACKUP_PATH" ] && mkdir -p "$BACKUP_PATH"
21+
22+
LOG_FILE="${BACKUP_PATH}/dgraph_backup_$(date +%Y%m%d_%H%M%S).log"
23+
24+
# Function to Backup DGraph Alpha and Zero Nodes
25+
backup_dgraph_node() {
26+
local host=$1
27+
local port=$2
28+
29+
echo "$(date) - Starting backup for DGraph node at ${host}:${port}" >> "${LOG_FILE}"
30+
# Replace with actual backup command for the node. Example:
31+
# curl "http://${host}:${port}/admin/backup"
32+
echo "$(date) - Backup completed for DGraph node at ${host}:${port}" >> "${LOG_FILE}"
33+
}
34+
35+
# Function to Backup DGraph Directory
36+
backup_dgraph_dir() {
37+
local dir=$1
38+
local dir_name=$(basename "$dir")
39+
echo "Starting Dgraph database Backup..."
40+
echo "$(date) - Starting backup for DGraph directory at $dir" >> "${LOG_FILE}"
41+
tar -czvf "${BACKUP_PATH}/${dir_name}_$(date +%Y%m%d_%H%M%S).tar.gz" -C "$dir" .
42+
echo "$(date) - Backup completed for DGraph directory at $dir" >> "${LOG_FILE}"
43+
echo "Database Dgraph Backup completed."
44+
}
45+
46+
47+
# Backup DGraph Nodes (Alphas and Zeros)
48+
IFS=',' read -r -a alpha_hosts <<< "$DGRAPH_ALPHA_HOSTS"
49+
IFS=',' read -r -a alpha_ports <<< "$DGRAPH_ALPHA_PORTS"
50+
IFS=',' read -r -a zero_hosts <<< "$DGRAPH_ZERO_HOSTS"
51+
IFS=',' read -r -a zero_ports <<< "$DGRAPH_ZERO_PORTS"
52+
53+
for i in "${!alpha_hosts[@]}"; do
54+
backup_dgraph_node "${alpha_hosts[i]}" "${alpha_ports[i]}"
55+
done
56+
57+
for i in "${!zero_hosts[@]}"; do
58+
backup_dgraph_node "${zero_hosts[i]}" "${zero_ports[i]}"
59+
done
60+
61+
# Backup DGraph Directories
62+
[ -d "$DATA_DGRAPH_ZERO_01_DIR" ] && backup_dgraph_dir "$DATA_DGRAPH_ZERO_01_DIR"
63+
[ -d "$DATA_DGRAPH_ALPHA_01_DIR" ] && backup_dgraph_dir "$DATA_DGRAPH_ALPHA_01_DIR"
64+
[ -d "$DATA_DGRAPH_ALPHA_02_DIR" ] && backup_dgraph_dir "$DATA_DGRAPH_ALPHA_02_DIR"
65+
[ -d "$DATA_DGRAPH_ALPHA_03_DIR" ] && backup_dgraph_dir "$DATA_DGRAPH_ALPHA_03_DIR"
66+
67+
# Function to Copy Backup to Remote Server
68+
copy_to_remote() {
69+
if [ -n "${REMOTE_SERVER}" ] && [ -n "${REMOTE_PATH}" ]; then
70+
echo "$(date) - Starting remote transfer" >> "${LOG_FILE}"
71+
scp "${BACKUP_PATH}/*_$(date +%Y%m%d).tar.gz" ${REMOTE_SERVER}:${REMOTE_PATH}
72+
echo "$(date) - Remote transfer completed" >> "${LOG_FILE}"
73+
else
74+
echo "$(date) - Remote server details not set. Skipping remote transfer." >> "${LOG_FILE}"
75+
fi
76+
}
77+
78+
# Main Execution
79+
copy_to_remote
80+
chmod -R 777 "$BACKUP_PATH"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,87 @@
1+
#!/bin/bash
2+
source ../conf.env
3+
#Backup Folder Name
4+
while true; do
5+
# Ask the user to enter a folder name
6+
echo "Backup folder Path:- ${DGRAPH_BACKUP_DIRECTORY}"
7+
pushd ${DGRAPH_BACKUP_DIRECTORY}
8+
echo
9+
echo "Recent 5 Backups list"
10+
ls -lt --time=creation --sort=time | grep '^d' | tail -n 5
11+
echo
12+
popd
13+
read -p "Please enter your Dgraph Backup Folder Name: " BACKUP_FOLDER_NAME
14+
15+
# Check if the folder exists
16+
if [ -d "${DGRAPH_BACKUP_DIRECTORY}/$BACKUP_FOLDER_NAME" ]; then
17+
echo "Folder '$BACKUP_FOLDER_NAME' exists!"
18+
break # Exit the loop if the folder exists
19+
else
20+
echo "Folder '$BACKUP_FOLDER_NAME' does not exist, at ${DGRAPH_BACKUP_DIRECTORY}. "
21+
echo "Please try again"
22+
fi
23+
done
24+
25+
pushd ../deployment/reboot
26+
source d-stack-3-reboot.sh
27+
popd
28+
echo "Please wait for sometime..."
29+
sleep 60
30+
# BACKUP_FOLDER_NAME="20240202_123952"
31+
# Load Environment Variables for DGraph Alpha and Zero Nodes
32+
DGRAPH_ALPHA_HOSTS="${DGRAPH_HOSTS:-localhost}"
33+
DGRAPH_ALPHA_PORTS="${DGRAPH_PORTS:-8080}"
34+
DGRAPH_ZERO_HOSTS="${DGRAPH_ZERO_HOSTS:-localhost}"
35+
DGRAPH_ZERO_PORTS="${DGRAPH_ZERO_PORTS:-5080}"
36+
37+
# Load Environment Variables for Data Directories
38+
DATA_DGRAPH_ZERO_01_DIR="${DATA_DGRAPH_ZERO_01_DIR}"
39+
DATA_DGRAPH_ALPHA_01_DIR="${DATA_DGRAPH_ALPHA_01_DIR}"
40+
DATA_DGRAPH_ALPHA_02_DIR="${DATA_DGRAPH_ALPHA_02_DIR:-}" # Optional
41+
DATA_DGRAPH_ALPHA_03_DIR="${DATA_DGRAPH_ALPHA_03_DIR:-}" # Optional
42+
BACKUP_PATH="${DGRAPH_BACKUP_DIRECTORY}/$BACKUP_FOLDER_NAME"
43+
44+
REMOTE_SERVER="${DGRAPH_BACKUP_REMOTE_SERVER:-""}"
45+
REMOTE_PATH="${DGRAPH_BACKUP_REMOTE_PATH:-""}"
46+
# Function to Restore DGraph Directory
47+
restore_dgraph_dir() {
48+
local dir=$1
49+
local backup_file=$2
50+
echo $dir
51+
if [ "$(ls -A $dir)" ]; then
52+
echo "Directory $dir is not empty. Clearing existing files."
53+
sudo rm -rf $dir/*
54+
fi
55+
56+
echo$(date) - Starting restore for DGraph directory at $dir from $backup_file
57+
sudo tar -xzvf ${backup_file} -C $dir --strip-components=1
58+
echo$(date) - Restore completed for DGraph directory at $dir
59+
}
60+
# Restore Zero Nodes
61+
for backup_file in ${BACKUP_PATH}/zero*.tar.gz; do
62+
echo $backup_file
63+
# Assuming the first directory is for zero nodes
64+
restore_dgraph_dir $DATA_DGRAPH_ZERO_01_DIR $backup_file
65+
done
66+
# Restore Alpha Nodes
67+
for backup_file in ${BACKUP_PATH}/alpha*.tar.gz; do
68+
# Assuming the first directory is for alpha nodes
69+
restore_dgraph_dir $DATA_DGRAPH_ALPHA_01_DIR $backup_file
70+
done
71+
72+
copy_from_remote() {
73+
if [ -n "${REMOTE_SERVER}" ] && [ -n "${REMOTE_PATH}" ]; then
74+
echo "$(date) - Starting remote copy"
75+
scp ${REMOTE_SERVER}:${REMOTE_PATH}/*_$(date +%Y%m%d).tar.gz "${BACKUP_PATH}/"
76+
echo "$(date) - Remote copy completed"
77+
else
78+
echo "$(date) - Remote server details not set. Skipping remote copy."
79+
fi
80+
}
81+
# Main Execution
82+
copy_from_remote
83+
echo "Rebooting JeMPI"
84+
pushd ../deployment/reboot
85+
echo $PWD
86+
source d-stack-3-reboot.sh
87+
popd
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
#!/bin/bash
2+
source ../conf.env
3+
# Load Backup Directory from Environment Variable
4+
POSTGRES_BACKUP_PATH="${POSTGRES_BACKUP_DIRECTORY}"
5+
OLD_LOGS_DIR="${BACKUP_PATH}/old_logs"
6+
7+
# Create Logrotate Configuration File
8+
cat << EOF > /etc/logrotate.d/postgres-backup
9+
$POSTGRES_BACKUP_PATH/backup_*.log {
10+
rotate 30
11+
daily
12+
missingok
13+
notifempty
14+
compress
15+
delaycompress
16+
create 640 root adm
17+
dateext
18+
dateformat -%Y%m%d
19+
olddir $OLD_LOGS_DIR
20+
}
21+
EOF
22+
23+
echo "Logrotate configuration for PostgreSQL backup updated."
24+
25+
DGRAPH_BACKUP_PATH="${DGRAPH_BACKUP_DIRECTORY}"
26+
OLD_LOGS_DIR="${BACKUP_PATH}/old_logs"
27+
28+
# Create Logrotate Configuration File
29+
cat << EOF > /etc/logrotate.d/dgraph-backup
30+
$DGRAPH_BACKUP_PATH/backup_*.log {
31+
rotate 30
32+
daily
33+
missingok
34+
notifempty
35+
compress
36+
delaycompress
37+
create 640 root adm
38+
dateext
39+
dateformat -%Y%m%d
40+
olddir $OLD_LOGS_DIR
41+
}
42+
EOF
43+
44+
echo "Logrotate configuration for DGraph backup updated."
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
#!/bin/bash
2+
source ../../base-docker-wsl/conf.env
3+
# Load Database Credentials from Environment Variables
4+
5+
DB_NAME="${POSTGRESQL_DATABASE}"
6+
DB_USER="${POSTGRESQL_USERNAME}"
7+
DB_PASSWORD="${POSTGRESQL_PASSWORD}"
8+
DB_HOST="${POSTGRES_HOST:-localhost}"
9+
BACKUP_PATH="${POSTGRES_BACKUP_DIRECTORY}/$(date +%Y%m%d_%H%M%S)"
10+
OLD_LOGS_DIR="${BACKUP_PATH}/old_logs" # Directory to store old logs
11+
12+
databases=("$POSTGRESQL_DATABASE" "$POSTGRESQL_USERS_DB" "$POSTGRESQL_NOTIFICATIONS_DB" "$POSTGRESQL_AUDIT_DB" "$POSTGRESQL_KC_TEST_DB")
13+
14+
# Check and Create Backup Directory and Old Logs Directory
15+
[ ! -d "$BACKUP_PATH" ] && mkdir -p "$BACKUP_PATH"
16+
[ ! -d "$OLD_LOGS_DIR" ] && mkdir -p "$OLD_LOGS_DIR"
17+
18+
LOG_FILE="${BACKUP_PATH}/$(date +%Y%m%d_%H%M%S).log"
19+
20+
# Check for Remote Server Details
21+
REMOTE_SERVER="${POSTGRES_BACKUP_REMOTE_SERVER}"
22+
REMOTE_PATH="${POSTGRES_BACKUP_REMOTE_PATH}"
23+
PGPORT="${PGPORT:-5432}"
24+
25+
# Function to Perform Backup
26+
backup_database() {
27+
echo "Starting Postgres database Backup..."
28+
# Loop through each database and dump it
29+
for db in "${databases[@]}"; do
30+
echo "db.. $db "
31+
backup_file="${BACKUP_PATH}/${db}--$(date +%Y%m%d_%H%M%S).sql"
32+
33+
echo "$(date) - Starting backup for database: ${db}" >> "${LOG_FILE}"
34+
PGPASSWORD=$DB_PASSWORD pg_dump -h $DB_HOST -U $DB_USER -d $db -F c -f "${BACKUP_PATH}/${db}--$(date +%Y%m%d_%H%M%S).sql"
35+
echo "$(date) - Backup completed for database: ${db}" >> "${LOG_FILE}"
36+
done
37+
echo "Database Postgres Backup completed."
38+
}
39+
40+
echo Function to Copy Backup to Remote Server
41+
copy_to_remote() {
42+
if [ -n "${REMOTE_SERVER}" ] && [ -n "${REMOTE_PATH}" ]; then
43+
for db in "${databases[@]}"; do
44+
echo "$(date) - Starting remote transfer" >> "${LOG_FILE}"
45+
scp "${BACKUP_PATH}/${db}_$(date +%Y%m%d_%H%M%S).sql" ${REMOTE_SERVER}:${REMOTE_PATH}
46+
echo "$(date) - Remote transfer completed" >> "${LOG_FILE}"
47+
done
48+
else
49+
echo "$(date) - Remote server details not set. Skipping remote transfer." >> "${LOG_FILE}"
50+
fi
51+
}
52+
53+
# # Main Execution
54+
backup_database
55+
copy_to_remote

0 commit comments

Comments
 (0)