Skip to content

Commit ca2148c

Browse files
MatthewErispeMatthewErispe
committed
code cleanup and nitpick
1 parent 1f75059 commit ca2148c

File tree

10 files changed

+65
-53
lines changed

10 files changed

+65
-53
lines changed

‎JeMPI_Apps/JeMPI_BackupRestoreAPI/src/main/java/org/jembi/jempi/backuprestoreapi/PsqlClient.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ boolean connect() {
3838
connection = DriverManager.getConnection(url, user, password);
3939
return connection.isValid(5);
4040
} catch (SQLException e) {
41-
LOGGER.error("{} {} {}", url, user, password);
41+
LOGGER.error("Connection error with URL: {}", url);
4242
LOGGER.error(e.getLocalizedMessage(), e);
4343
connection = null;
4444
return false;

‎JeMPI_Apps/JeMPI_LibAPI/src/main/java/org/jembi/jempi/libapi/PsqlClient.java

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ boolean connect() {
3838
connection = DriverManager.getConnection(url, user, password);
3939
return connection.isValid(5);
4040
} catch (SQLException e) {
41-
LOGGER.error("{} {} {}", url, user, password);
41+
LOGGER.error("Connection error with URL: {}", url);
4242
LOGGER.error(e.getLocalizedMessage(), e);
4343
connection = null;
4444
return false;

‎devops/linux/docker/backup_restore/dgraph-backup-api.sh

+12-14
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,9 @@ set -u
55
pushd .
66
BACKUP_DATE_TIME=$1
77
SCRIPT_DIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
8-
cd ${SCRIPT_DIR}/..
8+
cd "${SCRIPT_DIR}/.." || exit
99
JEMPI_DOCKER_HOME=$PWD
10-
# JEMPI_HOME = $1
10+
1111
down_dir="$JEMPI_DOCKER_HOME/deployment/down"
1212
reboot_dir="$JEMPI_DOCKER_HOME/deployment/reboot"
1313
backup_restore_dir="$JEMPI_DOCKER_HOME/backup_restore"
@@ -16,46 +16,44 @@ pushd .
1616
echo $python_cmd
1717
# Function to stop services
1818
stop_services() {
19-
pushd "$down_dir"
19+
pushd "$down_dir" || exit
2020
echo "Stopping API service"
2121
source d-stack-stop-services.sh
22-
popd
22+
popd || exit
2323
}
2424

2525
# Function to start backup restore API service
2626
start_backup_restore_service() {
27-
pushd "$reboot_dir"
27+
pushd "$reboot_dir" || exit
2828
echo "Starting Backup Restore API service"
2929
source d-stack-start-backup-restore-api-services.sh
30-
popd
30+
popd || exit
3131
}
3232

3333
# Function to backup data
3434
backup_data() {
35-
pushd "$backup_restore_dir"
35+
pushd "$backup_restore_dir" || exit
3636
sleep 20
3737
echo "Started Backup through API"
3838
$python_cmd dgraph-backup.py $BACKUP_DATE_TIME
3939
sleep 10
40-
# sudo bash dgraph-backup.sh
41-
# sudo bash postgres-backup.sh
42-
popd
40+
popd || exit
4341
}
4442

4543
# Function to start services
4644
start_services() {
47-
pushd "$reboot_dir"
45+
pushd "$reboot_dir" || exit
4846
echo "Starting API service"
4947
source d-stack-start-services.sh
50-
popd
48+
popd || exit
5149
}
5250

5351
# Function to stop backup restore API service
5452
stop_backup_restore_service() {
55-
pushd "$down_dir"
53+
pushd "$down_dir" || exit
5654
echo "Stopping Backup Restore API service"
5755
source d-stack-stop-backup-restore-api-services.sh
58-
popd
56+
popd || exit
5957
}
6058

6159
stop_services

‎devops/linux/docker/backup_restore/dgraph-backup.py

+12-10
Original file line numberDiff line numberDiff line change
@@ -26,8 +26,11 @@ def fetch_data_for_id(gid):
2626
payload = json.dumps({"gid": gid})
2727
headers = {'Content-Type': 'application/json'}
2828
response = requests.post(get_expanded_golden_record, headers=headers, data=payload)
29-
return response.json() if response.status_code == 200 else None
30-
29+
if response.status_code == 200:
30+
return response.json()
31+
else:
32+
print(f'Error fetching data for ID {gid}: {response.status_code}')
33+
return None
3134

3235
# Function to backup Dgraph data
3336
def backup_dgraph_data():
@@ -37,21 +40,20 @@ def backup_dgraph_data():
3740
if response.status_code == 200:
3841
new_golden_records = response.json()
3942
backup_data = []
40-
for gid in new_golden_records.get("records"):
41-
golden_record_data = fetch_data_for_id(gid)
42-
if golden_record_data:
43-
backup_data.append(golden_record_data)
44-
43+
for gid in new_golden_records.get("records", []):
44+
try:
45+
golden_record_data = fetch_data_for_id(gid)
46+
if golden_record_data:
47+
backup_data.append(golden_record_data)
48+
except Exception as e:
49+
print(f'Error processing ID {gid}: {e}')
4550
file_name = f'dgraph_backup_{current_datetime}.json'
4651
print(f'Total {str(len(backup_data))} Golden records backed up.')
47-
4852
backup_path_folder = create_backup_json(backup_data, file_name)
49-
5053
print(f'All data saved to {backup_path_folder + "/" + file_name}')
5154
else:
5255
print('Failed to retrieve list of IDs from the API')
5356

54-
5557
def create_backup_json(backup_data, file_name):
5658
backup_path_folder = os.path.join(backup_path, current_datetime)
5759
create_folder_if_not_exists(backup_path_folder)

‎devops/linux/docker/backup_restore/dgraph-restore-api.py

-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
import os
55
from datetime import datetime
66
from dotenv import dotenv_values
7-
import uuid
87

98
env_vars = dotenv_values('../conf.env')
109

‎devops/linux/docker/conf/stack/docker-stack-high-0.yml

+5-5
Original file line numberDiff line numberDiff line change
@@ -339,7 +339,7 @@ services:
339339
condition: on-failure
340340
placement:
341341
constraints:
342-
- node.labels.name == $PLACEMENT_NODE1
342+
- node.labels.name == ${PLACEMENT_NODE1}
343343
command: dgraph alpha --my=alpha-01:7080 --zero=zero-01:5080 --cache "size-mb=10240; percentage=50,30,20;" --security whitelist=0.0.0.0/0 --telemetry "reports=false; sentry=false;"
344344

345345
alpha-02:
@@ -372,7 +372,7 @@ services:
372372
condition: on-failure
373373
placement:
374374
constraints:
375-
- node.labels.name == $PLACEMENT_NODE2
375+
- node.labels.name == ${PLACEMENT_NODE2}
376376
command: dgraph alpha --my=alpha-02:7081 --zero=zero-01:5080 --cache "size-mb=10240; percentage=50,30,20;" --security whitelist=0.0.0.0/0 -o 1 --telemetry "reports=false; sentry=false;"
377377

378378
alpha-03:
@@ -405,7 +405,7 @@ services:
405405
condition: on-failure
406406
placement:
407407
constraints:
408-
- node.labels.name == $PLACEMENT_NODE3
408+
- node.labels.name == ${PLACEMENT_NODE3}
409409
command: dgraph alpha --my=alpha-03:7082 --zero=zero-01:5080 --cache "size-mb=10240; percentage=50,30,20;" --security whitelist=0.0.0.0/0 -o 2 --telemetry "reports=false; sentry=false;"
410410

411411
ratel:
@@ -822,11 +822,11 @@ services:
822822
DGRAPH_PORTS: ${DGRAPH_PORTS}
823823
deploy:
824824
mode: global
825-
825+
826826
backup-restore-api:
827827
image: ${IMAGE_REGISTRY}${BACKUP_RESTORE_API_IMAGE}
828828
environment:
829-
LOG4J2_LEVEL: "DEBUG"
829+
LOG4J2_LEVEL: "DEBUG"
830830
POSTGRESQL_IP: postgresql
831831
POSTGRESQL_PORT: 5432
832832
POSTGRESQL_USER: ${POSTGRESQL_USERNAME}

‎devops/linux/docker/conf/stack/docker-stack-high-1.yml

+9-9
Original file line numberDiff line numberDiff line change
@@ -128,7 +128,7 @@ services:
128128
condition: on-failure
129129
placement:
130130
constraints:
131-
- node.labels.name == $PLACEMENT_NODE1
131+
- node.labels.name == ${PLACEMENT_NODE1}
132132

133133
kafka-02:
134134
image: ${IMAGE_REGISTRY}$KAFKA_IMAGE
@@ -168,7 +168,7 @@ services:
168168
condition: on-failure
169169
placement:
170170
constraints:
171-
- node.labels.name == $PLACEMENT_NODE2
171+
- node.labels.name == ${PLACEMENT_NODE2}
172172

173173
kafka-03:
174174
image: ${IMAGE_REGISTRY}$KAFKA_IMAGE
@@ -208,7 +208,7 @@ services:
208208
condition: on-failure
209209
placement:
210210
constraints:
211-
- node.labels.name == $PLACEMENT_NODE3
211+
- node.labels.name == ${PLACEMENT_NODE3}
212212

213213
zero-01:
214214
image: ${IMAGE_REGISTRY}${DGRAPH_IMAGE}
@@ -240,7 +240,7 @@ services:
240240
condition: on-failure
241241
placement:
242242
constraints:
243-
- node.labels.name == $PLACEMENT_NODE1
243+
- node.labels.name == ${PLACEMENT_NODE1}
244244
command: dgraph zero --my=zero-01:5080 --replicas 1 --telemetry "reports=false; sentry=false;"
245245

246246
zero-02:
@@ -273,7 +273,7 @@ services:
273273
condition: on-failure
274274
placement:
275275
constraints:
276-
- node.labels.name == $PLACEMENT_NODE2
276+
- node.labels.name == ${PLACEMENT_NODE2}
277277
command: dgraph zero --my=zero-02:5080 --raft idx=2 --peer zero-01:5080 --replicas 1 --telemetry "reports=false; sentry=false;"
278278

279279
zero-03:
@@ -306,7 +306,7 @@ services:
306306
condition: on-failure
307307
placement:
308308
constraints:
309-
- node.labels.name == $PLACEMENT_NODE3
309+
- node.labels.name == ${PLACEMENT_NODE3}
310310
command: dgraph zero --my=zero-03:5080 --raft idx=3 --peer zero-01:5080 --replicas 1 --telemetry "reports=false; sentry=false;"
311311

312312
alpha-01:
@@ -339,7 +339,7 @@ services:
339339
condition: on-failure
340340
placement:
341341
constraints:
342-
- node.labels.name == $PLACEMENT_NODE1
342+
- node.labels.name == ${PLACEMENT_NODE1}
343343
command: dgraph alpha --my=alpha-01:7080 --zero=zero-01:5080 --cache "size-mb=10240; percentage=50,30,20;" --security whitelist=0.0.0.0/0 --telemetry "reports=false; sentry=false;"
344344

345345
alpha-02:
@@ -372,7 +372,7 @@ services:
372372
condition: on-failure
373373
placement:
374374
constraints:
375-
- node.labels.name == $PLACEMENT_NODE2
375+
- node.labels.name == ${PLACEMENT_NODE2}
376376
command: dgraph alpha --my=alpha-02:7081 --zero=zero-01:5080 --cache "size-mb=10240; percentage=50,30,20;" --security whitelist=0.0.0.0/0 -o 1 --telemetry "reports=false; sentry=false;"
377377

378378
alpha-03:
@@ -405,7 +405,7 @@ services:
405405
condition: on-failure
406406
placement:
407407
constraints:
408-
- node.labels.name == $PLACEMENT_NODE3
408+
- node.labels.name == ${PLACEMENT_NODE3}
409409
command: dgraph alpha --my=alpha-03:7082 --zero=zero-01:5080 --cache "size-mb=10240; percentage=50,30,20;" --security whitelist=0.0.0.0/0 -o 2 --telemetry "reports=false; sentry=false;"
410410

411411
ratel:

‎devops/linux/docker/deployment/deploy-local.sh

+7-6
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,8 @@ run_enviroment_configuration_and_helper_script(){
8484
# Navigate to environment configuration directory
8585
echo "Navigate to environment configuration directory"
8686
pushd "$JEMPI_HOME/devops/linux/docker/conf/env/"
87-
source $JEMPI_ENV_CONFIGURATION
87+
# shellcheck source=path/to/create-env-linux-low-1.sh
88+
source "$JEMPI_ENV_CONFIGURATION"
8889
popd
8990

9091
# Running Docker helper scripts
@@ -139,7 +140,7 @@ build_all_stack_and_reboot(){
139140
}
140141
initialize_db_build_all_stack_and_reboot(){
141142
echo "Create DB and Deploy"
142-
pushd "$JEMPI_HOME/devops/linux/docker/deployment/install_from_scratch"
143+
pushd "$JEMPI_HOME/devops/linux/docker/deployment/install_from_scratch" || exit
143144
yes | source d-stack-1-create-db-build-all-reboot.sh
144145
popd
145146
}
@@ -217,10 +218,10 @@ case $choice in
217218
6)
218219
BACKUP_DATE_TIME=$(date +%Y-%m-%d_%H%M%S)
219220
echo "Started Backup at- $BACKUP_DATE_TIME"
220-
pushd "$JEMPI_HOME/devops/linux/docker/backup_restore"
221-
source dgraph-backup-api.sh $BACKUP_DATE_TIME
222-
sudo bash postgres-backup.sh $BACKUP_DATE_TIME
223-
popd
221+
pushd "$JEMPI_HOME/devops/linux/docker/backup_restore" || exit
222+
source dgraph-backup-api.sh "$BACKUP_DATE_TIME" || { echo "Dgraph backup failed"; exit 1; }
223+
sudo bash postgres-backup.sh "$BACKUP_DATE_TIME" || { echo "Postgres backup failed"; exit 1; }
224+
popd || exit
224225

225226
;;
226227
7)

‎devops/linux/docker/deployment/reboot/d-stack-start-backup-restore-api-services.sh

+9-3
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,20 @@
22

33
set -e
44
set -u
5-
# SERVICE_NAME = $1
5+
66
pushd .
77
SCRIPT_DIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
8-
cd ${SCRIPT_DIR}/../..
8+
cd "${SCRIPT_DIR}/../.."
9+
10+
# Check if conf.env exists in the current directory
11+
if [ ! -f "./conf.env" ]; then
12+
echo "conf.env does not exist in the current directory."
13+
else
14+
echo "conf.env exists and can be sourced."
15+
fi
916

1017
source ./conf.env
1118

12-
# docker service scale ${SERVICE_NAME}=1
1319
docker service scale jempi_backup-restore-api=1
1420
echo
1521

‎devops/linux/docker/deployment/reboot/d-stack-start-services.sh

+9-3
Original file line numberDiff line numberDiff line change
@@ -2,14 +2,20 @@
22

33
set -e
44
set -u
5-
# SERVICE_NAME = $1
5+
66
pushd .
77
SCRIPT_DIR=$(cd -P -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd -P)
8-
cd ${SCRIPT_DIR}/../..
8+
cd "${SCRIPT_DIR}/../.."
9+
10+
# Check if conf.env exists in the current directory
11+
if [ ! -f "./conf.env" ]; then
12+
echo "conf.env does not exist in the current directory."
13+
else
14+
echo "conf.env exists and can be sourced."
15+
fi
916

1017
source ./conf.env
1118

12-
# docker service scale ${SERVICE_NAME}=1
1319
docker service scale jempi_api=1
1420
docker service scale jempi_async-receiver=1
1521
echo

0 commit comments

Comments
 (0)