Skip to content

Commit b718936

Browse files
casey-rapnicki-bixalcasey-rapnicki-bixal
committed
Database backup pipeline.
1 parent 675a103 commit b718936

File tree

4 files changed

+382
-0
lines changed

4 files changed

+382
-0
lines changed

‎.github/workflows/database-backup.yml

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Database Backup
2+
3+
on:
4+
push:
5+
branches:
6+
- feature/DIGITAL-178-database-backup
7+
workflow_call:
8+
secrets:
9+
CF_USER:
10+
required: true
11+
CF_PASSWORD:
12+
required: true
13+
CF_ORG:
14+
required: true
15+
PROJECT:
16+
required: true
17+
DATABASE_BACKUP_BASTION_NAME:
18+
required: true
19+
schedule:
20+
- cron: "0 0 * * *"
21+
22+
jobs:
23+
backup-database:
24+
runs-on: ubuntu-latest
25+
continue-on-error: true
26+
#if: github.ref_protected == true
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
- name: Set env
31+
run: |
32+
#BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
33+
BRANCH=develop
34+
COMPOSER_DEV=1
35+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_DEVELOPMENT_KEY }}
36+
case ${BRANCH} in
37+
develop)
38+
CF_SPACE="dev"
39+
DRUPAL_MEMORY=${{ vars.DEVELOP_CMS_MEMORY }}
40+
DRUPAL_INSTANCES=${{ vars.DEVELOP_INSTANCES }}
41+
;;
42+
main)
43+
CF_SPACE="prod"
44+
COMPOSER_DEV=0
45+
DRUPAL_MEMORY=${{ vars.MAIN_CMS_MEMORY }}
46+
DRUPAL_INSTANCES=${{ vars.MAIN_INSTANCES }}
47+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_PRODUCTION_KEY }}
48+
;;
49+
stage)
50+
CF_SPACE="staging"
51+
COMPOSER_DEV=0
52+
DRUPAL_MEMORY=${{ vars.STAGE_CMS_MEMORY }}
53+
DRUPAL_INSTANCES=${{ vars.STAGE_INSTANCES }}
54+
;;
55+
esac
56+
57+
echo "APP_NAME=drupal" | tee -a $GITHUB_ENV
58+
echo "BRANCH=${BRANCH}" | tee -a $GITHUB_ENV
59+
echo "BUILDPACK_PORT=${{ vars.BUILDPACK_PORT }}" | tee -a $GITHUB_ENV
60+
echo "CF_SPACE=${CF_SPACE}" | tee -a $GITHUB_ENV
61+
echo "COMPOSER_DEV=${COMPOSER_DEV}" | tee -a $GITHUB_ENV
62+
echo "DRUPAL_INSTANCES=${DRUPAL_INSTANCES}" | tee -a $GITHUB_ENV
63+
echo "DRUPAL_MEMORY=${DRUPAL_MEMORY}" | tee -a $GITHUB_ENV
64+
echo "GSA_AUTH_KEY=${GSA_AUTH_KEY}" | tee -a $GITHUB_ENV
65+
echo "HASH_SALT=${{ secrets.HASH_SALT }}" | tee -a $GITHUB_ENV
66+
echo "WAF_NAME=waf"| tee -a $GITHUB_ENV
67+
68+
if [ "${COMPOSER_DEV}" = "1" ]; then
69+
sed -i 's/--no-dev //' .bp-config/options.json || exit 0
70+
fi
71+
- name: Install basic dependancies
72+
run: ./scripts/pipeline/deb-basic-deps.sh
73+
- name: Install AWSCLI
74+
run: ./scripts/pipeline/deb-awscli.sh
75+
- name: Install MySQL Client
76+
run: ./scripts/pipeline/deb-mysql-client-install.sh
77+
- name: Install Cloudfoundry CLI
78+
run: ./scripts/pipeline/deb-cf-install.sh
79+
- name: Cloud.gov login
80+
env:
81+
CF_USER: "${{ secrets.CF_USER }}"
82+
CF_PASSWORD: "${{ secrets.CF_PASSWORD }}"
83+
CF_ORG: "${{ secrets.CF_ORG }}"
84+
PROJECT: "${{ secrets.PROJECT }}"
85+
run: |
86+
source ./scripts/pipeline/cloud-gov-login.sh
87+
- name: Start Bastion
88+
env:
89+
DATABASE_BACKUP_BASTION_NAME: "${{ secrets.DATABASE_BACKUP_BASTION_NAME }}"
90+
PROJECT: "${{ secrets.PROJECT }}"
91+
run: |
92+
cf start "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}" >/dev/null 2>&1
93+
./scripts/pipeline/cloud-gov-wait-for-app-start.sh "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}"
94+
- name: Backup Database (dev)
95+
id: backup
96+
shell: bash
97+
env:
98+
CF_USER: "${{ secrets.CF_USER }}"
99+
CF_PASSWORD: "${{ secrets.CF_PASSWORD }}"
100+
CF_ORG: "${{ secrets.CF_ORG }}"
101+
PROJECT: "${{ secrets.PROJECT }}"
102+
DATABASE_BACKUP_BASTION_NAME: "${{ secrets.DATABASE_BACKUP_BASTION_NAME }}"
103+
run: |
104+
export TIMESTAMP=$(date --utc +%FT%TZ | tr ':', '-')
105+
source ./scripts/pipeline/database-backup.sh
106+
source ./scripts/pipeline/s3-backup-upload.sh
107+
stopBastion:
108+
name: Stop Bastion
109+
runs-on: ubuntu-latest
110+
needs: backup-database
111+
#if: github.ref_protected == true && ${{ always() }}
112+
if: ${{ always() }}
113+
steps:
114+
- name: Checkout
115+
uses: actions/checkout@v4
116+
- name: Set env
117+
run: |
118+
#BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
119+
BRANCH=develop
120+
COMPOSER_DEV=1
121+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_DEVELOPMENT_KEY }}
122+
case ${BRANCH} in
123+
develop)
124+
CF_SPACE="dev"
125+
DRUPAL_MEMORY=${{ vars.DEVELOP_CMS_MEMORY }}
126+
DRUPAL_INSTANCES=${{ vars.DEVELOP_INSTANCES }}
127+
;;
128+
main)
129+
CF_SPACE="prod"
130+
COMPOSER_DEV=0
131+
DRUPAL_MEMORY=${{ vars.MAIN_CMS_MEMORY }}
132+
DRUPAL_INSTANCES=${{ vars.MAIN_INSTANCES }}
133+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_PRODUCTION_KEY }}
134+
;;
135+
stage)
136+
CF_SPACE="staging"
137+
COMPOSER_DEV=0
138+
DRUPAL_MEMORY=${{ vars.STAGE_CMS_MEMORY }}
139+
DRUPAL_INSTANCES=${{ vars.STAGE_INSTANCES }}
140+
;;
141+
esac
142+
143+
echo "APP_NAME=drupal" | tee -a $GITHUB_ENV
144+
echo "BRANCH=${BRANCH}" | tee -a $GITHUB_ENV
145+
echo "BUILDPACK_PORT=${{ vars.BUILDPACK_PORT }}" | tee -a $GITHUB_ENV
146+
echo "CF_SPACE=${CF_SPACE}" | tee -a $GITHUB_ENV
147+
echo "COMPOSER_DEV=${COMPOSER_DEV}" | tee -a $GITHUB_ENV
148+
echo "DRUPAL_INSTANCES=${DRUPAL_INSTANCES}" | tee -a $GITHUB_ENV
149+
echo "DRUPAL_MEMORY=${DRUPAL_MEMORY}" | tee -a $GITHUB_ENV
150+
echo "GSA_AUTH_KEY=${GSA_AUTH_KEY}" | tee -a $GITHUB_ENV
151+
echo "HASH_SALT=${{ secrets.HASH_SALT }}" | tee -a $GITHUB_ENV
152+
echo "WAF_NAME=waf"| tee -a $GITHUB_ENV
153+
154+
if [ "${COMPOSER_DEV}" = "1" ]; then
155+
sed -i 's/--no-dev //' .bp-config/options.json || exit 0
156+
fi
157+
- name: Install basic dependancies
158+
run: ./scripts/pipeline/deb-basic-deps.sh
159+
- name: Install Cloudfoundry CLI
160+
run: ./scripts/pipeline/deb-cf-install.sh
161+
- name: Cloud.gov login
162+
env:
163+
CF_USER: "${{ secrets.CF_USER }}"
164+
CF_PASSWORD: "${{ secrets.CF_PASSWORD }}"
165+
CF_ORG: "${{ secrets.CF_ORG }}"
166+
PROJECT: "${{ secrets.PROJECT }}"
167+
run: |
168+
source ./scripts/pipeline/cloud-gov-login.sh
169+
- name: Stop Bastion
170+
env:
171+
PROJECT: "${{ secrets.PROJECT }}"
172+
DATABASE_BACKUP_BASTION_NAME: "${{ secrets.DATABASE_BACKUP_BASTION_NAME }}"
173+
run: |
174+
cf stop "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}" >/dev/null 2>&1

‎scripts/pipeline/database-backup.sh

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,164 @@
1+
#!/bin/bash
2+
3+
kill_pids() {
4+
app=$1
5+
ids=$(ps aux | grep "${app}" | grep -v grep | awk '{print $2}')
6+
for id in ${ids}; do
7+
disown "${id}"
8+
kill -9 "${id}" >/dev/null 2>&1
9+
done
10+
}
11+
12+
## Wait for the tunnel to finish connecting.
13+
wait_for_tunnel() {
14+
while : ; do
15+
[ -n "$(grep 'Press Control-C to stop.' backup.txt)" ] && break
16+
echo "Waiting for tunnel..."
17+
sleep 1
18+
done
19+
}
20+
21+
date
22+
23+
## Create a tunnel through the application to pull the database.
24+
echo "Creating tunnel to database..."
25+
#cf enable-ssh "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}"
26+
#cf restart --strategy rolling "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}"
27+
cf connect-to-service --no-client "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}" "${PROJECT}-mysql-${CF_SPACE}" > backup.txt &
28+
29+
wait_for_tunnel
30+
31+
date
32+
33+
## Create variables and credential file for MySQL login.
34+
echo "Backing up '${CF_SPACE}' database..."
35+
{
36+
host=$(cat backup.txt | grep -i host | awk '{print $2}')
37+
port=$(cat backup.txt | grep -i port | awk '{print $2}')
38+
username=$(cat backup.txt | grep -i username | awk '{print $2}')
39+
password=$(cat backup.txt | grep -i password | awk '{print $2}')
40+
dbname=$(cat backup.txt | grep -i '^name' | awk '{print $2}')
41+
42+
mkdir ~/.mysql && chmod 0700 ~/.mysql
43+
44+
echo "[mysqldump]" > ~/.mysql/mysqldump.cnf
45+
echo "user=${username}" >> ~/.mysql/mysqldump.cnf
46+
echo "password=${password}" >> ~/.mysql/mysqldump.cnf
47+
chmod 400 ~/.mysql/mysqldump.cnf
48+
49+
## Exclude tables without data
50+
declare -a excluded_tables=(
51+
"cache_access_policy"
52+
"cache_bootstrap"
53+
"cache_config"
54+
"cache_container"
55+
"cache_default"
56+
"cache_discovery"
57+
"cache_dynamic_page_cache"
58+
"cache_entity"
59+
"cache_menu"
60+
"cache_page"
61+
"cache_render"
62+
"cache_tome_static"
63+
"cache_toolbar"
64+
"sessions"
65+
"watchdog"
66+
)
67+
68+
ignored_tables_string=''
69+
for table in "${excluded_tables[@]}"
70+
do
71+
ignored_tables_string+=" --ignore-table=${dbname}.${table}"
72+
done
73+
} >/dev/null 2>&1
74+
75+
echo "Dumping structure..."
76+
{
77+
## Dump structure
78+
mysqldump \
79+
--defaults-extra-file=~/.mysql/mysqldump.cnf \
80+
--host="${host}" \
81+
--port="${port}" \
82+
--protocol=TCP \
83+
--no-data \
84+
"${dbname}" > "backup_${CF_SPACE}.sql"
85+
} >/dev/null 2>&1
86+
87+
echo "Dumping content..."
88+
{
89+
## Dump content
90+
mysqldump \
91+
--defaults-extra-file=~/.mysql/mysqldump.cnf \
92+
--host="${host}" \
93+
--port="${port}" \
94+
--protocol=TCP \
95+
--no-create-info \
96+
--skip-triggers \
97+
${ignored_tables_string} \
98+
"${dbname}" >> "backup_${CF_SPACE}.sql"
99+
100+
## Patch out any MySQL 'SET' commands that require admin.
101+
sed -i 's/^SET /-- &/' "backup_${CF_SPACE}.sql"
102+
} >/dev/null 2>&1
103+
104+
date
105+
106+
## Kill the backgrounded SSH tunnel.
107+
echo "Cleaning up old connections..."
108+
{
109+
kill_pids "connect-to-service"
110+
}
111+
112+
## Disable ssh.
113+
#echo "Disabling ssh..."
114+
#cf disable-ssh "${PROJECT}-drupal-${CF_SPACE}"
115+
116+
rm -rf backup.txt ~/.mysql
117+
118+
echo "Compressing '${CF_SPACE}' database..."
119+
{
120+
mv "backup_${BRANCH}.sql" "${TIMESTAMP}.sql"
121+
gzip "${TIMESTAMP}.sql"
122+
} &> /dev/null
123+
124+
echo "Creating S3 credentials..."
125+
{
126+
cf target -s "${PROJECT}-${CF_SPACE}" >/dev/null 2>&1
127+
128+
export service="${PROJECT}-backup-${CF_SPACE}"
129+
export service_key="${service}-key"
130+
cf delete-service-key "${service}" "${service_key}" -f >/dev/null 2>&1
131+
cf create-service-key "${service}" "${service_key}" >/dev/null 2>&1
132+
sleep 2
133+
134+
s3_credentials=$(cf service-key "${service}" "${service_key}" | tail -n +2)
135+
export s3_credentials
136+
137+
AWS_ACCESS_KEY_ID=$(echo "${s3_credentials}" | jq -r '.credentials.access_key_id')
138+
export AWS_ACCESS_KEY_ID
139+
140+
bucket=$(echo "${s3_credentials}" | jq -r '.credentials.bucket')
141+
export bucket
142+
143+
AWS_DEFAULT_REGION=$(echo "${s3_credentials}" | jq -r '.credentials.region')
144+
export AWS_DEFAULT_REGION
145+
146+
AWS_SECRET_ACCESS_KEY=$(echo "${s3_credentials}" | jq -r '.credentials.secret_access_key')
147+
export AWS_SECRET_ACCESS_KEY
148+
} >/dev/null 2>&1
149+
150+
echo "Saving to backup bucket..."
151+
{
152+
# copy latest database to top level
153+
#gzip "backup_${CF_SPACE}.sql"
154+
#aws s3 cp "./backup_${CF_SPACE}.sql.gz" "s3://${bucket}/${CF_SPACE}/latest.sql.gz" --no-verify-ssl >/dev/null 2>&1 && echo "Successfully copied latest.sql.gz to S3!" || echo "Failed to copy latest.sql.gz to S3!"
155+
aws s3 cp "${TIMESTAMP}.sql.gz" "s3://${bucket}/$(date +%Y)/$(date +%m)/$(date +%d)/" --no-verify-ssl 2>/dev/null
156+
aws s3 cp "${TIMESTAMP}.sql.gz" "s3://${bucket}/latest.sql.gz" --no-verify-ssl 2>/dev/null
157+
} >/dev/null 2>&1
158+
159+
echo "Cleaning up credentials..."
160+
{
161+
cf delete-service-key "${service}" "${service_key}" -f >/dev/null 2>&1
162+
} >/dev/null 2>&1
163+
164+
date

‎scripts/pipeline/deb-cf-install.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,7 @@ echo "Installing CloudFoundry repository..."
1414
else
1515
mv cf cf8 /usr/local/bin
1616
fi
17+
18+
cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.4/cf-service-connect_linux_amd64
19+
1720
} >/dev/null 2>&1

‎scripts/pipeline/github-exports.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
3+
#BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
4+
BRANCH=develop
5+
COMPOSER_DEV=1
6+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_DEVELOPMENT_KEY }}
7+
case ${BRANCH} in
8+
develop)
9+
CF_SPACE="dev"
10+
DRUPAL_MEMORY=${{ vars.DEVELOP_CMS_MEMORY }}
11+
DRUPAL_INSTANCES=${{ vars.DEVELOP_INSTANCES }}
12+
;;
13+
main)
14+
CF_SPACE="prod"
15+
COMPOSER_DEV=0
16+
DRUPAL_MEMORY=${{ vars.MAIN_CMS_MEMORY }}
17+
DRUPAL_INSTANCES=${{ vars.MAIN_INSTANCES }}
18+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_PRODUCTION_KEY }}
19+
;;
20+
stage)
21+
CF_SPACE="staging"
22+
COMPOSER_DEV=0
23+
DRUPAL_MEMORY=${{ vars.STAGE_CMS_MEMORY }}
24+
DRUPAL_INSTANCES=${{ vars.STAGE_INSTANCES }}
25+
;;
26+
esac
27+
28+
echo "APP_NAME=drupal" | tee -a $GITHUB_ENV
29+
echo "BRANCH=${BRANCH}" | tee -a $GITHUB_ENV
30+
echo "BUILDPACK_PORT=${{ vars.BUILDPACK_PORT }}" | tee -a $GITHUB_ENV
31+
echo "CF_SPACE=${CF_SPACE}" | tee -a $GITHUB_ENV
32+
echo "COMPOSER_DEV=${COMPOSER_DEV}" | tee -a $GITHUB_ENV
33+
echo "DRUPAL_INSTANCES=${DRUPAL_INSTANCES}" | tee -a $GITHUB_ENV
34+
echo "DRUPAL_MEMORY=${DRUPAL_MEMORY}" | tee -a $GITHUB_ENV
35+
echo "GSA_AUTH_KEY=${GSA_AUTH_KEY}" | tee -a $GITHUB_ENV
36+
echo "HASH_SALT=${{ secrets.HASH_SALT }}" | tee -a $GITHUB_ENV
37+
echo "WAF_NAME=waf"| tee -a $GITHUB_ENV
38+
39+
if [ "${COMPOSER_DEV}" = "1" ]; then
40+
sed -i 's/--no-dev //' .bp-config/options.json || exit 0
41+
fi

0 commit comments

Comments
 (0)