Skip to content

Commit e4b404e

Browse files
casey-rapnicki-bixalcasey-rapnicki-bixal
committed
Database backup pipeline.
1 parent 675a103 commit e4b404e

File tree

4 files changed

+381
-0
lines changed

4 files changed

+381
-0
lines changed

‎.github/workflows/database-backup.yml

Lines changed: 174 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,174 @@
1+
name: Database Backup
2+
3+
on:
4+
push:
5+
branches:
6+
- feature/DIGITAL-178-database-backup
7+
workflow_call:
8+
secrets:
9+
CF_USER:
10+
required: true
11+
CF_PASSWORD:
12+
required: true
13+
CF_ORG:
14+
required: true
15+
PROJECT:
16+
required: true
17+
DATABASE_BACKUP_BASTION_NAME:
18+
required: true
19+
schedule:
20+
- cron: "0 0 * * *"
21+
22+
jobs:
23+
backup-database:
24+
runs-on: ubuntu-latest
25+
continue-on-error: true
26+
#if: github.ref_protected == true
27+
steps:
28+
- name: Checkout
29+
uses: actions/checkout@v4
30+
- name: Set env
31+
run: |
32+
#BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
33+
BRANCH=develop
34+
COMPOSER_DEV=1
35+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_DEVELOPMENT_KEY }}
36+
case ${BRANCH} in
37+
develop)
38+
CF_SPACE="dev"
39+
DRUPAL_MEMORY=${{ vars.DEVELOP_CMS_MEMORY }}
40+
DRUPAL_INSTANCES=${{ vars.DEVELOP_INSTANCES }}
41+
;;
42+
main)
43+
CF_SPACE="prod"
44+
COMPOSER_DEV=0
45+
DRUPAL_MEMORY=${{ vars.MAIN_CMS_MEMORY }}
46+
DRUPAL_INSTANCES=${{ vars.MAIN_INSTANCES }}
47+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_PRODUCTION_KEY }}
48+
;;
49+
stage)
50+
CF_SPACE="staging"
51+
COMPOSER_DEV=0
52+
DRUPAL_MEMORY=${{ vars.STAGE_CMS_MEMORY }}
53+
DRUPAL_INSTANCES=${{ vars.STAGE_INSTANCES }}
54+
;;
55+
esac
56+
57+
echo "APP_NAME=drupal" | tee -a $GITHUB_ENV
58+
echo "BRANCH=${BRANCH}" | tee -a $GITHUB_ENV
59+
echo "BUILDPACK_PORT=${{ vars.BUILDPACK_PORT }}" | tee -a $GITHUB_ENV
60+
echo "CF_SPACE=${CF_SPACE}" | tee -a $GITHUB_ENV
61+
echo "COMPOSER_DEV=${COMPOSER_DEV}" | tee -a $GITHUB_ENV
62+
echo "DRUPAL_INSTANCES=${DRUPAL_INSTANCES}" | tee -a $GITHUB_ENV
63+
echo "DRUPAL_MEMORY=${DRUPAL_MEMORY}" | tee -a $GITHUB_ENV
64+
echo "GSA_AUTH_KEY=${GSA_AUTH_KEY}" | tee -a $GITHUB_ENV
65+
echo "HASH_SALT=${{ secrets.HASH_SALT }}" | tee -a $GITHUB_ENV
66+
echo "WAF_NAME=waf"| tee -a $GITHUB_ENV
67+
68+
if [ "${COMPOSER_DEV}" = "1" ]; then
69+
sed -i 's/--no-dev //' .bp-config/options.json || exit 0
70+
fi
71+
- name: Install basic dependancies
72+
run: ./scripts/pipeline/deb-basic-deps.sh
73+
- name: Install AWSCLI
74+
run: ./scripts/pipeline/deb-awscli.sh
75+
- name: Install MySQL Client
76+
run: ./scripts/pipeline/deb-mysql-client-install.sh
77+
- name: Install Cloudfoundry CLI
78+
run: ./scripts/pipeline/deb-cf-install.sh
79+
- name: Cloud.gov login
80+
env:
81+
CF_USER: "${{ secrets.CF_USER }}"
82+
CF_PASSWORD: "${{ secrets.CF_PASSWORD }}"
83+
CF_ORG: "${{ secrets.CF_ORG }}"
84+
PROJECT: "${{ secrets.PROJECT }}"
85+
run: |
86+
source ./scripts/pipeline/cloud-gov-login.sh
87+
- name: Start Bastion
88+
env:
89+
DATABASE_BACKUP_BASTION_NAME: "${{ secrets.DATABASE_BACKUP_BASTION_NAME }}"
90+
PROJECT: "${{ secrets.PROJECT }}"
91+
run: |
92+
cf start "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}" >/dev/null 2>&1
93+
./scripts/pipeline/cloud-gov-wait-for-app-start.sh "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}"
94+
- name: Backup Database (dev)
95+
id: backup
96+
shell: bash
97+
env:
98+
CF_USER: "${{ secrets.CF_USER }}"
99+
CF_PASSWORD: "${{ secrets.CF_PASSWORD }}"
100+
CF_ORG: "${{ secrets.CF_ORG }}"
101+
PROJECT: "${{ secrets.PROJECT }}"
102+
DATABASE_BACKUP_BASTION_NAME: "${{ secrets.DATABASE_BACKUP_BASTION_NAME }}"
103+
run: |
104+
export TIMESTAMP=$(date --utc +%FT%TZ | tr ':', '-')
105+
source ./scripts/pipeline/database-backup.sh
106+
source ./scripts/pipeline/s3-backup-upload.sh
107+
stopBastion:
108+
name: Stop Bastion
109+
runs-on: ubuntu-latest
110+
needs: backup-database
111+
#if: github.ref_protected == true && ${{ always() }}
112+
if: ${{ always() }}
113+
steps:
114+
- name: Checkout
115+
uses: actions/checkout@v4
116+
- name: Set env
117+
run: |
118+
#BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
119+
BRANCH=develop
120+
COMPOSER_DEV=1
121+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_DEVELOPMENT_KEY }}
122+
case ${BRANCH} in
123+
develop)
124+
CF_SPACE="dev"
125+
DRUPAL_MEMORY=${{ vars.DEVELOP_CMS_MEMORY }}
126+
DRUPAL_INSTANCES=${{ vars.DEVELOP_INSTANCES }}
127+
;;
128+
main)
129+
CF_SPACE="prod"
130+
COMPOSER_DEV=0
131+
DRUPAL_MEMORY=${{ vars.MAIN_CMS_MEMORY }}
132+
DRUPAL_INSTANCES=${{ vars.MAIN_INSTANCES }}
133+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_PRODUCTION_KEY }}
134+
;;
135+
stage)
136+
CF_SPACE="staging"
137+
COMPOSER_DEV=0
138+
DRUPAL_MEMORY=${{ vars.STAGE_CMS_MEMORY }}
139+
DRUPAL_INSTANCES=${{ vars.STAGE_INSTANCES }}
140+
;;
141+
esac
142+
143+
echo "APP_NAME=drupal" | tee -a $GITHUB_ENV
144+
echo "BRANCH=${BRANCH}" | tee -a $GITHUB_ENV
145+
echo "BUILDPACK_PORT=${{ vars.BUILDPACK_PORT }}" | tee -a $GITHUB_ENV
146+
echo "CF_SPACE=${CF_SPACE}" | tee -a $GITHUB_ENV
147+
echo "COMPOSER_DEV=${COMPOSER_DEV}" | tee -a $GITHUB_ENV
148+
echo "DRUPAL_INSTANCES=${DRUPAL_INSTANCES}" | tee -a $GITHUB_ENV
149+
echo "DRUPAL_MEMORY=${DRUPAL_MEMORY}" | tee -a $GITHUB_ENV
150+
echo "GSA_AUTH_KEY=${GSA_AUTH_KEY}" | tee -a $GITHUB_ENV
151+
echo "HASH_SALT=${{ secrets.HASH_SALT }}" | tee -a $GITHUB_ENV
152+
echo "WAF_NAME=waf"| tee -a $GITHUB_ENV
153+
154+
if [ "${COMPOSER_DEV}" = "1" ]; then
155+
sed -i 's/--no-dev //' .bp-config/options.json || exit 0
156+
fi
157+
- name: Install basic dependancies
158+
run: ./scripts/pipeline/deb-basic-deps.sh
159+
- name: Install Cloudfoundry CLI
160+
run: ./scripts/pipeline/deb-cf-install.sh
161+
- name: Cloud.gov login
162+
env:
163+
CF_USER: "${{ secrets.CF_USER }}"
164+
CF_PASSWORD: "${{ secrets.CF_PASSWORD }}"
165+
CF_ORG: "${{ secrets.CF_ORG }}"
166+
PROJECT: "${{ secrets.PROJECT }}"
167+
run: |
168+
source ./scripts/pipeline/cloud-gov-login.sh
169+
- name: Stop Bastion
170+
env:
171+
PROJECT: "${{ secrets.PROJECT }}"
172+
DATABASE_BACKUP_BASTION_NAME: "${{ secrets.DATABASE_BACKUP_BASTION_NAME }}"
173+
run: |
174+
cf stop "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}" >/dev/null 2>&1

‎scripts/pipeline/database-backup.sh

Lines changed: 163 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,163 @@
1+
#!/bin/bash
2+
3+
kill_pids() {
4+
app=$1
5+
ids=$(ps aux | grep "${app}" | grep -v grep | awk '{print $2}')
6+
for id in ${ids}; do
7+
kill -9 "${id}" >/dev/null 2>&1
8+
done
9+
}
10+
11+
## Wait for the tunnel to finish connecting.
12+
wait_for_tunnel() {
13+
while : ; do
14+
[ -n "$(grep 'Press Control-C to stop.' backup.txt)" ] && break
15+
echo "Waiting for tunnel..."
16+
sleep 1
17+
done
18+
}
19+
20+
date
21+
22+
## Create a tunnel through the application to pull the database.
23+
echo "Creating tunnel to database..."
24+
#cf enable-ssh "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}"
25+
#cf restart --strategy rolling "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}"
26+
cf connect-to-service --no-client "${PROJECT}-${DATABASE_BACKUP_BASTION_NAME}-${CF_SPACE}" "${PROJECT}-mysql-${CF_SPACE}" > backup.txt &
27+
28+
wait_for_tunnel
29+
30+
date
31+
32+
## Create variables and credential file for MySQL login.
33+
echo "Backing up '${CF_SPACE}' database..."
34+
{
35+
host=$(cat backup.txt | grep -i host | awk '{print $2}')
36+
port=$(cat backup.txt | grep -i port | awk '{print $2}')
37+
username=$(cat backup.txt | grep -i username | awk '{print $2}')
38+
password=$(cat backup.txt | grep -i password | awk '{print $2}')
39+
dbname=$(cat backup.txt | grep -i '^name' | awk '{print $2}')
40+
41+
mkdir ~/.mysql && chmod 0700 ~/.mysql
42+
43+
echo "[mysqldump]" > ~/.mysql/mysqldump.cnf
44+
echo "user=${username}" >> ~/.mysql/mysqldump.cnf
45+
echo "password=${password}" >> ~/.mysql/mysqldump.cnf
46+
chmod 400 ~/.mysql/mysqldump.cnf
47+
48+
## Exclude tables without data
49+
declare -a excluded_tables=(
50+
"cache_access_policy"
51+
"cache_bootstrap"
52+
"cache_config"
53+
"cache_container"
54+
"cache_default"
55+
"cache_discovery"
56+
"cache_dynamic_page_cache"
57+
"cache_entity"
58+
"cache_menu"
59+
"cache_page"
60+
"cache_render"
61+
"cache_tome_static"
62+
"cache_toolbar"
63+
"sessions"
64+
"watchdog"
65+
)
66+
67+
ignored_tables_string=''
68+
for table in "${excluded_tables[@]}"
69+
do
70+
ignored_tables_string+=" --ignore-table=${dbname}.${table}"
71+
done
72+
} >/dev/null 2>&1
73+
74+
echo "Dumping structure..."
75+
{
76+
## Dump structure
77+
mysqldump \
78+
--defaults-extra-file=~/.mysql/mysqldump.cnf \
79+
--host="${host}" \
80+
--port="${port}" \
81+
--protocol=TCP \
82+
--no-data \
83+
"${dbname}" > "backup_${CF_SPACE}.sql"
84+
} >/dev/null 2>&1
85+
86+
echo "Dumping content..."
87+
{
88+
## Dump content
89+
mysqldump \
90+
--defaults-extra-file=~/.mysql/mysqldump.cnf \
91+
--host="${host}" \
92+
--port="${port}" \
93+
--protocol=TCP \
94+
--no-create-info \
95+
--skip-triggers \
96+
${ignored_tables_string} \
97+
"${dbname}" >> "backup_${CF_SPACE}.sql"
98+
99+
## Patch out any MySQL 'SET' commands that require admin.
100+
sed -i 's/^SET /-- &/' "backup_${CF_SPACE}.sql"
101+
} >/dev/null 2>&1
102+
103+
date
104+
105+
## Kill the backgrounded SSH tunnel.
106+
echo "Cleaning up old connections..."
107+
{
108+
kill_pids "connect-to-service"
109+
}
110+
111+
## Disable ssh.
112+
#echo "Disabling ssh..."
113+
#cf disable-ssh "${PROJECT}-drupal-${CF_SPACE}"
114+
115+
rm -rf backup.txt ~/.mysql
116+
117+
echo "Compressing '${CF_SPACE}' database..."
118+
{
119+
mv "backup_${CF_SPACE}.sql" "${TIMESTAMP}.sql"
120+
gzip "${TIMESTAMP}.sql"
121+
} &> /dev/null
122+
123+
echo "Creating S3 credentials..."
124+
{
125+
cf target -s "${PROJECT}-${CF_SPACE}" >/dev/null 2>&1
126+
127+
export service="${PROJECT}-backup-${CF_SPACE}"
128+
export service_key="${service}-key"
129+
cf delete-service-key "${service}" "${service_key}" -f
130+
cf create-service-key "${service}" "${service_key}"
131+
sleep 2
132+
133+
s3_credentials=$(cf service-key "${service}" "${service_key}" | tail -n +2)
134+
export s3_credentials
135+
136+
AWS_ACCESS_KEY_ID=$(echo "${s3_credentials}" | jq -r '.credentials.access_key_id')
137+
export AWS_ACCESS_KEY_ID
138+
139+
bucket=$(echo "${s3_credentials}" | jq -r '.credentials.bucket')
140+
export bucket
141+
142+
AWS_DEFAULT_REGION=$(echo "${s3_credentials}" | jq -r '.credentials.region')
143+
export AWS_DEFAULT_REGION
144+
145+
AWS_SECRET_ACCESS_KEY=$(echo "${s3_credentials}" | jq -r '.credentials.secret_access_key')
146+
export AWS_SECRET_ACCESS_KEY
147+
}
148+
149+
echo "Saving to backup bucket..."
150+
{
151+
# copy latest database to top level
152+
#gzip "backup_${CF_SPACE}.sql"
153+
#aws s3 cp "./backup_${CF_SPACE}.sql.gz" "s3://${bucket}/${CF_SPACE}/latest.sql.gz" --no-verify-ssl >/dev/null 2>&1 && echo "Successfully copied latest.sql.gz to S3!" || echo "Failed to copy latest.sql.gz to S3!"
154+
aws s3 cp "${TIMESTAMP}.sql.gz" "s3://${bucket}/$(date +%Y)/$(date +%m)/$(date +%d)/" --no-verify-ssl 2>/dev/null
155+
aws s3 cp "${TIMESTAMP}.sql.gz" "s3://${bucket}/latest.sql.gz" --no-verify-ssl 2>/dev/null
156+
} >/dev/null 2>&1
157+
158+
echo "Cleaning up credentials..."
159+
{
160+
cf delete-service-key "${service}" "${service_key}" -f >/dev/null 2>&1
161+
} >/dev/null 2>&1
162+
163+
date

‎scripts/pipeline/deb-cf-install.sh

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,7 @@ echo "Installing CloudFoundry repository..."
1414
else
1515
mv cf cf8 /usr/local/bin
1616
fi
17+
18+
cf install-plugin -f https://github.com/cloud-gov/cf-service-connect/releases/download/v1.1.4/cf-service-connect_linux_amd64
19+
1720
} >/dev/null 2>&1

‎scripts/pipeline/github-exports.sh

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
#!/bin/bash
2+
3+
#BRANCH=$(echo $GITHUB_REF | cut -d'/' -f 3)
4+
BRANCH=develop
5+
COMPOSER_DEV=1
6+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_DEVELOPMENT_KEY }}
7+
case ${BRANCH} in
8+
develop)
9+
CF_SPACE="dev"
10+
DRUPAL_MEMORY=${{ vars.DEVELOP_CMS_MEMORY }}
11+
DRUPAL_INSTANCES=${{ vars.DEVELOP_INSTANCES }}
12+
;;
13+
main)
14+
CF_SPACE="prod"
15+
COMPOSER_DEV=0
16+
DRUPAL_MEMORY=${{ vars.MAIN_CMS_MEMORY }}
17+
DRUPAL_INSTANCES=${{ vars.MAIN_INSTANCES }}
18+
GSA_AUTH_KEY=${{ secrets.GSA_AUTH_PRODUCTION_KEY }}
19+
;;
20+
stage)
21+
CF_SPACE="staging"
22+
COMPOSER_DEV=0
23+
DRUPAL_MEMORY=${{ vars.STAGE_CMS_MEMORY }}
24+
DRUPAL_INSTANCES=${{ vars.STAGE_INSTANCES }}
25+
;;
26+
esac
27+
28+
echo "APP_NAME=drupal" | tee -a $GITHUB_ENV
29+
echo "BRANCH=${BRANCH}" | tee -a $GITHUB_ENV
30+
echo "BUILDPACK_PORT=${{ vars.BUILDPACK_PORT }}" | tee -a $GITHUB_ENV
31+
echo "CF_SPACE=${CF_SPACE}" | tee -a $GITHUB_ENV
32+
echo "COMPOSER_DEV=${COMPOSER_DEV}" | tee -a $GITHUB_ENV
33+
echo "DRUPAL_INSTANCES=${DRUPAL_INSTANCES}" | tee -a $GITHUB_ENV
34+
echo "DRUPAL_MEMORY=${DRUPAL_MEMORY}" | tee -a $GITHUB_ENV
35+
echo "GSA_AUTH_KEY=${GSA_AUTH_KEY}" | tee -a $GITHUB_ENV
36+
echo "HASH_SALT=${{ secrets.HASH_SALT }}" | tee -a $GITHUB_ENV
37+
echo "WAF_NAME=waf"| tee -a $GITHUB_ENV
38+
39+
if [ "${COMPOSER_DEV}" = "1" ]; then
40+
sed -i 's/--no-dev //' .bp-config/options.json || exit 0
41+
fi

0 commit comments

Comments
 (0)