Skip to content

Commit b630efb

Browse files
egeguneshorsgithub-actions[bot]
authored
K8SPSMDB-1219: Minor test improvements (#1871)
* K8SPSMDB-1219: Minor test improvements * fix * fix physical restores with backup source * fix panics if annotations are nil * add pitr-physical-backup-source test * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/functions Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * Update e2e-tests/pitr-physical-backup-source/run Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com> * remove commented line * fix comments * fix pitr-physical-backup-source --------- Co-authored-by: Viacheslav Sarzhan <[email protected]> Co-authored-by: github-actions[bot] <41898282+github-actions[bot]@users.noreply.github.com>
1 parent e5678f0 commit b630efb

31 files changed

+3897
-155
lines changed

‎e2e-tests/demand-backup-physical-sharded/run

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ run_recovery_check() {
3636

3737
# we don't wait for cluster readiness here because the annotation gets removed then
3838
wait_restore "${backup_name}" "${cluster}" "ready" "0" "3000"
39-
kubectl_bin get psmdb ${cluster} -o yaml
39+
4040
if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
4141
echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
4242
exit 1

‎e2e-tests/functions

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1173,6 +1173,11 @@ cat_config() {
11731173
| yq eval '.spec.upgradeOptions.apply="Never"'
11741174
}
11751175

1176+
format_date() {
1177+
local timestamp=$1
1178+
echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S')
1179+
}
1180+
11761181
apply_cluster() {
11771182
if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
11781183
cat_config "$1" \
@@ -1681,6 +1686,37 @@ getUserData() {
16811686
urlencode "$(getSecretData "$secretName" "$dataKey")"
16821687
}
16831688

1689+
write_document() {
1690+
local cmp_postfix="$1"
1691+
local sleep_value=${2:-0}
1692+
1693+
desc 'write initial data, read from all'
1694+
run_mongos \
1695+
'use myApp\n db.test.insert({ x: 100500 })' \
1696+
"myApp:myPass@$cluster-mongos.$namespace"
1697+
sleep $sleep_value
1698+
1699+
compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" ${cmp_postfix}
1700+
}
1701+
1702+
write_initial_data() {
1703+
desc 'create user myApp'
1704+
run_mongos \
1705+
'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
1706+
"userAdmin:userAdmin123456@$cluster-mongos.$namespace"
1707+
sleep 2
1708+
}
1709+
1710+
reset_collection() {
1711+
desc 'reset data'
1712+
1713+
run_mongos \
1714+
'use myApp\n db.test.remove({})' \
1715+
"myApp:myPass@$cluster-mongos.$namespace"
1716+
sleep 2
1717+
write_document '' '120'
1718+
}
1719+
16841720
get_latest_oplog_chunk_ts() {
16851721
local cluster=$1
16861722
echo $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
@@ -1691,6 +1727,92 @@ format_date() {
16911727
echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S')
16921728
}
16931729

1730+
get_bucket_name() {
1731+
local backup_name=$1
1732+
1733+
kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.s3.bucket}'
1734+
}
1735+
1736+
check_recovery() {
1737+
local backup_name=$1
1738+
local restore_type=$2
1739+
local restore_date=$3
1740+
local cmp_postfix=$4
1741+
local cluster_name=$5
1742+
local backupSource=$6
1743+
1744+
local latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
1745+
1746+
desc "write more data before restore by $restore_type"
1747+
run_mongos \
1748+
'use myApp\n db.test.insert({ x: 100501 })' \
1749+
"myApp:myPass@$cluster-mongos.$namespace"
1750+
1751+
if [[ -n ${restore_date} ]]; then
1752+
desc "Restoring to time $(format_date ${restore_date})"
1753+
retries=0
1754+
until [[ ${latest_ts} -gt ${restore_date} ]]; do
1755+
if [[ $retries -gt 30 ]]; then
1756+
echo "Last oplog chunk ($(format_date ${latest_ts})) is not greater than restore target ($(format_date ${restore_date}))"
1757+
exit 1
1758+
fi
1759+
latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
1760+
retries=$((retries + 1))
1761+
echo "Waiting for last oplog chunk ($(format_date ${latest_ts})) to be greater than restore target ($(format_date ${restore_date}))"
1762+
sleep 10
1763+
done
1764+
else
1765+
desc "Restoring to latest"
1766+
local current_ts=$(get_latest_oplog_chunk_ts $cluster_name)
1767+
retries=0
1768+
until [[ ${latest_ts} -gt ${current_ts} ]]; do
1769+
if [[ $retries -gt 30 ]]; then
1770+
echo "Timeout while waiting for last oplog chunk ($(format_date ${latest_ts}))"
1771+
exit 1
1772+
fi
1773+
latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
1774+
retries=$((retries + 1))
1775+
echo "Waiting for last oplog chunk ($(format_date ${latest_ts})) to be 120 seconds older than starting chunk ($(format_date ${current_ts}))"
1776+
sleep 10
1777+
done
1778+
fi
1779+
1780+
if [ -z "$backupSource" ]; then
1781+
desc "check restore by $restore_type"
1782+
cat $test_dir/conf/restore.yml \
1783+
| $sed -e "s/name:/name: restore-$backup_name/" \
1784+
| $sed -e "s/backupName:/backupName: $backup_name/" \
1785+
| $sed -e "/backupSource/,+8d" \
1786+
| $sed -e "s/pitrType:/type: $restore_type/" \
1787+
| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $(format_date ${restore_date})/"; fi \
1788+
| kubectl_bin apply -f -
1789+
else
1790+
desc "check restore by $restore_type $backupSource"
1791+
backup_dest=$(get_backup_dest "$backup_name")
1792+
cat $test_dir/conf/restore.yml \
1793+
| $sed -e "s/name:/name: restore-$backup_name/" \
1794+
| $sed -e "/backupName/d" \
1795+
| $sed -e "s/pitrType:/type: $restore_type/" \
1796+
| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $(format_date ${restore_date})"/; fi \
1797+
| $sed -e "s|DESTINATION|$backup_dest|" \
1798+
| $sed -e "s|BUCKET-NAME|$(get_bucket_name "$backup_name")|" \
1799+
| if [ -n "$selective_collection" ]; then yq eval '.spec.selective = {"namespaces": ["myApp.test"], "withUsersAndRoles": true}'; else yq; fi \
1800+
| kubectl_bin apply -f -
1801+
fi
1802+
1803+
# fail faster if we don't reach requested status until some time
1804+
wait_restore "$backup_name" "$cluster_name" "requested" "0" "900"
1805+
echo
1806+
wait_restore "$backup_name" "$cluster_name" "ready" "0" "1600"
1807+
echo
1808+
set -o xtrace
1809+
1810+
wait_for_running $cluster-mongos 3
1811+
sleep 10
1812+
1813+
compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "$cmp_postfix"
1814+
}
1815+
16941816
run_pitr_check() {
16951817
local backup=$1
16961818
local cluster=$2

‎e2e-tests/multi-storage/run

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -283,12 +283,18 @@ wait_for_advanced_restorable_time backup-minio-3
283283
restore_time=$(get_latest_restorable_time backup-minio-3)
284284
check_recovery backup-minio-3 "$(format_pitr_target ${restore_time})"
285285

286+
log "sleeping for 60 seconds for resync to finish"
287+
sleep 60 # minio buckets are basically empty, resync should finish very quickly
288+
286289
log "changing main storage from minio-1 to minio-2"
287290
kubectl patch psmdb ${cluster} --type=json -p='[
288291
{"op": "remove", "path": "/spec/backup/storages/minio-1/main"},
289292
{"op": "add", "path": "/spec/backup/storages/minio-2/main", "value": true}
290293
]'
291294

295+
log "sleeping for 60 seconds for resync to finish"
296+
sleep 60 # minio buckets are basically empty, resync should finish very quickly
297+
292298
run_backup minio-2 backup-minio-2-1 logical
293299
wait_backup backup-minio-2-1
294300

‎e2e-tests/pitr-physical-backup-source/compare/find-2nd.json

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
switched to db myApp
2+
{ "_id" : , "x" : 100500 }
3+
{ "_id" : , "x" : 100500 }
4+
bye

‎e2e-tests/pitr-physical-backup-source/compare/find-3rd.json

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
switched to db myApp
2+
{ "_id" : , "x" : 100500 }
3+
{ "_id" : , "x" : 100500 }
4+
{ "_id" : , "x" : 100501 }
5+
bye

‎e2e-tests/pitr-physical-backup-source/compare/find.json

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
switched to db myApp
2+
{ "_id" : , "x" : 100500 }
3+
bye

0 commit comments

Comments
 (0)