K8SPSMDB-1219: Minor test improvements

e2e-tests/demand-backup-physical-sharded/run

@@ -36,7 +36,7 @@ run_recovery_check() {
 
 	# we don't wait for cluster readiness here because the annotation gets removed then
 	wait_restore "${backup_name}" "${cluster}" "ready" "0" "3000"
-	kubectl_bin get psmdb ${cluster} -o yaml
+
 	if [ $(kubectl_bin get psmdb ${cluster} -o yaml | yq '.metadata.annotations."percona.com/resync-pbm"') == null ]; then
 		echo "psmdb/${cluster} should be annotated with percona.com/resync-pbm after a physical restore"
 		exit 1

e2e-tests/functions

@@ -1173,6 +1173,11 @@ cat_config() {
 		| yq eval '.spec.upgradeOptions.apply="Never"'
 }
 
+format_date() {
+	local timestamp=$1
+	echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S')
+}
+
 apply_cluster() {
 	if [ -z "$SKIP_BACKUPS_TO_AWS_GCP_AZURE" ]; then
 		cat_config "$1" \
@@ -1681,6 +1686,37 @@ getUserData() {
 	urlencode "$(getSecretData "$secretName" "$dataKey")"
 }
 
+write_document() {
+	local cmp_postfix="$1"
+	local sleep_value=${2:-0}
+
+	desc 'write initial data, read from all'
+	run_mongos \
+		'use myApp\n db.test.insert({ x: 100500 })' \
+		"myApp:myPass@$cluster-mongos.$namespace"
+	sleep $sleep_value
+
+	compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" ${cmp_postfix}
+}
+
+write_initial_data() {
+	desc 'create user myApp'
+	run_mongos \
+		'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
+		"userAdmin:userAdmin123456@$cluster-mongos.$namespace"
+	sleep 2
+}
+
+reset_collection() {
+	desc 'reset data'
+
+	run_mongos \
+		'use myApp\n db.test.remove({})' \
+		"myApp:myPass@$cluster-mongos.$namespace"
+	sleep 2
+	write_document '' '120'
+}
+
 get_latest_oplog_chunk_ts() {
 	local cluster=$1
 	echo $(kubectl_bin exec ${cluster}-rs0-0 -c backup-agent -- pbm status -o json | jq '.backups.pitrChunks.pitrChunks | last | .range.end')
@@ -1691,6 +1727,92 @@ format_date() {
 	echo $(TZ=UTC $date -d@${timestamp} '+%Y-%m-%d %H:%M:%S')
 }
 
+get_bucket_name() {
+	local backup_name=$1
+
+	kubectl_bin get psmdb-backup $backup_name -o jsonpath='{.status.s3.bucket}'
+}
+
+check_recovery() {
+	local backup_name=$1
+	local restore_type=$2
+	local restore_date=$3
+	local cmp_postfix=$4
+	local cluster_name=$5
+	local backupSource=$6
+
+	local latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
+
+	desc "write more data before restore by $restore_type"
+	run_mongos \
+		'use myApp\n db.test.insert({ x: 100501 })' \
+		"myApp:myPass@$cluster-mongos.$namespace"
+
+	if [[ -n ${restore_date} ]]; then
+		desc "Restoring to time $(format_date ${restore_date})"
+		retries=0
+		until [[ ${latest_ts} -gt ${restore_date} ]]; do
+			if [[ $retries -gt 30 ]]; then
+				echo "Last oplog chunk ($(format_date ${latest_ts})) is not greater than restore target ($(format_date ${restore_date}))"
+				exit 1
+			fi
+			latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
+			retries=$((retries + 1))
+			echo "Waiting for last oplog chunk ($(format_date ${latest_ts})) to be greater than restore target ($(format_date ${restore_date}))"
+			sleep 10
+		done
+	else
+		desc "Restoring to latest"
+		local current_ts=$(get_latest_oplog_chunk_ts $cluster_name)
+		retries=0
+		until [[ ${latest_ts} -gt ${current_ts} ]]; do
+			if [[ $retries -gt 30 ]]; then
+				echo "Timeout while waiting for last oplog chunk ($(format_date ${latest_ts}))"
+				exit 1
+			fi
+			latest_ts=$(get_latest_oplog_chunk_ts $cluster_name)
+			retries=$((retries + 1))
+			echo "Waiting for last oplog chunk ($(format_date ${latest_ts})) to be 120 seconds older than starting chunk ($(format_date ${current_ts}))"
+			sleep 10
+		done
+	fi
+
+	if [ -z "$backupSource" ]; then
+		desc "check restore by $restore_type"
+		cat $test_dir/conf/restore.yml \
+			| $sed -e "s/name:/name: restore-$backup_name/" \
+			| $sed -e "s/backupName:/backupName: $backup_name/" \
+			| $sed -e "/backupSource/,+8d" \
+			| $sed -e "s/pitrType:/type: $restore_type/" \
+			| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $(format_date ${restore_date})/"; fi \
+			| kubectl_bin apply -f -
+	else
+		desc "check restore by $restore_type $backupSource"
+		backup_dest=$(get_backup_dest "$backup_name")
+		cat $test_dir/conf/restore.yml \
+			| $sed -e "s/name:/name: restore-$backup_name/" \
+			| $sed -e "/backupName/d" \
+			| $sed -e "s/pitrType:/type: $restore_type/" \
+			| if [ -z "$restore_date" ]; then $sed -e "/date:/d"; else $sed -e "s/date:/date: $(format_date ${restore_date})"/; fi \
+			| $sed -e "s|DESTINATION|$backup_dest|" \
+			| $sed -e "s|BUCKET-NAME|$(get_bucket_name "$backup_name")|" \
+			| if [ -n "$selective_collection" ]; then yq eval '.spec.selective = {"namespaces": ["myApp.test"], "withUsersAndRoles": true}'; else yq; fi \
+			| kubectl_bin apply -f -
+	fi
+
+	# fail faster if we don't reach requested status until some time
+	wait_restore "$backup_name" "$cluster_name" "requested" "0" "900"
+	echo
+	wait_restore "$backup_name" "$cluster_name" "ready" "0" "1600"
+	echo
+	set -o xtrace
+
+	wait_for_running $cluster-mongos 3
+	sleep 10
+
+	compare_mongos_cmd "find" "myApp:myPass@$cluster-mongos.$namespace" "$cmp_postfix"
+}
+
 run_pitr_check() {
 	local backup=$1
 	local cluster=$2

e2e-tests/multi-storage/run

@@ -283,12 +283,18 @@ wait_for_advanced_restorable_time backup-minio-3
 restore_time=$(get_latest_restorable_time backup-minio-3)
 check_recovery backup-minio-3 "$(format_pitr_target ${restore_time})"
 
+log "sleeping for 60 seconds for resync to finish"
+sleep 60 # minio buckets are basically empty, resync should finish very quickly
+
 log "changing main storage from minio-1 to minio-2"
 kubectl patch psmdb ${cluster} --type=json -p='[
   {"op": "remove", "path": "/spec/backup/storages/minio-1/main"},
   {"op": "add", "path": "/spec/backup/storages/minio-2/main", "value": true}
 ]'
 
+log "sleeping for 60 seconds for resync to finish"
+sleep 60 # minio buckets are basically empty, resync should finish very quickly
+
 run_backup minio-2 backup-minio-2-1 logical
 wait_backup backup-minio-2-1
 

e2e-tests/pitr-physical-backup-source/compare/find-2nd.json

@@ -0,0 +1,4 @@
+switched to db myApp
+{ "_id" : , "x" : 100500 }
+{ "_id" : , "x" : 100500 }
+bye

e2e-tests/pitr-physical-backup-source/compare/find-3rd.json

@@ -0,0 +1,5 @@
+switched to db myApp
+{ "_id" : , "x" : 100500 }
+{ "_id" : , "x" : 100500 }
+{ "_id" : , "x" : 100501 }
+bye

e2e-tests/pitr-physical-backup-source/compare/find.json

@@ -0,0 +1,3 @@
+switched to db myApp
+{ "_id" : , "x" : 100500 }
+bye