K8SPSMDB-1302: Fix stuck physical restore if update strategy is SmartUpdate (#1861)

egegunes · hors · web-flow · commit 9dbd866a99a6 · 2025-03-17T22:12:51.000+02:00
* K8SPSMDB-1302: Fix stuck physical restore if update strategy is SmartUpdate

* K8SPSMDB-1302 fix test

* wait for resync to finish before running backups

* physical restore improvements

* more compare

* fixes

* fix describe retry json error

* address review comments

* fix

---------

Co-authored-by: Viacheslav Sarzhan &lt;slava.sarzhan@percona.com&gt;
diff --git a/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml b/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml
@@ -80,6 +80,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml b/e2e-tests/demand-backup-incremental-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
@@ -80,6 +80,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore-arbiter-nv-oc.yml b/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore-arbiter-nv-oc.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore-arbiter-nv.yml b/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore-arbiter-nv.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore-oc.yml b/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore-oc.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore.yml b/e2e-tests/demand-backup-incremental/compare/statefulset_some-name-rs0_restore.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml b/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded-oc.yml
@@ -80,6 +80,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
diff --git a/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml b/e2e-tests/demand-backup-physical-sharded/compare/statefulset_some-name-rs0_restore_sharded.yml
@@ -80,6 +80,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -244,9 +248,7 @@ spec:
             defaultMode: 420
             secretName: some-name-pbm-config
   updateStrategy:
-    rollingUpdate:
-      partition: 0
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
     - metadata:
         name: mongod-data
diff --git a/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml b/e2e-tests/demand-backup-physical-sharded/conf/some-name-sharded.yml
@@ -6,6 +6,7 @@ spec:
   #platform: openshift
   image:
   imagePullPolicy: Always
+  updateStrategy: SmartUpdate
   backup:
     enabled: true
     image: perconalab/percona-server-mongodb-operator:1.1.0-backup
diff --git a/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore-arbiter-nv-oc.yml b/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore-arbiter-nv-oc.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -241,9 +245,7 @@ spec:
             defaultMode: 420
             secretName: some-name-pbm-config
   updateStrategy:
-    rollingUpdate:
-      partition: 0
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
     - metadata:
         name: mongod-data
diff --git a/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore-arbiter-nv.yml b/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore-arbiter-nv.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -243,9 +247,7 @@ spec:
             defaultMode: 420
             secretName: some-name-pbm-config
   updateStrategy:
-    rollingUpdate:
-      partition: 0
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
     - metadata:
         name: mongod-data
diff --git a/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore-oc.yml b/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore-oc.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -241,9 +245,7 @@ spec:
             defaultMode: 420
             secretName: some-name-pbm-config
   updateStrategy:
-    rollingUpdate:
-      partition: 0
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
     - metadata:
         name: mongod-data
diff --git a/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore.yml b/e2e-tests/demand-backup-physical/compare/statefulset_some-name-rs0_restore.yml
@@ -79,6 +79,10 @@ spec:
                   key: MONGODB_BACKUP_PASSWORD_ESCAPED
                   name: internal-some-name-users
                   optional: false
+            - name: PBM_AGENT_SIDECAR
+              value: "true"
+            - name: PBM_AGENT_SIDECAR_SLEEP
+              value: "5"
             - name: POD_NAME
               valueFrom:
                 fieldRef:
@@ -243,9 +247,7 @@ spec:
             defaultMode: 420
             secretName: some-name-pbm-config
   updateStrategy:
-    rollingUpdate:
-      partition: 0
-    type: RollingUpdate
+    type: OnDelete
   volumeClaimTemplates:
     - metadata:
         name: mongod-data
diff --git a/e2e-tests/demand-backup-physical/conf/some-name-arbiter-nv.yml b/e2e-tests/demand-backup-physical/conf/some-name-arbiter-nv.yml
@@ -8,6 +8,7 @@ spec:
   #platform: openshift
   image:
   imagePullPolicy: Always
+  updateStrategy: SmartUpdate
   backup:
     enabled: true
     image: perconalab/percona-server-mongodb-operator:1.1.0-backup
diff --git a/e2e-tests/demand-backup-physical/conf/some-name.yml b/e2e-tests/demand-backup-physical/conf/some-name.yml
@@ -8,6 +8,7 @@ spec:
   #platform: openshift
   image:
   imagePullPolicy: Always
+  updateStrategy: SmartUpdate
   backup:
     enabled: true
     image: perconalab/percona-server-mongodb-operator:1.1.0-backup
diff --git a/e2e-tests/demand-backup-physical/run b/e2e-tests/demand-backup-physical/run
@@ -68,6 +68,10 @@ echo "check if all pods started"
 wait_for_running ${cluster}-rs0 3
 wait_cluster_consistency ${cluster}
 
+sleep 60 # give time for resync to start
+
+wait_for_pbm_operations ${cluster}
+
 echo 'writing test data'
 run_mongo \
 	'db.createUser({user:"myApp",pwd:"myPass",roles:[{db:"myApp",role:"readWrite"}]})' \
diff --git a/pkg/controller/perconaservermongodb/pbm.go b/pkg/controller/perconaservermongodb/pbm.go
@@ -516,8 +516,6 @@ func (r *ReconcilePerconaServerMongoDB) resyncPBMIfNeeded(ctx context.Context, c
 			log.Error(err, "failed to delete annotation")
 			return
 		}
-
-		log.Info("resync is finished")
 	}()
 
 	return nil
diff --git a/pkg/controller/perconaservermongodb/smart.go b/pkg/controller/perconaservermongodb/smart.go
@@ -128,10 +128,10 @@ func (r *ReconcilePerconaServerMongoDB) smartUpdate(ctx context.Context, cr *api
 		return nil
 	}
 
-	if rsStatus, ok := cr.Status.Replsets[replset.Name]; ok {
+	if rsStatus, ok := cr.Status.Replsets[replset.Name]; ok && rsStatus.Members != nil {
 		for _, pod := range list.Items {
 			if _, ok := rsStatus.Members[pod.Name]; !ok {
-				log.Info("pod is not a member of replset, updating it", "pod", pod.Name)
+				log.Info("pod is not a member of replset, updating it", "pod", pod.Name, "members", rsStatus.Members)
 
 				if err := updatePod(&pod); err != nil {
 					return err
diff --git a/pkg/controller/perconaservermongodbrestore/physical.go b/pkg/controller/perconaservermongodbrestore/physical.go
diff --git a/pkg/naming/labels.go b/pkg/naming/labels.go
diff --git a/pkg/naming/naming.go b/pkg/naming/naming.go
diff --git a/pkg/psmdb/mongo/mongo.go b/pkg/psmdb/mongo/mongo.go
