K8SPG-713: handle errors on pgcluster deletion (#1025)

https://perconadev.atlassian.net/browse/K8SPG-713

Co-authored-by: Viacheslav Sarzhan <[email protected]>

Author: pooknull

internal/controller/postgrescluster/delete.go

@@ -100,5 +100,10 @@ func (r *Reconciler) handleDelete(
 		client.MergeFromWithOptions(before, client.MergeFromWithOptimisticLock{})))
 
 	// The caller should wait for further events or requeue upon error.
-	return &reconcile.Result{}, err
+
+	// K8SPG-713: For some reason we get a "not found" error when we try to remove finalizer.
+	//            This is unexpected because finalizers are designed to prevent objects from being deleted
+	//            until all finalizers have been deleted.
+
+	return &reconcile.Result{}, client.IgnoreNotFound(err)
 }

percona/controller/pgbackup/controller.go

@@ -89,14 +89,14 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re
 		return reconcile.Result{}, errors.Wrap(err, "ensure finalizers")
 	}
 
-	pgCluster := &v2.PerconaPGCluster{}
-	err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Spec.PGCluster, Namespace: request.Namespace}, pgCluster)
-	if err != nil {
-		return reconcile.Result{}, errors.Wrap(err, "get PostgresCluster")
-	}
-
 	switch pgBackup.Status.State {
 	case v2.BackupNew:
+		pgCluster := &v2.PerconaPGCluster{}
+		err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Spec.PGCluster, Namespace: request.Namespace}, pgCluster)
+		if err != nil {
+			return reconcile.Result{}, errors.Wrap(err, "get PostgresCluster")
+		}
+
 		if pgCluster.Spec.Pause != nil && *pgCluster.Spec.Pause {
 			log.Info("Can't start backup. PostgresCluster is paused", "pg-backup", pgBackup.Name, "cluster", pgCluster.Name)
 			return reconcile.Result{RequeueAfter: time.Second * 5}, nil
@@ -159,6 +159,12 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re
 		log.Info("Backup is starting", "backup", pgBackup.Name, "cluster", pgCluster.Name)
 		return reconcile.Result{}, nil
 	case v2.BackupStarting:
+		pgCluster := &v2.PerconaPGCluster{}
+		err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Spec.PGCluster, Namespace: request.Namespace}, pgCluster)
+		if err != nil {
+			return reconcile.Result{}, errors.Wrap(err, "get PostgresCluster")
+		}
+
 		job, err := findBackupJob(ctx, r.Client, pgCluster, pgBackup)
 		if err != nil {
 			if errors.Is(err, ErrBackupJobNotFound) {
@@ -247,6 +253,15 @@ func (r *PGBackupReconciler) Reconcile(ctx context.Context, request reconcile.Re
 
 		return reconcile.Result{}, nil
 	case v2.BackupSucceeded:
+		pgCluster := &v2.PerconaPGCluster{}
+		err := r.Client.Get(ctx, types.NamespacedName{Name: pgBackup.Spec.PGCluster, Namespace: request.Namespace}, pgCluster)
+		if err != nil {
+			if k8serrors.IsNotFound(err) {
+				return reconcile.Result{}, nil
+			}
+			return reconcile.Result{}, errors.Wrap(err, "get PostgresCluster")
+		}
+
 		execCli, err := clientcmd.NewClient()
 		if err != nil {
 			return reconcile.Result{}, errors.Wrap(err, "failed to create exec client")

percona/controller/pgcluster/controller.go

@@ -312,6 +312,9 @@ func (r *PGClusterReconciler) Reconcile(ctx context.Context, request reconcile.R
 	}
 
 	if err := r.Client.Get(ctx, client.ObjectKeyFromObject(postgresCluster), postgresCluster); err != nil {
+		if k8serrors.IsNotFound(err) {
+			return ctrl.Result{RequeueAfter: 1 * time.Second}, nil
+		}
 		return ctrl.Result{}, errors.Wrap(err, "get PostgresCluster")
 	}
 
@@ -371,6 +374,9 @@ func (r *PGClusterReconciler) reconcilePatroniVersionCheck(ctx context.Context,
 			},
 		}
 
+		if err := controllerutil.SetControllerReference(cr, p, r.Client.Scheme()); err != nil {
+			return errors.Wrap(err, "set controller reference")
+		}
 		if err := r.Client.Create(ctx, p); err != nil {
 			return errors.Wrap(err, "failed to create pod to check patroni version")
 		}

pkg/apis/pgv2.percona.com/v2/perconapgcluster_types.go

@@ -246,8 +246,9 @@ func (cr *PerconaPGCluster) ToCrunchy(ctx context.Context, postgresCluster *crun
 	if postgresCluster == nil {
 		postgresCluster = &crunchyv1beta1.PostgresCluster{
 			ObjectMeta: metav1.ObjectMeta{
-				Name:      cr.Name,
-				Namespace: cr.Namespace,
+				Name:       cr.Name,
+				Namespace:  cr.Namespace,
+				Finalizers: []string{naming.Finalizer},
 			},
 		}
 	}