Delete NovaCell cr only when deletion job pass

controllers/common.go

@@ -732,3 +732,9 @@ func ensureMemcached(
 
 	return memcached, err
 }
+
+func sortNovaCellListByName(cellList *novav1.NovaCellList) {
+	sort.SliceStable(cellList.Items, func(i, j int) bool {
+		return cellList.Items[i].Name < cellList.Items[j].Name
+	})
+}

controllers/nova_controller.go

@@ -20,7 +20,6 @@ import (
 	"context"
 	"errors"
 	"fmt"
-	"sort"
 	"strings"
 
 	batchv1 "k8s.io/api/batch/v1"
@@ -602,26 +601,20 @@ func (r *NovaReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 		return ctrl.Result{}, err
 	}
 
-	sortNovaCellListByName := func(cellList *novav1.NovaCellList) {
-		sort.SliceStable(cellList.Items, func(i, j int) bool {
-			return cellList.Items[i].Name < cellList.Items[j].Name
-		})
-	}
-
 	sortNovaCellListByName(novaCellList)
 
 	var deleteErrs []error
 
 	for _, cr := range novaCellList.Items {
 		_, ok := instance.Spec.CellTemplates[cr.Spec.CellName]
 		if !ok {
-			err := r.ensureCellDeleted(ctx, h, instance,
+			result, err := r.ensureCellDeleted(ctx, h, instance,
 				cr.Spec.CellName, apiTransportURL,
 				secret, apiDB, cellDBs[novav1.Cell0Name].Database.GetDatabaseHostname(), cells[novav1.Cell0Name])
 			if err != nil {
 				deleteErrs = append(deleteErrs, fmt.Errorf("Cell '%s' deletion failed, because: %w", cr.Spec.CellName, err))
-
-			} else {
+			}
+			if result == nova.CellDeleteComplete {
 				Log.Info("Cell deleted", "cell", cr.Spec.CellName)
 				delete(instance.Status.RegisteredCells, cr.Name)
 			}
@@ -687,7 +680,7 @@ func (r *NovaReconciler) ensureCellDeleted(
 	apiDB *mariadbv1.Database,
 	APIDatabaseHostname string,
 	cell0 *novav1.NovaCell,
-) error {
+) (nova.CellDeploymentStatus, error) {
 	Log := r.GetLogger(ctx)
 	cell := &novav1.NovaCell{}
 	fullCellName := types.NamespacedName{
@@ -700,29 +693,29 @@ func (r *NovaReconciler) ensureCellDeleted(
 		// We cannot do further cleanup of the MariaDBDatabase and
 		// MariaDBAccount as their name is only available in the NovaCell CR
 		// since the cell definition is removed from the Nova CR already.
-		return nil
+		return nova.CellDeleteComplete, nil
 	}
 	if err != nil {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 	// If it is not created by us, we don't touch it
 	if !OwnedBy(cell, instance) {
 		Log.Info("Cell isn't defined in the Nova, but there is a  "+
 			"Cell CR not owned by us. Not deleting it.",
 			"cell", cell)
-		return nil
+		return nova.CellDeleteComplete, nil
 	}
 
 	dbName, accountName := novaapi.ServiceName+"-"+cell.Spec.CellName, cell.Spec.CellDatabaseAccount
 
 	configHash, scriptName, configName, err := r.ensureNovaManageJobSecret(ctx, h, instance,
 		cell0, topLevelSecret, APIDatabaseHostname, apiTransportURL, apiDB)
 	if err != nil {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 	inputHash, err := util.HashOfInputHashes(configHash)
 	if err != nil {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 
 	labels := map[string]string{
@@ -734,24 +727,30 @@ func (r *NovaReconciler) ensureCellDeleted(
 		instance.Spec.PreserveJobs, r.RequeueTimeout,
 		inputHash)
 
-	_, err = job.DoJob(ctx, h)
+	result, err := job.DoJob(ctx, h)
 	if err != nil {
-		return err
+		return nova.CellDeleteFailed, err
+	}
+
+	if (result != ctrl.Result{}) {
+		// Job is still running. We can simply return as we will be reconciled
+		// when the Job status changes
+		return nova.CellDeleteInProgress, nil
 	}
 
 	secretName := getNovaCellCRName(instance.Name, cellName)
 	err = secret.DeleteSecretsWithName(ctx, h, secretName, instance.Namespace)
 	if err != nil && !k8s_errors.IsNotFound(err) {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 	configSecret, scriptSecret := r.getNovaManageJobSecretNames(cell)
 	err = secret.DeleteSecretsWithName(ctx, h, configSecret, instance.Namespace)
 	if err != nil && !k8s_errors.IsNotFound(err) {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 	err = secret.DeleteSecretsWithName(ctx, h, scriptSecret, instance.Namespace)
 	if err != nil && !k8s_errors.IsNotFound(err) {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 
 	// Delete transportURL cr
@@ -763,18 +762,18 @@ func (r *NovaReconciler) ensureCellDeleted(
 	}
 	err = r.Client.Delete(ctx, transportURL)
 	if err != nil && !k8s_errors.IsNotFound(err) {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 
 	err = mariadbv1.DeleteDatabaseAndAccountFinalizers(
 		ctx, h, dbName, accountName, instance.Namespace)
 	if err != nil {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 
 	err = r.ensureAccountDeletedIfOwned(ctx, h, instance, accountName)
 	if err != nil {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 
 	database := &mariadbv1.MariaDBDatabase{
@@ -785,7 +784,7 @@ func (r *NovaReconciler) ensureCellDeleted(
 	}
 	err = r.Client.Delete(ctx, database)
 	if err != nil && !k8s_errors.IsNotFound(err) {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 	Log.Info("Deleted MariaDBDatabase", "database", database)
 
@@ -794,11 +793,11 @@ func (r *NovaReconciler) ensureCellDeleted(
 	// what to clean up.
 	err = r.Client.Delete(ctx, cell)
 	if err != nil && k8s_errors.IsNotFound(err) {
-		return err
+		return nova.CellDeleteFailed, err
 	}
 
 	Log.Info("Cell isn't defined in the Nova CR, so it is deleted", "cell", cell)
-	return nil
+	return nova.CellDeleteComplete, nil
 }
 
 func (r *NovaReconciler) initStatus(

pkg/nova/common.go

@@ -96,6 +96,12 @@ const (
 	// CellComputeDiscoveryReady indicates that all NovaComputes in cell reached the Ready status and
 	// the hosts in the cell have been successfully discovered
 	CellComputeDiscoveryReady CellDeploymentStatus = iota
+	// CellDeleteInProgress indicates that the NovaCell deletion is in progress
+	CellDeleteInProgress CellDeploymentStatus = iota
+	// CellDeleteFailed indicates that the NovaCell deletion failed
+	CellDeleteFailed CellDeploymentStatus = iota
+	// CellDeleteComplete indicates that the NovaCell deletion is complete
+	CellDeleteComplete CellDeploymentStatus = iota
 )
 
 // Database -

test/functional/nova_multicell_test.go

@@ -1113,6 +1113,8 @@ var _ = Describe("Nova multi cell deletion", func() {
 				g.Expect(k8sClient.Update(ctx, nova)).To(Succeed())
 			}, timeout, interval).Should(Succeed())
 
+			th.SimulateJobSuccess(cell2.CellDeleteJobName)
+			th.SimulateJobSuccess(cell3.CellDeleteJobName)
 			Eventually(func(g Gomega) {
 				nova := GetNova(novaNames.NovaName)
 				g.Expect(nova.Status.RegisteredCells).To(HaveKey(cell2.CellCRName.Name))

test/functional/nova_reconfiguration_test.go

@@ -178,18 +178,28 @@ var _ = Describe("Nova reconfiguration", func() {
 				g.Expect(k8sClient.Update(ctx, nova)).To(Succeed())
 			}, timeout, interval).Should(Succeed())
 
-			Eventually(func(g Gomega) {
-				nova := GetNova(novaNames.NovaName)
-				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name))
-			}, timeout, interval).Should(Succeed())
-
-			NovaCellNotExists(cell1.CellCRName)
 			Eventually(func(g Gomega) {
 				mappingJob := th.GetJob(cell1.CellDeleteJobName)
 				newJobInputHash := GetEnvVarValue(
 					mappingJob.Spec.Template.Spec.Containers[0].Env, "INPUT_HASH", "")
 				g.Expect(newJobInputHash).NotTo(BeNil())
 			}, timeout, interval).Should(Succeed())
+
+			// Check that the cell status is not deleted
+			Consistently(func(g Gomega) {
+				nova := GetNova(novaNames.NovaName)
+				g.Expect(nova.Status.RegisteredCells).To(HaveKey(cell1.CellCRName.Name))
+			}, timeout, interval).Should(Succeed())
+
+			// Simulate the cell delete job success
+			th.SimulateJobSuccess(cell1.CellDeleteJobName)
+			Eventually(func(g Gomega) {
+				nova := GetNova(novaNames.NovaName)
+				g.Expect(nova.Status.RegisteredCells).NotTo(HaveKey(cell1.CellCRName.Name))
+			}, timeout, interval).Should(Succeed())
+
+			NovaCellNotExists(cell1.CellCRName)
+
 			th.AssertSecretDoesNotExist(cell1.InternalCellSecretName)
 
 			Eventually(func(g Gomega) {