[release-1.11] 🌱 Fix race condition on KCP initialized condition

controlplane/kubeadm/internal/controllers/status.go

@@ -155,6 +155,21 @@ func (r *KubeadmControlPlaneReconciler) updateStatus(ctx context.Context, contro
 // Note: This only gets initialized once and does not change if the kubeadm config map goes away.
 func setControlPlaneInitialized(ctx context.Context, controlPlane *internal.ControlPlane) error {
 	if !ptr.Deref(controlPlane.KCP.Status.Initialization.ControlPlaneInitialized, false) {
+		// If the control plane has only one machine, and this machine is marked for remediation or in the process of deleting,
+		// do not check for control plane initialized.
+		// This prevents an issue that happens if kubeadm init completes in the short timeframe between when machine deletion is triggered
+		// to when the machine goes away; this issue, if not properly handled, will lead to an inconsistent state where
+		// cluster is initialized, no CP machine exists, and the replacement CP machine fails when trying to join.
+		if len(controlPlane.Machines) == 1 {
+			m := controlPlane.Machines.UnsortedList()[0]
+			if collections.IsUnhealthyAndOwnerRemediated(m) {
+				return nil
+			}
+			if !m.DeletionTimestamp.IsZero() {
+				return nil
+			}
+		}
+
 		workloadCluster, err := controlPlane.GetWorkloadCluster(ctx)
 		if err != nil {
 			return errors.Wrap(err, "failed to create remote cluster client")

controlplane/kubeadm/internal/controllers/status_test.go

@@ -75,6 +75,9 @@ func TestKubeadmControlPlaneReconciler_setControlPlaneInitialized(t *testing.T)
 		controlPlane := &internal.ControlPlane{
 			Cluster: &clusterv1.Cluster{},
 			KCP:     &controlplanev1.KubeadmControlPlane{},
+			Machines: collections.FromMachines(
+				&clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1"}},
+			),
 		}
 		controlPlane.InjectTestManagementCluster(&fakeManagementCluster{
 			Workload: &fakeWorkloadCluster{
@@ -98,6 +101,85 @@ func TestKubeadmControlPlaneReconciler_setControlPlaneInitialized(t *testing.T)
 			Reason: controlplanev1.KubeadmControlPlaneInitializedReason,
 		}, conditions.IgnoreLastTransitionTime(true)))
 	})
+	t.Run("kubeadm config exists is ignored if there is a single CP machine and it is marked for remediation", func(t *testing.T) {
+		g := NewWithT(t)
+		controlPlane := &internal.ControlPlane{
+			Cluster: &clusterv1.Cluster{},
+			KCP:     &controlplanev1.KubeadmControlPlane{},
+			Machines: collections.FromMachines(
+				&clusterv1.Machine{
+					ObjectMeta: metav1.ObjectMeta{Name: "m1"},
+					Status: clusterv1.MachineStatus{
+						Conditions: []metav1.Condition{
+							{
+								Type:   clusterv1.MachineHealthCheckSucceededCondition,
+								Status: metav1.ConditionFalse,
+								Reason: clusterv1.MachineHealthCheckNodeDeletedReason,
+							},
+							{
+								Type:    clusterv1.MachineOwnerRemediatedCondition,
+								Status:  metav1.ConditionFalse,
+								Reason:  clusterv1.MachineOwnerRemediatedWaitingForRemediationReason,
+								Message: "Waiting for remediation",
+							},
+						},
+					},
+				},
+			),
+		}
+		controlPlane.InjectTestManagementCluster(&fakeManagementCluster{
+			Workload: &fakeWorkloadCluster{
+				Status: internal.ClusterStatus{
+					HasKubeadmConfig: true,
+				},
+			},
+		})
+
+		err := setControlPlaneInitialized(ctx, controlPlane)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		g.Expect(ptr.Deref(controlPlane.KCP.Status.Initialization.ControlPlaneInitialized, false)).To(BeFalse())
+
+		setInitializedCondition(ctx, controlPlane.KCP)
+		c := conditions.Get(controlPlane.KCP, controlplanev1.KubeadmControlPlaneInitializedCondition)
+		g.Expect(c).ToNot(BeNil())
+		g.Expect(*c).To(conditions.MatchCondition(metav1.Condition{
+			Type:   controlplanev1.KubeadmControlPlaneInitializedCondition,
+			Status: metav1.ConditionFalse,
+			Reason: controlplanev1.KubeadmControlPlaneNotInitializedReason,
+		}, conditions.IgnoreLastTransitionTime(true)))
+	})
+	t.Run("kubeadm config exists is ignored if there is a single CP machine and it is deleting", func(t *testing.T) {
+		g := NewWithT(t)
+		controlPlane := &internal.ControlPlane{
+			Cluster: &clusterv1.Cluster{},
+			KCP:     &controlplanev1.KubeadmControlPlane{},
+			Machines: collections.FromMachines(
+				&clusterv1.Machine{ObjectMeta: metav1.ObjectMeta{Name: "m1", DeletionTimestamp: ptr.To(metav1.Now())}},
+			),
+		}
+		controlPlane.InjectTestManagementCluster(&fakeManagementCluster{
+			Workload: &fakeWorkloadCluster{
+				Status: internal.ClusterStatus{
+					HasKubeadmConfig: true,
+				},
+			},
+		})
+
+		err := setControlPlaneInitialized(ctx, controlPlane)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		g.Expect(ptr.Deref(controlPlane.KCP.Status.Initialization.ControlPlaneInitialized, false)).To(BeFalse())
+
+		setInitializedCondition(ctx, controlPlane.KCP)
+		c := conditions.Get(controlPlane.KCP, controlplanev1.KubeadmControlPlaneInitializedCondition)
+		g.Expect(c).ToNot(BeNil())
+		g.Expect(*c).To(conditions.MatchCondition(metav1.Condition{
+			Type:   controlplanev1.KubeadmControlPlaneInitializedCondition,
+			Status: metav1.ConditionFalse,
+			Reason: controlplanev1.KubeadmControlPlaneNotInitializedReason,
+		}, conditions.IgnoreLastTransitionTime(true)))
+	})
 }
 
 func TestSetReplicas(t *testing.T) {