Skip to content

Commit d5ee623

Browse files
committed
updateRun e2e test
1 parent a38fb75 commit d5ee623

File tree

11 files changed

+863
-61
lines changed

11 files changed

+863
-61
lines changed

apis/placement/v1alpha1/stagedupdate_types.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,8 @@ import (
1818
// +kubebuilder:resource:scope=Cluster,categories={fleet,fleet-placement},shortName=crsur
1919
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
2020
// +kubebuilder:printcolumn:JSONPath=`.spec.placementName`,name="Placement",type=string
21-
// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot",type=string
22-
// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot",type=string
21+
// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot-Index",type=string
22+
// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot-Index",type=string
2323
// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Initialized")].status`,name="Initialized",type=string
2424
// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Succeeded")].status`,name="Succeeded",type=string
2525
// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date

apis/placement/v1beta1/stageupdate_types.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,8 @@ import (
1717
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
1818
// +kubebuilder:storageversion
1919
// +kubebuilder:printcolumn:JSONPath=`.spec.placementName`,name="Placement",type=string
20-
// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot",type=string
21-
// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot",type=string
20+
// +kubebuilder:printcolumn:JSONPath=`.spec.resourceSnapshotIndex`,name="Resource-Snapshot-Index",type=string
21+
// +kubebuilder:printcolumn:JSONPath=`.status.policySnapshotIndexUsed`,name="Policy-Snapshot-Index",type=string
2222
// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Initialized")].status`,name="Initialized",type=string
2323
// +kubebuilder:printcolumn:JSONPath=`.status.conditions[?(@.type=="Succeeded")].status`,name="Succeeded",type=string
2424
// +kubebuilder:printcolumn:JSONPath=`.metadata.creationTimestamp`,name="Age",type=date

config/crd/bases/placement.kubernetes-fleet.io_clusterstagedupdateruns.yaml

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -24,10 +24,10 @@ spec:
2424
name: Placement
2525
type: string
2626
- jsonPath: .spec.resourceSnapshotIndex
27-
name: Resource-Snapshot
27+
name: Resource-Snapshot-Index
2828
type: string
2929
- jsonPath: .status.policySnapshotIndexUsed
30-
name: Policy-Snapshot
30+
name: Policy-Snapshot-Index
3131
type: string
3232
- jsonPath: .status.conditions[?(@.type=="Initialized")].status
3333
name: Initialized
@@ -1244,10 +1244,10 @@ spec:
12441244
name: Placement
12451245
type: string
12461246
- jsonPath: .spec.resourceSnapshotIndex
1247-
name: Resource-Snapshot
1247+
name: Resource-Snapshot-Index
12481248
type: string
12491249
- jsonPath: .status.policySnapshotIndexUsed
1250-
name: Policy-Snapshot
1250+
name: Policy-Snapshot-Index
12511251
type: string
12521252
- jsonPath: .status.conditions[?(@.type=="Initialized")].status
12531253
name: Initialized

pkg/controllers/updaterun/execution.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -200,7 +200,8 @@ func (r *Reconciler) executeDeleteStage(
200200
for i := range existingDeleteStageStatus.Clusters {
201201
existingDeleteStageClusterMap[existingDeleteStageStatus.Clusters[i].ClusterName] = &existingDeleteStageStatus.Clusters[i]
202202
}
203-
deletingBinding := 0
203+
// Mark the delete stage as started in case it's not.
204+
markStageUpdatingStarted(updateRun.Status.DeletionStageStatus, updateRun.Generation)
204205
for _, binding := range toBeDeletedBindings {
205206
curCluster, exist := existingDeleteStageClusterMap[binding.Spec.TargetCluster]
206207
if !exist {
@@ -225,7 +226,6 @@ func (r *Reconciler) executeDeleteStage(
225226
klog.ErrorS(unexpectedErr, "The binding should be deleting before we mark a cluster deleting", "clusterStatus", curCluster, "clusterStagedUpdateRun", updateRunRef)
226227
return false, fmt.Errorf("%w: %s", errStagedUpdatedAborted, unexpectedErr.Error())
227228
}
228-
deletingBinding++
229229
continue
230230
}
231231
// The cluster status is not deleting yet
@@ -235,10 +235,6 @@ func (r *Reconciler) executeDeleteStage(
235235
}
236236
klog.V(2).InfoS("Deleted a binding pointing to a to be deleted cluster", "binding", klog.KObj(binding), "cluster", curCluster.ClusterName, "clusterStagedUpdateRun", updateRunRef)
237237
markClusterUpdatingStarted(curCluster, updateRun.Generation)
238-
if deletingBinding == 0 {
239-
markStageUpdatingStarted(updateRun.Status.DeletionStageStatus, updateRun.Generation)
240-
}
241-
deletingBinding++
242238
}
243239
// The rest of the clusters in the stage are not in the toBeDeletedBindings so it should be marked as delete succeeded.
244240
for _, clusterStatus := range existingDeleteStageClusterMap {

pkg/controllers/updaterun/initialization.go

Lines changed: 15 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -126,26 +126,23 @@ func (r *Reconciler) determinePolicySnapshot(
126126
}
127127
updateRun.Status.PolicySnapshotIndexUsed = policyIndex
128128

129-
// Get the cluster count from the policy snapshot.
130-
if latestPolicySnapshot.Spec.Policy == nil {
131-
nopolicyErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("policy snapshot `%s` does not have a policy", latestPolicySnapshot.Name))
132-
klog.ErrorS(nopolicyErr, "Failed to get the policy from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef)
133-
// no more retries here.
134-
return nil, -1, fmt.Errorf("%w: %s", errInitializedFailed, nopolicyErr.Error())
135-
}
136-
// for pickAll policy, the observed cluster count is not included in the policy snapshot. We set it to -1. It will be validated in the binding stages.
129+
// For pickAll policy, the observed cluster count is not included in the policy snapshot.
130+
// We set it to -1. It will be validated in the binding stages.
131+
// If policy is nil, it's default to pickAll.
137132
clusterCount := -1
138-
if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickNPlacementType {
139-
count, err := annotations.ExtractNumOfClustersFromPolicySnapshot(&latestPolicySnapshot)
140-
if err != nil {
141-
annErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("%w: the policy snapshot `%s` doesn't have valid cluster count annotation", err, latestPolicySnapshot.Name))
142-
klog.ErrorS(annErr, "Failed to get the cluster count from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef)
143-
// no more retries here.
144-
return nil, -1, fmt.Errorf("%w, %s", errInitializedFailed, annErr.Error())
133+
if latestPolicySnapshot.Spec.Policy != nil {
134+
if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickNPlacementType {
135+
count, err := annotations.ExtractNumOfClustersFromPolicySnapshot(&latestPolicySnapshot)
136+
if err != nil {
137+
annErr := controller.NewUnexpectedBehaviorError(fmt.Errorf("%w: the policy snapshot `%s` doesn't have valid cluster count annotation", err, latestPolicySnapshot.Name))
138+
klog.ErrorS(annErr, "Failed to get the cluster count from the latestPolicySnapshot", "clusterResourcePlacement", placementName, "latestPolicySnapshot", latestPolicySnapshot.Name, "clusterStagedUpdateRun", updateRunRef)
139+
// no more retries here.
140+
return nil, -1, fmt.Errorf("%w, %s", errInitializedFailed, annErr.Error())
141+
}
142+
clusterCount = count
143+
} else if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickFixedPlacementType {
144+
clusterCount = len(latestPolicySnapshot.Spec.Policy.ClusterNames)
145145
}
146-
clusterCount = count
147-
} else if latestPolicySnapshot.Spec.Policy.PlacementType == placementv1beta1.PickFixedPlacementType {
148-
clusterCount = len(latestPolicySnapshot.Spec.Policy.ClusterNames)
149146
}
150147
updateRun.Status.PolicyObservedClusterCount = clusterCount
151148
klog.V(2).InfoS("Found the latest policy snapshot", "latestPolicySnapshot", latestPolicySnapshot.Name, "observedClusterCount", updateRun.Status.PolicyObservedClusterCount, "clusterStagedUpdateRun", updateRunRef)

pkg/controllers/updaterun/initialization_integration_test.go

Lines changed: 0 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -220,30 +220,6 @@ var _ = Describe("Updaterun initialization tests", func() {
220220
Expect(k8sClient.Delete(ctx, snapshot2)).Should(Succeed())
221221
})
222222

223-
It("Should fail to initialize if the latest policy snapshot does not have index label", func() {
224-
By("Creating scheduling policy snapshot without index label")
225-
delete(policySnapshot.Labels, placementv1beta1.PolicyIndexLabel)
226-
Expect(k8sClient.Create(ctx, policySnapshot)).To(Succeed())
227-
228-
By("Creating a new clusterStagedUpdateRun")
229-
Expect(k8sClient.Create(ctx, updateRun)).To(Succeed())
230-
231-
By("Validating the initialization failed")
232-
validateFailedInitCondition(ctx, updateRun, "does not have a policy index label")
233-
})
234-
235-
It("Should fail to initialize if the latest policy snapshot has a nil policy", func() {
236-
By("Creating scheduling policy snapshot with nil policy")
237-
policySnapshot.Spec.Policy = nil
238-
Expect(k8sClient.Create(ctx, policySnapshot)).To(Succeed())
239-
240-
By("Creating a new clusterStagedUpdateRun")
241-
Expect(k8sClient.Create(ctx, updateRun)).To(Succeed())
242-
243-
By("Validating the initialization failed")
244-
validateFailedInitCondition(ctx, updateRun, "does not have a policy")
245-
})
246-
247223
It("Should fail to initialize if the latest policy snapshot does not have valid cluster count annotation", func() {
248224
By("Creating scheduling policy snapshot with invalid cluster count annotation")
249225
delete(policySnapshot.Annotations, placementv1beta1.NumberOfClustersAnnotation)

test/e2e/actuals_test.go

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -653,6 +653,7 @@ func crpStatusWithOverrideUpdatedFailedActual(
653653
return nil
654654
}
655655
}
656+
656657
func crpStatusWithWorkSynchronizedUpdatedFailedActual(
657658
wantSelectedResourceIdentifiers []placementv1beta1.ResourceIdentifier,
658659
wantSelectedClusters []string,
@@ -949,3 +950,184 @@ func validateCRPSnapshotRevisions(crpName string, wantPolicySnapshotRevision, wa
949950
}
950951
return nil
951952
}
953+
954+
func updateRunClusterRolloutSucceedConditions(generation int64) []metav1.Condition {
955+
return []metav1.Condition{
956+
{
957+
Type: string(placementv1beta1.ClusterUpdatingConditionStarted),
958+
Status: metav1.ConditionTrue,
959+
Reason: condition.ClusterUpdatingStartedReason,
960+
ObservedGeneration: generation,
961+
},
962+
{
963+
Type: string(placementv1beta1.ClusterUpdatingConditionSucceeded),
964+
Status: metav1.ConditionTrue,
965+
Reason: condition.ClusterUpdatingSucceededReason,
966+
ObservedGeneration: generation,
967+
},
968+
}
969+
}
970+
971+
func updateRunStageRolloutSucceedConditions(generation int64, wait bool) []metav1.Condition {
972+
startedCond := metav1.Condition{
973+
Type: string(placementv1beta1.StageUpdatingConditionProgressing),
974+
Status: metav1.ConditionTrue,
975+
Reason: condition.StageUpdatingStartedReason,
976+
ObservedGeneration: generation,
977+
}
978+
if wait {
979+
startedCond.Status = metav1.ConditionFalse
980+
startedCond.Reason = condition.StageUpdatingWaitingReason
981+
}
982+
return []metav1.Condition{
983+
startedCond,
984+
{
985+
Type: string(placementv1beta1.StageUpdatingConditionSucceeded),
986+
Status: metav1.ConditionTrue,
987+
Reason: condition.StageUpdatingSucceededReason,
988+
ObservedGeneration: generation,
989+
},
990+
}
991+
}
992+
993+
func updateRunAfterStageTaskSucceedConditions(generation int64, taskType placementv1beta1.AfterStageTaskType) []metav1.Condition {
994+
if taskType == placementv1beta1.AfterStageTaskTypeApproval {
995+
return []metav1.Condition{
996+
{
997+
Type: string(placementv1beta1.AfterStageTaskConditionApprovalRequestCreated),
998+
Status: metav1.ConditionTrue,
999+
Reason: condition.AfterStageTaskApprovalRequestCreatedReason,
1000+
ObservedGeneration: generation,
1001+
},
1002+
{
1003+
Type: string(placementv1beta1.AfterStageTaskConditionApprovalRequestApproved),
1004+
Status: metav1.ConditionTrue,
1005+
Reason: condition.AfterStageTaskApprovalRequestApprovedReason,
1006+
ObservedGeneration: generation,
1007+
},
1008+
}
1009+
}
1010+
return []metav1.Condition{
1011+
{
1012+
Type: string(placementv1beta1.AfterStageTaskConditionWaitTimeElapsed),
1013+
Status: metav1.ConditionTrue,
1014+
Reason: condition.AfterStageTaskWaitTimeElapsedReason,
1015+
ObservedGeneration: generation,
1016+
},
1017+
}
1018+
}
1019+
1020+
func updateRunSucceedConditions(generation int64) []metav1.Condition {
1021+
return []metav1.Condition{
1022+
{
1023+
Type: string(placementv1beta1.StagedUpdateRunConditionInitialized),
1024+
Status: metav1.ConditionTrue,
1025+
Reason: condition.UpdateRunInitializeSucceededReason,
1026+
ObservedGeneration: generation,
1027+
},
1028+
{
1029+
Type: string(placementv1beta1.StagedUpdateRunConditionProgressing),
1030+
Status: metav1.ConditionTrue,
1031+
Reason: condition.UpdateRunStartedReason,
1032+
ObservedGeneration: generation,
1033+
},
1034+
{
1035+
Type: string(placementv1beta1.StagedUpdateRunConditionSucceeded),
1036+
Status: metav1.ConditionTrue,
1037+
Reason: condition.UpdateRunSucceededReason,
1038+
ObservedGeneration: generation,
1039+
},
1040+
}
1041+
}
1042+
1043+
func updateRunStatusSucceededActual(
1044+
updateRunName string,
1045+
wantPolicyIndex string,
1046+
wantClusterCount int,
1047+
wantApplyStrategy *placementv1beta1.ApplyStrategy,
1048+
wantStrategySpec *placementv1beta1.StagedUpdateStrategySpec,
1049+
wantSelectedClusters [][]string,
1050+
wantUnscheduledClusters []string,
1051+
wantCROs map[string][]string,
1052+
wantROs map[string][]placementv1beta1.NamespacedName,
1053+
) func() error {
1054+
return func() error {
1055+
updateRun := &placementv1beta1.ClusterStagedUpdateRun{}
1056+
if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, updateRun); err != nil {
1057+
return err
1058+
}
1059+
1060+
wantStatus := placementv1beta1.StagedUpdateRunStatus{
1061+
PolicySnapshotIndexUsed: wantPolicyIndex,
1062+
PolicyObservedClusterCount: wantClusterCount,
1063+
ApplyStrategy: wantApplyStrategy.DeepCopy(),
1064+
StagedUpdateStrategySnapshot: wantStrategySpec,
1065+
}
1066+
stagesStatus := make([]placementv1beta1.StageUpdatingStatus, len(wantStrategySpec.Stages))
1067+
for i, stage := range wantStrategySpec.Stages {
1068+
stagesStatus[i].StageName = stage.Name
1069+
stagesStatus[i].Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantSelectedClusters[i]))
1070+
for j := range stagesStatus[i].Clusters {
1071+
stagesStatus[i].Clusters[j].ClusterName = wantSelectedClusters[i][j]
1072+
stagesStatus[i].Clusters[j].ClusterResourceOverrideSnapshots = wantCROs[wantSelectedClusters[i][j]]
1073+
stagesStatus[i].Clusters[j].ResourceOverrideSnapshots = wantROs[wantSelectedClusters[i][j]]
1074+
stagesStatus[i].Clusters[j].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.Generation)
1075+
}
1076+
stagesStatus[i].AfterStageTaskStatus = make([]placementv1beta1.AfterStageTaskStatus, len(stage.AfterStageTasks))
1077+
for j, task := range stage.AfterStageTasks {
1078+
stagesStatus[i].AfterStageTaskStatus[j].Type = task.Type
1079+
if task.Type == placementv1beta1.AfterStageTaskTypeApproval {
1080+
stagesStatus[i].AfterStageTaskStatus[j].ApprovalRequestName = fmt.Sprintf(placementv1beta1.ApprovalTaskNameFmt, updateRun.Name, stage.Name)
1081+
}
1082+
stagesStatus[i].AfterStageTaskStatus[j].Conditions = updateRunAfterStageTaskSucceedConditions(updateRun.Generation, task.Type)
1083+
}
1084+
stagesStatus[i].Conditions = updateRunStageRolloutSucceedConditions(updateRun.Generation, true)
1085+
}
1086+
1087+
deleteStageStatus := &placementv1beta1.StageUpdatingStatus{
1088+
StageName: "kubernetes-fleet.io/deleteStage",
1089+
}
1090+
deleteStageStatus.Clusters = make([]placementv1beta1.ClusterUpdatingStatus, len(wantUnscheduledClusters))
1091+
for i := range deleteStageStatus.Clusters {
1092+
deleteStageStatus.Clusters[i].ClusterName = wantUnscheduledClusters[i]
1093+
deleteStageStatus.Clusters[i].Conditions = updateRunClusterRolloutSucceedConditions(updateRun.Generation)
1094+
}
1095+
deleteStageStatus.Conditions = updateRunStageRolloutSucceedConditions(updateRun.Generation, false)
1096+
1097+
wantStatus.StagesStatus = stagesStatus
1098+
wantStatus.DeletionStageStatus = deleteStageStatus
1099+
wantStatus.Conditions = updateRunSucceedConditions(updateRun.Generation)
1100+
if diff := cmp.Diff(updateRun.Status, wantStatus, updateRunStatusCmpOption...); diff != "" {
1101+
return fmt.Errorf("CRP status diff (-got, +want): %s", diff)
1102+
}
1103+
return nil
1104+
}
1105+
}
1106+
1107+
func updateRunAndApprovalRequestsRemovedActual(updateRunName string) func() error {
1108+
return func() error {
1109+
if err := hubClient.Get(ctx, types.NamespacedName{Name: updateRunName}, &placementv1beta1.ClusterStagedUpdateRun{}); !errors.IsNotFound(err) {
1110+
return fmt.Errorf("UpdateRun still exists or an unexpected error occurred: %w", err)
1111+
}
1112+
1113+
appReqList := &placementv1beta1.ClusterApprovalRequestList{}
1114+
if err := hubClient.List(ctx, appReqList, client.MatchingLabels{
1115+
placementv1beta1.TargetUpdateRunLabel: updateRunName,
1116+
}); err != nil {
1117+
return fmt.Errorf("failed to list ClusterApprovalRequests: %w", err)
1118+
}
1119+
if len(appReqList.Items) > 0 {
1120+
return fmt.Errorf("ClusterApprovalRequests still exist: %v", appReqList.Items)
1121+
}
1122+
return nil
1123+
}
1124+
}
1125+
1126+
func updateRunStrategyRemovedActual(strategyName string) func() error {
1127+
return func() error {
1128+
if err := hubClient.Get(ctx, types.NamespacedName{Name: strategyName}, &placementv1beta1.ClusterStagedUpdateStrategy{}); !errors.IsNotFound(err) {
1129+
return fmt.Errorf("ClusterStagedUpdateStrategy still exists or an unexpected error occurred: %w", err)
1130+
}
1131+
return nil
1132+
}
1133+
}

test/e2e/resources_test.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ const (
3434
internalServiceImportNameTemplate = "isi-%d"
3535
endpointSliceExportNameTemplate = "ep-%d"
3636
crpEvictionNameTemplate = "crpe-%d"
37+
updateRunStrategyNameTemplate = "curs-%d"
38+
updateRunNameWithSubIndexTemplate = "cur-%d-%d"
3739

3840
customDeletionBlockerFinalizer = "custom-deletion-blocker-finalizer"
3941
workNamespaceLabelName = "process"

test/e2e/setup_test.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -211,6 +211,12 @@ var (
211211
ignoreClusterNameField,
212212
cmpopts.EquateEmpty(),
213213
}
214+
215+
updateRunStatusCmpOption = cmp.Options{
216+
utils.IgnoreConditionLTTAndMessageFields,
217+
cmpopts.IgnoreFields(placementv1beta1.StageUpdatingStatus{}, "StartTime", "EndTime"),
218+
cmpopts.EquateEmpty(),
219+
}
214220
)
215221

216222
// TestMain sets up the E2E test environment.

0 commit comments

Comments
 (0)