Skip to content

Commit 67ba387

Browse files
authored
fix: do not delete the workspace pod when the last update failed (#828)
### Proposed changes This PR fixes it so that the Workspace pod is not reclaimed when the update failed. We should only delete the pod if the sync was successful, and the update was also successful. ### Related issues (optional) Closes: #826
1 parent b00040c commit 67ba387

File tree

7 files changed

+24
-10
lines changed

7 files changed

+24
-10
lines changed

deploy/crds/pulumi.com_stacks.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,7 @@ spec:
773773
type: boolean
774774
workspaceReclaimPolicy:
775775
description: |-
776-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
776+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
777777
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
778778
enum:
779779
- Retain
@@ -10409,7 +10409,7 @@ spec:
1040910409
type: boolean
1041010410
workspaceReclaimPolicy:
1041110411
description: |-
10412-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
10412+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
1041310413
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
1041410414
enum:
1041510415
- Retain

deploy/helm/pulumi-operator/crds/pulumi.com_stacks.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,7 @@ spec:
773773
type: boolean
774774
workspaceReclaimPolicy:
775775
description: |-
776-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
776+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
777777
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
778778
enum:
779779
- Retain
@@ -10409,7 +10409,7 @@ spec:
1040910409
type: boolean
1041010410
workspaceReclaimPolicy:
1041110411
description: |-
10412-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
10412+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
1041310413
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
1041410414
enum:
1041510415
- Retain

docs/stacks.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -368,7 +368,7 @@ The default behavior is to create a stack if it doesn't exist.<br/>
368368
<td><b>workspaceReclaimPolicy</b></td>
369369
<td>enum</td>
370370
<td>
371-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
371+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
372372
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".<br/>
373373
<br/>
374374
<i>Enum</i>: Retain, Delete<br/>
@@ -19803,7 +19803,7 @@ The default behavior is to create a stack if it doesn't exist.<br/>
1980319803
<td><b>workspaceReclaimPolicy</b></td>
1980419804
<td>enum</td>
1980519805
<td>
19806-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
19806+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
1980719807
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".<br/>
1980819808
<br/>
1980919809
<i>Enum</i>: Retain, Delete<br/>

operator/api/pulumi/shared/stack_types.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,7 @@ type StackSpec struct {
155155
// +optional
156156
WorkspaceTemplate *WorkspaceApplyConfiguration `json:"workspaceTemplate,omitempty"`
157157

158-
// WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
158+
// WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
159159
// The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
160160
// +optional
161161
WorkspaceReclaimPolicy WorkspaceReclaimPolicy `json:"workspaceReclaimPolicy,omitempty"`

operator/config/crd/bases/pulumi.com_stacks.yaml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -773,7 +773,7 @@ spec:
773773
type: boolean
774774
workspaceReclaimPolicy:
775775
description: |-
776-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
776+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
777777
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
778778
enum:
779779
- Retain
@@ -10409,7 +10409,7 @@ spec:
1040910409
type: boolean
1041010410
workspaceReclaimPolicy:
1041110411
description: |-
10412-
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is synced.
10412+
WorkspaceReclaimPolicy specifies whether the workspace should be deleted or retained after the Stack is successfully updated.
1041310413
The default behavior is to retain the workspace. Valid values are one of "Retain" or "Delete".
1041410414
enum:
1041510415
- Retain

operator/internal/controller/pulumi/stack_controller.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -769,9 +769,11 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) (
769769
// Requeue reconciliation as necessary to detect branch updates and
770770
// resyncs. The logic finds the smallest requeue time (if any) among various polling loops.
771771
requeueAfter := time.Duration(0)
772+
updateFailed := false
772773

773774
// Try again with exponential backoff if the update failed.
774775
if instance.Status.LastUpdate.State == shared.FailedStackStateMessage {
776+
updateFailed = true
775777
requeueAfter = max(1*time.Second, time.Until(instance.Status.LastUpdate.LastResyncTime.Add(cooldown(instance))))
776778
}
777779
// Schedule another poll if ContinueResyncOnCommitMatch is set, for drift detection or to maintain dynamic resources.
@@ -802,7 +804,7 @@ func (r *StackReconciler) Reconcile(ctx context.Context, request ctrl.Request) (
802804
}
803805

804806
// Delete the workspace if the reclaim policy is set to delete.
805-
if instance.Spec.WorkspaceReclaimPolicy == shared.WorkspaceReclaimDelete {
807+
if !updateFailed && instance.Spec.WorkspaceReclaimPolicy == shared.WorkspaceReclaimDelete {
806808
log.Info("Deleting workspace as reclaim policy is set to delete")
807809
err := sess.DeleteWorkspace(ctx)
808810
if err != nil {

operator/internal/controller/pulumi/stack_controller_test.go

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -788,6 +788,18 @@ var _ = Describe("Stack Controller", func() {
788788
Expect(res.RequeueAfter).To(BeNumerically("~", 8*time.Minute, time.Minute))
789789
ByMarkingAsReconciling(pulumiv1.ReconcilingRetryReason, Equal("3 update failure(s)"))
790790
})
791+
792+
When("the WorkspaceReclaimPolicy is set to Delete", func() {
793+
BeforeEach(func(ctx context.Context) {
794+
obj.Spec.WorkspaceReclaimPolicy = shared.WorkspaceReclaimDelete
795+
})
796+
It("does not delete the workspace pod", func(ctx context.Context) {
797+
_, err := reconcileF(ctx)
798+
Expect(err).NotTo(HaveOccurred())
799+
By("not deleting the Workspace object")
800+
Expect(ws.GetName()).NotTo(BeEmpty())
801+
})
802+
})
791803
})
792804
When("done cooling down", func() {
793805
BeforeEach(func() {

0 commit comments

Comments
 (0)