Add JobGeneration

For tracking when jobs are recreated Signed-off-by: Fabricio Aguiar <[email protected]>
openstack-k8s-operators · Apr 2, 2024 · 6e8cdc0 · 6e8cdc0
1 parent 1003b1a
commit 6e8cdc0
Show file tree

Hide file tree

Showing 13 changed files with 181 additions and 0 deletions.
diff --git a/api/bases/ansibleee.openstack.org_openstackansibleees.yaml b/api/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1545,6 +1545,10 @@ spec:
             type: object
           status:
             properties:
+              JobGeneration:
+                default: 1
+                format: int64
+                type: integer
               JobStatus:
                 default: Pending
                 enum:

diff --git a/api/v1beta1/openstack_ansibleee_types.go b/api/v1beta1/openstack_ansibleee_types.go
@@ -134,6 +134,10 @@ type OpenStackAnsibleEEStatus struct {
 	// +kubebuilder:default:=Pending
 	// JobStatus status of the executed job (Pending/Running/Succeeded/Failed)
 	JobStatus string `json:"JobStatus,omitempty" optional:"true"`
+
+	// +kubebuilder:default:=1
+	// JobGeneration track the amount of job generations
+	JobGeneration int64 `json:"JobGeneration,omitempty"`
 }
 
 //+kubebuilder:object:root=true

diff --git a/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml b/config/crd/bases/ansibleee.openstack.org_openstackansibleees.yaml
@@ -1545,6 +1545,10 @@ spec:
             type: object
           status:
             properties:
+              JobGeneration:
+                default: 1
+                format: int64
+                type: integer
               JobStatus:
                 default: Pending
                 enum:

diff --git a/controllers/openstack_ansibleee_controller.go b/controllers/openstack_ansibleee_controller.go
@@ -202,6 +202,10 @@ func (r *OpenStackAnsibleEEReconciler) Reconcile(ctx context.Context, req ctrl.R
 		currentJobHash,
 	)
 
+	if len(currentJobHash) > 0 && ansibleeeJob.HasChanged() {
+		instance.Status.JobGeneration++
+	}
+
 	ctrlResult, err := ansibleeeJob.DoJob(
 		ctx,
 		helper,

diff --git a/docs/assemblies/openstack_ansibleee.adoc b/docs/assemblies/openstack_ansibleee.adoc
@@ -212,6 +212,11 @@ OpenStackAnsibleEEStatus defines the observed state of OpenStackAnsibleEE
 | JobStatus status of the executed job (Pending/Running/Succeeded/Failed)
 | string
 | false
+
+| JobGeneration
+| JobGeneration track the amount of job generations
+| int64
+| false
 |===
 
 <<custom-resources,Back to Custom Resources>>
diff --git a/tests/functional/ansibleee_controller_test.go b/tests/functional/ansibleee_controller_test.go
@@ -53,6 +53,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -72,6 +73,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Succeeded"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).To(HaveKey("ansibleee"))
 			})
@@ -93,6 +95,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -115,6 +118,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Failed"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 			})
@@ -145,6 +149,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Succeeded"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				originalInputHash := ansibleee.Status.Hash["input"]
 				Expect(ansibleee.Status.Hash).To(HaveKey("ansibleee"))
@@ -170,6 +175,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 
 				// simulate that the second job succeeds
 				th.SimulateJobSuccess(ansibleeeName)
@@ -182,6 +188,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Succeeded"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash["input"]).NotTo(Equal(originalInputHash))
 				Expect(ansibleee.Status.Hash["ansibleee"]).NotTo(Equal(originalJobHash))
 			})
@@ -227,6 +234,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -246,6 +254,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Succeeded"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).To(HaveKey("ansibleee"))
 			})
@@ -267,6 +276,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -289,6 +299,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Failed"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 			})
@@ -358,6 +369,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -377,6 +389,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Succeeded"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).To(HaveKey("ansibleee"))
 			})
@@ -398,6 +411,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -420,6 +434,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Failed"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 			})
@@ -506,6 +521,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -525,6 +541,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Succeeded"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).To(HaveKey("ansibleee"))
 			})
@@ -546,6 +563,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee := GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Running"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 
@@ -568,6 +586,7 @@ var _ = Describe("Ansibleee controller", func() {
 				)
 				ansibleee = GetAnsibleee(ansibleeeName)
 				Expect(ansibleee.Status.JobStatus).To(Equal("Failed"))
+				Expect(ansibleee.Status.JobGeneration).To(Equal(int64(1)))
 				Expect(ansibleee.Status.Hash).To(HaveKey("input"))
 				Expect(ansibleee.Status.Hash).NotTo(HaveKey("ansibleee"))
 			})

diff --git a/tests/kuttl/tests/run_failed_playbook/01-assert.yaml b/tests/kuttl/tests/run_failed_playbook/01-assert.yaml
@@ -23,6 +23,7 @@ spec:
   preserveJobs: true
 status:
   JobStatus: Failed
+  JobGeneration: 1
   conditions:
   - message: 'AnsibleExecutionJob error occured Internal error occurred: Job Failed.
       Check job logs'

diff --git a/tests/kuttl/tests/run_failed_playbook/02-assert.yaml b/tests/kuttl/tests/run_failed_playbook/02-assert.yaml
@@ -0,0 +1,117 @@
+#
+# Check for:
+#
+# - 1 OpenStackAnsibleEE CR
+# - 1 failed-play pod
+# - 1 failed-play job
+# - Correct output from ansible play
+#
+apiVersion: ansibleee.openstack.org/v1beta1
+kind: OpenStackAnsibleEE
+metadata:
+  name: failed-play
+spec:
+  name: openstackansibleee
+  play: |
+    - name: Execution failure
+      hosts: localhost
+      tasks:
+      - name: Copy absent file
+        ansible.builtin.shell: |
+            set -euxo pipefail
+            cp absent failed_op
+  preserveJobs: true
+status:
+  JobStatus: Failed
+  JobGeneration: 2
+  conditions:
+  - message: 'AnsibleExecutionJob error occured Internal error occurred: Job Failed.
+      Check job logs'
+    reason: Error
+    severity: Warning
+    status: "False"
+    type: Ready
+  - message: 'AnsibleExecutionJob error occured Internal error occurred: Job Failed.
+      Check job logs'
+    reason: Error
+    severity: Warning
+    status: "False"
+    type: AnsibleExecutionJobReady
+---
+apiVersion: v1
+kind: Pod
+metadata:
+  generateName: failed-play-
+  labels:
+    app: openstackansibleee
+    job-name: failed-play
+status:
+  phase: Failed
+---
+apiVersion: batch/v1
+kind: Job
+metadata:
+  labels:
+    app: openstackansibleee
+    job-name: failed-play
+    openstackansibleee_cr: failed-play
+    osaee: "true"
+  name: failed-play
+spec:
+  backoffLimit: 1
+  completionMode: NonIndexed
+  completions: 1
+  parallelism: 1
+  template:
+    metadata:
+      labels:
+        app: openstackansibleee
+        batch.kubernetes.io/job-name: failed-play
+        job-name: failed-play
+        openstackansibleee_cr: failed-play
+        osaee: "true"
+    spec:
+      containers:
+      - args:
+        - ansible-runner
+        - run
+        - /runner
+        - -p
+        - playbook.yaml
+        - -i
+        - failed-play
+        env:
+        - name: RUNNER_PLAYBOOK
+          value: |2+
+
+            - name: Execution failure
+              hosts: localhost
+              tasks:
+              - name: Copy absent file
+                ansible.builtin.shell: |
+                    set -euxo pipefail
+                    cp absent failed_op
+
+
+        - name: RUNNER_EXTRA_VARS
+          value: |2+
+
+            aaa: %!s(int=1)
+            bbb: %!s(int=2)
+            ccc: %!s(int=3)
+            ddd: %!s(int=4)
+
+
+        imagePullPolicy: Always
+        name: openstackansibleee
+      dnsPolicy: ClusterFirst
+      restartPolicy: Never
+      schedulerName: default-scheduler
+      terminationGracePeriodSeconds: 30
+status:
+  conditions:
+  - message: Job has reached the specified backoff limit
+    reason: BackoffLimitExceeded
+    status: "True"
+    type: Failed
+  failed: 2
diff --git a/tests/kuttl/tests/run_failed_playbook/02-patch_aee.yaml b/tests/kuttl/tests/run_failed_playbook/02-patch_aee.yaml
@@ -0,0 +1,19 @@
+apiVersion: ansibleee.openstack.org/v1beta1
+kind: OpenStackAnsibleEE
+metadata:
+  name: failed-play
+spec:
+  play: |
+    - name: Execution failure
+      hosts: localhost
+      tasks:
+      - name: Copy absent file
+        ansible.builtin.shell: |
+            set -euxo pipefail
+            cp absent failed_op
+  extraVars:
+    ddd: 4
+    aaa: 1
+    ccc: 3
+    bbb: 2
+  backoffLimit: 1
diff --git a/tests/kuttl/tests/run_simple_playbook/01-assert.yaml b/tests/kuttl/tests/run_simple_playbook/01-assert.yaml
@@ -21,6 +21,7 @@ spec:
           msg: "Hello, world this is ansibleee-play.yaml"
 status:
   JobStatus: Succeeded
+  JobGeneration: 1
 ---
 apiVersion: v1
 kind: Pod

diff --git a/tests/kuttl/tests/run_simple_playbook_debug/01-assert.yaml b/tests/kuttl/tests/run_simple_playbook_debug/01-assert.yaml
@@ -21,6 +21,7 @@ spec:
           msg: "Hello, world this is ansibleee-play-debug.yaml"
 status:
   JobStatus: Running
+  JobGeneration: 1
 ---
 apiVersion: v1
 kind: Pod

diff --git a/tests/kuttl/tests/run_simple_playbook_debug_cmdline/01-assert.yaml b/tests/kuttl/tests/run_simple_playbook_debug_cmdline/01-assert.yaml
@@ -21,6 +21,7 @@ spec:
           msg: "Hello, world this is ansibleee-play-debug.yaml"
 status:
   JobStatus: Running
+  JobGeneration: 1
 ---
 apiVersion: v1
 kind: Pod

diff --git a/tests/kuttl/tests/run_simple_playbook_extra_vars/01-assert.yaml b/tests/kuttl/tests/run_simple_playbook_extra_vars/01-assert.yaml
@@ -27,6 +27,7 @@ spec:
       why: "because"
 status:
   JobStatus: Succeeded
+  JobGeneration: 1
 ---
 apiVersion: v1
 kind: Pod