Introduce new parameter Expected Failures List

The Expected Failures List parameter will help with specifying set
of tests whose failures we want to ignore. By adding test to this
list, its failure will not have impact on the jobs pass/fail. It
can help with not having to re-run long running jobs and save
time and resources in specific cases, for example when dealing
with flaky tests.

Author: kstrenkova

api/bases/test.openstack.org_tempests.yaml

@@ -168,6 +168,11 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: The expected_failures.txt file contains tests that
+                      should not count as failures for the job, as they are expected
+                      to fail.
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +608,11 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: The expected_failures.txt file contains tests
+                            that should not count as failures for the job, as they
+                            are expected to fail.
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

api/v1beta1/tempest_types.go

@@ -151,6 +151,12 @@ type TempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList string `json:"excludeList"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The expected_failures.txt file contains tests that should not count
+	// as failures for the job, as they are expected to fail.
+	ExpectedFailuresList string `json:"expectedFailuresList"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// +kubebuilder:default:=0

api/v1beta1/tempest_types_workflow.go

@@ -30,6 +30,12 @@ type WorkflowTempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList *string `json:"excludeList,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// The expected_failures.txt file contains tests that should not count
+	// as failures for the job, as they are expected to fail.
+	ExpectedFailuresList *string `json:"expectedFailuresList,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// Concurrency value that is passed to tempest via --concurrency

api/v1beta1/zz_generated.deepcopy.go

@@ -168,6 +168,11 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: The expected_failures.txt file contains tests that
+                      should not count as failures for the job, as they are expected
+                      to fail.
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +608,11 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: The expected_failures.txt file contains tests
+                            that should not count as failures for the job, as they
+                            are expected to fail.
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

config/crd/bases/test.openstack.org_tempests.yaml

@@ -46,6 +46,8 @@ spec:
     concurrency: 8
     # excludeList: | # <-- Use | to preserve \n
     #   tempest.api.identity.v3.*
+    # expectedFailuresList: | # <-- Use | to preserve \n
+    #   tempest.api.identity.v3.*
     # workerFile: | # <-- Use | to preserve \n
     #   - worker:
     #     - tempest.api.*

config/samples/test_v1beta1_tempest.yaml

@@ -444,6 +444,13 @@ func (r *TempestReconciler) setTempestConfigVars(envVars map[string]string,
 		envVars["TEMPEST_EXCLUDE_LIST"] = testOperatorDir + excludeListFile
 	}
 
+	value = mergeWithWorkflow(tRun.ExpectedFailuresList, wtRun.ExpectedFailuresList)
+	if len(value) != 0 {
+		expectedFailuresListFile := "expected_failures.txt"
+		customData[expectedFailuresListFile] = value
+		envVats["TEMPEST_EXPECTED_FAILURES_LIST"] = testOperatorDir + expectedFailuresListFile
+	}
+
 	// Bool
 	tempestBoolEnvVars := map[string]bool{
 		"TEMPEST_SERIAL":     mergeWithWorkflow(tRun.Serial, wtRun.Serial),