Introduce new parameter Expected Failures List

The Expected Failures List parameter will help with specifying set
of tests whose failures we want to ignore. By adding test to this
list, its failure will not have impact on the jobs pass/fail. It
can help with not having to re-run long running jobs and save
time and resources in specific cases, for example when dealing
with flaky tests.

Author: kstrenkova

api/bases/test.openstack.org_tempests.yaml

@@ -168,6 +168,10 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: A content of expected_failures.txt file that is passed
+                      to tempest via --expected-failures-list
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +607,10 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: A content of expected_failures.txt file that
+                            is passed to tempest via --expected-failures-list
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

api/v1beta1/tempest_types.go

@@ -151,6 +151,12 @@ type TempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList string `json:"excludeList"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// A content of expected_failures.txt file that is passed to tempest via
+	// --expected-failures-list
+	ExpectedFailuresList string `json:"expectedFailuresList"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// +kubebuilder:default:=0

api/v1beta1/tempest_types_workflow.go

@@ -30,6 +30,12 @@ type WorkflowTempestRunSpec struct {
 	// A content of exclude.txt file that is passed to tempest via --exclude-list
 	ExcludeList *string `json:"excludeList,omitempty"`
 
+	// +kubebuilder:validation:Optional
+	// +operator-sdk:csv:customresourcedefinitions:type=spec
+	// A content of expected_failures.txt file that is passed to tempest via
+	// --expected-failures-list
+	ExpectedFailuresList *string `json:"expectedFailuresList,omitempty"`
+
 	// +kubebuilder:validation:Optional
 	// +operator-sdk:csv:customresourcedefinitions:type=spec
 	// Concurrency value that is passed to tempest via --concurrency

api/v1beta1/zz_generated.deepcopy.go

@@ -168,6 +168,10 @@ spec:
                     description: A content of exclude.txt file that is passed to tempest
                       via --exclude-list
                     type: string
+                  expectedFailuresList:
+                    description: A content of expected_failures.txt file that is passed
+                      to tempest via --expected-failures-list
+                    type: string
                   externalPlugin:
                     description: ExternalPlugin contains information about plugin
                       that should be installed within the tempest test pod. If this
@@ -603,6 +607,10 @@ spec:
                           description: A content of exclude.txt file that is passed
                             to tempest via --exclude-list
                           type: string
+                        expectedFailuresList:
+                          description: A content of expected_failures.txt file that
+                            is passed to tempest via --expected-failures-list
+                          type: string
                         externalPlugin:
                           description: ExternalPlugin contains information about plugin
                             that should be installed within the tempest test pod.

config/crd/bases/test.openstack.org_tempests.yaml

@@ -46,6 +46,8 @@ spec:
     concurrency: 8
     # excludeList: | # <-- Use | to preserve \n
     #   tempest.api.identity.v3.*
+    # expectedFailuresList: | # <-- Use | to preserve \n
+    #   tempest.api.identity.v3.*
     # workerFile: | # <-- Use | to preserve \n
     #   - worker:
     #     - tempest.api.*

config/samples/test_v1beta1_tempest.yaml

@@ -444,6 +444,13 @@ func (r *TempestReconciler) setTempestConfigVars(envVars map[string]string,
 		envVars["TEMPEST_EXCLUDE_LIST"] = testOperatorDir + excludeListFile
 	}
 
+	value = mergeWithWorkflow(tRun.ExpectedFailuresList, wtRun.ExpectedFailuresList)
+	if len(value) != 0 {
+		expectedFailuresListFile := "expected_failures.txt"
+		customData[expectedFailuresListFile] = value
+		envVats["TEMPEST_EXPECTED_FAILURES_LIST"] = testOperatorDir + expectedFailuresListFile
+	}
+
 	// Bool
 	tempestBoolEnvVars := map[string]bool{
 		"TEMPEST_SERIAL":     mergeWithWorkflow(tRun.Serial, wtRun.Serial),