From e200be9487ae6aa71e965a08cc2e4f4b1adad8b0 Mon Sep 17 00:00:00 2001 From: Katarina Strenkova Date: Wed, 23 Oct 2024 11:56:14 -0400 Subject: [PATCH] Introduce new parameter Expected Failures List The Expected Failures List parameter will help with specifying set of tests whose failures we want to ignore. By adding test to this list, its failure will not have impact on the jobs pass/fail. It can help with not having to re-run long running jobs and save time and resources in specific cases, for example when dealing with flaky tests. --- .zuul.yaml | 1 + api/bases/test.openstack.org_tempests.yaml | 12 ++++++++++++ api/v1beta1/tempest_types.go | 7 +++++++ api/v1beta1/tempest_types_workflow.go | 7 +++++++ api/v1beta1/zz_generated.deepcopy.go | 5 +++++ config/crd/bases/test.openstack.org_tempests.yaml | 12 ++++++++++++ config/samples/test_v1beta1_tempest.yaml | 2 ++ controllers/tempest_controller.go | 7 +++++++ 8 files changed, 53 insertions(+) diff --git a/.zuul.yaml b/.zuul.yaml index 9cf3fe9a..e04581be 100644 --- a/.zuul.yaml +++ b/.zuul.yaml @@ -57,6 +57,7 @@ tempest.api.identity.admin.v3.test_credentials.CredentialsTestJSON tempest.api.identity.admin.v3.test_tokens.TokensV3TestJSON.test_rescope_token tempest.api.identity.admin.v3.test_users.UsersV3TestJSON.test_update_user_password + cifmw_test_operator_tempest_expected_failures_list: | cifmw_tempest_tempestconf_config: overrides: | compute-feature-enabled.dhcp_domain '' diff --git a/api/bases/test.openstack.org_tempests.yaml b/api/bases/test.openstack.org_tempests.yaml index 48c2e5e4..5ee54c99 100644 --- a/api/bases/test.openstack.org_tempests.yaml +++ b/api/bases/test.openstack.org_tempests.yaml @@ -168,6 +168,12 @@ spec: description: A content of exclude.txt file that is passed to tempest via --exclude-list type: string + expectedFailuresList: + description: The expectedFailuresList parameter contains tests + that should not count as failures. When a test from this list + fails, the test pod ends with Completed state rather than with + Error state. + type: string externalPlugin: description: ExternalPlugin contains information about plugin that should be installed within the tempest test pod. If this @@ -603,6 +609,12 @@ spec: description: A content of exclude.txt file that is passed to tempest via --exclude-list type: string + expectedFailuresList: + description: The expectedFailuresList parameter contains + tests that should not count as failures. When a test from + this list fails, the test pod ends with Completed state + rather than with Error state. + type: string externalPlugin: description: ExternalPlugin contains information about plugin that should be installed within the tempest test pod. diff --git a/api/v1beta1/tempest_types.go b/api/v1beta1/tempest_types.go index 896ccb08..017e1647 100644 --- a/api/v1beta1/tempest_types.go +++ b/api/v1beta1/tempest_types.go @@ -151,6 +151,13 @@ type TempestRunSpec struct { // A content of exclude.txt file that is passed to tempest via --exclude-list ExcludeList string `json:"excludeList"` + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // The expectedFailuresList parameter contains tests that should not count + // as failures. When a test from this list fails, the test pod ends with + // Completed state rather than with Error state. + ExpectedFailuresList string `json:"expectedFailuresList"` + // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec // +kubebuilder:default:=0 diff --git a/api/v1beta1/tempest_types_workflow.go b/api/v1beta1/tempest_types_workflow.go index a1b74ae4..c7adeb96 100644 --- a/api/v1beta1/tempest_types_workflow.go +++ b/api/v1beta1/tempest_types_workflow.go @@ -30,6 +30,13 @@ type WorkflowTempestRunSpec struct { // A content of exclude.txt file that is passed to tempest via --exclude-list ExcludeList *string `json:"excludeList,omitempty"` + // +kubebuilder:validation:Optional + // +operator-sdk:csv:customresourcedefinitions:type=spec + // The expectedFailuresList parameter contains tests that should not count + // as failures. When a test from this list fails, the test pod ends with + // Completed state rather than with Error state. + ExpectedFailuresList *string `json:"expectedFailuresList,omitempty"` + // +kubebuilder:validation:Optional // +operator-sdk:csv:customresourcedefinitions:type=spec // Concurrency value that is passed to tempest via --concurrency diff --git a/api/v1beta1/zz_generated.deepcopy.go b/api/v1beta1/zz_generated.deepcopy.go index 211c2d77..0f1882d7 100644 --- a/api/v1beta1/zz_generated.deepcopy.go +++ b/api/v1beta1/zz_generated.deepcopy.go @@ -715,6 +715,11 @@ func (in *WorkflowTempestRunSpec) DeepCopyInto(out *WorkflowTempestRunSpec) { *out = new(string) **out = **in } + if in.ExpectedFailuresList != nil { + in, out := &in.ExpectedFailuresList, &out.ExpectedFailuresList + *out = new(string) + **out = **in + } if in.Concurrency != nil { in, out := &in.Concurrency, &out.Concurrency *out = new(int64) diff --git a/config/crd/bases/test.openstack.org_tempests.yaml b/config/crd/bases/test.openstack.org_tempests.yaml index 48c2e5e4..5ee54c99 100644 --- a/config/crd/bases/test.openstack.org_tempests.yaml +++ b/config/crd/bases/test.openstack.org_tempests.yaml @@ -168,6 +168,12 @@ spec: description: A content of exclude.txt file that is passed to tempest via --exclude-list type: string + expectedFailuresList: + description: The expectedFailuresList parameter contains tests + that should not count as failures. When a test from this list + fails, the test pod ends with Completed state rather than with + Error state. + type: string externalPlugin: description: ExternalPlugin contains information about plugin that should be installed within the tempest test pod. If this @@ -603,6 +609,12 @@ spec: description: A content of exclude.txt file that is passed to tempest via --exclude-list type: string + expectedFailuresList: + description: The expectedFailuresList parameter contains + tests that should not count as failures. When a test from + this list fails, the test pod ends with Completed state + rather than with Error state. + type: string externalPlugin: description: ExternalPlugin contains information about plugin that should be installed within the tempest test pod. diff --git a/config/samples/test_v1beta1_tempest.yaml b/config/samples/test_v1beta1_tempest.yaml index 47cc9e8d..4e20f0e2 100644 --- a/config/samples/test_v1beta1_tempest.yaml +++ b/config/samples/test_v1beta1_tempest.yaml @@ -46,6 +46,8 @@ spec: concurrency: 8 # excludeList: | # <-- Use | to preserve \n # tempest.api.identity.v3.* + # expectedFailuresList: | # <-- Use | to preserve \n + # tempest.api.identity.v3.test_tokens.TokensV3Test.test_create_token # workerFile: | # <-- Use | to preserve \n # - worker: # - tempest.api.* diff --git a/controllers/tempest_controller.go b/controllers/tempest_controller.go index 99757300..6a57b292 100644 --- a/controllers/tempest_controller.go +++ b/controllers/tempest_controller.go @@ -444,6 +444,13 @@ func (r *TempestReconciler) setTempestConfigVars(envVars map[string]string, envVars["TEMPEST_EXCLUDE_LIST"] = testOperatorDir + excludeListFile } + value = mergeWithWorkflow(tRun.ExpectedFailuresList, wtRun.ExpectedFailuresList) + if len(value) != 0 { + expectedFailuresListFile := "expected_failures.txt" + customData[expectedFailuresListFile] = value + envVars["TEMPEST_EXPECTED_FAILURES_LIST"] = testOperatorDir + expectedFailuresListFile + } + // Bool tempestBoolEnvVars := map[string]bool{ "TEMPEST_SERIAL": mergeWithWorkflow(tRun.Serial, wtRun.Serial),