continue to execute runs when exp failures did not fail during apply

Author: dsa0x

.changes/unreleased/ENHANCEMENTS-20250206-162053.yaml

@@ -1,5 +1,5 @@
 kind: ENHANCEMENTS
-body: Continue Test execution when a run fails due to an expected failure.
+body: Terraform Test: Continue subsequent test execution when an expected failure is not encountered.
 time: 2025-02-06T16:20:53.83763+01:00
 custom:
     Issue: "34969"

internal/backend/local/test.go

@@ -300,7 +300,6 @@ func (runner *TestFileRunner) Test(file *moduletest.File) {
 // walkGraph goes through the graph and execute each run it finds.
 func (runner *TestFileRunner) walkGraph(g *terraform.Graph, file *moduletest.File) tfdiags.Diagnostics {
 	sem := runner.Suite.semaphore
-	collectRunStatus, updateFileStatus := runner.trackRunStatuses(file)
 
 	// Walk the graph.
 	walkFn := func(v dag.Vertex) (diags tfdiags.Diagnostics) {
@@ -377,13 +376,9 @@ func (runner *TestFileRunner) walkGraph(g *terraform.Graph, file *moduletest.Fil
 			}
 
 			startTime := time.Now().UTC()
-			deferFileStatus := runner.run(run, file, startTime)
+			runner.run(run, file, startTime)
 			runner.Suite.View.Run(run, file, moduletest.Complete, 0)
-			// If the run block is done, but it was due to an expected failure, we
-			// don't want to update the file status immediately. We'll collect the
-			// status of this run block and update the file status at the end of the
-			// file execution.
-			collectRunStatus(run, deferFileStatus)
+			file.UpdateStatus(run.Status)
 		case graph.GraphNodeExecutable:
 			diags = v.Execute(runner.EvalContext)
 			return diags
@@ -394,12 +389,10 @@ func (runner *TestFileRunner) walkGraph(g *terraform.Graph, file *moduletest.Fil
 		return
 	}
 
-	diags := g.AcyclicGraph.Walk(walkFn)
-	updateFileStatus()
-	return diags
+	return g.AcyclicGraph.Walk(walkFn)
 }
 
-func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, startTime time.Time) (deferFileStatus bool) {
+func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, startTime time.Time) {
 	log.Printf("[TRACE] TestFileRunner: executing run block %s/%s", file.Name, run.Name)
 	defer func() {
 		// If we got far enough to actually execute the run then we'll give
@@ -491,9 +484,9 @@ func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, st
 
 	planScope, plan, planDiags := runner.plan(tfCtx, config, state, run, file, setVariables, references, start)
 	if run.Config.Command == configs.PlanTestCommand {
+		// Then we want to assess our conditions and diagnostics differently.
 		planDiags = run.ValidateExpectedFailures(planDiags)
 		run.Diagnostics = run.Diagnostics.Append(planDiags)
-		// Then we want to assess our conditions and diagnostics differently.
 		if planDiags.HasErrors() {
 			run.Status = moduletest.Error
 			return
@@ -540,21 +533,12 @@ func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, st
 		return
 	}
 
-	// Otherwise any error (expected or unexpected) during the planning prevents our apply from
+	// Otherwise any error during the planning prevents our apply from
 	// continuing which is an error.
 	planDiags = run.ExplainExpectedFailures(planDiags)
 	run.Diagnostics = run.Diagnostics.Append(planDiags)
 	if planDiags.HasErrors() {
 		run.Status = moduletest.Error
-		// If the plan failed, but all the failures were expected, then we don't
-		// want to mark the overall file as a failure, so that subsequent runs can
-		// still be executed.
-		// We will collect the status of this run instead of updating the file status.
-		// At the end of the file execution, we will update the file status based on the
-		// statuses of all the runs.
-		if !run.ValidateExpectedFailures(planDiags).HasErrors() {
-			deferFileStatus = true
-		}
 		return
 	}
 
@@ -575,11 +559,9 @@ func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, st
 	applyScope, updated, applyDiags := runner.apply(tfCtx, plan, state, run, file, moduletest.Running, start, variables)
 
 	// Remove expected diagnostics, and add diagnostics in case anything that should have failed didn't.
-	applyDiags = run.ValidateExpectedFailures(applyDiags)
-
-	run.Diagnostics = run.Diagnostics.Append(applyDiags)
-	if applyDiags.HasErrors() {
-		run.Status = moduletest.Error
+	// We'll also update the run status based on the presence of errors or missing expected failures.
+	failOrErr := runner.checkForMissingExpectedFailures(run, applyDiags)
+	if failOrErr {
 		// Even though the apply operation failed, the graph may have done
 		// partial updates and the returned state should reflect this.
 		runner.EvalContext.SetFileState(key, &graph.TestFileState{
@@ -639,6 +621,38 @@ func (runner *TestFileRunner) run(run *moduletest.Run, file *moduletest.File, st
 	return
 }
 
+// checkForMissingExpectedFailures checks for missing expected failures in the diagnostics.
+// It updates the run status based on the presence of errors or missing expected failures.
+func (runner *TestFileRunner) checkForMissingExpectedFailures(run *moduletest.Run, diags tfdiags.Diagnostics) (failOrErr bool) {
+	// If there are errors in the diagnostics, update the run status to error.
+	if diags.HasErrors() {
+		run.Status = moduletest.Error
+	}
+
+	// Retrieve diagnostics that are either unrelated to expected failures or report missing expected failures.
+	unexpectedDiags := run.ValidateExpectedFailures(diags)
+	var nonFailureDiags tfdiags.Diagnostics
+	for _, diag := range unexpectedDiags {
+		switch {
+		// If any diagnostic indicates a missing expected failure, update the run status to fail.
+		case diag.Description().Summary == moduletest.MissingFailureSummary:
+			diag.ExtraInfo()
+			run.Status = moduletest.Fail
+		default:
+			// Append other diagnostics.
+			nonFailureDiags = nonFailureDiags.Append(diag)
+		}
+	}
+	// If there are other errors, update the run status to error.
+	if nonFailureDiags.HasErrors() {
+		run.Status = moduletest.Error
+	}
+
+	// Append all diagnostics that are not expected failures to the run diagnostics.
+	run.Diagnostics = run.Diagnostics.Append(unexpectedDiags)
+	return run.Status > moduletest.Pass
+}
+
 func (runner *TestFileRunner) validate(run *moduletest.Run, file *moduletest.File, start int64) tfdiags.Diagnostics {
 	log.Printf("[TRACE] TestFileRunner: called validate for %s/%s", file.Name, run.Name)
 

internal/command/test_test.go

@@ -104,12 +104,6 @@ func TestTest_Runs(t *testing.T) {
 			expectedOut: []string{"1 passed, 0 failed."},
 			code:        0,
 		},
-		"expect_failures_continue": {
-			expectedOut: []string{"1 passed, 1 failed.", "Expected failure while planning"},
-			code:        1,
-			expectedErr: []string{"Module output value precondition failed"},
-			description: "continue test execution after an expected failure",
-		},
 		"expect_failures_resources": {
 			expectedOut: []string{"1 passed, 0 failed."},
 			code:        0,
@@ -399,14 +393,15 @@ func TestTest_Runs(t *testing.T) {
 				Meta: meta,
 			}
 
-			code := c.Run(tc.args)
+			code := c.Run(append(tc.args, "-no-color"))
 			output := done(t)
 
 			if code != tc.code {
 				t.Errorf("expected status code %d but got %d:\n\n%s", tc.code, code, output.All())
 			}
 
 			if len(tc.expectedOut) > 0 {
+				fmt.Println(output.All())
 				for _, expectedOut := range tc.expectedOut {
 					if !strings.Contains(output.Stdout(), expectedOut) {
 						t.Errorf("output didn't contain expected string:\n\n%s", output.Stdout())
@@ -1837,6 +1832,7 @@ the apply operation could not be executed and so the overall test case will
 be marked as a failure and the original diagnostic included in the test
 report.
 
+  run "no_run"... skip
 input.tftest.hcl... tearing down
 input.tftest.hcl... fail
 output.tftest.hcl... in progress
@@ -1869,7 +1865,7 @@ test report.
 resource.tftest.hcl... tearing down
 resource.tftest.hcl... fail
 
-Failure! 1 passed, 3 failed.
+Failure! 1 passed, 3 failed, 1 skipped.
 `
 	actualOut := output.Stdout()
 	if diff := cmp.Diff(expectedOut, actualOut); len(diff) > 0 {
@@ -1916,6 +1912,71 @@ input must contain the character 'b'
 	}
 }
 
+func TestTest_MissingExpectedFailuresDuringApply(t *testing.T) {
+	// Test asserting that the test run fails, but not errors out, when expected failures are not present during apply.
+	// This lets subsequent runs continue to execute and the file to be marked as failed.
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "expect_failures_during_apply")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+	view, done := testView(t)
+
+	c := &TestCommand{
+		Meta: Meta{
+			testingOverrides: metaOverridesForProvider(provider.Provider),
+			View:             view,
+		},
+	}
+
+	code := c.Run([]string{"-no-color"})
+	output := done(t)
+
+	if code == 0 {
+		t.Errorf("expected status code 0 but got %d", code)
+	}
+
+	expectedOut := `main.tftest.hcl... in progress
+  run "test"... fail
+  run "follow-up"... pass
+
+Warning: Value for undeclared variable
+
+  on main.tftest.hcl line 16, in run "follow-up":
+  16:     input = "does not matter"
+
+The module under test does not declare a variable named "input", but it is
+declared in run block "follow-up".
+
+main.tftest.hcl... tearing down
+main.tftest.hcl... fail
+
+Failure! 1 passed, 1 failed.
+`
+	actualOut := output.Stdout()
+	if diff := cmp.Diff(expectedOut, actualOut); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expectedOut, actualOut, diff)
+	}
+
+	expectedErr := `
+Error: Missing expected failure
+
+  on main.tftest.hcl line 7, in run "test":
+   7:     output.output
+
+The checkable object, output.output, was expected to report an error but did
+not.
+`
+	actualErr := output.Stderr()
+	if diff := cmp.Diff(actualErr, expectedErr); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expectedErr, actualErr, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}
+
 func TestTest_UnknownAndNulls(t *testing.T) {
 
 	tcs := map[string]struct {

internal/command/testdata/test/expect_failures_continue/main.tf

@@ -0,0 +1,13 @@
+
+locals {
+  input = uuid() # using UUID to ensure that plan phase will return an unknown value
+}
+
+output "output" {
+  value = local.input
+
+  precondition {
+    condition = local.input != ""
+    error_message = "this should not fail during the apply phase"
+  }
+}

internal/command/testdata/test/expect_failures_continue/main.tftest.hcl

@@ -0,0 +1,18 @@
+run "test" {
+
+  command = apply
+
+// We are expecting the output to fail during apply, but it will not, so the test will fail.
+  expect_failures = [
+    output.output
+  ]
+}
+
+// this should still run
+run "follow-up" {
+  command = apply
+
+  variables {
+    input = "does not matter"
+  }
+}

internal/command/testdata/test/expect_failures_during_apply/main.tf

@@ -14,3 +14,17 @@ run "input_failure" {
   ]
 
 }
+
+
+// This should not run because the previous run block is expected to error, thus
+// terminating the test file.
+run "no_run" {
+
+  variables {
+    input = "abc"
+  }
+  assert {
+    condition = var.input == "abc"
+    error_message = "should not run"
+  }
+}

internal/command/testdata/test/expect_failures_during_apply/main.tftest.hcl

@@ -20,7 +20,8 @@ import (
 )
 
 const (
-	MainStateIdentifier = ""
+	MainStateIdentifier   = ""
+	MissingFailureSummary = "Missing expected failure"
 )
 
 type Run struct {
@@ -545,7 +546,7 @@ func (run *Run) ValidateExpectedFailures(originals tfdiags.Diagnostics) tfdiags.
 			// diagnostics.
 			diags = diags.Append(&hcl.Diagnostic{
 				Severity: hcl.DiagError,
-				Summary:  "Missing expected failure",
+				Summary:  MissingFailureSummary,
 				Detail:   fmt.Sprintf("The checkable object, %s, was expected to report an error but did not.", addr.String()),
 				Subject:  sourceRanges.Get(addr).ToHCL().Ptr(),
 			})