Terraform test: Consolidate test execution procedure

internal/command/test_test.go

@@ -10,6 +10,7 @@ import (
 	"os"
 	"path"
 	"regexp"
+	"runtime"
 	"strings"
 	"testing"
 
@@ -801,7 +802,7 @@ func TestTest_ProviderAlias(t *testing.T) {
 
 	output := done(t)
 
-	if code := init.Run(nil); code != 0 {
+	if code := init.Run([]string{"-no-color"}); code != 0 {
 		t.Fatalf("expected status code 0 but got %d: %s", code, output.All())
 	}
 
@@ -814,7 +815,7 @@ func TestTest_ProviderAlias(t *testing.T) {
 		Meta: meta,
 	}
 
-	code := command.Run(nil)
+	code := command.Run([]string{"-no-color"})
 	output = done(t)
 
 	printedOutput := false
@@ -1127,6 +1128,23 @@ it has been removed. This occurs when a provider configuration is removed
 while objects created by that provider still exist in the state. Re-add the
 provider configuration to destroy test_resource.secondary, after which you
 can remove the provider configuration again.
+`,
+		},
+		"missing-provider-definition-in-file": {
+			expectedOut: `main.tftest.hcl... in progress
+  run "passes_validation"... fail
+main.tftest.hcl... tearing down
+main.tftest.hcl... fail
+
+Failure! 0 passed, 1 failed.
+`,
+			expectedErr: `
+Error: Missing provider definition for test
+
+  on main.tftest.hcl line 12, in run "passes_validation":
+  12:     test = test
+
+This provider block references a provider definition that does not exist.
 `,
 		},
 		"missing-provider-in-test-module": {
@@ -2465,6 +2483,83 @@ Success! 2 passed, 0 failed.
 	}
 }
 
+func TestTest_InvalidConfig(t *testing.T) {
+	td := t.TempDir()
+	testCopyDir(t, testFixturePath(path.Join("test", "invalid_config")), td)
+	defer testChdir(t, td)()
+
+	provider := testing_command.NewProvider(nil)
+
+	providerSource, close := newMockProviderSource(t, map[string][]string{
+		"test": {"1.0.0"},
+	})
+	defer close()
+
+	streams, done := terminal.StreamsForTesting(t)
+	view := views.NewView(streams)
+	ui := new(cli.MockUi)
+
+	meta := Meta{
+		Ui:             ui,
+		View:           view,
+		Streams:        streams,
+		ProviderSource: providerSource,
+	}
+
+	init := &InitCommand{
+		Meta: meta,
+	}
+
+	output := done(t)
+
+	if code := init.Run(nil); code != 0 {
+		t.Fatalf("expected status code 0 but got %d: %s", code, output.All())
+	}
+
+	// Reset the streams for the next command.
+	streams, done = terminal.StreamsForTesting(t)
+	meta.Streams = streams
+	meta.View = views.NewView(streams)
+
+	c := &TestCommand{
+		Meta: meta,
+	}
+
+	code := c.Run([]string{"-no-color"})
+	output = done(t)
+
+	if code != 1 {
+		t.Errorf("expected status code ! but got %d", code)
+	}
+
+	expected := `main.tftest.hcl... in progress
+  run "test"... fail
+
+Error: Failed to load plugin schemas
+
+Error while loading schemas for plugin components: Failed to obtain provider
+schema: Could not load the schema for provider
+registry.terraform.io/hashicorp/test: failed to instantiate provider
+"registry.terraform.io/hashicorp/test" to obtain schema: fork/exec
+.terraform/providers/registry.terraform.io/hashicorp/test/1.0.0/%s/terraform-provider-test_1.0.0:
+permission denied..
+main.tftest.hcl... tearing down
+main.tftest.hcl... fail
+
+Failure! 0 passed, 1 failed.
+`
+	expected = fmt.Sprintf(expected, runtime.GOOS+"_"+runtime.GOARCH)
+	actual := output.All()
+
+	if diff := cmp.Diff(actual, expected); len(diff) > 0 {
+		t.Errorf("output didn't match expected:\nexpected:\n%s\nactual:\n%s\ndiff:\n%s", expected, actual, diff)
+	}
+
+	if provider.ResourceCount() > 0 {
+		t.Errorf("should have deleted all resources on completion but left %v", provider.ResourceString())
+	}
+}
+
 func TestTest_RunBlocksInProviders(t *testing.T) {
 	td := t.TempDir()
 	testCopyDir(t, testFixturePath(path.Join("test", "provider_runs")), td)

internal/command/testdata/test/invalid_config/main.tf

@@ -0,0 +1,11 @@
+terraform {
+  required_providers {
+    test = {
+      source = "hashicorp/test"
+    }
+  }
+}
+
+resource "test_resource" "foo" {
+  nein = "foo"
+}

internal/command/testdata/test/invalid_config/main.tftest.hcl

@@ -0,0 +1,2 @@
+
+run "test" {}

internal/command/testdata/test/missing-provider-definition-in-file/main.tf

@@ -0,0 +1,17 @@
+terraform {
+  required_providers {
+    test = {
+      source = "hashicorp/test"
+      configuration_aliases = [ test.secondary, test ]
+    }
+  }
+}
+
+resource "test_resource" "primary" {
+  value = "foo"
+}
+
+resource "test_resource" "secondary" {
+  provider = test.secondary
+  value = "bar"
+}

internal/command/testdata/test/missing-provider-definition-in-file/main.tftest.hcl

@@ -0,0 +1,24 @@
+
+# provider "test" {}
+
+# provider "test" {
+#   alias = "secondary"
+# }
+
+run "passes_validation" {
+
+// references a provider that is not defined in the test file
+  providers = {
+    test = test
+  }
+
+  assert {
+    condition = test_resource.primary.value == "foo"
+    error_message = "primary contains invalid value"
+  }
+
+  assert {
+    condition = test_resource.secondary.value == "bar"
+    error_message = "secondary contains invalid value"
+  }
+}

internal/command/testdata/test/parallel_divided/main.tftest.hcl

@@ -84,7 +84,7 @@ run "main_fifth" {
 }
 
 run "main_sixth" {
-  state_key = "uniq_5"
+  state_key = "uniq_6"
   variables {
     input = "foo"
   }

internal/configs/test_file.go

@@ -138,6 +138,9 @@ type TestRun struct {
 	// configuration load process and should be used when the test is executed.
 	ConfigUnderTest *Config
 
+	// File is a reference to the parent TestFile that contains this run block.
+	File *TestFile
+
 	// ExpectFailures should be a list of checkable objects that are expected
 	// to report a failure from their custom conditions as part of this test
 	// run.
@@ -271,6 +274,23 @@ func (run *TestRun) Validate(config *Config) tfdiags.Diagnostics {
 		}
 	}
 
+	// All the providers defined within a run block should target an existing
+	// provider block within the test file.
+	for _, ref := range run.Providers {
+		_, ok := run.File.Providers[ref.InParent.String()]
+		if !ok {
+			// Then this reference was invalid as we didn't have the
+			// specified provider in the parent. This should have been
+			// caught earlier in validation anyway so is unlikely to happen.
+			diags = diags.Append(&hcl.Diagnostic{
+				Severity: hcl.DiagError,
+				Summary:  fmt.Sprintf("Missing provider definition for %s", ref.InParent.String()),
+				Detail:   "This provider block references a provider definition that does not exist.",
+				Subject:  ref.InParent.NameRange.Ptr(),
+			})
+		}
+	}
+
 	return diags
 }
 
@@ -306,7 +326,7 @@ type TestRunOptions struct {
 
 func loadTestFile(body hcl.Body) (*TestFile, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
-	tf := TestFile{
+	tf := &TestFile{
 		Providers: make(map[string]*Provider),
 		Overrides: addrs.MakeMap[addrs.Targetable, *Override](),
 	}
@@ -333,7 +353,7 @@ func loadTestFile(body hcl.Body) (*TestFile, hcl.Diagnostics) {
 	for _, block := range content.Blocks {
 		switch block.Type {
 		case "run":
-			run, runDiags := decodeTestRunBlock(block, tf.Config)
+			run, runDiags := decodeTestRunBlock(block, tf)
 			diags = append(diags, runDiags...)
 			if !runDiags.HasErrors() {
 				tf.Runs = append(tf.Runs, run)
@@ -455,7 +475,7 @@ func loadTestFile(body hcl.Body) (*TestFile, hcl.Diagnostics) {
 		}
 	}
 
-	return &tf, diags
+	return tf, diags
 }
 
 func decodeFileConfigBlock(fileContent *hcl.BodyContent) (*TestFileConfig, hcl.Diagnostics) {
@@ -495,19 +515,19 @@ func decodeFileConfigBlock(fileContent *hcl.BodyContent) (*TestFileConfig, hcl.D
 	return ret, diags
 }
 
-func decodeTestRunBlock(block *hcl.Block, fileConfig *TestFileConfig) (*TestRun, hcl.Diagnostics) {
+func decodeTestRunBlock(block *hcl.Block, file *TestFile) (*TestRun, hcl.Diagnostics) {
 	var diags hcl.Diagnostics
 
 	content, contentDiags := block.Body.Content(testRunBlockSchema)
 	diags = append(diags, contentDiags...)
 
 	r := TestRun{
-		Overrides: addrs.MakeMap[addrs.Targetable, *Override](),
-
+		Overrides:     addrs.MakeMap[addrs.Targetable, *Override](),
+		File:          file,
 		Name:          block.Labels[0],
 		NameDeclRange: block.LabelRanges[0],
 		DeclRange:     block.DefRange,
-		Parallel:      fileConfig != nil && fileConfig.Parallel,
+		Parallel:      file.Config != nil && file.Config.Parallel,
 	}
 
 	if !hclsyntax.ValidIdentifier(r.Name) {