feat: Support project-level Terraform distribution selection (#5167)

Signed-off-by: Andrew Borg <[email protected]>
Co-authored-by: PePe Amengual <[email protected]>
Co-authored-by: Simon Heather <[email protected]>

Author: 3 people

Diff for: cmd/server.go

@@ -72,6 +72,7 @@ const (
 	CheckoutStrategyFlag             = "checkout-strategy"
 	ConfigFlag                       = "config"
 	DataDirFlag                      = "data-dir"
+	DefaultTFDistributionFlag        = "default-tf-distribution"
 	DefaultTFVersionFlag             = "default-tf-version"
 	DisableApplyAllFlag              = "disable-apply-all"
 	DisableAutoplanFlag              = "disable-autoplan"
@@ -141,7 +142,7 @@ const (
 	SSLCertFileFlag                  = "ssl-cert-file"
 	SSLKeyFileFlag                   = "ssl-key-file"
 	RestrictFileList                 = "restrict-file-list"
-	TFDistributionFlag               = "tf-distribution"
+	TFDistributionFlag               = "tf-distribution" // deprecated for DefaultTFDistributionFlag
 	TFDownloadFlag                   = "tf-download"
 	TFDownloadURLFlag                = "tf-download-url"
 	UseTFPluginCache                 = "use-tf-plugin-cache"
@@ -421,8 +422,8 @@ var stringFlags = map[string]stringFlag{
 		description: fmt.Sprintf("File containing x509 private key matching --%s.", SSLCertFileFlag),
 	},
 	TFDistributionFlag: {
-		description:  fmt.Sprintf("Which TF distribution to use. Can be set to %s or %s.", TFDistributionTerraform, TFDistributionOpenTofu),
-		defaultValue: DefaultTFDistribution,
+		description: "[Deprecated for --default-tf-distribution].",
+		hidden:      true,
 	},
 	TFDownloadURLFlag: {
 		description:  "Base URL to download Terraform versions from.",
@@ -437,6 +438,10 @@ var stringFlags = map[string]stringFlag{
 			" Only set if using TFC/E as a remote backend." +
 			" Should be specified via the ATLANTIS_TFE_TOKEN environment variable for security.",
 	},
+	DefaultTFDistributionFlag: {
+		description:  fmt.Sprintf("Which TF distribution to use. Can be set to %s or %s.", TFDistributionTerraform, TFDistributionOpenTofu),
+		defaultValue: DefaultTFDistribution,
+	},
 	DefaultTFVersionFlag: {
 		description: "Terraform version to default to (ex. v0.12.0). Will download if not yet on disk." +
 			" If not set, Atlantis uses the terraform binary in its PATH.",
@@ -840,12 +845,13 @@ func (s *ServerCmd) run() error {
 
 	// Config looks good. Start the server.
 	server, err := s.ServerCreator.NewServer(userConfig, server.Config{
-		AllowForkPRsFlag:        AllowForkPRsFlag,
-		AtlantisURLFlag:         AtlantisURLFlag,
-		AtlantisVersion:         s.AtlantisVersion,
-		DefaultTFVersionFlag:    DefaultTFVersionFlag,
-		RepoConfigJSONFlag:      RepoConfigJSONFlag,
-		SilenceForkPRErrorsFlag: SilenceForkPRErrorsFlag,
+		AllowForkPRsFlag:          AllowForkPRsFlag,
+		AtlantisURLFlag:           AtlantisURLFlag,
+		AtlantisVersion:           s.AtlantisVersion,
+		DefaultTFDistributionFlag: DefaultTFDistributionFlag,
+		DefaultTFVersionFlag:      DefaultTFVersionFlag,
+		RepoConfigJSONFlag:        RepoConfigJSONFlag,
+		SilenceForkPRErrorsFlag:   SilenceForkPRErrorsFlag,
 	})
 
 	if err != nil {
@@ -921,8 +927,11 @@ func (s *ServerCmd) setDefaults(c *server.UserConfig, v *viper.Viper) {
 	if c.RedisPort == 0 {
 		c.RedisPort = DefaultRedisPort
 	}
-	if c.TFDistribution == "" {
-		c.TFDistribution = DefaultTFDistribution
+	if c.TFDistribution != "" && c.DefaultTFDistribution == "" {
+		c.DefaultTFDistribution = c.TFDistribution
+	}
+	if c.DefaultTFDistribution == "" {
+		c.DefaultTFDistribution = DefaultTFDistribution
 	}
 	if c.TFDownloadURL == "" {
 		c.TFDownloadURL = DefaultTFDownloadURL
@@ -953,7 +962,7 @@ func (s *ServerCmd) validate(userConfig server.UserConfig) error {
 		return fmt.Errorf("invalid log level: must be one of %v", ValidLogLevels)
 	}
 
-	if userConfig.TFDistribution != TFDistributionTerraform && userConfig.TFDistribution != TFDistributionOpenTofu {
+	if userConfig.DefaultTFDistribution != TFDistributionTerraform && userConfig.DefaultTFDistribution != TFDistributionOpenTofu {
 		return fmt.Errorf("invalid tf distribution: expected one of %s or %s",
 			TFDistributionTerraform, TFDistributionOpenTofu)
 	}
@@ -1172,6 +1181,10 @@ func (s *ServerCmd) deprecationWarnings(userConfig *server.UserConfig) error {
 	//       }
 	//
 
+	if userConfig.TFDistribution != "" {
+		deprecatedFlags = append(deprecatedFlags, TFDistributionFlag)
+	}
+
 	if len(deprecatedFlags) > 0 {
 		warning := "WARNING: "
 		if len(deprecatedFlags) == 1 {

Diff for: cmd/server_test.go

@@ -73,6 +73,7 @@ var testFlags = map[string]interface{}{
 	CheckoutStrategyFlag:             CheckoutStrategyMerge,
 	CheckoutDepthFlag:                0,
 	DataDirFlag:                      "/path",
+	DefaultTFDistributionFlag:        "terraform",
 	DefaultTFVersionFlag:             "v0.11.0",
 	DisableApplyAllFlag:              true,
 	DisableMarkdownFoldingFlag:       true,
@@ -977,6 +978,46 @@ func TestExecute_AutoplanFileList(t *testing.T) {
 	}
 }
 
+func TestExecute_ValidateDefaultTFDistribution(t *testing.T) {
+	cases := []struct {
+		description string
+		flags       map[string]interface{}
+		expectErr   string
+	}{
+		{
+			"terraform",
+			map[string]interface{}{
+				DefaultTFDistributionFlag: "terraform",
+			},
+			"",
+		},
+		{
+			"opentofu",
+			map[string]interface{}{
+				DefaultTFDistributionFlag: "opentofu",
+			},
+			"",
+		},
+		{
+			"errs on invalid distribution",
+			map[string]interface{}{
+				DefaultTFDistributionFlag: "invalid_distribution",
+			},
+			"invalid tf distribution: expected one of terraform or opentofu",
+		},
+	}
+	for _, testCase := range cases {
+		t.Log("Should validate default tf distribution when " + testCase.description)
+		c := setupWithDefaults(testCase.flags, t)
+		err := c.Execute()
+		if testCase.expectErr != "" {
+			ErrEquals(t, testCase.expectErr, err)
+		} else {
+			Ok(t, err)
+		}
+	}
+}
+
 func setup(flags map[string]interface{}, t *testing.T) *cobra.Command {
 	vipr := viper.New()
 	for k, v := range flags {

Diff for: runatlantis.io/docs/repo-level-atlantis-yaml.md

@@ -66,6 +66,7 @@ projects:
   branch: /main/
   dir: .
   workspace: default
+  terraform_distribution: terraform
   terraform_version: v0.11.0
   delete_source_branch_on_merge: true
   repo_locking: true # deprecated: use repo_locks instead
@@ -262,6 +263,20 @@ See [Custom Workflow Use Cases: Terragrunt](custom-workflows.md#terragrunt)
 
 See [Custom Workflow Use Cases: Running custom commands](custom-workflows.md#running-custom-commands)
 
+### Terraform Distributions
+
+If you'd like to use a different distribution of Terraform than what is set
+by the `--default-tf-version` flag, then set the `terraform_distribution` key:
+
+```yaml
+version: 3
+projects:
+- dir: project1
+  terraform_distribution: opentofu
+```
+
+Atlantis will automatically download and use this distribution. Valid values are `terraform` and `opentofu`.
+
 ### Terraform Versions
 
 If you'd like to use a different version of Terraform than what is in Atlantis'

Diff for: runatlantis.io/docs/server-configuration.md

@@ -386,6 +386,16 @@ and set `--autoplan-modules` to `false`.
   Note that the atlantis user is restricted to `~/.atlantis`.
   If you set the `--data-dir` flag to a path outside of Atlantis its home directory, ensure that you grant the atlantis user the correct permissions.
 
+### `--default-tf-distribution`
+
+  ```bash
+  atlantis server --default-tf-distribution="terraform"
+  # or
+  ATLANTIS_DEFAULT_TF_DISTRIBUTION="terraform"
+  ```
+
+  Which TF distribution to use. Can be set to `terraform` or `opentofu`.
+
 ### `--default-tf-version`
 
   ```bash
@@ -1259,13 +1269,8 @@ This is useful when you have many projects and want to keep the pull request cle
 
 ### `--tf-distribution`
 
-  ```bash
-  atlantis server --tf-distribution="terraform"
-  # or
-  ATLANTIS_TF_DISTRIBUTION="terraform"
-  ```
-
-  Which TF distribution to use. Can be set to `terraform` or `opentofu`.
+  <Badge text="Deprecated" type="warn"/>
+  Deprecated for `--default-tf-distribution`.
 
 ### `--tf-download`
 

Diff for: server/controllers/events/events_controller_e2e_test.go

@@ -29,6 +29,7 @@ import (
 	mock_policy "github.com/runatlantis/atlantis/server/core/runtime/policy/mocks"
 	"github.com/runatlantis/atlantis/server/core/terraform"
 	terraform_mocks "github.com/runatlantis/atlantis/server/core/terraform/mocks"
+	"github.com/runatlantis/atlantis/server/core/terraform/tfclient"
 	"github.com/runatlantis/atlantis/server/events"
 	"github.com/runatlantis/atlantis/server/events/command"
 	"github.com/runatlantis/atlantis/server/events/mocks"
@@ -1319,7 +1320,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 	mockDownloader := terraform_mocks.NewMockDownloader()
 	distribution := terraform.NewDistributionTerraformWithDownloader(mockDownloader)
 
-	terraformClient, err := terraform.NewClient(logger, distribution, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", true, false, projectCmdOutputHandler)
+	terraformClient, err := tfclient.NewClient(logger, distribution, binDir, cacheDir, "", "", "", "default-tf-version", "https://releases.hashicorp.com", true, false, projectCmdOutputHandler)
 	Ok(t, err)
 	boltdb, err := db.New(dataDir)
 	Ok(t, err)
@@ -1346,6 +1347,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 		}
 	}
 
+	defaultTFDistribution := terraformClient.DefaultDistribution()
 	defaultTFVersion := terraformClient.DefaultVersion()
 	locker := events.NewDefaultWorkingDirLocker()
 	parser := &config.ParserValidator{}
@@ -1429,7 +1431,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 		terraformClient,
 	)
 
-	showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFVersion)
+	showStepRunner, err := runtime.NewShowStepRunner(terraformClient, defaultTFDistribution, defaultTFVersion)
 
 	Ok(t, err)
 
@@ -1440,6 +1442,7 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 	conftextExec.VersionCache = &LocalConftestCache{}
 
 	policyCheckRunner, err := runtime.NewPolicyCheckStepRunner(
+		defaultTFDistribution,
 		defaultTFVersion,
 		conftextExec,
 	)
@@ -1451,11 +1454,13 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 		Locker:           projectLocker,
 		LockURLGenerator: &mockLockURLGenerator{},
 		InitStepRunner: &runtime.InitStepRunner{
-			TerraformExecutor: terraformClient,
-			DefaultTFVersion:  defaultTFVersion,
+			TerraformExecutor:     terraformClient,
+			DefaultTFDistribution: defaultTFDistribution,
+			DefaultTFVersion:      defaultTFVersion,
 		},
 		PlanStepRunner: runtime.NewPlanStepRunner(
 			terraformClient,
+			defaultTFDistribution,
 			defaultTFVersion,
 			statusUpdater,
 			asyncTfExec,
@@ -1465,10 +1470,11 @@ func setupE2E(t *testing.T, repoDir string, opt setupOption) (events_controllers
 		ApplyStepRunner: &runtime.ApplyStepRunner{
 			TerraformExecutor: terraformClient,
 		},
-		ImportStepRunner:  runtime.NewImportStepRunner(terraformClient, defaultTFVersion),
-		StateRmStepRunner: runtime.NewStateRmStepRunner(terraformClient, defaultTFVersion),
+		ImportStepRunner:  runtime.NewImportStepRunner(terraformClient, defaultTFDistribution, defaultTFVersion),
+		StateRmStepRunner: runtime.NewStateRmStepRunner(terraformClient, defaultTFDistribution, defaultTFVersion),
 		RunStepRunner: &runtime.RunStepRunner{
 			TerraformExecutor:       terraformClient,
+			DefaultTFDistribution:   defaultTFDistribution,
 			DefaultTFVersion:        defaultTFVersion,
 			ProjectCmdOutputHandler: projectCmdOutputHandler,
 		},

Diff for: server/core/config/parser_validator_test.go

@@ -610,6 +610,31 @@ workflows:
 				},
 			},
 		},
+		{
+			description: "project field with terraform_distribution set to opentofu",
+			input: `
+version: 3
+projects:
+- dir: .
+  workspace: myworkspace
+  terraform_distribution: opentofu
+`,
+			exp: valid.RepoCfg{
+				Version: 3,
+				Projects: []valid.Project{
+					{
+						Dir:                   ".",
+						Workspace:             "myworkspace",
+						TerraformDistribution: String("opentofu"),
+						Autoplan: valid.Autoplan{
+							WhenModified: raw.DefaultAutoPlanWhenModified,
+							Enabled:      true,
+						},
+					},
+				},
+				Workflows: make(map[string]valid.Workflow),
+			},
+		},
 		{
 			description: "project dir with ..",
 			input: `

Diff for: server/core/config/raw/project.go

@@ -26,6 +26,7 @@ type Project struct {
 	Dir                       *string    `yaml:"dir,omitempty"`
 	Workspace                 *string    `yaml:"workspace,omitempty"`
 	Workflow                  *string    `yaml:"workflow,omitempty"`
+	TerraformDistribution     *string    `yaml:"terraform_distribution,omitempty"`
 	TerraformVersion          *string    `yaml:"terraform_version,omitempty"`
 	Autoplan                  *Autoplan  `yaml:"autoplan,omitempty"`
 	PlanRequirements          []string   `yaml:"plan_requirements,omitempty"`
@@ -86,6 +87,7 @@ func (p Project) Validate() error {
 		validation.Field(&p.PlanRequirements, validation.By(validPlanReq)),
 		validation.Field(&p.ApplyRequirements, validation.By(validApplyReq)),
 		validation.Field(&p.ImportRequirements, validation.By(validImportReq)),
+		validation.Field(&p.TerraformDistribution, validation.By(validDistribution)),
 		validation.Field(&p.TerraformVersion, validation.By(VersionValidator)),
 		validation.Field(&p.DependsOn, validation.By(DependsOn)),
 		validation.Field(&p.Name, validation.By(validName)),
@@ -118,6 +120,9 @@ func (p Project) ToValid() valid.Project {
 	if p.TerraformVersion != nil {
 		v.TerraformVersion, _ = version.NewVersion(*p.TerraformVersion)
 	}
+	if p.TerraformDistribution != nil {
+		v.TerraformDistribution = p.TerraformDistribution
+	}
 	if p.Autoplan == nil {
 		v.Autoplan = DefaultAutoPlan()
 	} else {
@@ -202,3 +207,11 @@ func validImportReq(value interface{}) error {
 	}
 	return nil
 }
+
+func validDistribution(value interface{}) error {
+	distribution := value.(*string)
+	if distribution != nil && *distribution != "terraform" && *distribution != "opentofu" {
+		return fmt.Errorf("'%s' is not a valid terraform_distribution, only '%s' and '%s' are supported", *distribution, "terraform", "opentofu")
+	}
+	return nil
+}

Diff for: server/core/config/valid/global_cfg.go

@@ -105,6 +105,7 @@ type MergedProjectCfg struct {
 	AutoplanEnabled           bool
 	AutoMergeDisabled         bool
 	AutoMergeMethod           string
+	TerraformDistribution     *string
 	TerraformVersion          *version.Version
 	RepoCfgVersion            int
 	PolicySets                PolicySets
@@ -412,6 +413,7 @@ func (g GlobalCfg) MergeProjectCfg(log logging.SimpleLogging, repoID string, pro
 		DependsOn:                 proj.DependsOn,
 		Name:                      proj.GetName(),
 		AutoplanEnabled:           proj.Autoplan.Enabled,
+		TerraformDistribution:     proj.TerraformDistribution,
 		TerraformVersion:          proj.TerraformVersion,
 		RepoCfgVersion:            rCfg.Version,
 		PolicySets:                g.PolicySets,
@@ -438,6 +440,7 @@ func (g GlobalCfg) DefaultProjCfg(log logging.SimpleLogging, repoID string, repo
 		Workspace:                 workspace,
 		Name:                      "",
 		AutoplanEnabled:           DefaultAutoPlanEnabled,
+		TerraformDistribution:     nil,
 		TerraformVersion:          nil,
 		PolicySets:                g.PolicySets,
 		DeleteSourceBranchOnMerge: deleteSourceBranchOnMerge,

Diff for: server/core/config/valid/repo_cfg.go

@@ -147,6 +147,7 @@ type Project struct {
 	Workspace                 string
 	Name                      *string
 	WorkflowName              *string
+	TerraformDistribution     *string
 	TerraformVersion          *version.Version
 	Autoplan                  Autoplan
 	PlanRequirements          []string