Add tf import support to conditionally honor all props in spec (#740)

Signed-off-by: Steven Kaufer <[email protected]>

Author: kaufers

pkg/provider/terraform/instance/README.md

@@ -105,19 +105,66 @@ and update its state.  When an instance is removed, Terraform will do the same b
 and update its state.
 
 
-## Running
+## Runtime Options
+
+The plugin support the following `Launch` `inproc` `Options`:
+
+* `Dir`: Directory that will be used to contain the `tfstate` and `tf.json` files
+* `PollInterval`: Frequency that `terraform apply` is invoked; note that it is only invoked on the leader
+manager (unless `standalone=true`, see below)
+* `Standalone`: If `true` then manager leadership is not verified prior to invoking `terraform apply`
+(default is `false`)
+* `Envs`: Array of environment variables to include when invoking the `terraform` commands
+
+The plugin also supports importing existing resources into terraform; this can be used to import the
+initial manager into terraform. Once the resource is imported into terraform, a corresponding `.tf.json`
+file is also created. The following optional fields are used for this purpose:
+* `ImportGroupSpecURL`: The group specification URL that contains a nested instance specification; the
+`.tf.json` file for the imported resource contains the properties in the instance specification
+* `ImportGroupID`: Optional group ID that the imported resource should be tagged with
+* `ImportResources`: An array of resources to import into terraform, these resources must correspond
+with those in the instance specification (nested in the group specification). Each element contains:
+  * `ResourceType`: The terraform resource type being imported
+  * `ResourceID`: The resource ID being imported
+  * `ResourceName`: The terraform resource name to assign to the the resource being imported; this value
+  must match the name in the instance spec (required if there is more then one resource of the same type
+  in the specification)
+  * `ExcludePropIDs`: An array of property IDs in the instance specification that should _not_ be
+  included in the corresponding `.tf.json` file
+
+For example:
 
-Begin by building plugin [binaries](/README.md#binaries).
+```
+{
+  "Key" : "terraform",
+  "Launch" : {
+    "inproc": {
+      "Kind" : "terraform",
+      "Options" : {
+        "Dir": "/infrakit",
+        "PollInterval": "60s",
+        "Standalone": false,
+        "ImportGroupSpecURL" : "file://defn-mgr-group.json",
+        "ImportGroupID": "managers",
+        "ImportResources": [
+          {
+            "ResourceType": "ibm_compute_vm_instance",
+            "ResourceID": "123456"
+          },
+          {
+            "ResourceType": "ibm_subnet",
+            "ResourceID": "abc-123-xyz"
+          }
+        ]
+      }
+    }
+  }
+}
+```
 
-The supported fields are:
-* `dir`: Directory that will be used to contain the `tfstate` and `tf.json` files
-* `poll-interval`: Frequency that `terraform apply` is invoked; note that it is only invoked on the leader manager (unless `standalone=true`, see below)
-* `standalone`: If `true` then manager leadership is not verified prior to invoking `terraform apply` (default is `false`)
+## Running
 
-The plugin also supports importing an existing resource into terraform; this can be used to import the initial manager into terraform. Once the resource is imported into terraform, a corresponding `.tf.json` file is also created. The following optional fields are used for this purpose:
-* `import-group-spec-url`: The group specification URL that contains a nested instance specification; the `.tf.json` file for the imported resource contains the properties in the instance specification
-* `import-resource`: The terraform resource type, optional resource name, and resource ID to import into in the form of `<type>:[<name>:]<id>`. The `<type>` and `<name>` must match a resource in the instance specification; the `<name>` is required if there is more then one resource of the given type in the specification. This value may be specified multiple times.
-* `import-group-id`: Optional group ID that the imported resource should be tagged with
+Begin by building plugin [binaries](/README.md#binaries).
 
 The plugin also checks to make sure it can call `terraform`. Install Terraform [here](https://www.terraform.io/downloads.html) if you haven't done so.
 

pkg/provider/terraform/instance/plugin.go

@@ -78,6 +78,7 @@ type ImportResource struct {
 	ResourceID           *string
 	ResourceType         *TResourceType
 	ResourceName         *TResourceName      // Name of resource in the instance spec
+	ExcludePropIDs       *[]string           // Property IDs that exist in the instance spec that should be excluded
 	ResourceProps        TResourceProperties // Populated via tf show
 	SpecProps            TResourceProperties // Parsed from instance spec
 	FinalProps           TResourceProperties // Properties for the tf.json.new file
@@ -1767,20 +1768,33 @@ func setFinalResourceAndFilename(resource *ImportResource, filename string, reso
 func determineFinalPropsForImport(res *ImportResource) {
 	log.Infof("Using spec for %v import: %v", string(*res.ResourceType), res.SpecProps)
 	finalProps := TResourceProperties{}
-	for k := range res.SpecProps {
+	for k, specVal := range res.SpecProps {
 		// Ignore certain keys in spec
 		if k == PropScope {
 			continue
 		}
 		if k == PropHostnamePrefix {
 			k = "hostname"
 		}
-		v, has := res.ResourceProps[k]
-		if !has {
-			log.Warningf("Imported terraform resource missing '%s' property, not setting", k)
-			continue
+		if res.ExcludePropIDs != nil && len(*res.ExcludePropIDs) > 0 {
+			exclude := false
+			for _, propID := range *res.ExcludePropIDs {
+				if propID == k {
+					exclude = true
+					log.Infof("Excluding spec property '%s' for resource type %v", propID, string(*res.ResourceType))
+					break
+				}
+			}
+			if exclude {
+				continue
+			}
+		}
+		if v, has := res.ResourceProps[k]; has {
+			finalProps[k] = v
+		} else {
+			log.Warningf("Imported terraform resource missing '%s' property, using spec value: %v", k, specVal)
+			finalProps[k] = specVal
 		}
-		finalProps[k] = v
 	}
 	// Always keep the tags, even if the spec does not have them as a property
 	if _, has := finalProps["tags"]; !has {

pkg/provider/terraform/instance/plugin_test.go

@@ -3592,11 +3592,13 @@ func TestWriteTfJSONFilesForImportMultipleFiles(t *testing.T) {
 
 func TestDetermineFinalPropsForImport(t *testing.T) {
 	specProps := TResourceProperties{
-		PropHostnamePrefix: "some-prefix",
-		PropScope:          "some-scope",
-		"ssh-key-ids":      []interface{}{789},
-		"datacenter":       "some-datacenter",
-		"z-other":          "not-imported",
+		PropHostnamePrefix:      "some-prefix",
+		PropScope:               "some-scope",
+		"ssh-key-ids":           []interface{}{789},
+		"datacenter":            "some-datacenter",
+		"some-key-in-spec-only": "some-key-value",
+		"lifecycle":             map[string][]string{"ignore_changes": {"static-key"}},
+		"exclude-prop":          "exclude-val",
 	}
 	// Include tags
 	resourceProps := TResourceProperties{
@@ -3611,10 +3613,13 @@ func TestDetermineFinalPropsForImport(t *testing.T) {
 		"z-imported": "imported-but-not-in-spec",
 	}
 	resType := VMIBMCloud
+	// With excluded props
+	excludePropIDs := []string{"exclude-prop"}
 	res := ImportResource{
-		ResourceType:  &resType,
-		ResourceProps: resourceProps,
-		SpecProps:     specProps,
+		ResourceType:   &resType,
+		ExcludePropIDs: &excludePropIDs,
+		ResourceProps:  resourceProps,
+		SpecProps:      specProps,
 	}
 	determineFinalPropsForImport(&res)
 	expectedProps := TResourceProperties{
@@ -3625,18 +3630,42 @@ func TestDetermineFinalPropsForImport(t *testing.T) {
 			"actual-tag1:actual-val1",
 			"actual-tag2:actual-val2",
 		},
+		"some-key-in-spec-only": "some-key-value",
+		"lifecycle":             map[string][]string{"ignore_changes": {"static-key"}},
 	}
 	require.Equal(t, expectedProps, res.FinalProps)
 	// Without tags
 	delete(resourceProps, "tags")
 	res = ImportResource{
-		ResourceType:  &resType,
-		ResourceProps: resourceProps,
-		SpecProps:     specProps,
+		ResourceType:   &resType,
+		ExcludePropIDs: &excludePropIDs,
+		ResourceProps:  resourceProps,
+		SpecProps:      specProps,
 	}
 	determineFinalPropsForImport(&res)
 	delete(expectedProps, "tags")
 	require.Equal(t, expectedProps, res.FinalProps)
+	// Without excluded props
+	excludePropIDs = []string{}
+	res = ImportResource{
+		ResourceType:   &resType,
+		ExcludePropIDs: &excludePropIDs,
+		ResourceProps:  resourceProps,
+		SpecProps:      specProps,
+	}
+	determineFinalPropsForImport(&res)
+	expectedProps["exclude-prop"] = "exclude-val"
+	require.Equal(t, expectedProps, res.FinalProps)
+	// nil excluded props
+	res = ImportResource{
+		ResourceType:   &resType,
+		ExcludePropIDs: nil,
+		ResourceProps:  resourceProps,
+		SpecProps:      specProps,
+	}
+	determineFinalPropsForImport(&res)
+	expectedProps["exclude-prop"] = "exclude-val"
+	require.Equal(t, expectedProps, res.FinalProps)
 }
 
 func TestImportNoVm(t *testing.T) {

pkg/run/v0/terraform/terraform.go

@@ -49,6 +49,9 @@ type ImportResourceOptions struct {
 
 	// ID of the resource to import
 	ResourceID string
+
+	// IDs of the properties to exclude from the instance spec
+	ExcludePropIDs []string
 }
 
 // Options capture the options for starting up the plugin.
@@ -74,7 +77,7 @@ type Options struct {
 	// NewOption is an example... see the plugins.json file in this directory.
 	NewOption string
 
-	// Envs are the environemtn variables to include when invoking terraform
+	// Envs are the environment variables to include when invoking terraform
 	Envs types.Any
 }
 
@@ -121,10 +124,12 @@ func Run(plugins func() discovery.Plugins, name plugin.Name,
 		resType := terraform.TResourceType(importResource.ResourceType)
 		resName := terraform.TResourceName(importResource.ResourceName)
 		resID := importResource.ResourceID
+		excludePropIDs := importResource.ExcludePropIDs
 		res := terraform.ImportResource{
-			ResourceType: &resType,
-			ResourceName: &resName,
-			ResourceID:   &resID,
+			ResourceType:   &resType,
+			ResourceName:   &resName,
+			ResourceID:     &resID,
+			ExcludePropIDs: &excludePropIDs,
 		}
 		resources = append(resources, &res)
 	}