Ensure ephemeral mark is assigned to WriteOnly attribute values

internal/backend/local/backend_apply.go

@@ -106,7 +106,7 @@ func (b *Local) opApply(
 			// actions but that any it does include will be properly-formed.
 			// plan.Errored will be true in this case, which our plan
 			// renderer can rely on to tailor its messaging.
-			if plan != nil && (len(plan.Changes.Resources) != 0 || len(plan.Changes.Outputs) != 0) {
+			if plan != nil && plan.Changes != nil && (len(plan.Changes.Resources) != 0 || len(plan.Changes.Outputs) != 0) {
 				op.View.Plan(plan, schemas)
 			}
 			op.ReportResult(runningOp, diags)

internal/configs/configschema/implied_type.go

@@ -59,6 +59,30 @@ func (b *Block) ContainsSensitive() bool {
 	return false
 }
 
+// ContainsWriteOnly returns true if any of the attributes of the receiving
+// block or any of its descendant blocks are marked as WriteOnly.
+//
+// Blocks themselves cannot be WriteOnly as a whole -- sensitivity is a
+// per-attribute idea -- but sometimes we want to include a whole object
+// decoded from a block in some UI output, and that is safe to do only if
+// none of the contained attributes are WriteOnly.
+func (b *Block) ContainsWriteOnly() bool {
+	for _, attrS := range b.Attributes {
+		if attrS.WriteOnly {
+			return true
+		}
+		if attrS.NestedType != nil && attrS.NestedType.ContainsWriteOnly() {
+			return true
+		}
+	}
+	for _, blockS := range b.BlockTypes {
+		if blockS.ContainsWriteOnly() {
+			return true
+		}
+	}
+	return false
+}
+
 // ImpliedType returns the cty.Type that would result from decoding a Block's
 // ImpliedType and getting the resulting AttributeType.
 //
@@ -134,3 +158,17 @@ func (o *Object) ContainsSensitive() bool {
 	}
 	return false
 }
+
+// ContainsWriteOnly returns true if any of the attributes of the receiving
+// Object are marked as WriteOnly.
+func (o *Object) ContainsWriteOnly() bool {
+	for _, attrS := range o.Attributes {
+		if attrS.WriteOnly {
+			return true
+		}
+		if attrS.NestedType != nil && attrS.NestedType.ContainsWriteOnly() {
+			return true
+		}
+	}
+	return false
+}

internal/configs/configschema/marks.go

@@ -84,6 +84,72 @@ func (b *Block) SensitivePaths(val cty.Value, basePath cty.Path) []cty.Path {
 	return ret
 }
 
+// WriteOnlyPaths returns a set of paths into the given value that should
+// be marked as WriteOnly based on the static declarations in the schema.
+func (b *Block) WriteOnlyPaths(val cty.Value, basePath cty.Path) []cty.Path {
+	var ret []cty.Path
+
+	// We can mark attributes as WriteOnly even if the value is null
+	for name, attrS := range b.Attributes {
+		if attrS.WriteOnly {
+			attrPath := copyAndExtendPath(basePath, cty.GetAttrStep{Name: name})
+			ret = append(ret, attrPath)
+		}
+	}
+
+	// If the value is null, no other marks are possible
+	if val.IsNull() {
+		return ret
+	}
+
+	// Extract marks for nested attribute type values
+	for name, attrS := range b.Attributes {
+		// If the attribute has no nested type, or the nested type doesn't
+		// contain any write-only attributes, skip inspecting it
+		if attrS.NestedType == nil || !attrS.NestedType.ContainsWriteOnly() {
+			continue
+		}
+
+		// Create a copy of the path, with this step added, to add to our PathValueMarks slice
+		attrPath := copyAndExtendPath(basePath, cty.GetAttrStep{Name: name})
+		ret = append(ret, attrS.NestedType.WriteOnlyPaths(val.GetAttr(name), attrPath)...)
+	}
+
+	// Extract marks for nested blocks
+	for name, blockS := range b.BlockTypes {
+		// If our block doesn't contain any WriteOnly attributes, skip inspecting it
+		if !blockS.Block.ContainsWriteOnly() {
+			continue
+		}
+
+		blockV := val.GetAttr(name)
+		if blockV.IsNull() || !blockV.IsKnown() {
+			continue
+		}
+
+		// Create a copy of the path, with this step added, to add to our PathValueMarks slice
+		blockPath := copyAndExtendPath(basePath, cty.GetAttrStep{Name: name})
+
+		switch blockS.Nesting {
+		case NestingSingle, NestingGroup:
+			ret = append(ret, blockS.Block.WriteOnlyPaths(blockV, blockPath)...)
+		case NestingList, NestingMap, NestingSet:
+			blockV, _ = blockV.Unmark() // peel off one level of marking so we can iterate
+			for it := blockV.ElementIterator(); it.Next(); {
+				idx, blockEV := it.Element()
+				// Create a copy of the path, with this block instance's index
+				// step added, to add to our PathValueMarks slice
+				blockInstancePath := copyAndExtendPath(blockPath, cty.IndexStep{Key: idx})
+				morePaths := blockS.Block.WriteOnlyPaths(blockEV, blockInstancePath)
+				ret = append(ret, morePaths...)
+			}
+		default:
+			panic(fmt.Sprintf("unsupported nesting mode %s", blockS.Nesting))
+		}
+	}
+	return ret
+}
+
 // SensitivePaths returns a set of paths into the given value that should be
 // marked as sensitive based on the static declarations in the schema.
 func (o *Object) SensitivePaths(val cty.Value, basePath cty.Path) []cty.Path {
@@ -140,3 +206,60 @@ func (o *Object) SensitivePaths(val cty.Value, basePath cty.Path) []cty.Path {
 	}
 	return ret
 }
+
+// WriteOnlyPaths returns a set of paths into the given value that should be
+// marked as WriteOnly based on the static declarations in the schema.
+func (o *Object) WriteOnlyPaths(val cty.Value, basePath cty.Path) []cty.Path {
+	var ret []cty.Path
+
+	if val.IsNull() || !val.IsKnown() {
+		return ret
+	}
+
+	for name, attrS := range o.Attributes {
+		// Skip attributes which can never produce WriteOnly path value marks
+		if !attrS.WriteOnly && (attrS.NestedType == nil || !attrS.NestedType.ContainsWriteOnly()) {
+			continue
+		}
+
+		switch o.Nesting {
+		case NestingSingle, NestingGroup:
+			// Create a path to this attribute
+			attrPath := copyAndExtendPath(basePath, cty.GetAttrStep{Name: name})
+
+			if attrS.WriteOnly {
+				// If the entire attribute is WriteOnly, mark it so
+				ret = append(ret, attrPath)
+			} else {
+				// The attribute has a nested type which contains WriteOnly
+				// attributes, so recurse
+				ret = append(ret, attrS.NestedType.WriteOnlyPaths(val.GetAttr(name), attrPath)...)
+			}
+		case NestingList, NestingMap, NestingSet:
+			// For nested attribute types which have a non-single nesting mode,
+			// we add path value marks for each element of the collection
+			val, _ = val.Unmark() // peel off one level of marking so we can iterate
+			for it := val.ElementIterator(); it.Next(); {
+				idx, attrEV := it.Element()
+				attrV := attrEV.GetAttr(name)
+
+				// Create a path to this element of the attribute's collection. Note
+				// that the path is extended in opposite order to the iteration order
+				// of the loops: index into the collection, then the contained
+				// attribute name. This is because we have one type
+				// representing multiple collection elements.
+				attrPath := copyAndExtendPath(basePath, cty.IndexStep{Key: idx}, cty.GetAttrStep{Name: name})
+
+				if attrS.WriteOnly {
+					// If the entire attribute is WriteOnly, mark it so
+					ret = append(ret, attrPath)
+				} else {
+					ret = append(ret, attrS.NestedType.WriteOnlyPaths(attrV, attrPath)...)
+				}
+			}
+		default:
+			panic(fmt.Sprintf("unsupported nesting mode %s", attrS.NestedType.Nesting))
+		}
+	}
+	return ret
+}

internal/plans/changes.go

@@ -611,22 +611,27 @@ func (c *Change) Encode(ty cty.Type) (*ChangeSrc, error) {
 	// We don't accept any other marks here. The caller should have dealt
 	// with those somehow and replaced them with unmarked placeholders before
 	// writing the value into the state.
-	unmarkedBefore, marksesBefore := c.Before.UnmarkDeepWithPaths()
-	unmarkedAfter, marksesAfter := c.After.UnmarkDeepWithPaths()
-	sensitiveAttrsBefore, unsupportedMarksesBefore := marks.PathsWithMark(marksesBefore, marks.Sensitive)
-	sensitiveAttrsAfter, unsupportedMarksesAfter := marks.PathsWithMark(marksesAfter, marks.Sensitive)
-	if len(unsupportedMarksesBefore) != 0 {
+	unmarkedBefore, marksBefore := c.Before.UnmarkDeepWithPaths()
+	unmarkedAfter, marksAfter := c.After.UnmarkDeepWithPaths()
+
+	sensitiveAttrsBefore, remainingMarksBefore := marks.PathsWithMark(marksBefore, marks.Sensitive)
+	sensitiveAttrsAfter, remainingMarksAfter := marks.PathsWithMark(marksAfter, marks.Sensitive)
+
+	writeOnlyAttrsBefore, remainingMarksBefore := marks.PathsWithMark(remainingMarksBefore, marks.Ephemeral)
+	writeOnlyAttrsAfter, remainingMarksAfter := marks.PathsWithMark(remainingMarksAfter, marks.Ephemeral)
+
+	if len(remainingMarksBefore) != 0 {
 		return nil, fmt.Errorf(
 			"prior value %s: can't serialize value marked with %#v (this is a bug in Terraform)",
-			tfdiags.FormatCtyPath(unsupportedMarksesBefore[0].Path),
-			unsupportedMarksesBefore[0].Marks,
+			tfdiags.FormatCtyPath(remainingMarksBefore[0].Path),
+			remainingMarksBefore[0].Marks,
 		)
 	}
-	if len(unsupportedMarksesAfter) != 0 {
+	if len(remainingMarksAfter) != 0 {
 		return nil, fmt.Errorf(
 			"new value %s: can't serialize value marked with %#v (this is a bug in Terraform)",
-			tfdiags.FormatCtyPath(unsupportedMarksesAfter[0].Path),
-			unsupportedMarksesAfter[0].Marks,
+			tfdiags.FormatCtyPath(remainingMarksAfter[0].Path),
+			remainingMarksAfter[0].Marks,
 		)
 	}
 
@@ -645,6 +650,8 @@ func (c *Change) Encode(ty cty.Type) (*ChangeSrc, error) {
 		After:                afterDV,
 		BeforeSensitivePaths: sensitiveAttrsBefore,
 		AfterSensitivePaths:  sensitiveAttrsAfter,
+		BeforeWriteOnlyPaths: writeOnlyAttrsBefore,
+		AfterWriteOnlyPaths:  writeOnlyAttrsAfter,
 		Importing:            c.Importing.Encode(),
 		GeneratedConfig:      c.GeneratedConfig,
 	}, nil

internal/plans/changes_src.go

@@ -388,6 +388,14 @@ type ChangeSrc struct {
 	// the serialized change.
 	BeforeSensitivePaths, AfterSensitivePaths []cty.Path
 
+	// BeforeWriteOnlyPaths and AfterWriteOnlyPaths are the paths for any
+	// values in Before or After (respectively) that are considered to be
+	// WriteOnly. The WriteOnly marks are removed from the in-memory values
+	// to enable encoding (marked values cannot be marshalled), and so we
+	// store the WriteOnly paths to allow re-marking later when we decode
+	// the serialized change.
+	BeforeWriteOnlyPaths, AfterWriteOnlyPaths []cty.Path
+
 	// Importing is present if the resource is being imported as part of this
 	// change.
 	//

internal/states/instance_object.go

@@ -98,11 +98,34 @@ func (o *ResourceInstanceObject) Encode(ty cty.Type, schemaVersion uint64) (*Res
 	// If it contains marks, remove these marks before traversing the
 	// structure with UnknownAsNull, and save the PathValueMarks
 	// so we can save them in state.
-	val, sensitivePaths, err := unmarkValueForStorage(o.Value)
+	val, remainingMarks := o.Value.UnmarkDeepWithPaths()
+
+	sensitivePaths, remainingMarks := marks.PathsWithMark(remainingMarks, marks.Sensitive)
+
+	writeOnlyPaths, remainingMarks := marks.PathsWithMark(remainingMarks, marks.Ephemeral)
+	// ensure we aren't attempting to persist real values after unmarking
+	err := cty.Walk(val, func(p cty.Path, v cty.Value) (bool, error) {
+		for _, woPath := range writeOnlyPaths {
+			if p.Equals(woPath) && !v.IsNull() {
+				return false, fmt.Errorf(
+					"%s: cannot serialize ephemeral value %#v for inclusion in a state snapshot (this is a bug in Terraform)",
+					tfdiags.FormatCtyPath(woPath), val.GoString(),
+				)
+			}
+		}
+		return true, nil
+	})
 	if err != nil {
 		return nil, err
 	}
 
+	if len(remainingMarks) != 0 {
+		return nil, fmt.Errorf(
+			"%s: cannot serialize value marked as %#v for inclusion in a state snapshot (this is a bug in Terraform)",
+			tfdiags.FormatCtyPath(remainingMarks[0].Path), remainingMarks[0].Marks,
+		)
+	}
+
 	// Our state serialization can't represent unknown values, so we convert
 	// them to nulls here. This is lossy, but nobody should be writing unknown
 	// values here and expecting to get them out again later.
@@ -157,24 +180,3 @@ func (o *ResourceInstanceObject) AsTainted() *ResourceInstanceObject {
 	ret.Status = ObjectTainted
 	return ret
 }
-
-// unmarkValueForStorage takes a value that possibly contains marked values
-// and returns an equal value without markings along with the separated mark
-// metadata that should be stored alongside the value in another field.
-//
-// This function only accepts the marks that are valid to store, and so will
-// return an error if other marks are present. Marks that this package doesn't
-// know how to store must be dealt with somehow by a caller -- presumably by
-// replacing each marked value with some sort of storage placeholder -- before
-// writing a value into the state.
-func unmarkValueForStorage(v cty.Value) (unmarkedV cty.Value, sensitivePaths []cty.Path, err error) {
-	val, pvms := v.UnmarkDeepWithPaths()
-	sensitivePaths, withOtherMarks := marks.PathsWithMark(pvms, marks.Sensitive)
-	if len(withOtherMarks) != 0 {
-		return cty.NilVal, nil, fmt.Errorf(
-			"%s: cannot serialize value marked as %#v for inclusion in a state snapshot (this is a bug in Terraform)",
-			tfdiags.FormatCtyPath(withOtherMarks[0].Path), withOtherMarks[0].Marks,
-		)
-	}
-	return val, sensitivePaths, nil
-}

internal/terraform/node_resource_abstract_instance.go

@@ -748,6 +748,9 @@ func (n *NodeAbstractResourceInstance) refresh(ctx EvalContext, deposedKey state
 	if moreSensitivePaths := schema.SensitivePaths(ret.Value, nil); len(moreSensitivePaths) != 0 {
 		ret.Value = marks.MarkPaths(ret.Value, marks.Sensitive, moreSensitivePaths)
 	}
+	if writeOnlyPaths := schema.WriteOnlyPaths(ret.Value, nil); len(writeOnlyPaths) != 0 {
+		ret.Value = marks.MarkPaths(ret.Value, marks.Ephemeral, writeOnlyPaths)
+	}
 
 	return ret, deferred, diags
 }
@@ -1067,6 +1070,9 @@ func (n *NodeAbstractResourceInstance) plan(
 	if sensitivePaths := schema.SensitivePaths(plannedNewVal, nil); len(sensitivePaths) != 0 {
 		plannedNewVal = marks.MarkPaths(plannedNewVal, marks.Sensitive, sensitivePaths)
 	}
+	if writeOnlyPaths := schema.WriteOnlyPaths(plannedNewVal, nil); len(writeOnlyPaths) != 0 {
+		plannedNewVal = marks.MarkPaths(plannedNewVal, marks.Ephemeral, writeOnlyPaths)
+	}
 
 	reqRep, reqRepDiags := getRequiredReplaces(unmarkedPriorVal, unmarkedPlannedNewVal, resp.RequiresReplace, n.ResolvedProvider.Provider, n.Addr)
 	diags = diags.Append(reqRepDiags)
@@ -2646,6 +2652,9 @@ func (n *NodeAbstractResourceInstance) apply(
 	if sensitivePaths := schema.SensitivePaths(newVal, nil); len(sensitivePaths) != 0 {
 		newVal = marks.MarkPaths(newVal, marks.Sensitive, sensitivePaths)
 	}
+	if writeOnlyPaths := schema.WriteOnlyPaths(newVal, nil); len(writeOnlyPaths) != 0 {
+		newVal = marks.MarkPaths(newVal, marks.Ephemeral, writeOnlyPaths)
+	}
 
 	if change.Action != plans.Delete && !diags.HasErrors() {
 		// Only values that were marked as unknown in the planned value are allowed

internal/terraform/node_resource_plan_partialexp.go

@@ -278,6 +278,9 @@ func (n *nodePlannablePartialExpandedResource) managedResourceExecute(ctx EvalCo
 	if sensitivePaths := schema.SensitivePaths(plannedNewVal, nil); len(sensitivePaths) != 0 {
 		plannedNewVal = marks.MarkPaths(plannedNewVal, marks.Sensitive, sensitivePaths)
 	}
+	if writeOnlyPaths := schema.WriteOnlyPaths(plannedNewVal, nil); len(writeOnlyPaths) != 0 {
+		plannedNewVal = marks.MarkPaths(plannedNewVal, marks.Ephemeral, writeOnlyPaths)
+	}
 
 	change.After = plannedNewVal
 	change.Private = resp.PlannedPrivate