add support for marshaling zng.Value fields (#2693)

This commit adds support for marshaling and unmarshaling zng Values
that are embedded in Go structs.  This allows us to encode arbitrary
zng.Values as part of native Go data structures, e.g., to encode
the upper/lower range boundary as zng.Values in the metadata struct
of an arbitrary Zed lake data object.

While here, we also improved the error message when trying to decode
a non-record Zed value into a Go struct.

Author: mccanne

zson/marshal.go

@@ -275,6 +275,7 @@ func (m *MarshalZNGContext) NamedBindings(bindings []Binding) error {
 }
 
 var nanoTsType = reflect.TypeOf(nano.Ts(0))
+var zngValueType = reflect.TypeOf(zng.Value{})
 
 func (m *MarshalZNGContext) encodeValue(v reflect.Value) (zng.Type, error) {
 	typ, err := m.encodeAny(v)
@@ -292,8 +293,10 @@ func (m *MarshalZNGContext) encodeValue(v reflect.Value) (zng.Type, error) {
 		kind := v.Kind().String()
 		if name != "" && name != kind {
 			// We do not want to further decorate nano.Ts as
-			// it's already been converted to a Zed time.
-			if v.Type() == nanoTsType {
+			// it's already been converted to a Zed time;
+			// likewise for zng.Value, which gets encoded as
+			// itself and its own alias type if it has one.
+			if t := v.Type(); t == nanoTsType || t == zngValueType {
 				return typ, nil
 			}
 			path := v.Type().PkgPath()
@@ -320,10 +323,22 @@ func (m *MarshalZNGContext) encodeAny(v reflect.Value) (zng.Type, error) {
 	if v.Type().Implements(marshalerTypeZNG) {
 		return v.Interface().(ZNGMarshaler).MarshalZNG(m)
 	}
-	if v, ok := v.Interface().(nano.Ts); ok {
-		m.Builder.AppendPrimitive(zng.EncodeTime(v))
+	if ts, ok := v.Interface().(nano.Ts); ok {
+		m.Builder.AppendPrimitive(zng.EncodeTime(ts))
 		return zng.TypeTime, nil
 	}
+	if zv, ok := v.Interface().(zng.Value); ok {
+		typ, err := m.TranslateType(zv.Type)
+		if err != nil {
+			return nil, err
+		}
+		if zng.IsContainerType(typ) {
+			m.Builder.AppendContainer(zv.Bytes)
+		} else {
+			m.Builder.AppendPrimitive(zv.Bytes)
+		}
+		return typ, nil
+	}
 	switch v.Kind() {
 	case reflect.Array:
 		if v.Type().Elem().Kind() == reflect.Uint8 {
@@ -609,6 +624,12 @@ func (u *UnmarshalZNGContext) decodeAny(zv zng.Value, v reflect.Value) error {
 		v.Set(reflect.ValueOf(x))
 		return err
 	}
+	if _, ok := v.Interface().(zng.Value); ok {
+		// For zng.Values we simply set the reflect value to the
+		// zng.Value that has been decoded.
+		v.Set(reflect.ValueOf(zv))
+		return nil
+	}
 	switch v.Kind() {
 	case reflect.Array:
 		return u.decodeArray(zv, v)
@@ -750,7 +771,7 @@ func (u *UnmarshalZNGContext) decodeIP(zv zng.Value, v reflect.Value) error {
 func (u *UnmarshalZNGContext) decodeRecord(zv zng.Value, sval reflect.Value) error {
 	recType, ok := zng.AliasOf(zv.Type).(*zng.TypeRecord)
 	if !ok {
-		return errors.New("not a record")
+		return fmt.Errorf("cannot unmarshal Zed type %q into Go struct", FormatType(zv.Type))
 	}
 	nameToField := make(map[string]int)
 	stype := sval.Type()

zson/marshal_test.go

@@ -235,3 +235,50 @@ func TestUnexported(t *testing.T) {
 	_, err := m.Marshal(f)
 	require.NoError(t, err)
 }
+
+type ZNGValueField struct {
+	Name  string
+	Field zng.Value `zng:"field"`
+}
+
+func TestZNGValueField(t *testing.T) {
+	// Include a Zed int64 inside a Go struct as a zng.Value field.
+	zngValueField := &ZNGValueField{
+		Name:  "test1",
+		Field: zng.Value{zng.TypeInt64, zng.EncodeInt(123)},
+	}
+	m := zson.NewZNGMarshaler()
+	m.Decorate(zson.StyleSimple)
+	zv, err := m.Marshal(zngValueField)
+	require.NoError(t, err)
+	expected := `{Name:"test1",field:123} (=ZNGValueField)`
+	actual, err := zson.FormatValue(zv)
+	require.NoError(t, err)
+	assert.Equal(t, trim(expected), trim(actual))
+	u := zson.NewZNGUnmarshaler()
+	var out ZNGValueField
+	err = u.Unmarshal(zv, &out)
+	require.NoError(t, err)
+	assert.Equal(t, *zngValueField, out)
+	// Include a Zed record inside a Go struct in a zng.Value field.
+	z := `{s:"foo",a:[1,2,3]}`
+	zv2, err := zson.ParseValue(zson.NewContext(), z)
+	require.NoError(t, err)
+	zngValueField2 := &ZNGValueField{
+		Name:  "test2",
+		Field: zv2,
+	}
+	m2 := zson.NewZNGMarshaler()
+	m2.Decorate(zson.StyleSimple)
+	zv3, err := m2.Marshal(zngValueField2)
+	require.NoError(t, err)
+	expected2 := `{Name:"test2",field:{s:"foo",a:[1,2,3]}} (=ZNGValueField)`
+	actual2, err := zson.FormatValue(zv3)
+	require.NoError(t, err)
+	assert.Equal(t, trim(expected2), trim(actual2))
+	u2 := zson.NewZNGUnmarshaler()
+	var out2 ZNGValueField
+	err = u2.Unmarshal(zv3, &out2)
+	require.NoError(t, err)
+	assert.Equal(t, *zngValueField2, out2)
+}

zson/zson.go

@@ -10,6 +10,7 @@ import (
 	"strings"
 
 	"github.com/brimdata/zed/compiler/ast/zed"
+	"github.com/brimdata/zed/zcode"
 	"github.com/brimdata/zed/zng"
 )
 
@@ -68,6 +69,22 @@ func ParseType(zctx *Context, zson string) (zng.Type, error) {
 	return NewAnalyzer().convertType(zctx, ast)
 }
 
+func ParseValue(zctx *Context, zson string) (zng.Value, error) {
+	zp, err := NewParser(strings.NewReader(zson))
+	if err != nil {
+		return zng.Value{}, err
+	}
+	ast, err := zp.ParseValue()
+	if err != nil {
+		return zng.Value{}, err
+	}
+	val, err := NewAnalyzer().ConvertValue(zctx, ast)
+	if err != nil {
+		return zng.Value{}, err
+	}
+	return Build(zcode.NewBuilder(), val)
+}
+
 func TranslateType(zctx *Context, astType zed.Type) (zng.Type, error) {
 	return NewAnalyzer().convertType(zctx, astType)
 }