Merge pull request #86 from gregdhill/grant-version

increment hoard version in grant to preserve backwards compatability

Author: Silas Davis

CHANGELOG.md

@@ -1,4 +1,9 @@
 # [Monax Hoard](https://github.com/monax/hoard) Changelog
+## [7.2.0] - 2019-12-17
+### Fixed
+- Symmetric grants are now versioned to preserve backwards compatability
+
+
 ## [7.1.0] - 2019-12-12
 ### Changed
 - [CMD] Secret keys can now be loaded from the environment
@@ -153,6 +158,7 @@ This is the first Hoard open source release and includes:
 	- Hoar-Daemon hoard
 	- Hoar-Control hoarctl CLI
 
+[7.2.0]: https://github.com/monax/hoard/compare/v7.1.0...v7.2.0
 [7.1.0]: https://github.com/monax/hoard/compare/v7.0.0...v7.1.0
 [7.0.0]: https://github.com/monax/hoard/compare/v6.0.0...v7.0.0
 [6.0.0]: https://github.com/monax/hoard/compare/v5.1.0...v6.0.0

NOTES.md

@@ -1,6 +1,3 @@
-### Changed
-- [CMD] Secret keys can now be loaded from the environment
-
 ### Fixed
-- [ENCRYPTION] Secret keys are no longer derived at runtime due to high scrypt memory overhead
+- Symmetric grants are now versioned to preserve backwards compatability
 

cmd/hoard/config.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"strings"
 
-	b64 "encoding/base64"
 	"github.com/cep21/xdgbasedir"
 	cli "github.com/jawher/mow.cli"
 	"github.com/monax/hoard/v7/config"
@@ -49,7 +48,7 @@ func Config(cmd *cli.Cmd) {
 		conf.ChunkSize = *chunkSizeOpt
 		if len(*secretsOpt) > 0 {
 			conf.Secrets = &config.Secrets{
-				Symmetric: make([]config.SymmetricSecret, len(*secretsOpt)),
+				Symmetric: make([]*config.SymmetricSecret, len(*secretsOpt)),
 			}
 			for i, ss := range *secretsOpt {
 				pair := strings.Split(ss, ":")
@@ -66,8 +65,9 @@ func Config(cmd *cli.Cmd) {
 					fatalf("could not derive secret key for config: %v", err)
 				}
 
+				conf.Secrets.Symmetric[i] = new(config.SymmetricSecret)
 				conf.Secrets.Symmetric[i].PublicID = pair[0]
-				conf.Secrets.Symmetric[i].SecretKey = b64.StdEncoding.EncodeToString(data)
+				conf.Secrets.Symmetric[i].SecretKey = data
 			}
 		}
 	}

config/secrets.go

@@ -11,14 +11,62 @@ import (
 // Symmetric secrets are those local to the running daemon
 // and OpenPGP identifies an entity in the given keyring
 type Secrets struct {
-	Symmetric []SymmetricSecret
+	Symmetric []*SymmetricSecret
 	OpenPGP   *OpenPGPSecret
 }
 
 type SymmetricSecret struct {
 	// An identifier for this secret that will be stored in the clear with the grant
-	PublicID  string
-	SecretKey string
+	PublicID string
+	// We expect this to be base64 encoded
+	SecretKey SecretKey
+	// Needed for backwards compatability
+	Passphrase string
+}
+
+// SecretKey allows us to encode yaml and toml as base64
+type SecretKey []byte
+
+// MarshalText should fulfil most serialization interfaces to ensure that the
+// secret key in the config is always base64 encoded
+func (sec SecretKey) MarshalText() ([]byte, error) {
+	data := b64.StdEncoding.EncodeToString(sec)
+	return []byte(data), nil
+}
+
+func (sec *SymmetricSecret) UnmarshalTOML(in interface{}) error {
+	if sec == nil {
+		sec = new(SymmetricSecret)
+	}
+
+	data, _ := in.(map[string]interface{})
+	sec.PublicID, _ = data["PublicID"].(string)
+	sec.Passphrase, _ = data["Passphrase"].(string)
+
+	secret, _ := data["SecretKey"].(string)
+	key, err := b64.StdEncoding.DecodeString(secret)
+	sec.SecretKey = key
+	return err
+}
+
+func (sec *SymmetricSecret) UnmarshalYAML(unmarshal func(interface{}) error) error {
+	if sec == nil {
+		sec = new(SymmetricSecret)
+	}
+
+	secret := &struct {
+		PublicID   string
+		SecretKey  string
+		Passphrase string
+	}{}
+	if err := unmarshal(secret); err != nil {
+		return err
+	}
+	sec.PublicID = secret.PublicID
+	sec.Passphrase = secret.Passphrase
+	key, err := b64.StdEncoding.DecodeString(secret.SecretKey)
+	sec.SecretKey = key
+	return err
 }
 
 type OpenPGPSecret struct {
@@ -34,7 +82,7 @@ type SecretsManager struct {
 	OpenPGP  *OpenPGPSecret
 }
 
-type SymmetricProvider func(secretID string) ([]byte, error)
+type SymmetricProvider func(secretID string) (SymmetricSecret, error)
 
 // NoopSecretManager is an empty secret manager
 var NoopSecretManager = SecretsManager{
@@ -43,35 +91,36 @@ var NoopSecretManager = SecretsManager{
 }
 
 // NoopSymmetricProvider returns an empty provider
-func NoopSymmetricProvider(_ string) ([]byte, error) {
-	return nil, fmt.Errorf("no secrets provided to hoard")
+func NoopSymmetricProvider(_ string) (SymmetricSecret, error) {
+	return SymmetricSecret{}, fmt.Errorf("no secrets provided to hoard")
 }
 
 // ProviderFromConfig creates a secret reader from a set of symmetric secrets
 func NewSymmetricProvider(conf *Secrets, fromEnv bool) (SymmetricProvider, error) {
 	if conf == nil || len(conf.Symmetric) == 0 {
 		return NoopSymmetricProvider, nil
 	}
-	secs := make(map[string][]byte, len(conf.Symmetric))
+	secs := make(map[string]SymmetricSecret, len(conf.Symmetric))
 	for _, s := range conf.Symmetric {
 		if fromEnv {
 			// sometimes we don't want to specify these in the config
-			s.SecretKey = os.Getenv(s.PublicID)
+			secret := os.Getenv(s.PublicID)
+			s.SecretKey = []byte(secret)
+			s.Passphrase = secret
 		}
-		secret, err := b64.StdEncoding.DecodeString(s.SecretKey)
-		if err != nil {
-			return nil, err
+		secs[s.PublicID] = SymmetricSecret{
+			Passphrase: s.Passphrase,
+			SecretKey:  s.SecretKey,
 		}
-		secs[s.PublicID] = secret
 	}
-	return func(id string) ([]byte, error) {
+	return func(id string) (SymmetricSecret, error) {
 		if id == "" {
-			return nil, fmt.Errorf("empty secret ID passed to provider")
+			return SymmetricSecret{}, fmt.Errorf("empty secret ID passed to provider")
 		}
 		if val, ok := secs[id]; ok {
 			return val, nil
 		}
-		return nil, fmt.Errorf("could not find symmetric secret with ID '%s'", id)
+		return SymmetricSecret{}, fmt.Errorf("could not find symmetric secret with ID '%s'", id)
 	}, nil
 }
 

config/secrets_test.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"bytes"
+	"encoding/json"
+	"testing"
+
+	"github.com/BurntSushi/toml"
+	"github.com/monax/hoard/v7/encryption"
+	"github.com/stretchr/testify/assert"
+	yaml "gopkg.in/yaml.v2"
+)
+
+func TestSecretKeyMarshal(t *testing.T) {
+	salt := make([]byte, encryption.NonceSize)
+	key, err := encryption.DeriveSecretKey([]byte("hello"), salt)
+	assert.NoError(t, err)
+
+	secret := SecretKey(key)
+	data, err := secret.MarshalText()
+	assert.NoError(t, err)
+	expected := "bFQ+wRhNaOgC4fNcliGFaZ5Xr3wOywYJZP1eqj6SDCk="
+	assert.Equal(t, expected, string(data))
+
+	inSecret := new(SymmetricSecret)
+	inSecret.SecretKey = secret
+	outSecret := new(SymmetricSecret)
+
+	data, err = json.Marshal(inSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, "{\"PublicID\":\"\",\"SecretKey\":\"bFQ+wRhNaOgC4fNcliGFaZ5Xr3wOywYJZP1eqj6SDCk=\",\"Passphrase\":\"\"}", string(data))
+	err = json.Unmarshal(data, outSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, key, []byte(outSecret.SecretKey))
+
+	data, err = yaml.Marshal(inSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, "publicid: \"\"\nsecretkey: bFQ+wRhNaOgC4fNcliGFaZ5Xr3wOywYJZP1eqj6SDCk=\npassphrase: \"\"\n", string(data))
+	err = yaml.Unmarshal(data, outSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, key, []byte(outSecret.SecretKey))
+
+	buf := new(bytes.Buffer)
+	encoder := toml.NewEncoder(buf)
+	err = encoder.Encode(inSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, "PublicID = \"\"\nSecretKey = \"bFQ+wRhNaOgC4fNcliGFaZ5Xr3wOywYJZP1eqj6SDCk=\"\nPassphrase = \"\"\n", buf.String())
+	err = toml.Unmarshal(buf.Bytes(), outSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, key, []byte(outSecret.SecretKey))
+
+	err = toml.Unmarshal([]byte("SecretKey = \"bFQ+wRhNaOgC4fNcliGFaZ5Xr3wOywYJZP1eqj6SDCk=\"\n"), outSecret)
+	assert.NoError(t, err)
+	assert.Equal(t, key, []byte(outSecret.SecretKey))
+	assert.Equal(t, "", outSecret.PublicID)
+	assert.Equal(t, "", outSecret.Passphrase)
+
+	err = toml.Unmarshal([]byte("PublicID = \"\"\nSecretKey = \"badkey=\"\n"), outSecret)
+	assert.Error(t, err)
+}

go.mod

@@ -19,11 +19,16 @@ require (
 	github.com/h2non/filetype v1.0.10
 	github.com/jawher/mow.cli v1.1.0
 	github.com/monax/relic v2.0.0+incompatible
+	github.com/naoina/go-stringutil v0.1.0 // indirect
+	github.com/naoina/toml v0.1.1
+	github.com/pelletier/go-toml v1.6.0
 	github.com/stretchr/testify v1.4.0
+	github.com/test-go/testify v1.1.4
 	gocloud.dev v0.18.0
 	golang.org/x/crypto v0.0.0-20191122220453-ac88ee75c92c
 	golang.org/x/oauth2 v0.0.0-20191122200657-5d9234df094c
 	google.golang.org/api v0.6.0
 	google.golang.org/grpc v1.25.1
+	gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0
 	gopkg.in/yaml.v2 v2.2.7
 )

go.sum

@@ -127,6 +127,12 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/monax/relic v2.0.0+incompatible h1:5q+fw8Y7UJJuOBzGV5bZNlBk9k9ii6fzmdpwXsZKMdg=
 github.com/monax/relic v2.0.0+incompatible/go.mod h1:ZJcXg8m9tYkd2h6VeEZruhRUQPklFKbzFaTxyXrXxVk=
+github.com/naoina/go-stringutil v0.1.0 h1:rCUeRUHjBjGTSHl0VC00jUPLz8/F9dDzYI70Hzifhks=
+github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0=
+github.com/naoina/toml v0.1.1 h1:PT/lllxVVN0gzzSqSlHEmP8MJB4MY2U7STGxiouV4X8=
+github.com/naoina/toml v0.1.1/go.mod h1:NBIhNtsFMo3G2szEBne+bO4gS192HuIYRqfvOWb4i1E=
+github.com/pelletier/go-toml v1.6.0 h1:aetoXYr0Tv7xRU/V4B4IZJ2QcbtMUFoNb3ORp7TzIK4=
+github.com/pelletier/go-toml v1.6.0/go.mod h1:5N711Q9dKgbdkxHL+MEfF31hpT7l0S0s/t2kKREewys=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -140,6 +146,8 @@ github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXf
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
 github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/test-go/testify v1.1.4 h1:Tf9lntrKUMHiXQ07qBScBTSA0dhYQlu83hswqelv1iE=
+github.com/test-go/testify v1.1.4/go.mod h1:rH7cfJo/47vWGdi4GPj16x3/t1xGOj2YxzmNQzk2ghU=
 go.opencensus.io v0.15.0/go.mod h1:UffZAU+4sDEINUGP/B7UfBBkq4fqLu9zXAX7ke6CHW0=
 go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU=
 go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4=
@@ -237,8 +245,11 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0 h1:POO/ycCATvegFmVuPpQzZFJ+pGZeX22Ufu6fibxDVjU=
+gopkg.in/yaml.v1 v1.0.0-20140924161607-9f9df34309c0/go.mod h1:WDnlLJ4WF5VGsH/HVa3CI79GS0ol3YnhVnKP89i0kNg=
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo=
 gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

grant/grant.go

@@ -7,9 +7,11 @@ import (
 	"github.com/monax/hoard/v7/reference"
 )
 
+const defaultGrantVersion = 1
+
 // Seal this reference into a Grant as specified by Spec
 func Seal(secret config.SecretsManager, ref *reference.Ref, spec *Spec) (*Grant, error) {
-	grt := &Grant{Spec: spec}
+	grt := &Grant{Spec: spec, Version: defaultGrantVersion}
 
 	if s := spec.GetPlaintext(); s != nil {
 		grt.EncryptedReference = PlaintextGrant(ref)
@@ -18,7 +20,7 @@ func Seal(secret config.SecretsManager, ref *reference.Ref, spec *Spec) (*Grant,
 		if err != nil {
 			return nil, err
 		}
-		encRef, err := SymmetricGrant(ref, secret)
+		encRef, err := SymmetricGrant(ref, secret.SecretKey)
 		if err != nil {
 			return nil, err
 		}
@@ -45,7 +47,12 @@ func Unseal(secret config.SecretsManager, grt *Grant) (*reference.Ref, error) {
 		if err != nil {
 			return nil, err
 		}
-		return SymmetricReference(grt.EncryptedReference, secret)
+		switch grt.GetVersion() {
+		case 0:
+			return SymmetricReferenceV0(grt.EncryptedReference, []byte(secret.Passphrase))
+		default:
+			return SymmetricReferenceV1(grt.EncryptedReference, secret.SecretKey)
+		}
 	} else if s := grt.Spec.GetOpenPGP(); s != nil {
 		return OpenPGPReference(grt.EncryptedReference, secret.OpenPGP)
 	} else {