From fd28158042dc75452e45b2798c631bd04d15ccdc Mon Sep 17 00:00:00 2001 From: Matt Calhoun Date: Mon, 3 Feb 2025 14:48:56 -0500 Subject: [PATCH] add redis store --- go.mod | 2 + go.sum | 11 + pkg/store/redis_store.go | 151 +++++++++++ pkg/store/redis_store_test.go | 243 ++++++++++++++++++ pkg/store/registry.go | 16 +- pkg/store/store.go | 20 ++ .../projects/configuration/stores.mdx | 64 +++++ 7 files changed, 506 insertions(+), 1 deletion(-) create mode 100644 pkg/store/redis_store.go create mode 100644 pkg/store/redis_store_test.go diff --git a/go.mod b/go.mod index ee4413e265..ed065fbf9e 100644 --- a/go.mod +++ b/go.mod @@ -46,6 +46,7 @@ require ( github.com/open-policy-agent/opa v1.1.0 github.com/otiai10/copy v1.14.1 github.com/pkg/errors v0.9.1 + github.com/redis/go-redis/v9 v9.7.0 github.com/samber/lo v1.49.1 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 @@ -128,6 +129,7 @@ require ( github.com/containerd/stargz-snapshotter/estargz v0.16.3 // indirect github.com/cyphar/filepath-securejoin v0.3.6 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect + github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect github.com/dimchansky/utfbom v1.1.1 // indirect github.com/dlclark/regexp2 v1.11.0 // indirect github.com/docker/cli v27.5.0+incompatible // indirect diff --git a/go.sum b/go.sum index c6e4efc175..829218c7ac 100644 --- a/go.sum +++ b/go.sum @@ -834,6 +834,13 @@ github.com/boombuler/barcode v1.0.0/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl github.com/boombuler/barcode v1.0.1/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bradleyjkemp/cupaloy/v2 v2.8.0 h1:any4BmKE+jGIaMpnU8YgH/I2LPiLBufr6oMMlVBbn9M= github.com/bradleyjkemp/cupaloy/v2 v2.8.0/go.mod h1:bm7JXdkRd4BHJk9HpwqAI8BoAY1lps46Enkdqw6aRX0= +github.com/brianvoe/gofakeit/v5 v5.11.2/go.mod h1:/ZENnKqX+XrN8SORLe/fu5lZDIo1tuPncWuRD+eyhSI= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= +github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= +github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs= +github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c= +github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA= +github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2 h1:3uZCA/BLTIu+DqCfguByNMJa2HVHpXvjfy0Dy7g6fuA= github.com/bytecodealliance/wasmtime-go/v3 v3.0.2/go.mod h1:RnUjnIXxEJcL6BgCvNyzCCRzZcxCgsZCi+RNlvYor5Q= github.com/catppuccin/go v0.2.0 h1:ktBeIrIP42b/8FGiScP9sgrWOss3lw0Z5SktRoithGA= @@ -934,6 +941,8 @@ github.com/dgraph-io/badger/v4 v4.5.1 h1:7DCIXrQjo1LKmM96YD+hLVJ2EEsyyoWxJfpdd56 github.com/dgraph-io/badger/v4 v4.5.1/go.mod h1:qn3Be0j3TfV4kPbVoK0arXCD1/nr1ftth6sbL5jxdoA= github.com/dgraph-io/ristretto/v2 v2.1.0 h1:59LjpOJLNDULHh8MC4UaegN52lC4JnO2dITsie/Pa8I= github.com/dgraph-io/ristretto/v2 v2.1.0/go.mod h1:uejeqfYXpUomfse0+lO+13ATz4TypQYLJZzBSAemuB4= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= +github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54 h1:SG7nF6SRlWhcT7cNTs5R6Hk4V2lcmLz2NsG2VnInyNo= github.com/dgryski/trifles v0.0.0-20230903005119-f50d829f2e54/go.mod h1:if7Fbed8SFyPtHLHbg49SI7NAdJiC5WIA09pe59rfAA= github.com/dimchansky/utfbom v1.1.0/go.mod h1:rO41eb7gLfo8SF1jd9F8HplJm1Fewwi4mQvIirEdv+8= @@ -1661,6 +1670,8 @@ github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0leargg github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0 h1:MkV+77GLUNo5oJ0jf870itWm3D0Sjh7+Za9gazKc5LQ= github.com/rcrowley/go-metrics v0.0.0-20200313005456-10cdbea86bc0/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= +github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E= +github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw= github.com/remyoudompheng/bigfft v0.0.0-20200410134404-eec4a21b6bb0/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo= github.com/rivo/uniseg v0.1.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc= diff --git a/pkg/store/redis_store.go b/pkg/store/redis_store.go new file mode 100644 index 0000000000..ea0d1f84cd --- /dev/null +++ b/pkg/store/redis_store.go @@ -0,0 +1,151 @@ +package store + +import ( + "context" + "encoding/json" + "fmt" + "os" + "strings" + "time" + + "github.com/redis/go-redis/v9" +) + +type RedisStore struct { + prefix string + repoName string + redisClient RedisClient + stackDelimiter *string +} + +type RedisStoreOptions struct { + Prefix *string `mapstructure:"prefix"` + StackDelimiter *string `mapstructure:"stack_delimiter"` + URL *string `mapstructure:"url"` +} + +// RedisClient interface allows us to mock the Redis Client in test with only the methods we are using in the +// RedisStore. +type RedisClient interface { + Get(ctx context.Context, key string) *redis.StringCmd + Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd +} + +// Ensure RedisStore implements the store.Store interface. +var _ Store = (*RedisStore)(nil) + +func getRedisOptions(options *RedisStoreOptions) (*redis.Options, error) { + if options.URL != nil { + opts, err := redis.ParseURL(*options.URL) + if err != nil { + return &redis.Options{}, fmt.Errorf("failed to parse redis url: %v", err) + } + + return opts, nil + } + + if os.Getenv("ATMOS_REDIS_URL") != "" { + return redis.ParseURL(os.Getenv("ATMOS_REDIS_URL")) + } + + return &redis.Options{}, fmt.Errorf("either url must be set in options or REDIS_URL environment variable must be set") +} + +func NewRedisStore(options RedisStoreOptions) (Store, error) { + prefix := "" + if options.Prefix != nil { + prefix = *options.Prefix + } + + stackDelimiter := "/" + if options.StackDelimiter != nil { + stackDelimiter = *options.StackDelimiter + } + + opts, err := getRedisOptions(&options) + if err != nil { + return nil, fmt.Errorf("failed to parse redis url: %v", err) + } + + redisClient := redis.NewClient(opts) + + return &RedisStore{ + prefix: prefix, + redisClient: redisClient, + stackDelimiter: &stackDelimiter, + }, nil +} + +func (s *RedisStore) getKey(stack string, component string, key string) (string, error) { + if s.stackDelimiter == nil { + return "", fmt.Errorf("stack delimiter is not set") + } + + prefixParts := []string{s.repoName, s.prefix} + prefix := strings.Join(prefixParts, "/") + + return getKey(prefix, *s.stackDelimiter, stack, component, key, "/") +} + +func (s *RedisStore) Get(stack string, component string, key string) (interface{}, error) { + if stack == "" { + return nil, fmt.Errorf("stack cannot be empty") + } + + if component == "" { + return nil, fmt.Errorf("component cannot be empty") + } + + if key == "" { + return nil, fmt.Errorf("key cannot be empty") + } + + paramName, err := s.getKey(stack, component, key) + if err != nil { + return nil, fmt.Errorf("failed to get key: %v", err) + } + + ctx := context.Background() + jsonData, err := s.redisClient.Get(ctx, paramName).Result() + if err != nil { + return nil, fmt.Errorf("failed to get key: %v", err) + } + + var result interface{} + err = json.Unmarshal([]byte(jsonData), &result) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal file: %v", err) + } + + return result, nil +} + +func (s *RedisStore) Set(stack string, component string, key string, value interface{}) error { + if stack == "" { + return fmt.Errorf("stack cannot be empty") + } + + if component == "" { + return fmt.Errorf("component cannot be empty") + } + + if key == "" { + return fmt.Errorf("key cannot be empty") + } + + // Construct the full parameter name using getKey + paramName, err := s.getKey(stack, component, key) + if err != nil { + return fmt.Errorf("failed to get key: %v", err) + } + + jsonData, err := json.Marshal(value) + if err != nil { + return fmt.Errorf("failed to marshal value: %v", err) + } + + ctx := context.Background() + err = s.redisClient.Set(ctx, paramName, jsonData, 0).Err() + + return err +} diff --git a/pkg/store/redis_store_test.go b/pkg/store/redis_store_test.go new file mode 100644 index 0000000000..fa860e3d59 --- /dev/null +++ b/pkg/store/redis_store_test.go @@ -0,0 +1,243 @@ +package store + +import ( + "context" + "encoding/json" + "testing" + "time" + + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" +) + +// MockRedisClient is a mock implementation of the RedisClient interface. +type MockRedisClient struct { + mock.Mock +} + +func (m *MockRedisClient) Get(ctx context.Context, key string) *redis.StringCmd { + args := m.Called(ctx, key) + cmd := redis.NewStringResult(args.String(0), args.Error(1)) + return cmd +} + +func (m *MockRedisClient) Set(ctx context.Context, key string, value interface{}, expiration time.Duration) *redis.StatusCmd { + args := m.Called(ctx, key, value, expiration) + cmd := redis.NewStatusResult(args.String(0), args.Error(1)) + return cmd +} + +func ptr(s string) *string { + return &s +} + +func TestRedisStore_Get_Success(t *testing.T) { + // Arrange + mockClient := new(MockRedisClient) + store, err := NewRedisStore(RedisStoreOptions{ + Prefix: ptr("testprefix"), + StackDelimiter: ptr("/"), + URL: ptr("redis://localhost:6379"), + }) + assert.NoError(t, err) + + // Replace the real Redis client with the mock + redisStore, ok := store.(*RedisStore) + assert.True(t, ok) + redisStore.redisClient = mockClient + redisStore.repoName = "repo" + + stack := "mystack" + component := "mycomponent" + key := "mykey" + fullKey := "repo/testprefix/mystack/mycomponent/mykey" + + expectedValue := map[string]interface{}{ + "field": "value", + } + + jsonData, _ := json.Marshal(expectedValue) + + // Set up the expected calls and return values + mockClient.On("Get", context.Background(), fullKey).Return(string(jsonData), nil) + + // Act + result, err := redisStore.Get(stack, component, key) + + // Assert + assert.NoError(t, err) + assert.Equal(t, expectedValue, result) + mockClient.AssertExpectations(t) +} + +func TestRedisStore_Get_KeyNotFound(t *testing.T) { + // Arrange + mockClient := new(MockRedisClient) + store, err := NewRedisStore(RedisStoreOptions{ + Prefix: ptr("testprefix"), + StackDelimiter: ptr("/"), + URL: ptr("redis://localhost:6379"), + }) + assert.NoError(t, err) + + redisStore, ok := store.(*RedisStore) + assert.True(t, ok) + redisStore.redisClient = mockClient + redisStore.repoName = "repo" + + stack := "mystack" + component := "mycomponent" + key := "mykey" + fullKey := "repo/testprefix/mystack/mycomponent/mykey" + + // Set up the expected calls and return values + mockClient.On("Get", context.Background(), fullKey).Return("", redis.Nil) + + // Act + result, err := redisStore.Get(stack, component, key) + + // Assert + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "failed to get key") + mockClient.AssertExpectations(t) +} + +func TestRedisStore_Set_Success(t *testing.T) { + // Arrange + mockClient := new(MockRedisClient) + store, err := NewRedisStore(RedisStoreOptions{ + Prefix: ptr("testprefix"), + StackDelimiter: ptr("/"), + URL: ptr("redis://localhost:6379"), + }) + assert.NoError(t, err) + + redisStore, ok := store.(*RedisStore) + assert.True(t, ok) + redisStore.redisClient = mockClient + redisStore.repoName = "repo" + + stack := "mystack" + component := "mycomponent" + key := "mykey" + fullKey := "repo/testprefix/mystack/mycomponent/mykey" + + value := map[string]interface{}{ + "field": "value", + } + + jsonData, _ := json.Marshal(value) + + // Set up the expected calls and return values + mockClient.On("Set", context.Background(), fullKey, jsonData, time.Duration(0)).Return("OK", nil) + + // Act + err = redisStore.Set(stack, component, key, value) + + // Assert + assert.NoError(t, err) + mockClient.AssertExpectations(t) +} + +func TestRedisStore_Set_MarshalError(t *testing.T) { + // Arrange + mockClient := new(MockRedisClient) + store, err := NewRedisStore(RedisStoreOptions{ + Prefix: ptr("testprefix"), + StackDelimiter: ptr("/"), + URL: ptr("redis://localhost:6379"), + }) + assert.NoError(t, err) + + redisStore, ok := store.(*RedisStore) + assert.True(t, ok) + redisStore.redisClient = mockClient + redisStore.repoName = "repo" + + stack := "mystack" + component := "mycomponent" + key := "mykey" + + // Create a value that cannot be marshaled to JSON (e.g., a channel) + value := make(chan int) + + // Act + err = redisStore.Set(stack, component, key, value) + + // Assert + assert.Error(t, err) + assert.Contains(t, err.Error(), "failed to marshal value") +} + +func TestRedisStore_Get_UnmarshalError(t *testing.T) { + // Arrange + mockClient := new(MockRedisClient) + store, err := NewRedisStore(RedisStoreOptions{ + Prefix: ptr("testprefix"), + StackDelimiter: ptr("/"), + URL: ptr("redis://localhost:6379"), + }) + assert.NoError(t, err) + + redisStore, ok := store.(*RedisStore) + assert.True(t, ok) + redisStore.redisClient = mockClient + redisStore.repoName = "repo" + + stack := "mystack" + component := "mycomponent" + key := "mykey" + fullKey := "repo/testprefix/mystack/mycomponent/mykey" + + invalidJSON := "invalid_json" + + // Set up the expected calls and return values + mockClient.On("Get", context.Background(), fullKey).Return(invalidJSON, nil) + + // Act + result, err := redisStore.Get(stack, component, key) + + // Assert + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "failed to unmarshal file") + mockClient.AssertExpectations(t) +} + +func TestRedisStore_Get_GetKeyError(t *testing.T) { + // Arrange + mockClient := new(MockRedisClient) + store, err := NewRedisStore(RedisStoreOptions{ + Prefix: nil, // Prefix is nil + StackDelimiter: ptr("/"), + URL: ptr("redis://localhost:6379"), + }) + assert.NoError(t, err) + + redisStore, ok := store.(*RedisStore) + assert.True(t, ok) + redisStore.redisClient = mockClient + // repoName is not set, so prefixParts = ["", ""] and prefix = "/" + // Hence, the full key becomes "/mystack/mycomponent/mykey" + + stack := "mystack" + component := "mycomponent" + key := "mykey" + + // Expected full key based on getKey implementation + fullKey := "/mystack/mycomponent/mykey" + + // Set up the expected call to redisClient.Get with fullKey and return redis.Nil to simulate key not found + mockClient.On("Get", context.Background(), fullKey).Return("", redis.Nil) + + // Act + result, err := redisStore.Get(stack, component, key) + + // Assert + assert.Error(t, err) + assert.Nil(t, result) + assert.Contains(t, err.Error(), "failed to get key") + mockClient.AssertExpectations(t) +} diff --git a/pkg/store/registry.go b/pkg/store/registry.go index ae0a123f8c..88614ef783 100644 --- a/pkg/store/registry.go +++ b/pkg/store/registry.go @@ -1,6 +1,8 @@ package store -import "fmt" +import ( + "fmt" +) type StoreRegistry map[string]Store @@ -32,6 +34,18 @@ func NewStoreRegistry(config *StoresConfig) (StoreRegistry, error) { } registry[key] = store + case "redis": + var opts RedisStoreOptions + if err := parseOptions(storeConfig.Options, &opts); err != nil { + return nil, fmt.Errorf("failed to parse Redis store options: %w", err) + } + + store, err := NewRedisStore(opts) + if err != nil { + return nil, err + } + registry[key] = store + default: return nil, fmt.Errorf("store type %s not found", storeConfig.Type) } diff --git a/pkg/store/store.go b/pkg/store/store.go index dfe8aa0302..02fbcfcbd5 100644 --- a/pkg/store/store.go +++ b/pkg/store/store.go @@ -1,5 +1,7 @@ package store +import "strings" + // Store defines the common interface for all store implementations. type Store interface { Set(stack string, component string, key string, value interface{}) error @@ -8,3 +10,21 @@ type Store interface { // StoreFactory is a function type to initialize a new store. type StoreFactory func(options map[string]interface{}) (Store, error) + +// getKey generates a key for the store. First it splits the stack by the stack delimiter (from atmos.yaml) +// then it splits the component if it containts a "/" +// then it appends the key to the parts +// then it joins the parts with the final delimiter +func getKey(prefix string, stackDelimiter string, stack string, component string, key string, finalDelimiter string) (string, error) { + stackParts := strings.Split(stack, stackDelimiter) + componentParts := strings.Split(component, "/") + + parts := append([]string{prefix}, stackParts...) + parts = append(parts, componentParts...) + parts = append(parts, key) + + joinedKey := strings.Join(parts, finalDelimiter) + finalKey := strings.ReplaceAll(joinedKey, "//", "/") + + return finalKey, nil +} diff --git a/website/docs/core-concepts/projects/configuration/stores.mdx b/website/docs/core-concepts/projects/configuration/stores.mdx index 026a7e1390..41b2acf70b 100644 --- a/website/docs/core-concepts/projects/configuration/stores.mdx +++ b/website/docs/core-concepts/projects/configuration/stores.mdx @@ -17,6 +17,7 @@ Currently, the following stores are supported: - [AWS SSM Parameter Store](https://docs.aws.amazon.com/systems-manager/latest/userguide/systems-manager-parameter-store.html) - [Artifactory](https://jfrog.com/artifactory/) +- [Redis](https://redis.io/) Atmos stores are configured in the `atmos.yaml` file and available to use in stacks via the @@ -69,6 +70,12 @@ stores:
`stores.[store_name].options.url (required)`
The URL of the Artifactory instance.
+ +
`stores.[store_name].options.stacks_delimiter (optional)`
+
+ The delimiter that atmos is using to delimit stacks in the key path. This defaults to `-`. This is used to build the + key path for the store. +
#### Authentication @@ -108,6 +115,12 @@ stores:
`stores.[store_name].options.region (required)`
The AWS region to use for the SSM Parameter Store.
+ +
`stores.[store_name].options.stacks_delimiter (optional)`
+
+ The delimiter that atmos is using to delimit stacks in the key path. This defaults to `-`. This is used to build the + key path for the store. +
#### Authentication @@ -115,3 +128,54 @@ stores: The AWS SSM Parameter Store supports the standard AWS methods for authentication and the `AWS_ACCESS_KEY_ID`, `AWS_SECRET_ACCESS_KEY`, and `AWS_SESSION_TOKEN` environment variables. +### Redis + +```yaml +stores: + dev/redis: + type: redis + options: + url: redis://localhost:6379 + + stage/redis: + type: redis + options: + url: !env ATMOS_STAGE_REDIS_URL + + prod/redis: + type: redis + # The ATMOS_REDIS_URL environment variable will be used if no URL is specified in the options +``` + +
+
`stores.[store_name]`
+
This map key is the name of the store. It must be unique across all stores. This is how the store is referenced in the `store` function.
+ +
`stores.[store_name].type`
+
Must be set to `redis`
+ +
`stores.[store_name].options`
+
A map of options specific to the store type. For Redis, the following options are supported:
+ +
`stores.[store_name].options.prefix (optional)`
+
A prefix path that will be added to all keys stored or retreived from Redis. For example if the prefix + is `/atmos/infra-live/`, and if the stack is `plat-us2-dev`, the component is `vpc`, and the key is `vpc_id`, the full path + would be `/atmos/infra-live/plat-us2-dev/vpc/vpc_id`.
+ +
`stores.[store_name].options.url`
+
+ The URL of the Redis instance. This is optional and the `ATMOS_REDIS_URL` environment variable will be used if no + URL is specified in the options. +
+ +
`stores.[store_name].options.stacks_delimiter (optional)`
+
+ The delimiter that atmos is using to delimit stacks in the key path. This defaults to `-`. This is used to build the + key path for the store. +
+
+ +#### Authentication + +The Redis store supports authentication via the URL in options or via the `ATMOS_REDIS_URL` environment variable. The +URL format is described in the Redis [docs](https://redis.github.io/lettuce/user-guide/connecting-redis/).