diff --git a/.env.example b/.env.example index 72536cea..35496e05 100644 --- a/.env.example +++ b/.env.example @@ -10,6 +10,9 @@ HMAC_TIMESTAMP_AGE=5 ENVIRONMENT=local # local, staging, production SENTRY_DSN= HOST_DOMAIN=http://localhost:8000 +CURRENCIES_CACHE_DURATION=24 +PUBKEY_CACHE_DURATION=365 +INSTITUTIONS_CACHE_DURATION=24 # Database Config DB_NAME=paycrest @@ -25,6 +28,7 @@ REDIS_HOST=localhost REDIS_PORT=6379 REDIS_PASSWORD= REDIS_DB=0 +CACHE_VERSION=v1 # Order Config ORDER_FULFILLMENT_VALIDITY=10 # value in minutes diff --git a/config/redis.go b/config/redis.go index e7d32e56..00132d87 100644 --- a/config/redis.go +++ b/config/redis.go @@ -12,6 +12,7 @@ type RedisConfiguration struct { Port string Password string DB int + CacheVersion string } // RedisConfig retrieves the Redis configuration @@ -21,6 +22,7 @@ func RedisConfig() RedisConfiguration { Port: viper.GetString("REDIS_PORT"), Password: viper.GetString("REDIS_PASSWORD"), DB: viper.GetInt("REDIS_DB"), + CacheVersion: viper.GetString("CACHE_VERSION"), } } diff --git a/config/server.go b/config/server.go index 93119e5c..70221c0f 100644 --- a/config/server.go +++ b/config/server.go @@ -8,14 +8,17 @@ import ( // ServerConfiguration type defines the server configurations type ServerConfiguration struct { - Debug bool - Host string - Port string - Timezone string - AllowedHosts string - Environment string - SentryDSN string - HostDomain string + Debug bool + Host string + Port string + Timezone string + AllowedHosts string + Environment string + SentryDSN string + HostDomain string + CurrenciesCacheDuration int + InstitutionsCacheDuration int + PubKeyCacheDuration int } // ServerConfig sets the server configuration @@ -27,16 +30,22 @@ func ServerConfig() *ServerConfiguration { viper.SetDefault("ALLOWED_HOSTS", "*") viper.SetDefault("ENVIRONMENT", "local") viper.SetDefault("SENTRY_DSN", "") + viper.SetDefault("CURRENCIES_CACHE_DURATION", 24) + viper.SetDefault("INSTITUTIONS_CACHE_DURATION", 24) + viper.SetDefault("PUBKEY_CACHE_DURATION", 365) return &ServerConfiguration{ - Debug: viper.GetBool("DEBUG"), - Host: viper.GetString("SERVER_HOST"), - Port: viper.GetString("SERVER_PORT"), - Timezone: viper.GetString("SERVER_TIMEZONE"), - AllowedHosts: viper.GetString("ALLOWED_HOSTS"), - Environment: viper.GetString("ENVIRONMENT"), - SentryDSN: viper.GetString("SENTRY_DSN"), - HostDomain: viper.GetString("HOST_DOMAIN"), + Debug: viper.GetBool("DEBUG"), + Host: viper.GetString("SERVER_HOST"), + Port: viper.GetString("SERVER_PORT"), + Timezone: viper.GetString("SERVER_TIMEZONE"), + AllowedHosts: viper.GetString("ALLOWED_HOSTS"), + Environment: viper.GetString("ENVIRONMENT"), + SentryDSN: viper.GetString("SENTRY_DSN"), + HostDomain: viper.GetString("HOST_DOMAIN"), + CurrenciesCacheDuration: viper.GetInt("CURRENCIES_CACHE_DURATION"), + InstitutionsCacheDuration: viper.GetInt("INSTITUTIONS_CACHE_DURATION"), + PubKeyCacheDuration: viper.GetInt("PUBKEY_CACHE_DURATION"), } } diff --git a/go.mod b/go.mod index 9d5d126e..c33f15f5 100644 --- a/go.mod +++ b/go.mod @@ -3,13 +3,13 @@ module github.com/paycrest/protocol go 1.22.0 require ( + github.com/alicebob/miniredis/v2 v2.34.0 github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4 github.com/btcsuite/btcd/btcec/v2 v2.3.2 github.com/chadsr/logrus-sentry v0.4.1 github.com/getsentry/sentry-go v0.13.0 github.com/gin-gonic/gin v1.9.1 github.com/go-co-op/gocron v1.35.0 - github.com/go-redis/redismock/v9 v9.2.0 github.com/golang-jwt/jwt/v5 v5.0.0 github.com/jarcoal/httpmock v1.3.1 github.com/mailgun/mailgun-go/v3 v3.6.4 @@ -31,6 +31,7 @@ require ( github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e // indirect github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec // indirect github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/btcsuite/btcd v0.22.1 // indirect @@ -84,6 +85,7 @@ require ( github.com/tklauser/go-sysconf v0.3.12 // indirect github.com/tklauser/numcpus v0.6.1 // indirect github.com/tyler-smith/go-bip32 v1.0.0 // indirect + github.com/yuin/gopher-lua v1.1.1 // indirect github.com/yusufpapurcu/wmi v1.2.3 // indirect go.uber.org/atomic v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect diff --git a/go.sum b/go.sum index 4dbb0442..def291a3 100644 --- a/go.sum +++ b/go.sum @@ -63,6 +63,10 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= +github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= +github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4 h1:FUDNaUiPOxrVtUmsRSdx7hrvCKXpfQafPpPU0Yh27os= github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4/go.mod h1:glPG1rmt/bD3wEXWanFIuoPjC4MG+JEN+i7YhwEYA/Y= github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw= @@ -237,8 +241,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY= github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw= github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU= -github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw= -github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68= @@ -485,19 +487,13 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= -github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= -github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= -github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= -github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y= -github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/opus-domini/fast-shot v0.10.0 h1:zWbPy6KJZvNs0pUa0erF9TyeDsLZHDVZf4oDHOd6JGY= github.com/opus-domini/fast-shot v0.10.0/go.mod h1:sg5+f0VviAIIdrv24WLHL6kV7kWs4PsVDbSkr2TPYWw= github.com/paycrest/tron-wallet v1.0.13 h1:TEkjovg6i2zBTZFMfYpBrwswwnYAy6Q/Vc6g12PZh54= @@ -643,6 +639,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= +github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw= github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0= @@ -1028,7 +1026,6 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/routers/middleware/caching.go b/routers/middleware/caching.go index 072e11bb..b14c099f 100644 --- a/routers/middleware/caching.go +++ b/routers/middleware/caching.go @@ -4,6 +4,7 @@ import ( "context" "crypto/sha256" "encoding/hex" + "encoding/json" "fmt" "io/ioutil" "net/http" @@ -63,16 +64,17 @@ func NewCacheService(config config.RedisConfiguration) (*CacheService, error) { } func generateCacheKey(c *gin.Context) string { + conf := config.RedisConfig() path := c.Request.URL.Path switch { case path == "/v1/currencies": - return "api:currencies:list" + return fmt.Sprintf("%s:api:currencies:list", conf.CacheVersion) case path == "/v1/pubkey": - return "api:aggregator:pubkey" + return fmt.Sprintf("%s:api:aggregator:pubkey", conf.CacheVersion) case len(c.Param("currency_code")) > 0: - return fmt.Sprintf("api:institutions:%s", c.Param("currency_code")) + return fmt.Sprintf("%s:api:institutions:%s", conf.CacheVersion, c.Param("currency_code")) default: - return fmt.Sprintf("api:v1:%s:%s", c.Request.Method, path) + return fmt.Sprintf("%s:api:%s", conf.CacheVersion, path) } } @@ -142,18 +144,44 @@ func (w *cacheWriter) Write(b []byte) (int, error) { func (s *CacheService) WarmCache(ctx context.Context) error { conf := config.ServerConfig() baseURL := conf.HostDomain - if conf.HostDomain == "" { + if baseURL == "" { return fmt.Errorf("host domain is not set in the server configuration") } + // Fetch the list of supported currencies with a timeout + client := &http.Client{Timeout: 10 * time.Second} + currenciesURL := fmt.Sprintf("%s/v1/currencies", baseURL) + resp, err := client.Get(currenciesURL) + if err != nil { + return fmt.Errorf("failed to fetch currencies from %s: %v", currenciesURL, err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("failed to fetch currencies: status code %d", resp.StatusCode) + } + + var currencies []string + if err := json.NewDecoder(resp.Body).Decode(¤cies); err != nil { + return fmt.Errorf("failed to decode currencies response: %v", err) + } + + if len(currencies) == 0 { + fmt.Println("No currencies found. Using default currencies [USD, EUR, GBP].") + currencies = []string{"USD", "EUR", "GBP"} + } + + // Define static and dynamic endpoints endpoints := map[string]time.Duration{ - "currencies": 24 * time.Hour, - "pubkey": 365 * time.Hour, - "institutions/USD": 24 * time.Hour, - "institutions/EUR": 24 * time.Hour, - "institutions/GBP": 24 * time.Hour, + "currencies": time.Duration(conf.CurrenciesCacheDuration) * time.Hour, + "pubkey": time.Duration(conf.PubKeyCacheDuration) * time.Hour, + } + + for _, currency := range currencies { + endpoints[fmt.Sprintf("institutions/%s", currency)] = time.Duration(conf.InstitutionsCacheDuration) * time.Hour } + // Warm up cache for each endpoint for path, duration := range endpoints { urlStr := fmt.Sprintf("%s/v1/%s", baseURL, path) parsedURL, err := url.Parse(urlStr) diff --git a/routers/middleware/caching_test.go b/routers/middleware/caching_test.go new file mode 100644 index 00000000..3bed4953 --- /dev/null +++ b/routers/middleware/caching_test.go @@ -0,0 +1,102 @@ +package middleware + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/alicebob/miniredis/v2" + "github.com/gin-gonic/gin" + "github.com/paycrest/protocol/config" + "github.com/prometheus/client_golang/prometheus" + "github.com/redis/go-redis/v9" + "github.com/stretchr/testify/assert" +) + +func setupTestRedis() (*miniredis.Miniredis, *redis.Client) { + mr, err := miniredis.Run() + if err != nil { + panic(err) + } + + client := redis.NewClient(&redis.Options{ + Addr: mr.Addr(), + }) + + return mr, client +} + +func TestCacheMiddleware(t *testing.T) { + mr, client := setupTestRedis() + defer mr.Close() + + cacheService := &CacheService{ + client: client, + metrics: CacheMetrics{ + hits: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_hits_total"}), + misses: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_misses_total"}), + }, + } + + router := gin.Default() + router.GET("/v1/currencies", cacheService.CacheMiddleware(24*time.Hour), func(c *gin.Context) { + c.JSON(http.StatusOK, gin.H{"currencies": []string{"USD", "EUR", "GBP"}}) + }) + + // First request should be a cache miss + w := httptest.NewRecorder() + req, _ := http.NewRequest("GET", "/v1/currencies", nil) + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "MISS", w.Header().Get("X-Cache")) + + // Second request should be a cache hit + w = httptest.NewRecorder() + router.ServeHTTP(w, req) + + assert.Equal(t, http.StatusOK, w.Code) + assert.Equal(t, "HIT", w.Header().Get("X-Cache")) +} + +func TestWarmCache(t *testing.T) { + mr, client := setupTestRedis() + defer mr.Close() + + cacheService := &CacheService{ + client: client, + metrics: CacheMetrics{ + hits: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_hits_total"}), + misses: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_misses_total"}), + }, + } + + // Mock server to return currencies + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path == "/v1/currencies" { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode([]string{"USD", "EUR", "GBP"}) + } + })) + defer mockServer.Close() + + // Override the base URL in the server configuration + conf := config.ServerConfig() + conf.HostDomain = mockServer.URL + + ctx := context.Background() + err := cacheService.WarmCache(ctx) + assert.NoError(t, err) + + // Verify that the currencies are cached + for _, currency := range []string{"USD", "EUR", "GBP"} { + key := fmt.Sprintf("%s:api:institutions:%s", conf.CacheVersion, currency) + val, err := client.Get(ctx, key).Result() + assert.NoError(t, err) + assert.NotEmpty(t, val) + } +}