Skip to content

Commit

Permalink
feat: enhance caching mechanism with configurable durations and versi…
Browse files Browse the repository at this point in the history
…oning
  • Loading branch information
sundayonah committed Jan 22, 2025
1 parent 64ebee8 commit 27b22f8
Show file tree
Hide file tree
Showing 7 changed files with 180 additions and 36 deletions.
4 changes: 4 additions & 0 deletions .env.example
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,9 @@ HMAC_TIMESTAMP_AGE=5
ENVIRONMENT=local # local, staging, production
SENTRY_DSN=
HOST_DOMAIN=http://localhost:8000
CURRENCIES_CACHE_DURATION=24
PUBKEY_CACHE_DURATION=365
INSTITUTIONS_CACHE_DURATION=24

# Database Config
DB_NAME=paycrest
Expand All @@ -25,6 +28,7 @@ REDIS_HOST=localhost
REDIS_PORT=6379
REDIS_PASSWORD=
REDIS_DB=0
CACHE_VERSION=v1

# Order Config
ORDER_FULFILLMENT_VALIDITY=10 # value in minutes
Expand Down
2 changes: 2 additions & 0 deletions config/redis.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ type RedisConfiguration struct {
Port string
Password string
DB int
CacheVersion string
}

// RedisConfig retrieves the Redis configuration
Expand All @@ -21,6 +22,7 @@ func RedisConfig() RedisConfiguration {
Port: viper.GetString("REDIS_PORT"),
Password: viper.GetString("REDIS_PASSWORD"),
DB: viper.GetInt("REDIS_DB"),
CacheVersion: viper.GetString("CACHE_VERSION"),
}
}

Expand Down
41 changes: 25 additions & 16 deletions config/server.go
Original file line number Diff line number Diff line change
Expand Up @@ -8,14 +8,17 @@ import (

// ServerConfiguration type defines the server configurations
type ServerConfiguration struct {
Debug bool
Host string
Port string
Timezone string
AllowedHosts string
Environment string
SentryDSN string
HostDomain string
Debug bool
Host string
Port string
Timezone string
AllowedHosts string
Environment string
SentryDSN string
HostDomain string
CurrenciesCacheDuration int
InstitutionsCacheDuration int
PubKeyCacheDuration int
}

// ServerConfig sets the server configuration
Expand All @@ -27,16 +30,22 @@ func ServerConfig() *ServerConfiguration {
viper.SetDefault("ALLOWED_HOSTS", "*")
viper.SetDefault("ENVIRONMENT", "local")
viper.SetDefault("SENTRY_DSN", "")
viper.SetDefault("CURRENCIES_CACHE_DURATION", 24)
viper.SetDefault("INSTITUTIONS_CACHE_DURATION", 24)
viper.SetDefault("PUBKEY_CACHE_DURATION", 365)

return &ServerConfiguration{
Debug: viper.GetBool("DEBUG"),
Host: viper.GetString("SERVER_HOST"),
Port: viper.GetString("SERVER_PORT"),
Timezone: viper.GetString("SERVER_TIMEZONE"),
AllowedHosts: viper.GetString("ALLOWED_HOSTS"),
Environment: viper.GetString("ENVIRONMENT"),
SentryDSN: viper.GetString("SENTRY_DSN"),
HostDomain: viper.GetString("HOST_DOMAIN"),
Debug: viper.GetBool("DEBUG"),
Host: viper.GetString("SERVER_HOST"),
Port: viper.GetString("SERVER_PORT"),
Timezone: viper.GetString("SERVER_TIMEZONE"),
AllowedHosts: viper.GetString("ALLOWED_HOSTS"),
Environment: viper.GetString("ENVIRONMENT"),
SentryDSN: viper.GetString("SENTRY_DSN"),
HostDomain: viper.GetString("HOST_DOMAIN"),
CurrenciesCacheDuration: viper.GetInt("CURRENCIES_CACHE_DURATION"),
InstitutionsCacheDuration: viper.GetInt("INSTITUTIONS_CACHE_DURATION"),
PubKeyCacheDuration: viper.GetInt("PUBKEY_CACHE_DURATION"),
}
}

Expand Down
4 changes: 3 additions & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,13 @@ module github.com/paycrest/protocol
go 1.22.0

require (
github.com/alicebob/miniredis/v2 v2.34.0
github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4
github.com/btcsuite/btcd/btcec/v2 v2.3.2
github.com/chadsr/logrus-sentry v0.4.1
github.com/getsentry/sentry-go v0.13.0
github.com/gin-gonic/gin v1.9.1
github.com/go-co-op/gocron v1.35.0
github.com/go-redis/redismock/v9 v9.2.0
github.com/golang-jwt/jwt/v5 v5.0.0
github.com/jarcoal/httpmock v1.3.1
github.com/mailgun/mailgun-go/v3 v3.6.4
Expand All @@ -31,6 +31,7 @@ require (
github.com/FactomProject/basen v0.0.0-20150613233007-fe3947df716e // indirect
github.com/FactomProject/btcutilecc v0.0.0-20130527213604-d3a63a5752ec // indirect
github.com/Microsoft/go-winio v0.6.1 // indirect
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 // indirect
github.com/beorn7/perks v1.0.1 // indirect
github.com/bits-and-blooms/bitset v1.7.0 // indirect
github.com/btcsuite/btcd v0.22.1 // indirect
Expand Down Expand Up @@ -84,6 +85,7 @@ require (
github.com/tklauser/go-sysconf v0.3.12 // indirect
github.com/tklauser/numcpus v0.6.1 // indirect
github.com/tyler-smith/go-bip32 v1.0.0 // indirect
github.com/yuin/gopher-lua v1.1.1 // indirect
github.com/yusufpapurcu/wmi v1.2.3 // indirect
go.uber.org/atomic v1.9.0 // indirect
golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
Expand Down
15 changes: 6 additions & 9 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -63,6 +63,10 @@ github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBA
github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo=
github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558=
github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY=
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE=
github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0=
github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8=
github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4 h1:FUDNaUiPOxrVtUmsRSdx7hrvCKXpfQafPpPU0Yh27os=
github.com/anaskhan96/base58check v0.0.0-20181220122047-b05365d494c4/go.mod h1:glPG1rmt/bD3wEXWanFIuoPjC4MG+JEN+i7YhwEYA/Y=
github.com/apparentlymart/go-textseg/v13 v13.0.0 h1:Y+KvPE1NYz0xl601PVImeQfFyEy6iT90AvPUL1NNfNw=
Expand Down Expand Up @@ -237,8 +241,6 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw=
github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw=
github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0=
github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
Expand Down Expand Up @@ -485,19 +487,13 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl
github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
github.com/opus-domini/fast-shot v0.10.0 h1:zWbPy6KJZvNs0pUa0erF9TyeDsLZHDVZf4oDHOd6JGY=
github.com/opus-domini/fast-shot v0.10.0/go.mod h1:sg5+f0VviAIIdrv24WLHL6kV7kWs4PsVDbSkr2TPYWw=
github.com/paycrest/tron-wallet v1.0.13 h1:TEkjovg6i2zBTZFMfYpBrwswwnYAy6Q/Vc6g12PZh54=
Expand Down Expand Up @@ -643,6 +639,8 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M=
github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
github.com/yusufpapurcu/wmi v1.2.3 h1:E1ctvB7uKFMOJw3fdOW32DwGE9I7t++CRUEMKvFoFiw=
github.com/yusufpapurcu/wmi v1.2.3/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0=
github.com/zclconf/go-cty v1.13.2 h1:4GvrUxe/QUDYuJKAav4EYqdM47/kZa672LwmXFmEKT0=
Expand Down Expand Up @@ -1028,7 +1026,6 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
Expand Down
48 changes: 38 additions & 10 deletions routers/middleware/caching.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"context"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
Expand Down Expand Up @@ -63,16 +64,17 @@ func NewCacheService(config config.RedisConfiguration) (*CacheService, error) {
}

func generateCacheKey(c *gin.Context) string {
conf := config.RedisConfig()
path := c.Request.URL.Path
switch {
case path == "/v1/currencies":
return "api:currencies:list"
return fmt.Sprintf("%s:api:currencies:list", conf.CacheVersion)
case path == "/v1/pubkey":
return "api:aggregator:pubkey"
return fmt.Sprintf("%s:api:aggregator:pubkey", conf.CacheVersion)
case len(c.Param("currency_code")) > 0:
return fmt.Sprintf("api:institutions:%s", c.Param("currency_code"))
return fmt.Sprintf("%s:api:institutions:%s", conf.CacheVersion, c.Param("currency_code"))
default:
return fmt.Sprintf("api:v1:%s:%s", c.Request.Method, path)
return fmt.Sprintf("%s:api:%s", conf.CacheVersion, path)
}
}

Expand Down Expand Up @@ -142,18 +144,44 @@ func (w *cacheWriter) Write(b []byte) (int, error) {
func (s *CacheService) WarmCache(ctx context.Context) error {
conf := config.ServerConfig()
baseURL := conf.HostDomain
if conf.HostDomain == "" {
if baseURL == "" {
return fmt.Errorf("host domain is not set in the server configuration")
}

// Fetch the list of supported currencies with a timeout
client := &http.Client{Timeout: 10 * time.Second}
currenciesURL := fmt.Sprintf("%s/v1/currencies", baseURL)
resp, err := client.Get(currenciesURL)
if err != nil {
return fmt.Errorf("failed to fetch currencies from %s: %v", currenciesURL, err)
}
defer resp.Body.Close()

if resp.StatusCode != http.StatusOK {
return fmt.Errorf("failed to fetch currencies: status code %d", resp.StatusCode)
}

var currencies []string
if err := json.NewDecoder(resp.Body).Decode(&currencies); err != nil {
return fmt.Errorf("failed to decode currencies response: %v", err)
}

if len(currencies) == 0 {
fmt.Println("No currencies found. Using default currencies [USD, EUR, GBP].")
currencies = []string{"USD", "EUR", "GBP"}
}

// Define static and dynamic endpoints
endpoints := map[string]time.Duration{
"currencies": 24 * time.Hour,
"pubkey": 365 * time.Hour,
"institutions/USD": 24 * time.Hour,
"institutions/EUR": 24 * time.Hour,
"institutions/GBP": 24 * time.Hour,
"currencies": time.Duration(conf.CurrenciesCacheDuration) * time.Hour,
"pubkey": time.Duration(conf.PubKeyCacheDuration) * time.Hour,
}

for _, currency := range currencies {
endpoints[fmt.Sprintf("institutions/%s", currency)] = time.Duration(conf.InstitutionsCacheDuration) * time.Hour
}

// Warm up cache for each endpoint
for path, duration := range endpoints {
urlStr := fmt.Sprintf("%s/v1/%s", baseURL, path)
parsedURL, err := url.Parse(urlStr)
Expand Down
102 changes: 102 additions & 0 deletions routers/middleware/caching_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,102 @@
package middleware

import (
"context"
"encoding/json"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"

"github.com/alicebob/miniredis/v2"
"github.com/gin-gonic/gin"
"github.com/paycrest/protocol/config"
"github.com/prometheus/client_golang/prometheus"
"github.com/redis/go-redis/v9"
"github.com/stretchr/testify/assert"
)

func setupTestRedis() (*miniredis.Miniredis, *redis.Client) {
mr, err := miniredis.Run()
if err != nil {
panic(err)
}

client := redis.NewClient(&redis.Options{
Addr: mr.Addr(),
})

return mr, client
}

func TestCacheMiddleware(t *testing.T) {
mr, client := setupTestRedis()
defer mr.Close()

cacheService := &CacheService{
client: client,
metrics: CacheMetrics{
hits: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_hits_total"}),
misses: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_misses_total"}),
},
}

router := gin.Default()
router.GET("/v1/currencies", cacheService.CacheMiddleware(24*time.Hour), func(c *gin.Context) {
c.JSON(http.StatusOK, gin.H{"currencies": []string{"USD", "EUR", "GBP"}})
})

// First request should be a cache miss
w := httptest.NewRecorder()
req, _ := http.NewRequest("GET", "/v1/currencies", nil)
router.ServeHTTP(w, req)

assert.Equal(t, http.StatusOK, w.Code)
assert.Equal(t, "MISS", w.Header().Get("X-Cache"))

// Second request should be a cache hit
w = httptest.NewRecorder()
router.ServeHTTP(w, req)

assert.Equal(t, http.StatusOK, w.Code)
assert.Equal(t, "HIT", w.Header().Get("X-Cache"))
}

func TestWarmCache(t *testing.T) {
mr, client := setupTestRedis()
defer mr.Close()

cacheService := &CacheService{
client: client,
metrics: CacheMetrics{
hits: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_hits_total"}),
misses: prometheus.NewCounter(prometheus.CounterOpts{Name: "cache_misses_total"}),
},
}

// Mock server to return currencies
mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.URL.Path == "/v1/currencies" {
w.Header().Set("Content-Type", "application/json")
json.NewEncoder(w).Encode([]string{"USD", "EUR", "GBP"})
}
}))
defer mockServer.Close()

// Override the base URL in the server configuration
conf := config.ServerConfig()
conf.HostDomain = mockServer.URL

ctx := context.Background()
err := cacheService.WarmCache(ctx)
assert.NoError(t, err)

// Verify that the currencies are cached
for _, currency := range []string{"USD", "EUR", "GBP"} {
key := fmt.Sprintf("%s:api:institutions:%s", conf.CacheVersion, currency)
val, err := client.Get(ctx, key).Result()
assert.NoError(t, err)
assert.NotEmpty(t, val)
}
}

0 comments on commit 27b22f8

Please sign in to comment.