feat: add redismock dependency and enhance caching logic with improve…

…d error handling and response validation

go.mod

@@ -10,6 +10,7 @@ require (
 	github.com/getsentry/sentry-go v0.13.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-co-op/gocron v1.35.0
+	github.com/go-redis/redismock/v9 v9.2.0
 	github.com/golang-jwt/jwt/v5 v5.0.0
 	github.com/jarcoal/httpmock v1.3.1
 	github.com/mailgun/mailgun-go/v3 v3.6.4

go.sum

@@ -241,6 +241,8 @@ github.com/go-playground/universal-translator v0.18.1 h1:Bcnm0ZwsGyWbCzImXv+pAJn
 github.com/go-playground/universal-translator v0.18.1/go.mod h1:xekY+UJKNuX9WP91TpwSH2VMlDf28Uj24BCp08ZFTUY=
 github.com/go-playground/validator/v10 v10.15.0 h1:nDU5XeOKtB3GEa+uB7GNYwhVKsgjAR7VgKoNB6ryXfw=
 github.com/go-playground/validator/v10 v10.15.0/go.mod h1:9iXMNT7sEkjXb0I+enO7QXmzG6QCsPWY4zveKFVRSyU=
+github.com/go-redis/redismock/v9 v9.2.0 h1:ZrMYQeKPECZPjOj5u9eyOjg8Nnb0BS9lkVIZ6IpsKLw=
+github.com/go-redis/redismock/v9 v9.2.0/go.mod h1:18KHfGDK4Y6c2R0H38EUGWAdc7ZQS9gfYxc94k7rWT0=
 github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw=
 github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4=
 github.com/go-test/deep v1.0.3 h1:ZrJSEWsXzPOxaZnFteGEfooLba+ju3FYIbOrS+rQd68=
@@ -487,13 +489,19 @@ github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5Vgl
 github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w=
 github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w=
 github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c=
+github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE=
+github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+AU=
 github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec=
 github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.10.3/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
+github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
+github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
+github.com/onsi/gomega v1.25.0 h1:Vw7br2PCDYijJHSfBOWhov+8cAnUf8MfMaIOV323l6Y=
+github.com/onsi/gomega v1.25.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
 github.com/opus-domini/fast-shot v0.10.0 h1:zWbPy6KJZvNs0pUa0erF9TyeDsLZHDVZf4oDHOd6JGY=
 github.com/opus-domini/fast-shot v0.10.0/go.mod h1:sg5+f0VviAIIdrv24WLHL6kV7kWs4PsVDbSkr2TPYWw=
 github.com/paycrest/tron-wallet v1.0.13 h1:TEkjovg6i2zBTZFMfYpBrwswwnYAy6Q/Vc6g12PZh54=
@@ -1026,6 +1034,7 @@ gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce/go.mod h1:yeKp02qBN3iKW1OzL3MGk2IdtZzaj7SFntXj72NppTA=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8=
 gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k=
+gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ=
 gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

routers/middleware/caching.go

@@ -8,7 +8,6 @@ import (
 	"fmt"
 	"io/ioutil"
 	"net/http"
-	"net/url"
 	"time"
 
 	"github.com/gin-gonic/gin"
@@ -64,17 +63,18 @@ func NewCacheService(config config.RedisConfiguration) (*CacheService, error) {
 }
 
 func generateCacheKey(c *gin.Context) string {
-	conf := config.RedisConfig()
+	// conf := config.RedisConfig()
+	conf := "v1"
 	path := c.Request.URL.Path
 	switch {
 	case path == "/v1/currencies":
-		return fmt.Sprintf("%s:api:currencies:list", conf.CacheVersion)
+		return fmt.Sprintf("%s:api:currencies:list", conf)
 	case path == "/v1/pubkey":
-		return fmt.Sprintf("%s:api:aggregator:pubkey", conf.CacheVersion)
+		return fmt.Sprintf("%s:api:aggregator:pubkey", conf)
 	case len(c.Param("currency_code")) > 0:
-		return fmt.Sprintf("%s:api:institutions:%s", conf.CacheVersion, c.Param("currency_code"))
+		return fmt.Sprintf("%s:api:institutions:%s", conf, c.Param("currency_code"))
 	default:
-		return fmt.Sprintf("%s:api:%s", conf.CacheVersion, path)
+		return fmt.Sprintf("%s:api:%s", conf, path)
 	}
 }
 
@@ -115,6 +115,7 @@ func (s *CacheService) CacheMiddleware(duration time.Duration) gin.HandlerFunc {
 		}
 
 		s.metrics.misses.Inc()
+		c.Header("X-Cache", "MISS") // Add this line
 		c.Writer = &cacheWriter{ResponseWriter: c.Writer, body: make([]byte, 0)}
 		c.Next()
 
@@ -148,8 +149,10 @@ func (s *CacheService) WarmCache(ctx context.Context) error {
 		return fmt.Errorf("host domain is not set in the server configuration")
 	}
 
-	// Fetch the list of supported currencies with a timeout
+	// Create HTTP client with timeout
 	client := &http.Client{Timeout: 10 * time.Second}
+
+	// Fetch currencies first
 	currenciesURL := fmt.Sprintf("%s/v1/currencies", baseURL)
 	resp, err := client.Get(currenciesURL)
 	if err != nil {
@@ -166,32 +169,33 @@ func (s *CacheService) WarmCache(ctx context.Context) error {
 		return fmt.Errorf("failed to decode currencies response: %v", err)
 	}
 
+	// Use default currencies if none found
 	if len(currencies) == 0 {
-		fmt.Println("No currencies found. Using default currencies [USD, EUR, GBP].")
 		currencies = []string{"USD", "EUR", "GBP"}
 	}
 
-	// Define static and dynamic endpoints
-	endpoints := map[string]time.Duration{
-		"currencies": time.Duration(conf.CurrenciesCacheDuration) * time.Hour,
-		"pubkey":     time.Duration(conf.PubKeyCacheDuration) * time.Hour,
+	// Cache currencies
+	currenciesKey := fmt.Sprintf("v1:api:currencies:list")
+	currenciesData, err := json.Marshal(currencies)
+	if err != nil {
+		return fmt.Errorf("failed to marshal currencies: %v", err)
+	}
+	if err := s.client.Set(ctx, currenciesKey, string(currenciesData), time.Duration(conf.CurrenciesCacheDuration)*time.Hour).Err(); err != nil {
+		return fmt.Errorf("failed to cache currencies: %v", err)
 	}
 
-	for _, currency := range currencies {
-		endpoints[fmt.Sprintf("institutions/%s", currency)] = time.Duration(conf.InstitutionsCacheDuration) * time.Hour
+	// Cache pubkey
+	pubkeyURL := fmt.Sprintf("%s/v1/pubkey", baseURL)
+	if err := s.cacheEndpoint(ctx, pubkeyURL, "v1:api:aggregator:pubkey", time.Duration(conf.PubKeyCacheDuration)*time.Hour); err != nil {
+		return fmt.Errorf("failed to cache pubkey: %v", err)
 	}
 
-	// Warm up cache for each endpoint
-	for path, duration := range endpoints {
-		urlStr := fmt.Sprintf("%s/v1/%s", baseURL, path)
-		parsedURL, err := url.Parse(urlStr)
-		if err != nil {
-			fmt.Printf("Failed to parse URL %s: %v\n", urlStr, err)
-			continue
-		}
-		key := generateCacheKey(&gin.Context{Request: &http.Request{URL: parsedURL}})
-		if err := s.cacheEndpoint(ctx, urlStr, key, duration); err != nil {
-			fmt.Printf("Failed to cache %s: %v\n", path, err)
+	// Cache institutions for each currency
+	for _, currency := range currencies {
+		institutionsURL := fmt.Sprintf("%s/v1/institutions/%s", baseURL, currency)
+		key := fmt.Sprintf("v1:api:institutions:%s", currency)
+		if err := s.cacheEndpoint(ctx, institutionsURL, key, time.Duration(conf.InstitutionsCacheDuration)*time.Hour); err != nil {
+			return fmt.Errorf("failed to cache institutions for %s: %v", currency, err)
 		}
 	}
 
@@ -201,22 +205,35 @@ func (s *CacheService) WarmCache(ctx context.Context) error {
 func (s *CacheService) cacheEndpoint(ctx context.Context, url, key string, duration time.Duration) error {
 	resp, err := http.Get(url)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to fetch from %s: %v", url, err)
 	}
 	defer resp.Body.Close()
 
 	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("failed to fetch data from %s: status code %d", url, resp.StatusCode)
+		return fmt.Errorf("received non-200 status code (%d) from %s", resp.StatusCode, url)
 	}
 
 	body, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
-		return err
+		return fmt.Errorf("failed to read response body from %s: %v", url, err)
 	}
 
+	// Verify the response is valid JSON
+	var jsonCheck interface{}
+	if err := json.Unmarshal(body, &jsonCheck); err != nil {
+		return fmt.Errorf("invalid JSON response from %s: %v", url, err)
+	}
+
+	// Generate and store ETag
 	etag := generateETag(body)
-	s.client.Set(ctx, key, string(body), duration)
-	s.client.Set(ctx, key+":etag", etag, duration)
+	if err := s.client.Set(ctx, key+":etag", etag, duration).Err(); err != nil {
+		return fmt.Errorf("failed to cache etag for %s: %v", url, err)
+	}
+
+	// Cache the response
+	if err := s.client.Set(ctx, key, string(body), duration).Err(); err != nil {
+		return fmt.Errorf("failed to cache response for %s: %v", url, err)
+	}
 
 	return nil
 }