Skip to content

Commit 89efbf0

Browse files
varonix0varonix0
authored
Merge pull request #59 from Infisical/retry-on-request-error
feat: retry on request error
2 parents 3f1b7f8 + 1731837 commit 89efbf0

File tree

4 files changed

+173
-20
lines changed

4 files changed

+173
-20
lines changed

‎client.go‎

Lines changed: 153 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,20 @@ import (
66
"crypto/x509"
77
"errors"
88
"fmt"
9+
910
"math"
1011
"math/rand"
1112
"net"
13+
"os"
1214
"reflect"
1315
"strconv"
1416
"strings"
1517
"sync"
1618
"time"
1719

20+
"github.com/rs/zerolog"
21+
"github.com/rs/zerolog/log"
22+
1823
"github.com/go-resty/resty/v2"
1924
"github.com/hashicorp/golang-lru/v2/expirable"
2025
api "github.com/infisical/go-sdk/packages/api/auth"
@@ -42,6 +47,8 @@ type InfisicalClient struct {
4247
dynamicSecrets DynamicSecretsInterface
4348
kms KmsInterface
4449
ssh SshInterface
50+
51+
logger zerolog.Logger
4552
}
4653

4754
type InfisicalClientInterface interface {
@@ -54,6 +61,56 @@ type InfisicalClientInterface interface {
5461
Ssh() SshInterface
5562
}
5663

64+
type ExponentialBackoffStrategy struct {
65+
// Base delay between retries. Defaults to 1 second
66+
BaseDelay time.Duration
67+
68+
// Maximum number of retries. Defaults to 3
69+
MaxRetries int
70+
71+
// Maximum delay between retries. Defaults to 30 seconds
72+
MaxDelay time.Duration
73+
}
74+
75+
func (s *ExponentialBackoffStrategy) GetDelay(retryCount int) time.Duration {
76+
77+
if s.BaseDelay == 0 {
78+
s.BaseDelay = 1 * time.Second
79+
}
80+
81+
if s.MaxDelay == 0 {
82+
s.MaxDelay = 30 * time.Second
83+
}
84+
85+
if s.MaxRetries == 0 {
86+
s.MaxRetries = 3
87+
}
88+
89+
delay := s.BaseDelay * time.Duration(math.Pow(2, float64(retryCount)))
90+
91+
// if delay is greater than the user-configured max delay, set the delay to the max delay
92+
if delay > s.MaxDelay {
93+
delay = s.MaxDelay
94+
}
95+
96+
return s.Jitter(delay)
97+
}
98+
99+
func (s *ExponentialBackoffStrategy) Jitter(delay time.Duration) time.Duration {
100+
// 20% jitter, negative and positive
101+
102+
jitterFactor := 0.2
103+
104+
// generates random value in [-0.2, +0.2] range
105+
randomFactor := (rand.Float64()*2 - 1) * jitterFactor
106+
jitter := time.Duration(randomFactor * float64(delay))
107+
return delay + jitter
108+
}
109+
110+
type RetryRequestsConfig struct {
111+
ExponentialBackoff *ExponentialBackoffStrategy
112+
}
113+
57114
type Config struct {
58115
SiteUrl string `default:"https://app.infisical.com"`
59116
CaCertificate string
@@ -62,6 +119,52 @@ type Config struct {
62119
SilentMode bool `default:"false"` // If enabled, the SDK will not print any warnings to the console.
63120
CacheExpiryInSeconds int // Defines how long certain API responses should be cached in memory, in seconds. When set to a positive value, responses from specific fetch API requests (like secret fetching) will be cached for this duration. Set to 0 to disable caching. Defaults to 0.
64121
CustomHeaders map[string]string
122+
RetryRequestsConfig *RetryRequestsConfig
123+
}
124+
125+
func setupLogger() zerolog.Logger {
126+
// very annoying but zerolog doesn't allow us to change one color without changing all of them
127+
// these are the default colors for each level, except for warn
128+
levelColors := map[string]string{
129+
"trace": "\033[35m", // magenta
130+
"debug": "\033[33m", // yellow
131+
"info": "\033[32m", // green
132+
"warn": "\033[33m", // yellow (this one is custom, the default is red \033[31m)
133+
"error": "\033[31m", // red
134+
"fatal": "\033[31m", // red
135+
"panic": "\033[31m", // red
136+
}
137+
138+
// map full level names to abbreviated forms (default zerolog behavior)
139+
// see consoleDefaultFormatLevel, in zerolog for example
140+
levelAbbrev := map[string]string{
141+
"trace": "TRC",
142+
"debug": "DBG",
143+
"info": "INF",
144+
"warn": "WRN",
145+
"error": "ERR",
146+
"fatal": "FTL",
147+
"panic": "PNC",
148+
}
149+
150+
logger := log.Output(zerolog.ConsoleWriter{
151+
Out: os.Stderr,
152+
TimeFormat: time.RFC3339,
153+
FormatLevel: func(i interface{}) string {
154+
level := fmt.Sprintf("%s", i)
155+
color := levelColors[level]
156+
if color == "" {
157+
color = "\033[0m" // no color for unknown levels
158+
}
159+
abbrev := levelAbbrev[level]
160+
if abbrev == "" {
161+
abbrev = strings.ToUpper(level) // fallback to uppercase if unknown
162+
}
163+
return color + abbrev + "\033[0m"
164+
},
165+
})
166+
167+
return logger
65168
}
66169

67170
func setDefaults(cfg *Config) {
@@ -133,7 +236,11 @@ func (c *InfisicalClient) setPlainAccessToken(accessToken string) {
133236
}
134237

135238
func NewInfisicalClient(context context.Context, config Config) InfisicalClientInterface {
136-
client := &InfisicalClient{}
239+
logger := setupLogger()
240+
241+
client := &InfisicalClient{
242+
logger: logger,
243+
}
137244
setDefaults(&config)
138245
client.UpdateConfiguration(config) // set httpClient and config
139246

@@ -169,16 +276,32 @@ func (c *InfisicalClient) UpdateConfiguration(config Config) {
169276
SetHeader("User-Agent", config.UserAgent).
170277
SetBaseURL(config.SiteUrl)
171278

172-
c.httpClient.SetRetryCount(3).
279+
maxRetries := 3
280+
maxWaitTime := 30 * time.Second
281+
282+
if config.RetryRequestsConfig != nil && config.RetryRequestsConfig.ExponentialBackoff != nil {
283+
maxRetries = config.RetryRequestsConfig.ExponentialBackoff.MaxRetries
284+
maxWaitTime = 10 * time.Minute
285+
}
286+
287+
c.httpClient.SetRetryCount(maxRetries).
173288
SetRetryWaitTime(1 * time.Second).
174-
SetRetryMaxWaitTime(30 * time.Second).
175-
SetRetryAfter(func(c *resty.Client, r *resty.Response) (time.Duration, error) {
289+
SetRetryMaxWaitTime(maxWaitTime).
290+
SetRetryAfter(func(rc *resty.Client, r *resty.Response) (time.Duration, error) {
291+
292+
if config.RetryRequestsConfig != nil && config.RetryRequestsConfig.ExponentialBackoff != nil {
293+
delay := config.RetryRequestsConfig.ExponentialBackoff.GetDelay(r.Request.Attempt)
294+
if !config.SilentMode {
295+
util.PrintWarning(c.logger, fmt.Sprintf("Request failed, [url=%s] [status=%d] [method=%s]\nRetrying in %s (attempt %d)", r.Request.URL, r.StatusCode(), r.Request.Method, delay.String(), r.Request.Attempt))
296+
}
297+
return delay, nil
298+
}
176299

177300
attempt := r.Request.Attempt + 1
178301
if attempt <= 0 {
179302
attempt = 1
180303
}
181-
waitTime := math.Min(float64(c.RetryWaitTime)*math.Pow(2, float64(attempt-1)), float64(c.RetryMaxWaitTime))
304+
waitTime := math.Min(float64(rc.RetryWaitTime)*math.Pow(2, float64(attempt-1)), float64(rc.RetryMaxWaitTime))
182305

183306
// Add jitter of +/-20%
184307
jitterFactor := 0.8 + (rand.Float64() * 0.4)
@@ -189,11 +312,19 @@ func (c *InfisicalClient) UpdateConfiguration(config Config) {
189312
}).
190313
AddRetryCondition(func(r *resty.Response, err error) bool {
191314
// don't retry if there's no error or it's a timeout
192-
if err == nil || errors.Is(err, context.DeadlineExceeded) {
315+
if errors.Is(err, context.DeadlineExceeded) {
316+
return false
317+
}
318+
319+
if err == nil && r == nil {
193320
return false
194321
}
195322

196-
errMsg := err.Error()
323+
if config.RetryRequestsConfig != nil && config.RetryRequestsConfig.ExponentialBackoff != nil {
324+
if (r != nil && r.IsError()) || err != nil {
325+
return r.Request.Attempt <= config.RetryRequestsConfig.ExponentialBackoff.MaxRetries
326+
}
327+
}
197328

198329
networkErrors := []string{
199330
"connection refused",
@@ -219,9 +350,14 @@ func (c *InfisicalClient) UpdateConfiguration(config Config) {
219350
return true
220351
}
221352

222-
for _, netErr := range networkErrors {
223-
if strings.Contains(strings.ToLower(errMsg), netErr) {
224-
isConditionMet = true
353+
if err != nil {
354+
for _, netErr := range networkErrors {
355+
errMsg := err.Error()
356+
357+
if strings.Contains(strings.ToLower(errMsg), netErr) {
358+
isConditionMet = true
359+
break
360+
}
225361
}
226362
}
227363

@@ -242,11 +378,11 @@ func (c *InfisicalClient) UpdateConfiguration(config Config) {
242378
if config.CaCertificate != "" {
243379
caCertPool, err := x509.SystemCertPool()
244380
if err != nil && !config.SilentMode {
245-
util.PrintWarning(fmt.Sprintf("failed to load system root CA pool: %v", err))
381+
util.PrintWarning(c.logger, fmt.Sprintf("failed to load system root CA pool: %v", err))
246382
}
247383

248384
if ok := caCertPool.AppendCertsFromPEM([]byte(config.CaCertificate)); !ok && !config.SilentMode {
249-
util.PrintWarning("failed to append CA certificate")
385+
util.PrintWarning(c.logger, "failed to append CA certificate")
250386
}
251387

252388
tlsConfig := &tls.Config{
@@ -371,7 +507,7 @@ func (c *InfisicalClient) handleTokenLifeCycle(context context.Context) {
371507

372508
if !config.SilentMode && !warningPrinted && tokenDetails.AccessTokenMaxTTL != 0 && tokenDetails.ExpiresIn != 0 {
373509
if tokenDetails.AccessTokenMaxTTL < 60 || tokenDetails.ExpiresIn < 60 {
374-
util.PrintWarning("Machine Identity access token TTL or max TTL is less than 60 seconds. This may cause excessive API calls, and you may be subject to rate-limits.")
510+
util.PrintWarning(c.logger, "Machine Identity access token TTL or max TTL is less than 60 seconds. This may cause excessive API calls, and you may be subject to rate-limits.")
375511
}
376512
warningPrinted = true
377513
}
@@ -387,7 +523,7 @@ func (c *InfisicalClient) handleTokenLifeCycle(context context.Context) {
387523
newToken, err := authStrategies[c.authMethod](clientCredential)
388524

389525
if err != nil && !config.SilentMode {
390-
util.PrintWarning(fmt.Sprintf("Failed to re-authenticate: %s\n", err.Error()))
526+
util.PrintWarning(c.logger, fmt.Sprintf("Failed to re-authenticate: %s\n", err.Error()))
391527
} else {
392528
c.setAccessToken(newToken, c.credential, c.authMethod)
393529
c.mu.Lock()
@@ -403,7 +539,7 @@ func (c *InfisicalClient) handleTokenLifeCycle(context context.Context) {
403539
// If renewing would exceed max TTL, directly re-authenticate
404540
newToken, err := authStrategies[c.authMethod](clientCredential)
405541
if err != nil && !config.SilentMode {
406-
util.PrintWarning(fmt.Sprintf("Failed to re-authenticate: %s\n", err.Error()))
542+
util.PrintWarning(c.logger, fmt.Sprintf("Failed to re-authenticate: %s\n", err.Error()))
407543
} else {
408544
c.setAccessToken(newToken, c.credential, c.authMethod)
409545
c.mu.Lock()
@@ -416,12 +552,12 @@ func (c *InfisicalClient) handleTokenLifeCycle(context context.Context) {
416552

417553
if err != nil {
418554
if !config.SilentMode {
419-
util.PrintWarning(fmt.Sprintf("Failed to renew access token: %s\n\nAttempting to re-authenticate.", err.Error()))
555+
util.PrintWarning(c.logger, fmt.Sprintf("Failed to renew access token: %s\n\nAttempting to re-authenticate.", err.Error()))
420556
}
421557

422558
newToken, err := authStrategies[c.authMethod](clientCredential)
423559
if err != nil && !config.SilentMode {
424-
util.PrintWarning(fmt.Sprintf("Failed to re-authenticate: %s\n", err.Error()))
560+
util.PrintWarning(c.logger, fmt.Sprintf("Failed to re-authenticate: %s\n", err.Error()))
425561
} else {
426562
c.setAccessToken(newToken, c.credential, c.authMethod)
427563
c.mu.Lock()

‎go.mod‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,10 @@ require (
3535
github.com/google/s2a-go v0.1.7 // indirect
3636
github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect
3737
github.com/googleapis/gax-go/v2 v2.12.5 // indirect
38+
github.com/mattn/go-colorable v0.1.13 // indirect
39+
github.com/mattn/go-isatty v0.0.19 // indirect
3840
github.com/oracle/oci-go-sdk/v65 v65.95.2 // indirect
41+
github.com/rs/zerolog v1.34.0 // indirect
3942
github.com/sony/gobreaker v0.5.0 // indirect
4043
github.com/youmark/pkcs8 v0.0.0-20240726163527-a2c0da244d78 // indirect
4144
go.opencensus.io v0.24.0 // indirect

‎go.sum‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,7 @@ github.com/aws/smithy-go v1.20.2/go.mod h1:krry+ya/rV9RDcV/Q16kpu6ypI4K2czasz0NC
3737
github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
3838
github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
3939
github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
40+
github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
4041
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
4142
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
4243
github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
@@ -53,6 +54,7 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag=
5354
github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE=
5455
github.com/go-resty/resty/v2 v2.13.1 h1:x+LHXBI2nMB1vqndymf26quycC4aggYJ7DECYbiz03g=
5556
github.com/go-resty/resty/v2 v2.13.1/go.mod h1:GznXlLxkq6Nh4sU59rPmUw3VtgpO3aS96ORAI6Q7d+0=
57+
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
5658
github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw=
5759
github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU=
5860
github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
@@ -88,11 +90,20 @@ github.com/googleapis/gax-go/v2 v2.12.5 h1:8gw9KZK8TiVKB6q3zHY3SBzLnrGp6HQjyfYBY
8890
github.com/googleapis/gax-go/v2 v2.12.5/go.mod h1:BUDKcWo+RaKq5SC9vVYL0wLADa3VcfswbOMMRmB9H3E=
8991
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
9092
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
93+
github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA=
94+
github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg=
95+
github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
96+
github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA=
97+
github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
9198
github.com/oracle/oci-go-sdk/v65 v65.95.2 h1:0HJ0AgpLydp/DtvYrF2d4str2BjXOVAeNbuW7E07g94=
9299
github.com/oracle/oci-go-sdk/v65 v65.95.2/go.mod h1:u6XRPsw9tPziBh76K7GrrRXPa8P8W3BQeqJ6ZZt9VLA=
100+
github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
93101
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
94102
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
95103
github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
104+
github.com/rs/xid v1.6.0/go.mod h1:7XoLgs4eV+QndskICGsho+ADou8ySMSjJKDIan90Nz0=
105+
github.com/rs/zerolog v1.34.0 h1:k43nTLIwcTVQAncfCw4KZ2VY6ukYoZaBPNOE8txlOeY=
106+
github.com/rs/zerolog v1.34.0/go.mod h1:bJsvje4Z08ROH4Nhs5iH600c3IkWhwp44iRc54W6wYQ=
96107
github.com/sony/gobreaker v0.5.0 h1:dRCvqm0P490vZPmy7ppEk2qCnCieBooFJ+YoXGYB+yg=
97108
github.com/sony/gobreaker v0.5.0/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY=
98109
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
@@ -166,8 +177,11 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w
166177
golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
167178
golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
168179
golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
180+
golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
169181
golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
182+
golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
170183
golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
184+
golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
171185
golang.org/x/sys v0.17.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
172186
golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
173187
golang.org/x/sys v0.20.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=

‎packages/util/helper.go‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6,13 +6,13 @@ import (
66
"encoding/hex"
77
"encoding/json"
88
"fmt"
9-
"os"
109
"sort"
1110
"strings"
1211
"time"
1312

1413
"github.com/go-resty/resty/v2"
1514
"github.com/infisical/go-sdk/packages/models"
15+
"github.com/rs/zerolog"
1616
)
1717

1818
func AppendAPIEndpoint(siteUrl string) string {
@@ -28,8 +28,8 @@ func AppendAPIEndpoint(siteUrl string) string {
2828
return siteUrl + "/api"
2929
}
3030

31-
func PrintWarning(message string) {
32-
fmt.Fprintf(os.Stderr, "[Infisical] Warning: %v \n", message)
31+
func PrintWarning(logger zerolog.Logger, message string) {
32+
logger.Warn().Msgf("[Infisical] Warning: %v", message)
3333
}
3434

3535
func EnsureUniqueSecretsByKey(secrets *[]models.Secret, skipUniqueKey bool) {

0 commit comments

Comments
 (0)