Skip to content

Commit

Permalink
fix(retry): add WithMaxElapsedTime to retry options and refactor to u…
Browse files Browse the repository at this point in the history 
…se functional options
  • Loading branch information
@matt-FFFFFF
matt-FFFFFF committed Jan 16, 2025
1 parent 327514e commit e20c92b
Show file tree
Hide file tree
Showing 12 changed files with 244 additions and 132 deletions.
56 changes: 46 additions & 10 deletions internal/clients/data_plane_client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ import (

"github.com/Azure/terraform-provider-azapi/internal/clients"
"github.com/Azure/terraform-provider-azapi/internal/services/parse"
"github.com/cenkalti/backoff/v4"
"github.com/stretchr/testify/assert"
)

Expand All @@ -27,9 +28,16 @@ import (
//
// We check these timings are expected using the assert.InDeltaSlice function.
func TestRetryDataPlaneClient(t *testing.T) {
t.Parallel()
mock := NewMockDataPlaneClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 30, 2, 0.0, []string{"retry error"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(30*time.Second),
backoff.WithMultiplier(2),
backoff.WithRandomizationFactor(0.0),
)
rexs := clients.StringSliceToRegexpSliceMust([]string{"retry error"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), parse.DataPlaneResourceId{}, clients.DefaultRequestOptions())
assert.NoError(t, err)
assert.Equal(t, 3, mock.requestCount)
Expand All @@ -47,36 +55,64 @@ func TestRetryDataPlaneClient(t *testing.T) {
}

func TestRetryDataPlaneClientRegexp(t *testing.T) {
t.Parallel()
mock := NewMockDataPlaneClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 5, 1.5, 0.0, []string{"^retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(5*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
rexs := clients.StringSliceToRegexpSliceMust([]string{"^retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), parse.DataPlaneResourceId{}, clients.DefaultRequestOptions())
assert.NoError(t, err)
assert.Equal(t, 3, mock.RequestCount())
}

func TestRetryDataPlaneClientMultiRegexp(t *testing.T) {
t.Parallel()
mock := NewMockDataPlaneClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 5, 1.5, 0.0, []string{"nomatch", "^retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(5*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
rexs := clients.StringSliceToRegexpSliceMust([]string{"nomatch", "^retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), parse.DataPlaneResourceId{}, clients.DefaultRequestOptions())
assert.NoError(t, err)
assert.Equal(t, 3, mock.RequestCount())
}

func TestRetryDataPlaneClientMultiRegexpNoMatchWithPermError(t *testing.T) {
t.Parallel()
mock := NewMockDataPlaneClient(t, nil, errors.New("perm error"), 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 5, 1.5, 0.0, []string{"retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(5*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
rexs := clients.StringSliceToRegexpSliceMust([]string{"retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), parse.DataPlaneResourceId{}, clients.DefaultRequestOptions())
assert.ErrorContains(t, err, "perm error")
assert.Equal(t, 3, mock.RequestCount())
}

func TestRetryDataPlaneClientContextDeadline(t *testing.T) {
t.Parallel()
mock := NewMockDataPlaneClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(60, 60, 1.5, 0.0, []string{"^retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(60*time.Second),
backoff.WithMaxInterval(60*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
rexs := clients.StringSliceToRegexpSliceMust([]string{"^retry"})
retryClient := clients.NewDataPlaneClientRetryableErrors(mock, bkof, rexs, nil, nil)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
start := time.Now()
Expand Down
38 changes: 24 additions & 14 deletions internal/clients/resource_client.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -84,20 +84,30 @@ func NewResourceClient(credential azcore.TokenCredential, opt *arm.ClientOptions
}, nil
}

// NewRetryableErrors creates the backoff and error regexs for retryable errors.
func NewRetryableErrors(intervalSeconds, maxIntervalSeconds int, multiplier, randomizationFactor float64, errorRegexs []string) (*backoff.ExponentialBackOff, []regexp.Regexp) {
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(time.Duration(intervalSeconds)*time.Second),
backoff.WithRandomizationFactor(randomizationFactor),
backoff.WithMaxInterval(time.Duration(maxIntervalSeconds)*time.Second),
backoff.WithRandomizationFactor(randomizationFactor),
backoff.WithMultiplier(multiplier),
)
res := make([]regexp.Regexp, len(errorRegexs))
for i, e := range errorRegexs {
res[i] = *regexp.MustCompile(e) // MustCompile as schema has custom validation so we know it's valid
}
return bkof, res
// // NewRetryableErrors creates the backoff and error regexs for retryable errors.
// func NewRetryableErrors(intervalSeconds, maxIntervalSeconds int, multiplier, randomizationFactor float64, errorRegexs []string) (*backoff.ExponentialBackOff, []regexp.Regexp) {
// bkof := backoff.NewExponentialBackOff(
// backoff.WithInitialInterval(time.Duration(intervalSeconds)*time.Second),
// backoff.WithRandomizationFactor(randomizationFactor),
// backoff.WithMaxInterval(time.Duration(maxIntervalSeconds)*time.Second),
// backoff.WithRandomizationFactor(randomizationFactor),
// backoff.WithMultiplier(multiplier),
// )
// res := make([]regexp.Regexp, len(errorRegexs))
// for i, e := range errorRegexs {
// res[i] = *regexp.MustCompile(e) // MustCompile as schema has custom validation so we know it's valid
// }
// return bkof, res
// }

// StringSliceToRegexpSliceMust converts a slice of strings to a slice of regexps.
// It panics if any of the strings are invalid regexps.
func StringSliceToRegexpSliceMust(ss []string) []regexp.Regexp {
res := make([]regexp.Regexp, len(ss))
for i, e := range ss {
res[i] = *regexp.MustCompile(e)
}
return res
}

// WithRetry configures the retryable errors for the client.
Expand Down
56 changes: 46 additions & 10 deletions internal/clients/resource_client_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"time"

"github.com/Azure/terraform-provider-azapi/internal/clients"
"github.com/cenkalti/backoff/v4"
"github.com/stretchr/testify/assert"
)

Expand All @@ -26,9 +27,16 @@ import (
//
// We check these timings are expected using the assert.InDeltaSlice function.
func TestRetryClient(t *testing.T) {
t.Parallel()
mock := NewMockResourceClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 30, 2, 0.0, []string{"retry error"})
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
rexs := clients.StringSliceToRegexpSliceMust([]string{"retry error"})
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(30*time.Second),
backoff.WithMultiplier(2),
backoff.WithRandomizationFactor(0.0),
)
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), "", "", clients.DefaultRequestOptions())
assert.NoError(t, err)
assert.Equal(t, 3, mock.requestCount)
Expand All @@ -46,36 +54,64 @@ func TestRetryClient(t *testing.T) {
}

func TestRetryClientRegexp(t *testing.T) {
t.Parallel()
mock := NewMockResourceClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 5, 1.5, 0.0, []string{"^retry"})
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
rexs := clients.StringSliceToRegexpSliceMust([]string{"^retry"})
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(5*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), "", "", clients.DefaultRequestOptions())
assert.NoError(t, err)
assert.Equal(t, 3, mock.RequestCount())
}

func TestRetryClientMultiRegexp(t *testing.T) {
t.Parallel()
mock := NewMockResourceClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 5, 1.5, 0.0, []string{"nomatch", "^retry"})
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
rexs := clients.StringSliceToRegexpSliceMust([]string{"nomatch", "^retry"})
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(5*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), "", "", clients.DefaultRequestOptions())
assert.NoError(t, err)
assert.Equal(t, 3, mock.RequestCount())
}

func TestRetryClientMultiRegexpNoMatchWithPermError(t *testing.T) {
t.Parallel()
mock := NewMockResourceClient(t, nil, errors.New("perm error"), 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(1, 5, 1.5, 0.0, []string{"retry"})
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
rexs := clients.StringSliceToRegexpSliceMust([]string{"retry"})
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(1*time.Second),
backoff.WithMaxInterval(5*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, rexs, nil, nil)
_, err := retryClient.Get(context.Background(), "", "", clients.DefaultRequestOptions())
assert.ErrorContains(t, err, "perm error")
assert.Equal(t, 3, mock.RequestCount())
}

func TestRetryClientContextDeadline(t *testing.T) {
t.Parallel()
mock := NewMockResourceClient(t, nil, nil, 3, errors.New("retry error"))
bkof, errRegExps := clients.NewRetryableErrors(60, 60, 1.5, 0.0, []string{"^retry"})
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, errRegExps, nil, nil)
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(60*time.Second),
backoff.WithMaxInterval(60*time.Second),
backoff.WithMultiplier(1.5),
backoff.WithRandomizationFactor(0.0),
)
rexs := clients.StringSliceToRegexpSliceMust([]string{"^retry"})
retryClient := clients.NewResourceClientRetryableErrors(mock, bkof, rexs, nil, nil)
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
defer cancel()
start := time.Now()
Expand Down
9 changes: 9 additions & 0 deletions internal/retry/retryable_errors.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"math/big"
"regexp"
"strings"
"time"

"github.com/hashicorp/terraform-plugin-framework/attr"
"github.com/hashicorp/terraform-plugin-framework/diag"
Expand Down Expand Up @@ -606,10 +607,18 @@ func (v RetryValue) GetIntervalSeconds() int {
return v.getInt64AttrValue(intervalSecondsAttributeName)
}

func (v RetryValue) GetIntervalSecondsAsDuration() time.Duration {
return time.Duration(v.IntervalSeconds.ValueInt64()) * time.Second
}

func (v RetryValue) GetMaxIntervalSeconds() int {
return v.getInt64AttrValue(maxIntervalSecondsAttributeName)
}

func (v RetryValue) GetMaxIntervalSecondsAsDuration() time.Duration {
return time.Duration(v.MaxIntervalSeconds.ValueInt64()) * time.Second
}

func (v RetryValue) GetMultiplier() float64 {
return v.getNumberAttrValue(multiplierAttributeName)
}
Expand Down
55 changes: 30 additions & 25 deletions internal/services/azapi_data_plane_resource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -374,19 +374,6 @@ func (r *DataPlaneResource) CreateUpdate(ctx context.Context, plan tfsdk.Plan, s

ctx = tflog.SetField(ctx, "resource_id", id.ID())

var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() {
bkof, regexps := clients.NewRetryableErrors(
model.Retry.GetIntervalSeconds(),
model.Retry.GetMaxIntervalSeconds(),
model.Retry.GetMultiplier(),
model.Retry.GetRandomizationFactor(),
model.Retry.GetErrorMessages(),
)
tflog.Debug(ctx, "azapi_data_plane_resource.CreateUpdate is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
}
isNewResource := state == nil || state.Raw.IsNull()

var timeout time.Duration
Expand All @@ -402,6 +389,22 @@ func (r *DataPlaneResource) CreateUpdate(ctx context.Context, plan tfsdk.Plan, s
return
}
}

var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() {
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(timeout),
)
tflog.Debug(ctx, "azapi_data_plane_resource.CreateUpdate is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
}

ctx, cancel := context.WithTimeout(ctx, timeout)
defer cancel()

Expand Down Expand Up @@ -500,12 +503,13 @@ func (r *DataPlaneResource) Read(ctx context.Context, request resource.ReadReque
var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
bkof, regexps := clients.NewRetryableErrors(
model.Retry.GetIntervalSeconds(),
model.Retry.GetMaxIntervalSeconds(),
model.Retry.GetMultiplier(),
model.Retry.GetRandomizationFactor(),
model.Retry.GetErrorMessages(),
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(readTimeout),
)
tflog.Debug(ctx, "azapi_data_plane_resource.Read is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
Expand Down Expand Up @@ -592,12 +596,13 @@ func (r *DataPlaneResource) Delete(ctx context.Context, request resource.DeleteR
var client clients.DataPlaneRequester
client = r.ProviderData.DataPlaneClient
if !model.Retry.IsNull() && !model.Retry.IsUnknown() {
bkof, regexps := clients.NewRetryableErrors(
model.Retry.GetIntervalSeconds(),
model.Retry.GetMaxIntervalSeconds(),
model.Retry.GetMultiplier(),
model.Retry.GetRandomizationFactor(),
model.Retry.GetErrorMessages(),
regexps := clients.StringSliceToRegexpSliceMust(model.Retry.GetErrorMessages())
bkof := backoff.NewExponentialBackOff(
backoff.WithInitialInterval(model.Retry.GetIntervalSecondsAsDuration()),
backoff.WithMaxInterval(model.Retry.GetMaxIntervalSecondsAsDuration()),
backoff.WithMultiplier(model.Retry.GetMultiplier()),
backoff.WithRandomizationFactor(model.Retry.GetRandomizationFactor()),
backoff.WithMaxElapsedTime(deleteTimeout),
)
tflog.Debug(ctx, "azapi_data_plane_resource.Delete is using retry")
client = r.ProviderData.DataPlaneClient.WithRetry(bkof, regexps, nil, nil)
Expand Down
Loading

0 comments on commit e20c92b

Please sign in to comment.