🐛 Clear credential cache on InvalidClientTokenId to allow retry on next reconcile

After the AWS SDK v1 to v2 migration, aws.NewCredentialsCache caches
failed AssumeRole results. When a freshly-created IAM access key has not
yet propagated (AWS IAM eventual consistency), the first AssumeRole call
fails with InvalidClientTokenId and this error is cached indefinitely.
Unlike SDK v1 where IsExpired() returned true after failure (triggering
a fresh attempt), SDK v2's CredentialsCache retains the error.

This fix clears p.credentials when InvalidClientTokenId is returned, so
the next controller-runtime reconcile loop creates a fresh credentials
cache and retries the AssumeRole call, restoring the pre-migration
recovery behavior.

Fixes #6123

Author: omargallob

pkg/cloud/identity/identity.go

@@ -22,13 +22,16 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/gob"
+	"errors"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/aws/smithy-go"
 	corev1 "k8s.io/api/core/v1"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
@@ -180,5 +183,30 @@ func (p *AWSRolePrincipalTypeProvider) Retrieve(ctx context.Context) (aws.Creden
 		// Update credentials
 		p.credentials = creds
 	}
-	return p.credentials.Retrieve(ctx)
+	result, err := p.credentials.Retrieve(ctx)
+	if err != nil && IsInvalidClientTokenIDError(err) {
+		// Clear the cached credentials so the next reconcile creates a fresh
+		// cache and retries AssumeRole. This handles AWS IAM eventual
+		// consistency for freshly-created access keys, where the first
+		// AssumeRole call may fail with InvalidClientTokenId before the key
+		// has propagated. Without this, the CredentialsCache retains the
+		// failed result and subsequent reconciles never recover.
+		p.log.Info("Transient InvalidClientTokenId error from AssumeRole, clearing credential cache to allow retry on next reconcile")
+		p.credentials = nil
+	}
+	return result, err
+}
+
+// IsInvalidClientTokenIDError reports whether err is an AWS STS
+// InvalidClientTokenId error, which is raised when an access key has not yet
+// propagated through IAM's eventual consistency.
+func IsInvalidClientTokenIDError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidClientTokenId" {
+		return true
+	}
+	return strings.Contains(err.Error(), "InvalidClientTokenId")
 }

pkg/cloud/identity/identity_test.go

@@ -18,19 +18,23 @@ package identity
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	ststypes "github.com/aws/aws-sdk-go-v2/service/sts/types"
+	"github.com/aws/smithy-go"
 	"github.com/golang/mock/gomock"
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/sts/mock_stsiface"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger"
 )
 
 func TestAWSStaticPrincipalTypeProvider(t *testing.T) {
@@ -193,3 +197,112 @@ func TestAWSStaticPrincipalTypeProvider(t *testing.T) {
 		})
 	}
 }
+
+func TestAWSRolePrincipalTypeProvider_ClearsCredentialCacheOnInvalidClientTokenId(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	g := NewWithT(t)
+
+	secret := &corev1.Secret{
+		Data: map[string][]byte{
+			"AccessKeyID":     []byte("test-key"),
+			"SecretAccessKey": []byte("test-secret"),
+		},
+	}
+	staticProvider := NewAWSStaticPrincipalTypeProvider(&infrav1.AWSClusterStaticIdentity{}, secret)
+
+	stsMock := mock_stsiface.NewMockSTSClient(mockCtrl)
+	roleIdentity := &infrav1.AWSClusterRoleIdentity{
+		Spec: infrav1.AWSClusterRoleIdentitySpec{
+			AWSRoleSpec: infrav1.AWSRoleSpec{
+				RoleArn:         "arn:aws:iam::123456789012:role/test-role",
+				SessionName:     "test-session",
+				DurationSeconds: 900,
+			},
+		},
+	}
+
+	testLog := logger.NewLogger(klog.NewKlogr())
+	roleProvider := &AWSRolePrincipalTypeProvider{
+		credentials:    nil,
+		Principal:      roleIdentity,
+		region:         "us-east-1",
+		sourceProvider: staticProvider,
+		stsClient:      stsMock,
+		log:            testLog.WithName("test"),
+	}
+
+	// First call: return InvalidClientTokenId error
+	invalidTokenErr := &smithy.GenericAPIError{
+		Code:    "InvalidClientTokenId",
+		Message: "The security token included in the request is invalid",
+	}
+	stsMock.EXPECT().AssumeRole(gomock.Any(), gomock.Any()).Return(nil, invalidTokenErr)
+
+	_, err := roleProvider.Retrieve(context.TODO())
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(ContainSubstring("InvalidClientTokenId"))
+	// Credentials cache should be cleared
+	g.Expect(roleProvider.credentials).To(BeNil())
+
+	// Second call: succeed — proves the cache was cleared and a fresh attempt is made
+	stsMock.EXPECT().AssumeRole(gomock.Any(), gomock.Any()).Return(&sts.AssumeRoleOutput{
+		Credentials: &ststypes.Credentials{
+			AccessKeyId:     aws.String("new-key"),
+			SecretAccessKey: aws.String("new-secret"),
+			SessionToken:    aws.String("new-token"),
+			Expiration:      aws.Time(time.Now().Add(1 * time.Hour)),
+		},
+	}, nil)
+
+	creds, err := roleProvider.Retrieve(context.TODO())
+	g.Expect(err).To(BeNil())
+	g.Expect(creds.AccessKeyID).To(Equal("new-key"))
+}
+
+func TestIsInvalidClientTokenIDError(t *testing.T) {
+	tests := []struct {
+		name   string
+		err    error
+		expect bool
+	}{
+		{
+			name:   "nil error",
+			err:    nil,
+			expect: false,
+		},
+		{
+			name: "typed smithy APIError with InvalidClientTokenId",
+			err: &smithy.GenericAPIError{
+				Code:    "InvalidClientTokenId",
+				Message: "The security token included in the request is invalid",
+			},
+			expect: true,
+		},
+		{
+			name:   "error containing InvalidClientTokenId in message",
+			err:    fmt.Errorf("operation failed: InvalidClientTokenId: token not valid"),
+			expect: true,
+		},
+		{
+			name:   "unrelated error",
+			err:    fmt.Errorf("connection timeout"),
+			expect: false,
+		},
+		{
+			name: "different API error code",
+			err: &smithy.GenericAPIError{
+				Code:    "AccessDenied",
+				Message: "not authorized",
+			},
+			expect: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			g.Expect(IsInvalidClientTokenIDError(tt.err)).To(Equal(tt.expect))
+		})
+	}
+}