feat: clear credential cache on InvalidClientTokenId to allow retry on next reconcile

Author: omargallob

pkg/cloud/identity/identity.go

@@ -22,13 +22,16 @@ import (
 	"context"
 	"crypto/sha256"
 	"encoding/gob"
+	"errors"
+	"strings"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/config"
 	"github.com/aws/aws-sdk-go-v2/credentials"
 	"github.com/aws/aws-sdk-go-v2/credentials/stscreds"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
+	"github.com/aws/smithy-go"
 	corev1 "k8s.io/api/core/v1"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
@@ -180,5 +183,30 @@ func (p *AWSRolePrincipalTypeProvider) Retrieve(ctx context.Context) (aws.Creden
 		// Update credentials
 		p.credentials = creds
 	}
-	return p.credentials.Retrieve(ctx)
+	result, err := p.credentials.Retrieve(ctx)
+	if err != nil && IsInvalidClientTokenIDError(err) {
+		// Clear the cached credentials so the next reconcile creates a fresh
+		// cache and retries AssumeRole. This handles AWS IAM eventual
+		// consistency for freshly-created access keys, where the first
+		// AssumeRole call may fail with InvalidClientTokenId before the key
+		// has propagated. Without this, the CredentialsCache retains the
+		// failed result and subsequent reconciles never recover.
+		p.log.Info("Transient InvalidClientTokenId error from AssumeRole, clearing credential cache to allow retry on next reconcile")
+		p.credentials = nil
+	}
+	return result, err
+}
+
+// IsInvalidClientTokenIDError reports whether err is an AWS STS
+// InvalidClientTokenId error, which is raised when an access key has not yet
+// propagated through IAM's eventual consistency.
+func IsInvalidClientTokenIDError(err error) bool {
+	if err == nil {
+		return false
+	}
+	var apiErr smithy.APIError
+	if errors.As(err, &apiErr) && apiErr.ErrorCode() == "InvalidClientTokenId" {
+		return true
+	}
+	return strings.Contains(err.Error(), "InvalidClientTokenId")
 }

pkg/cloud/identity/identity_test.go

@@ -18,19 +18,23 @@ package identity
 
 import (
 	"context"
+	"fmt"
 	"testing"
 	"time"
 
 	"github.com/aws/aws-sdk-go-v2/aws"
 	"github.com/aws/aws-sdk-go-v2/service/sts"
 	ststypes "github.com/aws/aws-sdk-go-v2/service/sts/types"
+	"github.com/aws/smithy-go"
 	"github.com/golang/mock/gomock"
 	"github.com/google/go-cmp/cmp"
 	. "github.com/onsi/gomega"
 	corev1 "k8s.io/api/core/v1"
+	"k8s.io/klog/v2"
 
 	infrav1 "sigs.k8s.io/cluster-api-provider-aws/v2/api/v1beta2"
 	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/cloud/services/sts/mock_stsiface"
+	"sigs.k8s.io/cluster-api-provider-aws/v2/pkg/logger"
 )
 
 func TestAWSStaticPrincipalTypeProvider(t *testing.T) {
@@ -193,3 +197,112 @@ func TestAWSStaticPrincipalTypeProvider(t *testing.T) {
 		})
 	}
 }
+
+func TestAWSRolePrincipalTypeProvider_ClearsCredentialCacheOnInvalidClientTokenId(t *testing.T) {
+	mockCtrl := gomock.NewController(t)
+	defer mockCtrl.Finish()
+
+	g := NewWithT(t)
+
+	secret := &corev1.Secret{
+		Data: map[string][]byte{
+			"AccessKeyID":     []byte("test-key"),
+			"SecretAccessKey": []byte("test-secret"),
+		},
+	}
+	staticProvider := NewAWSStaticPrincipalTypeProvider(&infrav1.AWSClusterStaticIdentity{}, secret)
+
+	stsMock := mock_stsiface.NewMockSTSClient(mockCtrl)
+	roleIdentity := &infrav1.AWSClusterRoleIdentity{
+		Spec: infrav1.AWSClusterRoleIdentitySpec{
+			AWSRoleSpec: infrav1.AWSRoleSpec{
+				RoleArn:         "arn:aws:iam::123456789012:role/test-role",
+				SessionName:     "test-session",
+				DurationSeconds: 900,
+			},
+		},
+	}
+
+	testLog := logger.NewLogger(klog.NewKlogr())
+	roleProvider := &AWSRolePrincipalTypeProvider{
+		credentials:    nil,
+		Principal:      roleIdentity,
+		region:         "us-east-1",
+		sourceProvider: staticProvider,
+		stsClient:      stsMock,
+		log:            testLog.WithName("test"),
+	}
+
+	// First call: return InvalidClientTokenId error
+	invalidTokenErr := &smithy.GenericAPIError{
+		Code:    "InvalidClientTokenId",
+		Message: "The security token included in the request is invalid",
+	}
+	stsMock.EXPECT().AssumeRole(gomock.Any(), gomock.Any()).Return(nil, invalidTokenErr)
+
+	_, err := roleProvider.Retrieve(context.TODO())
+	g.Expect(err).To(HaveOccurred())
+	g.Expect(err.Error()).To(ContainSubstring("InvalidClientTokenId"))
+	// Credentials cache should be cleared
+	g.Expect(roleProvider.credentials).To(BeNil())
+
+	// Second call: succeed — proves the cache was cleared and a fresh attempt is made
+	stsMock.EXPECT().AssumeRole(gomock.Any(), gomock.Any()).Return(&sts.AssumeRoleOutput{
+		Credentials: &ststypes.Credentials{
+			AccessKeyId:     aws.String("new-key"),
+			SecretAccessKey: aws.String("new-secret"),
+			SessionToken:    aws.String("new-token"),
+			Expiration:      aws.Time(time.Now().Add(1 * time.Hour)),
+		},
+	}, nil)
+
+	creds, err := roleProvider.Retrieve(context.TODO())
+	g.Expect(err).To(BeNil())
+	g.Expect(creds.AccessKeyID).To(Equal("new-key"))
+}
+
+func TestIsInvalidClientTokenIDError(t *testing.T) {
+	tests := []struct {
+		name   string
+		err    error
+		expect bool
+	}{
+		{
+			name:   "nil error",
+			err:    nil,
+			expect: false,
+		},
+		{
+			name: "typed smithy APIError with InvalidClientTokenId",
+			err: &smithy.GenericAPIError{
+				Code:    "InvalidClientTokenId",
+				Message: "The security token included in the request is invalid",
+			},
+			expect: true,
+		},
+		{
+			name:   "error containing InvalidClientTokenId in message",
+			err:    fmt.Errorf("operation failed: InvalidClientTokenId: token not valid"),
+			expect: true,
+		},
+		{
+			name:   "unrelated error",
+			err:    fmt.Errorf("connection timeout"),
+			expect: false,
+		},
+		{
+			name: "different API error code",
+			err: &smithy.GenericAPIError{
+				Code:    "AccessDenied",
+				Message: "not authorized",
+			},
+			expect: false,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			g.Expect(IsInvalidClientTokenIDError(tt.err)).To(Equal(tt.expect))
+		})
+	}
+}

test/e2e/shared/aws.go

@@ -937,6 +937,12 @@ func newUserAccessKey(ctx context.Context, cfg *aws.Config, userName string) *ia
 // key may not be immediately usable across all AWS services. STS GetCallerIdentity
 // is a lightweight call that validates the credential without requiring any IAM
 // permissions.
+//
+// Note: the controller itself also handles this scenario gracefully by clearing
+// the credential cache when AssumeRole returns InvalidClientTokenId (see
+// AWSRolePrincipalTypeProvider.Retrieve in pkg/cloud/identity/identity.go).
+// This e2e wait is kept as defense-in-depth to avoid unnecessary reconcile
+// churn during tests.
 func waitForAccessKeyPropagation(cfg *aws.Config) {
 	By("Waiting for access key to propagate via STS GetCallerIdentity...")
 	stsSvc := sts.NewFromConfig(*cfg)