Skip to content

Commit 7aca2c5

Browse files
authored
fix: refresh AWS credentials on every signing request to survive rotation (#2756)
Replaces credentials.NewStaticCredentialsProvider with a dynamic provider (connectionConfigCredentialsProvider) that re-reads the connection config on every signing call. The static provider froze credential values at aws.Config construction time, so in-flight goroutines kept signing with stale tokens across credential rotation — surfacing as ExpiredToken errors for long-running queries and queries that started before a rotation reached the pod. Also bumps steampipe-plugin-sdk to v6.0.0 to take the upstream race fix on Connection.Config (SDK now guards the field with a sync.RWMutex and exposes it via GetConfig/SetConfig accessors). Adds a CredentialsCache integration test that rotates config mid-test and asserts the cache returns the rotated credentials after Expires elapses. Closes #2756.
1 parent 441b718 commit 7aca2c5

597 files changed

Lines changed: 2234 additions & 1840 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

aws/backup_tags.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,7 @@ import (
77
"github.com/aws/aws-sdk-go-v2/aws"
88
"github.com/aws/aws-sdk-go-v2/service/backup"
99
"github.com/aws/smithy-go"
10-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
10+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
1111
)
1212

1313
func getAwsBackupResourceTags(ctx context.Context, d *plugin.QueryData, arn string) (interface{}, error) {

aws/canonical_policy.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,8 @@ import (
1010
"strings"
1111

1212
"github.com/turbot/go-kit/types"
13-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
14-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
13+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
14+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin/transform"
1515
)
1616

1717
//

aws/cloudwatch_metric.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
"github.com/aws/aws-sdk-go-v2/aws"
1010
"github.com/aws/aws-sdk-go-v2/service/cloudwatch"
1111
"github.com/aws/aws-sdk-go-v2/service/cloudwatch/types"
12-
"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
13-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
12+
"github.com/turbot/steampipe-plugin-sdk/v6/grpc/proto"
13+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
1414
)
1515

1616
// append the common cloudwatch metric columns onto the column list

aws/common_columns.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -6,10 +6,10 @@ import (
66
"strings"
77

88
"github.com/aws/aws-sdk-go-v2/service/sts"
9-
"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
10-
"github.com/turbot/steampipe-plugin-sdk/v5/memoize"
11-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
12-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin/transform"
9+
"github.com/turbot/steampipe-plugin-sdk/v6/grpc/proto"
10+
"github.com/turbot/steampipe-plugin-sdk/v6/memoize"
11+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
12+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin/transform"
1313
)
1414

1515
// Columns defined on every account-level resource (e.g. aws_iam_access_key)

aws/connection_config.go

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import (
44
"fmt"
55
"strings"
66

7-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
7+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
88
)
99

1010
type awsConfig struct {
@@ -28,10 +28,14 @@ func ConfigInstance() interface{} {
2828

2929
// GetConfig :: retrieve and cast connection config from query data
3030
func GetConfig(connection *plugin.Connection) awsConfig {
31-
if connection == nil || connection.Config == nil {
31+
if connection == nil {
3232
return awsConfig{}
3333
}
34-
config, _ := connection.Config.(awsConfig)
34+
raw := connection.GetConfig()
35+
if raw == nil {
36+
return awsConfig{}
37+
}
38+
config, _ := raw.(awsConfig)
3539

3640
if config.Regions != nil {
3741
if len(config.Regions) == 0 {

aws/cost_explorer.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -9,8 +9,8 @@ import (
99
"github.com/aws/aws-sdk-go-v2/service/costexplorer/types"
1010
"github.com/golang/protobuf/ptypes/timestamp"
1111

12-
"github.com/turbot/steampipe-plugin-sdk/v5/grpc/proto"
13-
"github.com/turbot/steampipe-plugin-sdk/v5/plugin"
12+
"github.com/turbot/steampipe-plugin-sdk/v6/grpc/proto"
13+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
1414
)
1515

1616
// AllCostMetrics is a constant returning all the cost metrics

aws/credentials_provider.go

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
package aws
2+
3+
import (
4+
"context"
5+
"errors"
6+
"fmt"
7+
"time"
8+
9+
"github.com/aws/aws-sdk-go-v2/aws"
10+
"github.com/turbot/steampipe-plugin-sdk/v6/plugin"
11+
)
12+
13+
// connectionConfigCredentialsProvider is an aws.CredentialsProvider that reads
14+
// access_key / secret_key / session_token from the steampipe connection config
15+
// on every Retrieve() call (subject to the SDK's CredentialsCache, which we
16+
// hint at via a short Expires below).
17+
//
18+
// Background
19+
//
20+
// The steampipe plugin SDK mutates the connection config in place when a new
21+
// connection config arrives via UpdateConnectionConfigs (see
22+
// steampipe-plugin-sdk/plugin/plugin_connection_config.go upsertConnectionData,
23+
// which calls d.Connection.SetConfig(configStruct) under a write lock). The
24+
// SDK comment at that site explicitly acknowledges that a query may already be
25+
// executing with this Connection object, and that the AWS plugin in particular
26+
// "may refresh the Client using the previous credentials" — which is the bug
27+
// this provider fixes.
28+
//
29+
// A credentials.NewStaticCredentialsProvider built once at aws.Config
30+
// construction time captures the original token values. A goroutine holding
31+
// that aws.Config keeps signing requests with the original token regardless of
32+
// rotation. When the original token expires at AWS, every subsequent request
33+
// from that goroutine fails with ExpiredToken — even if a fresh valid token
34+
// has been delivered to Connection.Config by the SDK.
35+
//
36+
// By re-reading the connection config on every Retrieve(), in-flight goroutines
37+
// holding the same aws.Config pick up rotated credentials on the next signing
38+
// operation (modulo the CredentialsCache TTL we set below).
39+
type connectionConfigCredentialsProvider struct {
40+
connection *plugin.Connection
41+
}
42+
43+
// credentialsExpiresInterval is how long the returned aws.Credentials are
44+
// considered fresh by the SDK's CredentialsCache. Overridable in tests.
45+
var credentialsExpiresInterval = 60 * time.Second
46+
47+
// Retrieve implements aws.CredentialsProvider. Called by the AWS SDK on every
48+
// signed request, subject to the wrapping CredentialsCache.
49+
func (p *connectionConfigCredentialsProvider) Retrieve(_ context.Context) (creds aws.Credentials, err error) {
50+
// Belt-and-suspenders: the SDK's Connection.GetConfig acquires an RLock so
51+
// torn interface reads cannot happen via that path. This recover still
52+
// converts any other panic inside Retrieve into a clean error the AWS SDK
53+
// can retry, rather than propagating through the signing middleware.
54+
defer func() {
55+
if r := recover(); r != nil {
56+
err = fmt.Errorf("connectionConfigCredentialsProvider: panic during Retrieve for connection %q: %v", p.connectionName(), r)
57+
creds = aws.Credentials{}
58+
}
59+
}()
60+
61+
if p.connection == nil {
62+
return aws.Credentials{}, errors.New("connectionConfigCredentialsProvider: connection is nil")
63+
}
64+
65+
// Read the raw config through the SDK's Connection.GetConfig accessor
66+
// (which acquires the RLock) and type-assert directly here. Avoid the
67+
// local aws/connection_config.go GetConfig helper — it normalizes the
68+
// Regions slice in place and panics on regions = []. Neither belongs in
69+
// the AWS request signing path: Retrieve only needs the credential fields,
70+
// and a malformed connection config should not crash signing goroutines
71+
// deep inside the AWS SDK middleware.
72+
raw := p.connection.GetConfig()
73+
cfg, ok := raw.(awsConfig)
74+
if !ok {
75+
return aws.Credentials{}, fmt.Errorf("connectionConfigCredentialsProvider: connection %q config is %T, expected awsConfig", p.connection.Name, raw)
76+
}
77+
78+
if cfg.AccessKey == nil {
79+
return aws.Credentials{}, fmt.Errorf("connectionConfigCredentialsProvider: connection %q has no access_key in config", p.connection.Name)
80+
}
81+
if cfg.SecretKey == nil {
82+
return aws.Credentials{}, fmt.Errorf("connectionConfigCredentialsProvider: connection %q has no secret_key in config", p.connection.Name)
83+
}
84+
85+
var sessionToken string
86+
if cfg.SessionToken != nil {
87+
sessionToken = *cfg.SessionToken
88+
}
89+
90+
return aws.Credentials{
91+
AccessKeyID: *cfg.AccessKey,
92+
SecretAccessKey: *cfg.SecretKey,
93+
SessionToken: sessionToken,
94+
Source: "connectionConfigCredentialsProvider",
95+
// config.WithCredentialsProvider wraps any provider in a
96+
// CredentialsCache. The cache will NOT call Retrieve again until the
97+
// cached value's Expires has passed (or until cache invalidation),
98+
// so setting Expires too far out defeats the rotation-pickup goal:
99+
// the cache would hold the original creds in memory long after they
100+
// were rotated in Connection.Config.
101+
//
102+
// 60 seconds matches what the standalone reproduction harness
103+
// validated to keep rotation latency bounded. Reading
104+
// Connection.Config is an in-memory type assertion + struct copy,
105+
// so a short interval is essentially free.
106+
CanExpire: true,
107+
Expires: time.Now().Add(credentialsExpiresInterval),
108+
}, nil
109+
}
110+
111+
func (p *connectionConfigCredentialsProvider) connectionName() string {
112+
if p.connection == nil {
113+
return "<nil>"
114+
}
115+
return p.connection.Name
116+
}

0 commit comments

Comments
 (0)