Skip to content

Commit 0625062

Browse files
ameukamxmudrii
andauthored
Use ephemeral S3 buckets for E2E tests (#17157)
* Use ephemeral S3 buckets for E2E tests Use S3 buckets created during the lifecycle of a test instead of a static one and provide the capability to make them read-only public. Signed-off-by: Arnaud Meukam <[email protected]> * Improve ephemeral S3 buckets implementation for tests Signed-off-by: Marko Mudrinić <[email protected]> Signed-off-by: Arnaud Meukam <[email protected]> * Base S3 bucket name on ProwJob ID Signed-off-by: Marko Mudrinić <[email protected]> --------- Signed-off-by: Arnaud Meukam <[email protected]> Signed-off-by: Marko Mudrinić <[email protected]> Co-authored-by: Marko Mudrinić <[email protected]>
1 parent 4539d80 commit 0625062

File tree

6 files changed

+263
-14
lines changed

6 files changed

+263
-14
lines changed

tests/e2e/go.mod

+4-4
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,10 @@ go 1.23.5
55
replace k8s.io/kops => ../../.
66

77
require (
8+
github.com/aws/aws-sdk-go-v2 v1.31.0
9+
github.com/aws/aws-sdk-go-v2/config v1.27.38
10+
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2
11+
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2
812
github.com/blang/semver/v4 v4.0.0
913
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
1014
github.com/octago/sflags v0.2.0
@@ -66,9 +70,7 @@ require (
6670
github.com/aliyun/credentials-go v1.2.3 // indirect
6771
github.com/apparentlymart/go-cidr v1.1.0 // indirect
6872
github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
69-
github.com/aws/aws-sdk-go-v2 v1.31.0 // indirect
7073
github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.5 // indirect
71-
github.com/aws/aws-sdk-go-v2/config v1.27.38 // indirect
7274
github.com/aws/aws-sdk-go-v2/credentials v1.17.36 // indirect
7375
github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.14 // indirect
7476
github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.18 // indirect
@@ -82,10 +84,8 @@ require (
8284
github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.3.20 // indirect
8385
github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.20 // indirect
8486
github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.17.18 // indirect
85-
github.com/aws/aws-sdk-go-v2/service/s3 v1.63.2 // indirect
8687
github.com/aws/aws-sdk-go-v2/service/sso v1.23.2 // indirect
8788
github.com/aws/aws-sdk-go-v2/service/ssooidc v1.27.2 // indirect
88-
github.com/aws/aws-sdk-go-v2/service/sts v1.31.2 // indirect
8989
github.com/aws/smithy-go v1.21.0 // indirect
9090
github.com/awslabs/amazon-ecr-credential-helper/ecr-login v0.0.0-20220228164355-396b2034c795 // indirect
9191
github.com/beorn7/perks v1.0.1 // indirect

tests/e2e/kubetest2-kops/aws/s3.go

+194
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,194 @@
1+
/*
2+
Copyright 2024 The Kubernetes Authors.
3+
4+
Licensed under the Apache License, Version 2.0 (the "License");
5+
you may not use this file except in compliance with the License.
6+
You may obtain a copy of the License at
7+
8+
http://www.apache.org/licenses/LICENSE-2.0
9+
10+
Unless required by applicable law or agreed to in writing, software
11+
distributed under the License is distributed on an "AS IS" BASIS,
12+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
See the License for the specific language governing permissions and
14+
limitations under the License.
15+
*/
16+
17+
package aws
18+
19+
import (
20+
"context"
21+
"errors"
22+
"fmt"
23+
"os"
24+
"regexp"
25+
"strings"
26+
"time"
27+
28+
"github.com/aws/aws-sdk-go-v2/aws"
29+
awsconfig "github.com/aws/aws-sdk-go-v2/config"
30+
"github.com/aws/aws-sdk-go-v2/service/s3"
31+
"github.com/aws/aws-sdk-go-v2/service/s3/types"
32+
"github.com/aws/aws-sdk-go-v2/service/sts"
33+
"k8s.io/klog/v2"
34+
)
35+
36+
// defaultRegion is the region to query the AWS APIs through, this can be any AWS region is required even if we are not
37+
// running on AWS.
38+
const defaultRegion = "us-east-2"
39+
40+
// Client contains S3 and STS clients that are used to perform bucket and object actions.
41+
type Client struct {
42+
s3Client *s3.Client
43+
stsClient *sts.Client
44+
}
45+
46+
// NewAWSClient returns a new instance of awsClient configured to work in the default region (us-east-2).
47+
func NewClient(ctx context.Context) (*Client, error) {
48+
cfg, err := awsconfig.LoadDefaultConfig(ctx,
49+
awsconfig.WithRegion(defaultRegion))
50+
if err != nil {
51+
return nil, fmt.Errorf("loading AWS config: %w", err)
52+
}
53+
54+
return &Client{
55+
s3Client: s3.NewFromConfig(cfg),
56+
stsClient: sts.NewFromConfig(cfg),
57+
}, nil
58+
}
59+
60+
// BucketName constructs an unique bucket name using the AWS account ID in the default region (us-east-2).
61+
func (c Client) BucketName(ctx context.Context) (string, error) {
62+
// Construct the bucket name based on the ProwJob ID (if running in Prow) or AWS account ID (if running outside
63+
// Prow) and the current timestamp
64+
var identifier string
65+
if jobID := os.Getenv("PROW_JOB_ID"); len(jobID) >= 4 {
66+
identifier = jobID[:4]
67+
} else {
68+
callerIdentity, err := c.stsClient.GetCallerIdentity(ctx, &sts.GetCallerIdentityInput{})
69+
if err != nil {
70+
return "", fmt.Errorf("building AWS STS presigned request: %w", err)
71+
}
72+
identifier = *callerIdentity.Account
73+
}
74+
timestamp := time.Now().Format("20060102150405")
75+
bucket := fmt.Sprintf("k8s-infra-kops-%s-%s", identifier, timestamp)
76+
77+
bucket = strings.ToLower(bucket)
78+
// Only allow lowercase letters, numbers, and hyphens
79+
bucket = regexp.MustCompile("[^a-z0-9-]").ReplaceAllString(bucket, "")
80+
81+
if len(bucket) > 63 {
82+
bucket = bucket[:63] // Max length is 63
83+
}
84+
85+
return bucket, nil
86+
}
87+
88+
// EnsureS3Bucket creates a new S3 bucket with the given name and public read permissions.
89+
func (c Client) EnsureS3Bucket(ctx context.Context, bucketName string, publicRead bool) error {
90+
bucketName = strings.TrimPrefix(bucketName, "s3://")
91+
_, err := c.s3Client.CreateBucket(ctx, &s3.CreateBucketInput{
92+
Bucket: aws.String(bucketName),
93+
CreateBucketConfiguration: &types.CreateBucketConfiguration{
94+
LocationConstraint: defaultRegion,
95+
},
96+
})
97+
if err != nil {
98+
var exists *types.BucketAlreadyExists
99+
if errors.As(err, &exists) {
100+
klog.Infof("Bucket %s already exists\n", bucketName)
101+
} else {
102+
klog.Infof("Error creating bucket %s, err: %v\n", bucketName, err)
103+
}
104+
105+
return fmt.Errorf("creating bucket %s: %w", bucketName, err)
106+
}
107+
108+
// Wait for the bucket to be created
109+
err = s3.NewBucketExistsWaiter(c.s3Client).Wait(
110+
ctx, &s3.HeadBucketInput{
111+
Bucket: aws.String(bucketName),
112+
},
113+
time.Minute)
114+
if err != nil {
115+
klog.Infof("Failed attempt to wait for bucket %s to exist, err: %v", bucketName, err)
116+
117+
return fmt.Errorf("waiting for bucket %s to exist: %w", bucketName, err)
118+
}
119+
120+
klog.Infof("Bucket %s created successfully", bucketName)
121+
122+
if publicRead {
123+
err = c.setPublicReadPolicy(ctx, bucketName)
124+
if err != nil {
125+
klog.Errorf("Failed to set public read policy on bucket %s, err: %v", bucketName, err)
126+
127+
return fmt.Errorf("setting public read policy for bucket %s: %w", bucketName, err)
128+
}
129+
130+
klog.Infof("Public read policy set on bucket %s", bucketName)
131+
}
132+
133+
return nil
134+
}
135+
136+
// DeleteS3Bucket deletes a S3 bucket with the given name.
137+
func (c Client) DeleteS3Bucket(ctx context.Context, bucketName string) error {
138+
bucketName = strings.TrimPrefix(bucketName, "s3://")
139+
_, err := c.s3Client.DeleteBucket(ctx, &s3.DeleteBucketInput{
140+
Bucket: aws.String(bucketName),
141+
})
142+
if err != nil {
143+
var noBucket *types.NoSuchBucket
144+
if errors.As(err, &noBucket) {
145+
klog.Infof("Bucket %s does not exits.", bucketName)
146+
147+
return nil
148+
} else {
149+
klog.Infof("Couldn't delete bucket %s, err: %v", bucketName, err)
150+
151+
return fmt.Errorf("deleting bucket %s: %w", bucketName, err)
152+
}
153+
}
154+
155+
err = s3.NewBucketNotExistsWaiter(c.s3Client).Wait(
156+
ctx, &s3.HeadBucketInput{
157+
Bucket: aws.String(bucketName),
158+
},
159+
time.Minute)
160+
if err != nil {
161+
klog.Infof("Failed attempt to wait for bucket %s to be deleted, err: %v", bucketName, err)
162+
163+
return fmt.Errorf("waiting for bucket %s to be deleted, err: %w", bucketName, err)
164+
}
165+
166+
klog.Infof("Bucket %s deleted", bucketName)
167+
168+
return nil
169+
}
170+
171+
func (c Client) setPublicReadPolicy(ctx context.Context, bucketName string) error {
172+
policy := fmt.Sprintf(`{
173+
"Version": "2012-10-17",
174+
"Statement": [
175+
{
176+
"Sid": "PublicReadGetObject",
177+
"Effect": "Allow",
178+
"Principal": "*",
179+
"Action": "s3:GetObject",
180+
"Resource": "arn:aws:s3:::%s/*"
181+
}
182+
]
183+
}`, bucketName)
184+
185+
_, err := c.s3Client.PutBucketPolicy(ctx, &s3.PutBucketPolicyInput{
186+
Bucket: aws.String(bucketName),
187+
Policy: aws.String(policy),
188+
})
189+
if err != nil {
190+
return fmt.Errorf("putting bucket policy for %s: %w", bucketName, err)
191+
}
192+
193+
return nil
194+
}

tests/e2e/kubetest2-kops/deployer/common.go

+29-1
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,15 @@ limitations under the License.
1717
package deployer
1818

1919
import (
20+
"context"
2021
"errors"
2122
"fmt"
2223
"os"
2324
"path/filepath"
2425
"strings"
2526

2627
"k8s.io/klog/v2"
28+
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
2729
"k8s.io/kops/tests/e2e/kubetest2-kops/gce"
2830
"k8s.io/kops/tests/e2e/pkg/target"
2931
"k8s.io/kops/tests/e2e/pkg/util"
@@ -51,6 +53,12 @@ func (d *deployer) initialize() error {
5153

5254
switch d.CloudProvider {
5355
case "aws":
56+
client, err := aws.NewClient(context.Background())
57+
if err != nil {
58+
return fmt.Errorf("init failed to build AWS client: %w", err)
59+
}
60+
d.aws = client
61+
5462
if d.SSHPrivateKeyPath == "" {
5563
d.SSHPrivateKeyPath = os.Getenv("AWS_SSH_PRIVATE_KEY_FILE")
5664
}
@@ -316,34 +324,53 @@ func defaultClusterName(cloudProvider string) (string, error) {
316324
// stateStore returns the kops state store to use
317325
// defaulting to values used in prow jobs
318326
func (d *deployer) stateStore() string {
327+
if d.stateStoreName != "" {
328+
return d.stateStoreName
329+
}
319330
ss := os.Getenv("KOPS_STATE_STORE")
320331
if ss == "" {
321332
switch d.CloudProvider {
322333
case "aws":
323-
ss = "s3://k8s-kops-prow"
334+
ctx := context.Background()
335+
bucketName, err := d.aws.BucketName(ctx)
336+
if err != nil {
337+
klog.Fatalf("Failed to generate bucket name: %v", err)
338+
return ""
339+
}
340+
d.createBucket = true
341+
ss = "s3://" + bucketName
324342
case "gce":
325343
d.createBucket = true
326344
ss = "gs://" + gce.GCSBucketName(d.GCPProject, "state")
327345
case "digitalocean":
328346
ss = "do://e2e-kops-space"
329347
}
330348
}
349+
350+
d.stateStoreName = ss
331351
return ss
332352
}
333353

334354
// discoveryStore returns the VFS path to use for public OIDC documents
335355
func (d *deployer) discoveryStore() string {
356+
if d.discoveryStoreName != "" {
357+
return d.discoveryStoreName
358+
}
336359
discovery := os.Getenv("KOPS_DISCOVERY_STORE")
337360
if discovery == "" {
338361
switch d.CloudProvider {
339362
case "aws":
340363
discovery = "s3://k8s-kops-ci-prow"
341364
}
342365
}
366+
d.discoveryStoreName = discovery
343367
return discovery
344368
}
345369

346370
func (d *deployer) stagingStore() string {
371+
if d.stagingStoreName != "" {
372+
return d.stagingStoreName
373+
}
347374
sb := os.Getenv("KOPS_STAGING_BUCKET")
348375
if sb == "" {
349376
switch d.CloudProvider {
@@ -352,6 +379,7 @@ func (d *deployer) stagingStore() string {
352379
sb = "gs://" + gce.GCSBucketName(d.GCPProject, "staging")
353380
}
354381
}
382+
d.stagingStoreName = sb
355383
return sb
356384
}
357385

tests/e2e/kubetest2-kops/deployer/deployer.go

+12-3
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"github.com/octago/sflags/gen/gpflag"
2626
"github.com/spf13/pflag"
2727
"k8s.io/klog/v2"
28+
"k8s.io/kops/tests/e2e/kubetest2-kops/aws"
2829
"k8s.io/kops/tests/e2e/kubetest2-kops/builder"
2930
"k8s.io/kops/tests/e2e/pkg/target"
3031

@@ -57,7 +58,6 @@ type deployer struct {
5758
CreateArgs string `flag:"create-args" desc:"Extra space-separated arguments passed to 'kops create cluster'"`
5859
KopsBinaryPath string `flag:"kops-binary-path" desc:"The path to kops executable used for testing"`
5960
KubernetesFeatureGates string `flag:"kubernetes-feature-gates" desc:"Feature Gates to enable on Kubernetes components"`
60-
createBucket bool `flag:"-"`
6161

6262
// ControlPlaneCount specifies the number of VMs in the control-plane.
6363
ControlPlaneCount int `flag:"control-plane-count" desc:"Number of control-plane instances"`
@@ -90,6 +90,13 @@ type deployer struct {
9090
manifestPath string
9191
terraform *target.Terraform
9292

93+
aws *aws.Client
94+
95+
createBucket bool
96+
stateStoreName string
97+
discoveryStoreName string
98+
stagingStoreName string
99+
93100
// boskos struct field will be non-nil when the deployer is
94101
// using boskos to acquire a GCP project
95102
boskos *client.Client
@@ -106,8 +113,10 @@ type deployer struct {
106113
var _ types.NewDeployer = New
107114

108115
// assert that deployer implements types.Deployer
109-
var _ types.Deployer = &deployer{}
110-
var _ types.DeployerWithPostTester = &deployer{}
116+
var (
117+
_ types.Deployer = &deployer{}
118+
_ types.DeployerWithPostTester = &deployer{}
119+
)
111120

112121
func (d *deployer) Provider() string {
113122
return Name

tests/e2e/kubetest2-kops/deployer/down.go

+12-3
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@ limitations under the License.
1717
package deployer
1818

1919
import (
20+
"context"
2021
"fmt"
2122
"strings"
2223

@@ -72,9 +73,17 @@ func (d *deployer) Down() error {
7273
return err
7374
}
7475

75-
if d.CloudProvider == "gce" && d.createBucket {
76-
gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
77-
gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
76+
if d.createBucket {
77+
switch d.CloudProvider {
78+
case "aws":
79+
ctx := context.Background()
80+
if err := d.aws.DeleteS3Bucket(ctx, d.stateStore()); err != nil {
81+
return err
82+
}
83+
case "gce":
84+
gce.DeleteGCSBucket(d.stateStore(), d.GCPProject)
85+
gce.DeleteGCSBucket(d.stagingStore(), d.GCPProject)
86+
}
7887
}
7988

8089
if d.boskos != nil {

0 commit comments

Comments
 (0)