Merge pull request #2 from base-org/s3-iam-support

Add support for IAM access to S3

Author: danyalprout

api/Makefile

@@ -10,4 +10,4 @@ test:
 .PHONY: \
 	blob-api \
 	clean \
-	test
+	test

common/flags/config.go

@@ -8,32 +8,44 @@ import (
 )
 
 type DataStorage string
+type S3CredentialType string
 
 const (
-	DataStorageUnknown DataStorage = "unknown"
-	DataStorageS3      DataStorage = "s3"
-	DataStorageFile    DataStorage = "file"
+	DataStorageUnknown  DataStorage      = "unknown"
+	DataStorageS3       DataStorage      = "s3"
+	DataStorageFile     DataStorage      = "file"
+	S3CredentialUnknown S3CredentialType = "unknown"
+	S3CredentialStatic  S3CredentialType = "static"
+	S3CredentialIAM     S3CredentialType = "iam"
 )
 
 type S3Config struct {
-	Endpoint        string
-	AccessKey       string
-	SecretAccessKey string
-	UseHttps        bool
-	Bucket          string
+	Endpoint string
+	UseHttps bool
+	Bucket   string
+
+	S3CredentialType S3CredentialType
+	AccessKey        string
+	SecretAccessKey  string
 }
 
 func (c S3Config) check() error {
 	if c.Endpoint == "" {
 		return errors.New("s3 endpoint must be set")
 	}
 
-	if c.AccessKey == "" {
-		return errors.New("s3 access key must be set")
+	if c.S3CredentialType == S3CredentialUnknown {
+		return errors.New("s3 credential type must be set")
 	}
 
-	if c.SecretAccessKey == "" {
-		return errors.New("s3 secret access key must be set")
+	if c.S3CredentialType == S3CredentialStatic {
+		if c.AccessKey == "" {
+			return errors.New("s3 access key must be set")
+		}
+
+		if c.SecretAccessKey == "" {
+			return errors.New("s3 secret access key must be set")
+		}
 	}
 
 	if c.Bucket == "" {
@@ -85,12 +97,22 @@ func toDataStorage(s string) DataStorage {
 
 func readS3Config(ctx *cli.Context) S3Config {
 	return S3Config{
-		Endpoint:        ctx.String(S3EndpointFlagName),
-		AccessKey:       ctx.String(S3AccessKeyFlagName),
-		SecretAccessKey: ctx.String(S3SecretAccessKeyFlagName),
-		UseHttps:        ctx.Bool(S3EndpointHttpsFlagName),
-		Bucket:          ctx.String(S3BucketFlagName),
+		Endpoint:         ctx.String(S3EndpointFlagName),
+		AccessKey:        ctx.String(S3AccessKeyFlagName),
+		SecretAccessKey:  ctx.String(S3SecretAccessKeyFlagName),
+		UseHttps:         ctx.Bool(S3EndpointHttpsFlagName),
+		Bucket:           ctx.String(S3BucketFlagName),
+		S3CredentialType: toS3CredentialType(ctx.String(S3CredentialTypeFlagName)),
+	}
+}
+
+func toS3CredentialType(s string) S3CredentialType {
+	if s == string(S3CredentialStatic) {
+		return S3CredentialStatic
+	} else if s == string(S3CredentialIAM) {
+		return S3CredentialIAM
 	}
+	return S3CredentialUnknown
 }
 
 func (c BeaconConfig) Check() error {

common/flags/flags.go

@@ -9,6 +9,7 @@ const (
 	BeaconHttpFlagName              = "l1-beacon-http"
 	BeaconHttpClientTimeoutFlagName = "l1-beacon-client-timeout"
 	DataStoreFlagName               = "data-store"
+	S3CredentialTypeFlagName        = "s3-credential-type"
 	S3EndpointFlagName              = "s3-endpoint"
 	S3EndpointHttpsFlagName         = "s3-endpoint-https"
 	S3AccessKeyFlagName             = "s3-access-key"
@@ -34,6 +35,11 @@ func CLIFlags(envPrefix string) []cli.Flag {
 		},
 		// Optional Flags
 		// S3 Data Store Flags
+		&cli.StringFlag{
+			Name:    S3CredentialTypeFlagName,
+			Usage:   "The way to authenticate to S3, options are [iam, static]",
+			EnvVars: opservice.PrefixEnvVar(envPrefix, "S3_CREDENTIAL_TYPE"),
+		},
 		&cli.StringFlag{
 			Name:    S3EndpointFlagName,
 			Usage:   "The URL for the S3 bucket (without the scheme http or https specified)",

common/storage/s3.go

@@ -19,8 +19,15 @@ type S3Storage struct {
 }
 
 func NewS3Storage(cfg flags.S3Config, l log.Logger) (*S3Storage, error) {
+	var c *credentials.Credentials
+	if cfg.S3CredentialType == flags.S3CredentialStatic {
+		c = credentials.NewStaticV4(cfg.AccessKey, cfg.SecretAccessKey, "")
+	} else {
+		c = credentials.NewIAM("")
+	}
+
 	client, err := minio.New(cfg.Endpoint, &minio.Options{
-		Creds:  credentials.NewStaticV4(cfg.AccessKey, cfg.SecretAccessKey, ""),
+		Creds:  c,
 		Secure: cfg.UseHttps,
 	})
 

common/storage/s3_test.go

@@ -24,11 +24,12 @@ func setupS3(t *testing.T) *S3Storage {
 	l := testlog.Logger(t, log.LvlInfo)
 
 	s3, err := NewS3Storage(flags.S3Config{
-		Endpoint:        "localhost:9000",
-		AccessKey:       "admin",
-		SecretAccessKey: "password",
-		UseHttps:        false,
-		Bucket:          "blobs",
+		Endpoint:         "localhost:9000",
+		AccessKey:        "admin",
+		SecretAccessKey:  "password",
+		UseHttps:         false,
+		Bucket:           "blobs",
+		S3CredentialType: flags.S3CredentialStatic,
 	}, l)
 
 	require.NoError(t, err)