From 293a26989e0a6c4fc6b3dc510a8ad6c5bb999587 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Tue, 25 Feb 2025 16:01:28 +0100 Subject: [PATCH 1/5] feat(object): add support for IAM Access Key syntax https://www.scaleway.com/en/docs/iam/api-cli/using-api-key-object-storage/#overriding-the-preferred-project-when-making-a-call --- .../namespaces/object/v1/custom_config_get.go | 6 ++++++ .../object/v1/custom_config_get_test.go | 1 + .../object/v1/custom_config_install.go | 6 ++++++ internal/namespaces/object/v1/s3configfile.go | 16 +++++++++++++--- .../testdata/test-config-get-default-mc.golden | 4 ++-- .../test-config-get-default-rclone.golden | 4 ++-- .../test-config-get-default-s3cmd.golden | 4 ++-- .../test-config-get-with-region-mc.golden | 4 ++-- .../test-config-get-with-region-rclone.golden | 4 ++-- .../test-config-get-with-region-s3cmd.golden | 4 ++-- 10 files changed, 38 insertions(+), 15 deletions(-) diff --git a/internal/namespaces/object/v1/custom_config_get.go b/internal/namespaces/object/v1/custom_config_get.go index a78515154e..30e286e5ed 100644 --- a/internal/namespaces/object/v1/custom_config_get.go +++ b/internal/namespaces/object/v1/custom_config_get.go @@ -37,6 +37,12 @@ func configGetCommand() *core.Command { Required: false, Default: core.DefaultValueSetter("scaleway"), }, + { + Name: "project-id", + Short: "Scaleway project ID to use with IAM Access Key syntax", + Required: false, + ValidateFunc: core.ValidateProjectID(), + }, core.RegionArgSpec(scw.RegionFrPar, scw.RegionNlAms), }, Examples: []*core.Example{ diff --git a/internal/namespaces/object/v1/custom_config_get_test.go b/internal/namespaces/object/v1/custom_config_get_test.go index 84fef2277f..c5784f5e64 100644 --- a/internal/namespaces/object/v1/custom_config_get_test.go +++ b/internal/namespaces/object/v1/custom_config_get_test.go @@ -16,6 +16,7 @@ func Test_ConfigGet(t *testing.T) { "11111111-1111-1111-1111-111111111111", ), scw.WithDefaultOrganizationID("11111111-1111-1111-1111-111111111111"), + scw.WithDefaultProjectID("11111111-1111-1111-1111-111111111111"), scw.WithDefaultZone(scw.ZoneFrPar1), scw.WithDefaultRegion(scw.RegionFrPar), ) diff --git a/internal/namespaces/object/v1/custom_config_install.go b/internal/namespaces/object/v1/custom_config_install.go index a4cccedc7b..f7bdef9da1 100644 --- a/internal/namespaces/object/v1/custom_config_install.go +++ b/internal/namespaces/object/v1/custom_config_install.go @@ -41,6 +41,12 @@ func configInstallCommand() *core.Command { Required: false, Default: core.DefaultValueSetter("scaleway"), }, + { + Name: "project-id", + Short: "Scaleway project ID to use with IAM Access Key syntax", + Required: false, + ValidateFunc: core.ValidateProjectID(), + }, core.RegionArgSpec(scw.RegionFrPar, scw.RegionNlAms), }, Examples: []*core.Example{ diff --git a/internal/namespaces/object/v1/s3configfile.go b/internal/namespaces/object/v1/s3configfile.go index 28f54582e2..fcf81f76c6 100644 --- a/internal/namespaces/object/v1/s3configfile.go +++ b/internal/namespaces/object/v1/s3configfile.go @@ -37,6 +37,7 @@ type s3config struct { Region scw.Region Name string ctx context.Context + ProjectID string } const ( @@ -55,7 +56,7 @@ const s3cmdTemplate = `# Generated by scaleway-cli command # Configuration file for s3cmd https://s3tools.org/s3cmd # Default location: $HOME/.s3cfg [default] -access_key = {{ .AccessKey }} +access_key = {{ .AccessKey }}@{{ .ProjectID }} bucket_location = {{ .Region }} host_base = s3.{{ .Region }}.scw.cloud host_bucket = %(bucket)s.s3.{{ .Region }}.scw.cloud @@ -71,7 +72,7 @@ type = s3 provider = Scaleway env_auth = false endpoint = s3.{{ .Region }}.scw.cloud -access_key_id = {{ .AccessKey }} +access_key_id = {{ .AccessKey }}@{{ .ProjectID }} secret_access_key = {{ .SecretKey }} region = {{ .Region }} location_constraint = @@ -87,16 +88,24 @@ func newS3Config(ctx context.Context, region scw.Region, name string) (s3config, if !accessExists { return s3config{}, errors.New("no access key found") } + secretKey, secretExists := client.GetSecretKey() if !secretExists { return s3config{}, errors.New("no secret key found") } + + projectID, exists := client.GetDefaultProjectID() + if !exists { + return s3config{}, errors.New("no project ID found") + } + config := s3config{ AccessKey: accessKey, SecretKey: secretKey, Region: region, Name: name, ctx: ctx, + ProjectID: projectID, } return config, nil @@ -144,6 +153,7 @@ func (c s3config) getConfigFile(tool s3tool) (core.RawResult, error) { AccessKey string `json:"accessKey"` SecretKey string `json:"secretKey"` API string `json:"api"` + ProjectID string `json:"projectID"` } m := struct { Version string `json:"version"` @@ -153,7 +163,7 @@ func (c s3config) getConfigFile(tool s3tool) (core.RawResult, error) { Hosts: map[string]hostconfig{ c.Name: { URL: "https://s3." + c.Region.String() + ".scw.cloud", - AccessKey: c.AccessKey, + AccessKey: c.AccessKey + "@" + c.ProjectID, SecretKey: c.SecretKey, API: "S3v4", }, diff --git a/internal/namespaces/object/v1/testdata/test-config-get-default-mc.golden b/internal/namespaces/object/v1/testdata/test-config-get-default-mc.golden index 5734bd2208..f2dbdbaafb 100644 --- a/internal/namespaces/object/v1/testdata/test-config-get-default-mc.golden +++ b/internal/namespaces/object/v1/testdata/test-config-get-default-mc.golden @@ -1,5 +1,5 @@ 🎲🎲🎲 EXIT CODE: 0 🎲🎲🎲 🟩🟩🟩 STDOUT️ 🟩🟩🟩️ -{"version":"9","hosts":{"scaleway":{"url":"https://s3.fr-par.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4"}}} +{"version":"9","hosts":{"scaleway":{"url":"https://s3.fr-par.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4","projectID":""}}} 🟩🟩🟩 JSON STDOUT 🟩🟩🟩 -{"version":"9","hosts":{"scaleway":{"url":"https://s3.fr-par.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4"}}} +{"version":"9","hosts":{"scaleway":{"url":"https://s3.fr-par.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4","projectID":""}}} diff --git a/internal/namespaces/object/v1/testdata/test-config-get-default-rclone.golden b/internal/namespaces/object/v1/testdata/test-config-get-default-rclone.golden index fbefdbbf60..012e5df6b3 100644 --- a/internal/namespaces/object/v1/testdata/test-config-get-default-rclone.golden +++ b/internal/namespaces/object/v1/testdata/test-config-get-default-rclone.golden @@ -8,7 +8,7 @@ type = s3 provider = Scaleway env_auth = false endpoint = s3.fr-par.scw.cloud -access_key_id = SCWXXXXXXXXXXXXXXXXX +access_key_id = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 secret_access_key = 11111111-1111-1111-1111-111111111111 region = fr-par location_constraint = @@ -25,7 +25,7 @@ type = s3 provider = Scaleway env_auth = false endpoint = s3.fr-par.scw.cloud -access_key_id = SCWXXXXXXXXXXXXXXXXX +access_key_id = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 secret_access_key = 11111111-1111-1111-1111-111111111111 region = fr-par location_constraint = diff --git a/internal/namespaces/object/v1/testdata/test-config-get-default-s3cmd.golden b/internal/namespaces/object/v1/testdata/test-config-get-default-s3cmd.golden index 2f84f5d7ee..284f491807 100644 --- a/internal/namespaces/object/v1/testdata/test-config-get-default-s3cmd.golden +++ b/internal/namespaces/object/v1/testdata/test-config-get-default-s3cmd.golden @@ -4,7 +4,7 @@ # Configuration file for s3cmd https://s3tools.org/s3cmd # Default location: $HOME/.s3cfg [default] -access_key = SCWXXXXXXXXXXXXXXXXX +access_key = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 bucket_location = fr-par host_base = s3.fr-par.scw.cloud host_bucket = %(bucket)s.s3.fr-par.scw.cloud @@ -15,7 +15,7 @@ use_https = True # Configuration file for s3cmd https://s3tools.org/s3cmd # Default location: $HOME/.s3cfg [default] -access_key = SCWXXXXXXXXXXXXXXXXX +access_key = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 bucket_location = fr-par host_base = s3.fr-par.scw.cloud host_bucket = %(bucket)s.s3.fr-par.scw.cloud diff --git a/internal/namespaces/object/v1/testdata/test-config-get-with-region-mc.golden b/internal/namespaces/object/v1/testdata/test-config-get-with-region-mc.golden index ff758202a2..4c0d4252ee 100644 --- a/internal/namespaces/object/v1/testdata/test-config-get-with-region-mc.golden +++ b/internal/namespaces/object/v1/testdata/test-config-get-with-region-mc.golden @@ -1,5 +1,5 @@ 🎲🎲🎲 EXIT CODE: 0 🎲🎲🎲 🟩🟩🟩 STDOUT️ 🟩🟩🟩️ -{"version":"9","hosts":{"scaleway":{"url":"https://s3.nl-ams.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4"}}} +{"version":"9","hosts":{"scaleway":{"url":"https://s3.nl-ams.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4","projectID":""}}} 🟩🟩🟩 JSON STDOUT 🟩🟩🟩 -{"version":"9","hosts":{"scaleway":{"url":"https://s3.nl-ams.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4"}}} +{"version":"9","hosts":{"scaleway":{"url":"https://s3.nl-ams.scw.cloud","accessKey":"SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111","secretKey":"11111111-1111-1111-1111-111111111111","api":"S3v4","projectID":""}}} diff --git a/internal/namespaces/object/v1/testdata/test-config-get-with-region-rclone.golden b/internal/namespaces/object/v1/testdata/test-config-get-with-region-rclone.golden index c8bf893230..ebe4a2dd53 100644 --- a/internal/namespaces/object/v1/testdata/test-config-get-with-region-rclone.golden +++ b/internal/namespaces/object/v1/testdata/test-config-get-with-region-rclone.golden @@ -8,7 +8,7 @@ type = s3 provider = Scaleway env_auth = false endpoint = s3.nl-ams.scw.cloud -access_key_id = SCWXXXXXXXXXXXXXXXXX +access_key_id = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 secret_access_key = 11111111-1111-1111-1111-111111111111 region = nl-ams location_constraint = @@ -25,7 +25,7 @@ type = s3 provider = Scaleway env_auth = false endpoint = s3.nl-ams.scw.cloud -access_key_id = SCWXXXXXXXXXXXXXXXXX +access_key_id = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 secret_access_key = 11111111-1111-1111-1111-111111111111 region = nl-ams location_constraint = diff --git a/internal/namespaces/object/v1/testdata/test-config-get-with-region-s3cmd.golden b/internal/namespaces/object/v1/testdata/test-config-get-with-region-s3cmd.golden index d4051acce6..1fdffc3306 100644 --- a/internal/namespaces/object/v1/testdata/test-config-get-with-region-s3cmd.golden +++ b/internal/namespaces/object/v1/testdata/test-config-get-with-region-s3cmd.golden @@ -4,7 +4,7 @@ # Configuration file for s3cmd https://s3tools.org/s3cmd # Default location: $HOME/.s3cfg [default] -access_key = SCWXXXXXXXXXXXXXXXXX +access_key = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 bucket_location = nl-ams host_base = s3.nl-ams.scw.cloud host_bucket = %(bucket)s.s3.nl-ams.scw.cloud @@ -15,7 +15,7 @@ use_https = True # Configuration file for s3cmd https://s3tools.org/s3cmd # Default location: $HOME/.s3cfg [default] -access_key = SCWXXXXXXXXXXXXXXXXX +access_key = SCWXXXXXXXXXXXXXXXXX@11111111-1111-1111-1111-111111111111 bucket_location = nl-ams host_base = s3.nl-ams.scw.cloud host_bucket = %(bucket)s.s3.nl-ams.scw.cloud From 18d2c7b60268e6c9f39effedb32ba84e9d3203a8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Tue, 25 Feb 2025 16:18:22 +0100 Subject: [PATCH 2/5] Fix --- internal/namespaces/object/v1/custom_config_install_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/internal/namespaces/object/v1/custom_config_install_test.go b/internal/namespaces/object/v1/custom_config_install_test.go index f4a5981665..226aefdb9e 100644 --- a/internal/namespaces/object/v1/custom_config_install_test.go +++ b/internal/namespaces/object/v1/custom_config_install_test.go @@ -18,6 +18,7 @@ func Test_ConfigInstall(t *testing.T) { "11111111-1111-1111-1111-111111111111", ), scw.WithDefaultOrganizationID("11111111-1111-1111-1111-111111111111"), + scw.WithDefaultProjectID("11111111-1111-1111-1111-111111111111"), scw.WithDefaultZone(scw.ZoneFrPar1), scw.WithDefaultRegion(scw.RegionFrPar), ) From 627da63eed49d16238427c325e31dc539abf8d9b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Tue, 25 Feb 2025 16:24:03 +0100 Subject: [PATCH 3/5] Fix --- docs/commands/object.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/commands/object.md b/docs/commands/object.md index 20d9ae96b7..347262413a 100644 --- a/docs/commands/object.md +++ b/docs/commands/object.md @@ -146,6 +146,7 @@ scw object config get [arg=value ...] |------|---|-------------| | type | Required
One of: `rclone`, `s3cmd`, `mc` | Type of S3 tool you want to generate a config for | | name | Default: `scaleway` | Name of the s3 remote you want to generate | +| project-id | | Scaleway project ID to use with IAM Access Key syntax | | region | Default: `fr-par`
One of: `fr-par`, `nl-ams` | Region to target. If none is passed will use default region from the config | @@ -187,6 +188,7 @@ scw object config install [arg=value ...] |------|---|-------------| | type | Required
One of: `rclone`, `s3cmd`, `mc` | Type of S3 tool you want to generate a config for | | name | Default: `scaleway` | Name of the s3 remote you want to generate | +| project-id | | Scaleway project ID to use with IAM Access Key syntax | | region | Default: `fr-par`
One of: `fr-par`, `nl-ams` | Region to target. If none is passed will use default region from the config | From 45e78ed9501692a0ca0474e94e81a53c2d391f16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Wed, 26 Feb 2025 09:18:06 +0100 Subject: [PATCH 4/5] Update internal/namespaces/object/v1/s3configfile.go Co-authored-by: Olivier Cano --- internal/namespaces/object/v1/s3configfile.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/internal/namespaces/object/v1/s3configfile.go b/internal/namespaces/object/v1/s3configfile.go index fcf81f76c6..63809b1c03 100644 --- a/internal/namespaces/object/v1/s3configfile.go +++ b/internal/namespaces/object/v1/s3configfile.go @@ -99,13 +99,15 @@ func newS3Config(ctx context.Context, region scw.Region, name string) (s3config, return s3config{}, errors.New("no project ID found") } + if projectID != "" { + accessKey+="@"+projectID + } config := s3config{ AccessKey: accessKey, SecretKey: secretKey, Region: region, Name: name, ctx: ctx, - ProjectID: projectID, } return config, nil From 15d29003ef980dafb76de3ed6823b6ed8c0af2c4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?R=C3=A9my=20L=C3=A9one?= Date: Wed, 26 Feb 2025 09:18:16 +0100 Subject: [PATCH 5/5] Update internal/namespaces/object/v1/s3configfile.go Co-authored-by: Olivier Cano --- internal/namespaces/object/v1/s3configfile.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/namespaces/object/v1/s3configfile.go b/internal/namespaces/object/v1/s3configfile.go index 63809b1c03..dc862ca396 100644 --- a/internal/namespaces/object/v1/s3configfile.go +++ b/internal/namespaces/object/v1/s3configfile.go @@ -56,7 +56,7 @@ const s3cmdTemplate = `# Generated by scaleway-cli command # Configuration file for s3cmd https://s3tools.org/s3cmd # Default location: $HOME/.s3cfg [default] -access_key = {{ .AccessKey }}@{{ .ProjectID }} +access_key = {{ .AccessKey }} bucket_location = {{ .Region }} host_base = s3.{{ .Region }}.scw.cloud host_bucket = %(bucket)s.s3.{{ .Region }}.scw.cloud