diff --git a/.github/workflows/test-aws-sdk-clients.yaml b/.github/workflows/test-aws-sdk-clients.yaml new file mode 100644 index 0000000000..96cafaf6e8 --- /dev/null +++ b/.github/workflows/test-aws-sdk-clients.yaml @@ -0,0 +1,62 @@ +name: Test AWS SDK Clients +on: [push, pull_request, workflow_dispatch] +# it should run nightly, in the push only for CR step +# on: +# schedule: +# - cron: "0 1 * * *" +# workflow_dispatch: + +jobs: + test-aws-sdk-clients: + runs-on: ubuntu-latest + timeout-minutes: 90 + concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + repository: 'noobaa/noobaa-core' + path: 'noobaa-core' + + - name: Run AWS SDK Clients + run: | + set -x + cd ./noobaa-core + make test-aws-sdk-clients + + # # this should be removed + # - name: Test slack webhook + # uses: slackapi/slack-github-action@v2.0.0 + # with: + # webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + # webhook-type: incoming-webhook + # payload: | + # text: "Hello world (Shira is testing webhook)" + + # this should be removed + - name: Test slack webhook 2 + run: | + echo "check the variable" + echo "${{ secrets.SLACK_WEBHOOK_URL }}" | wc -c + echo "send the massage" + curl -X POST ${{ secrets.SLACK_WEBHOOK_URL }} \ + -H "Content-type: application/json" \ + -d '{"text": "hello_world (Shira) with curl in github action"}' + + # this should be added + # later we would add the if: ${{ failure() }} condition + # - name: Post a message in a channel + # uses: slackapi/slack-github-action@v2.0.0 + # with: + # webhook: ${{ secrets.SLACK_WEBHOOK_URL }} + # webhook-type: incoming-webhook + # payload: | + # text: "*GitHub Action Test AWS SDK Clients Result*: ${{ job.status }}\n" + # blocks: + # - type: "section" + # text: + # type: "mrkdwn" + # text: "GitHub Action build result: ${{ job.status }}\n" + diff --git a/Makefile b/Makefile index aab584e828..6419fc2602 100644 --- a/Makefile +++ b/Makefile @@ -229,6 +229,12 @@ build-ssl-postgres: tester @echo "##\033[1;32m Build image postgres:ssl done.\033[0m" .PHONY: build-ssl-postgres +build-aws-client: noobaa + @echo "\n##\033[1;32m Build image for AWS Client tests ...\033[0m" + $(CONTAINER_ENGINE) build $(CONTAINER_PLATFORM_FLAG) $(CPUSET) -f src/deploy/NVA_build/AWSClient.Dockerfile $(CACHE_FLAG) $(NETWORK_FLAG) -t noobaa-aws-client . $(REDIRECT_STDOUT) + @echo "\033[1;32mBuild image for AWS Client tests done.\033[0m" +.PHONY: build-aws-client + test: tester @echo "\033[1;34mRunning tests with Mongo.\033[0m" @$(call create_docker_network) @@ -363,6 +369,17 @@ test-external-pg-sanity: build-ssl-postgres @$(call remove_docker_network) .PHONY: test-external-pg-sanity +test-aws-sdk-clients: build-aws-client + @echo "\033[1;34mRunning tests with Postgres.\033[0m" + @$(call create_docker_network) + @$(call run_postgres) + @echo "\033[1;34mRunning aws sdk clients tests\033[0m" + $(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "NOOBAA_LOG_LEVEL=all" -v $(PWD)/logs:/logs noobaa-aws-client ./src/test/unit_tests/run_npm_test_on_test_container.sh -c ./node_modules/mocha/bin/mocha.js src/test/unit_tests/different_clients/test_go_sdkv2_script.js + @$(call stop_noobaa) + @$(call stop_postgres) + @$(call remove_docker_network) +.PHONY: test-aws-sdk-clients + clean: @echo Stopping and Deleting containers @$(CONTAINER_ENGINE) ps -a | grep noobaa_ | awk '{print $1}' | xargs $(CONTAINER_ENGINE) stop &> /dev/null diff --git a/src/deploy/NVA_build/AWSClient.Dockerfile b/src/deploy/NVA_build/AWSClient.Dockerfile new file mode 100644 index 0000000000..918c50bbab --- /dev/null +++ b/src/deploy/NVA_build/AWSClient.Dockerfile @@ -0,0 +1,33 @@ +FROM noobaa + +USER 0:0 + +ENV container=docker +ENV TEST_CONTAINER=true + +WORKDIR /root/node_modules/noobaa-core/ + +# check npm version (installing dev dependency in different between npm version pre version 7 that was --only=dev) +RUN npm -v +RUN npm install --omit=prod + +############################################################################ +# Layers: +# Title: Install go and modules +# for testing with AWS SDK GO client with most updated version + +############################################################################ + +# installing go +RUN dnf install -y golang +# verify go installation +RUN go version +# set the PATH for go +ENV PATH="/usr/local/go/bin:$PATH" + +# install the needed modules +# note: the files go.mod and go.sum will be automatically created after this step in the WORKDIR +RUN go mod init src/test/unit_tests/different_clients +RUN go mod tidy + +USER 10001:0 diff --git a/src/test/unit_tests/coretest.js b/src/test/unit_tests/coretest.js index f140ac0c90..1872b732b1 100644 --- a/src/test/unit_tests/coretest.js +++ b/src/test/unit_tests/coretest.js @@ -233,6 +233,7 @@ function setup(options = {}) { }); rpc_client.options.auth_token = token; await overwrite_system_address(SYSTEM); + console.log('pools_to_create.length', pools_to_create.length); if (pools_to_create.length > 0) { await announce('setup_pools()'); await setup_pools(pools_to_create); diff --git a/src/test/unit_tests/different_clients/go_aws_sdkv2_client.go b/src/test/unit_tests/different_clients/go_aws_sdkv2_client.go new file mode 100644 index 0000000000..37d29302dd --- /dev/null +++ b/src/test/unit_tests/different_clients/go_aws_sdkv2_client.go @@ -0,0 +1,345 @@ +package main + +import ( + "context" + "crypto/tls" + "flag" + "fmt" + "log" + "net/http" + "os" + "strings" + + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/credentials" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/aws-sdk-go-v2/service/s3/types" +) + +var ( + accessKeyId string + secretKeyId string + awsRegion string + passingTests []string + failingTests []string + skippedTests []string + bucket string + key string + mpu string + endpointAddress string + disableDeletion bool +) + +type testName string + +const ( + createBucketTest testName = "createBucket" + deleteBucketTest testName = "deleteBucket" + putObjectTest testName = "putObject" + deleteObjectTest testName = "deleteObject" + createMultipartUploadTest testName = "createMultipartUpload" + uploadPartTest testName = "uploadPart" + completeMultiPartUploadTest testName = "completeMultiPartUpload" + deleteMultipartUploadObjectTest testName = "deleteMultipartUploadObject" +) + +func init() { + flag.StringVar(&bucket, "bucket", "", "The `name` of the S3 bucket to create and add objects to.") + flag.StringVar(&key, "key", "", "The `name` of the S3 object to put.") + flag.StringVar(&mpu, "mpu", "", "The `name` of the S3 multi part upload.") + flag.StringVar(&endpointAddress, "endpoint", "", "Set if you want non-AWS S3 endpoint.") + flag.BoolVar(&disableDeletion, "disable-deletion", false, "Set to true if you want to disable deletion of the the objects and the bucket.") +} + +// before running the script need to run: +// if it's AWS - use AWS credentials and details +// if it's NooBaa - use admin credentials and details from NooBaa system +// $ export AWS_ACCESS_KEY_ID= +// $ export AWS_SECRET_ACCESS_KEY= +// $ export AWS_DEFAULT_REGION= + +// tested with the following command: +// on AWS: +// go run script/client_script.go -bucket -key -mpu -endpoint "" [-disable-deletion] +// for example: +// go run script/client_script.go -bucket shira-test02 -key test-key -mpu test-mpu-key -endpoint "" -disable-deletion +// on NooBaa: +// go run script/client_script.go -bucket -key -mpu -endpoint "" [-disable-deletion] +// for example: +// (MCG with admin credentials and kubectl port-forward -n test3 service/s3 12443:443) +// go run script/client_script.go -bucket second.bucket -key test-key -mpu test-mpu-key -endpoint "https://localhost:12443" -disable-deletion +func main() { + printTestIntroduction() + flag.Parse() + checkRequiredFlags() + client := configureS3Client() + runTests(client) + deleteObjectsAndBucket(client) + printTestSummary() +} + +// readEnv will read the environment variable and return its value +func readEnv(envVar string) string { + value, exists := os.LookupEnv(envVar) + if !exists { + fmt.Printf("Env %s is not set\n", envVar) + } + return value +} + +// checkRequiredFlag will check if the flag is empty and print the default flags +func checkRequiredFlag(flagName string) { + if len(flagName) == 0 { + flag.PrintDefaults() + log.Fatalf("invalid parameters, %s name required", flagName) + } +} + +// checkRequiredFlags will check if each of the required flags is empty +func checkRequiredFlags() { + checkRequiredFlag(bucket) + checkRequiredFlag(key) + checkRequiredFlag(mpu) +} + +// printTestIntroduction will print the introduction of the test +func printTestIntroduction() { + fmt.Println("Running a couple of tests using AWS SDK Go V2...") + fmt.Println("--------------------------------------------------") +} + +// printTestSummary will print the summary of the test (passing, failing and skipped tests) +func printTestSummary() { + totalTests := len(passingTests) + len(failingTests) + fmt.Println("--------------------------------------------------") + fmt.Println("\nTotal Tests:", totalTests, + "\nPassing Tests: ", len(passingTests), passingTests, + "\nFailing Tests: ", len(failingTests), failingTests, + "\nSkipped Tests: ", len(skippedTests), skippedTests, + ) +} + +// runTests will run the tests +// according to the following list: Create Bucket, Put Object, Put Multipart Objects +// the deletion operation (Delete Object, Delete Multipart Upload Object and Delete Bucket) were moved to a separate function +func runTests(client *s3.Client) { + createBucket(client, bucket) + putObject(client, bucket, key) + uploadId := createMultipartUpload(client, bucket, mpu) + if uploadId != nil { + completedParts := uploadPart(client, bucket, mpu, uploadId) + if completedParts != nil { + completeMultiPartUpload(client, bucket, mpu, uploadId, completedParts) + } else { + skippedTests = append(skippedTests, "completeMultiPartUpload") + } + } else { + skippedTests = append(skippedTests, "uploadPart") + skippedTests = append(skippedTests, "completeMultiPartUpload") + } +} + +// deleteObjectsAndBucket will delete the 2 created objects (one from Put Objects and the other from Multipart Upload Objects) and the bucket +func deleteObjectsAndBucket(client *s3.Client) { + if disableDeletion { + fmt.Println("--------------------------------------------------") + fmt.Println("disable-deletion flag was set, will not operate delete commands") + } + enableDeletion := !disableDeletion + if enableDeletion { + deleteObject(client, bucket, key, string(deleteObjectTest)) + deleteObject(client, bucket, mpu, string(deleteMultipartUploadObjectTest)) + deleteBucket(client, bucket) + } +} + +// configureS3Client will configure the S3 client according to the endpoint address +func configureS3Client() *s3.Client { + var client *s3.Client + accessKeyId = readEnv("AWS_ACCESS_KEY_ID") + secretKeyId = readEnv("AWS_SECRET_ACCESS_KEY") + awsRegion = readEnv("AWS_DEFAULT_REGION") + + if endpointAddress == "" { + fmt.Println("Running on AWS endpoint") + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, secretKeyId, "")), + config.WithRegion(awsRegion)) + if err != nil { + log.Fatalf("failed to load SDK configuration (AWS endpoint), %v", err) + } + client = s3.NewFromConfig(cfg) + } else { + fmt.Println("Running on configured endpoint", endpointAddress) + cfg, err := config.LoadDefaultConfig( + context.TODO(), + config.WithCredentialsProvider(credentials.NewStaticCredentialsProvider(accessKeyId, secretKeyId, "")), + config.WithHTTPClient(&http.Client{Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}}), + ) + if err != nil { + log.Fatalf("failed to load SDK configuration (non AWS endpoint with address %s), %v", endpointAddress, err) + } + client = s3.NewFromConfig(cfg, func(o *s3.Options) { + o.BaseEndpoint = &endpointAddress + o.UsePathStyle = true + }) + } + return client +} + +// createBucket will create a bucket with the given name +func createBucket(client *s3.Client, bucketName string) { + fmt.Printf("\ncreating bucket %s\n", bucketName) + + params := &s3.CreateBucketInput{ + Bucket: &bucketName, + } + _, err := client.CreateBucket(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to create bucket %s. error: \n%v\n", bucketName, err) + failingTests = append(failingTests, string(createBucketTest)) + } else { + fmt.Printf("succeeded in create bucket %s.\n", bucketName) + passingTests = append(passingTests, string(createBucketTest)) + } +} + +// deleteBucket will delete the bucket with the given name +func deleteBucket(client *s3.Client, bucketName string) { + fmt.Printf("\ndeleting bucket %s\n", bucketName) + + params := &s3.DeleteBucketInput{ + Bucket: &bucketName, + } + _, err := client.DeleteBucket(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to delete bucket %s. error: \n%v\n", bucketName, err) + failingTests = append(failingTests, string(deleteBucketTest)) + } else { + fmt.Printf("succeeded in delete bucket %s.\n", bucketName) + passingTests = append(passingTests, string(deleteBucketTest)) + } +} + +// putObject will put an object with the given key in the given bucket +func putObject(client *s3.Client, bucketName, keyName string) { + fmt.Printf("\nput object %s in bucket %s\n", keyName, bucketName) + content := "body for example" + + params := &s3.PutObjectInput{ + Bucket: &bucketName, + Key: &keyName, + Body: strings.NewReader(content), // pass string directly as io.Reader + } + _, err := client.PutObject(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to put object %s in bucket %s. error: \n%v\n", keyName, bucketName, err) + failingTests = append(failingTests, string(putObjectTest)) + } else { + fmt.Printf("succeeded in put object %s bucket %s.\n", keyName, bucketName) + passingTests = append(passingTests, string(putObjectTest)) + } +} + +// deleteObject will delete the object with the given key in the given bucket +func deleteObject(client *s3.Client, bucketName, keyName, testNameOperation string) { + fmt.Printf("\ndelete object %s in bucket %s\n", keyName, bucketName) + + params := &s3.DeleteObjectInput{ + Bucket: &bucketName, + Key: &keyName, + } + _, err := client.DeleteObject(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to delete object %s in bucket %s. error: \n%v\n", keyName, bucketName, err) + failingTests = append(failingTests, string(testNameOperation)) + } else { + fmt.Printf("succeeded in delete object %s bucket %s.\n", keyName, bucketName) + passingTests = append(passingTests, string(testNameOperation)) + } +} + +// deleteObject will delete the object with the given key in the given bucket +func createMultipartUpload(client *s3.Client, bucketName, keyName string) *string { + fmt.Printf("\ncreate multi part upload %s in bucket %s\n", keyName, bucketName) + var output *s3.CreateMultipartUploadOutput + + params := &s3.CreateMultipartUploadInput{ + Bucket: &bucketName, + Key: &keyName, + } + output, err := client.CreateMultipartUpload(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to create multipart upload %s in bucket %s. error: \n%v\n", keyName, bucketName, err) + failingTests = append(failingTests, string(createMultipartUploadTest)) + return nil + } else { + fmt.Printf("succeeded in create multipart upload %s bucket %s.\n", keyName, bucketName) + passingTests = append(passingTests, string(createMultipartUploadTest)) + return output.UploadId + } +} + +// uploadPart will upload a part with the given key in the given bucket +func uploadPart(client *s3.Client, bucketName, keyName string, uploadId *string) []types.CompletedPart { + fmt.Printf("\nupload part %s in bucket %s\n", keyName, bucketName) + content := "body for example mpu" + var partNumber int32 = 1 + var output *s3.UploadPartOutput + + params := &s3.UploadPartInput{ + Bucket: &bucketName, + Key: &keyName, + PartNumber: &partNumber, + UploadId: uploadId, + Body: strings.NewReader(content), // pass string directly as io.Reader + } + var completedParts []types.CompletedPart + + output, err := client.UploadPart(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to upload part %d %s in bucket %s. error: \n%v\n", partNumber, keyName, bucketName, err) + failingTests = append(failingTests, string(uploadPartTest)) + return nil + } else { + fmt.Printf("succeeded in upload part %d %s in bucket %s.\n", partNumber, keyName, bucketName) + passingTests = append(passingTests, string(uploadPartTest)) + completedParts = append(completedParts, types.CompletedPart{ + ETag: output.ETag, + PartNumber: &partNumber, + }) + return completedParts + } +} + +// completeMultiPartUpload will complete the multi part upload with the given key in the given bucket +func completeMultiPartUpload(client *s3.Client, bucketName, keyName string, uploadId *string, completedParts []types.CompletedPart) { + fmt.Printf("\ncomplete multi part %s in bucket %s\n", keyName, bucketName) + + params := &s3.CompleteMultipartUploadInput{ + Bucket: &bucketName, + Key: &keyName, + UploadId: uploadId, + MultipartUpload: &types.CompletedMultipartUpload{ + Parts: completedParts, + }, + } + _, err := client.CompleteMultipartUpload(context.TODO(), params) + if err != nil { + fmt.Printf("got error when trying to complete multi part upload %s in bucket %s. error: \n%v\n", keyName, bucketName, err) + failingTests = append(failingTests, string(completeMultiPartUploadTest)) + } else { + fmt.Printf("succeeded in upload part %s in bucket %s.\n", keyName, bucketName) + passingTests = append(passingTests, string(completeMultiPartUploadTest)) + } +} + +// useful links: +// I started from this example: https://github.com/aws/aws-sdk-go-v2/blob/main/example/service/s3/listObjects/listObjects.go +// configurations: https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-gosdk.html +// endpoint configuration (took the simple approach): https://docs.aws.amazon.com/sdk-for-go/v2/developer-guide/configure-endpoints.html +// disable tls certificate verification: https://github.com/aws/aws-sdk-go-v2/issues/1295#issuecomment-860487390 +// AWS code examples with AWS SDK Go V2: https://docs.aws.amazon.com/code-library/latest/ug/go_2_s3_code_examples.html +// complete MPU: https://medium.com/@hirok4/implementation-of-multipart-upload-in-go-19eeb456d723 diff --git a/src/test/unit_tests/different_clients/run_go_sdkv2_client_script.js b/src/test/unit_tests/different_clients/run_go_sdkv2_client_script.js new file mode 100644 index 0000000000..39047006cf --- /dev/null +++ b/src/test/unit_tests/different_clients/run_go_sdkv2_client_script.js @@ -0,0 +1,34 @@ +/* Copyright (C) 2025 NooBaa */ +'use strict'; + +const child_process = require('child_process'); +const util = require('util'); + +const async_exec = util.promisify(child_process.exec); + +/** + * run_go_sdk_v2_client_script will run the aws_sdkv2_client go script + * @param {string} bucket_name + * @param {string} key_name + * @param {string} mpu_key_name + * @param {string} endpoint + */ +async function run_go_sdk_v2_client_script(bucket_name, key_name, mpu_key_name, endpoint) { + try { + // check go version + const res = await async_exec('go version'); + console.log('Go version', res.stdout.trim()); + + // run the script + const command_to_run_go_script = `go run ` + + `./src/test/unit_tests/different_clients/go_aws_sdkv2_client.go ` + + `-bucket ${bucket_name} -key ${key_name} -mpu ${mpu_key_name} -endpoint ${endpoint}`; + const { stdout } = await async_exec(command_to_run_go_script, { env: { ...process.env } }); + return stdout; + } catch (err) { + console.log('go run exec failed with err:,', err); + throw new Error(`go run script.go exec failed ${err.stderr || err.message}`); + } +} + +exports.run_go_sdk_v2_client_script = run_go_sdk_v2_client_script; diff --git a/src/test/unit_tests/different_clients/test_go_sdkv2_script.js b/src/test/unit_tests/different_clients/test_go_sdkv2_script.js new file mode 100644 index 0000000000..a0722184ba --- /dev/null +++ b/src/test/unit_tests/different_clients/test_go_sdkv2_script.js @@ -0,0 +1,53 @@ +/* Copyright (C) 2025 NooBaa */ +'use strict'; + +// disabling init_rand_seed as it takes longer than the actual test execution +process.env.DISABLE_INIT_RANDOM_SEED = "true"; + +// setup coretest first to prepare the env +const coretest = require('../coretest'); +coretest.setup({ pools_to_create: [coretest.POOL_LIST[1]] }); +const { rpc_client, EMAIL } = coretest; + +const mocha = require('mocha'); +const assert = require('assert'); +const config = require('../../../../config'); +const { run_go_sdk_v2_client_script } = require('./run_go_sdkv2_client_script'); + +mocha.describe('Go AWS SDK V2 Client script execution', function() { + + mocha.before(async () => { + const account_info = await rpc_client.account.read_account({ email: EMAIL, }); + console.log('test_go_sdkv2_script: account_info', account_info); + const admin_access_key = account_info.access_keys[0].access_key.unwrap(); + const admin_secret_key = account_info.access_keys[0].secret_key.unwrap(); + + process.env.AWS_ACCESS_KEY_ID = admin_access_key; + process.env.AWS_SECRET_ACCESS_KEY = admin_secret_key; + process.env.AWS_DEFAULT_REGION = config.DEFAULT_REGION; + }); + + mocha.it('All test summary should pass (no failing tests)', async function() { + this.timeout(600000); // eslint-disable-line no-invalid-this + try { + const bucket_name = 'lala-bucket'; + const key_name = 'test-key'; + const mpu_key_name = 'test-mpu-key'; + const endpoint = coretest.get_http_address(); + console.log('test_go_sdkv2_script: endpoint', endpoint); + + const output = await run_go_sdk_v2_client_script(bucket_name, key_name, mpu_key_name, endpoint); + console.log('\nTest output:\n'); + console.log(output); + // here we will fail the test in case of at least one failing test printing + const match_failing_tests_statement = output.match(/Failing Tests:\s*(\d+)/i); + // we don't see the failing tests output we will set a number higher than 1 to fail the test on purpose + const number_of_failing_tests = match_failing_tests_statement ? parseInt(match_failing_tests_statement[1], 10) : 1; + assert.equal(number_of_failing_tests, 0); + } catch (err) { + console.log('while execution of aws go client had the err', err); + assert.fail(`test aws go client failed with: ${err}, ${err.stack}`); + } + }); +}); +