CI | Data Buckets | Test AWS SDK Clients

.github/workflows/test-aws-sdk-clients.yaml

@@ -0,0 +1,62 @@
+name: Test AWS SDK Clients
+on: [push, pull_request, workflow_dispatch]
+# it should run nightly, in the push only for CR step
+# on: 
+#   schedule:
+#     - cron: "0 1 * * *"
+#   workflow_dispatch:
+
+jobs:
+  test-aws-sdk-clients:
+    runs-on: ubuntu-latest
+    timeout-minutes: 90
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}
+      cancel-in-progress: true
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4
+        with:
+          repository: 'noobaa/noobaa-core'
+          path: 'noobaa-core'
+
+      - name: Run AWS SDK Clients
+        run: |
+          set -x
+          cd ./noobaa-core
+          make test-aws-sdk-clients
+
+      # # this should be removed
+      # - name: Test slack webhook
+      #   uses: slackapi/[email protected]
+      #   with:
+      #     webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+      #     webhook-type: incoming-webhook
+      #     payload: |
+      #       text: "Hello world (Shira is testing webhook)"
+
+      # this should be removed
+      - name: Test slack webhook 2
+        run: | 
+          echo "check the variable" 
+          echo "${{ secrets.SLACK_WEBHOOK_URL }}" | wc -c
+          echo "send the massage"
+          curl -X POST ${{ secrets.SLACK_WEBHOOK_URL }} \
+          -H "Content-type: application/json" \
+          -d '{"text": "hello_world (Shira) with curl in github action"}'
+
+      # this should be added
+      # later we would add the if: ${{ failure() }} condition
+      # - name: Post a message in a channel
+      #   uses: slackapi/[email protected]
+      #   with:
+      #     webhook: ${{ secrets.SLACK_WEBHOOK_URL }}
+      #     webhook-type: incoming-webhook
+      #     payload: |
+      #       text: "*GitHub Action Test AWS SDK Clients Result*: ${{ job.status }}\n"
+      #       blocks:
+      #         - type: "section"
+      #           text:
+      #           type: "mrkdwn"
+      #           text: "GitHub Action build result: ${{ job.status }}\n"
+

Makefile

@@ -229,6 +229,12 @@ build-ssl-postgres: tester
 	@echo "##\033[1;32m Build image postgres:ssl done.\033[0m"
 .PHONY: build-ssl-postgres
 
+build-aws-client: noobaa
+	@echo "\n##\033[1;32m Build image for AWS Client tests ...\033[0m"
+	$(CONTAINER_ENGINE) build $(CONTAINER_PLATFORM_FLAG) $(CPUSET) -f src/deploy/NVA_build/AWSClient.Dockerfile $(CACHE_FLAG) $(NETWORK_FLAG) -t noobaa-aws-client . $(REDIRECT_STDOUT)
+	@echo "\033[1;32mBuild image for AWS Client tests done.\033[0m"
+.PHONY: build-aws-client
+
 test: tester
 	@echo "\033[1;34mRunning tests with Mongo.\033[0m"
 	@$(call create_docker_network)
@@ -363,6 +369,17 @@ test-external-pg-sanity: build-ssl-postgres
 	@$(call remove_docker_network)
 .PHONY: test-external-pg-sanity
 
+test-aws-sdk-clients: build-aws-client
+	@echo "\033[1;34mRunning tests with Postgres.\033[0m"
+	@$(call create_docker_network)
+	@$(call run_postgres)
+	@echo "\033[1;34mRunning aws sdk clients tests\033[0m"
+	$(CONTAINER_ENGINE) run $(CPUSET) --network noobaa-net --name noobaa_$(GIT_COMMIT)_$(NAME_POSTFIX) --env "SUPPRESS_LOGS=$(SUPPRESS_LOGS)" --env "POSTGRES_HOST=coretest-postgres-$(GIT_COMMIT)-$(NAME_POSTFIX)" --env "POSTGRES_USER=noobaa" --env "DB_TYPE=postgres" --env "POSTGRES_DBNAME=coretest" --env "NOOBAA_LOG_LEVEL=all" -v $(PWD)/logs:/logs  noobaa-aws-client ./src/test/unit_tests/run_npm_test_on_test_container.sh -c ./node_modules/mocha/bin/mocha.js src/test/unit_tests/different_clients/test_go_sdkv2_script.js
+	@$(call stop_noobaa)
+	@$(call stop_postgres)
+	@$(call remove_docker_network)
+.PHONY: test-aws-sdk-clients
+
 clean:
 	@echo Stopping and Deleting containers
 	@$(CONTAINER_ENGINE) ps -a | grep noobaa_ | awk '{print $1}' | xargs $(CONTAINER_ENGINE) stop &> /dev/null

src/deploy/NVA_build/AWSClient.Dockerfile

@@ -0,0 +1,33 @@
+FROM noobaa
+
+USER 0:0
+
+ENV container=docker
+ENV TEST_CONTAINER=true
+
+WORKDIR /root/node_modules/noobaa-core/
+
+# check npm version (installing dev dependency in different between npm version pre version 7 that was --only=dev)
+RUN npm -v
+RUN npm install --omit=prod
+
+############################################################################
+# Layers:
+#   Title: Install go and modules
+#           for testing with AWS SDK GO client with most updated version
+
+############################################################################
+
+# installing go
+RUN dnf install -y golang
+# verify go installation
+RUN go version
+# set the PATH for go
+ENV PATH="/usr/local/go/bin:$PATH"
+
+# install the needed modules
+# note: the files go.mod and go.sum will be automatically created after this step in the WORKDIR
+RUN go mod init src/test/unit_tests/different_clients
+RUN go mod tidy
+
+USER 10001:0

src/test/unit_tests/coretest.js

@@ -233,6 +233,7 @@ function setup(options = {}) {
         });
         rpc_client.options.auth_token = token;
         await overwrite_system_address(SYSTEM);
+        console.log('pools_to_create.length', pools_to_create.length);
         if (pools_to_create.length > 0) {
             await announce('setup_pools()');
             await setup_pools(pools_to_create);