add bats tests for alibabacloud secret store csi provider

Makefile

@@ -94,6 +94,7 @@ ENVSUBST := envsubst
 EKSCTL := eksctl
 AWS_CLI := aws
 YQ := yq
+ALIYUNCLI := aliyun
 
 # Test variables
 KIND_VERSION ?= 0.13.0
@@ -214,6 +215,9 @@ $(PROTOC): ## Install protoc
 $(YQ): ## Install yq for running the tests
 	curl -LO https://github.com/mikefarah/yq/releases/download/$(YQ_VERSION)/yq_linux_amd64 && chmod +x ./yq_linux_amd64 && mv yq_linux_amd64 /usr/local/bin/yq
 
+$(ALIYUNCLI): ## Install aliyun for running the tests
+	curl -LO https://github.com/aliyun/aliyun-cli/releases/download/v3.0.134/aliyun-cli-linux-3.0.134-amd64.tgz && tar xzvf aliyun-cli-linux-3.0.134-amd64.tgz && chmod +x ./aliyun && cp aliyun /usr/local/bin
+
 SHELLCHECK := $(TOOLS_BIN_DIR)/shellcheck-$(SHELLCHECK_VER)
 $(SHELLCHECK): OS := $(shell uname | tr '[:upper:]' '[:lower:]')
 $(SHELLCHECK): ARCH := $(shell uname -m)
@@ -482,6 +486,10 @@ e2e-akeyless:
 e2e-gcp:
 	bats -t test/bats/gcp.bats
 
+.PHONY: e2e-alibabacloud
+e2e-alibabacloud:
+	bats -t test/bats/alibabacloud.bats
+
 .PHONY: e2e-aws
 e2e-aws:
 	bats -t test/bats/aws.bats

test/bats/alibabacloud.bats

@@ -0,0 +1,148 @@
+#!/usr/bin/env bats
+
+load helpers
+
+WAIT_TIME=120
+SLEEP_TIME=1
+NAMESPACE=kube-system
+POD_NAME=alibabacloud-basic-test-mount
+BATS_TEST_DIR=test/bats/tests/alibabacloud
+
+setup() {
+  if [[ -z "${ALIBABACLOUD_ACCESS_KEY}" ]] || [[ -z "${ALIBABACLOUD_ACCESS_SECRET}" ]]; then
+    echo "Error: ram ak/sk is not provided" >&2
+    return 1
+  fi
+}
+
+setup_file() {
+    #Configure aliyun cli profile
+    aliyun configure set --profile akProfile --mode AK --region us-west-1 --access-key-id ${ALIBABACLOUD_ACCESS_KEY} --access-key-secret ${ALIBABACLOUD_ACCESS_SECRET}
+
+    #Create test secrets
+    aliyun kms CreateSecret --SecretName testBasic --SecretData testValue --VersionId v1
+    aliyun kms CreateSecret --SecretName testSync --SecretData  syncValue --VersionId v1
+    aliyun kms CreateSecret --SecretName testRotation --SecretData beforeRotation --VersionId v1
+    aliyun kms CreateSecret --SecretName testJson --SecretData '{"username": "testUser", "password": "testPassword"}' --VersionId v1
+}
+
+teardown_file() {
+    aliyun kms DeleteSecret --SecretName testBasic --ForceDeleteWithoutRecovery true
+    aliyun kms DeleteSecret --SecretName testSync --ForceDeleteWithoutRecovery true
+    aliyun kms DeleteSecret --SecretName testRotation --ForceDeleteWithoutRecovery true
+    aliyun kms DeleteSecret --SecretName testJson --ForceDeleteWithoutRecovery true
+}
+
+validate_jsme_mount() {
+    result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$USERNAME_ALIAS)
+    [[ "${result//$'\r'}" == $USERNAME ]]
+
+    result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$PASSWORD_ALIAS)
+    [[ "${result//$'\r'}" == $PASSWORD ]]
+
+    result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/$SECRET_FILE_NAME)
+    [[ "${result//$'\r'}" == $SECRET_FILE_CONTENT ]]
+
+    run kubectl get secret --namespace $NAMESPACE $K8_SECRET_NAME
+    [ "$status" -eq 0 ]
+
+    result=$(kubectl --namespace=$NAMESPACE get secret $K8_SECRET_NAME -o jsonpath="{.data.username}" | base64 -d)
+    [[ "$result" == $USERNAME ]]
+
+    result=$(kubectl --namespace=$NAMESPACE get secret $K8_SECRET_NAME -o jsonpath="{.data.password}" | base64 -d)
+    [[ "$result" == $PASSWORD ]]
+}
+
+@test "Install alibabacloud provider" {
+    #Install csi secret driver
+    helm repo add csi-secrets-store-provider-alibabacloud https://raw.githubusercontent.com/AliyunContainerService/secrets-store-csi-driver-provider-alibaba-cloud/main/charts
+    helm install csi-secrets-store-provider-alibabacloud/csi-secrets-store-provider-alibabacloud --namespace kube-system --generate-name --set secrets-store-csi-driver.enableSecretRotation=true --set secrets-store-csi-driver.rotationPollInterval=10s --set secrets-store-csi-driver.syncSecret.enabled=true
+
+    cmd="kubectl --namespace $NAMESPACE wait --for=condition=Ready --timeout=60s pod -l app=csi-secrets-store-provider-alibabacloud"
+    wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
+
+    PROVIDER_POD=$(kubectl --namespace $NAMESPACE get pod -l app=csi-secrets-store-provider-alibabacloud -o jsonpath="{.items[0].metadata.name}")
+    run kubectl --namespace $NAMESPACE get pod/$PROVIDER_POD
+    assert_success
+}
+
+@test "secretproviderclasses crd is established" {
+    cmd="kubectl wait --namespace $NAMESPACE --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io"
+    wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
+
+    run kubectl get crd/secretproviderclasses.secrets-store.csi.x-k8s.io
+    assert_success
+}
+
+@test "create alibabacloud k8s secret" {
+  run kubectl create secret generic secrets-store-creds --from-literal access_key=${ALIBABACLOUD_ACCESS_KEY} --from-literal access_secret=${ALIBABACLOUD_ACCESS_SECRET} --namespace=$NAMESPACE
+  assert_success
+
+  # label the node publish secret ref secret
+  run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true --namespace=$NAMESPACE
+  assert_success
+}
+
+@test "deploy alibabacloud secretproviderclass crd" {
+    envsubst < $BATS_TEST_DIR/secretproviderclass.yaml | kubectl --namespace $NAMESPACE apply -f -
+
+    cmd="kubectl --namespace $NAMESPACE get secretproviderclasses.secrets-store.csi.x-k8s.io/alibabacloud-basic-test-mount-spc -o yaml | grep alibabacloud"
+    wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
+}
+
+@test "CSI inline volume test with pod portability" {
+   kubectl --namespace $NAMESPACE  apply -f $BATS_TEST_DIR/pod-inline-volume-secretproviderclass.yaml
+   cmd="kubectl --namespace $NAMESPACE  wait --for=condition=Ready --timeout=60s pod/alibabacloud-basic-test-mount"
+   wait_for_process $WAIT_TIME $SLEEP_TIME "$cmd"
+
+   run kubectl --namespace $NAMESPACE  get pod/$POD_NAME
+   assert_success
+}
+
+@test "CSI inline volume test with rotation" {
+   result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/testRotation)
+   [[ "${result//$'\r'}" == "beforeRotation" ]]
+
+   aliyun kms PutSecretValue --SecretName testRotation --SecretData afterRotation --VersionId v2
+   sleep 20
+   result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/testRotation)
+   [[ "${result//$'\r'}" == "afterRotation" ]]
+}
+
+@test "CSI inline volume test with pod portability - read secrets manager secrets from pod" {
+    result=$(kubectl --namespace $NAMESPACE exec $POD_NAME -- cat /mnt/secrets-store/testBasic)
+    [[ "${result//$'\r'}" == "testValue" ]]
+}
+
+@test "CSI inline volume test with pod portability - specify jsmePath for Secrets Manager secret with rotation" {
+
+    JSON_CONTENT='{"username": "testUser", "password": "testPassword"}'
+
+    USERNAME_ALIAS=mySecretUsername USERNAME=testUser PASSWORD_ALIAS=mySecretPassword \
+    PASSWORD=testPassword SECRET_FILE_NAME=testJson SECRET_FILE_CONTENT=$JSON_CONTENT
+    K8_SECRET_NAME=sm-secret-json validate_jsme_mount
+
+    UPDATED_JSON_CONTENT='{"username": "testUser", "password": "testPasswordUpdated"}'
+    aliyun kms PutSecretValue --SecretName testJson --SecretData "$UPDATED_JSON_CONTENT" --VersionId v2
+
+    sleep 20
+    USERNAME_ALIAS=mySecretUsername USERNAME=testUser PASSWORD_ALIAS=mySecretPassword \
+    PASSWORD=testPasswordUpdated SECRET_FILE_NAME=testJson SECRET_FILE_CONTENT=$UPDATED_JSON_CONTENT
+    K8_SECRET_NAME=sm-secret-json validate_jsme_mount
+}
+
+@test "Sync with Kubernetes Secret" {
+    run kubectl get secret --namespace $NAMESPACE  sm-secret
+    [ "$status" -eq 0 ]
+
+    result=$(kubectl --namespace=$NAMESPACE get secret sm-secret -o jsonpath="{.data.username}" | base64 -d)
+    [[ "$result" == "syncValue" ]]
+}
+
+@test "Sync with Kubernetes Secret - Delete deployment. Secret should also be deleted" {
+    run kubectl --namespace $NAMESPACE  delete -f $BATS_TEST_DIR/pod-inline-volume-secretproviderclass.yaml
+    assert_success
+
+    run wait_for_process $WAIT_TIME $SLEEP_TIME "check_secret_deleted secret $NAMESPACE"
+    assert_success
+}

test/bats/tests/alibabacloud/pod-inline-volume-secretproviderclass.yaml

@@ -0,0 +1,25 @@
+kind: Pod
+apiVersion: v1
+metadata:
+  name: alibabacloud-basic-test-mount
+spec:
+  containers:
+    - image: k8s.gcr.io/e2e-test-images/busybox:1.29
+      name: busybox
+      imagePullPolicy: IfNotPresent
+      command:
+        - "/bin/sleep"
+        - "10000"
+      volumeMounts:
+        - name: secrets-store-inline
+          mountPath: "/mnt/secrets-store"
+          readOnly: true
+  volumes:
+    - name: secrets-store-inline
+      csi:
+        driver: secrets-store.csi.k8s.io
+        readOnly: true
+        volumeAttributes:
+          secretProviderClass: "alibabacloud-basic-test-mount-spc"
+        nodePublishSecretRef:
+          name: secrets-store-creds

test/bats/tests/alibabacloud/secretproviderclass.yaml

@@ -0,0 +1,30 @@
+apiVersion: secrets-store.csi.x-k8s.io/v1
+kind: SecretProviderClass
+metadata:
+  name: alibabacloud-basic-test-mount-spc
+spec:
+  provider: alibabacloud
+  secretObjects:
+    - secretName: sm-secret
+      type: Opaque
+      data:
+        - objectName: testSync
+          key: username
+    - secretName: sm-secret-json
+      type: Opaque
+      data:
+        - objectName: mySecretUsername
+          key: username
+        - objectName: mySecretPassword
+          key: password
+  parameters:
+    objects: |
+      - objectName: testBasic
+      - objectName: testSync
+      - objectName: testRotation
+      - objectName: testJson
+        jmesPath:
+          - path: "username"
+            objectAlias: "mySecretUsername"
+          - path: "password"
+            objectAlias: "mySecretPassword"