Skip to content

Commit cef2865

Browse files
k8s-ci-robotk8s-ci-robot
authored
Merge pull request #1591 from aramase/automated-cherry-pick-of-#1578-upstream-release-1.4
Automated cherry pick of #1578: ci: migrate azure job to eks prow cluster
2 parents 859f39b + 7a6ed16 commit cef2865

14 files changed

+166
-328
lines changed

‎Makefile

Lines changed: 18 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -61,12 +61,12 @@ export GOPATH GOBIN GO111MODULE DOCKER_CLI_EXPERIMENTAL
6161

6262
# Generate all combination of all OS, ARCH, and OSVERSIONS for iteration
6363
ALL_OS = linux windows
64-
ALL_ARCH.linux = amd64 arm64
65-
ALL_OS_ARCH.linux = $(foreach arch, ${ALL_ARCH.linux}, linux-$(arch))
66-
ALL_ARCH.windows = amd64
67-
ALL_OSVERSIONS.windows := 1809 ltsc2022
68-
ALL_OS_ARCH.windows = $(foreach arch, $(ALL_ARCH.windows), $(foreach osversion, ${ALL_OSVERSIONS.windows}, windows-${osversion}-${arch}))
69-
ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH.${os}})
64+
ALL_ARCH_linux ?= amd64 arm64
65+
ALL_OS_ARCH_linux = $(foreach arch, ${ALL_ARCH_linux}, linux-$(arch))
66+
ALL_ARCH_windows = amd64
67+
ALL_OSVERSIONS_windows := 1809 ltsc2022
68+
ALL_OS_ARCH_windows = $(foreach arch, $(ALL_ARCH_windows), $(foreach osversion, ${ALL_OSVERSIONS_windows}, windows-${osversion}-${arch}))
69+
ALL_OS_ARCH = $(foreach os, $(ALL_OS), ${ALL_OS_ARCH_${os}})
7070

7171
# The current context of image building
7272
# The architecture of the image
@@ -328,24 +328,24 @@ docker-buildx-builder:
328328

329329
.PHONY: container-all
330330
container-all: docker-buildx-builder
331-
for arch in $(ALL_ARCH.linux); do \
331+
for arch in $(ALL_ARCH_linux); do \
332332
ARCH=$${arch} $(MAKE) container-linux; \
333333
ARCH=$${arch} $(MAKE) crd-container-linux; \
334334
done
335-
for osversion in $(ALL_OSVERSIONS.windows); do \
335+
for osversion in $(ALL_OSVERSIONS_windows); do \
336336
OSVERSION=$${osversion} $(MAKE) container-windows; \
337337
done
338338

339339
.PHONY: push-manifest
340340
push-manifest:
341341
docker manifest create --amend $(IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH), $(IMAGE_TAG)-${osarch})
342-
docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH.linux), $(CRD_IMAGE_TAG)-${osarch})
342+
docker manifest create --amend $(CRD_IMAGE_TAG) $(foreach osarch, $(ALL_OS_ARCH_linux), $(CRD_IMAGE_TAG)-${osarch})
343343
# add "os.version" field to windows images (based on https://github.com/kubernetes/kubernetes/blob/master/build/pause/Makefile)
344344
set -x; \
345345
registry_prefix=$(shell (echo ${REGISTRY} | grep -Eq ".*[\/\.].*") && echo "" || echo "docker.io/"); \
346346
manifest_image_folder=`echo "$${registry_prefix}${IMAGE_TAG}" | sed "s|/|_|g" | sed "s/:/-/"`; \
347-
for arch in $(ALL_ARCH.windows); do \
348-
for osversion in $(ALL_OSVERSIONS.windows); do \
347+
for arch in $(ALL_ARCH_windows); do \
348+
for osversion in $(ALL_OSVERSIONS_windows); do \
349349
BASEIMAGE=mcr.microsoft.com/windows/nanoserver:$${osversion}; \
350350
full_version=`docker manifest inspect $${BASEIMAGE} | jq -r '.manifests[0].platform["os.version"]'`; \
351351
sed -i -r "s/(\"os\"\:\"windows\")/\0,\"os.version\":\"$${full_version}\"/" "${HOME}/.docker/manifests/$${manifest_image_folder}/$${manifest_image_folder}-windows-$${osversion}-$${arch}"; \
@@ -359,11 +359,12 @@ push-manifest:
359359
## --------------------------------------
360360
## E2E Testing
361361
## --------------------------------------
362+
.PHONY: e2e-install-prerequisites
363+
e2e-install-prerequisites: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ)
364+
362365
.PHONY: e2e-bootstrap
363-
e2e-bootstrap: $(HELM) $(BATS) $(KIND) $(KUBECTL) $(ENVSUBST) $(YQ) #setup all required binaries and kind cluster for testing
364-
ifndef TEST_WINDOWS
366+
e2e-bootstrap: e2e-install-prerequisites #setup all required binaries and kind cluster for testing
365367
$(MAKE) setup-kind
366-
endif
367368
docker pull $(IMAGE_TAG) || $(MAKE) e2e-container
368369

369370
.PHONY: setup-kind
@@ -378,12 +379,8 @@ setup-eks-cluster: $(HELM) $(EKSCTL) $(BATS) $(ENVSUBST) $(YQ)
378379

379380
.PHONY: e2e-container
380381
e2e-container:
381-
ifdef TEST_WINDOWS
382-
$(MAKE) container-all push-manifest
383-
else
384382
$(MAKE) container
385383
kind load docker-image --name kind $(IMAGE_TAG) $(CRD_IMAGE_TAG)
386-
endif
387384

388385
.PHONY: e2e-mock-provider-container
389386
e2e-mock-provider-container:
@@ -437,7 +434,9 @@ e2e-helm-deploy:
437434
--set enableSecretRotation=true \
438435
--set rotationPollInterval=30s \
439436
--set tokenRequests[0].audience="aud1" \
440-
--set tokenRequests[1].audience="aud2"
437+
--set tokenRequests[1].audience="aud2" \
438+
--set tokenRequests[2].audience="conjur" \
439+
--set tokenRequests[3].audience="api://AzureADTokenExchange"
441440

442441
.PHONY: e2e-helm-upgrade
443442
e2e-helm-upgrade:

‎test/bats/azure.bats

Lines changed: 3 additions & 114 deletions
Original file line numberDiff line numberDiff line change
@@ -16,21 +16,10 @@ if [ $TEST_WINDOWS ]; then
1616
NODE_SELECTOR_OS=windows
1717
fi
1818

19-
if [ -z "$AUTO_ROTATE_SECRET_NAME" ]; then
20-
export AUTO_ROTATE_SECRET_NAME=secret-$(openssl rand -hex 6)
21-
fi
22-
23-
if [ -z "$IS_YAML_TEST" ]; then
24-
export IS_YAML_TEST=false
25-
fi
26-
27-
export KEYVAULT_NAME=${KEYVAULT_NAME:-csi-secrets-store-e2e}
19+
export KEYVAULT_NAME=${KEYVAULT_NAME:-secrets-store-csi-e2e}
2820
export SECRET_NAME=${KEYVAULT_SECRET_NAME:-secret1}
2921
export SECRET_VERSION=${KEYVAULT_SECRET_VERSION:-""}
3022
export SECRET_VALUE=${KEYVAULT_SECRET_VALUE:-"test"}
31-
export KEY_NAME=${KEYVAULT_KEY_NAME:-key1}
32-
export KEY_VERSION=${KEYVAULT_KEY_VERSION:-7cc095105411491b84fe1b92ebbcf01a}
33-
export KEY_VALUE_CONTAINS=${KEYVAULT_KEY_VALUE:-"LS0tLS1CRUdJTiBQVUJMSUMgS0VZLS0tLS0KTUlJQklqQU5CZ2txaGtpRzl3MEJBUUVGQUFPQ0FROEFNSUlCQ2dLQ0FRRUF4K2FadlhJN2FldG5DbzI3akVScgpheklaQ2QxUlBCQVZuQU1XcDhqY05TQk5MOXVuOVJrenJHOFd1SFBXUXNqQTA2RXRIOFNSNWtTNlQvaGQwMFNRCk1aODBMTlNxYkkwTzBMcWMzMHNLUjhTQ0R1cEt5dkpkb01LSVlNWHQzUlk5R2Ywam1ucHNKOE9WbDFvZlRjOTIKd1RINXYyT2I1QjZaMFd3d25MWlNiRkFnSE1uTHJtdEtwZTVNcnRGU21nZS9SL0J5ZXNscGU0M1FubnpndzhRTwpzU3ZMNnhDU21XVW9WQURLL1MxREU0NzZBREM2a2hGTjF5ZHUzbjVBcnREVGI0c0FjUHdTeXB3WGdNM3Y5WHpnClFKSkRGT0JJOXhSTW9UM2FjUWl0Z0c2RGZibUgzOWQ3VU83M0o3dUFQWUpURG1pZGhrK0ZFOG9lbjZWUG9YRy8KNXdJREFRQUIKLS0tLS1FTkQgUFVCTElDIEtFWS0tLS0t"}
3423
export LABEL_VALUE=${LABEL_VALUE:-"test"}
3524
export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
3625

@@ -39,8 +28,8 @@ export NODE_SELECTOR_OS=$NODE_SELECTOR_OS
3928
export API_VERSION=$(get_secrets_store_api_version)
4029

4130
setup() {
42-
if [[ -z "${AZURE_CLIENT_ID}" ]] || [[ -z "${AZURE_CLIENT_SECRET}" ]]; then
43-
echo "Error: Azure service principal is not provided" >&2
31+
if [[ -z "${IDENTITY_CLIENT_ID}" ]]; then
32+
echo "Error: Azure managed identity id is not provided" >&2
4433
return 1
4534
fi
4635
}
@@ -59,15 +48,6 @@ setup() {
5948
kubectl wait --for=condition=Ready --timeout=150s pods -l app=csi-secrets-store-provider-azure --namespace $NAMESPACE
6049
}
6150

62-
@test "create azure k8s secret" {
63-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET}
64-
assert_success
65-
66-
# label the node publish secret ref secret
67-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true
68-
assert_success
69-
}
70-
7151
@test "deploy azure secretproviderclass crd" {
7252
envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass.yaml | kubectl apply -f -
7353

@@ -96,12 +76,6 @@ setup() {
9676
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
9777
}
9878

99-
@test "CSI inline volume test with pod portability - read azure kv key from pod" {
100-
result=$(kubectl exec secrets-store-inline-crd -- cat /mnt/secrets-store/$KEY_NAME)
101-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
102-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
103-
}
104-
10579
@test "CSI inline volume test with pod portability - unmount succeeds" {
10680
# On Linux a failure to unmount the tmpfs will block the pod from being
10781
# deleted.
@@ -144,10 +118,6 @@ setup() {
144118
result=$(kubectl exec $POD -- cat /mnt/secrets-store/secretalias)
145119
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
146120

147-
result=$(kubectl exec $POD -- cat /mnt/secrets-store/$KEY_NAME)
148-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
149-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
150-
151121
result=$(kubectl get secret foosecret -o jsonpath="{.data.username}" | base64 -d)
152122
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
153123

@@ -184,13 +154,6 @@ setup() {
184154
run kubectl create ns test-ns
185155
assert_success
186156

187-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n test-ns
188-
assert_success
189-
190-
# label the node publish secret ref secret
191-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n test-ns
192-
assert_success
193-
194157
envsubst < $BATS_TESTS_DIR/azure_v1_secretproviderclass_ns.yaml | kubectl apply -f -
195158

196159
kubectl wait --for condition=established --timeout=60s crd/secretproviderclasses.secrets-store.csi.x-k8s.io
@@ -212,10 +175,6 @@ setup() {
212175
result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/secretalias)
213176
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
214177

215-
result=$(kubectl exec -n test-ns $POD -- cat /mnt/secrets-store/$KEY_NAME)
216-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
217-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
218-
219178
result=$(kubectl get secret foosecret -n test-ns -o jsonpath="{.data.username}" | base64 -d)
220179
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
221180

@@ -238,13 +197,6 @@ setup() {
238197
run kubectl create ns negative-test-ns
239198
assert_success
240199

241-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n negative-test-ns
242-
assert_success
243-
244-
# label the node publish secret ref secret
245-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n negative-test-ns
246-
assert_success
247-
248200
envsubst < $BATS_TESTS_DIR/deployment-synck8s-azure.yaml | kubectl apply -n negative-test-ns -f -
249201
sleep 5
250202

@@ -284,17 +236,9 @@ setup() {
284236
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/secretalias)
285237
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
286238

287-
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-0/$KEY_NAME)
288-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
289-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
290-
291239
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/secretalias)
292240
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
293241

294-
result=$(kubectl exec secrets-store-inline-multiple-crd -- cat /mnt/secrets-store-1/$KEY_NAME)
295-
result_base64_encoded=$(echo "${result//$'\r'}" | base64 ${BASE64_FLAGS})
296-
[[ "${result_base64_encoded}" == *"${KEY_VALUE_CONTAINS}"* ]]
297-
298242
result=$(kubectl get secret foosecret-0 -o jsonpath="{.data.username}" | base64 -d)
299243
[[ "${result//$'\r'}" == "${SECRET_VALUE}" ]]
300244

@@ -314,66 +258,11 @@ setup() {
314258
assert_success
315259
}
316260

317-
@test "Test auto rotation of mount contents and K8s secrets - Create deployment" {
318-
run kubectl create ns rotation
319-
assert_success
320-
321-
run kubectl create secret generic secrets-store-creds --from-literal clientid=${AZURE_CLIENT_ID} --from-literal clientsecret=${AZURE_CLIENT_SECRET} -n rotation
322-
assert_success
323-
324-
# label the node publish secret ref secret
325-
run kubectl label secret secrets-store-creds secrets-store.csi.k8s.io/used=true -n rotation
326-
assert_success
327-
328-
run az login -u ${AZURE_CLIENT_ID} -p ${AZURE_CLIENT_SECRET} -t ${TENANT_ID} --service-principal
329-
assert_success
330-
331-
run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value secret
332-
assert_success
333-
334-
envsubst < $BATS_TESTS_DIR/rotation/azure_synck8s_v1_secretproviderclass.yaml | kubectl apply -n rotation -f -
335-
envsubst < $BATS_TESTS_DIR/rotation/pod-synck8s-azure.yaml | kubectl apply -n rotation -f -
336-
337-
kubectl wait -n rotation --for=condition=Ready --timeout=60s pod/secrets-store-inline-rotation
338-
339-
run kubectl get pod/secrets-store-inline-rotation -n rotation
340-
assert_success
341-
}
342-
343-
@test "Test auto rotation of mount contents and K8s secrets" {
344-
result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
345-
[[ "${result//$'\r'}" == "secret" ]]
346-
347-
result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
348-
[[ "${result//$'\r'}" == "secret" ]]
349-
350-
run az keyvault secret set --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME} --value rotated
351-
assert_success
352-
353-
sleep 60
354-
355-
result=$(kubectl exec -n rotation secrets-store-inline-rotation -- cat /mnt/secrets-store/secretalias)
356-
[[ "${result//$'\r'}" == "rotated" ]]
357-
358-
result=$(kubectl get secret -n rotation rotationsecret -o jsonpath="{.data.username}" | base64 -d)
359-
[[ "${result//$'\r'}" == "rotated" ]]
360-
361-
run az keyvault secret delete --vault-name ${KEYVAULT_NAME} --name ${AUTO_ROTATE_SECRET_NAME}
362-
assert_success
363-
364-
run az logout
365-
assert_success
366-
}
367-
368261
teardown_file() {
369262
archive_provider "app=csi-secrets-store-provider-azure" || true
370263
archive_info || true
371264

372265
#cleanup
373-
run kubectl delete namespace rotation
374266
run kubectl delete namespace test-ns
375-
376-
run kubectl delete secret secrets-store-creds
377-
378267
run kubectl delete pods secrets-store-inline-crd secrets-store-inline-multiple-crd --force --grace-period 0
379268
}

‎test/bats/tests/azure/azure_synck8s_v1_secretproviderclass.yaml

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ spec:
1313
- objectName: secretalias # name of the mounted content to sync. this could be the object name or object alias
1414
key: username
1515
parameters:
16-
usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false"
16+
clientID: "$IDENTITY_CLIENT_ID"
1717
keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault
1818
objects: |
1919
array:
@@ -22,8 +22,4 @@ spec:
2222
objectType: secret # object types: secret, key or cert
2323
objectAlias: secretalias
2424
objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty
25-
- |
26-
objectName: $KEY_NAME
27-
objectType: key
28-
objectVersion: $KEY_VERSION
29-
tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
25+
tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault

‎test/bats/tests/azure/azure_v1_multiple_secretproviderclass.yaml

Lines changed: 4 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ spec:
1111
- objectName: secretalias
1212
key: username
1313
parameters:
14-
usePodIdentity: "false"
14+
clientID: "$IDENTITY_CLIENT_ID"
1515
keyvaultName: "$KEYVAULT_NAME"
1616
objects: |
1717
array:
@@ -20,11 +20,7 @@ spec:
2020
objectType: secret
2121
objectVersion: $SECRET_VERSION
2222
objectAlias: secretalias
23-
- |
24-
objectName: $KEY_NAME
25-
objectType: key
26-
objectVersion: $KEY_VERSION
27-
tenantId: "$TENANT_ID"
23+
tenantId: "$AZURE_TENANT_ID"
2824
---
2925
apiVersion: $API_VERSION
3026
kind: SecretProviderClass
@@ -39,7 +35,7 @@ spec:
3935
- objectName: secretalias
4036
key: username
4137
parameters:
42-
usePodIdentity: "false"
38+
clientID: "$IDENTITY_CLIENT_ID"
4339
keyvaultName: "$KEYVAULT_NAME"
4440
objects: |
4541
array:
@@ -48,8 +44,4 @@ spec:
4844
objectType: secret
4945
objectVersion: $SECRET_VERSION
5046
objectAlias: secretalias
51-
- |
52-
objectName: $KEY_NAME
53-
objectType: key
54-
objectVersion: $KEY_VERSION
55-
tenantId: "$TENANT_ID"
47+
tenantId: "$AZURE_TENANT_ID"

‎test/bats/tests/azure/azure_v1_secretproviderclass.yaml

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -5,16 +5,12 @@ metadata:
55
spec:
66
provider: azure
77
parameters:
8-
usePodIdentity: "false" # [OPTIONAL] if not provided, will default to "false"
8+
clientID: "$IDENTITY_CLIENT_ID"
99
keyvaultName: "$KEYVAULT_NAME" # the name of the KeyVault
1010
objects: |
1111
array:
1212
- |
1313
objectName: $SECRET_NAME
1414
objectType: secret # object types: secret, key or cert
1515
objectVersion: $SECRET_VERSION # [OPTIONAL] object versions, default to latest if empty
16-
- |
17-
objectName: $KEY_NAME
18-
objectType: key
19-
objectVersion: $KEY_VERSION
20-
tenantId: "$TENANT_ID" # the tenant ID of the KeyVault
16+
tenantId: "$AZURE_TENANT_ID" # the tenant ID of the KeyVault

0 commit comments

Comments
 (0)