Syncing latest changes from master for rook

.github/workflows/build.yml

@@ -30,7 +30,7 @@ jobs:
           go-version: "1.21"
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
         with:
           version: v3.6.2
 
@@ -60,7 +60,7 @@ jobs:
         run: tests/scripts/validate_modified_files.sh modcheck
 
       - name: run crds-gen
-        run: make csv-clean && GOPATH=$(go env GOPATH) make crds
+        run: GOPATH=$(go env GOPATH) make crds
 
       - name: validate crds-gen
         run: tests/scripts/validate_modified_files.sh crd
@@ -97,3 +97,9 @@ jobs:
       - name: build.all rook with go ${{ matrix.go-version }}
         run: |
           tests/scripts/github-action-helper.sh build_rook_all
+
+      - name: run gen-csv
+        run: make csv-clean && GOPATH=$(go env GOPATH) make gen-csv
+
+      - name: validate gen-csv
+        run: tests/scripts/validate_modified_files.sh gen-csv

.github/workflows/canary-integration-test.yml

@@ -4,7 +4,7 @@ on:
   workflow_call:
     inputs:
       ceph_images:
-        description: 'JSON list of Ceph images for creating Ceph cluster'
+        description: "JSON list of Ceph images for creating Ceph cluster"
         default: '["quay.io/ceph/ceph:v18"]'
         type: string
 
@@ -190,7 +190,7 @@ jobs:
 
       - name: validate-rgw-endpoint
         run: |
-          rgw_endpoint=$(kubectl get service -n rook-ceph |  awk '/rgw/ {print $3":80"}')
+          rgw_endpoint=$(kubectl get service -n rook-ceph -l rgw=store-a |  awk '/rgw/ {print $3":80"}')
           toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
           # pass the valid rgw-endpoint of same ceph cluster
           timeout 15 sh -c "until kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --rgw-endpoint $rgw_endpoint 2> output.txt; do sleep 1 && echo 'waiting for the rgw endpoint to be validated'; done"
@@ -229,6 +229,39 @@ jobs:
             echo "script failed because wrong realm was passed"
           fi
 
+      - name: test topology flags
+        run: |
+          toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
+          # create 3 replica-1 pools
+          sed -i 's/replicapool/replica1a/' deploy/examples/pool-test.yaml
+          kubectl create -f deploy/examples/pool-test.yaml
+          sed -i 's/replica1a/replica1b/' deploy/examples/pool-test.yaml
+          kubectl create -f deploy/examples/pool-test.yaml
+          sed -i 's/replica1b/replica1c/' deploy/examples/pool-test.yaml
+          kubectl create -f deploy/examples/pool-test.yaml
+          # bring back the original file
+          sed -i 's/replica1c/replicapool/' deploy/examples/pool-test.yaml
+
+          # check and wait for the pools to get ready
+          kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1a -nrook-ceph
+          kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1b -nrook-ceph
+          kubectl wait --for='jsonpath={.status.phase}=Ready' Cephblockpool/replica1c -nrook-ceph
+
+          # pass correct flags
+          kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03
+          # pass the pool which is not exists
+          if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools ab,cd,ef --topology-failure-domain-label hostname --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
+            echo "script run completed with stderr error after passing the wrong pools: $output"
+          else
+            echo "script failed because wrong pools doesn't exist"
+          fi
+          # dont pass all topology flags
+          if output=$(kubectl -n rook-ceph exec $toolbox -- python3 /etc/ceph/create-external-cluster-resources.py --rbd-data-pool-name replicapool --topology-pools replica1a,replica1b,replica1c --topology-failure-domain-values minikube,minikube-m02,minikube-m03); then
+            echo "script run completed with stderr error after passing the wrong flags: $output"
+          else
+            echo "script failed because topology-failure-domain-label is missing"
+          fi
+
       - name: test enable v2 mon port
         run: |
           toolbox=$(kubectl get pod -l app=rook-ceph-tools -n rook-ceph -o jsonpath='{.items[*].metadata.name}')
@@ -264,7 +297,7 @@ jobs:
         with:
           name: ${{ github.job }}-${{ matrix.ceph-image }}
 
-  raw-disk:
+  raw-disk-with-object:
     runs-on: ubuntu-20.04
     if: "!contains(github.event.pull_request.labels.*.name, 'skip-ci')"
     strategy:
@@ -310,7 +343,13 @@ jobs:
         run: tests/scripts/github-action-helper.sh wait_for_prepare_pod 2
 
       - name: wait for ceph to be ready
-        run: tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
+        run: |
+          tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
+
+      - name: wait for object stores to be ready
+        run: |
+          tests/scripts/validate_cluster.sh rgw store-a
+          tests/scripts/validate_cluster.sh rgw store-b
 
       - name: test toolbox-operator-image pod
         run: |
@@ -1018,7 +1057,7 @@ jobs:
       - name: wait for ceph to be ready
         run: |
           tests/scripts/github-action-helper.sh wait_for_ceph_to_be_ready osd 2
-          tests/scripts/validate_cluster.sh rgw
+          tests/scripts/validate_cluster.sh rgw my-store
           kubectl -n rook-ceph get pods
           kubectl -n rook-ceph get secrets
 
@@ -1039,7 +1078,7 @@ jobs:
           echo "wait for rgw pod to be deleted"
           kubectl wait --for=delete pod -l app=rook-ceph-rgw -n rook-ceph --timeout=100s
           kubectl create -f tests/manifests/test-object.yaml
-          tests/scripts/validate_cluster.sh rgw
+          tests/scripts/validate_cluster.sh rgw my-store
           tests/scripts/deploy-validate-vault.sh validate_rgw
 
       - name: collect common logs
@@ -1527,6 +1566,9 @@ jobs:
         shell: bash --noprofile --norc -eo pipefail -x {0}
         run: tests/scripts/github-action-helper.sh build_rook
 
+      - name: allow holder pod deployment
+        run: sed -i "s|CSI_DISABLE_HOLDER_PODS|# CSI_DISABLE_HOLDER_PODS|g" "deploy/examples/operator.yaml"
+
       - name: validate-yaml
         run: tests/scripts/github-action-helper.sh validate_yaml
 
@@ -1592,6 +1634,9 @@ jobs:
       - name: setup cluster resources
         uses: ./.github/workflows/canary-test-config
 
+      - name: allow holder pod deployment
+        run: sed -i "s|CSI_DISABLE_HOLDER_PODS|# CSI_DISABLE_HOLDER_PODS|g" "deploy/examples/operator.yaml"
+
       - name: set Ceph version in CephCluster manifest
         run: tests/scripts/github-action-helper.sh replace_ceph_image  "deploy/examples/cluster-test.yaml" "${{ github.event.inputs.ceph-image }}"
 

.github/workflows/canary-test-config/action.yaml

@@ -19,7 +19,7 @@ runs:
     - name: Setup Minikube
       shell: bash --noprofile --norc -eo pipefail -x {0}
       run: |
-        tests/scripts/github-action-helper.sh install_minikube_with_none_driver v1.29.1
+        tests/scripts/github-action-helper.sh install_minikube_with_none_driver v1.29.2
 
     - name: install deps
       shell: bash --noprofile --norc -eo pipefail -x {0}

.github/workflows/golangci-lint.yaml

@@ -54,6 +54,7 @@ jobs:
     steps:
       - uses: actions/setup-go@v5
         with:
-         go-version: "1.21.5"
+         go-version: "1.21"
+         check-latest: true
       - name: govulncheck
         uses: golang/govulncheck-action@v1

.github/workflows/helm-lint.yaml

@@ -31,7 +31,7 @@ jobs:
           fetch-depth: 0
 
       - name: Set up Helm
-        uses: azure/setup-helm@v3
+        uses: azure/setup-helm@v4
         with:
           version: v3.6.2
 

.github/workflows/integration-test-helm-suite.yaml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-mgr-suite.yaml

@@ -24,7 +24,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.29.1"]
+        kubernetes-versions: ["v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -36,7 +36,6 @@ jobs:
         with:
           use-tmate: ${{ secrets.USE_TMATE }}
 
-
       - name: setup cluster resources
         uses: ./.github/workflows/integration-test-config-latest-k8s
         with:

.github/workflows/integration-test-multi-cluster-suite.yaml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.29.1"]
+        kubernetes-versions: ["v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-object-suite.yaml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-smoke-suite.yaml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-test-upgrade-suite.yaml

@@ -25,7 +25,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -69,7 +69,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/integration-tests-on-release.yaml

@@ -18,7 +18,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.25.16", "v1.27.10", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -58,7 +58,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.25.16", "v1.27.10", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -99,7 +99,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.25.16", "v1.27.10", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -137,7 +137,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.25.16", "v1.27.10", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -175,7 +175,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.26.11", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.27.11", "v1.28.7", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4
@@ -216,7 +216,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        kubernetes-versions: ["v1.24.17", "v1.29.1"]
+        kubernetes-versions: ["v1.25.16", "v1.29.2"]
     steps:
       - name: checkout
         uses: actions/checkout@v4

.github/workflows/unit-test.yml

@@ -53,6 +53,8 @@ jobs:
       - name: run unit tests
         run: |
           export ROOK_UNIT_JQ_PATH="$(which jq)"
+          # AZURE_EXTENSION_DIR is set in GH action runners and affects Azure KMS unit tests; unset it
+          unset AZURE_EXTENSION_DIR
           GOPATH=$(go env GOPATH) make -j $(nproc) test | tee output.txt
 
       - name: check mds liveness probe script ran successfully

Dockerfile.bundle

@@ -0,0 +1,13 @@
+FROM scratch
+
+# Core bundle labels.
+LABEL operators.operatorframework.io.bundle.mediatype.v1=registry+v1
+LABEL operators.operatorframework.io.bundle.manifests.v1=manifests/
+LABEL operators.operatorframework.io.bundle.metadata.v1=metadata/
+LABEL operators.operatorframework.io.bundle.package.v1=rook-ceph-operator
+LABEL operators.operatorframework.io.bundle.channels.v1=alpha
+LABEL operators.operatorframework.io.bundle.channel.default.v1=alpha
+
+# Copy files to locations specified by labels.
+COPY build/csv/ceph /manifests/
+COPY build/bundle/annotations.yaml /metadata/

Documentation/CRDs/Cluster/.pages

@@ -4,5 +4,5 @@ nav:
     - host-cluster.md
     - pvc-cluster.md
     - stretch-cluster.md
-    - external-cluster.md
+    - external-cluster
     - ...