diff --git a/.github/workflows/operator_ci.yaml b/.github/workflows/operator_ci.yaml new file mode 100644 index 000000000..75e1d0a7f --- /dev/null +++ b/.github/workflows/operator_ci.yaml @@ -0,0 +1,181 @@ +name: Operator Build +on: + push: + branches: + - develop + - master + pull_request: + branches: + - develop + - master +jobs: + prepare_env: + name: "Prepare env" + runs-on: ubuntu-latest + steps: + - name: Checkout code + uses: actions/checkout@v2 + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.9.7' + - uses: FranzDiebold/github-env-vars-action@v2 + - name: Setup dependencies + id: setup_dependencies + run: | + build/ci/github_actions/setup_ci_dependencies.sh + env: + build_number: "${{ github.run_number }}" + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('dev-requirements.txt') }} + - name: Save dependencies files + uses: actions/upload-artifact@v2 + with: + name: ci_dependencies + path: | + /home/runner/.bash_profile + retention-days: 1 + outputs: + branch_image_tag: "${{ steps.setup_dependencies.outputs.branch_image_tag }}" + operator_specific_tag_for_test: "${{ steps.setup_dependencies.outputs.operator_specific_tag_for_test }}" + + unit_testing: + runs-on: ubuntu-latest + name: "Unit testing" + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run unit tests + run: | + mkdir -p build/_output/reports && chmod 777 build/_output/reports + ./build/ci/run_unittests.sh + + run_olm_validation: + runs-on: ubuntu-latest + name: "Run OLM validation" + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Run OLM validation + run: | + ./build/ci/run_olm_validation.sh + + build_and_push_images: + name: "Build and push images" + runs-on: ubuntu-latest + needs: + - unit_testing + - run_olm_validation + - prepare_env + strategy: + matrix: + image_type: ['operator'] + steps: + - name: Checkout + uses: actions/checkout@v2 + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + - name: Set up Docker Buildx + id: buildx + uses: docker/setup-buildx-action@v1.6.0 + - uses: FranzDiebold/github-env-vars-action@v2 + - name: Login to DockerHub + uses: docker/login-action@v1 + with: + username: '${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}' + password: '${{ secrets.CSIBLOCK_DOCKER_REGISTRY_PASSWORD }}' + - name: Cache Docker layers + uses: actions/cache@v2 + with: + path: /tmp/.buildx-${{ matrix.image_type }} + key: ${{ runner.os }}-buildx-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-buildx- + - name: Build and push ${{ matrix.image_type }} + uses: docker/build-push-action@v2 + with: + context: . + platforms: linux/amd64,linux/ppc64le,linux/s390x + push: true + tags: | + ${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}/ibm-block-csi-${{ matrix.image_type }}:${{ needs.prepare_env.outputs.operator_specific_tag_for_test }} + ${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}/ibm-block-csi-${{ matrix.image_type }}:${{ needs.prepare_env.outputs.branch_image_tag }} + file: build/Dockerfile.${{ matrix.image_type }} + cache-from: type=local,src=/tmp/.buildx-${{ matrix.image_type }} + cache-to: type=local,dest=/tmp/.buildx-new-${{ matrix.image_type }} + # Temp fix + # https://github.com/docker/build-push-action/issues/252 + # https://github.com/moby/buildkit/issues/1896 + - name: Move cache + run: | + [[ -G /tmp/.buildx-${{ matrix.image_type }} ]] && rm -rf /tmp/.buildx-${{ matrix.image_type }} + mv /tmp/.buildx-new-${{ matrix.image_type }} /tmp/.buildx-${{ matrix.image_type }} + outputs: + operator_specific_tag_for_test: "${{ needs.prepare_env.outputs.operator_specific_tag_for_test }}" + + deploy_csi_on_cluster: + runs-on: ubuntu-latest + name: "Deploy csi on cluster" + needs: build_and_push_images + env: + csiblock_docker_registry_username: ${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }} + controller_repository_for_test: '${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}/ibm-block-csi-controller' + node_repository_for_test: '${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}/ibm-block-csi-node' + operator_image_repository_for_test: '${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}/ibm-block-csi-operator' + operator_specific_tag_for_test: "${{ needs.build_and_push_images.outputs.operator_specific_tag_for_test }}" + operator_image_for_test: "${{ secrets.CSIBLOCK_DOCKER_REGISTRY_USERNAME }}/ibm-block-csi-operator:${{ needs.build_and_push_images.outputs.operator_specific_tag_for_test }}" + operator_yaml: "deploy/installer/generated/ibm-block-csi-operator.yaml" + cr_yaml: "config/samples/csi.ibm.com_v1_ibmblockcsi_cr.yaml" + timeout-minutes: 7 + steps: + - name: Checkout + uses: actions/checkout@v2 + - uses: FranzDiebold/github-env-vars-action@v2 + - name: Retrieve ci dependencies + uses: actions/download-artifact@v2 + with: + name: ci_dependencies + path: /home/runner + - name: Set up Python + uses: actions/setup-python@v2 + with: + python-version: '3.x' + - uses: actions/cache@v2 + with: + path: ${{ env.pythonLocation }} + key: ${{ env.pythonLocation }}-${{ hashFiles('dev-requirements.txt') }} + - name: Get driver images tag from branch + id: driver_images_tag_from_branch + run: | + build/ci/github_actions/driver/get_driver_images_tag_from_branch.sh + env: + github_token: ${{ secrets.CSIBLOCK_GITHUB_TOKEN }} + - name: Create k8s Kind Cluster + uses: helm/kind-action@v1.1.0 + with: + version: v0.11.1 + - name: Wait for k8s cluster to be ready + run: | + build/ci/github_actions/wait_for_k8s_ready.sh + - name: Deploy ibm block csi operator + run: | + build/ci/github_actions/operator/deploy_operator.sh + - name: Deploy ibm block csi driver + run: | + build/ci/github_actions/driver/deploy_driver.sh + env: + driver_images_tag: "${{ steps.driver_images_tag_from_branch.outputs.branch_image_tag }}" + - name: Save driver pods logs to artifacts + if: always() + run: | + build/ci/github_actions/driver/save_driver_logs_to_artifacts.sh + - name: Upload driver pods logs + if: always() + uses: actions/upload-artifact@v2 + with: + name: driver_logs + path: | + /tmp/*.txt + retention-days: 7 diff --git a/build/ci/build_push_image.sh b/build/ci/build_push_image.sh index be9e7637c..b849718ed 100755 --- a/build/ci/build_push_image.sh +++ b/build/ci/build_push_image.sh @@ -7,8 +7,8 @@ for envi in $MANDATORY_ENVS; do done # Prepare specific tag for the image -branch=`echo $GIT_BRANCH| sed 's|/|.|g'` #not sure if docker accept / in the version -specific_tag="${IMAGE_VERSION}_b${BUILD_NUMBER}_${branch}" +tags=$(build/ci/get_image_tags_from_branch.sh ${GIT_BRANCH} ${IMAGE_VERSION} ${BUILD_NUMBER} ${GIT_COMMIT}) +specific_tag=$(echo $tags | awk '{print$1}') # Set latest tag only if its from develop branch or master and prepare tags [ "$GIT_BRANCH" = "develop" -o "$GIT_BRANCH" = "origin/develop" -o "$GIT_BRANCH" = "master" ] && tag_latest="true" || tag_latest="false" diff --git a/build/ci/get_image_tags_from_branch.sh b/build/ci/get_image_tags_from_branch.sh new file mode 100755 index 000000000..4e4f13c80 --- /dev/null +++ b/build/ci/get_image_tags_from_branch.sh @@ -0,0 +1,9 @@ +#!/bin/bash -xe +GIT_BRANCH=$1 +IMAGE_VERSION=$2 +BUILD_NUMBER=$3 +COMMIT_HASH=${4:0:7} +branch_image_tag=$(echo $GIT_BRANCH| sed 's|/|.|g') #not sure if docker accept / in the version +specific_tag="${IMAGE_VERSION}_b${BUILD_NUMBER}_${COMMIT_HASH}_${branch_image_tag}" +echo $specific_tag +echo $branch_image_tag diff --git a/build/ci/github_actions/deployment.sh b/build/ci/github_actions/deployment.sh new file mode 100755 index 000000000..06d01ff03 --- /dev/null +++ b/build/ci/github_actions/deployment.sh @@ -0,0 +1,103 @@ +#!/bin/bash -xe +set +o pipefail + +is_driver_ready=false +actual_driver_running_time_in_seconds=0 +minimum_driver_running_time_in_seconds=10 +containers_prefix=ibm-block-csi +declare -a driver_pods_types=( + "controller" + "node" +) + +get_csi_pods (){ + kubectl get pod -A -l csi +} + +get_operator_pod (){ + kubectl get pod -A -l app.kubernetes.io/name=ibm-block-csi-operator +} + +get_image_pod_by_type (){ + pod_type=$1 + component_to_check=$2 + containers_images=$(kubectl get pods $(get_csi_pods | grep $pod_type | awk '{print$2}') -o jsonpath='{range .spec.containers[*]}{.name},{.image} {end}') + for containers_image in $containers_images + do + if [[ "$containers_image" =~ "$component_to_check," ]]; then + echo $containers_image | awk -F , '{print$2}' + break + fi + done +} + +wait_for_pod_to_start (){ + driver_pod_type=$1 + while [ "$(get_csi_pods | grep $driver_pod_type | wc -l)" -eq 0 ]; do + echo "The $driver_pod_type is not deployed" + sleep 1 + done +} + +wait_for_driver_deployment_to_start (){ + for driver_pods_type in "${driver_pods_types[@]}" + do + wait_for_pod_to_start $driver_pods_type + done +} + +wait_for_driver_deployment_to_finish (){ + wait_for_pods_to_finish get_csi_pods +} + +wait_for_operator_deployment_to_finish (){ + wait_for_pods_to_finish get_operator_pod +} + + +wait_for_pods_to_finish (){ + get_pods_command=$1 + while [ $is_driver_ready == "false" ]; do + if [ "$($get_pods_command | grep -iv running | grep -iv name | wc -l)" -eq 0 ]; then + ((++actual_driver_running_time_in_seconds)) + if [ $actual_driver_running_time_in_seconds -eq $minimum_driver_running_time_in_seconds ]; then + is_driver_ready=true + fi + else + actual_driver_running_time_in_seconds=0 + fi + get_csi_pods + sleep 1 + done +} + +assert_expected_image_in_pod (){ + pod_type=$1 + expected_pod_image=$2 + component_to_check=$containers_prefix-$pod_type + image_in_pod=$(get_image_pod_by_type $pod_type $component_to_check) + if [[ $image_in_pod != $expected_pod_image ]]; then + echo "$pod_type's image ($image_in_pod) is not the expected image ($expected_pod_image)" + exit 1 + fi +} + +assert_driver_images_in_pods (){ + expected_node_image=$1 + expected_controller_image=$2 + declare -A drivers_components_in_k8s=( + ["controller"]="$expected_controller_image" + ["node"]="$expected_node_image" + ) + for driver_component in ${!drivers_components_in_k8s[@]}; do + driver_component_expected_image=${drivers_components_in_k8s[${driver_component}]} + assert_expected_image_in_pod $driver_component $driver_component_expected_image + done +} + +assert_operator_image_in_pod (){ + operator_image_for_test=$1 + kubectl apply -f $operator_yaml + wait_for_pod_to_start "operator" + assert_expected_image_in_pod "operator" $operator_image_for_test +} diff --git a/build/ci/github_actions/driver/deploy_driver.sh b/build/ci/github_actions/driver/deploy_driver.sh new file mode 100755 index 000000000..00865dc0a --- /dev/null +++ b/build/ci/github_actions/driver/deploy_driver.sh @@ -0,0 +1,35 @@ +#!/bin/bash -xel +set +o pipefail + +expected_node_image=$node_repository_for_test:$driver_images_tag +expected_controller_image=$controller_repository_for_test:$driver_images_tag + +install_worker_prerequisites() { + kind_node_name=$(docker ps --format "{{.Names}}") + docker exec -i $kind_node_name apt-get update + docker exec -i $kind_node_name apt -y install open-iscsi +} + +edit_cr_images (){ + chmod 547 $cr_yaml + declare -A cr_image_fields=( + [".spec.controller.repository"]="$controller_repository_for_test" + [".spec.controller.tag"]="$driver_images_tag" + [".spec.node.repository"]="$node_repository_for_test" + [".spec.node.tag"]="$driver_images_tag" + ) + for image_field in ${!cr_image_fields[@]}; do + cr_image_value=${cr_image_fields[${image_field}]} + yq eval "${image_field} |= \"${cr_image_value}\"" $cr_yaml -i + done +} + +install_worker_prerequisites +edit_cr_images +cat $cr_yaml | grep repository: +cat $cr_yaml | grep tag: +kubectl apply -f $cr_yaml +source build/ci/github_actions/deployment.sh +wait_for_driver_deployment_to_start +assert_driver_images_in_pods $expected_node_image $expected_controller_image +wait_for_driver_deployment_to_finish diff --git a/build/ci/github_actions/driver/get_driver_images_tag_from_branch.sh b/build/ci/github_actions/driver/get_driver_images_tag_from_branch.sh new file mode 100755 index 000000000..de129d9d4 --- /dev/null +++ b/build/ci/github_actions/driver/get_driver_images_tag_from_branch.sh @@ -0,0 +1,28 @@ +#!/bin/bash -xe +set +o pipefail + +driver_images_tag_from_branch=latest +triggering_branch=$CI_ACTION_REF_NAME +target_image_tags=$(build/ci/get_image_tags_from_branch.sh ${triggering_branch}) +branch_image_tag=$(echo $branch_image_tag | awk '{print$2}') + +is_private_branch_component_image_exists(){ + driver_component=$1 + image_to_check=$csiblock_docker_registry_username/ibm-block-csi-$driver_component:$branch_image_tag + is_image_tag_exists=false + export driver_image_inspect=$(docker manifest inspect $image_to_check &> /dev/null; echo $?) + if [ $driver_image_inspect == "0" ]; then + echo true + else + echo false + fi +} + +is_controller_image_tag_exists=$(is_private_branch_component_image_exists controller) +is_node_image_tag_exists=$(is_private_branch_component_image_exists node) + +if [ $is_controller_image_tag_exists == "true" ] && [ $is_node_image_tag_exists == "true " ]; then + driver_images_tag_from_branch=$branch_image_tag +fi + +echo "::set-output name=branch_image_tag::${driver_images_tag_from_branch}" diff --git a/build/ci/github_actions/driver/save_driver_logs_to_artifacts.sh b/build/ci/github_actions/driver/save_driver_logs_to_artifacts.sh new file mode 100755 index 000000000..952fe7b03 --- /dev/null +++ b/build/ci/github_actions/driver/save_driver_logs_to_artifacts.sh @@ -0,0 +1,38 @@ +#!/bin/bash -x + +get_all_pods_by_type (){ + pod_type=$1 + kubectl get pod -l csi | grep $pod_type | awk '{print$1}' +} + +run_action_and_save_output (){ + pod_type=$1 + action=$2 + action_name=$3 + extra_args=$4 + container_name=$5 + pod_names=$(get_all_pods_by_type $pod_type) + kubectl $action $pod_names $extra_args > "/tmp/${pod_names}_${container_name}_${action_name}.txt" +} + +save_logs_of_all_containers_in_pod (){ + pod_type=$1 + pod_names=$(get_all_pods_by_type $pod_type) + containers=$(kubectl get pods $pod_names -o jsonpath='{.spec.containers[*].name}') + for container in $containers + do + run_action_and_save_output $pod_type logs "log" "-c $container" $container + done +} + +declare -a pod_types=( + "node" + "controller" + "operator" +) + +for pod_type in "${pod_types[@]}" +do + save_logs_of_all_containers_in_pod $pod_type + run_action_and_save_output $pod_type "describe pod" "describe" "" $pod_type +done diff --git a/build/ci/github_actions/operator/deploy_operator.sh b/build/ci/github_actions/operator/deploy_operator.sh new file mode 100755 index 000000000..691fe62ad --- /dev/null +++ b/build/ci/github_actions/operator/deploy_operator.sh @@ -0,0 +1,13 @@ +#!/bin/bash -xel +set +o pipefail + +edit_operator_yaml_image (){ + operator_image_in_branch=$(yq eval '(. | select(.kind == "Deployment") | .spec.template.spec.containers[0].image)' $operator_yaml) + sed -i "s+$operator_image_in_branch+$operator_image_for_test+g" $operator_yaml ## TODO: CSI-3223 avoid using sed +} + +edit_operator_yaml_image +cat $operator_yaml | grep image: +source build/ci/github_actions/deployment.sh +assert_operator_image_in_pod $operator_image_for_test +wait_for_operator_deployment_to_finish diff --git a/build/ci/github_actions/setup_ci_dependencies.sh b/build/ci/github_actions/setup_ci_dependencies.sh new file mode 100755 index 000000000..64e5f0e28 --- /dev/null +++ b/build/ci/github_actions/setup_ci_dependencies.sh @@ -0,0 +1,26 @@ +#!/bin/bash -xe +set +o pipefail + +install_ci_dependencies (){ + build/ci/github_actions/setup_yq.sh + python -m pip install --upgrade pip==21.2.4 + echo docker-hub==2.2.0 > dev-requirements.txt + pip install -r dev-requirements.txt +} + +install_ci_dependencies +triggering_branch=${CI_ACTION_REF_NAME} +# CSI-3173 - move image_version value into a common config file +image_version=$(cat version/version.go | grep -i driverversion | awk -F = '{print $2}') +image_version=$(echo ${image_version//\"}) +operator_image_tags_for_test=$(build/ci/get_image_tags_from_branch.sh ${triggering_branch} ${image_version} ${build_number} ${GITHUB_SHA}) +operator_specific_tag_for_test=$(echo $operator_image_tags_for_test | awk '{print$1}') + +if [ "$triggering_branch" == "develop" ]; then + branch_image_tag=latest +else + branch_image_tag=$(echo $operator_image_tags_for_test | awk '{print$2}') +fi + +echo "::set-output name=operator_specific_tag_for_test::${operator_specific_tag_for_test}" +echo "::set-output name=branch_image_tag::${branch_image_tag}" diff --git a/build/ci/github_actions/setup_yq.sh b/build/ci/github_actions/setup_yq.sh new file mode 100755 index 000000000..71966a5c2 --- /dev/null +++ b/build/ci/github_actions/setup_yq.sh @@ -0,0 +1,8 @@ +#!/bin/bash -xe +set +o pipefail + +cat >>/home/runner/.bash_profile <<'EOL' +yq() { + docker run --rm -i -v "${PWD}":/workdir mikefarah/yq "$@" +} +EOL diff --git a/build/ci/github_actions/wait_for_k8s_ready.sh b/build/ci/github_actions/wait_for_k8s_ready.sh new file mode 100755 index 000000000..69a2abd52 --- /dev/null +++ b/build/ci/github_actions/wait_for_k8s_ready.sh @@ -0,0 +1,25 @@ +#!/bin/bash -xe +set +o pipefail + +are_pods_ready (){ + pods=$@ + for pod in $pods; do + running_containers_count=$(echo $pod | awk -F / '{print$1}') + total_containers_count=$(echo $pod | awk -F / '{print$2}') + if [ $running_containers_count != $total_containers_count ]; then + echo false + break + fi + done + echo true +} + +is_kubernetes_cluster_ready (){ + pods=$(kubectl get pods -A | awk '{print$3}' | grep -iv ready) + are_all_pods_ready=$(are_pods_ready $pods) + echo $are_all_pods_ready +} + +while [[ $(is_kubernetes_cluster_ready) == "false" ]]; do + kubectl get pods -A +done