[CI] : add workflow to test upgrade (#636)

* add workflow to test upgrade

* test upgrade

* fetch tags as well

* run kubeadm-capl-cluster chainsaw test as well

* run local-release after checkout

* increase assert timeout to fix test failures

Author: rahulait

.github/workflows/e2e-upgrade-test.yaml

@@ -0,0 +1,100 @@
+name: Run e2e upgrade test
+
+on:
+  push:
+    branches:
+      - test-upgrade
+    tags:
+      - "*"
+  workflow_dispatch:
+
+permissions:
+  contents: read
+  pull-requests: read
+  actions: read
+
+concurrency:
+  group: e2e-upgrade-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  changes:
+    runs-on: ubuntu-latest
+    outputs:
+      # Expose matched filters as job 'src' output variable
+      paths: ${{ steps.filter.outputs.changes }}
+    steps:
+      - uses: actions/checkout@v4
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          disable-sudo: true
+          egress-policy: block
+          allowed-endpoints: >
+            api.github.com:443
+            github.com:443
+      - uses: dorny/paths-filter@v3
+        id: filter
+        with:
+          filters: .github/filters.yml
+  e2e-upgrade-test:
+    needs: changes
+    name: e2e-upgrade-test
+    if: ${{contains(fromJSON(needs.changes.outputs.paths), 'src')}}
+    runs-on: ubuntu-latest
+    env:
+      GITHUB_TOKEN: ${{ secrets.github_token }}
+      LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
+    steps:
+      - name: Harden Runner
+        uses: step-security/harden-runner@v2
+        with:
+          disable-sudo: true
+          egress-policy: audit
+          allowed-endpoints: >
+            *:6443
+            api.linode.com:443
+            api.github.com:443
+            github.com:443
+            gcr.io:443
+            ghcr.io:443
+            proxy.golang.org:443
+            sum.golang.org:443
+            *.githubusercontent.com:443
+            docker.io:443
+            registry-1.docker.io:443
+            auth.docker.io:443
+            production.cloudflare.docker.com:443
+            storage.googleapis.com:443
+            registry.k8s.io:443
+            *.pkg.dev:443
+            *.amazonaws.com:443
+            *.blob.core.windows.net:443
+            quay.io:443
+            *.quay.io:443
+            api.snapcraft.io:443
+            cloud.tilt.dev:443
+            kubernetes-sigs.github.io:443
+            charts.jetstack.io:443
+            helm.cilium.io:443
+            linode.github.io:443
+
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
+      - name: Run Upgrade Test
+        env:
+          LINODE_REGION: us-sea
+          LINODE_CONTROL_PLANE_MACHINE_TYPE: g6-standard-2
+          LINODE_MACHINE_TYPE: g6-standard-2
+          CLUSTERCTL_CONFIG: /home/runner/work/cluster-api-provider-linode/cluster-api-provider-linode/e2e/gha-clusterctl-config.yaml
+        run: make test-upgrade
+
+      - name: cleanup stale clusters
+        if: ${{ always() }}
+        run: make clean-child-clusters
+
+      - name: cleanup kind mgmt cluster
+        if: ${{ always() }}
+        run: make clean-kind-cluster

Makefile

@@ -159,10 +159,43 @@ test: generate fmt vet envtest ## Run tests.
 e2etest: generate local-release local-deploy chainsaw s5cmd
 	GIT_REF=$(GIT_REF) SSE_KEY=$$(openssl rand -base64 32) LOCALBIN=$(CACHE_BIN) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS)
 
-local-deploy: kind ctlptl tilt kustomize clusterctl
-	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
+.PHONY: local-deploy
+local-deploy: kind-cluster tilt kustomize clusterctl
 	$(TILT) ci -f Tiltfile
 
+.PHONY: kind-cluster
+kind-cluster: kind ctlptl
+	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
+
+##@ Test Upgrade:
+
+LATEST_REF         := $(shell git rev-parse --short HEAD)
+LAST_RELEASE       := $(shell git describe --abbrev=0 --tags)
+COMMON_CLUSTER_REF := $(shell echo "up-$(LATEST_REF)" | cut -c1-8)
+COMMON_NAMESPACE   := test-upgrade
+
+.PHONY: checkout-latest-commit
+checkout-latest-commit:
+	git checkout $(LATEST_REF)
+
+.PHONY: checkout-last-release
+checkout-last-release:
+	git checkout $(LAST_RELEASE)
+
+.PHONY: last-release-cluster
+last-release-cluster: kind ctlptl tilt kustomize clusterctl chainsaw kind-cluster checkout-last-release local-release local-deploy
+	GIT_REF=$(COMMON_CLUSTER_REF) LOCALBIN=$(CACHE_BIN) CLUSTERCTL_CONFIG=$(CLUSTERCTL_CONFIG) $(CHAINSAW) test --namespace $(COMMON_NAMESPACE) --assert-timeout 600s --skip-delete ./e2e/capl-cluster-flavors/kubeadm-capl-cluster
+
+.PHONY: test-upgrade
+test-upgrade: last-release-cluster checkout-latest-commit
+	$(MAKE) local-release
+	$(MAKE) local-deploy
+	GIT_REF=$(COMMON_CLUSTER_REF) LOCALBIN=$(CACHE_BIN) CLUSTERCTL_CONFIG=$(CLUSTERCTL_CONFIG) $(CHAINSAW) test --namespace $(COMMON_NAMESPACE) --assert-timeout 600s ./e2e/capl-cluster-flavors/kubeadm-capl-cluster
+
+.PHONY: clean-kind-cluster
+clean-kind-cluster: ctlptl
+	$(CTLPTL) delete -f .tilt/ctlptl-config.yaml
+
 ## --------------------------------------
 ## Build
 ## --------------------------------------

e2e/capl-cluster-flavors/kubeadm-capl-cluster/chainsaw-test.yaml

@@ -16,7 +16,9 @@ spec:
       value: (join('-', ['e2e', 'default-cluster', env('GIT_REF')]))
     - name: cluster
       # Format the cluster name
-      value: (trim((truncate(($run), `32`)), '-'))
+      # linode firewall has limit of max 32 chars, so we truncate the cluster name to 29 chars
+      # for firewall named as <cluster>-nb
+      value: (trim((truncate(($run), `29`)), '-'))
   template: true
   steps:
     - name: Check if CAPI provider resources exist
@@ -33,10 +35,12 @@ spec:
                 value: ($namespace)
               - name: CLUSTERCTL_CONFIG
                 value: (env('CLUSTERCTL_CONFIG'))
+              - name: KUBERNETES_VERSION
+                value: (env('KUBERNETES_VERSION') || 'v1.29.1')
             content: |
               set -e
               clusterctl generate cluster $CLUSTER -n $NAMESPACE \
-              --kubernetes-version v1.29.1 \
+              --kubernetes-version $KUBERNETES_VERSION \
               --infrastructure local-linode:v0.0.0 \
               --control-plane-machine-count 1 --worker-machine-count 1 \
               --config ${CLUSTERCTL_CONFIG:=${HOME}/.cluster-api/clusterctl.yaml} > default-cluster.yaml