Adding an E2E test that runs a full CAPL cluster test (default, k3s, & rke2) on merging to main

.github/workflows/build_test_ci.yml

@@ -165,11 +165,19 @@ jobs:
       with:
         key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }}
 
-    - name: E2E test
+    - name: Complete E2E Test
+      if: github.ref == 'refs/heads/main' 
       run: make e2etest
       env:
         GITHUB_TOKEN: ${{ secrets.github_token }}
         LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
+
+    - name: Quick E2E Test
+      if: github.ref != 'refs/heads/main'
+      run: make e2etest-quick
+      env:
+        GITHUB_TOKEN: ${{ secrets.github_token }}
+        LINODE_TOKEN: ${{ secrets.LINODE_TOKEN }}
 
     - name: Copy logs
       if: ${{ always() }}

Makefile

@@ -147,7 +147,11 @@ test: generate fmt vet envtest ## Run tests.
 
 .PHONY: e2etest
 e2etest: generate local-deploy chainsaw
-	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e
+	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e/default-capl-cluster --assert-timeout 600s --cluster child-cluster=/e2e/default-capl-cluster/child-cluster-kubeconfig.yaml
+
+.PHONY: e2etest-pr
+e2etest-quick: generate local-deploy chainsaw
+	GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e --selector quick
 
 local-deploy: kind ctlptl tilt kustomize clusterctl
 	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode

e2e/default-CAPL-cluster/assert-child-cluster-resources.yaml

@@ -0,0 +1,94 @@
+---
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  region: us-ord
+  type: g6-standard-2
+status:
+  ready: true
+  instanceState: running
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Machine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  clusterName: ($cluster)
+status:
+  bootstrapReady: true
+  infrastructureReady: true
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: MachineDeployment
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+spec:
+  clusterName: ($cluster)
+  replicas: 1
+status:
+  readyReplicas: 1
+  unavailableReplicas: 0
+  availableReplicas: 1
+---
+apiVersion: controlplane.cluster.x-k8s.io/v1beta1
+kind: KubeadmControlPlane
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+status:
+  readyReplicas: 1
+  unavailableReplicas: 0
+  ready: true
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+    helmreleaseproxy.addons.cluster.x-k8s.io/helmchartproxy-name: linode-cloud-controller-manager
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+    helmreleaseproxy.addons.cluster.x-k8s.io/helmchartproxy-name: cilium
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed
+---
+apiVersion: addons.cluster.x-k8s.io/v1alpha1
+kind: HelmReleaseProxy
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)
+    helmreleaseproxy.addons.cluster.x-k8s.io/helmchartproxy-name: csi-driver-linode
+status:
+  conditions:
+  - type: Ready
+    status: "True"
+  - type: ClusterAvailable
+    status: "True"
+  - type: HelmReleaseReady
+    status: "True"
+  status: deployed

e2e/default-CAPL-cluster/chainsaw-test.yaml

@@ -0,0 +1,103 @@
+# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json
+apiVersion: chainsaw.kyverno.io/v1alpha1
+kind: Test
+metadata:
+  creationTimestamp: null
+  name: default-capl-cluster
+spec:
+  bindings:
+    # A short identifier for the E2E test run
+  - name: run
+    value: (join('-', ['e2e', 'min-lm', env('GIT_REF')]))
+  - name: cluster
+    # Format the cluster name
+    value: (trim((truncate(($run), `32`)), '-'))
+  template: true
+  steps:
+  - name: step-00
+    try:
+    - assert:
+        file: assert-capi-resources.yaml
+  - name: Generate cluster using clusterctl
+    try:
+    - script:
+        env:
+        - name: CLUSTER
+          value: ($cluster)
+        - name: NAMESPACE
+          value: ($namespace)
+        - name: FLAVOR
+          value: (env('FLAVOR'))
+        content: |
+          set -e
+          clusterctl generate cluster $CLUSTER -n $NAMESPACE $FLAVOR --kubernetes-version v1.29.1 --infrastructure linode:v0.0.0 --control-plane-machine-count 1 --worker-machine-count 1 > default-cluster.yaml
+        check:
+          ($error == null): true
+  - name: Apply generated cluster yaml
+    try:
+    - apply:
+        file: 01-create-cluster.yaml
+    - assert:
+        file: assert-child-cluster-resources.yaml
+  - name: Check if the linodes are created
+    try:
+    - script:
+        env:
+        - name: TARGET_API
+          value: api.linode.com
+        - name: TARGET_API_VERSION
+          value: v4beta
+        - name: URI
+          value: linode/instances
+        - name: FILTER
+          value: (to_string({"tags":($cluster)}))
+        content: |
+          set -e
+          curl -s \
+            -H "Authorization: Bearer $LINODE_TOKEN" \
+            -H "X-Filter: $FILTER" \
+            -H "Content-Type: application/json" \
+            "https://$TARGET_API/$TARGET_API_VERSION/$URI"
+        check:
+          ($error): ~
+          (json_parse($stdout)):
+            results: 2
+  - name: Delete child cluster
+    try:
+    - delete:
+        ref:
+          apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Cluster
+          name: ($cluster)
+    - error:
+        file: check-child-cluster-deleted.yaml
+  - name: Check if the linodes are deleted
+    try:
+    - script:
+        env:
+        - name: TARGET_API
+          value: api.linode.com
+        - name: TARGET_API_VERSION
+          value: v4beta
+        - name: URI
+          value: linode/instances
+        - name: FILTER
+          value: (to_string({"tags":($cluster)}))
+        content: |
+          set -e
+          curl -s \
+            -H "Authorization: Bearer $LINODE_TOKEN" \
+            -H "X-Filter: $FILTER" \
+            -H "Content-Type: application/json" \
+            "https://$TARGET_API/$TARGET_API_VERSION/$URI"
+        check:
+          ($error): ~
+          (json_parse($stdout)):
+            results: 0
+  - name: Delete generated child cluster manifest yaml
+    try:
+    - script:
+        content: |
+          rm -f e2e-auto-gen*.yaml
+        check:
+          ($error == null): true

e2e/default-CAPL-cluster/check-child-cluster-deleted.yaml

@@ -0,0 +1,5 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  labels:
+    cluster.x-k8s.io/cluster-name: ($cluster)

e2e/linodecluster-controller/minimal-linodecluster/assert-capi-resources.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capl-controller-manager
+  namespace: capl-system
+status:
+  availableReplicas: 1

e2e/linodecluster-controller/minimal-linodecluster/chainsaw-test.yaml

@@ -4,6 +4,9 @@ kind: Test
 metadata:
   creationTimestamp: null
   name: minimal-linodecluster
+  # Label to trigger the test on every PR
+  labels:
+    quick:
 spec:
   bindings:
     # A short identifier for the E2E test run
@@ -16,21 +19,21 @@ spec:
     value: ($cluster)
   template: true
   steps:
-  - name: step-00
+  - name: Check if CAPI provider resources exist
     try:
     - assert:
-        file: 00-assert.yaml
-  - name: step-01
+        file: assert-capi-resources.yaml
+  - name: Create Cluster resource
     try:
     - apply:
-        file: 01-create-cluster.yaml
-  - name: step-02
+        file: create-cluster.yaml
+  - name: Create LinodeCluster resource
     try:
     - apply:
-        file: 02-create-linodecluster.yaml
+        file: create-linodecluster.yaml
     - assert:
-        file: 02-assert.yaml
-  - name: step-03
+        file: assert-linodecluster.yaml
+  - name: Check if the nodebalancer exists
     try:
     - script:
         env:
@@ -53,16 +56,16 @@ spec:
           ($error): ~
           (json_parse($stdout)):
             results: 1
-  - name: step-04
+  - name: Delete Cluster resource
     try:
     - delete:
         ref:
           apiVersion: cluster.x-k8s.io/v1beta1
           kind: Cluster
           name: ($cluster)
     - error:
-        file: 04-error.yaml
-  - name: step-05
+        file: check-linodecluster-deleted.yaml
+  - name: Check if the nodebalancer is deleted
     try:
     - script:
         env:
@@ -85,7 +88,3 @@ spec:
           ($error): ~
           (json_parse($stdout)):
             results: 0
-  - name: step-06
-    try:
-    - apply:
-        file: 06-cleanup-cluster.yaml

e2e/linodemachine-controller/minimal-linodemachine/chainsaw-test.yaml

@@ -4,6 +4,9 @@ kind: Test
 metadata:
   creationTimestamp: null
   name: minimal-linodemachine
+  # Label to trigger the test on every PR
+  labels:
+    quick:
 spec:
   bindings:
     # A short identifier for the E2E test run

e2e/linodemachine-controller/vpc-integration/chainsaw-test.yaml

@@ -4,6 +4,10 @@ kind: Test
 metadata:
   creationTimestamp: null
   name: vpc-integration
+  # Label to trigger the test on every PR
+  labels:
+    quick:
+    vpc-integration:
 spec:
   bindings:
     # A short identifier for the E2E test run

e2e/linodeobjectstoragebucket-controller/minimal-linodeobjectstoragebucket/chainsaw-test.yaml

@@ -3,6 +3,9 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
   name: minimal-linodeobjectstoragebucket
+  # Label to trigger the test on every PR
+  labels:
+    quick:
 spec:
   bindings:
     # A short identifier for the E2E test run

e2e/linodevpc-controller/minimal-linodevpc/chainsaw-test.yaml

@@ -3,6 +3,10 @@ apiVersion: chainsaw.kyverno.io/v1alpha1
 kind: Test
 metadata:
   name: minimal-linodevpc
+  # Label to trigger the test on every PR
+  labels:
+    quick:
+    min-vpc:
 spec:
   bindings:
     # A short identifier for the E2E test run

e2e/rke2-capl-cluster/assert-capi-resources.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capl-controller-manager
+  namespace: capl-system
+status:
+  availableReplicas: 1