Choice for downstream clusters cleanup

Signed-off-by: Chandan Pinjani <[email protected]>
rancher · Feb 19, 2024 · 9d8a546 · 9d8a546
1 parent a2145f6
commit 9d8a546
Show file tree

Hide file tree

Showing 17 changed files with 106 additions and 58 deletions.
diff --git a/.github/workflows/aks.yaml b/.github/workflows/aks.yaml
@@ -17,6 +17,10 @@ on:
         required: true
         type: string
         default: v1.27.9+k3s1
+      runner_template:
+        description: Runner template to use
+        default: hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1
+        type: string
       operator_nightly_chart:
         description: Install hosted-provider nightly chart
         default: true
@@ -26,10 +30,10 @@ on:
         description: Destroy the auto-generated self-hosted runner
         default: true
         type: boolean
-      runner_template:
-        description: Runner template to use
-        default: hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1
-        type: string
+      downstream_cluster_cleanup:
+        description: Cleanup downstream clusters after test
+        default: true
+        type: boolean
       downstream_k8s_version:
         description: Downstream cluster K8s version to test
         default: 1.26.6
@@ -60,3 +64,4 @@ jobs:
       runner_template: ${{ inputs.runner_template || 'hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1' }}
       downstream_k8s_version: ${{ inputs.downstream_k8s_version || '1.26.6' }}
       rancher_installed: ${{ inputs.rancher_installed || 'hostname/password' }}
+      downstream_cluster_cleanup: ${{ inputs.downstream_cluster_cleanup == true || (github.event_name == 'schedule' && true) }}
diff --git a/.github/workflows/eks.yaml b/.github/workflows/eks.yaml
@@ -17,6 +17,10 @@ on:
         required: true
         type: string
         default: v1.27.9+k3s1
+      runner_template:
+        description: Runner template to use
+        default: hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1
+        type: string
       operator_nightly_chart:
         description: Install hosted-provider nightly chart
         default: true
@@ -26,10 +30,10 @@ on:
         description: Destroy the auto-generated self-hosted runner
         default: true
         type: boolean
-      runner_template:
-        description: Runner template to use
-        default: hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1
-        type: string
+      downstream_cluster_cleanup:
+        description: Cleanup downstream clusters after test
+        default: true
+        type: boolean
       downstream_k8s_version:
         description: Downstream cluster K8s version to test
         default: 1.26
@@ -60,3 +64,4 @@ jobs:
       runner_template: ${{ inputs.runner_template || 'hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1' }}
       downstream_k8s_version: ${{ inputs.downstream_k8s_version || '1.26' }}
       rancher_installed: ${{ inputs.rancher_installed || 'hostname/password' }}
+      downstream_cluster_cleanup: ${{ inputs.downstream_cluster_cleanup == true || (github.event_name == 'schedule' && true) }}
diff --git a/.github/workflows/gke.yaml b/.github/workflows/gke.yaml
@@ -17,6 +17,10 @@ on:
         required: true
         type: string
         default: v1.27.9+k3s1
+      runner_template:
+        description: Runner template to use
+        default: hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1
+        type: string
       operator_nightly_chart:
         description: Install hosted-provider nightly chart
         default: true
@@ -26,10 +30,10 @@ on:
         description: Destroy the auto-generated self-hosted runner
         default: true
         type: boolean
-      runner_template:
-        description: Runner template to use
-        default: hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1
-        type: string
+      downstream_cluster_cleanup:
+        description: Cleanup downstream clusters after test
+        default: true
+        type: boolean
       downstream_k8s_version:
         description: Downstream cluster K8s version to test
         default: 1.27.3-gke.100
@@ -60,3 +64,4 @@ jobs:
       runner_template: ${{ inputs.runner_template || 'hosted-prov-e2e-ci-runner-spot-n2-highmem-16-gl-template-v1' }}
       downstream_k8s_version: ${{ inputs.downstream_k8s_version || '1.27.3-gke.100' }}
       rancher_installed: ${{ inputs.rancher_installed || 'hostname/password' }}
+      downstream_cluster_cleanup: ${{ inputs.downstream_cluster_cleanup == true || (github.event_name == 'schedule' && true) }}
diff --git a/.github/workflows/main.yaml b/.github/workflows/main.yaml
@@ -45,6 +45,9 @@ on:
       downstream_k8s_version:
         description: Downstream cluster K8s version to test
         type: string
+      downstream_cluster_cleanup:
+        description: Cleanup downstream clusters after test
+        type: boolean
 
 env:
   AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
@@ -61,6 +64,7 @@ env:
   RANCHER_LOG_COLLECTOR: ${{ github.workspace }}/.github/scripts/collect-rancher-logs.sh
   GCP_RUNNER_ZONE: asia-south2-c
   DOWNSTREAM_KUBERNETES_VERSION: ${{ inputs.downstream_k8s_version }}
+  DOWNSTREAM_CLUSTER_CLEANUP: ${{ inputs.downstream_cluster_cleanup }}
 jobs:
   create-runner:
     runs-on: ubuntu-latest

diff --git a/hosted/aks/p0/p0_importing_test.go b/hosted/aks/p0/p0_importing_test.go
@@ -54,10 +54,12 @@ var _ = Describe("P0Importing", func() {
 			cluster.AKSConfig = cluster.AKSStatus.UpstreamSpec
 		})
 		AfterEach(func() {
-			err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
-			Expect(err).To(BeNil())
-			err = helper.DeleteAKSClusteronAzure(clusterName)
-			Expect(err).To(BeNil())
+			if ctx.ClusterCleanup {
+				err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
+				Expect(err).To(BeNil())
+				err = helper.DeleteAKSClusteronAzure(clusterName)
+				Expect(err).To(BeNil())
+			}
 		})
 		It("should successfully import the cluster & add, delete, scale nodepool", func() {
 

diff --git a/hosted/aks/p0/p0_provisioning_test.go b/hosted/aks/p0/p0_provisioning_test.go
@@ -52,10 +52,12 @@ var _ = Describe("P0Provisioning", func() {
 			Expect(err).To(BeNil())
 		})
 		AfterEach(func() {
-			err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
-			Expect(err).To(BeNil())
-			err = helper.DeleteAKSClusteronAzure(clusterName)
-			Expect(err).To(BeNil())
+			if ctx.ClusterCleanup {
+				err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
+				Expect(err).To(BeNil())
+				err = helper.DeleteAKSClusteronAzure(clusterName)
+				Expect(err).To(BeNil())
+			}
 		})
 		It("should successfully provision the cluster & add, delete, scale nodepool", func() {
 

diff --git a/hosted/aks/support_matrix/support_matrix_importing_test.go b/hosted/aks/support_matrix/support_matrix_importing_test.go
@@ -59,10 +59,12 @@ var _ = Describe("SupportMatrixImporting", func() {
 				Expect(err).To(BeNil())
 			})
 			AfterEach(func() {
-				err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
-				Expect(err).To(BeNil())
-				err = helper.DeleteAKSClusteronAzure(clusterName)
-				Expect(err).To(BeNil())
+				if ctx.ClusterCleanup {
+					err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
+					Expect(err).To(BeNil())
+					err = helper.DeleteAKSClusteronAzure(clusterName)
+					Expect(err).To(BeNil())
+				}
 			})
 
 			It("should successfully import the cluster", func() {

diff --git a/hosted/aks/support_matrix/support_matrix_provisioning_test.go b/hosted/aks/support_matrix/support_matrix_provisioning_test.go
@@ -60,10 +60,12 @@ var _ = Describe("SupportMatrixProvisioning", func() {
 				Expect(err).To(BeNil())
 			})
 			AfterEach(func() {
-				err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
-				Expect(err).To(BeNil())
-				err = helper.DeleteAKSClusteronAzure(clusterName)
-				Expect(err).To(BeNil())
+				if ctx.ClusterCleanup {
+					err := helper.DeleteAKSHostCluster(cluster, ctx.RancherClient)
+					Expect(err).To(BeNil())
+					err = helper.DeleteAKSClusteronAzure(clusterName)
+					Expect(err).To(BeNil())
+				}
 			})
 
 			It("should successfully provision the cluster", func() {

diff --git a/hosted/eks/p0/p0_importing_test.go b/hosted/eks/p0/p0_importing_test.go
@@ -54,10 +54,12 @@ var _ = Describe("P0Importing", func() {
 			cluster.EKSConfig = cluster.EKSStatus.UpstreamSpec
 		})
 		AfterEach(func() {
-			err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
-			Expect(err).To(BeNil())
-			err = helper.DeleteEKSClusterOnAWS(region, clusterName)
-			Expect(err).To(BeNil())
+			if ctx.ClusterCleanup {
+				err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
+				Expect(err).To(BeNil())
+				err = helper.DeleteEKSClusterOnAWS(region, clusterName)
+				Expect(err).To(BeNil())
+			}
 			// TODO: Force delete EKS cluster
 		})
 

diff --git a/hosted/eks/p0/p0_provisioning_test.go b/hosted/eks/p0/p0_provisioning_test.go
@@ -49,8 +49,10 @@ var _ = Describe("P0Provisioning", func() {
 		})
 		AfterEach(func() {
 			// TODO Check if EKS cluster deleted on AWS
-			err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
-			Expect(err).To(BeNil())
+			if ctx.ClusterCleanup {
+				err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
+				Expect(err).To(BeNil())
+			}
 		})
 
 		It("should successfully provision the cluster & add, delete, scale nodepool", func() {

diff --git a/hosted/eks/support_matrix/support_matrix_importing_test.go b/hosted/eks/support_matrix/support_matrix_importing_test.go
@@ -59,10 +59,12 @@ var _ = Describe("SupportMatrixImporting", func() {
 				Expect(err).To(BeNil())
 			})
 			AfterEach(func() {
-				err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
-				Expect(err).To(BeNil())
-				err = helper.DeleteEKSClusterOnAWS(region, clusterName)
-				Expect(err).To(BeNil())
+				if ctx.ClusterCleanup {
+					err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
+					Expect(err).To(BeNil())
+					err = helper.DeleteEKSClusterOnAWS(region, clusterName)
+					Expect(err).To(BeNil())
+				}
 			})
 
 			It("should successfully import the cluster", func() {

diff --git a/hosted/eks/support_matrix/support_matrix_provisioning_test.go b/hosted/eks/support_matrix/support_matrix_provisioning_test.go
@@ -57,8 +57,10 @@ var _ = Describe("SupportMatrixProvisioning", func() {
 				Expect(err).To(BeNil())
 			})
 			AfterEach(func() {
-				err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
-				Expect(err).To(BeNil())
+				if ctx.ClusterCleanup {
+					err := helper.DeleteEKSHostCluster(cluster, ctx.RancherClient)
+					Expect(err).To(BeNil())
+				}
 			})
 
 			It("should successfully provision the cluster", func() {

diff --git a/hosted/gke/p0/p0_importing_test.go b/hosted/gke/p0/p0_importing_test.go
@@ -57,10 +57,12 @@ var _ = Describe("P0Importing", func() {
 			cluster.GKEConfig = cluster.GKEStatus.UpstreamSpec
 		})
 		AfterEach(func() {
-			err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
-			Expect(err).To(BeNil())
-			err = helper.DeleteGKEClusterOnGCloud(zone, project, clusterName)
-			Expect(err).To(BeNil())
+			if ctx.ClusterCleanup {
+				err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
+				Expect(err).To(BeNil())
+				err = helper.DeleteGKEClusterOnGCloud(zone, project, clusterName)
+				Expect(err).To(BeNil())
+			}
 		})
 
 		It("should successfully import the cluster & add, delete, scale nodepool", func() {

diff --git a/hosted/gke/p0/p0_provisioning_test.go b/hosted/gke/p0/p0_provisioning_test.go
@@ -54,8 +54,10 @@ var _ = Describe("P0Provisioning", func() {
 			Expect(err).To(BeNil())
 		})
 		AfterEach(func() {
-			err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
-			Expect(err).To(BeNil())
+			if ctx.ClusterCleanup {
+				err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
+				Expect(err).To(BeNil())
+			}
 		})
 		It("should successfully provision the cluster & add, delete, scale nodepool", func() {
 

diff --git a/hosted/gke/support_matrix/support_matrix_importing_test.go b/hosted/gke/support_matrix/support_matrix_importing_test.go
@@ -61,10 +61,12 @@ var _ = Describe("SupportMatrixImporting", func() {
 				Expect(err).To(BeNil())
 			})
 			AfterEach(func() {
-				err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
-				Expect(err).To(BeNil())
-				err = helper.DeleteGKEClusterOnGCloud(zone, project, clusterName)
-				Expect(err).To(BeNil())
+				if ctx.ClusterCleanup {
+					err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
+					Expect(err).To(BeNil())
+					err = helper.DeleteGKEClusterOnGCloud(zone, project, clusterName)
+					Expect(err).To(BeNil())
+				}
 			})
 			It("should successfully import the cluster", func() {
 				By("checking cluster name is same", func() {

diff --git a/hosted/gke/support_matrix/support_matrix_provisioning_test.go b/hosted/gke/support_matrix/support_matrix_provisioning_test.go
@@ -59,8 +59,10 @@ var _ = Describe("SupportMatrixProvisioning", func() {
 				Expect(err).To(BeNil())
 			})
 			AfterEach(func() {
-				err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
-				Expect(err).To(BeNil())
+				if ctx.ClusterCleanup {
+					err := helper.DeleteGKEHostCluster(cluster, ctx.RancherClient)
+					Expect(err).To(BeNil())
+				}
 			})
 			It("should successfully provision the cluster", func() {
 				By("checking cluster name is same", func() {

diff --git a/hosted/helpers/helper_common.go b/hosted/helpers/helper_common.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 	"os/user"
+	"strconv"
 	"strings"
 	"time"
 
@@ -35,17 +36,20 @@ const (
 var (
 	rancherPassword = os.Getenv("RANCHER_PASSWORD")
 	rancherHostname = os.Getenv("RANCHER_HOSTNAME")
+	clusterCleanup  = os.Getenv("DOWNSTREAM_CLUSTER_CLEANUP")
 	cloudCredential *cloudcredentials.CloudCredential
 )
 
 type Context struct {
-	CloudCred     *cloudcredentials.CloudCredential
-	RancherClient *rancher.Client
-	Session       *session.Session
+	CloudCred      *cloudcredentials.CloudCredential
+	RancherClient  *rancher.Client
+	Session        *session.Session
+	ClusterCleanup bool
 }
 
 func CommonBeforeSuite(cloud string) (Context, error) {
 
+	clusterCleanup, _ := strconv.ParseBool(clusterCleanup)
 	rancherConfig := new(rancher.Config)
 	Eventually(rancherConfig, "10s").ShouldNot(BeNil())
 
@@ -102,9 +106,10 @@ func CommonBeforeSuite(cloud string) (Context, error) {
 	}
 
 	return Context{
-		CloudCred:     cloudCredential,
-		RancherClient: rancherClient,
-		Session:       testSession,
+		CloudCred:      cloudCredential,
+		RancherClient:  rancherClient,
+		Session:        testSession,
+		ClusterCleanup: clusterCleanup,
 	}, nil
 }