Skip to content

Commit

Permalink
Refactor main branch to run tests on multiple Rancher versions (#251)
Browse files Browse the repository at this point in the history 
* Refactor main branch

Signed-off-by: Chandan Pinjani <[email protected]>

* Add SkipUpgradeTestsLog & logging for AfterEach

Signed-off-by: Chandan Pinjani <[email protected]>

---------

Signed-off-by: Chandan Pinjani <[email protected]>
  • Loading branch information
@cpinjani
cpinjani authored Feb 6, 2025
1 parent ec07398 commit a45154a
Show file tree
Hide file tree
Showing 29 changed files with 249 additions and 128 deletions.
49 changes: 19 additions & 30 deletions .github/workflows/main.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -123,6 +123,9 @@ jobs:
env:
# For some reason the go doesn't link by default against system libresolv library
CGO_LDFLAGS: -O2 -g -lresolv
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
INSTALL_K3S_VERSION: ${{ inputs.k3s_version }}
RANCHER_VERSION: ${{ inputs.rancher_version }}
outputs:
PUBLIC_IP: ${{ steps.runner-ip.outputs.PUBLIC_IP }}
steps:
Expand Down Expand Up @@ -213,9 +216,6 @@ jobs:
id: prepare-rancher
if: ${{ inputs.rancher_installed == 'hostname/password' }}
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
INSTALL_K3S_VERSION: ${{ inputs.k3s_version }}
RANCHER_VERSION: ${{ inputs.rancher_version }}
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
PROXY_HOST: ${{ env.PROXY_HOST }}
Expand Down Expand Up @@ -256,7 +256,7 @@ jobs:
rm -rf awscliv2.zip aws/
- name: Provisioning cluster tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'p0_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'p0_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -266,7 +266,7 @@ jobs:
make e2e-provisioning-tests
- name: Import cluster tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'p0_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'p0_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -276,7 +276,7 @@ jobs:
make e2e-import-tests
- name: Provisioning cluster P1 tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'p1_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'p1_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -286,7 +286,7 @@ jobs:
make e2e-p1-provisioning-tests
- name: Import cluster P1 tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'p1_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'p1_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -296,7 +296,7 @@ jobs:
make e2e-p1-import-tests
- name: Support matrix provisioning tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'support_matrix_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'support_matrix_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -306,7 +306,7 @@ jobs:
make e2e-support-matrix-provisioning-tests
- name: Support matrix import tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'support_matrix_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'support_matrix_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -316,29 +316,27 @@ jobs:
make e2e-support-matrix-import-tests
- name: K8s Chart Support provisioning tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'k8s_chart_support_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'k8s_chart_support_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-provisioning.yaml
QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
make e2e-k8s-chart-support-provisioning-tests
- name: K8s Chart Support import tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'k8s_chart_support_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'k8s_chart_support_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-import.yaml
QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
make e2e-k8s-chart-support-import-tests
- name: Sync provisioning tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'sync_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'sync_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -348,7 +346,7 @@ jobs:
make e2e-sync-provisioning-tests
- name: Sync import tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'sync_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'sync_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
Expand All @@ -358,54 +356,46 @@ jobs:
make e2e-sync-import-tests
- name: Backup/Restore provisioning tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'backup_restore_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'backup_restore_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
RANCHER_VERSION: ${{ inputs.rancher_version }}
BACKUP_OPERATOR_VERSION: ${{ inputs.backup_operator_version }}
CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-provisioning.yaml
QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
make e2e-backup-restore-provisioning-tests
- name: Backup/Restore import tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'backup_restore_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'backup_restore_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
RANCHER_VERSION: ${{ inputs.rancher_version }}
BACKUP_OPERATOR_VERSION: ${{ inputs.backup_operator_version }}
CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-import.yaml
QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
run: |
make e2e-backup-restore-import-tests
- name: K8s Chart Support Upgrade provisioning tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'k8s_chart_support_upgrade_provisioning') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'k8s_chart_support_upgrade_provisioning') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-provisioning.yaml
QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
RANCHER_VERSION: ${{ inputs.rancher_version }}
RANCHER_UPGRADE_VERSION: ${{ inputs.rancher_upgrade_version }}
K8S_UPGRADE_MINOR_VERSION: ${{ inputs.k8s_upgrade_minor_version }}
run: |
make e2e-k8s-chart-support-provisioning-tests-upgrade
- name: K8s Chart Support Upgrade import tests
if: ${{ !cancelled() && steps.prepare-rancher.outcome != 'failure' && contains(inputs.tests_to_run, 'k8s_chart_support_upgrade_import') }}
if: ${{ !cancelled() && steps.prepare-rancher.outcome == 'success' && contains(inputs.tests_to_run, 'k8s_chart_support_upgrade_import') }}
env:
RANCHER_HOSTNAME: ${{ env.RANCHER_HOSTNAME }}
RANCHER_PASSWORD: ${{ env.RANCHER_PASSWORD }}
CATTLE_TEST_CONFIG: ${{ github.workspace }}/cattle-config-import.yaml
QASE_RUN_ID: ${{ steps.qase.outputs.qase_run_id }}
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
RANCHER_VERSION: ${{ inputs.rancher_version }}
RANCHER_UPGRADE_VERSION: ${{ inputs.rancher_upgrade_version }}
K8S_UPGRADE_MINOR_VERSION: ${{ inputs.k8s_upgrade_minor_version }}
run: |
Expand Down Expand Up @@ -451,8 +441,7 @@ jobs:
if-no-files-found: ignore

- name: Add summary
env:
KUBECONFIG: /etc/rancher/k3s/k3s.yaml
shell: bash
if: ${{ always() && steps.prepare-rancher.outcome == 'success' }}
run: |
# Add summary
Expand All @@ -462,7 +451,7 @@ jobs:
echo "Tests run: ${{ inputs.tests_to_run }}" >> ${GITHUB_STEP_SUMMARY}
OPERATOR_HELM_VERSION=$(helm get metadata rancher-${{ inputs.hosted_provider }}-operator -n cattle-system -o json | jq -r .version)
echo "Installed rancher-${{ inputs.hosted_provider }}-operator chart version: $OPERATOR_HELM_VERSION" >> ${GITHUB_STEP_SUMMARY}
if [ ${{ inputs.tests_to_run }} == "backup_restore" ]; then
if [[ ${{ inputs.tests_to_run }} =~ "backup_restore" ]]; then
BR_OPERATOR_HELM_VERSION=$(helm get metadata rancher-backup -n cattle-resources-system -o json | jq -r .version)
echo "Installed backup-restore operator chart version: $BR_OPERATOR_HELM_VERSION" >> ${GITHUB_STEP_SUMMARY}
fi
Expand Down
2 changes: 1 addition & 1 deletion hosted/aks/helper/helper_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -448,7 +448,7 @@ func UpdateAutoScaling(cluster *management.Cluster, client *rancher.Client, enab
}
}
return true
}, tools.SetTimeout(7*time.Minute), 5*time.Second).Should(BeTrue())
}, tools.SetTimeout(10*time.Minute), 15*time.Second).Should(BeTrue())
}
return cluster, nil
}
Expand Down
16 changes: 10 additions & 6 deletions hosted/aks/p0/p0_import_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,9 +48,10 @@ var _ = Describe("P0Import", func() {
} {
testData := testData
When("a cluster is imported", func() {
var cluster *management.Cluster

BeforeEach(func() {
if testData.isUpgrade && helpers.SkipUpgradeTests {
Skip("Skipping upgrade tests ...")
}
k8sVersion, err := helper.GetK8sVersion(ctx.RancherAdminClient, ctx.CloudCredID, location, testData.isUpgrade)
Expect(err).NotTo(HaveOccurred())
GinkgoLogr.Info(fmt.Sprintf("Using K8s version %s for cluster %s", k8sVersion, clusterName))
Expand All @@ -65,10 +66,13 @@ var _ = Describe("P0Import", func() {
})

AfterEach(func() {
if ctx.ClusterCleanup && cluster != nil {
err := helper.DeleteAKSHostCluster(cluster, ctx.RancherAdminClient)
Expect(err).To(BeNil())
err = helper.DeleteAKSClusteronAzure(clusterName)
if ctx.ClusterCleanup {
if cluster != nil && cluster.ID != "" {
GinkgoLogr.Info(fmt.Sprintf("Cleaning up resource cluster: %s %s", cluster.Name, cluster.ID))
err := helper.DeleteAKSHostCluster(cluster, ctx.RancherAdminClient)
Expect(err).To(BeNil())
}
err := helper.DeleteAKSClusteronAzure(clusterName)
Expect(err).To(BeNil())
} else {
fmt.Println("Skipping downstream cluster deletion: ", clusterName)
Expand Down
14 changes: 9 additions & 5 deletions hosted/aks/p0/p0_provisioning_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,9 +49,10 @@ var _ = Describe("P0Provisioning", func() {
} {
testData := testData
When("a cluster is created", func() {
var cluster *management.Cluster

BeforeEach(func() {
if testData.isUpgrade && helpers.SkipUpgradeTests {
Skip("Skipping upgrade tests ...")
}
k8sVersion, err := helper.GetK8sVersion(ctx.RancherAdminClient, ctx.CloudCredID, location, testData.isUpgrade)
Expect(err).NotTo(HaveOccurred())
GinkgoLogr.Info(fmt.Sprintf("Using K8s version %s for cluster %s", k8sVersion, clusterName))
Expand All @@ -62,9 +63,12 @@ var _ = Describe("P0Provisioning", func() {
Expect(err).To(BeNil())
})
AfterEach(func() {
if ctx.ClusterCleanup && cluster != nil {
err := helper.DeleteAKSHostCluster(cluster, ctx.RancherAdminClient)
Expect(err).To(BeNil())
if ctx.ClusterCleanup {
if cluster != nil && cluster.ID != "" {
GinkgoLogr.Info(fmt.Sprintf("Cleaning up resource cluster: %s %s", cluster.Name, cluster.ID))
err := helper.DeleteAKSHostCluster(cluster, ctx.RancherAdminClient)
Expect(err).To(BeNil())
}
} else {
fmt.Println("Skipping downstream cluster deletion: ", clusterName)
}
Expand Down
3 changes: 3 additions & 0 deletions hosted/aks/p0/p0_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ const (

var (
ctx helpers.RancherContext
cluster *management.Cluster
clusterName string
testCaseID int64
location = helpers.GetAKSLocation()
Expand All @@ -53,6 +54,8 @@ var _ = SynchronizedBeforeSuite(func() []byte {
})

var _ = BeforeEach(func() {
// Setting this to nil ensures we do not use the `cluster` variable value from another test running in parallel with this one.
cluster = nil
clusterName = namegen.AppendRandomString(helpers.ClusterNamePrefix)
})

Expand Down
28 changes: 22 additions & 6 deletions hosted/aks/p1/p1_import_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -6,17 +6,13 @@ import (

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
management "github.com/rancher/shepherd/clients/rancher/generated/management/v3"

"github.com/rancher/hosted-providers-e2e/hosted/aks/helper"
"github.com/rancher/hosted-providers-e2e/hosted/helpers"
)

var _ = Describe("P1Import", func() {
var (
cluster *management.Cluster
k8sVersion string
)
var k8sVersion string
BeforeEach(func() {
GinkgoLogr.Info(fmt.Sprintf("Running on process: %d", GinkgoParallelProcess()))

Expand Down Expand Up @@ -78,6 +74,9 @@ var _ = Describe("P1Import", func() {
})

It("should be able to update cluster monitoring", func() {
if helpers.SkipUpgradeTests {
Skip(helpers.SkipUpgradeTestsLog)
}
testCaseID = 271
updateMonitoringCheck(cluster, ctx.RancherAdminClient)
})
Expand All @@ -99,7 +98,7 @@ var _ = Describe("P1Import", func() {
// Wait until the cluster no longer exists
_, err := ctx.RancherAdminClient.Management.Cluster.ByID(clusterID)
return err
}, "10s", "1s").ShouldNot(BeNil())
}, "30s", "3s").ShouldNot(BeNil())
cluster, err = helper.ImportAKSHostedCluster(ctx.RancherAdminClient, clusterName, ctx.CloudCredID, location, helpers.GetCommonMetadataLabels())
Expect(err).To(BeNil())
cluster, err = helpers.WaitUntilClusterIsReady(cluster, ctx.RancherAdminClient)
Expand All @@ -108,6 +107,9 @@ var _ = Describe("P1Import", func() {
})

It("should successfully Import a cluster in Region without AZ", func() {
if helpers.SkipUpgradeTests {
Skip(helpers.SkipUpgradeTestsLog)
}
location = "ukwest"
testCaseID = 276

Expand Down Expand Up @@ -142,6 +144,10 @@ var _ = Describe("P1Import", func() {
When("a cluster with custom kubelet and os config is created and imported for upgrade", func() {
var upgradeToVersion string
BeforeEach(func() {
if helpers.SkipUpgradeTests {
Skip(helpers.SkipUpgradeTestsLog)
}

kubeletConfigJsonData := `{"cpuManagerPolicy": "static", "cpuCfsQuota": true, "cpuCfsQuotaPeriod": "200ms", "imageGcHighThreshold": 90, "imageGcLowThreshold": 70, "topologyManagerPolicy": "best-effort", "allowedUnsafeSysctls": ["kernel.msg*","net.*"], "failSwapOn": false}`
kubeletConfigDotJson, err := os.CreateTemp("", "custom-kubelet-*.json")
Expect(err).ToNot(HaveOccurred())
Expand Down Expand Up @@ -201,6 +207,9 @@ var _ = Describe("P1Import", func() {
})

It("should not be able to remove system nodepool", func() {
if helpers.SkipTest {
Skip("Skipping test for v2.8, v2.9 ...")
}
testCaseID = 267
removeSystemNpCheck(cluster, ctx.RancherAdminClient)
})
Expand All @@ -226,6 +235,10 @@ var _ = Describe("P1Import", func() {
When("a cluster is created and imported for upgrade", func() {
var upgradeK8sVersion string
BeforeEach(func() {
if helpers.SkipUpgradeTests {
Skip(helpers.SkipUpgradeTestsLog)
}

var err error
k8sVersion, err = helper.GetK8sVersion(ctx.RancherAdminClient, ctx.CloudCredID, location, true)
Expect(err).NotTo(HaveOccurred())
Expand All @@ -244,6 +257,9 @@ var _ = Describe("P1Import", func() {
})

It("NP cannot be upgraded to k8s version greater than CP k8s version", func() {
if helpers.SkipUpgradeTests {
Skip(helpers.SkipUpgradeTestsLog)
}
testCaseID = 269
npUpgradeToVersionGTCPCheck(cluster, ctx.RancherAdminClient, upgradeK8sVersion)
})
Expand Down
Loading

0 comments on commit a45154a

Please sign in to comment.