From aa3026dfcafffd28d456e5ff59de1e80e0205fd8 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Fri, 6 Oct 2023 13:44:01 +0100 Subject: [PATCH 1/8] Test Instascale machine set functionality Co-authored-by: Karel Suta ksuta@redhat.com Co-authored-by: Fiona Waters fwaters@redhat.com --- test/e2e/instascale_machineset_test.go | 54 ++++++++++++++++++++++++++ test/support/client.go | 13 +++++++ test/support/machinesets.go | 36 +++++++++++++++++ 3 files changed, 103 insertions(+) create mode 100644 test/e2e/instascale_machineset_test.go create mode 100644 test/support/machinesets.go diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go new file mode 100644 index 000000000..08805555f --- /dev/null +++ b/test/e2e/instascale_machineset_test.go @@ -0,0 +1,54 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +func TestInstascaleMachineSet(t *testing.T) { + test := With(t) + test.T().Parallel() + + // skip test if not using machine sets + if !MachineSetsExist(test) { + test.T().Skip("Skipping test as machine sets don't exist") + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + // look for machine set with aw name - expect to find it + test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + // look for machine set replica - expect not to find it + test.Expect(MachineExists(test)).Should(BeFalse()) + + // // Setup batch job and AppWrapper + _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + test.Expect(err).NotTo(HaveOccurred()) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + //look for machine set replica - expect to find it + test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue()) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for machine set replica - expect not to find it + test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse()) + +} diff --git a/test/support/client.go b/test/support/client.go index f5696bde8..d911c7c2c 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/clientcmd" imagev1 "github.com/openshift/client-go/image/clientset/versioned" + machinev1 "github.com/openshift/client-go/machine/clientset/versioned" routev1 "github.com/openshift/client-go/route/clientset/versioned" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,6 +34,7 @@ import ( type Client interface { Core() kubernetes.Interface + Machine() machinev1.Interface Route() routev1.Interface Image() imagev1.Interface MCAD() mcadclient.Interface @@ -42,6 +44,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + machine machinev1.Interface route routev1.Interface image imagev1.Interface mcad mcadclient.Interface @@ -55,6 +58,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Machine() machinev1.Interface { + return t.machine +} + func (t *testClient) Route() routev1.Interface { return t.route } @@ -88,6 +95,11 @@ func newTestClient() (Client, error) { return nil, err } + machineClient, err := machinev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + routeClient, err := routev1.NewForConfig(cfg) if err != nil { return nil, err @@ -115,6 +127,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + machine: machineClient, route: routeClient, image: imageClient, mcad: mcadClient, diff --git a/test/support/machinesets.go b/test/support/machinesets.go new file mode 100644 index 000000000..b45455360 --- /dev/null +++ b/test/support/machinesets.go @@ -0,0 +1,36 @@ +package support + +import ( + "github.com/onsi/gomega" + + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + v1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func MachineSets(t Test) ([]v1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func MachineExists(t Test) (foundReplica bool) { + machineSetList, err := MachineSets(t) + t.Expect(err).NotTo(gomega.HaveOccurred()) + for _, ms := range machineSetList { + if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) { + foundReplica = true + } + } + return foundReplica +} + +func MachineSetId(machineSet v1beta1.MachineSet) string { + return machineSet.Name +} + +func MachineSetsExist(t Test) bool { + ms, err := MachineSets(t) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms != nil +} From 77e0f91da41920b107af9bc6882ea3de08e5c480 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Wed, 11 Oct 2023 10:17:53 +0100 Subject: [PATCH 2/8] catching 404 error when machine sets not in use --- test/e2e/instascale_machineset_test.go | 5 ++++- test/support/machinesets.go | 8 ++++++-- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 08805555f..0a1975a48 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + "k8s.io/apimachinery/pkg/api/errors" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -14,7 +16,8 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - if !MachineSetsExist(test) { + ms, err := MachineSetsExist(test) + if !ms || err != nil && errors.IsNotFound(err) { test.T().Skip("Skipping test as machine sets don't exist") } diff --git a/test/support/machinesets.go b/test/support/machinesets.go index b45455360..85259536c 100644 --- a/test/support/machinesets.go +++ b/test/support/machinesets.go @@ -3,6 +3,7 @@ package support import ( "github.com/onsi/gomega" + "k8s.io/apimachinery/pkg/api/errors" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1beta1 "github.com/openshift/api/machine/v1beta1" @@ -29,8 +30,11 @@ func MachineSetId(machineSet v1beta1.MachineSet) string { return machineSet.Name } -func MachineSetsExist(t Test) bool { +func MachineSetsExist(t Test) (bool, error) { ms, err := MachineSets(t) + if err != nil && errors.IsNotFound(err) { + return false, err + } t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil + return ms != nil, err } From c8ec1488f148bc685ba77b1e11a888ba46a3f7ee Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Wed, 11 Oct 2023 17:52:30 +0200 Subject: [PATCH 3/8] Use generic function to retrieve machines --- test/e2e/instascale_machineset_test.go | 16 +++++----- test/support/machine.go | 42 ++++++++++++++++++++++++++ test/support/machinesets.go | 40 ------------------------ 3 files changed, 50 insertions(+), 48 deletions(-) create mode 100644 test/support/machine.go delete mode 100644 test/support/machinesets.go diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 0a1975a48..df79aec2f 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -32,9 +32,9 @@ func TestInstascaleMachineSet(t *testing.T) { }) // look for machine set with aw name - expect to find it - test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) - // look for machine set replica - expect not to find it - test.Expect(MachineExists(test)).Should(BeFalse()) + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + // look for machine belonging to the machine set, there should be none + test.Expect(GetMachines(test, "test-instascale")).Should(BeEmpty()) // // Setup batch job and AppWrapper _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) @@ -44,14 +44,14 @@ func TestInstascaleMachineSet(t *testing.T) { test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) - //look for machine set replica - expect to find it - test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue()) + // look for machine belonging to the machine set - expect to find it + test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(HaveLen(1)) // assert that the AppWrapper goes to "Completed" state - test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort). + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) - // look for machine set replica - expect not to find it - test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse()) + // look for machine belonging to the machine set - there should be none + test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(BeEmpty()) } diff --git a/test/support/machine.go b/test/support/machine.go new file mode 100644 index 000000000..f6be3750d --- /dev/null +++ b/test/support/machine.go @@ -0,0 +1,42 @@ +package support + +import ( + "github.com/onsi/gomega" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine { + return func(g gomega.Gomega) []machinev1beta1.Machine { + machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return machine.Items + } +} + +func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { + t.T().Helper() + return Machines(t, machineSetName)(t) +} + +func MachineSetId(machineSet machinev1beta1.MachineSet) string { + return machineSet.Name +} + +func MachineSetsExist(t Test) (bool, error) { + ms, err := GetMachineSets(t) + if err != nil && errors.IsNotFound(err) { + return false, err + } + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms != nil, err +} diff --git a/test/support/machinesets.go b/test/support/machinesets.go deleted file mode 100644 index 85259536c..000000000 --- a/test/support/machinesets.go +++ /dev/null @@ -1,40 +0,0 @@ -package support - -import ( - "github.com/onsi/gomega" - - "k8s.io/apimachinery/pkg/api/errors" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" - - v1beta1 "github.com/openshift/api/machine/v1beta1" -) - -func MachineSets(t Test) ([]v1beta1.MachineSet, error) { - ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{}) - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms.Items, err -} - -func MachineExists(t Test) (foundReplica bool) { - machineSetList, err := MachineSets(t) - t.Expect(err).NotTo(gomega.HaveOccurred()) - for _, ms := range machineSetList { - if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) { - foundReplica = true - } - } - return foundReplica -} - -func MachineSetId(machineSet v1beta1.MachineSet) string { - return machineSet.Name -} - -func MachineSetsExist(t Test) (bool, error) { - ms, err := MachineSets(t) - if err != nil && errors.IsNotFound(err) { - return false, err - } - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil, err -} From 3e187a32f2076fd2e3c59008534b8ddb5a432356 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 11:41:42 +0100 Subject: [PATCH 4/8] determining cluster type --- test/e2e/instascale_machineset_test.go | 8 +++---- test/support/environment.go | 30 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index df79aec2f..aef1dfe7a 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,8 +6,6 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" - "k8s.io/apimachinery/pkg/api/errors" - . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -16,9 +14,9 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - ms, err := MachineSetsExist(test) - if !ms || err != nil && errors.IsNotFound(err) { - test.T().Skip("Skipping test as machine sets don't exist") + clusterType := DetermineClusterType() + if clusterType != OcpCluster { + test.T().Skip("Skipping test as not running on an OCP cluster") } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index 540749823..2455603af 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -38,6 +38,15 @@ const ( // Cluster ID for OSD cluster used in tests, used for testing InstaScale OsdClusterID = "CLUSTERID" + + // Type of cluster test is run on + ClusterType = "CLUSTER_TYPE" + // OpenShift Dedicated Cluster + OsdCluster = "OSD" + // OpenShift Container Platform Cluster + OcpCluster = "OCP" + // ROSA Hosted Hypershift Cluster + HypershiftCluster = "HYPERSHIFT" ) func GetCodeFlareSDKVersion() string { @@ -65,6 +74,10 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } +func GetClusterType() (string, bool) { + return os.LookupEnv(ClusterType) +} + func IsOsd() bool { osdClusterId, found := GetOsdClusterId() if found && osdClusterId != "" { @@ -79,3 +92,20 @@ func lookupEnvOrDefault(key, value string) string { } return value } + +func DetermineClusterType() string { + clusterType, ok := GetClusterType() + if !ok { + return "" + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + return "" + } +} From da8c04c15ff9b2bb107e9848edd8c24b27f1661f Mon Sep 17 00:00:00 2001 From: Karel Suta Date: Thu, 12 Oct 2023 16:00:50 +0200 Subject: [PATCH 5/8] Use enum for cluster type in e2e tests --- test/e2e/instascale_machineset_test.go | 4 +- test/support/environment.go | 53 +++++++++++++------------- 2 files changed, 29 insertions(+), 28 deletions(-) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index aef1dfe7a..8e23d4e7c 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -14,9 +14,9 @@ func TestInstascaleMachineSet(t *testing.T) { test.T().Parallel() // skip test if not using machine sets - clusterType := DetermineClusterType() + clusterType := GetClusterType(test) if clusterType != OcpCluster { - test.T().Skip("Skipping test as not running on an OCP cluster") + test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index 2455603af..cdafca6e9 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -40,13 +40,16 @@ const ( OsdClusterID = "CLUSTERID" // Type of cluster test is run on - ClusterType = "CLUSTER_TYPE" - // OpenShift Dedicated Cluster - OsdCluster = "OSD" - // OpenShift Container Platform Cluster - OcpCluster = "OCP" - // ROSA Hosted Hypershift Cluster - HypershiftCluster = "HYPERSHIFT" + ClusterTypeEnvVar = "CLUSTER_TYPE" +) + +type ClusterType string + +const ( + OsdCluster ClusterType = "OSD" + OcpCluster ClusterType = "OCP" + HypershiftCluster ClusterType = "HYPERSHIFT" + UndefinedCluster ClusterType = "UNDEFINED" ) func GetCodeFlareSDKVersion() string { @@ -74,8 +77,23 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } -func GetClusterType() (string, bool) { - return os.LookupEnv(ClusterType) +func GetClusterType(t Test) ClusterType { + clusterType, ok := os.LookupEnv(ClusterTypeEnvVar) + if !ok { + t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar) + return UndefinedCluster + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType) + return UndefinedCluster + } } func IsOsd() bool { @@ -92,20 +110,3 @@ func lookupEnvOrDefault(key, value string) string { } return value } - -func DetermineClusterType() string { - clusterType, ok := GetClusterType() - if !ok { - return "" - } - switch clusterType { - case "OSD": - return OsdCluster - case "OCP": - return OcpCluster - case "HYPERSHIFT": - return HypershiftCluster - default: - return "" - } -} From 457c533cf8b8ccb16aef510564d508c4113a5ece Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 15:20:14 +0100 Subject: [PATCH 6/8] updating machinepool test skip --- test/e2e/instascale_machinepool_test.go | 5 +++-- test/support/environment.go | 8 -------- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index b6ad0782c..4c2cbab58 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -29,8 +29,9 @@ func TestInstascaleMachinePool(t *testing.T) { test := With(t) test.T().Parallel() - if !IsOsd() { - test.T().Skip("Skipping test as not running on an OSD cluster") + clusterType := GetClusterType(test) + if clusterType != OsdCluster { + test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() diff --git a/test/support/environment.go b/test/support/environment.go index cdafca6e9..7d8e6bc80 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -96,14 +96,6 @@ func GetClusterType(t Test) ClusterType { } } -func IsOsd() bool { - osdClusterId, found := GetOsdClusterId() - if found && osdClusterId != "" { - return true - } - return false -} - func lookupEnvOrDefault(key, value string) string { if v, ok := os.LookupEnv(key); ok { return v From 9ab7a797d2259f63da4a052c08ee8c0e05e02b70 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Thu, 12 Oct 2023 15:35:01 +0100 Subject: [PATCH 7/8] removing unused function --- test/support/machine.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/test/support/machine.go b/test/support/machine.go index f6be3750d..26acb4bce 100644 --- a/test/support/machine.go +++ b/test/support/machine.go @@ -3,7 +3,6 @@ package support import ( "github.com/onsi/gomega" - "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" machinev1beta1 "github.com/openshift/api/machine/v1beta1" @@ -31,12 +30,3 @@ func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { func MachineSetId(machineSet machinev1beta1.MachineSet) string { return machineSet.Name } - -func MachineSetsExist(t Test) (bool, error) { - ms, err := GetMachineSets(t) - if err != nil && errors.IsNotFound(err) { - return false, err - } - t.Expect(err).NotTo(gomega.HaveOccurred()) - return ms != nil, err -} From ea9ee6abcbbdccd08a72d204b75e7b3bdfd508e4 Mon Sep 17 00:00:00 2001 From: Fiona Waters Date: Mon, 16 Oct 2023 12:53:18 +0100 Subject: [PATCH 8/8] refactor to use aw.Name --- test/e2e/instascale_app_wrapper.go | 8 +++----- test/e2e/instascale_machinepool_test.go | 7 ++++++- test/e2e/instascale_machineset_test.go | 17 +++++++++++------ 3 files changed, 20 insertions(+), 12 deletions(-) diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index f9ec3bd0c..805df1bc3 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -27,7 +27,7 @@ import ( . "github.com/project-codeflare/codeflare-operator/test/support" ) -func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { +func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -89,7 +89,7 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - // create an appwrapper + // AppWrapper aw := &mcadv1beta1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ Name: "test-instascale", @@ -136,7 +136,5 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - - return job, aw, err + return aw } diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index 4c2cbab58..386b224db 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -22,6 +22,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -54,7 +56,10 @@ func TestInstascaleMachinePool(t *testing.T) { ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go index 8e23d4e7c..fad7717b2 100644 --- a/test/e2e/instascale_machineset_test.go +++ b/test/e2e/instascale_machineset_test.go @@ -6,6 +6,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -29,13 +31,16 @@ func TestInstascaleMachineSet(t *testing.T) { "mnist.py": ReadFile(test, "mnist.py"), }) + // // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + // look for machine set with aw name - expect to find it - test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale")))) + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name)))) // look for machine belonging to the machine set, there should be none - test.Expect(GetMachines(test, "test-instascale")).Should(BeEmpty()) + test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty()) - // // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) // assert that AppWrapper goes to "Running" state @@ -43,13 +48,13 @@ func TestInstascaleMachineSet(t *testing.T) { Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) // look for machine belonging to the machine set - expect to find it - test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(HaveLen(1)) + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1)) // assert that the AppWrapper goes to "Completed" state test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) // look for machine belonging to the machine set - there should be none - test.Eventually(Machines(test, "test-instascale"), TestTimeoutLong).Should(BeEmpty()) + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty()) }