diff --git a/test/e2e/instascale_app_wrapper.go b/test/e2e/instascale_app_wrapper.go index f9ec3bd0c..805df1bc3 100644 --- a/test/e2e/instascale_app_wrapper.go +++ b/test/e2e/instascale_app_wrapper.go @@ -27,7 +27,7 @@ import ( . "github.com/project-codeflare/codeflare-operator/test/support" ) -func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) { +func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper { // Batch Job job := &batchv1.Job{ TypeMeta: metav1.TypeMeta{ @@ -89,7 +89,7 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - // create an appwrapper + // AppWrapper aw := &mcadv1beta1.AppWrapper{ ObjectMeta: metav1.ObjectMeta{ Name: "test-instascale", @@ -136,7 +136,5 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi }, } - _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) - - return job, aw, err + return aw } diff --git a/test/e2e/instascale_machinepool_test.go b/test/e2e/instascale_machinepool_test.go index b6ad0782c..386b224db 100644 --- a/test/e2e/instascale_machinepool_test.go +++ b/test/e2e/instascale_machinepool_test.go @@ -22,6 +22,8 @@ import ( . "github.com/onsi/gomega" mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + . "github.com/project-codeflare/codeflare-operator/test/support" ) @@ -29,8 +31,9 @@ func TestInstascaleMachinePool(t *testing.T) { test := With(t) test.T().Parallel() - if !IsOsd() { - test.T().Skip("Skipping test as not running on an OSD cluster") + clusterType := GetClusterType(test) + if clusterType != OsdCluster { + test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType) } namespace := test.NewTestNamespace() @@ -53,7 +56,10 @@ func TestInstascaleMachinePool(t *testing.T) { ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge")))) // Setup batch job and AppWrapper - _, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm) + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) test.Expect(err).NotTo(HaveOccurred()) test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name) diff --git a/test/e2e/instascale_machineset_test.go b/test/e2e/instascale_machineset_test.go new file mode 100644 index 000000000..fad7717b2 --- /dev/null +++ b/test/e2e/instascale_machineset_test.go @@ -0,0 +1,60 @@ +package e2e + +import ( + "testing" + + . "github.com/onsi/gomega" + mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + . "github.com/project-codeflare/codeflare-operator/test/support" +) + +func TestInstascaleMachineSet(t *testing.T) { + test := With(t) + test.T().Parallel() + + // skip test if not using machine sets + clusterType := GetClusterType(test) + if clusterType != OcpCluster { + test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType) + } + + namespace := test.NewTestNamespace() + + // Test configuration + cm := CreateConfigMap(test, namespace.Name, map[string][]byte{ + // pip requirements + "requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"), + // MNIST training script + "mnist.py": ReadFile(test, "mnist.py"), + }) + + // // Setup batch job and AppWrapper + aw := instaScaleJobAppWrapper(test, namespace, cm) + + // look for machine set with aw name - expect to find it + test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name)))) + // look for machine belonging to the machine set, there should be none + test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty()) + + // apply AppWrapper to cluster + _, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{}) + test.Expect(err).NotTo(HaveOccurred()) + + // assert that AppWrapper goes to "Running" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive))) + + // look for machine belonging to the machine set - expect to find it + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1)) + + // assert that the AppWrapper goes to "Completed" state + test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium). + Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted))) + + // look for machine belonging to the machine set - there should be none + test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty()) + +} diff --git a/test/support/client.go b/test/support/client.go index f5696bde8..d911c7c2c 100644 --- a/test/support/client.go +++ b/test/support/client.go @@ -26,6 +26,7 @@ import ( "k8s.io/client-go/tools/clientcmd" imagev1 "github.com/openshift/client-go/image/clientset/versioned" + machinev1 "github.com/openshift/client-go/machine/clientset/versioned" routev1 "github.com/openshift/client-go/route/clientset/versioned" // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // to ensure that exec-entrypoint and run can make use of them. @@ -33,6 +34,7 @@ import ( type Client interface { Core() kubernetes.Interface + Machine() machinev1.Interface Route() routev1.Interface Image() imagev1.Interface MCAD() mcadclient.Interface @@ -42,6 +44,7 @@ type Client interface { type testClient struct { core kubernetes.Interface + machine machinev1.Interface route routev1.Interface image imagev1.Interface mcad mcadclient.Interface @@ -55,6 +58,10 @@ func (t *testClient) Core() kubernetes.Interface { return t.core } +func (t *testClient) Machine() machinev1.Interface { + return t.machine +} + func (t *testClient) Route() routev1.Interface { return t.route } @@ -88,6 +95,11 @@ func newTestClient() (Client, error) { return nil, err } + machineClient, err := machinev1.NewForConfig(cfg) + if err != nil { + return nil, err + } + routeClient, err := routev1.NewForConfig(cfg) if err != nil { return nil, err @@ -115,6 +127,7 @@ func newTestClient() (Client, error) { return &testClient{ core: kubeClient, + machine: machineClient, route: routeClient, image: imageClient, mcad: mcadClient, diff --git a/test/support/environment.go b/test/support/environment.go index 540749823..7d8e6bc80 100644 --- a/test/support/environment.go +++ b/test/support/environment.go @@ -38,6 +38,18 @@ const ( // Cluster ID for OSD cluster used in tests, used for testing InstaScale OsdClusterID = "CLUSTERID" + + // Type of cluster test is run on + ClusterTypeEnvVar = "CLUSTER_TYPE" +) + +type ClusterType string + +const ( + OsdCluster ClusterType = "OSD" + OcpCluster ClusterType = "OCP" + HypershiftCluster ClusterType = "HYPERSHIFT" + UndefinedCluster ClusterType = "UNDEFINED" ) func GetCodeFlareSDKVersion() string { @@ -65,12 +77,23 @@ func GetOsdClusterId() (string, bool) { return os.LookupEnv(OsdClusterID) } -func IsOsd() bool { - osdClusterId, found := GetOsdClusterId() - if found && osdClusterId != "" { - return true +func GetClusterType(t Test) ClusterType { + clusterType, ok := os.LookupEnv(ClusterTypeEnvVar) + if !ok { + t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar) + return UndefinedCluster + } + switch clusterType { + case "OSD": + return OsdCluster + case "OCP": + return OcpCluster + case "HYPERSHIFT": + return HypershiftCluster + default: + t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType) + return UndefinedCluster } - return false } func lookupEnvOrDefault(key, value string) string { diff --git a/test/support/machine.go b/test/support/machine.go new file mode 100644 index 000000000..26acb4bce --- /dev/null +++ b/test/support/machine.go @@ -0,0 +1,32 @@ +package support + +import ( + "github.com/onsi/gomega" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + machinev1beta1 "github.com/openshift/api/machine/v1beta1" +) + +func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) { + ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{}) + t.Expect(err).NotTo(gomega.HaveOccurred()) + return ms.Items, err +} + +func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine { + return func(g gomega.Gomega) []machinev1beta1.Machine { + machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName}) + g.Expect(err).NotTo(gomega.HaveOccurred()) + return machine.Items + } +} + +func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine { + t.T().Helper() + return Machines(t, machineSetName)(t) +} + +func MachineSetId(machineSet machinev1beta1.MachineSet) string { + return machineSet.Name +}