Test Instascale machine set functionality

Co-authored-by: Karel Suta [email protected]
Co-authored-by: Fiona Waters [email protected]

Author: Fiona-Waters

test/e2e/instascale_machineset_test.go

@@ -0,0 +1,54 @@
+package e2e
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+
+	. "github.com/project-codeflare/codeflare-operator/test/support"
+)
+
+func TestInstascaleMachineSet(t *testing.T) {
+	test := With(t)
+	test.T().Parallel()
+
+	// skip test if not using machine sets
+	if !MachineSetsExist(test) {
+		test.T().Skip("Skipping test as machine sets don't exist")
+	}
+
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	cm := CreateConfigMap(test, namespace.Name, map[string][]byte{
+		// pip requirements
+		"requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"),
+		// MNIST training script
+		"mnist.py": ReadFile(test, "mnist.py"),
+	})
+
+	// look for machine set with aw name - expect to find it
+	test.Expect(MachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal("test-instascale"))))
+	// look for machine set replica - expect not to find it
+	test.Expect(MachineExists(test)).Should(BeFalse())
+
+	// // Setup batch job and AppWrapper
+	_, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm)
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// assert that AppWrapper goes to "Running" state
+	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning).
+		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
+
+	//look for machine set replica - expect to find it
+	test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeTrue())
+
+	// assert that the AppWrapper goes to "Completed" state
+	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort).
+		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)))
+
+	// look for machine set replica - expect not to find it
+	test.Eventually(MachineExists(test), TestTimeoutLong).Should(BeFalse())
+
+}

test/support/client.go

@@ -26,13 +26,15 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 
 	imagev1 "github.com/openshift/client-go/image/clientset/versioned"
+	machinev1 "github.com/openshift/client-go/machine/clientset/versioned"
 	routev1 "github.com/openshift/client-go/route/clientset/versioned"
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 )
 
 type Client interface {
 	Core() kubernetes.Interface
+	Machine() machinev1.Interface
 	Route() routev1.Interface
 	Image() imagev1.Interface
 	MCAD() mcadclient.Interface
@@ -42,6 +44,7 @@ type Client interface {
 
 type testClient struct {
 	core    kubernetes.Interface
+	machine machinev1.Interface
 	route   routev1.Interface
 	image   imagev1.Interface
 	mcad    mcadclient.Interface
@@ -55,6 +58,10 @@ func (t *testClient) Core() kubernetes.Interface {
 	return t.core
 }
 
+func (t *testClient) Machine() machinev1.Interface {
+	return t.machine
+}
+
 func (t *testClient) Route() routev1.Interface {
 	return t.route
 }
@@ -88,6 +95,11 @@ func newTestClient() (Client, error) {
 		return nil, err
 	}
 
+	machineClient, err := machinev1.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
 	routeClient, err := routev1.NewForConfig(cfg)
 	if err != nil {
 		return nil, err
@@ -115,6 +127,7 @@ func newTestClient() (Client, error) {
 
 	return &testClient{
 		core:    kubeClient,
+		machine: machineClient,
 		route:   routeClient,
 		image:   imageClient,
 		mcad:    mcadClient,

test/support/machinesets.go

@@ -0,0 +1,36 @@
+package support
+
+import (
+	"github.com/onsi/gomega"
+
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	v1beta1 "github.com/openshift/api/machine/v1beta1"
+)
+
+func MachineSets(t Test) ([]v1beta1.MachineSet, error) {
+	ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), v1.ListOptions{})
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+	return ms.Items, err
+}
+
+func MachineExists(t Test) (foundReplica bool) {
+	machineSetList, err := MachineSets(t)
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+	for _, ms := range machineSetList {
+		if ms.Name == "test-instascale" && &ms.Status.AvailableReplicas == Ptr(int32(1)) {
+			foundReplica = true
+		}
+	}
+	return foundReplica
+}
+
+func MachineSetId(machineSet v1beta1.MachineSet) string {
+	return machineSet.Name
+}
+
+func MachineSetsExist(t Test) bool {
+	ms, err := MachineSets(t)
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+	return ms != nil
+}