Test Instascale machine set functionality

test/e2e/instascale_app_wrapper.go

@@ -27,7 +27,7 @@ import (
 	. "github.com/project-codeflare/codeflare-operator/test/support"
 )
 
-func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) (*batchv1.Job, *mcadv1beta1.AppWrapper, error) {
+func instaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, config *corev1.ConfigMap) *mcadv1beta1.AppWrapper {
 	// Batch Job
 	job := &batchv1.Job{
 		TypeMeta: metav1.TypeMeta{
@@ -89,7 +89,7 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi
 		},
 	}
 
-	// create an appwrapper
+	// AppWrapper
 	aw := &mcadv1beta1.AppWrapper{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      "test-instascale",
@@ -136,7 +136,5 @@ func createInstaScaleJobAppWrapper(test Test, namespace *corev1.Namespace, confi
 		},
 	}
 
-	_, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
-
-	return job, aw, err
+	return aw
 }

test/e2e/instascale_machinepool_test.go

@@ -22,15 +22,18 @@ import (
 	. "github.com/onsi/gomega"
 	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
 
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
 	. "github.com/project-codeflare/codeflare-operator/test/support"
 )
 
 func TestInstascaleMachinePool(t *testing.T) {
 	test := With(t)
 	test.T().Parallel()
 
-	if !IsOsd() {
-		test.T().Skip("Skipping test as not running on an OSD cluster")
+	clusterType := GetClusterType(test)
+	if clusterType != OsdCluster {
+		test.T().Skipf("Skipping test as not running on an OSD cluster, resolved cluster type: %s", clusterType)
 	}
 
 	namespace := test.NewTestNamespace()
@@ -53,7 +56,10 @@ func TestInstascaleMachinePool(t *testing.T) {
 		ShouldNot(ContainElement(WithTransform(MachinePoolId, Equal("test-instascale-g4dn-xlarge"))))
 
 	// Setup batch job and AppWrapper
-	_, aw, err := createInstaScaleJobAppWrapper(test, namespace, cm)
+	aw := instaScaleJobAppWrapper(test, namespace, cm)
+
+	// apply AppWrapper to cluster
+	_, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
 	test.Expect(err).NotTo(HaveOccurred())
 	test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name)
 

test/e2e/instascale_machineset_test.go

@@ -0,0 +1,60 @@
+package e2e
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/project-codeflare/codeflare-operator/test/support"
+)
+
+func TestInstascaleMachineSet(t *testing.T) {
+	test := With(t)
+	test.T().Parallel()
+
+	// skip test if not using machine sets
+	clusterType := GetClusterType(test)
+	if clusterType != OcpCluster {
+		test.T().Skipf("Skipping test as not running on an OCP cluster, resolved cluster type: %s", clusterType)
+	}
+
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	cm := CreateConfigMap(test, namespace.Name, map[string][]byte{
+		// pip requirements
+		"requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"),
+		// MNIST training script
+		"mnist.py": ReadFile(test, "mnist.py"),
+	})
+
+	// // Setup batch job and AppWrapper
+	aw := instaScaleJobAppWrapper(test, namespace, cm)
+
+	// look for machine set with aw name - expect to find it
+	test.Expect(GetMachineSets(test)).Should(ContainElement(WithTransform(MachineSetId, Equal(aw.Name))))
+	// look for machine belonging to the machine set, there should be none
+	test.Expect(GetMachines(test, aw.Name)).Should(BeEmpty())
+
+	// apply AppWrapper to cluster
+	_, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+
+	// assert that AppWrapper goes to "Running" state
+	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning).
+		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
+
+	// look for machine belonging to the machine set - expect to find it
+	test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(HaveLen(1))
+
+	// assert that the AppWrapper goes to "Completed" state
+	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutMedium).
+		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)))
+
+	// look for machine belonging to the machine set - there should be none
+	test.Eventually(Machines(test, aw.Name), TestTimeoutLong).Should(BeEmpty())
+
+}

test/support/client.go

@@ -26,13 +26,15 @@ import (
 	"k8s.io/client-go/tools/clientcmd"
 
 	imagev1 "github.com/openshift/client-go/image/clientset/versioned"
+	machinev1 "github.com/openshift/client-go/machine/clientset/versioned"
 	routev1 "github.com/openshift/client-go/route/clientset/versioned"
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
 )
 
 type Client interface {
 	Core() kubernetes.Interface
+	Machine() machinev1.Interface
 	Route() routev1.Interface
 	Image() imagev1.Interface
 	MCAD() mcadclient.Interface
@@ -42,6 +44,7 @@ type Client interface {
 
 type testClient struct {
 	core    kubernetes.Interface
+	machine machinev1.Interface
 	route   routev1.Interface
 	image   imagev1.Interface
 	mcad    mcadclient.Interface
@@ -55,6 +58,10 @@ func (t *testClient) Core() kubernetes.Interface {
 	return t.core
 }
 
+func (t *testClient) Machine() machinev1.Interface {
+	return t.machine
+}
+
 func (t *testClient) Route() routev1.Interface {
 	return t.route
 }
@@ -88,6 +95,11 @@ func newTestClient() (Client, error) {
 		return nil, err
 	}
 
+	machineClient, err := machinev1.NewForConfig(cfg)
+	if err != nil {
+		return nil, err
+	}
+
 	routeClient, err := routev1.NewForConfig(cfg)
 	if err != nil {
 		return nil, err
@@ -115,6 +127,7 @@ func newTestClient() (Client, error) {
 
 	return &testClient{
 		core:    kubeClient,
+		machine: machineClient,
 		route:   routeClient,
 		image:   imageClient,
 		mcad:    mcadClient,

test/support/environment.go

@@ -38,6 +38,18 @@ const (
 
 	// Cluster ID for OSD cluster used in tests, used for testing InstaScale
 	OsdClusterID = "CLUSTERID"
+
+	// Type of cluster test is run on
+	ClusterTypeEnvVar = "CLUSTER_TYPE"
+)
+
+type ClusterType string
+
+const (
+	OsdCluster        ClusterType = "OSD"
+	OcpCluster        ClusterType = "OCP"
+	HypershiftCluster ClusterType = "HYPERSHIFT"
+	UndefinedCluster  ClusterType = "UNDEFINED"
 )
 
 func GetCodeFlareSDKVersion() string {
@@ -65,12 +77,23 @@ func GetOsdClusterId() (string, bool) {
 	return os.LookupEnv(OsdClusterID)
 }
 
-func IsOsd() bool {
-	osdClusterId, found := GetOsdClusterId()
-	if found && osdClusterId != "" {
-		return true
+func GetClusterType(t Test) ClusterType {
+	clusterType, ok := os.LookupEnv(ClusterTypeEnvVar)
+	if !ok {
+		t.T().Logf("Expected environment variable %s not found, cluster type is not defined.", ClusterTypeEnvVar)
+		return UndefinedCluster
+	}
+	switch clusterType {
+	case "OSD":
+		return OsdCluster
+	case "OCP":
+		return OcpCluster
+	case "HYPERSHIFT":
+		return HypershiftCluster
+	default:
+		t.T().Logf("Expected environment variable %s contains unexpected value: '%s'", ClusterTypeEnvVar, clusterType)
+		return UndefinedCluster
 	}
-	return false
 }
 
 func lookupEnvOrDefault(key, value string) string {

test/support/machine.go

@@ -0,0 +1,32 @@
+package support
+
+import (
+	"github.com/onsi/gomega"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	machinev1beta1 "github.com/openshift/api/machine/v1beta1"
+)
+
+func GetMachineSets(t Test) ([]machinev1beta1.MachineSet, error) {
+	ms, err := t.Client().Machine().MachineV1beta1().MachineSets("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{})
+	t.Expect(err).NotTo(gomega.HaveOccurred())
+	return ms.Items, err
+}
+
+func Machines(t Test, machineSetName string) func(g gomega.Gomega) []machinev1beta1.Machine {
+	return func(g gomega.Gomega) []machinev1beta1.Machine {
+		machine, err := t.Client().Machine().MachineV1beta1().Machines("openshift-machine-api").List(t.Ctx(), metav1.ListOptions{LabelSelector: "machine.openshift.io/cluster-api-machineset=" + machineSetName})
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+		return machine.Items
+	}
+}
+
+func GetMachines(t Test, machineSetName string) []machinev1beta1.Machine {
+	t.T().Helper()
+	return Machines(t, machineSetName)(t)
+}
+
+func MachineSetId(machineSet machinev1beta1.MachineSet) string {
+	return machineSet.Name
+}