instascale hypershift/nodepool e2e test

Author: Fiona-Waters

test/e2e/instascale_nodepool_test.go

@@ -0,0 +1,67 @@
+package e2e
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+	mcadv1beta1 "github.com/project-codeflare/multi-cluster-app-dispatcher/pkg/apis/controller/v1beta1"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	. "github.com/project-codeflare/codeflare-operator/test/support"
+)
+
+func TestInstascaleNodepool(t *testing.T) {
+
+	test := With(t)
+	test.T().Parallel()
+
+	clusterType := GetClusterType(test)
+	if clusterType != HypershiftCluster {
+		test.T().Skipf("Skipping test as not running on an Hypershift cluster, resolved cluster type: %s", clusterType)
+	}
+
+	namespace := test.NewTestNamespace()
+
+	// Test configuration
+	cm := CreateConfigMap(test, namespace.Name, map[string][]byte{
+		// pip requirements
+		"requirements.txt": ReadFile(test, "mnist_pip_requirements.txt"),
+		// MNIST training script
+		"mnist.py": ReadFile(test, "mnist.py"),
+	})
+
+	//create OCM connection
+	connection := CreateOCMConnection(test)
+	defer connection.Close()
+
+	// check existing cluster resources
+	// look for node pool with aw name - expect not to find it
+	test.Expect(GetNodePools(test, connection)).
+		ShouldNot(ContainElement(WithTransform(NodePoolId, Equal("test-instascale-g4dn-xlarge"))))
+
+	// Setup batch job and AppWrapper
+	aw := instaScaleJobAppWrapper(test, namespace, cm)
+
+	// apply AppWrapper to cluster
+	_, err := test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
+	test.Expect(err).NotTo(HaveOccurred())
+	test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name)
+
+	// assert that AppWrapper goes to "Running" state
+	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutGpuProvisioning).
+		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
+
+	// look for node pool with aw name - expect to find it
+	test.Eventually(NodePools(test, connection), TestTimeoutLong).
+		Should(ContainElement(WithTransform(NodePoolId, Equal("test-instascale-g4dn-xlarge"))))
+
+	// assert that the AppWrapper goes to "Completed" state
+	test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort).
+		Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)))
+
+	// look for node pool with aw name - expect not to find it
+	test.Eventually(NodePools(test, connection), TestTimeoutLong).
+		ShouldNot(ContainElement(WithTransform(NodePoolId, Equal("test-instascale-g4dn-xlarge"))))
+
+}

test/support/environment.go

@@ -36,8 +36,8 @@ const (
 	// The namespace where a secret containing InstaScale OCM token is stored and the secret name.
 	InstaScaleOcmSecret = "INSTASCALE_OCM_SECRET"
 
-	// Cluster ID for OSD cluster used in tests, used for testing InstaScale
-	OsdClusterID = "CLUSTERID"
+	// Cluster ID for OSD and OCP clusters used in tests, used for testing InstaScale
+	ClusterID = "CLUSTERID"
 
 	// Type of cluster test is run on
 	ClusterTypeEnvVar = "CLUSTER_TYPE"
@@ -73,8 +73,8 @@ func GetInstascaleOcmSecret() (string, string) {
 	return res[0], res[1]
 }
 
-func GetOsdClusterId() (string, bool) {
-	return os.LookupEnv(OsdClusterID)
+func GetClusterId() (string, bool) {
+	return os.LookupEnv(ClusterID)
 }
 
 func GetClusterType(t Test) ClusterType {

test/support/ocm.go

@@ -54,11 +54,11 @@ func buildOCMConnection(secret string) (*ocmsdk.Connection, error) {
 }
 
 func MachinePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.MachinePool {
-	osdClusterId, found := GetOsdClusterId()
+	clusterId, found := GetClusterId()
 	t.Expect(found).To(gomega.BeTrue(), "OSD cluster id not found, please configure environment properly")
 
 	return func(g gomega.Gomega) []*cmv1.MachinePool {
-		machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(osdClusterId).MachinePools().List().Send()
+		machinePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(clusterId).MachinePools().List().Send()
 		g.Expect(err).NotTo(gomega.HaveOccurred())
 		return machinePoolsListResponse.Items().Slice()
 	}
@@ -72,3 +72,23 @@ func GetMachinePools(t Test, connection *ocmsdk.Connection) []*cmv1.MachinePool
 func MachinePoolId(machinePool *cmv1.MachinePool) string {
 	return machinePool.ID()
 }
+
+func NodePools(t Test, connection *ocmsdk.Connection) func(g gomega.Gomega) []*cmv1.NodePool {
+	clusterId, found := GetClusterId()
+	t.Expect(found).To(gomega.BeTrue(), "Cluster id not found, please configure environment properly")
+
+	return func(g gomega.Gomega) []*cmv1.NodePool {
+		nodePoolsListResponse, err := connection.ClustersMgmt().V1().Clusters().Cluster(clusterId).NodePools().List().Send()
+		g.Expect(err).NotTo(gomega.HaveOccurred())
+		return nodePoolsListResponse.Items().Slice()
+	}
+}
+
+func GetNodePools(t Test, connection *ocmsdk.Connection) []*cmv1.NodePool {
+	t.T().Helper()
+	return NodePools(t, connection)(t)
+}
+
+func NodePoolId(nodePool *cmv1.NodePool) string {
+	return nodePool.ID()
+}