Skip to content

Commit 242cf34

Browse files
Fiona-WatersFiona-Waters
committed
refactored to check for specific machine names
1 parent bed345d commit 242cf34

File tree

3 files changed

+91
-72
lines changed

3 files changed

+91
-72
lines changed

‎test/e2e/instascale_test.go

Lines changed: 57 additions & 61 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package e2e
22

33
import (
4-
"sync"
54
"testing"
65
"time"
76

@@ -18,7 +17,6 @@ var (
1817
machinePoolsExist bool
1918
numInitialNodePools int
2019
numInitialMachineSets int
21-
wg = &sync.WaitGroup{}
2220
)
2321

2422
func TestInstascale(t *testing.T) {
@@ -65,24 +63,27 @@ func TestInstascale(t *testing.T) {
6563
}
6664
defer connection.Close()
6765

68-
machinePoolsExist = true
6966
// check existing cluster resources
70-
numInitialMachinePools, err := MachinePoolsCount(connection)
71-
if err != nil {
72-
test.T().Errorf("Unable to count machine pools - Error : %v", err)
73-
}
67+
machinePoolsExist, err := MachinePoolsExist(connection)
68+
test.Expect(err).NotTo(HaveOccurred())
69+
nodePoolsExist, err := NodePoolsExist(connection)
70+
test.Expect(err).NotTo(HaveOccurred())
7471

75-
if numInitialMachinePools == 0 {
76-
machinePoolsExist = false
77-
numInitialNodePools, err = NodePoolsCount(connection)
72+
if machinePoolsExist {
73+
// look for machine pool with aw name - expect not to find it
74+
foundMachinePool, err := CheckMachinePools(connection, "test-instascale")
75+
test.Expect(err).NotTo(HaveOccurred())
76+
test.Expect(foundMachinePool).To(BeFalse())
77+
} else if nodePoolsExist {
78+
// look for node pool with aw name - expect not to find it
79+
foundNodePool, err := CheckNodePools(connection, "test-instascale")
80+
test.Expect(err).NotTo(HaveOccurred())
81+
test.Expect(foundNodePool).To(BeFalse())
82+
} else {
83+
// TODO update to foundMachineSet
84+
numInitialMachineSets, err = MachineSetsCount()
7885
if err != nil {
79-
test.T().Errorf("Unable to count node pools - Error : %v", err)
80-
}
81-
if numInitialNodePools == 0 {
82-
numInitialMachineSets, err = MachineSetsCount()
83-
if err != nil {
84-
test.T().Errorf("Unable to count machine sets - Error : %v", err)
85-
}
86+
test.T().Errorf("Unable to count machine sets - Error : %v", err)
8687
}
8788
}
8889

@@ -103,13 +104,13 @@ func TestInstascale(t *testing.T) {
103104
Spec: corev1.PodSpec{
104105
Containers: []corev1.Container{
105106
{
106-
Name: "job",
107-
Image: GetPyTorchImage(),
107+
Name: "job",
108+
Image: GetPyTorchImage(),
108109
Env: []corev1.EnvVar{
109110
corev1.EnvVar{Name: "PYTHONUSERBASE", Value: "/test2"},
110111
},
111112
Command: []string{"/bin/sh", "-c", "pip install -r /test/requirements.txt && torchrun /test/mnist.py"},
112-
Args: []string{"$PYTHONUSERBASE"},
113+
Args: []string{"$PYTHONUSERBASE"},
113114
VolumeMounts: []corev1.VolumeMount{
114115
{
115116
Name: "test",
@@ -184,46 +185,44 @@ func TestInstascale(t *testing.T) {
184185
},
185186
},
186187
},
187-
GenericTemplate: Raw(test, job),
188+
GenericTemplate: Raw(test, job),
189+
CompletionStatus: "Complete",
188190
},
189191
},
190192
},
191193
},
192194
}
193195

194-
_, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
196+
_, err = test.Client().MCAD().WorkloadV1beta1().AppWrappers(namespace.Name).Create(test.Ctx(), aw, metav1.CreateOptions{})
195197
test.Expect(err).NotTo(HaveOccurred())
196198
test.T().Logf("AppWrapper created successfully %s/%s", aw.Namespace, aw.Name)
197199

198200
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort).
199201
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateActive)))
200202

201-
// wait for required resources to be created before checking them again
202-
time.Sleep(TestTimeoutShort)
203-
if !machinePoolsExist {
204-
numNodePools, err := NodePoolsCount(connection)
205-
if err != nil {
206-
test.T().Errorf("Unable to count node pools - Error : %v", err)
207-
}
208-
test.Expect(numNodePools).To(BeNumerically(">", numInitialNodePools))
209-
test.T().Logf("number of node pools increased from %d to %d", numInitialNodePools, numNodePools)
203+
// time.Sleep is used twice throughout the test, each for 30 seconds. Can look into using sync package waitGroup instead if that makes more sense
204+
// wait for required resources to scale up before checking them again
205+
time.Sleep(TestTimeoutThirtySeconds)
210206

211-
} else if machinePoolsExist {
212-
numMachinePools, err := MachinePoolsCount(connection)
213-
if err != nil {
214-
test.T().Errorf("Unable to count machine pools - Error : %v", err)
215-
}
216-
test.Expect(numMachinePools).To(BeNumerically(">", numInitialMachinePools))
217-
test.T().Logf("number of machine pools increased from %d to %d", numInitialMachinePools, numMachinePools)
207+
if machinePoolsExist {
208+
// look for machine pool with aw name - expect to find it
209+
foundMachinePool, err := CheckMachinePools(connection, "test-instascale")
210+
test.Expect(err).NotTo(HaveOccurred())
211+
test.Expect(foundMachinePool).To(BeTrue())
212+
} else if nodePoolsExist {
213+
// look for node pool with aw name - expect to find it
214+
foundNodePool, err := CheckNodePools(connection, "test-instascale")
215+
test.Expect(err).NotTo(HaveOccurred())
216+
test.Expect(foundNodePool).To(BeTrue())
218217
} else {
219-
numMachineSets, err := MachineSetsCount()
218+
// TODO update to foundMachineSet
219+
numInitialMachineSets, err = MachineSetsCount()
220220
if err != nil {
221221
test.T().Errorf("Unable to count machine sets - Error : %v", err)
222222
}
223-
test.Expect(numMachineSets).To(BeNumerically(">", numInitialMachineSets))
224-
test.T().Logf("number of machine sets increased from %d to %d", numInitialMachineSets, numMachineSets)
225223
}
226-
224+
225+
// Assert that the job has completed
227226
test.T().Logf("Waiting for Job %s/%s to complete", job.Namespace, job.Name)
228227
test.Eventually(Job(test, job.Namespace, job.Name), TestTimeoutLong).Should(
229228
Or(
@@ -235,30 +234,27 @@ func TestInstascale(t *testing.T) {
235234
test.Expect(GetJob(test, job.Namespace, job.Name)).
236235
To(WithTransform(ConditionStatus(batchv1.JobComplete), Equal(corev1.ConditionTrue)))
237236

238-
// AppWrapper not being updated to complete once job is finished
237+
test.Eventually(AppWrapper(test, namespace, aw.Name), TestTimeoutShort).
238+
Should(WithTransform(AppWrapperState, Equal(mcadv1beta1.AppWrapperStateCompleted)))
239239

240-
time.Sleep(TestTimeoutMedium)
241-
if !machinePoolsExist {
242-
numNodePoolsFinal, err := NodePoolsCount(connection)
243-
if err != nil {
244-
test.T().Errorf("Unable to count node pools - Error : %v", err)
245-
}
246-
test.Expect(numNodePoolsFinal).To(BeNumerically("==", numInitialNodePools))
247-
test.T().Logf("number of machine pools decreased")
240+
// allow time for the resources to scale down before checking them again
241+
time.Sleep(TestTimeoutThirtySeconds)
248242

249-
} else if machinePoolsExist {
250-
numMachinePoolsFinal, err := MachinePoolsCount(connection)
251-
if err != nil {
252-
test.T().Errorf("Unable to count machine pools - Error : %v", err)
253-
}
254-
test.Expect(numMachinePoolsFinal).To(BeNumerically("==", numInitialMachinePools))
255-
test.T().Logf("number of machine pools decreased")
243+
if machinePoolsExist {
244+
// look for machine pool with aw name - expect to find it
245+
foundMachinePool, err := CheckMachinePools(connection, "test-instascale")
246+
test.Expect(err).NotTo(HaveOccurred())
247+
test.Expect(foundMachinePool).To(BeFalse())
248+
} else if nodePoolsExist {
249+
// look for node pool with aw name - expect to find it
250+
foundNodePool, err := CheckNodePools(connection, "test-instascale")
251+
test.Expect(err).NotTo(HaveOccurred())
252+
test.Expect(foundNodePool).To(BeFalse())
256253
} else {
257-
numMachineSetsFinal, err := MachineSetsCount()
254+
// TODO update to foundMachineSet
255+
numInitialMachineSets, err = MachineSetsCount()
258256
if err != nil {
259257
test.T().Errorf("Unable to count machine sets - Error : %v", err)
260258
}
261-
test.Expect(numMachineSetsFinal).To(BeNumerically("==", numInitialMachineSets))
262-
test.T().Logf("number of machine sets decreased")
263259
}
264260
}

‎test/support/clusterpools.go

Lines changed: 33 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -4,15 +4,17 @@ import (
44
"context"
55
"fmt"
66
"os"
7+
"strings"
78

89
ocmsdk "github.com/openshift-online/ocm-sdk-go"
10+
cmv1 "github.com/openshift-online/ocm-sdk-go/clustersmgmt/v1"
911
mapiclientset "github.com/openshift/client-go/machine/clientset/versioned"
1012
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
1113
)
1214

1315
var (
14-
ClusterID string = os.Getenv("CLUSTERID")
15-
machineClient mapiclientset.Interface
16+
ClusterID string = os.Getenv("CLUSTERID")
17+
machineClient mapiclientset.Interface
1618
)
1719

1820
const (
@@ -39,30 +41,50 @@ func CreateOCMConnection(secret string) (*ocmsdk.Connection, error) {
3941
return connection, nil
4042
}
4143

44+
func MachinePoolsExist(connection *ocmsdk.Connection) (bool, error) {
45+
machinePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools()
46+
return machinePools != nil, nil
47+
}
48+
49+
func NodePoolsExist(connection *ocmsdk.Connection) (bool, error) {
50+
nodePools := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools()
51+
return nodePools != nil, nil
52+
}
4253

43-
func MachinePoolsCount(connection *ocmsdk.Connection) (numMachinePools int, err error) {
54+
func CheckMachinePools(connection *ocmsdk.Connection, awName string) (foundMachinePool bool, err error) {
4455
machinePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).MachinePools().List()
4556
machinePoolsListResponse, err := machinePoolsConnection.Send()
4657
if err != nil {
47-
return 0, fmt.Errorf("unable to send request, error: %v", err)
58+
return false, fmt.Errorf("unable to send request, error: %v", err)
4859
}
4960
machinePoolsList := machinePoolsListResponse.Items()
50-
numMachinePools = machinePoolsList.Len()
61+
machinePoolsList.Range(func(index int, item *cmv1.MachinePool) bool {
62+
instanceName, _ := item.GetID()
63+
if strings.Contains(instanceName, awName) {
64+
foundMachinePool = true
65+
}
66+
return true
67+
})
5168

52-
return numMachinePools, nil
69+
return foundMachinePool, err
5370
}
5471

55-
func NodePoolsCount(connection *ocmsdk.Connection) (numNodePools int, err error) {
72+
func CheckNodePools(connection *ocmsdk.Connection, awName string) (foundNodePool bool, err error) {
5673
nodePoolsConnection := connection.ClustersMgmt().V1().Clusters().Cluster(ClusterID).NodePools().List()
5774
nodePoolsListResponse, err := nodePoolsConnection.SendContext(context.Background())
5875
if err != nil {
59-
return 0, fmt.Errorf("unable to send request, error: %v", err)
76+
return false, fmt.Errorf("unable to send request, error: %v", err)
6077
}
6178
nodePoolsList := nodePoolsListResponse.Items()
62-
numNodePools = nodePoolsList.Len()
63-
fmt.Println(numNodePools)
79+
nodePoolsList.Range(func(index int, item *cmv1.NodePool) bool {
80+
instanceName, _ := item.GetID()
81+
if strings.Contains(instanceName, awName) {
82+
foundNodePool = true
83+
}
84+
return true
85+
})
6486

65-
return numNodePools, nil
87+
return foundNodePool, err
6688
}
6789

6890
func MachineSetsCount() (numMachineSets int, err error) {

‎test/support/support.go

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@ import (
2929

3030
var (
3131
TestTimeoutShort = 1 * time.Minute
32+
TestTimeoutThirtySeconds = 30 * time.Second
3233
TestTimeoutMedium = 2 * time.Minute
3334
TestTimeoutLong = 5 * time.Minute
3435

0 commit comments

Comments
 (0)