Skip to content

Commit df77406

Browse files
jentingroboquat
jenting
authored and
roboquat
committed
test: restart control plane components after the volume snapshot is finished
Signed-off-by: JenTing Hsiao <[email protected]>
1 parent e345597 commit df77406

File tree

2 files changed

+139
-16
lines changed

2 files changed

+139
-16
lines changed

‎test/pkg/integration/apis.go

Lines changed: 49 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,10 +34,13 @@ import (
3434
"google.golang.org/grpc/credentials"
3535
"google.golang.org/grpc/credentials/insecure"
3636
"google.golang.org/grpc/status"
37+
appsv1 "k8s.io/api/apps/v1"
3738
corev1 "k8s.io/api/core/v1"
3839
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
3940
"k8s.io/apimachinery/pkg/labels"
41+
"k8s.io/apimachinery/pkg/types"
4042
"sigs.k8s.io/e2e-framework/klient"
43+
"sigs.k8s.io/e2e-framework/klient/k8s"
4144

4245
// Gitpod uses mysql, so it makes sense to make this DB driver available
4346
// by default.
@@ -1153,3 +1156,49 @@ func (c *ComponentAPI) IsPVCExist(pvcName string) bool {
11531156
var pvc corev1.PersistentVolumeClaim
11541157
return c.client.Resources().Get(context.Background(), pvcName, c.namespace, &pvc) == nil
11551158
}
1159+
1160+
// RestartDeployment rollout restart the deployment by updating the
1161+
// spec.template.metadata.annotations["kubectl.kubernetes.io/restartedAt"] = time.Now()
1162+
func (c *ComponentAPI) RestartDeployment(deployName, namespace string, wait bool) error {
1163+
var deploy appsv1.Deployment
1164+
if err := c.client.Resources().WithNamespace(namespace).Get(context.Background(), deployName, namespace, &deploy); err != nil {
1165+
return err
1166+
}
1167+
1168+
patchData := map[string]interface{}{
1169+
"spec": map[string]interface{}{
1170+
"template": map[string]interface{}{
1171+
"metadata": map[string]interface{}{
1172+
"annotations": map[string]interface{}{
1173+
"kubectl.kubernetes.io/restartedAt": time.Now().Format(time.Stamp),
1174+
},
1175+
},
1176+
},
1177+
},
1178+
}
1179+
1180+
encodedPatchData, err := json.Marshal(patchData)
1181+
if err != nil {
1182+
return err
1183+
}
1184+
1185+
if err := c.client.Resources().WithNamespace(namespace).Patch(context.Background(), &deploy, k8s.Patch{PatchType: types.MergePatchType, Data: encodedPatchData}); err != nil {
1186+
return err
1187+
}
1188+
1189+
if !wait {
1190+
return nil
1191+
}
1192+
1193+
// waits for the deployment rollout status, maximum to one minute
1194+
for i := 0; i < 10; i++ {
1195+
if err := c.client.Resources().WithNamespace(namespace).Get(context.Background(), deployName, namespace, &deploy); err != nil {
1196+
return err
1197+
}
1198+
if deploy.Status.UnavailableReplicas == 0 {
1199+
break
1200+
}
1201+
time.Sleep(6 * time.Second)
1202+
}
1203+
return nil
1204+
}

‎test/tests/components/ws-manager/prebuild_test.go

Lines changed: 90 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -201,6 +201,13 @@ const (
201201
// - make sure the regular workspace PVC object should exist or not
202202
// - make sure the file foobar.txt exists
203203
func TestOpenWorkspaceFromPrebuild(t *testing.T) {
204+
var (
205+
snapshotcontrollerDeployment string = "snapshot-controller"
206+
snapshotcontrollerNamespace string = "kube-system"
207+
wsmanagerDeployment string = "ws-manager"
208+
wsmanagerNamespace string = "default"
209+
)
210+
204211
f := features.New("prebuild").
205212
WithLabel("component", "ws-manager").
206213
Assess("it should open workspace from prebuild successfully", func(_ context.Context, t *testing.T, cfg *envconf.Config) context.Context {
@@ -231,6 +238,8 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
231238

232239
for _, test := range tests {
233240
t.Run(test.Name, func(t *testing.T) {
241+
isPVCEnable := reflect.DeepEqual(test.FF, []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM})
242+
234243
api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client())
235244
t.Cleanup(func() {
236245
api.Done(t)
@@ -268,8 +277,28 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
268277
}
269278

270279
t.Logf("prebuild snapshot: %s", prebuildSnapshot)
271-
if vsInfo != nil {
272-
t.Logf("vsName: %s, vsHandle: %s", vsInfo.VolumeSnapshotName, vsInfo.VolumeSnapshotHandle)
280+
checkSnapshot(t, vsInfo, isPVCEnable)
281+
282+
// restart the ws-manager and volume snapshot after the volume snapshot is finished
283+
if isPVCEnable {
284+
t.Logf("rollout restart deployment %s/%s after the volume snapshot is finished", wsmanagerDeployment, wsmanagerNamespace)
285+
if err := api.RestartDeployment(wsmanagerDeployment, wsmanagerNamespace, true); err != nil {
286+
t.Errorf("cannot restart deployment %s/%s", wsmanagerDeployment, wsmanagerNamespace)
287+
}
288+
t.Logf("rollout restart deployment %s/%s is finished", wsmanagerDeployment, wsmanagerNamespace)
289+
290+
t.Logf("rollout restart deployment %s/%s after the volume snapshot is finished", snapshotcontrollerDeployment, snapshotcontrollerNamespace)
291+
if err := api.RestartDeployment(snapshotcontrollerDeployment, snapshotcontrollerNamespace, true); err != nil {
292+
t.Errorf("cannot restart deployment %s/%s", snapshotcontrollerDeployment, snapshotcontrollerNamespace)
293+
}
294+
t.Logf("rollout restart deployment %s/%s is finished", snapshotcontrollerDeployment, snapshotcontrollerNamespace)
295+
296+
// recreate a new API because the ws-manager restarted
297+
api1 := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client())
298+
api = api1
299+
t.Cleanup(func() {
300+
api1.Done(t)
301+
})
273302
}
274303

275304
// launch the workspace from prebuild
@@ -282,7 +311,7 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
282311
Prebuild: &csapi.PrebuildInitializer{
283312
Prebuild: &csapi.SnapshotInitializer{
284313
Snapshot: prebuildSnapshot,
285-
FromVolumeSnapshot: reflect.DeepEqual(test.FF, []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM}),
314+
FromVolumeSnapshot: isPVCEnable,
286315
},
287316
Git: []*csapi.GitInitializer{
288317
{
@@ -303,7 +332,7 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
303332
}
304333

305334
// check the PVC object should exist or not
306-
checkPVCObject(t, api, test.FF, regularPrefix+ws.Req.Id)
335+
checkPVCObject(t, api, isPVCEnable, regularPrefix+ws.Req.Id)
307336

308337
defer func() {
309338
// stop workspace in defer function to prevent we forget to stop the workspace
@@ -352,12 +381,32 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
352381
t.Fatal(err)
353382
}
354383

355-
if vsInfo != nil {
356-
t.Logf("vsName: %s, vsHandle: %s", vsInfo.VolumeSnapshotName, vsInfo.VolumeSnapshotHandle)
384+
checkSnapshot(t, vsInfo, isPVCEnable)
385+
386+
// restart the ws-manager and volume snapshot after the volume snapshot is finished
387+
if isPVCEnable {
388+
t.Logf("rollout restart deployment %s/%s after the volume snapshot is finished", wsmanagerDeployment, wsmanagerNamespace)
389+
if err := api.RestartDeployment(wsmanagerDeployment, wsmanagerNamespace, true); err != nil {
390+
t.Errorf("cannot restart deployment %s/%s", wsmanagerDeployment, wsmanagerNamespace)
391+
}
392+
t.Logf("rollout restart deployment %s/%s is finished", wsmanagerDeployment, wsmanagerNamespace)
393+
394+
t.Logf("rollout restart deployment %s/%s after the volume snapshot is finished", snapshotcontrollerDeployment, snapshotcontrollerNamespace)
395+
if err := api.RestartDeployment(snapshotcontrollerDeployment, snapshotcontrollerNamespace, true); err != nil {
396+
t.Errorf("cannot restart deployment %s/%s", snapshotcontrollerDeployment, snapshotcontrollerNamespace)
397+
}
398+
t.Logf("rollout restart deployment %s/%s is finished", snapshotcontrollerDeployment, snapshotcontrollerNamespace)
399+
400+
// recreate a new API because the ws-manager restarted
401+
sapi1 := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client())
402+
sapi = sapi1
403+
t.Cleanup(func() {
404+
sapi.Done(t)
405+
})
357406
}
358407

359408
// reopen the workspace and make sure the file foobar.txt exists
360-
ws1, stopWs1, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
409+
ws1, stopWs1, err := integration.LaunchWorkspaceDirectly(t, ctx, sapi, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
361410
req.ServicePrefix = ws.Req.ServicePrefix
362411
req.Metadata.MetaId = ws.Req.Metadata.MetaId
363412
req.Metadata.Owner = ws.Req.Metadata.Owner
@@ -366,7 +415,7 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
366415
Spec: &csapi.WorkspaceInitializer_Backup{
367416
Backup: &csapi.FromBackupInitializer{
368417
CheckoutLocation: test.CheckoutLocation,
369-
FromVolumeSnapshot: reflect.DeepEqual(test.FF, []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM}),
418+
FromVolumeSnapshot: isPVCEnable,
370419
},
371420
},
372421
}
@@ -379,7 +428,7 @@ func TestOpenWorkspaceFromPrebuild(t *testing.T) {
379428
}
380429

381430
// check the PVC object should exist or not
382-
checkPVCObject(t, api, test.FF, regularPrefix+ws1.Req.Id)
431+
checkPVCObject(t, sapi, isPVCEnable, regularPrefix+ws1.Req.Id)
383432

384433
defer func() {
385434
// stop workspace in defer function to prevent we forget to stop the workspace
@@ -447,6 +496,7 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
447496
ContextURL string
448497
WorkspaceRoot string
449498
CheckoutLocation string
499+
FF []wsmanapi.WorkspaceFeatureFlag
450500
ShouldFailToOpenRegularWorkspace bool
451501
}{
452502
{
@@ -458,6 +508,7 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
458508
ContextURL: "https://github.com/gitpod-io/empty",
459509
CheckoutLocation: "empty",
460510
WorkspaceRoot: "/workspace/empty",
511+
FF: []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM},
461512
},
462513
{
463514
Name: "prebuild-large-regular-small-workspace-class",
@@ -468,6 +519,7 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
468519
ContextURL: "https://github.com/gitpod-io/empty",
469520
CheckoutLocation: "empty",
470521
WorkspaceRoot: "/workspace/empty",
522+
FF: []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM},
471523
ShouldFailToOpenRegularWorkspace: true,
472524
},
473525
}
@@ -477,6 +529,8 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
477529

478530
for _, test := range tests {
479531
t.Run(test.Name, func(t *testing.T) {
532+
isPVCEnable := reflect.DeepEqual(test.FF, []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM})
533+
480534
api := integration.NewComponentAPI(ctx, cfg.Namespace(), kubeconfig, cfg.Client())
481535
t.Cleanup(func() {
482536
api.Done(t)
@@ -490,7 +544,7 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
490544
Name: "GITPOD_TASKS",
491545
Value: fmt.Sprintf(`[{ "init": %q }]`, initTask),
492546
})
493-
req.Spec.FeatureFlags = []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM}
547+
req.Spec.FeatureFlags = test.FF
494548
req.Spec.Initializer = &csapi.WorkspaceInitializer{
495549
Spec: &csapi.WorkspaceInitializer_Git{
496550
Git: &csapi.GitInitializer{
@@ -513,14 +567,12 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
513567
}
514568

515569
t.Logf("prebuild snapshot: %s", prebuildSnapshot)
516-
if vsInfo != nil {
517-
t.Logf("vsName: %s, vsHandle: %s", vsInfo.VolumeSnapshotName, vsInfo.VolumeSnapshotHandle)
518-
}
570+
checkSnapshot(t, vsInfo, isPVCEnable)
519571

520572
// launch the workspace from prebuild
521573
ws, stopWs, err := integration.LaunchWorkspaceDirectly(t, ctx, api, integration.WithRequestModifier(func(req *wsmanapi.StartWorkspaceRequest) error {
522574
req.Spec.Class = test.RegularWorkspaceClass
523-
req.Spec.FeatureFlags = []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM}
575+
req.Spec.FeatureFlags = test.FF
524576
req.Spec.Initializer = &csapi.WorkspaceInitializer{
525577
Spec: &csapi.WorkspaceInitializer_Prebuild{
526578
Prebuild: &csapi.PrebuildInitializer{
@@ -585,9 +637,31 @@ func TestPrebuildAndRegularWorkspaceDifferentWorkspaceClass(t *testing.T) {
585637
testEnv.Test(t, f)
586638
}
587639

640+
// checkSnapshot checks the volume snapshot information is valid or not
641+
func checkSnapshot(t *testing.T, vsInfo *wsmanapi.VolumeSnapshotInfo, isPVCEnable bool) {
642+
if !isPVCEnable {
643+
if vsInfo == nil {
644+
return
645+
}
646+
if vsInfo.VolumeSnapshotName == "" && vsInfo.VolumeSnapshotHandle == "" {
647+
return
648+
}
649+
t.Fatalf("it should not contain volume snapshot name %s and volume snapshot handle %s without PVC", vsInfo.VolumeSnapshotName, vsInfo.VolumeSnapshotHandle)
650+
} else {
651+
if vsInfo == nil {
652+
t.Fatal("it should contain volume snapshot info with PVC")
653+
}
654+
if vsInfo.VolumeSnapshotName != "" && vsInfo.VolumeSnapshotHandle != "" {
655+
t.Logf("vsName: %s, vsHandle: %s", vsInfo.VolumeSnapshotName, vsInfo.VolumeSnapshotHandle)
656+
return
657+
}
658+
t.Fatalf("it should contain volume snapshot name %s and volume snapshot handle %s without PVC", vsInfo.VolumeSnapshotName, vsInfo.VolumeSnapshotHandle)
659+
}
660+
}
661+
588662
// checkPVCObject checks the PVC object should exist or not
589-
func checkPVCObject(t *testing.T, api *integration.ComponentAPI, ff []wsmanapi.WorkspaceFeatureFlag, pvcName string) {
590-
if reflect.DeepEqual(ff, []wsmanapi.WorkspaceFeatureFlag{wsmanapi.WorkspaceFeatureFlag_PERSISTENT_VOLUME_CLAIM}) {
663+
func checkPVCObject(t *testing.T, api *integration.ComponentAPI, isPVCEnable bool, pvcName string) {
664+
if isPVCEnable {
591665
// check the PVC object exist
592666
if !api.IsPVCExist(pvcName) {
593667
t.Fatal("prebuild PVC object should exist")

0 commit comments

Comments
 (0)