cmd/runqemubuildlet: restart unresponsive qemu processes

Expose the healthz port from the buildlet running under QEMU, and
periodically check it for a successful response. If it has been failing
for longer than ten minutes, try to restart the VM. This should
successfully restart VMs that failed to boot, failed to shut down, or
are otherwise unresponsive.

For golang/go#47018

Change-Id: I9218f94ee24de6e0a56ad60a18e075ce48893938
Reviewed-on: https://go-review.googlesource.com/c/build/+/336109
Trust: Alexander Rakoczy <[email protected]>
Run-TryBot: Alexander Rakoczy <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
Reviewed-by: Carlos Amedee <[email protected]>

Author: toothrot

cmd/runqemubuildlet/heartbeat.go

@@ -0,0 +1,70 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.16
+// +build go1.16
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/http"
+	"time"
+
+	"golang.org/x/build/internal"
+)
+
+// buildletHealthTimeout is the maximum time to wait for a
+// checkBuildletHealth request to complete.
+const buildletHealthTimeout = 10 * time.Second
+
+// checkBuildletHealth performs a GET request against URL, and returns
+// an error if an http.StatusOK isn't returned before
+// buildletHealthTimeout has elapsed.
+func checkBuildletHealth(ctx context.Context, url string) error {
+	ctx, cancel := context.WithTimeout(ctx, buildletHealthTimeout)
+	defer cancel()
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
+	if err != nil {
+		return err
+	}
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		return err
+	}
+	defer resp.Body.Close()
+	if _, err := io.Copy(io.Discard, resp.Body); err != nil {
+		return err
+	}
+	if resp.StatusCode != http.StatusOK {
+		return fmt.Errorf("resp.StatusCode = %d, wanted %d", resp.StatusCode, http.StatusOK)
+	}
+	return nil
+}
+
+// heartbeatContext calls f every period. If f consistently returns an
+// error for longer than the provided timeout duration, the context
+// returned by heartbeatContext will be cancelled, and
+// heartbeatContext will stop sending requests.
+//
+// A single call to f that does not return an error will reset the
+// timeout window, unless heartbeatContext has already timed out.
+func heartbeatContext(ctx context.Context, period time.Duration, timeout time.Duration, f func(context.Context) error) (context.Context, func()) {
+	ctx, cancel := context.WithCancel(ctx)
+
+	lastSuccess := time.Now()
+	go internal.PeriodicallyDo(ctx, period, func(ctx context.Context, t time.Time) {
+		err := f(ctx)
+		if err != nil && t.Sub(lastSuccess) > timeout {
+			cancel()
+		}
+		if err == nil {
+			lastSuccess = t
+		}
+	})
+
+	return ctx, cancel
+}

cmd/runqemubuildlet/heartbeat_test.go

@@ -0,0 +1,94 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+//go:build go1.16
+// +build go1.16
+
+package main
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+	"time"
+)
+
+func TestCheckBuildletHealth(t *testing.T) {
+	cases := []struct {
+		desc     string
+		respCode int
+		wantErr  bool
+	}{
+		{
+			desc:     "success",
+			respCode: http.StatusOK,
+		},
+		{
+			desc:     "failure",
+			respCode: http.StatusBadGateway,
+			wantErr:  true,
+		},
+	}
+	for _, c := range cases {
+		t.Run(c.desc, func(t *testing.T) {
+			m := http.NewServeMux()
+			m.HandleFunc("/healthz", func(w http.ResponseWriter, req *http.Request) {
+				w.WriteHeader(c.respCode)
+				fmt.Sprintln(w, "ok")
+			})
+			s := httptest.NewServer(m)
+			defer s.Close()
+			u, err := url.Parse(s.URL)
+			if err != nil {
+				t.Fatalf("url.Parse(%q) = %v, wanted no error", s.URL, err)
+			}
+			u.Path = "/healthz"
+
+			if err := checkBuildletHealth(context.Background(), u.String()); (err != nil) != c.wantErr {
+				t.Errorf("checkBuildletHealth(_, %q) = %v, wantErr: %t", s.URL, err, c.wantErr)
+			}
+		})
+	}
+}
+
+func TestHeartbeatContext(t *testing.T) {
+	ctx := context.Background()
+
+	didWork := make(chan interface{}, 2)
+	done := make(chan interface{})
+	ctx, cancel := heartbeatContext(ctx, time.Millisecond, 100*time.Millisecond, func(context.Context) error {
+		select {
+		case <-done:
+			return errors.New("heartbeat stopped")
+		case didWork <- nil:
+		default:
+		}
+		return nil
+	})
+	defer cancel()
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Errorf("heatbeatContext() never called f, wanted at least one call")
+	case <-didWork:
+	}
+
+	select {
+	case <-done:
+		t.Errorf("heartbeatContext() finished early, wanted it to still be testing")
+	case <-didWork:
+		close(done)
+	}
+
+	select {
+	case <-time.After(5 * time.Second):
+		t.Errorf("heartbeatContext() did not timeout, wanted timeout after failing over %v", time.Second)
+	case <-ctx.Done():
+		// heartbeatContext() successfully timed out after failing
+	}
+}

cmd/runqemubuildlet/main.go

@@ -24,6 +24,7 @@ import (
 
 var (
 	windows10Path = flag.String("windows-10-path", defaultWindowsDir(), "Path to Windows image and QEMU dependencies.")
+	healthzURL    = flag.String("buildlet-healthz-url", "http://localhost:8080/healthz", "URL to buildlet /healthz endpoint.")
 )
 
 func main() {
@@ -33,22 +34,32 @@ func main() {
 	defer stop()
 
 	for ctx.Err() == nil {
-		cmd := windows10Cmd(*windows10Path)
-		log.Printf("Starting VM: %s", cmd.String())
-		cmd.Stdout = os.Stdout
-		cmd.Stderr = os.Stderr
-		if err := cmd.Start(); err != nil {
-			log.Printf("cmd.Start() = %v. Retrying in 10 seconds.", err)
+		if err := runWindows10(ctx); err != nil {
+			log.Printf("runWindows10() = %v. Retrying in 10 seconds.", err)
 			time.Sleep(10 * time.Second)
 			continue
 		}
-		if err := internal.WaitOrStop(ctx, cmd, os.Interrupt, time.Minute); err != nil {
-			log.Printf("waitOrStop(_, %v, %v, %v) = %v. Retrying in 10 seconds.", cmd, os.Interrupt, time.Minute, err)
-			time.Sleep(10 * time.Second)
-		}
 	}
 }
 
+func runWindows10(ctx context.Context) error {
+	cmd := windows10Cmd(*windows10Path)
+	log.Printf("Starting VM: %s", cmd.String())
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Start(); err != nil {
+		return fmt.Errorf("cmd.Start() = %w", err)
+	}
+	ctx, cancel := heartbeatContext(ctx, 30*time.Second, 10*time.Minute, func(ctx context.Context) error {
+		return checkBuildletHealth(ctx, *healthzURL)
+	})
+	defer cancel()
+	if err := internal.WaitOrStop(ctx, cmd, os.Interrupt, time.Minute); err != nil {
+		return fmt.Errorf("WaitOrStop(_, %v, %v, %v) = %w", cmd, os.Interrupt, time.Minute, err)
+	}
+	return nil
+}
+
 // defaultWindowsDir returns a default path for a Windows VM.
 //
 // The directory should contain the Windows VM image, and UTM
@@ -81,7 +92,7 @@ func windows10Cmd(base string) *exec.Cmd {
 		"-device", "usb-mouse,bus=usb-bus.0",
 		"-device", "usb-kbd,bus=usb-bus.0",
 		"-device", "virtio-net-pci,netdev=net0",
-		"-netdev", "user,id=net0",
+		"-netdev", "user,id=net0,hostfwd=tcp::8080-:8080",
 		"-bios", filepath.Join(base, "Images/QEMU_EFI.fd"),
 		"-device", "nvme,drive=drive0,serial=drive0,bootindex=0",
 		"-drive", fmt.Sprintf("if=none,media=disk,id=drive0,file=%s,cache=writethrough", filepath.Join(base, "Images/win10.qcow2")),