all: remove macstadium integration and monitoring

We no longer use this hosting provider.

Removes builders depending on this host type, including Windows ARM
builders.

Fixes golang/go#57562
Updates golang/go#47019

Change-Id: I1b2d3dfb0540cdf1ebc1ce2cfabbcd7d56c30145
Reviewed-on: https://go-review.googlesource.com/c/build/+/460295
TryBot-Result: Gopher Robot <[email protected]>
Run-TryBot: Jenny Rakoczy <[email protected]>
Reviewed-by: Heschi Kreinick <[email protected]>

Author: toothrot

cmd/buildlet/buildlet.go

@@ -34,7 +34,6 @@ import (
 	"os/exec"
 	"path"
 	"path/filepath"
-	"regexp"
 	"runtime"
 	"strconv"
 	"strings"
@@ -126,9 +125,6 @@ const (
 
 func main() {
 	builderEnv := os.Getenv("GO_BUILDER_ENV")
-	if builderEnv == "macstadium_vm" {
-		configureMacStadium()
-	}
 	onGCE := metadata.OnGCE()
 	switch runtime.GOOS {
 	case "plan9":
@@ -1737,103 +1733,6 @@ func killProcessTreeUnix(p *os.Process) error {
 	return p.Kill()
 }
 
-// configureMacStadium configures the buildlet flags for use on a Mac
-// VM running on MacStadium under VMWare.
-func configureMacStadium() {
-	*haltEntireOS = true
-
-	// TODO: setup RAM disk for tmp and set *workDir
-
-	disableMacScreensaver()
-	enableMacDeveloperMode()
-
-	version, err := exec.Command("sw_vers", "-productVersion").Output()
-	if err != nil {
-		log.Fatalf("failed to find sw_vers -productVersion: %v", err)
-	}
-	majorMinor := regexp.MustCompile(`^(\d+)\.(\d+)`)
-	m := majorMinor.FindStringSubmatch(string(version))
-	if m == nil {
-		log.Fatalf("unsupported sw_vers version %q", version)
-	}
-	major, minor := m[1], m[2] // "10", "12"
-
-	// As of macOS 11 (Big Sur), the major digits are used to indicate the version of
-	// macOS. This version also introduced support for multiple architectures. This
-	// takes into account the need to distinguish between versions and architectures for
-	// the later versions.
-	mj, err := strconv.Atoi(major)
-	if err != nil {
-		log.Fatalf("unable to parse major version %q", major)
-	}
-	if mj >= 13 {
-		*reverseType = fmt.Sprintf("host-darwin-%s-%s", runtime.GOARCH, major)
-	} else if mj >= 11 {
-		// The darwin-amd64-{11,12}_0 hosts have a "_0" suffix as a historical
-		// relic from when it distinguished macOS 10.15, 10.14 and older, as below.
-		*reverseType = fmt.Sprintf("host-darwin-%s-%s_0", runtime.GOARCH, major)
-	} else {
-		*reverseType = fmt.Sprintf("host-darwin-%s-%s_%s", runtime.GOARCH, major, minor)
-	}
-	*coordinator = "farmer.golang.org:443"
-
-	// guestName is set by cmd/makemac to something like
-	// "mac_10_10_host01b" or "mac_10_12_host01a", which encodes
-	// three things: the mac OS version of the guest VM, which
-	// physical machine it's on (1 to 10, currently) and which of
-	// two possible VMs on that host is running (a or b). For
-	// monitoring purposes, we want stable hostnames and don't
-	// care which OS version is currently running (which changes
-	// constantly), so normalize these to only have the host
-	// number and side (a or b), without the OS version. The
-	// buildlet will report the OS version to the coordinator
-	// anyway. We could in theory do this normalization in the
-	// coordinator, but we don't want to put buildlet-specific
-	// knowledge there, and this file already contains a bunch of
-	// buildlet host-specific configuration, so normalize it here.
-	guestName := vmwareGetInfo("guestinfo.name") // "mac_10_12_host01a"
-	hostPos := strings.Index(guestName, "_host")
-	if hostPos == -1 {
-		// Assume cmd/makemac changed its conventions.
-		// Maybe all this normalization belongs there anyway,
-		// but normalizing here is a safer first step.
-		*hostname = guestName
-	} else {
-		*hostname = "macstadium" + guestName[hostPos:] // "macstadium_host01a"
-	}
-}
-
-func disableMacScreensaver() {
-	err := exec.Command("defaults", "-currentHost", "write", "com.apple.screensaver", "idleTime", "0").Run()
-	if err != nil {
-		log.Printf("disabling screensaver: %v", err)
-	}
-}
-
-// enableMacDeveloperMode enables developer mode on macOS for the
-// runtime tests. (Issue 31123)
-//
-// It is best effort; errors are logged but otherwise ignored.
-func enableMacDeveloperMode() {
-	// Macs are configured with password-less sudo. Without sudo we get prompts
-	// that "SampleTools wants to make changes" that block the buildlet from starting.
-	// But oddly, not via gomote. Only during startup. The environment must be different
-	// enough that in one case macOS asks for permission (because it can use the GUI?)
-	// and in the gomote case (where the environment is largely scrubbed) it can't do
-	// the GUI dialog somehow and must just try to do it anyway and finds that passwordless
-	// sudo works. But using sudo seems to make it always work.
-	// For extra paranoia, use a context to not block start-up.
-	ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
-	defer cancel()
-
-	out, err := exec.CommandContext(ctx, "/usr/bin/sudo", "/usr/sbin/DevToolsSecurity", "-enable").CombinedOutput()
-	if err != nil {
-		log.Printf("Error enabling developer mode: %v, %s", err, out)
-		return
-	}
-	log.Printf("DevToolsSecurity: %s", out)
-}
-
 func vmwareGetInfo(key string) string {
 	cmd := exec.Command("/Library/Application Support/VMware Tools/vmware-tools-daemon",
 		"--cmd",

cmd/buildlet/reverse.go

@@ -33,14 +33,6 @@ func keyForMode(mode string) (string, error) {
 	if isDevReverseMode() {
 		return devBuilderKey(mode), nil
 	}
-	if os.Getenv("GO_BUILDER_ENV") == "macstadium_vm" {
-		infoKey := "guestinfo.key-" + mode
-		key := vmwareGetInfo(infoKey)
-		if key == "" {
-			return "", fmt.Errorf("no build key found for VMWare info-get key %q", infoKey)
-		}
-		return key, nil
-	}
 	keyPath := filepath.Join(homedir(), ".gobuildkey-"+mode)
 	if v := os.Getenv("GO_BUILD_KEY_PATH"); v != "" {
 		keyPath = v

cmd/buildlet/stage0/Makefile

@@ -4,9 +4,6 @@
 # The only reason we upload them to GCS is because that's where the
 # automated VM/container creation scripts download them from.
 #
-# The notable exception is our Mac VMs on MacStadium. In those cases,
-# the images actually do download the stage0 binary on start-up.
-#
 
 GO=go1.19
 

cmd/buildlet/stage0/stage0.go

@@ -26,7 +26,6 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strings"
 	"time"
 
 	"cloud.google.com/go/compute/metadata"
@@ -75,7 +74,6 @@ func main() {
 	}
 	log.Printf("bootstrap binary running")
 
-	var isMacStadiumVM bool
 	switch osArch {
 	case "linux/arm":
 		if onGCE {
@@ -97,15 +95,6 @@ func main() {
 		default:
 			panic(fmt.Sprintf("unknown/unspecified $GO_BUILDER_ENV value %q", env))
 		}
-	case "darwin/amd64":
-		// The MacStadium builders' baked-in stage0.sh
-		// bootstrap file doesn't set GO_BUILDER_ENV
-		// unfortunately, so use the filename it runs its
-		// downloaded bootstrap URL to determine whether we're
-		// in that environment.
-		isMacStadiumVM = len(os.Args) > 0 && strings.HasSuffix(os.Args[0], "run-builder")
-		log.Printf("isMacStadiumVM = %v", isMacStadiumVM)
-		os.Setenv("GO_BUILDER_ENV", "macstadium_vm")
 	}
 
 	if !awaitNetwork() {
@@ -115,7 +104,6 @@ func main() {
 	netDelay := prettyDuration(timeNetwork.Sub(timeStart))
 	log.Printf("network up after %v", netDelay)
 
-Download:
 	// Note: we name it ".exe" for Windows, but the name also
 	// works fine on Linux, etc.
 	target := filepath.FromSlash("./buildlet.exe")
@@ -202,15 +190,6 @@ Download:
 	case "solaris/amd64", "illumos/amd64":
 		hostType := buildEnv
 		cmd.Args = append(cmd.Args, reverseHostTypeArgs(hostType)...)
-	case "windows/arm64":
-		switch buildEnv {
-		case "host-windows-arm64-mini", "host-windows11-arm64-mini":
-			cmd.Args = append(cmd.Args,
-				"--halt=true",
-				"--reverse-type="+buildEnv,
-				"--coordinator=farmer.golang.org:443",
-			)
-		}
 	}
 	// Release the serial port (if we opened it) so the buildlet
 	// process can open & write to it. At least on Windows, only
@@ -219,24 +198,6 @@ Download:
 		closeSerialLogOutput()
 	}
 	err := cmd.Run()
-	if isMacStadiumVM {
-		if err != nil {
-			log.Printf("error running buildlet: %v", err)
-			log.Printf("restarting in 2 seconds.")
-			time.Sleep(2 * time.Second) // in case we're spinning, slow it down
-		} else {
-			log.Printf("buildlet process exited; restarting.")
-		}
-		// Some of the MacStadium VM environments reuse their
-		// environment. Re-download the buildlet (if it
-		// changed-- httpdl does conditional downloading) and
-		// then re-run. At least on Sierra we never get this
-		// far because the buildlet will halt the machine
-		// before we get here. (and then cmd/makemac will
-		// recreate the VM)
-		// But if we get here, restart the process.
-		goto Download
-	}
 	if err != nil {
 		if configureSerialLogOutput != nil {
 			configureSerialLogOutput()

cmd/coordinator/coordinator.go

@@ -360,7 +360,6 @@ func main() {
 	mux.HandleFunc("/style.css", handleStyleCSS)
 	mux.HandleFunc("/try", serveTryStatus(false))
 	mux.HandleFunc("/try.json", serveTryStatus(true))
-	mux.HandleFunc("/status/reverse.json", pool.ReversePool().ServeReverseStatusJSON)
 	mux.HandleFunc("/status/post-submit-active.json", handlePostSubmitActiveJSON)
 	mux.Handle("/dashboard", dashV2)
 	mux.HandleFunc("/queues", handleQueues)

cmd/coordinator/coordinator_test.go

@@ -250,36 +250,36 @@ func TestIssue42084(t *testing.T) {
 
 	// First, determine builds without try messages. Our target SlowBot shouldn't be included.
 	ts := newTrySet(work)
-	hasWindowsARM64Builder := false
+	hasLinuxArmBuilder := false
 	for _, bs := range ts.builds {
 		v := bs.NameAndBranch()
-		if v == "windows-arm64-10" {
-			hasWindowsARM64Builder = true
+		if v == "linux-arm" {
+			hasLinuxArmBuilder = true
 		}
 	}
-	if hasWindowsARM64Builder {
-		// This test relies on windows-arm64-10 builder not being a default
+	if hasLinuxArmBuilder {
+		// This test relies on linux-arm builder not being a default
 		// TryBot to provide coverage for issue 42084. If the build policy
 		// changes, need to pick another builder to use in this test.
-		t.Fatal("windows-arm64-10 builder was included even without TRY= message")
+		t.Fatal("linux-arm builder was included even without TRY= message")
 	}
 
 	// Next, add try messages, and check that the SlowBot is now included.
 	work.TryMessage = []*apipb.TryVoteMessage{
-		{Message: "TRY=windows-amd64", AuthorId: 1234, Version: 1},
-		{Message: "TRY=windows-arm64-10", AuthorId: 1234, Version: 1},
+		{Message: "TRY=linux", AuthorId: 1234, Version: 1},
+		{Message: "TRY=linux-arm", AuthorId: 1234, Version: 1},
 	}
 	ts = newTrySet(work)
-	hasWindowsARM64Builder = false
+	hasLinuxArmBuilder = false
 	for i, bs := range ts.builds {
 		v := bs.NameAndBranch()
 		t.Logf("build[%d]: %s", i, v)
-		if v == "windows-arm64-10" {
-			hasWindowsARM64Builder = true
+		if v == "linux-arm-aws" {
+			hasLinuxArmBuilder = true
 		}
 	}
-	if !hasWindowsARM64Builder {
-		t.Error("windows-arm64-10 SlowBot was not included")
+	if !hasLinuxArmBuilder {
+		t.Error("linux-arm SlowBot was not included")
 	}
 }
 

cmd/coordinator/status.go

@@ -13,7 +13,6 @@ import (
 	"bytes"
 	"context"
 	_ "embed"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"html"
@@ -147,7 +146,6 @@ func addHealthChecker(mux *http.ServeMux, hc *healthChecker) {
 var basePinErr atomic.Value
 
 func addHealthCheckers(ctx context.Context, mux *http.ServeMux, sc *secret.Client) {
-	addHealthChecker(mux, newMacHealthChecker())
 	addHealthChecker(mux, newMacOSARM64Checker())
 	addHealthChecker(mux, newOSUPPC64leChecker())
 	addHealthChecker(mux, newOSUPPC64lePower9Checker())
@@ -292,81 +290,6 @@ func newGitMirrorChecker() *healthChecker {
 	}
 }
 
-func newMacHealthChecker() *healthChecker {
-	var hosts []string
-	const numMacHosts = 8 // physical Mac Pros, not reverse buildlet connections. M1 Macs will be included in separate checks.
-	for i := 1; i <= numMacHosts; i++ {
-		for _, suf := range []string{"a", "b"} {
-			name := fmt.Sprintf("macstadium_host%02d%s", i, suf)
-			hosts = append(hosts, name)
-		}
-	}
-	checkHosts := reverseHostChecker(hosts)
-
-	// And check that the makemac daemon is listening.
-	var makeMac struct {
-		sync.Mutex
-		lastCheck  time.Time // currently unused
-		lastErrors []string
-		lastWarns  []string
-	}
-	setMakeMacStatus := func(errs, warns []string) {
-		makeMac.Lock()
-		defer makeMac.Unlock()
-		makeMac.lastCheck = time.Now()
-		makeMac.lastErrors = errs
-		makeMac.lastWarns = warns
-	}
-	go func() {
-		for {
-			errs, warns := fetchMakeMacStatus()
-			setMakeMacStatus(errs, warns)
-			time.Sleep(15 * time.Second)
-		}
-	}()
-	return &healthChecker{
-		ID:     "macs",
-		Title:  "MacStadium Mac VMs",
-		DocURL: "https://github.com/golang/build/tree/master/env/darwin/macstadium",
-		Check: func(w *checkWriter) {
-			// Check hosts.
-			checkHosts(w)
-			// Check makemac daemon.
-			makeMac.Lock()
-			defer makeMac.Unlock()
-			for _, v := range makeMac.lastWarns {
-				w.warnf("makemac daemon: %v", v)
-			}
-			for _, v := range makeMac.lastErrors {
-				w.errorf("makemac daemon: %v", v)
-			}
-		},
-	}
-}
-
-func fetchMakeMacStatus() (errs, warns []string) {
-	c := &http.Client{Timeout: 15 * time.Second}
-	res, err := c.Get("http://macstadiumd.golang.org:8713")
-	if err != nil {
-		return []string{fmt.Sprintf("failed to fetch status: %v", err)}, nil
-	}
-	defer res.Body.Close()
-	if res.StatusCode != 200 {
-		return []string{fmt.Sprintf("HTTP status %v", res.Status)}, nil
-	}
-	if res.Header.Get("Content-Type") != "application/json" {
-		return []string{fmt.Sprintf("unexpected content-type %q; want JSON", res.Header.Get("Content-Type"))}, nil
-	}
-	var resj struct {
-		Errors   []string
-		Warnings []string
-	}
-	if err := json.NewDecoder(res.Body).Decode(&resj); err != nil {
-		return []string{fmt.Sprintf("reading status response body: %v", err)}, nil
-	}
-	return resj.Errors, resj.Warnings
-}
-
 func newMacOSARM64Checker() *healthChecker {
 	var expect int // Number of expected darwin/arm64 reverse builders based on x/build/dashboard.
 	for hostType, hc := range dashboard.Hosts {

cmd/coordinator/status_test.go

@@ -95,8 +95,6 @@ func TestHandleStatus_HealthFormatting(t *testing.T) {
 		t.Fatalf("output didn't contain %q: %s", suf, got)
 	}
 	for _, sub := range []string{
-		`<a href="/status/macs">MacStadium Mac VMs</a> [`,
-		`<li>macstadium_host06a not yet connected</li>`,
 		`<a href="/status/allgood">All Good Test</a>: ok`,
 		`<li>test-info</li>`,
 		`<li><span style='color: orange'>test-warn</span></li>`,