cmd/coordinator, dashboard: scale test exec timeouts by GO_TEST_TIMEOUT_SCALE

So slow builders can finish their cgo tests, etc.

Updates golang/go#29177 etc

Change-Id: If5ee8d2201dc53ab506d4f5e31406d6aae240352
Reviewed-on: https://go-review.googlesource.com/c/162540
Run-TryBot: Brad Fitzpatrick <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: bradfitz

cmd/coordinator/coordinator.go

@@ -2836,11 +2836,6 @@ func parseOutputAndBanner(b []byte) (banner string, out []byte) {
 // (A communication error)
 const maxTestExecErrors = 3
 
-func execTimeout(testNames []string) time.Duration {
-	// TODO(bradfitz): something smarter probably.
-	return 20 * time.Minute
-}
-
 // runTestsOnBuildlet runs tis on bc, using the optional goroot & gopath environment variables.
 func (st *buildStatus) runTestsOnBuildlet(bc *buildlet.Client, tis []*testItem, goroot, gopath string) {
 	names := make([]string, len(tis))
@@ -2871,7 +2866,7 @@ func (st *buildStatus) runTestsOnBuildlet(bc *buildlet.Client, tis []*testItem,
 	args = append(args, names...)
 	var buf bytes.Buffer
 	t0 := time.Now()
-	timeout := execTimeout(names)
+	timeout := st.conf.DistTestsExecTimeout(names)
 	var remoteErr, err error
 	if ti := tis[0]; ti.bench != nil {
 		pbr, perr := st.parentRev()

dashboard/builders.go

@@ -12,6 +12,7 @@ import (
 	"sort"
 	"strconv"
 	"strings"
+	"time"
 
 	"golang.org/x/build/buildenv"
 	"golang.org/x/build/types"
@@ -681,6 +682,8 @@ type BuildConfig struct {
 
 	env           []string // extra environment ("key=value") pairs
 	allScriptArgs []string
+
+	testHostConf *HostConfig // override HostConfig for testing, at least for now
 }
 
 func (c *BuildConfig) Env() []string {
@@ -720,7 +723,42 @@ func (c *BuildConfig) FilePathJoin(x ...string) string {
 	return strings.Join(x, "/")
 }
 
+// DistTestsExecTimeout returns how long the coordinator should wait
+// for a cmd/dist test execution to run the provided dist test names.
+func (c *BuildConfig) DistTestsExecTimeout(distTests []string) time.Duration {
+	// TODO: consider using distTests? We never did before, but
+	// now we have the TestStats in the coordinator. Pass in a
+	// *buildstats.TestStats and use historical data times some
+	// fudge factor? For now just use the old 20 minute limit
+	// we've used since 2014, but scale it by the
+	// GO_TEST_TIMEOUT_SCALE for the super slow builders which
+	// struggle with, say, the cgo tests. (which should be broken
+	// up into separate dist tests or shards, like the test/ dir
+	// was)
+	d := 20 * time.Minute
+	d *= time.Duration(c.timeoutScale())
+	return d
+}
+
+// timeoutScale returns this builder's GO_TEST_TIMEOUT_SCALE value, or 1.
+func (c *BuildConfig) timeoutScale() int {
+	const pfx = "GO_TEST_TIMEOUT_SCALE="
+	for _, env := range [][]string{c.env, c.hostConf().env} {
+		for _, kv := range env {
+			if strings.HasPrefix(kv, pfx) {
+				if n, err := strconv.Atoi(kv[len(pfx):]); err == nil && n > 0 {
+					return n
+				}
+			}
+		}
+	}
+	return 1
+}
+
 func (c *BuildConfig) hostConf() *HostConfig {
+	if c.testHostConf != nil {
+		return c.testHostConf
+	}
 	if c, ok := Hosts[c.HostType]; ok {
 		return c
 	}

dashboard/builders_test.go

@@ -7,6 +7,7 @@ package dashboard
 import (
 	"strings"
 	"testing"
+	"time"
 )
 
 func TestOSARCHAccessors(t *testing.T) {
@@ -21,6 +22,53 @@ func TestOSARCHAccessors(t *testing.T) {
 	}
 }
 
+func TestDistTestsExecTimeout(t *testing.T) {
+	tests := []struct {
+		c    *BuildConfig
+		want time.Duration
+	}{
+		{
+			&BuildConfig{
+				env:          []string{},
+				testHostConf: &HostConfig{},
+			},
+			20 * time.Minute,
+		},
+		{
+			&BuildConfig{
+				env:          []string{"GO_TEST_TIMEOUT_SCALE=2"},
+				testHostConf: &HostConfig{},
+			},
+			40 * time.Minute,
+		},
+		{
+			&BuildConfig{
+				env: []string{},
+				testHostConf: &HostConfig{
+					env: []string{"GO_TEST_TIMEOUT_SCALE=3"},
+				},
+			},
+			60 * time.Minute,
+		},
+		// BuildConfig's env takes precedence:
+		{
+			&BuildConfig{
+				env: []string{"GO_TEST_TIMEOUT_SCALE=2"},
+				testHostConf: &HostConfig{
+					env: []string{"GO_TEST_TIMEOUT_SCALE=3"},
+				},
+			},
+			40 * time.Minute,
+		},
+	}
+	for i, tt := range tests {
+		got := tt.c.DistTestsExecTimeout(nil)
+		if got != tt.want {
+			t.Errorf("%d. got %v; want %v", i, got, tt.want)
+		}
+	}
+}
+
 func TestListTrybots(t *testing.T) {
 	forProj := func(proj string) {
 		t.Run(proj, func(t *testing.T) {