cmd/coordinator: enable EC2 buildlet pool

This change enables the EC2 buildlet pools.

Updates golang/go#38337
Fixes golang/go#36841

Change-Id: I3f9384c039dce8fab529650bc6acdbeda40f5819
Reviewed-on: https://go-review.googlesource.com/c/build/+/247908
Run-TryBot: Carlos Amedee <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Alexander Rakoczy <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: cagedmantis

cmd/coordinator/coordinator.go

@@ -60,6 +60,7 @@ import (
 	"golang.org/x/build/gerrit"
 	"golang.org/x/build/internal/buildgo"
 	"golang.org/x/build/internal/buildstats"
+	"golang.org/x/build/internal/cloud"
 	"golang.org/x/build/internal/coordinator/pool"
 	"golang.org/x/build/internal/secret"
 	"golang.org/x/build/internal/singleflight"
@@ -115,6 +116,7 @@ var (
 	mode           = flag.String("mode", "", "Valid modes are 'dev', 'prod', or '' for auto-detect. dev means localhost development, not be confused with staging on go-dashboard-dev, which is still the 'prod' mode.")
 	buildEnvName   = flag.String("env", "", "The build environment configuration to use. Not required if running on GCE.")
 	devEnableGCE   = flag.Bool("dev_gce", false, "Whether or not to enable the GCE pool when in dev mode. The pool is enabled by default in prod mode.")
+	devEnableEC2   = flag.Bool("dev_ec2", false, "Whether or not to enable the EC2 pool when in dev mode. The pool is enabled by default in prod mode.")
 	shouldRunBench = flag.Bool("run_bench", false, "Whether or not to run benchmarks on trybot commits. Override by GCE project attribute 'farmer-run-bench'.")
 	perfServer     = flag.String("perf_server", "", "Upload benchmark results to `server`. Overrides buildenv default for testing.")
 )
@@ -296,6 +298,13 @@ func main() {
 		log.Printf("Kube support disabled due to error initializing Kubernetes: %v", err)
 	}
 
+	if *mode == "prod" || (*mode == "dev" && *devEnableEC2) {
+		// TODO(golang.org/issues/38337) the coordinator will use a package scoped pool
+		// until the coordinator is refactored to not require them.
+		ec2Pool := mustCreateEC2BuildletPool(sc)
+		defer ec2Pool.Close()
+	}
+
 	go updateInstanceRecord()
 
 	switch *mode {
@@ -1634,6 +1643,8 @@ func poolForConf(conf *dashboard.HostConfig) pool.Buildlet {
 		panic("nil conf")
 	}
 	switch {
+	case conf.IsEC2():
+		return pool.EC2BuildetPool()
 	case conf.IsVM():
 		return pool.NewGCEConfiguration().BuildletPool()
 	case conf.IsContainer():
@@ -1751,6 +1762,9 @@ func (st *buildStatus) expectedBuildletStartDuration() time.Duration {
 			return 2 * time.Minute
 		}
 		return time.Minute
+	case *pool.EC2Buildlet:
+		// lack of historical data. 2 * time.Minute is a safe overestimate
+		return 2 * time.Minute
 	case *pool.ReverseBuildletPool:
 		goos, arch := st.conf.GOOS(), st.conf.GOARCH()
 		if goos == "darwin" {
@@ -4048,3 +4062,26 @@ func mustCreateSecretClientOnGCE() *secret.Client {
 	}
 	return client
 }
+
+func mustCreateEC2BuildletPool(sc *secret.Client) *pool.EC2Buildlet {
+	awsKeyID, err := sc.Retrieve(context.Background(), secret.NameAWSKeyID)
+	if err != nil {
+		log.Fatalf("unable to retrieve secret %q: %s", secret.NameAWSKeyID, err)
+	}
+
+	awsAccessKey, err := sc.Retrieve(context.Background(), secret.NameAWSAccessKey)
+	if err != nil {
+		log.Fatalf("unable to retrieve secret %q: %s", secret.NameAWSAccessKey, err)
+	}
+
+	awsClient, err := cloud.NewAWSClient(buildenv.Production.AWSRegion, awsKeyID, awsAccessKey)
+	if err != nil {
+		log.Fatalf("unable to create AWS client: %s", err)
+	}
+
+	ec2Pool, err := pool.NewEC2Buildlet(awsClient, buildenv.Production, dashboard.Hosts, isGCERemoteBuildlet)
+	if err != nil {
+		log.Fatalf("unable to create EC2 buildlet pool: %s", err)
+	}
+	return ec2Pool
+}

cmd/coordinator/status.go

@@ -659,6 +659,11 @@ func handleStatus(w http.ResponseWriter, r *http.Request) {
 	data.GCEPoolStatus = template.HTML(buf.String())
 	buf.Reset()
 
+	buf.Reset()
+	pool.EC2BuildetPool().WriteHTMLStatus(&buf)
+	data.EC2PoolStatus = template.HTML(buf.String())
+	buf.Reset()
+
 	pool.KubePool().WriteHTMLStatus(&buf)
 	data.KubePoolStatus = template.HTML(buf.String())
 	buf.Reset()
@@ -727,6 +732,7 @@ type statusData struct {
 	TrybotsErr        string
 	Trybots           template.HTML
 	GCEPoolStatus     template.HTML // TODO: embed template
+	EC2PoolStatus     template.HTML // TODO: embed template
 	KubePoolStatus    template.HTML // TODO: embed template
 	ReversePoolStatus template.HTML // TODO: embed template
 	RemoteBuildlets   template.HTML
@@ -790,6 +796,7 @@ var statusTmpl = template.Must(template.New("status").Parse(`
 <h2 id=pools>Buildlet pools <a href='#pools'>¶</a></h2>
 <ul>
 	<li>{{.GCEPoolStatus}}</li>
+	<li>{{.EC2PoolStatus}}</li>
 	<li>{{.KubePoolStatus}}</li>
 	<li>{{.ReversePoolStatus}}</li>
 </ul>