all: move services to new Kubernetes cluster

Our makefiles install xb from the repository, so this has to be checked
in before I do anything else, and nobody else should deploy anything
until I'm done.

Because the new cluster is an Autopilot cluster, it lives in a region
(us-central1) and that's what you pass to the kubectl command, etc.
Move Region/Zone into the individual KubeConfigs and use the correct
ones as appropriate.

For golang/go#48408.

Change-Id: Iceacfe68305a3744aa87ce0fef777b977a252586
Reviewed-on: https://go-review.googlesource.com/c/build/+/350137
Trust: Heschi Kreinick <[email protected]>
Run-TryBot: Heschi Kreinick <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: heschi

buildenv/envs.go

@@ -14,7 +14,6 @@ import (
 	"math/rand"
 	"os"
 	"path/filepath"
-	"strings"
 	"sync"
 
 	"golang.org/x/oauth2"
@@ -42,13 +41,32 @@ type KubeConfig struct {
 	// MachineType is the GCE machine type to use for the Kubernetes cluster nodes.
 	MachineType string
 
+	// The zone of the cluster. Autopilot clusters have no single zone.
+	Zone string
+
+	// The region of the cluster.
+	Region string
+
 	// Name is the name of the Kubernetes cluster that will be created.
 	Name string
 
 	// Namespace is the Kubernetes namespace to use within the cluster.
 	Namespace string
 }
 
+// ZoneOrRegion returns the zone or if unset, the region of the cluster.
+// This is the string to use as the "zone" of the cluster when connecting to it
+// with the Kubernetes API.
+func (kc KubeConfig) ZoneOrRegion() string {
+	if kc.Zone != "" {
+		return kc.Zone
+	}
+	if kc.Region != "" {
+		return kc.Region
+	}
+	panic(fmt.Sprintf("KubeConfig has neither zone nor region: %#v", kc))
+}
+
 // Environment describes the configuration of the infrastructure for a
 // coordinator and its buildlet resources running on Google Cloud Platform.
 // Staging and Production are the two common build environments.
@@ -72,11 +90,6 @@ type Environment struct {
 	// disabled and the coordinator serves on 8119.
 	IsProd bool
 
-	// ControlZone is the GCE zone that the coordinator instance and Kubernetes cluster
-	// will run in. This field may be overridden as necessary without impacting
-	// other fields.
-	ControlZone string
-
 	// VMZones are the GCE zones that the VMs will be deployed to. These
 	// GCE zones will be periodically cleaned by deleting old VMs. The zones
 	// should all exist within a single region.
@@ -90,10 +103,10 @@ type Environment struct {
 	// MachineType is the GCE machine type to use for the coordinator.
 	MachineType string
 
-	// KubeBuild is the Kubernetes config for the buildlet cluster.
+	// KubeBuild is the cluster that runs buildlets.
 	KubeBuild KubeConfig
-	// KubeTools is the Kubernetes config for the tools cluster.
-	KubeTools KubeConfig
+	// KubeServices is the cluster that runs the coordinator and other services.
+	KubeServices KubeConfig
 
 	// PreferContainersOnCOS controls whether we do most builds on
 	// Google's Container-Optimized OS Linux image running on a VM
@@ -154,19 +167,10 @@ func (e Environment) ComputePrefix() string {
 }
 
 // RandomVMZone returns a randomly selected zone from the zones in VMZones.
-// The Zone value will be returned if VMZones is not set.
 func (e Environment) RandomVMZone() string {
-	if len(e.VMZones) == 0 {
-		return e.ControlZone
-	}
 	return e.VMZones[rand.Intn(len(e.VMZones))]
 }
 
-// Region returns the GCE region, derived from its zone.
-func (e Environment) Region() string {
-	return e.ControlZone[:strings.LastIndex(e.ControlZone, "-")]
-}
-
 // SnapshotURL returns the absolute URL of the .tar.gz containing a
 // built Go tree for the builderType and Go rev (40 character Git
 // commit hash). The tarball is suitable for passing to
@@ -248,20 +252,23 @@ var Staging = &Environment{
 	ProjectNumber:         302018677728,
 	GoProjectName:         "go-dashboard-dev",
 	IsProd:                true,
-	ControlZone:           "us-central1-f",
 	VMZones:               []string{"us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"},
 	StaticIP:              "104.154.113.235",
 	MachineType:           "n1-standard-1",
 	PreferContainersOnCOS: true,
 	KubeBuild: KubeConfig{
 		MinNodes:    1,
 		MaxNodes:    1, // auto-scaling disabled
+		Zone:        "us-central1-f",
+		Region:      "us-central1",
 		Name:        "buildlets",
 		MachineType: "n1-standard-4", // only used for make.bash due to PreferContainersOnCOS
 	},
-	KubeTools: KubeConfig{
+	KubeServices: KubeConfig{
 		MinNodes:    3,
 		MaxNodes:    3,
+		Zone:        "us-central1-f",
+		Region:      "us-central1",
 		Name:        "go",
 		MachineType: "n1-standard-4",
 		Namespace:   "default",
@@ -284,21 +291,23 @@ var Production = &Environment{
 	ProjectNumber:         872405196845,
 	GoProjectName:         "golang-org",
 	IsProd:                true,
-	ControlZone:           "us-central1-f",
 	VMZones:               []string{"us-central1-a", "us-central1-b", "us-central1-c", "us-central1-f"},
 	StaticIP:              "107.178.219.46",
 	MachineType:           "n1-standard-4",
 	PreferContainersOnCOS: true,
 	KubeBuild: KubeConfig{
 		MinNodes:    2,
 		MaxNodes:    2, // auto-scaling disabled
+		Zone:        "us-central1-f",
+		Region:      "us-central1",
 		Name:        "buildlets",
 		MachineType: "n1-standard-4", // only used for make.bash due to PreferContainersOnCOS
 	},
-	KubeTools: KubeConfig{
+	KubeServices: KubeConfig{
 		MinNodes:    4,
 		MaxNodes:    4,
-		Name:        "go",
+		Region:      "us-central1",
+		Name:        "services",
 		MachineType: "n1-standard-4",
 		Namespace:   "prod",
 	},

buildenv/envs_test.go

@@ -9,44 +9,13 @@ import (
 )
 
 func TestEnvironmentNextZone(t *testing.T) {
-	testCases := []struct {
-		name      string
-		env       Environment
-		wantOneOf []string // desired zone should appear in this slice
-	}{
-		{
-			name: "zones-not-set",
-			env: Environment{
-				ControlZone: "kentucky",
-				VMZones:     []string{},
-			},
-			wantOneOf: []string{"kentucky"},
-		},
-		{
-			name: "zone-and-zones-set",
-			env: Environment{
-				ControlZone: "kentucky",
-				VMZones:     []string{"texas", "california", "washington"},
-			},
-
-			wantOneOf: []string{"texas", "california", "washington"},
-		},
-		{
-			name: "zones-only-contains-one-entry",
-			env: Environment{
-				ControlZone: "kentucky",
-				VMZones:     []string{"texas"},
-			},
-			wantOneOf: []string{"texas"},
-		},
+	env := Environment{
+		VMZones: []string{"texas", "california", "washington"},
 	}
-	for _, tc := range testCases {
-		t.Run(tc.name, func(t *testing.T) {
-			got := tc.env.RandomVMZone()
-			if !containsString(got, tc.wantOneOf) {
-				t.Errorf("got=%q; want %v", got, tc.wantOneOf)
-			}
-		})
+	wantOneOf := []string{"texas", "california", "washington"}
+	got := env.RandomVMZone()
+	if !containsString(got, wantOneOf) {
+		t.Errorf("got=%q; want %v", got, wantOneOf)
 	}
 }
 

cmd/gcpinit/gcpinit.go

@@ -32,7 +32,7 @@ resources:
 - name: "{{ .Kube.Name }}"
   type: container.v1.cluster
   properties:
-    zone: "{{ .Env.ControlZone }}"
+    zone: "{{ .Env.KubeServices.Zone }}"
     cluster:
       initial_node_count: {{ .Kube.MinNodes }}
       network: "default"
@@ -77,7 +77,7 @@ func main() {
 		log.Fatalf("could not create client: %v", err)
 	}
 
-	for _, c := range []*buildenv.KubeConfig{&buildEnv.KubeBuild, &buildEnv.KubeTools} {
+	for _, c := range []*buildenv.KubeConfig{&buildEnv.KubeBuild, &buildEnv.KubeServices} {
 		err := createCluster(bgc, c)
 		if err != nil {
 			log.Fatalf("Error creating Kubernetes cluster %q: %v", c.Name, err)

cmd/xb/xb.go

@@ -56,16 +56,14 @@ func main() {
 	case "kubectl":
 		env := getEnv()
 		curCtx := kubeCurrentContext()
-		wantCtx := fmt.Sprintf("gke_%s_%s_go", env.ProjectName, env.ControlZone)
+		wantCtx := fmt.Sprintf("gke_%s_%s_%s", env.ProjectName, env.KubeServices.ZoneOrRegion(), env.KubeServices.Name)
 		if curCtx != wantCtx {
 			log.SetFlags(0)
-			log.Fatalf("Wrong kubectl context; currently using %q; want %q\nRun:\n  gcloud container clusters get-credentials --project=%s --zone=%s go",
+			log.Fatalf("Wrong kubectl context; currently using %q; want %q\nRun:\n  gcloud container clusters get-credentials --project=%s --zone=%s %s",
 				curCtx, wantCtx,
-				env.ProjectName, env.ControlZone,
+				env.ProjectName, env.KubeServices.ZoneOrRegion(), env.KubeServices.Name,
 			)
 		}
-		// gcloud container clusters get-credentials --zone=us-central1-f go
-		// gcloud container clusters get-credentials --zone=us-central1-f buildlets
 		runCmd()
 	case "docker":
 		runDocker()

internal/buildgo/basepin.go

@@ -82,7 +82,7 @@ func (c *Client) MakeBasepinDisks(ctx context.Context) error {
 				delete(need, d.SourceImage)
 				continue
 			}
-			if zone != c.Env.ControlZone {
+			if zone != c.Env.KubeBuild.Zone {
 				log.Printf("basepin: deleting unnecessary disk %v in zone %v", d.Name, zone)
 				op, err := svc.Disks.Delete(c.Env.ProjectName, zone, d.Name).Do()
 				if err != nil {

internal/coordinator/pool/gce.go

@@ -133,7 +133,7 @@ func InitGCE(sc *secret.Client, vmDeleteTimeout time.Duration, tFiles map[string
 
 		// Convert the zone from "projects/1234/zones/us-central1-a" to "us-central1-a".
 		projectZone = path.Base(projectZone)
-		buildEnv.ControlZone = projectZone
+		buildEnv.KubeBuild.Zone = projectZone
 
 		if buildEnv.StaticIP == "" {
 			buildEnv.StaticIP, err = metadata.ExternalIP()
@@ -349,9 +349,9 @@ func (p *GCEBuildlet) pollQuotaLoop() {
 
 func (p *GCEBuildlet) pollQuota() {
 	gceAPIGate()
-	reg, err := computeService.Regions.Get(buildEnv.ProjectName, buildEnv.Region()).Do()
+	reg, err := computeService.Regions.Get(buildEnv.ProjectName, buildEnv.KubeBuild.Region).Do()
 	if err != nil {
-		log.Printf("Failed to get quota for %s/%s: %v", buildEnv.ProjectName, buildEnv.Region(), err)
+		log.Printf("Failed to get quota for %s/%s: %v", buildEnv.ProjectName, buildEnv.KubeBuild.Region, err)
 		return
 	}
 	p.mu.Lock()

internal/coordinator/pool/kube.go

@@ -74,17 +74,17 @@ func InitKube(monitorGitMirror MonitorGitMirrorFunc) error {
 	var err error
 	buildletsKubeClient, err = gke.NewClient(ctx,
 		gceBuildEnv.KubeBuild.Name,
-		gke.OptZone(gceBuildEnv.ControlZone),
+		gke.OptZone(gceBuildEnv.KubeBuild.ZoneOrRegion()),
 		gke.OptProject(gceBuildEnv.ProjectName),
 		gke.OptTokenSource(gce.GCPCredentials().TokenSource))
 	if err != nil {
 		return err
 	}
 
 	goKubeClient, err = gke.NewClient(ctx,
-		gceBuildEnv.KubeTools.Name,
-		gke.OptNamespace(gceBuildEnv.KubeTools.Namespace),
-		gke.OptZone(gceBuildEnv.ControlZone),
+		gceBuildEnv.KubeServices.Name,
+		gke.OptNamespace(gceBuildEnv.KubeServices.Namespace),
+		gke.OptZone(gceBuildEnv.KubeServices.ZoneOrRegion()),
 		gke.OptProject(gceBuildEnv.ProjectName),
 		gke.OptTokenSource(gce.GCPCredentials().TokenSource))
 	if err != nil {
@@ -172,12 +172,12 @@ func (p *kubeBuildletPool) pollCapacity(ctx context.Context) {
 	gceBuildEnv := NewGCEConfiguration().BuildEnv()
 	nodes, err := buildletsKubeClient.GetNodes(ctx)
 	if err != nil {
-		log.Printf("failed to retrieve nodes to calculate cluster capacity for %s/%s: %v", gceBuildEnv.ProjectName, gceBuildEnv.Region(), err)
+		log.Printf("failed to retrieve nodes to calculate cluster capacity for %s/%s: %v", gceBuildEnv.ProjectName, gceBuildEnv.KubeBuild.Region, err)
 		return
 	}
 	pods, err := buildletsKubeClient.GetPods(ctx)
 	if err != nil {
-		log.Printf("failed to retrieve pods to calculate cluster capacity for %s/%s: %v", gceBuildEnv.ProjectName, gceBuildEnv.Region(), err)
+		log.Printf("failed to retrieve pods to calculate cluster capacity for %s/%s: %v", gceBuildEnv.ProjectName, gceBuildEnv.KubeBuild.Region, err)
 		return
 	}
 
@@ -474,7 +474,7 @@ func (p *kubeBuildletPool) cleanUpOldPods(ctx context.Context) {
 				}
 				if err == nil && time.Now().Unix() > unixDeadline {
 					stats.DeletedOld++
-					log.Printf("cleanUpOldPods: Deleting expired pod %q in zone %q ...", pod.Name, NewGCEConfiguration().BuildEnv().ControlZone)
+					log.Printf("cleanUpOldPods: Deleting expired pod %q...", pod.Name)
 					err = buildletsKubeClient.DeletePod(ctx, pod.Name)
 					if err != nil {
 						log.Printf("cleanUpOldPods: problem deleting old pod %q: %v", pod.Name, err)