internal/task: update dependency in master branch after minor release

1. Extract the awaitSubmission method so that it can be shared by
   multiple workflows.
2. Add an executeAndMonitorChange helper, to run a script and return
   any change to the file list provided.
3. Add a task to the gopls release flow to upgrade dependency for
   x/tools' gopls submodule in master branch. (only for minor version
   release)
4. Update the gopls release flow to accept CL reviewers as a parameter.

A local relui screenshot is at https://go.dev/issue/57643#issuecomment-2294232653

For golang/go#57643

Change-Id: I9cb6495956409ce9a3712584278ba0e031e48cfc
Reviewed-on: https://go-review.googlesource.com/c/build/+/606336
Auto-Submit: Hongxiang Jiang <[email protected]>
Reviewed-by: Robert Findley <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>
LUCI-TryBot-Result: Go LUCI <[email protected]>

Author: h9jiang

cmd/relui/main.go

@@ -320,6 +320,12 @@ func main() {
 	}
 	dh.RegisterDefinition("Prepare a pre-release gopls candidate", prereleaseGoplsTasks.NewDefinition())
 
+	releaseGoplsTasks := task.ReleaseGoplsTasks{
+		Gerrit:     gerritClient,
+		CloudBuild: cloudBuildClient,
+	}
+	dh.RegisterDefinition("Release gopls", releaseGoplsTasks.NewDefinition())
+
 	privateSyncTask := &task.PrivateMasterSyncTask{
 		Git:              gitClient,
 		PrivateGerritURL: "https://go-internal.googlesource.com/golang/go-private",

internal/task/releasegopls.go

@@ -43,10 +43,10 @@ func (r *PrereleaseGoplsTasks) NewDefinition() *wf.Definition {
 	branchCreated := wf.Action1(wd, "creating new branch if minor release", r.createBranchIfMinor, semv, wf.After(issue))
 
 	configChangeID := wf.Task3(wd, "updating branch's codereview.cfg", r.updateCodeReviewConfig, semv, reviewers, issue, wf.After(branchCreated))
-	configCommit := wf.Task1(wd, "await config CL submission", r.AwaitSubmission, configChangeID)
+	configCommit := wf.Task1(wd, "await config CL submission", clAwaiter{r.Gerrit}.awaitSubmission, configChangeID)
 
 	dependencyChangeID := wf.Task4(wd, "updating gopls' x/tools dependency", r.updateXToolsDependency, semv, prerelease, reviewers, issue, wf.After(configCommit))
-	dependencyCommit := wf.Task1(wd, "await gopls' x/tools dependency CL submission", r.AwaitSubmission, dependencyChangeID)
+	dependencyCommit := wf.Task1(wd, "await gopls' x/tools dependency CL submission", clAwaiter{r.Gerrit}.awaitSubmission, dependencyChangeID)
 
 	verified := wf.Action1(wd, "verify installing latest gopls using release branch dependency commit", r.verifyGoplsInstallation, dependencyCommit)
 	prereleaseVersion := wf.Task3(wd, "tag pre-release", r.tagPrerelease, semv, dependencyCommit, prerelease, wf.After(verified))
@@ -154,10 +154,10 @@ func (r *PrereleaseGoplsTasks) createBranchIfMinor(ctx *wf.TaskContext, semv sem
 //
 // It returns an empty string if no such CL is found, otherwise it returns the
 // CL's change ID.
-func (r *PrereleaseGoplsTasks) openCL(ctx *wf.TaskContext, branch, title string) (string, error) {
+func openCL(ctx *wf.TaskContext, gerrit GerritClient, branch, title string) (string, error) {
 	// Query for an existing pending config CL, to avoid duplication.
 	query := fmt.Sprintf(`message:%q status:open owner:[email protected] repo:tools branch:%q -age:7d`, title, branch)
-	changes, err := r.Gerrit.QueryChanges(ctx, query)
+	changes, err := gerrit.QueryChanges(ctx, query)
 	if err != nil {
 		return "", err
 	}
@@ -178,7 +178,7 @@ func (r *PrereleaseGoplsTasks) updateCodeReviewConfig(ctx *wf.TaskContext, semv
 	branch := goplsReleaseBranchName(semv)
 	clTitle := fmt.Sprintf("all: update %s for %s", configFile, branch)
 
-	openCL, err := r.openCL(ctx, branch, clTitle)
+	openCL, err := openCL(ctx, r.Gerrit, branch, clTitle)
 	if err != nil {
 		return "", fmt.Errorf("failed to find the open CL of title %q in branch %q: %w", clTitle, branch, err)
 	}
@@ -275,7 +275,7 @@ func (r *PrereleaseGoplsTasks) updateXToolsDependency(ctx *wf.TaskContext, semv
 
 	branch := goplsReleaseBranchName(semv)
 	clTitle := fmt.Sprintf("gopls: update go.mod for v%v.%v.%v-%s", semv.Major, semv.Minor, semv.Patch, pre)
-	openCL, err := r.openCL(ctx, branch, clTitle)
+	openCL, err := openCL(ctx, r.Gerrit, branch, clTitle)
 	if err != nil {
 		return "", fmt.Errorf("failed to find the open CL of title %q in branch %q: %w", clTitle, branch, err)
 	}
@@ -284,38 +284,23 @@ func (r *PrereleaseGoplsTasks) updateXToolsDependency(ctx *wf.TaskContext, semv
 		return openCL, nil
 	}
 
-	outputFiles := []string{"gopls/go.mod.before", "gopls/go.mod", "gopls/go.sum.before", "gopls/go.sum"}
-	const scriptFmt = `git checkout %s
-git rev-parse --abbrev-ref HEAD
-cp gopls/go.mod gopls/go.mod.before
-cp gopls/go.sum gopls/go.sum.before
-cd gopls
-go mod edit -dropreplace=golang.org/x/tools
-go get golang.org/x/tools@%s
-go mod tidy -compat=1.19
-`
 	head, err := r.Gerrit.ReadBranchHead(ctx, "tools", branch)
 	if err != nil {
 		return "", err
 	}
-	build, err := r.CloudBuild.RunScript(ctx, fmt.Sprintf(scriptFmt, branch, head), "tools", outputFiles)
-	if err != nil {
-		return "", err
-	}
+	// TODO(hxjiang): Remove -compat flag once gopls no longer supports building
+	// with older Go versions.
+	script := fmt.Sprintf(`cd gopls
+go mod edit -dropreplace=golang.org/x/tools
+go get golang.org/x/tools@%s
+go mod tidy -compat=1.19
+`, head)
 
-	outputs, err := buildToOutputs(ctx, r.CloudBuild, build)
+	changedFiles, err := executeAndMonitorChange(ctx, r.CloudBuild, branch, script, []string{"gopls/go.mod", "gopls/go.sum"})
 	if err != nil {
 		return "", err
 	}
 
-	changedFiles := map[string]string{}
-	for i := 0; i < len(outputFiles); i += 2 {
-		before, after := outputs[outputFiles[i]], outputs[outputFiles[i+1]]
-		if before != after {
-			changedFiles[outputFiles[i+1]] = after
-		}
-	}
-
 	// Skip CL creation as nothing changed.
 	if len(changedFiles) == 0 {
 		return "", nil
@@ -324,7 +309,7 @@ go mod tidy -compat=1.19
 	changeInput := gerrit.ChangeInput{
 		Project: "tools",
 		Branch:  branch,
-		Subject: fmt.Sprintf("%s\n\nThis is an automated CL which updates the go.mod go.sum.\n\nFor golang/go#%v", clTitle, issue),
+		Subject: fmt.Sprintf("%s\n\nThis is an automated CL which updates the go.mod and go.sum.\n\nFor golang/go#%v", clTitle, issue),
 	}
 
 	ctx.Printf("creating auto-submit change under branch %q in x/tools repo.", branch)
@@ -391,21 +376,6 @@ func (r *PrereleaseGoplsTasks) tagPrerelease(ctx *wf.TaskContext, semv semversio
 	return version, nil
 }
 
-// AwaitSubmission waits for the CL with the given change ID to be submitted.
-//
-// The return value is the submitted commit hash, or "" if changeID is "".
-func (r *PrereleaseGoplsTasks) AwaitSubmission(ctx *wf.TaskContext, changeID string) (string, error) {
-	if changeID == "" {
-		ctx.Printf("not awaiting: no CL was created")
-		return "", nil
-	}
-
-	ctx.Printf("awaiting review/submit of %v", ChangeLink(changeID))
-	return AwaitCondition(ctx, 10*time.Second, func() (string, bool, error) {
-		return r.Gerrit.Submitted(ctx, changeID, "")
-	})
-}
-
 func (r *PrereleaseGoplsTasks) isValidVersion(ctx *wf.TaskContext, ver string) (semversion, error) {
 	if !semver.IsValid(ver) {
 		return semversion{}, fmt.Errorf("the input %q version does not follow semantic version schema", ver)
@@ -529,17 +499,22 @@ func (r *PrereleaseGoplsTasks) possibleGoplsVersions(ctx *wf.TaskContext) ([]str
 // ReleaseGoplsTasks implements a new workflow definition include all the tasks
 // to release gopls.
 type ReleaseGoplsTasks struct {
-	Gerrit GerritClient
+	Gerrit     GerritClient
+	CloudBuild CloudBuildClient
 }
 
 // NewDefinition create a new workflow definition for gopls release.
 func (r *ReleaseGoplsTasks) NewDefinition() *wf.Definition {
 	wd := wf.New(wf.ACL{Groups: []string{groups.ToolsTeam}})
 
 	version := wf.Param(wd, wf.ParamDef[string]{Name: "pre-release version"})
+	reviewers := wf.Param(wd, reviewersParam)
 
 	semv := wf.Task1(wd, "validating input version", r.isValidPrereleaseVersion, version)
-	_ = wf.Action1(wd, "tag the release", r.tagRelease, semv)
+	tagged := wf.Action1(wd, "tag the release", r.tagRelease, semv)
+
+	changeID := wf.Task2(wd, "updating x/tools dependency in master branch in gopls sub dir", r.updateDependencyIfMinor, reviewers, semv, wf.After(tagged))
+	_ = wf.Task1(wd, "await x/tools gopls dependency CL submission in gopls sub dir", clAwaiter{r.Gerrit}.awaitSubmission, changeID)
 
 	return wd
 }
@@ -637,3 +612,120 @@ func (r *ReleaseGoplsTasks) tagRelease(ctx *wf.TaskContext, semv semversion) err
 	ctx.Printf("tagged commit %s with tag %s", info.Revision, releaseTag)
 	return nil
 }
+
+// updateDependencyIfMinor update the dependency of x/tools repo in master
+// branch.
+//
+// Returns the change ID.
+func (r *ReleaseGoplsTasks) updateDependencyIfMinor(ctx *wf.TaskContext, reviewers []string, semv semversion) (string, error) {
+	if semv.Patch != 0 {
+		return "", nil
+	}
+
+	clTitle := fmt.Sprintf("gopls/go.mod: update dependencies following the v%v.%v.%v release", semv.Major, semv.Minor, semv.Patch)
+	openCL, err := openCL(ctx, r.Gerrit, "master", clTitle)
+	if err != nil {
+		return "", fmt.Errorf("failed to find the open CL of title %q in master branch: %w", clTitle, err)
+	}
+	if openCL != "" {
+		ctx.Printf("not creating CL: found existing CL %s", openCL)
+		return openCL, nil
+	}
+
+	// TODO(hxjiang): Remove -compat flag once gopls no longer supports building
+	// with older Go versions.
+	const script = `cd gopls
+pwd
+go get -u all
+go mod tidy -compat=1.19
+`
+	changed, err := executeAndMonitorChange(ctx, r.CloudBuild, "master", script, []string{"gopls/go.mod", "gopls/go.sum"})
+	if err != nil {
+		return "", err
+	}
+
+	// Skip CL creation as nothing changed.
+	if len(changed) == 0 {
+		return "", nil
+	}
+
+	changeInput := gerrit.ChangeInput{
+		Project: "tools",
+		Branch:  "master",
+		Subject: fmt.Sprintf("%s\n\nThis is an automated CL which updates the go.mod and go.sum.", clTitle),
+	}
+
+	ctx.Printf("creating auto-submit change under master branch in x/tools repo.")
+	return r.Gerrit.CreateAutoSubmitChange(ctx, changeInput, reviewers, changed)
+}
+
+// executeAndMonitorChange runs the specified script on the designated branch,
+// tracking changes to the provided files.
+//
+// Returns a map where keys are the filenames of modified files and values are
+// their corresponding content after script execution.
+func executeAndMonitorChange(ctx *wf.TaskContext, cloudBuild CloudBuildClient, branch, script string, watchFiles []string) (map[string]string, error) {
+	// Checkout to the provided branch.
+	fullScript := fmt.Sprintf(`git checkout %s
+git rev-parse --abbrev-ref HEAD
+git rev-parse --ref HEAD
+`, branch)
+	// Make a copy of all file that need to watch.
+	// If the file does not exist, create a empty file and a empty before file.
+	for _, file := range watchFiles {
+		if strings.Contains(file, "'") {
+			return nil, fmt.Errorf("file name %q contains '", file)
+		}
+		fullScript += fmt.Sprintf(`if [ -f '%[1]s' ]; then
+    cp '%[1]s' '%[1]s.before'
+else
+    touch '%[1]s' '%[1]s.before'
+fi
+`, file)
+	}
+	// Execute the script provided.
+	fullScript += script
+
+	// Output files before the script execution and after the script execution.
+	outputFiles := []string{}
+	for _, file := range watchFiles {
+		outputFiles = append(outputFiles, file+".before")
+		outputFiles = append(outputFiles, file)
+	}
+	build, err := cloudBuild.RunScript(ctx, fullScript, "tools", outputFiles)
+	if err != nil {
+		return nil, err
+	}
+
+	outputs, err := buildToOutputs(ctx, cloudBuild, build)
+	if err != nil {
+		return nil, err
+	}
+
+	changed := map[string]string{}
+	for i := 0; i < len(outputFiles); i += 2 {
+		if before, after := outputs[outputFiles[i]], outputs[outputFiles[i+1]]; before != after {
+			changed[outputFiles[i+1]] = after
+		}
+	}
+
+	return changed, nil
+}
+
+// A clAwaiter closes over a GerritClient to provide a reusable workflow task
+// for awaiting the submission of a Gerrit change.
+type clAwaiter struct {
+	GerritClient
+}
+
+func (c clAwaiter) awaitSubmission(ctx *wf.TaskContext, changeID string) (string, error) {
+	if changeID == "" {
+		ctx.Printf("not awaiting: no CL was created")
+		return "", nil
+	}
+
+	ctx.Printf("awaiting review/submit of %v", ChangeLink(changeID))
+	return AwaitCondition(ctx, 10*time.Second, func() (string, bool, error) {
+		return c.Submitted(ctx, changeID, "")
+	})
+}

internal/task/releasegopls_test.go

@@ -7,6 +7,7 @@ package task
 import (
 	"context"
 	"fmt"
+	"reflect"
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
@@ -871,3 +872,85 @@ func TestTagRelease(t *testing.T) {
 		})
 	}
 }
+
+func TestExecuteAndMonitorChange(t *testing.T) {
+	mustHaveShell(t)
+
+	testcases := []struct {
+		name   string
+		branch string
+		script string
+		watch  []string
+		want   map[string]string
+	}{
+		{
+			name:   "write all three files with different content",
+			branch: "master",
+			script: `echo -n "foo" > file_a
+echo -n "foo" > file_b
+echo -n "foo" > file_c
+`,
+			watch: []string{"file_a", "file_b", "file_c"},
+			want:  map[string]string{"file_a": "foo", "file_b": "foo", "file_c": "foo"},
+		},
+		{
+			name:   "ignore file_c changes",
+			branch: "master",
+			script: `echo -n "foo" > file_a
+echo -n "foo" > file_b
+echo -n "foo" > file_c
+`,
+			watch: []string{"file_a", "file_b"},
+			want:  map[string]string{"file_a": "foo", "file_b": "foo"},
+		},
+		{
+			name:   "write two files with different content",
+			branch: "master",
+			script: `echo -n "foo" > file_a
+echo -n "foo" > file_b
+`,
+			watch: []string{"file_a", "file_b", "file_c"},
+			want:  map[string]string{"file_a": "foo", "file_b": "foo"},
+		},
+		{
+			name:   "write one file with different content in foo branch",
+			branch: "foo",
+			script: `echo -n "foo" > file_a`,
+			watch:  []string{"file_a", "file_b", "file_c"},
+			want:   map[string]string{"file_a": "foo"},
+		},
+		{
+			name:   "create a file in foo branch",
+			branch: "foo",
+			script: `echo -n "foo" > file_d`,
+			watch:  []string{"file_a", "file_b", "file_c", "file_d"},
+			want:   map[string]string{"file_d": "foo"},
+		},
+	}
+
+	for _, tc := range testcases {
+		t.Run(tc.name, func(t *testing.T) {
+			tools := NewFakeRepo(t, "tools")
+			initial := tools.Commit(map[string]string{
+				"gopls/go.mod": "module golang.org/x/tools\n",
+				"gopls/go.sum": "\n",
+				"file_a":       "file_a",
+				"file_b":       "file_b",
+				"file_c":       "file_c",
+			})
+			if tc.branch != "master" {
+				tools.Branch(tc.branch, initial)
+			}
+
+			cloudBuild := NewFakeCloudBuild(t, NewFakeGerrit(t, tools), "", nil, fakeGo)
+			got, err := executeAndMonitorChange(&workflow.TaskContext{Context: context.Background(), Logger: &testLogger{t, ""}}, cloudBuild, tc.branch, tc.script, tc.watch)
+			if err != nil {
+				t.Fatal(err)
+			}
+
+			if !reflect.DeepEqual(got, tc.want) {
+				t.Errorf("executeAndMonitorChange() = %v want = %v", got, tc.want)
+			}
+		})
+	}
+}