internal/task: replace use of go.dev/dl/?mode=json API with local params

The tweet tasks are some of the earliest we've automated for relui.
In fact, they were ready for use with the release process ahead of
the rest of relui, so the only place to add them was the CLI-based
x/build/cmd/releasebot command.

Since release coordinators needed to provide all of the task inputs as
a JSON object over CLI, at the time, it was easier to fetch information
about the Go release artifacts that had been published for the very Go
release being made by making a network call to the go.dev/dl/?mode=json
API.

By now, relui is complete and has all the steps interconnected without
toil of invoking individual release steps manually, so it's easy for it
to pass the release artifacts to the tweeting tasks directly, and avoid
having to ask the website for the release it itself published (silly!).

We can also drop the lightweight checking of bad version strings done
in oneOrTwoGoVersions. Those typos can't happen now since relui picks
the next version to release automatically.

This CL does the re-wiring to drop the use of API from TweetRelease but
makes almost no changes to the current comprehensive tweet tests, and
they're still passing. The next change simplifies the tests to drop
their temporary use of the API.

For golang/go#57062.

Change-Id: I60efc8c8c96152ff52c9fb5cc0f91262f0d8f494
Reviewed-on: https://go-review.googlesource.com/c/build/+/504519
Reviewed-by: Dmitri Shuralyov <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Auto-Submit: Dmitri Shuralyov <[email protected]>
Reviewed-by: Heschi Kreinick <[email protected]>
Run-TryBot: Dmitri Shuralyov <[email protected]>

Author: dmitshur

internal/relui/workflows.go

@@ -296,15 +296,15 @@ func registerProdReleaseWorkflows(ctx context.Context, h *DefinitionHolder, buil
 
 		coordinators := wf.Param(wd, releaseCoordinators)
 
-		versionPublished := addSingleReleaseWorkflow(build, milestone, version, wd, r.major, r.kind, coordinators)
+		published := addSingleReleaseWorkflow(build, milestone, version, wd, r.major, r.kind, coordinators)
 
 		securitySummary := wf.Const("")
 		securityFixes := wf.Slice[string]()
 		if r.kind == task.KindCurrentMinor || r.kind == task.KindPrevMinor {
 			securitySummary = wf.Param(wd, securitySummaryParameter)
 			securityFixes = wf.Param(wd, securityFixesParameter)
 		}
-		addCommTasks(wd, build, comm, r.kind, wf.Slice(versionPublished), securitySummary, securityFixes, coordinators)
+		addCommTasks(wd, build, comm, r.kind, wf.Slice(published), securitySummary, securityFixes, coordinators)
 
 		h.RegisterDefinition(fmt.Sprintf("Go 1.%d %s", r.major, r.suffix), wd)
 	}
@@ -343,26 +343,26 @@ func createMinorReleaseWorkflow(build *BuildReleaseTasks, milestone *task.Milest
 	wd := wf.New()
 
 	coordinators := wf.Param(wd, releaseCoordinators)
-	v1Published := addSingleReleaseWorkflow(build, milestone, version, wd.Sub(fmt.Sprintf("Go 1.%d", currentMajor)), currentMajor, task.KindCurrentMinor, coordinators)
-	v2Published := addSingleReleaseWorkflow(build, milestone, version, wd.Sub(fmt.Sprintf("Go 1.%d", prevMajor)), prevMajor, task.KindPrevMinor, coordinators)
+	currPublished := addSingleReleaseWorkflow(build, milestone, version, wd.Sub(fmt.Sprintf("Go 1.%d", currentMajor)), currentMajor, task.KindCurrentMinor, coordinators)
+	prevPublished := addSingleReleaseWorkflow(build, milestone, version, wd.Sub(fmt.Sprintf("Go 1.%d", prevMajor)), prevMajor, task.KindPrevMinor, coordinators)
 
 	securitySummary := wf.Param(wd, securitySummaryParameter)
 	securityFixes := wf.Param(wd, securityFixesParameter)
-	addCommTasks(wd, build, comm, task.KindCurrentMinor, wf.Slice(v1Published, v2Published), securitySummary, securityFixes, coordinators)
+	addCommTasks(wd, build, comm, task.KindCurrentMinor, wf.Slice(currPublished, prevPublished), securitySummary, securityFixes, coordinators)
 
 	return wd, nil
 }
 
 func addCommTasks(
 	wd *wf.Definition, build *BuildReleaseTasks, comm task.CommunicationTasks,
-	kind task.ReleaseKind, versions wf.Value[[]string], securitySummary wf.Value[string], securityFixes, coordinators wf.Value[[]string],
+	kind task.ReleaseKind, published wf.Value[[]task.Published], securitySummary wf.Value[string], securityFixes, coordinators wf.Value[[]string],
 ) {
-	okayToAnnounceAndTweet := wf.Action0(wd, "Wait to Announce", build.ApproveAction, wf.After(versions))
+	okayToAnnounceAndTweet := wf.Action0(wd, "Wait to Announce", build.ApproveAction, wf.After(published))
 
 	// Announce that a new Go release has been published.
-	sentMail := wf.Task4(wd, "mail-announcement", comm.AnnounceRelease, wf.Const(kind), versions, securityFixes, coordinators, wf.After(okayToAnnounceAndTweet))
+	sentMail := wf.Task4(wd, "mail-announcement", comm.AnnounceRelease, wf.Const(kind), published, securityFixes, coordinators, wf.After(okayToAnnounceAndTweet))
 	announcementURL := wf.Task1(wd, "await-announcement", comm.AwaitAnnounceMail, sentMail)
-	tweetURL := wf.Task3(wd, "post-tweet", comm.TweetRelease, versions, securitySummary, announcementURL, wf.After(okayToAnnounceAndTweet))
+	tweetURL := wf.Task3(wd, "post-tweet", comm.TweetRelease, published, securitySummary, announcementURL, wf.After(okayToAnnounceAndTweet))
 
 	wf.Output(wd, "Announcement URL", announcementURL)
 	wf.Output(wd, "Tweet URL", tweetURL)
@@ -379,7 +379,7 @@ func now(_ context.Context) (time.Time, error) {
 func addSingleReleaseWorkflow(
 	build *BuildReleaseTasks, milestone *task.MilestoneTasks, version *task.VersionTasks,
 	wd *wf.Definition, major int, kind task.ReleaseKind, coordinators wf.Value[[]string],
-) (versionPublished wf.Value[string]) {
+) wf.Value[task.Published] {
 	kindVal := wf.Const(kind)
 	branch := fmt.Sprintf("release-branch.go1.%d", major)
 	if kind == task.KindBeta {
@@ -425,14 +425,14 @@ func addSingleReleaseWorkflow(
 	uploadedMods := wf.Action2(wd, "Upload modules to CDN", build.uploadModules, nextVersion, modules, wf.After(tagged))
 	availableOnProxy := wf.Action2(wd, "Wait for modules on proxy.golang.org", build.awaitProxy, nextVersion, modules, wf.After(uploadedMods))
 	pushed := wf.Action3(wd, "Push issues", milestone.PushIssues, milestones, nextVersion, kindVal, wf.After(tagged))
-	versionPublished = wf.Task2(wd, "Publish to website", build.publishArtifacts, nextVersion, signedAndTestedArtifacts, wf.After(uploaded, availableOnProxy, pushed))
+	published := wf.Task2(wd, "Publish to website", build.publishArtifacts, nextVersion, signedAndTestedArtifacts, wf.After(uploaded, availableOnProxy, pushed))
 	if kind == task.KindMajor {
-		goimportsCL := wf.Task2(wd, fmt.Sprintf("Mail goimports CL for 1.%d", major), version.CreateUpdateStdlibIndexCL, coordinators, versionPublished)
+		goimportsCL := wf.Task2(wd, fmt.Sprintf("Mail goimports CL for 1.%d", major), version.CreateUpdateStdlibIndexCL, coordinators, nextVersion, wf.After(published))
 		goimportsCommit := wf.Task2(wd, "Wait for goimports CL submission", version.AwaitCL, goimportsCL, wf.Const(""))
 		wf.Output(wd, "goimports CL submitted", goimportsCommit)
 	}
-	wf.Output(wd, "Released version", versionPublished)
-	return versionPublished
+	wf.Output(wd, "Published to website", published)
+	return published
 }
 
 type moduleArtifact struct {
@@ -1334,11 +1334,12 @@ func uploadFile(scratchFS, servingFS fs.FS, scratch, filename string) error {
 }
 
 // publishArtifacts publishes artifacts for version (typically so they appear at https://go.dev/dl/).
-// It returns version, the Go version that has been successfully published.
-//
-// The version string uses the same format as Go tags. For example, "go1.19rc1".
-func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version string, artifacts []artifact) (publishedVersion string, _ error) {
-	for _, a := range artifacts {
+// It returns the Go version and files that have been successfully published.
+func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version string, artifacts []artifact) (task.Published, error) {
+	// Each release artifact corresponds to a single website file.
+	var files = make([]task.WebsiteFile, len(artifacts))
+	for i, a := range artifacts {
+		// Define website file metadata.
 		f := task.WebsiteFile{
 			Filename:       a.Filename,
 			Version:        version,
@@ -1360,12 +1361,16 @@ func (tasks *BuildReleaseTasks) publishArtifacts(ctx *wf.TaskContext, version st
 		case "msi", "pkg":
 			f.Kind = "installer"
 		}
+
+		// Publish it.
 		if err := tasks.PublishFile(f); err != nil {
-			return "", err
+			return task.Published{}, err
 		}
+		ctx.Printf("Published %q.", f.Filename)
+		files[i] = f
 	}
-	ctx.Printf("Published %v artifacts for %v", len(artifacts), version)
-	return version, nil
+	ctx.Printf("Published all %d files for %s.", len(files), version)
+	return task.Published{Version: version, Files: files}, nil
 }
 
 // cutPrefix returns s without the provided leading prefix string

internal/task/announce.go

@@ -131,13 +131,14 @@ type SentMail struct {
 	Subject string // Subject of the email. Expected to be unique so it can be used to identify the email.
 }
 
-// AnnounceRelease sends an email announcing a Go release to Google Groups.
-func (t AnnounceMailTasks) AnnounceRelease(ctx *workflow.TaskContext, kind ReleaseKind, versions []string, security []string, users []string) (SentMail, error) {
+// AnnounceRelease sends an email to Google Groups
+// announcing that a Go release has been published.
+func (t AnnounceMailTasks) AnnounceRelease(ctx *workflow.TaskContext, kind ReleaseKind, published []Published, security []string, users []string) (SentMail, error) {
 	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
 		return SentMail{}, fmt.Errorf("insufficient time for announce release task; a minimum of a minute left on context is required")
 	}
-	if err := oneOrTwoGoVersions(versions); err != nil {
-		return SentMail{}, err
+	if len(published) < 1 || len(published) > 2 {
+		return SentMail{}, fmt.Errorf("got %d published Go releases, AnnounceRelease supports only 1 or 2 at once", len(published))
 	}
 	names, err := coordinatorFirstNames(users)
 	if err != nil {
@@ -146,12 +147,12 @@ func (t AnnounceMailTasks) AnnounceRelease(ctx *workflow.TaskContext, kind Relea
 
 	r := releaseAnnouncement{
 		Kind:     kind,
-		Version:  versions[0],
+		Version:  published[0].Version,
 		Security: security,
 		Names:    names,
 	}
-	if len(versions) == 2 {
-		r.SecondaryVersion = versions[1]
+	if len(published) == 2 {
+		r.SecondaryVersion = published[1].Version
 	}
 
 	// Generate the announcement email.
@@ -198,8 +199,8 @@ func (t AnnounceMailTasks) PreAnnounceRelease(ctx *workflow.TaskContext, version
 	if deadline, ok := ctx.Deadline(); ok && time.Until(deadline) < time.Minute {
 		return SentMail{}, fmt.Errorf("insufficient time for pre-announce release task; a minimum of a minute left on context is required")
 	}
-	if err := oneOrTwoGoVersions(versions); err != nil {
-		return SentMail{}, err
+	if len(versions) < 1 || len(versions) > 2 {
+		return SentMail{}, fmt.Errorf("got %d planned Go releases, PreAnnounceRelease supports only 1 or 2 at once", len(versions))
 	}
 	now := time.Now().UTC()
 	if t.testHookNow != nil {

internal/task/announce_test.go

@@ -24,7 +24,7 @@ import (
 func TestAnnounceReleaseShortContext(t *testing.T) {
 	ctx, cancel := context.WithTimeout(context.Background(), time.Second)
 	defer cancel()
-	_, err := (AnnounceMailTasks{}).AnnounceRelease(&workflow.TaskContext{Context: ctx}, KindCurrentMinor, []string{"go1.18.1", "go1.17.8"}, nil, nil)
+	_, err := (AnnounceMailTasks{}).AnnounceRelease(&workflow.TaskContext{Context: ctx}, KindCurrentMinor, []Published{{Version: "go1.18.1"}, {Version: "go1.17.8"}}, nil, nil)
 	if err == nil {
 		t.Errorf("want non-nil error")
 	} else if !strings.HasPrefix(err.Error(), "insufficient time") {
@@ -195,7 +195,7 @@ func TestAnnounceRelease(t *testing.T) {
 	tests := [...]struct {
 		name         string
 		kind         ReleaseKind
-		versions     []string
+		published    []Published
 		security     []string
 		coordinators []string
 		want         SentMail
@@ -204,7 +204,7 @@ func TestAnnounceRelease(t *testing.T) {
 		{
 			name:         "minor",
 			kind:         KindCurrentMinor,
-			versions:     []string{"go1.18.1", "go1.17.8"}, // Intentionally not 1.17.9 so the real email doesn't get in the way.
+			published:    []Published{{Version: "go1.18.1"}, {Version: "go1.17.8"}}, // Intentionally not 1.17.9 so the real email doesn't get in the way.
 			coordinators: []string{"heschi", "dmitshur"},
 			want:         SentMail{Subject: "Go 1.18.1 and Go 1.17.8 are released"},
 			wantLog: `announcement subject: Go 1.18.1 and Go 1.17.8 are released
@@ -264,7 +264,7 @@ Heschi and Dmitri for the Go team` + "\n",
 			}
 			var buf bytes.Buffer
 			ctx := &workflow.TaskContext{Context: context.Background(), Logger: fmtWriter{&buf}}
-			sentMail, err := tasks.AnnounceRelease(ctx, tc.kind, tc.versions, tc.security, tc.coordinators)
+			sentMail, err := tasks.AnnounceRelease(ctx, tc.kind, tc.published, tc.security, tc.coordinators)
 			if err != nil {
 				if fe := (fetchError{}); errors.As(err, &fe) && fe.PossiblyRetryable {
 					t.Skip("test run produced no actionable signal due to a transient network error:", err) // See go.dev/issue/60541.

internal/task/dlcl.go

@@ -63,24 +63,6 @@ func (t *VersionTasks) MailDLCL(ctx *workflow.TaskContext, major int, kind Relea
 	return t.Gerrit.CreateAutoSubmitChange(ctx, changeInput, reviewers, files)
 }
 
-// oneOrTwoGoVersions returns true iff len(versions) is exactly 1 or 2
-// and each version passes some lightweight checks that catch problems.
-func oneOrTwoGoVersions(versions []string) error {
-	if len(versions) < 1 || len(versions) > 2 {
-		return fmt.Errorf("got %d Go versions, want 1 or 2", len(versions))
-	}
-	for _, ver := range versions {
-		if ver != strings.ToLower(ver) {
-			return fmt.Errorf("version %q is not lowercase", ver)
-		} else if strings.Contains(ver, " ") {
-			return fmt.Errorf("version %q contains a space", ver)
-		} else if !strings.HasPrefix(ver, "go") {
-			return fmt.Errorf("version %q doesn't have the 'go' prefix", ver)
-		}
-	}
-	return nil
-}
-
 func docLink(major int, kind ReleaseKind, ver string) string {
 	if kind == KindCurrentMinor || kind == KindPrevMinor {
 		return fmt.Sprintf("https://go.dev/doc/devel/release#%v", ver)

internal/task/task.go

@@ -14,6 +14,18 @@ import (
 	wf "golang.org/x/build/internal/workflow"
 )
 
+// Published holds information for a Go release
+// that by this time has already been published.
+//
+// Published in this context refers to it being
+// available for download at https://go.dev/dl/.
+// It doesn't mean it has been announced by now;
+// that step happens sometime after publication.
+type Published struct {
+	Version string        // Version that's published, in the same format as Go tags. For example, "go1.21rc1".
+	Files   []WebsiteFile // Files that are published.
+}
+
 // CommunicationTasks combines communication tasks together.
 type CommunicationTasks struct {
 	AnnounceMailTasks

internal/task/tweet.go

@@ -84,20 +84,20 @@ type TweetTasks struct {
 	RandomSeed int64
 }
 
-// TweetRelease posts a tweet announcing a Go release.
+// TweetRelease posts a tweet announcing that a Go release has been published.
 // ErrTweetTooLong is returned if the inputs result in a tweet that's too long.
-func (t TweetTasks) TweetRelease(ctx *workflow.TaskContext, versions []string, security string, announcement string) (tweetURL string, _ error) {
-	if err := oneOrTwoGoVersions(versions); err != nil {
-		return "", err
+func (t TweetTasks) TweetRelease(ctx *workflow.TaskContext, published []Published, security string, announcement string) (tweetURL string, _ error) {
+	if len(published) < 1 || len(published) > 2 {
+		return "", fmt.Errorf("got %d published Go releases, TweetRelease supports only 1 or 2 at once", len(published))
 	}
 
 	r := releaseTweet{
-		Version:      versions[0],
+		Version:      published[0].Version,
 		Security:     security,
 		Announcement: announcement,
 	}
-	if len(versions) == 2 {
-		r.SecondaryVersion = versions[1]
+	if len(published) == 2 {
+		r.SecondaryVersion = published[1].Version
 	}
 
 	seed := t.RandomSeed
@@ -114,7 +114,7 @@ func (t TweetTasks) TweetRelease(ctx *workflow.TaskContext, versions []string, s
 	ctx.Printf("tweet text:\n%s\n", tweetText)
 
 	// Generate tweet image.
-	imagePNG, imageText, err := tweetImage(r.Version, rnd)
+	imagePNG, imageText, err := tweetImage(published[0], rnd)
 	if err != nil {
 		return "", err
 	}
@@ -320,19 +320,18 @@ var emoji = map[string][]string{
 }
 
 // tweetImage generates an image to use in the announcement
-// tweet for goVersion. It returns the image encoded as PNG,
+// tweet for published. It returns the image encoded as PNG,
 // and the text displayed in the image.
 //
-// tweetImage makes an HTTP GET request to the go.dev/dl/?mode=json
-// read-only API to select a random release archive to highlight.
-func tweetImage(goVersion string, rnd *rand.Rand) (imagePNG []byte, imageText string, _ error) {
-	a, err := fetchRandomArchive(goVersion, rnd)
+// tweetImage selects a random release archive to highlight.
+func tweetImage(published Published, rnd *rand.Rand) (imagePNG []byte, imageText string, _ error) {
+	a, err := pickRandomArchive(published, rnd)
 	if err != nil {
 		return nil, "", err
 	}
 	var buf bytes.Buffer
 	if err := goCmdTmpl.Execute(&buf, map[string]string{
-		"GoVer":    goVersion,
+		"GoVer":    published.Version,
 		"GOOS":     a.OS,
 		"GOARCH":   a.GOARCH(),
 		"Filename": a.Filename,
@@ -375,57 +374,22 @@ func digits(i int64) int {
 	return n
 }
 
-// fetchRandomArchive downloads all release archives for Go version goVer,
-// and selects a random archive to showcase in the image that displays
-// sample output from the 'go install golang.org/dl/...@latest' command.
-func fetchRandomArchive(goVer string, rnd *rand.Rand) (archive WebsiteFile, _ error) {
-	archives, err := fetchReleaseArchives(goVer)
-	if err != nil {
-		return WebsiteFile{}, err
-	}
-	return archives[rnd.Intn(len(archives))], nil
-}
-
-func fetchReleaseArchives(goVer string) (archives []WebsiteFile, _ error) {
-	url := "https://go.dev/dl/?mode=json"
-	if strings.Contains(goVer, "beta") || strings.Contains(goVer, "rc") ||
-		goVer == "go1.17" || goVer == "go1.17.1" || goVer == "go1.11.1" /* For TestTweetRelease. */ {
-
-		url += "&include=all"
-	}
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, err
-	}
-	defer resp.Body.Close()
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("non-200 OK status code: %v", resp.Status)
-	} else if ct := resp.Header.Get("Content-Type"); ct != "application/json" {
-		return nil, fmt.Errorf("got Content-Type %q, want %q", ct, "application/json")
-	}
-	var releases []WebsiteRelease
-	err = json.NewDecoder(resp.Body).Decode(&releases)
-	if err != nil {
-		return nil, err
-	}
-	for _, r := range releases {
-		if r.Version != goVer {
+// pickRandomArchive picks one random release archive
+// to showcase in an image showing sample output from
+// 'go install golang.org/dl/...@latest'.
+func pickRandomArchive(published Published, rnd *rand.Rand) (archive WebsiteFile, _ error) {
+	var archives []WebsiteFile
+	for _, f := range published.Files {
+		if f.Kind != "archive" {
+			// Not an archive type of file, skip. The golang.org/dl commands use archives only.
 			continue
 		}
-		var archives []WebsiteFile
-		for _, f := range r.Files {
-			if f.Kind != "archive" {
-				continue
-			}
-			archives = append(archives, f)
-		}
-		if len(archives) == 0 {
-			return nil, fmt.Errorf("release version %q has 0 archive files", goVer)
-		}
-		// Return archives.
-		return archives, nil
+		archives = append(archives, f)
 	}
-	return nil, fmt.Errorf("release version %q not found", goVer)
+	if len(archives) == 0 {
+		return WebsiteFile{}, fmt.Errorf("release version %q has 0 archive files", published.Version)
+	}
+	return archives[rnd.Intn(len(archives))], nil
 }
 
 // drawTerminal draws an image of a terminal window