internal/relui: wait for module files to appear on proxy.golang.org

At some point the modules will be a key part of the release, just like
the website entries are now. Make sure that they're available on
proxy.golang.org before the release is announced.

For golang/go#58659.

Change-Id: I744c067f6efa06f3ca4cdbaf40a351f583331c16
Reviewed-on: https://go-review.googlesource.com/c/build/+/489415
Run-TryBot: Heschi Kreinick <[email protected]>
TryBot-Result: Gopher Robot <[email protected]>
Auto-Submit: Heschi Kreinick <[email protected]>
Reviewed-by: Carlos Amedee <[email protected]>
Reviewed-by: Dmitri Shuralyov <[email protected]>

Author: heschi

cmd/relui/main.go

@@ -177,6 +177,7 @@ func main() {
 		ScratchURL:       *scratchFilesBase,
 		ServingURL:       *servingFilesBase,
 		DownloadURL:      *edgeCacheURL,
+		ProxyPrefix:      "https://proxy.golang.org/golang.org/toolchain/@v",
 		PublishFile: func(f *task.WebsiteFile) error {
 			return publishFile(*websiteUploadURL, userPassAuth, f)
 		},

internal/relui/buildrelease_test.go

@@ -220,6 +220,7 @@ esac
 		CreateBuildlet:   fakeBuildlets.CreateBuildlet,
 		SignService:      task.NewFakeSignService(t),
 		DownloadURL:      dlServer.URL,
+		ProxyPrefix:      dlServer.URL,
 		PublishFile:      publishFile,
 		ApproveAction: func(ctx *workflow.TaskContext) error {
 			if strings.Contains(ctx.TaskName, "Release Coordinator Approval") {

internal/relui/workflows.go

@@ -423,8 +423,9 @@ func addSingleReleaseWorkflow(
 	tagged := wf.Action2(wd, "Tag version", version.TagRelease, nextVersion, tagCommit, wf.After(okayToTagAndPublish))
 	uploaded := wf.Action1(wd, "Upload artifacts to CDN", build.uploadArtifacts, signedAndTestedArtifacts, wf.After(tagged))
 	uploadedMods := wf.Action2(wd, "Upload modules to CDN", build.uploadModules, nextVersion, modules, wf.After(tagged))
+	availableOnProxy := wf.Action2(wd, "Wait for modules on proxy.golang.org", build.awaitProxy, nextVersion, modules, wf.After(uploadedMods))
 	pushed := wf.Action3(wd, "Push issues", milestone.PushIssues, milestones, nextVersion, kindVal, wf.After(tagged))
-	versionPublished = wf.Task2(wd, "Publish to website", build.publishArtifacts, nextVersion, signedAndTestedArtifacts, wf.After(uploaded, uploadedMods, pushed))
+	versionPublished = wf.Task2(wd, "Publish to website", build.publishArtifacts, nextVersion, signedAndTestedArtifacts, wf.After(uploaded, availableOnProxy, pushed))
 	if kind == task.KindMajor {
 		goimportsCL := wf.Task2(wd, fmt.Sprintf("Mail goimports CL for 1.%d", major), version.CreateUpdateStdlibIndexCL, coordinators, versionPublished)
 		goimportsCommit := wf.Task2(wd, "Wait for goimports CL submission", version.AwaitCL, goimportsCL, wf.Const(""))
@@ -559,6 +560,7 @@ type BuildReleaseTasks struct {
 	GCSClient              *storage.Client
 	ScratchURL, ServingURL string // ScratchURL is a gs:// or file:// URL, no trailing slash. E.g., "gs://golang-release-staging/relui-scratch".
 	DownloadURL            string
+	ProxyPrefix            string // ProxyPrefix is the prefix at which module files are published, e.g. https://proxy.golang.org/golang.org/toolchain/@v
 	PublishFile            func(*task.WebsiteFile) error
 	CreateBuildlet         func(context.Context, string) (buildlet.RemoteClient, error)
 	SignService            sign.Service
@@ -1237,7 +1239,7 @@ func (tasks *BuildReleaseTasks) uploadModules(ctx *wf.TaskContext, version strin
 	}
 	want := map[string]bool{} // URLs we're waiting on becoming available.
 	for _, mod := range modules {
-		base := fmt.Sprintf("v0.0.1-%v.%v-%v", version, mod.Target.GOOS, mod.Target.GOARCH)
+		base := task.ToolchainModuleVersion(mod.Target, version)
 		if err := uploadFile(scratchFS, servingFS, mod.ZipScratch, fmt.Sprintf(base+".zip")); err != nil {
 			return err
 		}
@@ -1255,6 +1257,16 @@ func (tasks *BuildReleaseTasks) uploadModules(ctx *wf.TaskContext, version strin
 	return err
 }
 
+func (tasks *BuildReleaseTasks) awaitProxy(ctx *wf.TaskContext, version string, modules []moduleArtifact) error {
+	want := map[string]bool{}
+	for _, mod := range modules {
+		url := fmt.Sprintf("%v/%v.info", tasks.ProxyPrefix, task.ToolchainModuleVersion(mod.Target, version))
+		want[url] = true
+	}
+	_, err := task.AwaitCondition(ctx, 30*time.Second, checkFiles(ctx, want))
+	return err
+}
+
 func checkFiles(ctx context.Context, want map[string]bool) func() (int, bool, error) {
 	found := map[string]bool{}
 	return func() (int, bool, error) {

internal/task/dl2mod.go

@@ -25,17 +25,17 @@ const (
 )
 
 func ToolchainZipPrefix(target *releasetargets.Target, version string) string {
-	return modulePath + "@" + toolchainModuleVersion(target, version)
+	return modulePath + "@" + ToolchainModuleVersion(target, version)
 }
 
-func toolchainModuleVersion(target *releasetargets.Target, version string) string {
+func ToolchainModuleVersion(target *releasetargets.Target, version string) string {
 	return fmt.Sprintf("%v-%v.%v-%v", moduleVersion, version, target.GOOS, target.GOARCH)
 }
 
 // TarToModFiles converts the distribution archive with the given name and content
 // to a collection of module files.
 func TarToModFiles(target *releasetargets.Target, version string, t time.Time, tgz io.Reader, w io.Writer) (mod string, info string, _ error) {
-	vers := toolchainModuleVersion(target, version)
+	vers := ToolchainModuleVersion(target, version)
 	zipPrefix := ToolchainZipPrefix(target, version)
 
 	// rename takes the name of a file found in a distribution archive