internal/fetch: collapse identical build contexts

If all the build contexts for a package produce the same documentation,
then return a single Documentation with build context all/all.

This will save a lot of space in the documentation table, since
most packages will fit into this case.

For golang/go#37232

Change-Id: I15237242e0c3ca3c7b8f8c8944b227afebd23785
Reviewed-on: https://go-review.googlesource.com/c/pkgsite/+/289680
Trust: Jonathan Amsterdam <[email protected]>
Run-TryBot: Jonathan Amsterdam <[email protected]>
TryBot-Result: kokoro <[email protected]>
Reviewed-by: Julie Qiu <[email protected]>

Author: jba

internal/fetch/fetchdata_test.go

@@ -315,9 +315,28 @@ var moduleBuildConstraints = &testModule{
 						Name: "cpu",
 						Path: "build.constraints/module/cpu",
 					},
-					Documentation: []*internal.Documentation{{
-						Synopsis: "Package cpu implements processor feature detection used by the Go standard library.",
-					}},
+					Documentation: []*internal.Documentation{
+						{
+							GOOS:     "linux",
+							GOARCH:   "amd64",
+							Synopsis: "Package cpu implements processor feature detection used by the Go standard library.",
+						},
+						{
+							GOOS:     "windows",
+							GOARCH:   "amd64",
+							Synopsis: "Package cpu implements processor feature detection used by the Go standard library.",
+						},
+						{
+							GOOS:     "darwin",
+							GOARCH:   "amd64",
+							Synopsis: "Package cpu implements processor feature detection used by the Go standard library.",
+						},
+						{
+							GOOS:     "js",
+							GOARCH:   "wasm",
+							Synopsis: "Package cpu implements processor feature detection used by the Go standard library.",
+						},
+					},
 				},
 			},
 		},
@@ -609,7 +628,6 @@ var moduleWasm = &testModule{
 	mod: &proxy.Module{
 		ModulePath: "github.com/my/module/js",
 		Files: map[string]string{
-
 			"README.md": "THIS IS A README",
 			"LICENSE":   testhelper.BSD0License,
 			"js/js.go": `
@@ -666,6 +684,8 @@ var moduleStd = &testModule{
 	mod: &proxy.Module{
 		ModulePath: stdlib.ModulePath,
 		Version:    "v1.12.5",
+		// No files necessary because the internal/stdlib package will read from
+		// internal/stdlib/testdata.
 	},
 	fr: &FetchResult{
 		Module: &internal.Module{
@@ -679,8 +699,7 @@ var moduleStd = &testModule{
 			Units: []*internal.Unit{
 				{
 					UnitMeta: internal.UnitMeta{
-						Path: "std",
-
+						Path:              "std",
 						IsRedistributable: true,
 					},
 					Readme: &internal.Readme{
@@ -713,9 +732,31 @@ var moduleStd = &testModule{
 						Filepath: "cmd/pprof/README",
 						Contents: "This directory is the copy of Google's pprof shipped as part of the Go distribution.\n",
 					},
-					Documentation: []*internal.Documentation{{
-						Synopsis: "Pprof interprets and displays profiles of Go programs.",
-					}},
+					// cmd/pprof has a file with a build constraint that does not include js/wasm.
+					// Since the set files isn't the same across all build contexts, we represent
+					// every build context.
+					Documentation: []*internal.Documentation{
+						{
+							GOOS:     "linux",
+							GOARCH:   "amd64",
+							Synopsis: "Pprof interprets and displays profiles of Go programs.",
+						},
+						{
+							GOOS:     "windows",
+							GOARCH:   "amd64",
+							Synopsis: "Pprof interprets and displays profiles of Go programs.",
+						},
+						{
+							GOOS:     "darwin",
+							GOARCH:   "amd64",
+							Synopsis: "Pprof interprets and displays profiles of Go programs.",
+						},
+						{
+							GOOS:     "js",
+							GOARCH:   "wasm",
+							Synopsis: "Pprof interprets and displays profiles of Go programs.",
+						},
+					},
 					Imports: []string{
 						"cmd/internal/objfile",
 						"crypto/tls",

internal/fetch/helper_test.go

@@ -73,12 +73,6 @@ func cleanFetchResult(t *testing.T, fr *FetchResult, detector *licenses.Detector
 			IsRedistributable: u.IsRedistributable,
 			Licenses:          u.Licenses,
 		}
-		if len(u.Documentation) > 0 {
-			if u.Documentation[0].GOOS == "" {
-				u.Documentation[0].GOOS = "linux"
-				u.Documentation[0].GOARCH = "amd64"
-			}
-		}
 		if u.IsPackage() && shouldSetPVS {
 			fr.PackageVersionStates = append(
 				fr.PackageVersionStates, &internal.PackageVersionState{

internal/fetch/load.go

@@ -103,8 +103,9 @@ func loadPackage(ctx context.Context, zipGoFiles []*zip.File, innerPath string,
 			// No package for this build context.
 			continue
 		case errors.Is(err, godoc.ErrTooLarge):
-			// The doc for this build context is too large. Remember that and
-			// return the package for this build context; ignore the others.
+			// The doc for this build context is too large. To keep things
+			// simple, return a single package with this error that will be used
+			// for all build contexts, and ignore the others.
 			return &goPackage{
 				err:     err,
 				path:    importPath,
@@ -162,6 +163,7 @@ func loadPackage(ctx context.Context, zipGoFiles []*zip.File, innerPath string,
 
 // mapKeyForFiles generates a value that corresponds to the given set of file
 // names and can be used as a map key.
+// It assumes the filenames do not contain spaces.
 func mapKeyForFiles(files map[string][]byte) string {
 	var names []string
 	for n := range files {

internal/fetch/unit.go

@@ -61,8 +61,7 @@ func moduleUnits(modulePath, version string,
 		if pkg, ok := pkgLookup[dirPath]; ok {
 			dir.Name = pkg.name
 			dir.Imports = pkg.imports
-			// TODO(golang/go#37232): keep all docs
-			dir.Documentation = pkg.docs[:1]
+			dir.Documentation = pkg.docs
 		}
 		units = append(units, dir)
 	}