cmd/go/internal/modindex: ignore non-source files for index

We were saving non-go file information in the module index files,
leading in an unnecessary increase in memory usage in modules
containing many non-go files. This was a bug because this information
is never used. Don't save that information.

For #54226

Change-Id: I0644064f83f96e3a9f43b7e66ca94d69d9603376
Reviewed-on: https://go-review.googlesource.com/c/go/+/439118
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>
Run-TryBot: Michael Matloob <[email protected]>
Reviewed-by: Bryan Mills <[email protected]>

Author: matloob

src/cmd/go/internal/modindex/build.go

@@ -477,13 +477,18 @@ type fileEmbed struct {
 	pos     token.Position
 }
 
+var errNonSource = errors.New("non source file")
+
 // getFileInfo extracts the information needed from each go file for the module
 // index.
 //
 // If Name denotes a Go program, matchFile reads until the end of the
 // Imports and returns that section of the file in the FileInfo's Header field,
 // even though it only considers text until the first non-comment
 // for +build lines.
+//
+// getFileInfo will return errNonSource if the file is not a source or object
+// file and shouldn't even be added to IgnoredFiles.
 func getFileInfo(dir, name string, fset *token.FileSet) (*fileInfo, error) {
 	if strings.HasPrefix(name, "_") ||
 		strings.HasPrefix(name, ".") {
@@ -498,7 +503,7 @@ func getFileInfo(dir, name string, fset *token.FileSet) (*fileInfo, error) {
 
 	if ext != ".go" && fileListForExt(&dummyPkg, ext) == nil {
 		// skip
-		return nil, nil
+		return nil, errNonSource
 	}
 
 	info := &fileInfo{name: filepath.Join(dir, name), fset: fset}

src/cmd/go/internal/modindex/index_test.go

@@ -85,3 +85,20 @@ func TestIndex(t *testing.T) {
 		}
 	})
 }
+
+func TestImportRaw_IgnoreNonGo(t *testing.T) {
+	path := filepath.Join("testdata", "ignore_non_source")
+	p := importRaw(path, ".")
+
+	wantFiles := []string{"a.syso", "b.go", "c.c"}
+
+	var gotFiles []string
+	for i := range p.sourceFiles {
+		gotFiles = append(gotFiles, p.sourceFiles[i].name)
+	}
+
+	if !reflect.DeepEqual(gotFiles, wantFiles) {
+		t.Errorf("names of files in importRaw(testdata/ignore_non_source): got %v; want %v",
+			gotFiles, wantFiles)
+	}
+}

src/cmd/go/internal/modindex/scan.go

@@ -210,7 +210,10 @@ func importRaw(modroot, reldir string) *rawPackage {
 			continue
 		}
 		info, err := getFileInfo(absdir, name, fset)
-		if err != nil {
+		if err == errNonSource {
+			// not a source or object file. completely ignore in the index
+			continue
+		} else if err != nil {
 			p.sourceFiles = append(p.sourceFiles, &rawFile{name: name, error: err.Error()})
 			continue
 		} else if info == nil {

src/cmd/go/internal/modindex/testdata/ignore_non_source/a.syso

@@ -0,0 +1 @@
+package ignore_non_source