go/internal/gcimporter: update to anticipate missing targets and .as

This cl updates go/internal/gcimporter the to anticiapte missing
targets and .a files the same way CL 442303 updates the two other
versions of gcimporter to do the same. It also adds a couple of
helpers to create importcfg files for the compiler and list the
locations of cached stdlib .a files in internal/goroot and
internal/testenv, the analogues of their import paths in the go
distribution.

Change-Id: Ie207882c13df0e886a51d31e7957a1e508331f10
Reviewed-on: https://go-review.googlesource.com/c/tools/+/445455
TryBot-Result: Gopher Robot <[email protected]>
Reviewed-by: Bryan Mills <[email protected]>
Run-TryBot: Michael Matloob <[email protected]>
gopls-CI: kokoro <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>
(cherry picked from commit e4bb343)
Reviewed-on: https://go-review.googlesource.com/c/tools/+/446858
Reviewed-by: Robert Findley <[email protected]>

Author: matloob

go/internal/gcimporter/gcimporter.go

@@ -22,11 +22,14 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"path"
 	"path/filepath"
 	"sort"
 	"strconv"
 	"strings"
 	"text/scanner"
+
+	"golang.org/x/tools/internal/goroot"
 )
 
 const (
@@ -38,6 +41,25 @@ const (
 	trace = false
 )
 
+func lookupGorootExport(pkgpath, srcRoot, srcDir string) (string, bool) {
+	pkgpath = filepath.ToSlash(pkgpath)
+	m, err := goroot.PkgfileMap()
+	if err != nil {
+		return "", false
+	}
+	if export, ok := m[pkgpath]; ok {
+		return export, true
+	}
+	vendorPrefix := "vendor"
+	if strings.HasPrefix(srcDir, filepath.Join(srcRoot, "cmd")) {
+		vendorPrefix = path.Join("cmd", vendorPrefix)
+	}
+	pkgpath = path.Join(vendorPrefix, pkgpath)
+	fmt.Fprintln(os.Stderr, "looking up ", pkgpath)
+	export, ok := m[pkgpath]
+	return export, ok
+}
+
 var pkgExts = [...]string{".a", ".o"}
 
 // FindPkg returns the filename and unique package id for an import
@@ -60,11 +82,18 @@ func FindPkg(path, srcDir string) (filename, id string) {
 		}
 		bp, _ := build.Import(path, srcDir, build.FindOnly|build.AllowBinary)
 		if bp.PkgObj == "" {
-			id = path // make sure we have an id to print in error message
-			return
+			var ok bool
+			if bp.Goroot {
+				filename, ok = lookupGorootExport(path, bp.SrcRoot, srcDir)
+			}
+			if !ok {
+				id = path // make sure we have an id to print in error message
+				return
+			}
+		} else {
+			noext = strings.TrimSuffix(bp.PkgObj, ".a")
+			id = bp.ImportPath
 		}
-		noext = strings.TrimSuffix(bp.PkgObj, ".a")
-		id = bp.ImportPath
 
 	case build.IsLocalImport(path):
 		// "./x" -> "/this/directory/x.ext", "/this/directory/x"
@@ -85,6 +114,12 @@ func FindPkg(path, srcDir string) (filename, id string) {
 		}
 	}
 
+	if filename != "" {
+		if f, err := os.Stat(filename); err == nil && !f.IsDir() {
+			return
+		}
+	}
+
 	// try extensions
 	for _, ext := range pkgExts {
 		filename = noext + ext

go/internal/gcimporter/gcimporter_test.go

@@ -44,25 +44,31 @@ func needsCompiler(t *testing.T, compiler string) {
 
 // compile runs the compiler on filename, with dirname as the working directory,
 // and writes the output file to outdirname.
-func compile(t *testing.T, dirname, filename, outdirname string) string {
-	return compilePkg(t, dirname, filename, outdirname, "p")
+// compile gives the resulting package a packagepath of p.
+func compile(t *testing.T, dirname, filename, outdirname string, packagefiles map[string]string) string {
+	return compilePkg(t, dirname, filename, outdirname, packagefiles, "p")
 }
 
-func compilePkg(t *testing.T, dirname, filename, outdirname, pkg string) string {
+func compilePkg(t *testing.T, dirname, filename, outdirname string, packagefiles map[string]string, pkg string) string {
 	testenv.NeedsGoBuild(t)
 
 	// filename must end with ".go"
-	if !strings.HasSuffix(filename, ".go") {
+	basename := strings.TrimSuffix(filepath.Base(filename), ".go")
+	ok := filename != basename
+	if !ok {
 		t.Fatalf("filename doesn't end in .go: %s", filename)
 	}
-	basename := filepath.Base(filename)
-	outname := filepath.Join(outdirname, basename[:len(basename)-2]+"o")
-	cmd := exec.Command("go", "tool", "compile", "-p="+pkg, "-o", outname, filename)
+	objname := basename + ".o"
+	outname := filepath.Join(outdirname, objname)
+	importcfgfile := filepath.Join(outdirname, basename) + ".importcfg"
+	testenv.WriteImportcfg(t, importcfgfile, packagefiles)
+	importreldir := strings.ReplaceAll(outdirname, string(os.PathSeparator), "/")
+	cmd := exec.Command("go", "tool", "compile", "-p", pkg, "-D", importreldir, "-importcfg", importcfgfile, "-o", outname, filename)
 	cmd.Dir = dirname
 	out, err := cmd.CombinedOutput()
 	if err != nil {
 		t.Logf("%s", out)
-		t.Fatalf("(cd %v && %v) failed: %s", cmd.Dir, cmd, err)
+		t.Fatalf("go tool compile %s failed: %s", filename, err)
 	}
 	return outname
 }
@@ -129,7 +135,7 @@ func TestImportTestdata(t *testing.T) {
 	tmpdir := mktmpdir(t)
 	defer os.RemoveAll(tmpdir)
 
-	compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata"))
+	compile(t, "testdata", testfile, filepath.Join(tmpdir, "testdata"), nil)
 
 	// filename should end with ".go"
 	filename := testfile[:len(testfile)-3]
@@ -459,8 +465,8 @@ func TestIssue13566(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	compilePkg(t, "testdata", "a.go", testoutdir, apkg(testoutdir))
-	compile(t, testoutdir, bpath, testoutdir)
+	compilePkg(t, "testdata", "a.go", testoutdir, nil, apkg(testoutdir))
+	compile(t, testoutdir, bpath, testoutdir, map[string]string{apkg(testoutdir): filepath.Join(testoutdir, "a.o")})
 
 	// import must succeed (test for issue at hand)
 	pkg := importPkg(t, "./testdata/b", tmpdir)
@@ -528,7 +534,7 @@ func TestIssue15517(t *testing.T) {
 	tmpdir := mktmpdir(t)
 	defer os.RemoveAll(tmpdir)
 
-	compile(t, "testdata", "p.go", filepath.Join(tmpdir, "testdata"))
+	compile(t, "testdata", "p.go", filepath.Join(tmpdir, "testdata"), nil)
 
 	// Multiple imports of p must succeed without redeclaration errors.
 	// We use an import path that's not cleaned up so that the eventual
@@ -619,8 +625,8 @@ func TestIssue51836(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	compilePkg(t, dir, "a.go", testoutdir, apkg(testoutdir))
-	compile(t, testoutdir, bpath, testoutdir)
+	compilePkg(t, dir, "a.go", testoutdir, nil, apkg(testoutdir))
+	compile(t, testoutdir, bpath, testoutdir, map[string]string{apkg(testoutdir): filepath.Join(testoutdir, "a.o")})
 
 	// import must succeed (test for issue at hand)
 	_ = importPkg(t, "./testdata/aa", tmpdir)
@@ -646,7 +652,7 @@ func importPkg(t *testing.T, path, srcDir string) *types.Package {
 func compileAndImportPkg(t *testing.T, name string) *types.Package {
 	tmpdir := mktmpdir(t)
 	defer os.RemoveAll(tmpdir)
-	compile(t, "testdata", name+".go", filepath.Join(tmpdir, "testdata"))
+	compile(t, "testdata", name+".go", filepath.Join(tmpdir, "testdata"), nil)
 	return importPkg(t, "./testdata/"+name, tmpdir)
 }
 

internal/goroot/importcfg.go

@@ -0,0 +1,71 @@
+// Copyright 2022 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package goroot is a copy of package internal/goroot
+// in the main GO repot. It provides a utility to produce
+// an importcfg and import path to package file map mapping
+// standard library packages to the locations of their export
+// data files.
+package goroot
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"strings"
+	"sync"
+)
+
+// Importcfg returns an importcfg file to be passed to the
+// Go compiler that contains the cached paths for the .a files for the
+// standard library.
+func Importcfg() (string, error) {
+	var icfg bytes.Buffer
+
+	m, err := PkgfileMap()
+	if err != nil {
+		return "", err
+	}
+	fmt.Fprintf(&icfg, "# import config")
+	for importPath, export := range m {
+		if importPath != "unsafe" && export != "" { // unsafe
+			fmt.Fprintf(&icfg, "\npackagefile %s=%s", importPath, export)
+		}
+	}
+	s := icfg.String()
+	return s, nil
+}
+
+var (
+	stdlibPkgfileMap map[string]string
+	stdlibPkgfileErr error
+	once             sync.Once
+)
+
+// PkgfileMap returns a map of package paths to the location on disk
+// of the .a file for the package.
+// The caller must not modify the map.
+func PkgfileMap() (map[string]string, error) {
+	once.Do(func() {
+		m := make(map[string]string)
+		output, err := exec.Command("go", "list", "-export", "-e", "-f", "{{.ImportPath}} {{.Export}}", "std", "cmd").Output()
+		if err != nil {
+			stdlibPkgfileErr = err
+		}
+		for _, line := range strings.Split(string(output), "\n") {
+			if line == "" {
+				continue
+			}
+			sp := strings.SplitN(line, " ", 2)
+			if len(sp) != 2 {
+				err = fmt.Errorf("determining pkgfile map: invalid line in go list output: %q", line)
+				return
+			}
+			importPath, export := sp[0], sp[1]
+			m[importPath] = export
+		}
+		stdlibPkgfileMap = m
+	})
+	return stdlibPkgfileMap, stdlibPkgfileErr
+}

internal/testenv/testenv.go

@@ -16,8 +16,11 @@ import (
 	"runtime/debug"
 	"strings"
 	"sync"
+	"testing"
 	"time"
 
+	"golang.org/x/tools/internal/goroot"
+
 	exec "golang.org/x/sys/execabs"
 )
 
@@ -329,3 +332,20 @@ func Deadline(t Testing) (time.Time, bool) {
 	}
 	return td.Deadline()
 }
+
+// WriteImportcfg writes an importcfg file used by the compiler or linker to
+// dstPath containing entries for the packages in std and cmd in addition
+// to the package to package file mappings in additionalPackageFiles.
+func WriteImportcfg(t testing.TB, dstPath string, additionalPackageFiles map[string]string) {
+	importcfg, err := goroot.Importcfg()
+	for k, v := range additionalPackageFiles {
+		importcfg += fmt.Sprintf("\npackagefile %s=%s", k, v)
+	}
+	if err != nil {
+		t.Fatalf("preparing the importcfg failed: %s", err)
+	}
+	ioutil.WriteFile(dstPath, []byte(importcfg), 0655)
+	if err != nil {
+		t.Fatalf("writing the importcfg failed: %s", err)
+	}
+}