Skip to content

Commit fd080ea

Browse files
author
Bryan C. Mills
committed
cmd/go: resolve non-standard imports from within GOROOT/src using vendor directories
Updates #30228 Fixes #26924 Change-Id: Ie625c64721559c7633396342320536396cd1fcf5 Reviewed-on: https://go-review.googlesource.com/c/go/+/164621 Run-TryBot: Bryan C. Mills <[email protected]> TryBot-Result: Gobot Gobot <[email protected]> Reviewed-by: Jay Conrod <[email protected]>
1 parent 1c2d4da commit fd080ea

File tree

13 files changed

+276
-56
lines changed

13 files changed

+276
-56
lines changed

‎src/cmd/go/internal/cfg/cfg.go

+1-1
Original file line numberDiff line numberDiff line change
@@ -38,7 +38,7 @@ var (
3838
BuildWork bool // -work flag
3939
BuildX bool // -x flag
4040

41-
CmdName string // "build", "install", "list", etc.
41+
CmdName string // "build", "install", "list", "mod tidy", etc.
4242

4343
DebugActiongraph string // -debug-actiongraph flag (undocumented, unstable)
4444
)

‎src/cmd/go/internal/load/pkg.go

+18-20
Original file line numberDiff line numberDiff line change
@@ -32,14 +32,14 @@ var (
3232
ModInit func()
3333

3434
// module hooks; nil if module use is disabled
35-
ModBinDir func() string // return effective bin directory
36-
ModLookup func(path string) (dir, realPath string, err error) // lookup effective meaning of import
37-
ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct
38-
ModImportPaths func(args []string) []*search.Match // expand import paths
39-
ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
40-
ModInfoProg func(info string) []byte // wrap module info in .go code for binary
41-
ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files
42-
ModDirImportPath func(string) string // return effective import path for directory
35+
ModBinDir func() string // return effective bin directory
36+
ModLookup func(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) // lookup effective meaning of import
37+
ModPackageModuleInfo func(path string) *modinfo.ModulePublic // return module info for Package struct
38+
ModImportPaths func(args []string) []*search.Match // expand import paths
39+
ModPackageBuildInfo func(main string, deps []string) string // return module info to embed in binary
40+
ModInfoProg func(info string) []byte // wrap module info in .go code for binary
41+
ModImportFromFiles func([]string) // update go.mod to add modules for imports in these files
42+
ModDirImportPath func(string) string // return effective import path for directory
4343
)
4444

4545
var IgnoreImports bool // control whether we ignore imports in packages
@@ -483,8 +483,10 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
483483
}
484484

485485
parentPath := ""
486+
parentIsStd := false
486487
if parent != nil {
487488
parentPath = parent.ImportPath
489+
parentIsStd = parent.Standard
488490
}
489491

490492
// Determine canonical identifier for this package.
@@ -501,7 +503,7 @@ func LoadImport(path, srcDir string, parent *Package, stk *ImportStack, importPo
501503
importPath = dirToImportPath(filepath.Join(srcDir, path))
502504
} else if cfg.ModulesEnabled {
503505
var p string
504-
modDir, p, modErr = ModLookup(path)
506+
modDir, p, modErr = ModLookup(parentPath, parentIsStd, path)
505507
if modErr == nil {
506508
importPath = p
507509
}
@@ -641,7 +643,13 @@ func isDir(path string) bool {
641643
// Go 1.11 module legacy conversion (golang.org/issue/25069).
642644
func ResolveImportPath(parent *Package, path string) (found string) {
643645
if cfg.ModulesEnabled {
644-
if _, p, e := ModLookup(path); e == nil {
646+
parentPath := ""
647+
parentIsStd := false
648+
if parent != nil {
649+
parentPath = parent.ImportPath
650+
parentIsStd = parent.Standard
651+
}
652+
if _, p, e := ModLookup(parentPath, parentIsStd, path); e == nil {
645653
return p
646654
}
647655
return path
@@ -1401,16 +1409,6 @@ func (p *Package) load(stk *ImportStack, bp *build.Package, err error) {
14011409
continue
14021410
}
14031411
p1 := LoadImport(path, p.Dir, p, stk, p.Internal.Build.ImportPos[path], ResolveImport)
1404-
if p.Standard && p.Error == nil && !p1.Standard && p1.Error == nil {
1405-
p.Error = &PackageError{
1406-
ImportStack: stk.Copy(),
1407-
Err: fmt.Sprintf("non-standard import %q in standard package %q", path, p.ImportPath),
1408-
}
1409-
pos := p.Internal.Build.ImportPos[path]
1410-
if len(pos) > 0 {
1411-
p.Error.Pos = pos[0].String()
1412-
}
1413-
}
14141412

14151413
path = p1.ImportPath
14161414
importPaths[i] = path

‎src/cmd/go/internal/modload/build.go

-3
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,6 @@ func findStandardImportPath(path string) string {
3838
if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) {
3939
return filepath.Join(cfg.GOROOT, "src", path)
4040
}
41-
if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, "vendor/"+path) {
42-
return filepath.Join(cfg.GOROOT, "src/vendor", path)
43-
}
4441
}
4542
return ""
4643
}

‎src/cmd/go/internal/modload/import.go

+13
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,19 @@ func Import(path string) (m module.Version, dir string, err error) {
6464
if search.IsStandardImportPath(path) {
6565
if goroot.IsStandardPackage(cfg.GOROOT, cfg.BuildContext.Compiler, path) {
6666
dir := filepath.Join(cfg.GOROOT, "src", path)
67+
68+
// If the main module is in the standard library, attribute its packages
69+
// to that module.
70+
switch Target.Path {
71+
case "cmd":
72+
if strings.HasPrefix(path, "cmd") {
73+
return Target, dir, nil
74+
}
75+
case "std":
76+
if !strings.HasPrefix(path, "cmd") {
77+
return Target, dir, nil
78+
}
79+
}
6780
return module.Version{}, dir, nil
6881
}
6982
}

‎src/cmd/go/internal/modload/init.go

+6
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@ import (
1818
"cmd/go/internal/mvs"
1919
"cmd/go/internal/renameio"
2020
"cmd/go/internal/search"
21+
"cmd/go/internal/str"
2122
"encoding/json"
2223
"fmt"
2324
"go/build"
@@ -380,6 +381,11 @@ func InitMod() {
380381
// modFileToBuildList initializes buildList from the modFile.
381382
func modFileToBuildList() {
382383
Target = modFile.Module.Mod
384+
if (str.HasPathPrefix(Target.Path, "std") || str.HasPathPrefix(Target.Path, "cmd")) &&
385+
search.InDir(cwd, cfg.GOROOTsrc) == "" {
386+
base.Fatalf("go: reserved module path %s not allow outside of GOROOT/src", Target.Path)
387+
}
388+
383389
list := []module.Version{Target}
384390
for _, r := range modFile.Require {
385391
list = append(list, r.Mod)

‎src/cmd/go/internal/modload/load.go

+81-12
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ import (
1212
"io/ioutil"
1313
"os"
1414
"path"
15+
pathpkg "path"
1516
"path/filepath"
1617
"sort"
1718
"strings"
@@ -90,7 +91,9 @@ func ImportPaths(patterns []string) []*search.Match {
9091
// the exact version of a particular module increases during
9192
// the loader iterations.
9293
m.Pkgs = str.StringList(fsDirs[i])
93-
for j, pkg := range m.Pkgs {
94+
pkgs := m.Pkgs
95+
m.Pkgs = m.Pkgs[:0]
96+
for _, pkg := range pkgs {
9497
dir := pkg
9598
if !filepath.IsAbs(dir) {
9699
dir = filepath.Join(cwd, pkg)
@@ -108,6 +111,16 @@ func ImportPaths(patterns []string) []*search.Match {
108111
if strings.HasPrefix(suffix, "/vendor/") {
109112
// TODO getmode vendor check
110113
pkg = strings.TrimPrefix(suffix, "/vendor/")
114+
} else if Target.Path == "std" {
115+
// Don't add the prefix "std/" to packages in the "std" module.
116+
// It's the one module path that isn't a prefix of its packages.
117+
pkg = strings.TrimPrefix(suffix, "/")
118+
if pkg == "builtin" {
119+
// "builtin" is a pseudo-package with a real source file.
120+
// It's not included in "std", so it shouldn't be included in
121+
// "./..." within module "std" either.
122+
continue
123+
}
111124
} else {
112125
pkg = Target.Path + suffix
113126
}
@@ -129,10 +142,10 @@ func ImportPaths(patterns []string) []*search.Match {
129142
// After loader is done iterating, we still need to return the
130143
// path, so that "go list -e" produces valid output.
131144
if iterating {
132-
pkg = ""
145+
continue
133146
}
134147
}
135-
m.Pkgs[j] = pkg
148+
m.Pkgs = append(m.Pkgs, pkg)
136149
}
137150

138151
case strings.Contains(m.Pattern, "..."):
@@ -163,9 +176,7 @@ func ImportPaths(patterns []string) []*search.Match {
163176
updateMatches(true)
164177
for _, m := range matches {
165178
for _, pkg := range m.Pkgs {
166-
if pkg != "" {
167-
roots = append(roots, pkg)
168-
}
179+
roots = append(roots, pkg)
169180
}
170181
}
171182
return roots
@@ -394,13 +405,17 @@ func ModuleUsedDirectly(path string) bool {
394405
}
395406

396407
// Lookup returns the source directory, import path, and any loading error for
397-
// the package at path.
408+
// the package at path as imported from the package in parentDir.
398409
// Lookup requires that one of the Load functions in this package has already
399410
// been called.
400-
func Lookup(path string) (dir, realPath string, err error) {
411+
func Lookup(parentPath string, parentIsStd bool, path string) (dir, realPath string, err error) {
401412
if path == "" {
402413
panic("Lookup called with empty package path")
403414
}
415+
416+
if parentIsStd {
417+
path = loaded.stdVendor(parentPath, path)
418+
}
404419
pkg, ok := loaded.pkgCache.Get(path).(*loadPkg)
405420
if !ok {
406421
// The loader should have found all the relevant paths.
@@ -434,10 +449,11 @@ func Lookup(path string) (dir, realPath string, err error) {
434449
// TODO(rsc): It might be nice to make the loader take and return
435450
// a buildList rather than hard-coding use of the global.
436451
type loader struct {
437-
tags map[string]bool // tags for scanDir
438-
testRoots bool // include tests for roots
439-
isALL bool // created with LoadALL
440-
testAll bool // include tests for all packages
452+
tags map[string]bool // tags for scanDir
453+
testRoots bool // include tests for roots
454+
isALL bool // created with LoadALL
455+
testAll bool // include tests for all packages
456+
forceStdVendor bool // if true, load standard-library dependencies from the vendor subtree
441457

442458
// reset on each iteration
443459
roots []*loadPkg
@@ -457,6 +473,17 @@ func newLoader() *loader {
457473
ld := new(loader)
458474
ld.tags = imports.Tags()
459475
ld.testRoots = LoadTests
476+
477+
switch Target.Path {
478+
case "std", "cmd":
479+
// Inside the "std" and "cmd" modules, we prefer to use the vendor directory
480+
// unless the command explicitly changes the module graph.
481+
// TODO(golang.org/issue/30240): Remove this special case.
482+
if cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ") {
483+
ld.forceStdVendor = true
484+
}
485+
}
486+
460487
return ld
461488
}
462489

@@ -631,7 +658,11 @@ func (ld *loader) doPkg(item interface{}) {
631658
}
632659
}
633660

661+
inStd := (search.IsStandardImportPath(pkg.path) && search.InDir(pkg.dir, cfg.GOROOTsrc) != "")
634662
for _, path := range imports {
663+
if inStd {
664+
path = ld.stdVendor(pkg.path, path)
665+
}
635666
pkg.imports = append(pkg.imports, ld.pkg(path, false))
636667
}
637668

@@ -642,6 +673,30 @@ func (ld *loader) doPkg(item interface{}) {
642673
}
643674
}
644675

676+
// stdVendor returns the canonical import path for the package with the given
677+
// path when imported from the standard-library package at parentPath.
678+
func (ld *loader) stdVendor(parentPath, path string) string {
679+
if search.IsStandardImportPath(path) {
680+
return path
681+
}
682+
683+
if str.HasPathPrefix(parentPath, "cmd") && (Target.Path != "cmd" || ld.forceStdVendor) {
684+
vendorPath := pathpkg.Join("cmd", "vendor", path)
685+
if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
686+
return vendorPath
687+
}
688+
}
689+
if Target.Path != "std" || ld.forceStdVendor {
690+
vendorPath := pathpkg.Join("vendor", path)
691+
if _, err := os.Stat(filepath.Join(cfg.GOROOTsrc, filepath.FromSlash(vendorPath))); err == nil {
692+
return vendorPath
693+
}
694+
}
695+
696+
// Not vendored: resolve from modules.
697+
return path
698+
}
699+
645700
// computePatternAll returns the list of packages matching pattern "all",
646701
// starting with a list of the import paths for the packages in the main module.
647702
func (ld *loader) computePatternAll(paths []string) []string {
@@ -932,6 +987,20 @@ func (r *mvsReqs) required(mod module.Version) ([]module.Version, error) {
932987
return vendorList, nil
933988
}
934989

990+
switch Target.Path {
991+
case "std", "cmd":
992+
// When inside "std" or "cmd", only fetch and read go.mod files if we're
993+
// explicitly running a command that can change the module graph. If we have
994+
// to resolve a new dependency, we might pick the wrong version, but 'go mod
995+
// tidy' will fix it — and new standard-library dependencies should be rare
996+
// anyway.
997+
//
998+
// TODO(golang.org/issue/30240): Drop this special-case.
999+
if cfg.CmdName != "get" && !strings.HasPrefix(cfg.CmdName, "mod ") {
1000+
return nil, nil
1001+
}
1002+
}
1003+
9351004
origPath := mod.Path
9361005
if repl := Replacement(mod); repl.Path != "" {
9371006
if repl.Version == "" {

‎src/cmd/go/internal/modload/query.go

+5
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ import (
99
"cmd/go/internal/modfetch/codehost"
1010
"cmd/go/internal/module"
1111
"cmd/go/internal/semver"
12+
"cmd/go/internal/str"
1213
"fmt"
1314
pathpkg "path"
1415
"strings"
@@ -131,6 +132,10 @@ func Query(path, query string, allowed func(module.Version) bool) (*modfetch.Rev
131132
return &modfetch.RevInfo{Version: Target.Version}, nil
132133
}
133134

135+
if str.HasPathPrefix(path, "std") || str.HasPathPrefix(path, "cmd") {
136+
return nil, fmt.Errorf("explicit requirement on standard-library module %s not allowed", path)
137+
}
138+
134139
// Load versions and execute query.
135140
repo, err := modfetch.Lookup(path)
136141
if err != nil {

0 commit comments

Comments
 (0)