cmd/go: support module deprecation

A module is deprecated if its author adds a comment containing a
paragraph starting with "Deprecated:" to its go.mod file. The comment
must appear immediately before the "module" directive or as a suffix
on the same line. The deprecation message runs from just after
"Deprecated:" to the end of the paragraph. This is implemented in
CL 301089.

'go list -m -u' loads deprecation messages from the latest version of
each module, not considering retractions (i.e., deprecations and
retractions are loaded from the same version). By default, deprecated
modules are printed with a "(deprecated)" suffix. The full deprecation
message is available in the -f and -json output.

'go get' prints deprecation warnings for modules named on the command
line. It also prints warnings for modules needed to build packages
named on the command line if those modules are direct dependencies of
the main module.

For #40357

Change-Id: Id81fb2b24710681b025becd6cd74f746f4378e78
Reviewed-on: https://go-review.googlesource.com/c/go/+/306334
Trust: Jay Conrod <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>

Author: Jay Conrod

src/cmd/go/alldocs.go

@@ -348,7 +348,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 		if *listM {
 			*listFmt = "{{.String}}"
 			if *listVersions {
-				*listFmt = `{{.Path}}{{range .Versions}} {{.}}{{end}}`
+				*listFmt = `{{.Path}}{{range .Versions}} {{.}}{{end}}{{if .Deprecated}} (deprecated){{end}}`
 			}
 		} else {
 			*listFmt = "{{.ImportPath}}"
@@ -453,7 +453,7 @@ func runList(ctx context.Context, cmd *base.Command, args []string) {
 
 		var mode modload.ListMode
 		if *listU {
-			mode |= modload.ListU | modload.ListRetracted
+			mode |= modload.ListU | modload.ListRetracted | modload.ListDeprecated
 		}
 		if *listRetracted {
 			mode |= modload.ListRetracted

src/cmd/go/internal/list/list.go

@@ -86,7 +86,7 @@ writing it back to go.mod. The JSON output corresponds to these Go types:
 
 	type Module struct {
 		Path string
-		Version string
+		Deprecated string
 	}
 
 	type GoMod struct {
@@ -450,14 +450,19 @@ func flagDropRetract(arg string) {
 
 // fileJSON is the -json output data structure.
 type fileJSON struct {
-	Module  module.Version
+	Module  editModuleJSON
 	Go      string `json:",omitempty"`
 	Require []requireJSON
 	Exclude []module.Version
 	Replace []replaceJSON
 	Retract []retractJSON
 }
 
+type editModuleJSON struct {
+	Path       string
+	Deprecated string `json:",omitempty"`
+}
+
 type requireJSON struct {
 	Path     string
 	Version  string `json:",omitempty"`
@@ -479,7 +484,10 @@ type retractJSON struct {
 func editPrintJSON(modFile *modfile.File) {
 	var f fileJSON
 	if modFile.Module != nil {
-		f.Module = modFile.Module.Mod
+		f.Module = editModuleJSON{
+			Path:       modFile.Module.Mod.Path,
+			Deprecated: modFile.Module.Deprecated,
+		}
 	}
 	if modFile.Go != nil {
 		f.Go = modFile.Go.Version

src/cmd/go/internal/modcmd/edit.go

@@ -354,7 +354,7 @@ func runGet(ctx context.Context, cmd *base.Command, args []string) {
 			pkgPatterns = append(pkgPatterns, q.pattern)
 		}
 	}
-	r.checkPackagesAndRetractions(ctx, pkgPatterns)
+	r.checkPackageProblems(ctx, pkgPatterns)
 
 	// We've already downloaded modules (and identified direct and indirect
 	// dependencies) by loading packages in findAndUpgradeImports.
@@ -1463,25 +1463,31 @@ func (r *resolver) chooseArbitrarily(cs pathSet) (isPackage bool, m module.Versi
 	return false, cs.mod
 }
 
-// checkPackagesAndRetractions reloads packages for the given patterns and
-// reports missing and ambiguous package errors. It also reports loads and
-// reports retractions for resolved modules and modules needed to build
-// named packages.
+// checkPackageProblems reloads packages for the given patterns and reports
+// missing and ambiguous package errors. It also reports retractions and
+// deprecations for resolved modules and modules needed to build named packages.
 //
 // We skip missing-package errors earlier in the process, since we want to
 // resolve pathSets ourselves, but at that point, we don't have enough context
 // to log the package-import chains leading to each error.
-func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns []string) {
+func (r *resolver) checkPackageProblems(ctx context.Context, pkgPatterns []string) {
 	defer base.ExitIfErrors()
 
-	// Build a list of modules to load retractions for. Start with versions
-	// selected based on command line queries.
-	//
-	// This is a subset of the build list. If the main module has a lot of
-	// dependencies, loading retractions for the entire build list would be slow.
-	relevantMods := make(map[module.Version]struct{})
+	// Gather information about modules we might want to load retractions and
+	// deprecations for. Loading this metadata requires at least one version
+	// lookup per module, and we don't want to load information that's neither
+	// relevant nor actionable.
+	type modFlags int
+	const (
+		resolved modFlags = 1 << iota // version resolved by 'go get'
+		named                         // explicitly named on command line or provides a named package
+		hasPkg                        // needed to build named packages
+		direct                        // provides a direct dependency of the main module
+	)
+	relevantMods := make(map[module.Version]modFlags)
 	for path, reason := range r.resolvedVersion {
-		relevantMods[module.Version{Path: path, Version: reason.version}] = struct{}{}
+		m := module.Version{Path: path, Version: reason.version}
+		relevantMods[m] |= resolved
 	}
 
 	// Reload packages, reporting errors for missing and ambiguous imports.
@@ -1518,44 +1524,89 @@ func (r *resolver) checkPackagesAndRetractions(ctx context.Context, pkgPatterns
 				base.SetExitStatus(1)
 				if ambiguousErr := (*modload.AmbiguousImportError)(nil); errors.As(err, &ambiguousErr) {
 					for _, m := range ambiguousErr.Modules {
-						relevantMods[m] = struct{}{}
+						relevantMods[m] |= hasPkg
 					}
 				}
 			}
 			if m := modload.PackageModule(pkg); m.Path != "" {
-				relevantMods[m] = struct{}{}
+				relevantMods[m] |= hasPkg
+			}
+		}
+		for _, match := range matches {
+			for _, pkg := range match.Pkgs {
+				m := modload.PackageModule(pkg)
+				relevantMods[m] |= named
 			}
 		}
 	}
 
-	// Load and report retractions.
-	type retraction struct {
-		m   module.Version
-		err error
-	}
-	retractions := make([]retraction, 0, len(relevantMods))
+	reqs := modload.LoadModFile(ctx)
 	for m := range relevantMods {
-		retractions = append(retractions, retraction{m: m})
+		if reqs.IsDirect(m.Path) {
+			relevantMods[m] |= direct
+		}
 	}
-	sort.Slice(retractions, func(i, j int) bool {
-		return retractions[i].m.Path < retractions[j].m.Path
-	})
-	for i := 0; i < len(retractions); i++ {
+
+	// Load retractions for modules mentioned on the command line and modules
+	// needed to build named packages. We care about retractions of indirect
+	// dependencies, since we might be able to upgrade away from them.
+	type modMessage struct {
+		m       module.Version
+		message string
+	}
+	retractions := make([]modMessage, 0, len(relevantMods))
+	for m, flags := range relevantMods {
+		if flags&(resolved|named|hasPkg) != 0 {
+			retractions = append(retractions, modMessage{m: m})
+		}
+	}
+	sort.Slice(retractions, func(i, j int) bool { return retractions[i].m.Path < retractions[j].m.Path })
+	for i := range retractions {
 		i := i
 		r.work.Add(func() {
 			err := modload.CheckRetractions(ctx, retractions[i].m)
 			if retractErr := (*modload.ModuleRetractedError)(nil); errors.As(err, &retractErr) {
-				retractions[i].err = err
+				retractions[i].message = err.Error()
 			}
 		})
 	}
+
+	// Load deprecations for modules mentioned on the command line. Only load
+	// deprecations for indirect dependencies if they're also direct dependencies
+	// of the main module. Deprecations of purely indirect dependencies are
+	// not actionable.
+	deprecations := make([]modMessage, 0, len(relevantMods))
+	for m, flags := range relevantMods {
+		if flags&(resolved|named) != 0 || flags&(hasPkg|direct) == hasPkg|direct {
+			deprecations = append(deprecations, modMessage{m: m})
+		}
+	}
+	sort.Slice(deprecations, func(i, j int) bool { return deprecations[i].m.Path < deprecations[j].m.Path })
+	for i := range deprecations {
+		i := i
+		r.work.Add(func() {
+			deprecation, err := modload.CheckDeprecation(ctx, deprecations[i].m)
+			if err != nil || deprecation == "" {
+				return
+			}
+			deprecations[i].message = modload.ShortMessage(deprecation, "")
+		})
+	}
+
 	<-r.work.Idle()
+
+	// Report deprecations, then retractions.
+	for _, mm := range deprecations {
+		if mm.message != "" {
+			fmt.Fprintf(os.Stderr, "go: warning: module %s is deprecated: %s\n", mm.m.Path, mm.message)
+		}
+	}
 	var retractPath string
-	for _, r := range retractions {
-		if r.err != nil {
-			fmt.Fprintf(os.Stderr, "go: warning: %v\n", r.err)
+	for _, mm := range retractions {
+		if mm.message != "" {
+			fmt.Fprintf(os.Stderr, "go: warning: %v\n", mm.message)
 			if retractPath == "" {
-				retractPath = r.m.Path
+				retractPath = mm.m.Path
 			} else {
 				retractPath = "<module>"
 			}

src/cmd/go/internal/modget/get.go

@@ -10,19 +10,20 @@ import "time"
 // and the fields are documented in the help text in ../list/list.go
 
 type ModulePublic struct {
-	Path      string        `json:",omitempty"` // module path
-	Version   string        `json:",omitempty"` // module version
-	Versions  []string      `json:",omitempty"` // available module versions
-	Replace   *ModulePublic `json:",omitempty"` // replaced by this module
-	Time      *time.Time    `json:",omitempty"` // time version was created
-	Update    *ModulePublic `json:",omitempty"` // available update (with -u)
-	Main      bool          `json:",omitempty"` // is this the main module?
-	Indirect  bool          `json:",omitempty"` // module is only indirectly needed by main module
-	Dir       string        `json:",omitempty"` // directory holding local copy of files, if any
-	GoMod     string        `json:",omitempty"` // path to go.mod file describing module, if any
-	GoVersion string        `json:",omitempty"` // go version used in module
-	Retracted []string      `json:",omitempty"` // retraction information, if any (with -retracted or -u)
-	Error     *ModuleError  `json:",omitempty"` // error loading module
+	Path       string        `json:",omitempty"` // module path
+	Version    string        `json:",omitempty"` // module version
+	Versions   []string      `json:",omitempty"` // available module versions
+	Replace    *ModulePublic `json:",omitempty"` // replaced by this module
+	Time       *time.Time    `json:",omitempty"` // time version was created
+	Update     *ModulePublic `json:",omitempty"` // available update (with -u)
+	Main       bool          `json:",omitempty"` // is this the main module?
+	Indirect   bool          `json:",omitempty"` // module is only indirectly needed by main module
+	Dir        string        `json:",omitempty"` // directory holding local copy of files, if any
+	GoMod      string        `json:",omitempty"` // path to go.mod file describing module, if any
+	GoVersion  string        `json:",omitempty"` // go version used in module
+	Retracted  []string      `json:",omitempty"` // retraction information, if any (with -retracted or -u)
+	Deprecated string        `json:",omitempty"` // deprecation message, if any (with -u)
+	Error      *ModuleError  `json:",omitempty"` // error loading module
 }
 
 type ModuleError struct {
@@ -45,6 +46,9 @@ func (m *ModulePublic) String() string {
 			s += " [" + versionString(m.Update) + "]"
 		}
 	}
+	if m.Deprecated != "" {
+		s += " (deprecated)"
+	}
 	if m.Replace != nil {
 		s += " => " + m.Replace.Path
 		if m.Replace.Version != "" {
@@ -53,6 +57,9 @@ func (m *ModulePublic) String() string {
 				s += " [" + versionString(m.Replace.Update) + "]"
 			}
 		}
+		if m.Replace.Deprecated != "" {
+			s += " (deprecated)"
+		}
 	}
 	return s
 }

src/cmd/go/internal/modinfo/info.go

@@ -112,8 +112,8 @@ func addUpdate(ctx context.Context, m *modinfo.ModulePublic) {
 	info, err := Query(ctx, m.Path, "upgrade", m.Version, CheckAllowed)
 	var noVersionErr *NoMatchingVersionError
 	if errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) {
-		// Ignore "not found" and "no matching version" errors. This usually means
-		// the user is offline or the proxy doesn't have a matching version.
+		// Ignore "not found" and "no matching version" errors.
+		// This means the proxy has no matching version or no versions at all.
 		//
 		// We should report other errors though. An attacker that controls the
 		// network shouldn't be able to hide versions by interfering with
@@ -163,9 +163,8 @@ func addRetraction(ctx context.Context, m *modinfo.ModulePublic) {
 	var noVersionErr *NoMatchingVersionError
 	var retractErr *ModuleRetractedError
 	if err == nil || errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) {
-		// Ignore "not found" and "no matching version" errors. This usually means
-		// the user is offline or the proxy doesn't have a go.mod file that could
-		// contain retractions.
+		// Ignore "not found" and "no matching version" errors.
+		// This means the proxy has no matching version or no versions at all.
 		//
 		// We should report other errors though. An attacker that controls the
 		// network shouldn't be able to hide versions by interfering with
@@ -184,6 +183,31 @@ func addRetraction(ctx context.Context, m *modinfo.ModulePublic) {
 	}
 }
 
+// addDeprecation fills in m.Deprecated if the module was deprecated by its
+// author. m.Error is set if there's an error loading deprecation information.
+func addDeprecation(ctx context.Context, m *modinfo.ModulePublic) {
+	deprecation, err := CheckDeprecation(ctx, module.Version{Path: m.Path, Version: m.Version})
+	var noVersionErr *NoMatchingVersionError
+	if errors.Is(err, fs.ErrNotExist) || errors.As(err, &noVersionErr) {
+		// Ignore "not found" and "no matching version" errors.
+		// This means the proxy has no matching version or no versions at all.
+		//
+		// We should report other errors though. An attacker that controls the
+		// network shouldn't be able to hide versions by interfering with
+		// the HTTPS connection. An attacker that controls the proxy may still
+		// hide versions, since the "list" and "latest" endpoints are not
+		// authenticated.
+		return
+	}
+	if err != nil {
+		if m.Error == nil {
+			m.Error = &modinfo.ModuleError{Err: err.Error()}
+		}
+		return
+	}
+	m.Deprecated = deprecation
+}
+
 // moduleInfo returns information about module m, loaded from the requirements
 // in rs (which may be nil to indicate that m was not loaded from a requirement
 // graph).

src/cmd/go/internal/modload/build.go

@@ -191,6 +191,12 @@ func (rs *Requirements) Graph(ctx context.Context) (*ModuleGraph, error) {
 	return cached.mg, cached.err
 }
 
+// IsDirect returns whether the given module provides a package directly
+// imported by a package or test in the main module.
+func (rs *Requirements) IsDirect(path string) bool {
+	return rs.direct[path]
+}
+
 // A ModuleGraph represents the complete graph of module dependencies
 // of a main module.
 //

src/cmd/go/internal/modload/buildlist.go

@@ -25,6 +25,7 @@ type ListMode int
 const (
 	ListU ListMode = 1 << iota
 	ListRetracted
+	ListDeprecated
 	ListVersions
 	ListRetractedVersions
 )
@@ -52,6 +53,9 @@ func ListModules(ctx context.Context, args []string, mode ListMode) ([]*modinfo.
 					if mode&ListRetracted != 0 {
 						addRetraction(ctx, m)
 					}
+					if mode&ListDeprecated != 0 {
+						addDeprecation(ctx, m)
+					}
 					<-sem
 				}()
 			}