modfile: support retract directive and version intervals

This CL adds support for parsing and programmatically adding and
removing a new directive, "retract", as described in golang/go#24031.

The "retract" directive comes in two forms:

    retract v1.0.0           // single version
    retract [v1.1.0, v1.2.0] // closed interval

Updates golang/go#24031

Change-Id: I1236c7d89e7674abf694e49e9b4869b14a59fac0
Reviewed-on: https://go-review.googlesource.com/c/mod/+/228039
Run-TryBot: Jay Conrod <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Michael Matloob <[email protected]>

Author: Jay Conrod

modfile/rule.go

@@ -30,6 +30,7 @@ import (
 
 	"golang.org/x/mod/internal/lazyregexp"
 	"golang.org/x/mod/module"
+	"golang.org/x/mod/semver"
 )
 
 // A File is the parsed, interpreted form of a go.mod file.
@@ -39,6 +40,7 @@ type File struct {
 	Require []*Require
 	Exclude []*Exclude
 	Replace []*Replace
+	Retract []*Retract
 
 	Syntax *FileSyntax
 }
@@ -75,6 +77,21 @@ type Replace struct {
 	Syntax *Line
 }
 
+// A Retract is a single retract statement.
+type Retract struct {
+	VersionInterval
+	Rationale string
+	Syntax    *Line
+}
+
+// A VersionInterval represents a range of versions with upper and lower bounds.
+// Intervals are closed: both bounds are included. When Low is equal to High,
+// the interval may refer to a single version ('v1.2.3') or an interval
+// ('[v1.2.3, v1.2.3]'); both have the same representation.
+type VersionInterval struct {
+	Low, High string
+}
+
 func (f *File) AddModuleStmt(path string) error {
 	if f.Syntax == nil {
 		f.Syntax = new(FileSyntax)
@@ -138,7 +155,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File
 	for _, x := range fs.Stmt {
 		switch x := x.(type) {
 		case *Line:
-			f.add(&errs, x, x.Token[0], x.Token[1:], fix, strict)
+			f.add(&errs, nil, x, x.Token[0], x.Token[1:], fix, strict)
 
 		case *LineBlock:
 			if len(x.Token) > 1 {
@@ -161,9 +178,9 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File
 					})
 				}
 				continue
-			case "module", "require", "exclude", "replace":
+			case "module", "require", "exclude", "replace", "retract":
 				for _, l := range x.Line {
-					f.add(&errs, l, x.Token[0], l.Token, fix, strict)
+					f.add(&errs, x, l, x.Token[0], l.Token, fix, strict)
 				}
 			}
 		}
@@ -177,7 +194,7 @@ func parseToFile(file string, data []byte, fix VersionFixer, strict bool) (*File
 
 var GoVersionRE = lazyregexp.New(`^([1-9][0-9]*)\.(0|[1-9][0-9]*)$`)
 
-func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix VersionFixer, strict bool) {
+func (f *File) add(errs *ErrorList, block *LineBlock, line *Line, verb string, args []string, fix VersionFixer, strict bool) {
 	// If strict is false, this module is a dependency.
 	// We ignore all unknown directives as well as main-module-only
 	// directives like replace and exclude. It will work better for
@@ -186,7 +203,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix
 	// and simply ignore those statements.
 	if !strict {
 		switch verb {
-		case "module", "require", "go":
+		case "go", "module", "retract", "require":
 			// want these even for dependency go.mods
 		default:
 			return
@@ -232,6 +249,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix
 
 		f.Go = &Go{Syntax: line}
 		f.Go.Version = args[0]
+
 	case "module":
 		if f.Module != nil {
 			errorf("repeated module statement")
@@ -248,6 +266,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix
 			return
 		}
 		f.Module.Mod = module.Version{Path: s}
+
 	case "require", "exclude":
 		if len(args) != 2 {
 			errorf("usage: %s module/path v1.2.3", verb)
@@ -284,6 +303,7 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix
 				Syntax: line,
 			})
 		}
+
 	case "replace":
 		arrow := 2
 		if len(args) >= 2 && args[1] == "=>" {
@@ -347,6 +367,33 @@ func (f *File) add(errs *ErrorList, line *Line, verb string, args []string, fix
 			New:    module.Version{Path: ns, Version: nv},
 			Syntax: line,
 		})
+
+	case "retract":
+		rationale := parseRetractRationale(block, line)
+		vi, err := parseVersionInterval(verb, &args, fix)
+		if err != nil {
+			if strict {
+				wrapError(err)
+				return
+			} else {
+				// Only report errors parsing intervals in the main module. We may
+				// support additional syntax in the future, such as open and half-open
+				// intervals. Those can't be supported now, because they break the
+				// go.mod parser, even in lax mode.
+				return
+			}
+		}
+		if len(args) > 0 && strict {
+			// In the future, there may be additional information after the version.
+			errorf("unexpected token after version: %q", args[0])
+			return
+		}
+		retract := &Retract{
+			VersionInterval: vi,
+			Rationale:       rationale,
+			Syntax:          line,
+		}
+		f.Retract = append(f.Retract, retract)
 	}
 }
 
@@ -444,6 +491,53 @@ func AutoQuote(s string) string {
 	return s
 }
 
+func parseVersionInterval(verb string, args *[]string, fix VersionFixer) (VersionInterval, error) {
+	toks := *args
+	if len(toks) == 0 || toks[0] == "(" {
+		return VersionInterval{}, fmt.Errorf("expected '[' or version")
+	}
+	if toks[0] != "[" {
+		v, err := parseVersion(verb, "", &toks[0], fix)
+		if err != nil {
+			return VersionInterval{}, err
+		}
+		*args = toks[1:]
+		return VersionInterval{Low: v, High: v}, nil
+	}
+	toks = toks[1:]
+
+	if len(toks) == 0 {
+		return VersionInterval{}, fmt.Errorf("expected version after '['")
+	}
+	low, err := parseVersion(verb, "", &toks[0], fix)
+	if err != nil {
+		return VersionInterval{}, err
+	}
+	toks = toks[1:]
+
+	if len(toks) == 0 || toks[0] != "," {
+		return VersionInterval{}, fmt.Errorf("expected ',' after version")
+	}
+	toks = toks[1:]
+
+	if len(toks) == 0 {
+		return VersionInterval{}, fmt.Errorf("expected version after ','")
+	}
+	high, err := parseVersion(verb, "", &toks[0], fix)
+	if err != nil {
+		return VersionInterval{}, err
+	}
+	toks = toks[1:]
+
+	if len(toks) == 0 || toks[0] != "]" {
+		return VersionInterval{}, fmt.Errorf("expected ']' after version")
+	}
+	toks = toks[1:]
+
+	*args = toks
+	return VersionInterval{Low: low, High: high}, nil
+}
+
 func parseString(s *string) (string, error) {
 	t := *s
 	if strings.HasPrefix(t, `"`) {
@@ -461,6 +555,27 @@ func parseString(s *string) (string, error) {
 	return t, nil
 }
 
+// parseRetractRationale extracts the rationale for a retract directive from the
+// surrounding comments. If the line does not have comments and is part of a
+// block that does have comments, the block's comments are used.
+func parseRetractRationale(block *LineBlock, line *Line) string {
+	comments := line.Comment()
+	if block != nil && len(comments.Before) == 0 && len(comments.Suffix) == 0 {
+		comments = block.Comment()
+	}
+	groups := [][]Comment{comments.Before, comments.Suffix}
+	var lines []string
+	for _, g := range groups {
+		for _, c := range g {
+			if !strings.HasPrefix(c.Token, "//") {
+				continue // blank line
+			}
+			lines = append(lines, strings.TrimSpace(strings.TrimPrefix(c.Token, "//")))
+		}
+	}
+	return strings.Join(lines, "\n")
+}
+
 type ErrorList []Error
 
 func (e ErrorList) Error() string {
@@ -494,6 +609,8 @@ func (e *Error) Error() string {
 	var directive string
 	if e.ModPath != "" {
 		directive = fmt.Sprintf("%s %s: ", e.Verb, e.ModPath)
+	} else if e.Verb != "" {
+		directive = fmt.Sprintf("%s: ", e.Verb)
 	}
 
 	return pos + directive + e.Err.Error()
@@ -585,6 +702,15 @@ func (f *File) Cleanup() {
 	}
 	f.Replace = f.Replace[:w]
 
+	w = 0
+	for _, r := range f.Retract {
+		if r.Low != "" || r.High != "" {
+			f.Retract[w] = r
+			w++
+		}
+	}
+	f.Retract = f.Retract[:w]
+
 	f.Syntax.Cleanup()
 }
 
@@ -778,6 +904,34 @@ func (f *File) DropReplace(oldPath, oldVers string) error {
 	return nil
 }
 
+func (f *File) AddRetract(vi VersionInterval, rationale string) error {
+	r := &Retract{
+		VersionInterval: vi,
+	}
+	if vi.Low == vi.High {
+		r.Syntax = f.Syntax.addLine(nil, "retract", AutoQuote(vi.Low))
+	} else {
+		r.Syntax = f.Syntax.addLine(nil, "retract", "[", AutoQuote(vi.Low), ",", AutoQuote(vi.High), "]")
+	}
+	if rationale != "" {
+		for _, line := range strings.Split(rationale, "\n") {
+			com := Comment{Token: "// " + line}
+			r.Syntax.Comment().Before = append(r.Syntax.Comment().Before, com)
+		}
+	}
+	return nil
+}
+
+func (f *File) DropRetract(vi VersionInterval) error {
+	for _, r := range f.Retract {
+		if r.VersionInterval == vi {
+			f.Syntax.removeLine(r.Syntax)
+			*r = Retract{}
+		}
+	}
+	return nil
+}
+
 func (f *File) SortBlocks() {
 	f.removeDups() // otherwise sorting is unsafe
 
@@ -786,28 +940,38 @@ func (f *File) SortBlocks() {
 		if !ok {
 			continue
 		}
-		sort.Slice(block.Line, func(i, j int) bool {
-			li := block.Line[i]
-			lj := block.Line[j]
-			for k := 0; k < len(li.Token) && k < len(lj.Token); k++ {
-				if li.Token[k] != lj.Token[k] {
-					return li.Token[k] < lj.Token[k]
-				}
-			}
-			return len(li.Token) < len(lj.Token)
+		less := lineLess
+		if block.Token[0] == "retract" {
+			less = lineRetractLess
+		}
+		sort.SliceStable(block.Line, func(i, j int) bool {
+			return less(block.Line[i], block.Line[j])
 		})
 	}
 }
 
+// removeDups removes duplicate exclude and replace directives.
+//
+// Earlier exclude directives take priority.
+//
+// Later replace directives take priority.
+//
+// require directives are not de-duplicated. That's left up to higher-level
+// logic (MVS).
+//
+// retract directives are not de-duplicated since comments are
+// meaningful, and versions may be retracted multiple times.
 func (f *File) removeDups() {
-	have := make(map[module.Version]bool)
 	kill := make(map[*Line]bool)
+
+	// Remove duplicate excludes.
+	haveExclude := make(map[module.Version]bool)
 	for _, x := range f.Exclude {
-		if have[x.Mod] {
+		if haveExclude[x.Mod] {
 			kill[x.Syntax] = true
 			continue
 		}
-		have[x.Mod] = true
+		haveExclude[x.Mod] = true
 	}
 	var excl []*Exclude
 	for _, x := range f.Exclude {
@@ -817,15 +981,16 @@ func (f *File) removeDups() {
 	}
 	f.Exclude = excl
 
-	have = make(map[module.Version]bool)
+	// Remove duplicate replacements.
 	// Later replacements take priority over earlier ones.
+	haveReplace := make(map[module.Version]bool)
 	for i := len(f.Replace) - 1; i >= 0; i-- {
 		x := f.Replace[i]
-		if have[x.Old] {
+		if haveReplace[x.Old] {
 			kill[x.Syntax] = true
 			continue
 		}
-		have[x.Old] = true
+		haveReplace[x.Old] = true
 	}
 	var repl []*Replace
 	for _, x := range f.Replace {
@@ -835,6 +1000,9 @@ func (f *File) removeDups() {
 	}
 	f.Replace = repl
 
+	// Duplicate require and retract directives are not removed.
+
+	// Drop killed statements from the syntax tree.
 	var stmts []Expr
 	for _, stmt := range f.Syntax.Stmt {
 		switch stmt := stmt.(type) {
@@ -858,3 +1026,38 @@ func (f *File) removeDups() {
 	}
 	f.Syntax.Stmt = stmts
 }
+
+// lineLess returns whether li should be sorted before lj. It sorts
+// lexicographically without assigning any special meaning to tokens.
+func lineLess(li, lj *Line) bool {
+	for k := 0; k < len(li.Token) && k < len(lj.Token); k++ {
+		if li.Token[k] != lj.Token[k] {
+			return li.Token[k] < lj.Token[k]
+		}
+	}
+	return len(li.Token) < len(lj.Token)
+}
+
+// lineRetractLess returns whether li should be sorted before lj for lines in
+// a "retract" block. It treats each line as a version interval. Single versions
+// are compared as if they were intervals with the same low and high version.
+// Intervals are sorted in descending order, first by low version, then by
+// high version, using semver.Compare.
+func lineRetractLess(li, lj *Line) bool {
+	interval := func(l *Line) VersionInterval {
+		if len(l.Token) == 1 {
+			return VersionInterval{Low: l.Token[0], High: l.Token[0]}
+		} else if len(l.Token) == 5 && l.Token[0] == "[" && l.Token[2] == "," && l.Token[4] == "]" {
+			return VersionInterval{Low: l.Token[1], High: l.Token[3]}
+		} else {
+			// Line in unknown format. Treat as an invalid version.
+			return VersionInterval{}
+		}
+	}
+	vii := interval(li)
+	vij := interval(lj)
+	if cmp := semver.Compare(vii.Low, vij.Low); cmp != 0 {
+		return cmp > 0
+	}
+	return semver.Compare(vii.High, vij.High) > 0
+}