[dev.inline] cmd/internal/src: replace src.Pos with syntax.Pos

This replaces the src.Pos LineHist-based position tracking with
the syntax.Pos implementation and updates all uses.

The LineHist table is not used anymore - the respective code is still
there but should be removed eventually. CL forthcoming.

Passes toolstash -cmp when comparing to the master repo (with the
exception of a couple of swapped assembly instructions, likely due
to different instruction scheduling because the line-based sorting
has changed; though this is won't affect correctness).

The sizes of various important compiler data structures have increased
significantly (see the various sizes_test.go files); this is probably
the reason for an increase of compilation times (to be addressed). Here
are the results of compilebench -count 5, run on a "quiet" machine (no
apps running besides a terminal):

name       old time/op     new time/op     delta
Template       256ms ± 1%      280ms ±15%  +9.54%          (p=0.008 n=5+5)
Unicode        132ms ± 1%      132ms ± 1%    ~             (p=0.690 n=5+5)
GoTypes        891ms ± 1%      917ms ± 2%  +2.88%          (p=0.008 n=5+5)
Compiler       3.84s ± 2%      3.99s ± 2%  +3.95%          (p=0.016 n=5+5)
MakeBash       47.1s ± 1%      47.2s ± 2%    ~             (p=0.841 n=5+5)

name       old user-ns/op  new user-ns/op  delta
Template        309M ± 1%       326M ± 2%  +5.18%          (p=0.008 n=5+5)
Unicode         165M ± 1%       168M ± 4%    ~             (p=0.421 n=5+5)
GoTypes        1.14G ± 2%      1.18G ± 1%  +3.47%          (p=0.008 n=5+5)
Compiler       5.00G ± 1%      5.16G ± 1%  +3.12%          (p=0.008 n=5+5)

Change-Id: I241c4246cdff627d7ecb95cac23060b38f9775ec
Reviewed-on: https://go-review.googlesource.com/34273
Run-TryBot: Robert Griesemer <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Matthew Dempsky <[email protected]>

Author: griesemer

src/cmd/asm/internal/asm/asm.go

@@ -13,7 +13,6 @@ import (
 	"cmd/asm/internal/flags"
 	"cmd/asm/internal/lex"
 	"cmd/internal/obj"
-	"cmd/internal/src"
 	"cmd/internal/sys"
 )
 
@@ -62,7 +61,7 @@ func (p *Parser) append(prog *obj.Prog, cond string, doLabel bool) {
 	}
 	prog.Pc = p.pc
 	if *flags.Debug {
-		fmt.Println(p.histLineNum, prog)
+		fmt.Println(p.lineNum, prog)
 	}
 	if testOut != nil {
 		fmt.Fprintln(testOut, prog)
@@ -164,7 +163,7 @@ func (p *Parser) asmText(word string, operands [][]lex.Token) {
 	prog := &obj.Prog{
 		Ctxt: p.ctxt,
 		As:   obj.ATEXT,
-		Pos:  src.MakePos(p.histLineNum),
+		Pos:  p.pos(),
 		From: nameAddr,
 		From3: &obj.Addr{
 			Type:   obj.TYPE_CONST,
@@ -297,7 +296,7 @@ func (p *Parser) asmPCData(word string, operands [][]lex.Token) {
 	prog := &obj.Prog{
 		Ctxt: p.ctxt,
 		As:   obj.APCDATA,
-		Pos:  src.MakePos(p.histLineNum),
+		Pos:  p.pos(),
 		From: key,
 		To:   value,
 	}
@@ -327,7 +326,7 @@ func (p *Parser) asmFuncData(word string, operands [][]lex.Token) {
 	prog := &obj.Prog{
 		Ctxt: p.ctxt,
 		As:   obj.AFUNCDATA,
-		Pos:  src.MakePos(p.histLineNum),
+		Pos:  p.pos(),
 		From: valueAddr,
 		To:   nameAddr,
 	}
@@ -342,7 +341,7 @@ func (p *Parser) asmJump(op obj.As, cond string, a []obj.Addr) {
 	var target *obj.Addr
 	prog := &obj.Prog{
 		Ctxt: p.ctxt,
-		Pos:  src.MakePos(p.histLineNum),
+		Pos:  p.pos(),
 		As:   op,
 	}
 	switch len(a) {
@@ -470,7 +469,7 @@ func (p *Parser) asmInstruction(op obj.As, cond string, a []obj.Addr) {
 	// fmt.Printf("%s %+v\n", op, a)
 	prog := &obj.Prog{
 		Ctxt: p.ctxt,
-		Pos:  src.MakePos(p.histLineNum),
+		Pos:  p.pos(),
 		As:   op,
 	}
 	switch len(a) {

src/cmd/asm/internal/asm/endtoend_test.go

@@ -26,10 +26,9 @@ import (
 // result against a golden file.
 
 func testEndToEnd(t *testing.T, goarch, file string) {
-	lex.InitHist()
 	input := filepath.Join("testdata", file+".s")
 	architecture, ctxt := setArch(goarch)
-	lexer := lex.NewLexer(input, ctxt)
+	lexer := lex.NewLexer(input)
 	parser := NewParser(ctxt, architecture, lexer)
 	pList := obj.Linknewplist(ctxt)
 	var ok bool
@@ -264,10 +263,9 @@ var (
 )
 
 func testErrors(t *testing.T, goarch, file string) {
-	lex.InitHist()
 	input := filepath.Join("testdata", file+".s")
 	architecture, ctxt := setArch(goarch)
-	lexer := lex.NewLexer(input, ctxt)
+	lexer := lex.NewLexer(input)
 	parser := NewParser(ctxt, architecture, lexer)
 	pList := obj.Linknewplist(ctxt)
 	var ok bool

src/cmd/asm/internal/asm/parse.go

@@ -19,14 +19,14 @@ import (
 	"cmd/asm/internal/flags"
 	"cmd/asm/internal/lex"
 	"cmd/internal/obj"
+	"cmd/internal/src"
 	"cmd/internal/sys"
 )
 
 type Parser struct {
 	lex           lex.TokenReader
 	lineNum       int   // Line number in source file.
-	histLineNum   int32 // Cumulative line number across source files.
-	errorLine     int32 // (Cumulative) line number of last error.
+	errorLine     int   // Line number of last error.
 	errorCount    int   // Number of errors.
 	pc            int64 // virtual PC; count of Progs; doesn't advance for GLOBL or DATA.
 	input         []lex.Token
@@ -60,19 +60,19 @@ func NewParser(ctxt *obj.Link, ar *arch.Arch, lexer lex.TokenReader) *Parser {
 	}
 }
 
-// panicOnError is enable when testing to abort execution on the first error
+// panicOnError is enabled when testing to abort execution on the first error
 // and turn it into a recoverable panic.
 var panicOnError bool
 
 func (p *Parser) errorf(format string, args ...interface{}) {
 	if panicOnError {
 		panic(fmt.Errorf(format, args...))
 	}
-	if p.histLineNum == p.errorLine {
+	if p.lineNum == p.errorLine {
 		// Only one error per line.
 		return
 	}
-	p.errorLine = p.histLineNum
+	p.errorLine = p.lineNum
 	if p.lex != nil {
 		// Put file and line information on head of message.
 		format = "%s:%d: " + format + "\n"
@@ -85,6 +85,10 @@ func (p *Parser) errorf(format string, args ...interface{}) {
 	}
 }
 
+func (p *Parser) pos() src.Pos {
+	return src.MakePos(p.lex.Base(), uint(p.lineNum), 0)
+}
+
 func (p *Parser) Parse() (*obj.Prog, bool) {
 	for p.line() {
 	}
@@ -105,7 +109,6 @@ func (p *Parser) line() bool {
 		// are labeled with this line. Otherwise we complain after we've absorbed
 		// the terminating newline and the line numbers are off by one in errors.
 		p.lineNum = p.lex.Line()
-		p.histLineNum = lex.HistLine()
 		switch tok {
 		case '\n', ';':
 			continue

src/cmd/asm/internal/asm/pseudo_test.go

@@ -56,7 +56,6 @@ func TestErroneous(t *testing.T) {
 	for _, test := range tests {
 		parser.errorCount = 0
 		parser.lineNum++
-		parser.histLineNum++
 		if !parser.pseudo(test.pseudo, tokenize(test.operands)) {
 			t.Fatalf("Wrong pseudo-instruction: %s", test.pseudo)
 		}

src/cmd/asm/internal/lex/input.go

@@ -13,6 +13,7 @@ import (
 	"text/scanner"
 
 	"cmd/asm/internal/flags"
+	"cmd/internal/src"
 )
 
 // Input is the main input: a stack of readers and some macro definitions.
@@ -290,7 +291,7 @@ func lookup(args []string, arg string) int {
 func (in *Input) invokeMacro(macro *Macro) {
 	// If the macro has no arguments, just substitute the text.
 	if macro.args == nil {
-		in.Push(NewSlice(in.File(), in.Line(), macro.tokens))
+		in.Push(NewSlice(in.Base(), in.Line(), macro.tokens))
 		return
 	}
 	tok := in.Stack.Next()
@@ -300,7 +301,7 @@ func (in *Input) invokeMacro(macro *Macro) {
 		in.peekToken = tok
 		in.peekText = in.text
 		in.peek = true
-		in.Push(NewSlice(in.File(), in.Line(), []Token{Make(macroName, macro.name)}))
+		in.Push(NewSlice(in.Base(), in.Line(), []Token{Make(macroName, macro.name)}))
 		return
 	}
 	actuals := in.argsFor(macro)
@@ -317,7 +318,7 @@ func (in *Input) invokeMacro(macro *Macro) {
 		}
 		tokens = append(tokens, substitution...)
 	}
-	in.Push(NewSlice(in.File(), in.Line(), tokens))
+	in.Push(NewSlice(in.Base(), in.Line(), tokens))
 }
 
 // argsFor returns a map from formal name to actual value for this argumented macro invocation.
@@ -452,8 +453,8 @@ func (in *Input) line() {
 	if tok != '\n' {
 		in.Error("unexpected token at end of #line: ", tok)
 	}
-	linkCtxt.LineHist.Update(histLine, file, line)
-	in.Stack.SetPos(line, file)
+	pos := src.MakePos(in.Base(), uint(in.Line()), uint(in.Col()))
+	in.Stack.SetBase(src.NewLinePragmaBase(pos, file, uint(line)))
 }
 
 // #undef processing

src/cmd/asm/internal/lex/lex.go

@@ -12,7 +12,7 @@ import (
 	"strings"
 	"text/scanner"
 
-	"cmd/internal/obj"
+	"cmd/internal/src"
 )
 
 // A ScanToken represents an input item. It is a simple wrapping of rune, as
@@ -57,23 +57,8 @@ func (t ScanToken) String() string {
 	}
 }
 
-var (
-	// It might be nice if these weren't global.
-	linkCtxt *obj.Link     // The link context for all instructions.
-	histLine int       = 1 // The cumulative count of lines processed.
-)
-
-// HistLine reports the cumulative source line number of the token,
-// for use in the Prog structure for the linker. (It's always handling the
-// instruction from the current lex line.)
-// It returns int32 because that's what type ../asm prefers.
-func HistLine() int32 {
-	return int32(histLine)
-}
-
 // NewLexer returns a lexer for the named file and the given link context.
-func NewLexer(name string, ctxt *obj.Link) TokenReader {
-	linkCtxt = ctxt
+func NewLexer(name string) TokenReader {
 	input := NewInput(name)
 	fd, err := os.Open(name)
 	if err != nil {
@@ -83,16 +68,11 @@ func NewLexer(name string, ctxt *obj.Link) TokenReader {
 	return input
 }
 
-// InitHist sets the line count to 1, for reproducible testing.
-func InitHist() {
-	histLine = 1
-}
-
 // The other files in this directory each contain an implementation of TokenReader.
 
 // A TokenReader is like a reader, but returns lex tokens of type Token. It also can tell you what
 // the text of the most recently returned token is, and where it was found.
-// The underlying scanner elides all spaces except newline, so the input looks like a  stream of
+// The underlying scanner elides all spaces except newline, so the input looks like a stream of
 // Tokens; original spacing is lost but we don't need it.
 type TokenReader interface {
 	// Next returns the next token.
@@ -102,12 +82,14 @@ type TokenReader interface {
 	Text() string
 	// File reports the source file name of the token.
 	File() string
+	// Base reports the position base of the token.
+	Base() *src.PosBase
+	// SetBase sets the position base.
+	SetBase(*src.PosBase)
 	// Line reports the source line number of the token.
 	Line() int
 	// Col reports the source column number of the token.
 	Col() int
-	// SetPos sets the file and line number.
-	SetPos(line int, file string)
 	// Close does any teardown required.
 	Close()
 }

src/cmd/asm/internal/lex/slice.go

@@ -4,22 +4,26 @@
 
 package lex
 
-import "text/scanner"
+import (
+	"text/scanner"
+
+	"cmd/internal/src"
+)
 
 // A Slice reads from a slice of Tokens.
 type Slice struct {
-	tokens   []Token
-	fileName string
-	line     int
-	pos      int
+	tokens []Token
+	base   *src.PosBase
+	line   int
+	pos    int
 }
 
-func NewSlice(fileName string, line int, tokens []Token) *Slice {
+func NewSlice(base *src.PosBase, line int, tokens []Token) *Slice {
 	return &Slice{
-		tokens:   tokens,
-		fileName: fileName,
-		line:     line,
-		pos:      -1, // Next will advance to zero.
+		tokens: tokens,
+		base:   base,
+		line:   line,
+		pos:    -1, // Next will advance to zero.
 	}
 }
 
@@ -36,7 +40,17 @@ func (s *Slice) Text() string {
 }
 
 func (s *Slice) File() string {
-	return s.fileName
+	return s.base.Filename()
+}
+
+func (s *Slice) Base() *src.PosBase {
+	return s.base
+}
+
+func (s *Slice) SetBase(base *src.PosBase) {
+	// Cannot happen because we only have slices of already-scanned text,
+	// but be prepared.
+	s.base = base
 }
 
 func (s *Slice) Line() int {
@@ -56,12 +70,5 @@ func (s *Slice) Col() int {
 	return s.pos
 }
 
-func (s *Slice) SetPos(line int, file string) {
-	// Cannot happen because we only have slices of already-scanned
-	// text, but be prepared.
-	s.line = line
-	s.fileName = file
-}
-
 func (s *Slice) Close() {
 }

src/cmd/asm/internal/lex/stack.go

@@ -4,7 +4,11 @@
 
 package lex
 
-import "text/scanner"
+import (
+	"text/scanner"
+
+	"cmd/internal/src"
+)
 
 // A Stack is a stack of TokenReaders. As the top TokenReader hits EOF,
 // it resumes reading the next one down.
@@ -34,7 +38,15 @@ func (s *Stack) Text() string {
 }
 
 func (s *Stack) File() string {
-	return s.tr[len(s.tr)-1].File()
+	return s.Base().Filename()
+}
+
+func (s *Stack) Base() *src.PosBase {
+	return s.tr[len(s.tr)-1].Base()
+}
+
+func (s *Stack) SetBase(base *src.PosBase) {
+	s.tr[len(s.tr)-1].SetBase(base)
 }
 
 func (s *Stack) Line() int {
@@ -45,9 +57,5 @@ func (s *Stack) Col() int {
 	return s.tr[len(s.tr)-1].Col()
 }
 
-func (s *Stack) SetPos(line int, file string) {
-	s.tr[len(s.tr)-1].SetPos(line, file)
-}
-
 func (s *Stack) Close() { // Unused.
 }