encoding/json: calculate correct SyntaxError.Offset in the stream

Stream decoder does not count whitespaces, empty objects and arrays
in syntax error offset. This change removes offset tracking from the
scanner and relies on the calling code to provide the correct value.

Fixes #44811, #34543

Author: AlexanderYastrebov

src/encoding/json/decode_test.go

@@ -452,6 +452,8 @@ var unmarshalTests = []unmarshalTest{
 	{in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}},
 	{in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}},
 	{in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true},
+	{in: ``, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 0}},
+	{in: ` `, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 1}},
 	{in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}},
 	{in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}},
 
@@ -1085,6 +1087,11 @@ func equalError(a, b error) bool {
 	if b == nil {
 		return a == nil
 	}
+	ase, aok := a.(*SyntaxError)
+	bse, bok := b.(*SyntaxError)
+	if aok || bok {
+		return aok && bok && *ase == *bse
+	}
 	return a.Error() == b.Error()
 }
 

src/encoding/json/indent.go

@@ -19,7 +19,9 @@ func compact(dst *bytes.Buffer, src []byte, escape bool) error {
 	scan := newScanner()
 	defer freeScanner(scan)
 	start := 0
+	var offset int64
 	for i, c := range src {
+		offset++
 		if escape && (c == '<' || c == '>' || c == '&') {
 			if start < i {
 				dst.Write(src[start:i])
@@ -51,7 +53,7 @@ func compact(dst *bytes.Buffer, src []byte, escape bool) error {
 	}
 	if scan.eof() == scanError {
 		dst.Truncate(origLen)
-		return scan.err
+		return &SyntaxError{msg: scan.errMsg, Offset: offset}
 	}
 	if start < len(src) {
 		dst.Write(src[start:])
@@ -84,8 +86,9 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
 	defer freeScanner(scan)
 	needIndent := false
 	depth := 0
+	var offset int64
 	for _, c := range src {
-		scan.bytes++
+		offset++
 		v := scan.step(scan, c)
 		if v == scanSkipSpace {
 			continue
@@ -137,7 +140,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
 	}
 	if scan.eof() == scanError {
 		dst.Truncate(origLen)
-		return scan.err
+		return &SyntaxError{msg: scan.errMsg, Offset: offset}
 	}
 	return nil
 }

src/encoding/json/scanner.go

@@ -29,14 +29,13 @@ func Valid(data []byte) bool {
 // scan is passed in for use by checkValid to avoid an allocation.
 func checkValid(data []byte, scan *scanner) error {
 	scan.reset()
-	for _, c := range data {
-		scan.bytes++
+	for i, c := range data {
 		if scan.step(scan, c) == scanError {
-			return scan.err
+			return &SyntaxError{msg: scan.errMsg, Offset: int64(i + 1)}
 		}
 	}
 	if scan.eof() == scanError {
-		return scan.err
+		return &SyntaxError{msg: scan.errMsg, Offset: int64(len(data))}
 	}
 	return nil
 }
@@ -75,11 +74,7 @@ type scanner struct {
 	parseState []int
 
 	// Error that happened, if any.
-	err error
-
-	// total bytes consumed, updated by decoder.Decode (and deliberately
-	// not set to zero by scan.reset)
-	bytes int64
+	errMsg string
 }
 
 var scannerPool = sync.Pool{
@@ -90,8 +85,6 @@ var scannerPool = sync.Pool{
 
 func newScanner() *scanner {
 	scan := scannerPool.Get().(*scanner)
-	// scan.reset by design doesn't set bytes to zero
-	scan.bytes = 0
 	scan.reset()
 	return scan
 }
@@ -148,14 +141,14 @@ const maxNestingDepth = 10000
 func (s *scanner) reset() {
 	s.step = stateBeginValue
 	s.parseState = s.parseState[0:0]
-	s.err = nil
+	s.errMsg = ""
 	s.endTop = false
 }
 
 // eof tells the scanner that the end of input has been reached.
 // It returns a scan status just as s.step does.
 func (s *scanner) eof() int {
-	if s.err != nil {
+	if s.errMsg != "" {
 		return scanError
 	}
 	if s.endTop {
@@ -165,8 +158,8 @@ func (s *scanner) eof() int {
 	if s.endTop {
 		return scanEnd
 	}
-	if s.err == nil {
-		s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
+	if s.errMsg == "" {
+		s.errMsg = "unexpected end of JSON input"
 	}
 	return scanError
 }
@@ -588,7 +581,7 @@ func stateError(s *scanner, c byte) int {
 // error records an error and switches to the error state.
 func (s *scanner) error(c byte, context string) int {
 	s.step = stateError
-	s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
+	s.errMsg = "invalid character " + quoteChar(c) + " " + context
 	return scanError
 }
 

src/encoding/json/scanner_test.go

@@ -178,18 +178,20 @@ func TestIndentBig(t *testing.T) {
 	}
 }
 
-type indentErrorTest struct {
+type errorTest struct {
 	in  string
 	err error
 }
 
-var indentErrorTests = []indentErrorTest{
+var errorTests = []errorTest{
 	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
 	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
+	{``, &SyntaxError{"unexpected end of JSON input", 0}},
+	{`  [  `, &SyntaxError{"unexpected end of JSON input", 5}},
 }
 
 func TestIndentErrors(t *testing.T) {
-	for i, tt := range indentErrorTests {
+	for i, tt := range errorTests {
 		slice := make([]uint8, 0)
 		buf := bytes.NewBuffer(slice)
 		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
@@ -201,6 +203,17 @@ func TestIndentErrors(t *testing.T) {
 	}
 }
 
+func TestCompactErrors(t *testing.T) {
+	for i, tt := range errorTests {
+		var buf bytes.Buffer
+		err := Compact(&buf, []byte(tt.in))
+		if !reflect.DeepEqual(err, tt.err) {
+			t.Errorf("#%d: Compact: expected %#v, got: %#v", i, tt.err, err)
+			continue
+		}
+	}
+}
+
 func diff(t *testing.T, a, b []byte) {
 	for i := 0; ; i++ {
 		if i >= len(a) || i >= len(b) || a[i] != b[i] {

src/encoding/json/stream.go

@@ -99,13 +99,8 @@ Input:
 		// Look in the buffer for a new value.
 		for ; scanp < len(dec.buf); scanp++ {
 			c := dec.buf[scanp]
-			dec.scan.bytes++
 			switch dec.scan.step(&dec.scan, c) {
 			case scanEnd:
-				// scanEnd is delayed one byte so we decrement
-				// the scanner bytes count by 1 to ensure that
-				// this value is correct in the next call of Decode.
-				dec.scan.bytes--
 				break Input
 			case scanEndObject, scanEndArray:
 				// scanEnd is delayed one byte.
@@ -116,8 +111,8 @@ Input:
 					break Input
 				}
 			case scanError:
-				dec.err = dec.scan.err
-				return 0, dec.scan.err
+				dec.err = &SyntaxError{msg: dec.scan.errMsg, Offset: dec.scanned + int64(scanp) + 1}
+				return 0, dec.err
 			}
 		}
 

src/encoding/json/stream_test.go

@@ -397,10 +397,14 @@ var tokenStreamCases = []tokenStreamCase{
 	}},
 	{json: `{ "\a" }`, expTokens: []interface{}{
 		Delim('{'),
-		&SyntaxError{"invalid character 'a' in string escape code", 3},
+		&SyntaxError{"invalid character 'a' in string escape code", 5},
 	}},
 	{json: ` \a`, expTokens: []interface{}{
-		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
+		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 2},
+	}},
+	{json: `{ } [] nil`, expTokens: []interface{}{
+		Delim('{'), Delim('}'), Delim('['), Delim(']'),
+		&SyntaxError{"invalid character 'i' in literal null (expecting 'u')", 9},
 	}},
 }