encoding/json: calculate correct SyntaxError.Offset in the stream

src/encoding/json/decode_test.go

@@ -452,6 +452,8 @@ var unmarshalTests = []unmarshalTest{
 	{in: `{"X": "foo", "Y"}`, err: &SyntaxError{"invalid character '}' after object key", 17}},
 	{in: `[1, 2, 3+]`, err: &SyntaxError{"invalid character '+' after array element", 9}},
 	{in: `{"X":12x}`, err: &SyntaxError{"invalid character 'x' after object key:value pair", 8}, useNumber: true},
+	{in: ``, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 0}},
+	{in: ` `, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 1}},
 	{in: `[2, 3`, err: &SyntaxError{msg: "unexpected end of JSON input", Offset: 5}},
 	{in: `{"F3": -}`, ptr: new(V), out: V{F3: Number("-")}, err: &SyntaxError{msg: "invalid character '}' in numeric literal", Offset: 9}},
 
@@ -1085,6 +1087,11 @@ func equalError(a, b error) bool {
 	if b == nil {
 		return a == nil
 	}
+	ase, aok := a.(*SyntaxError)
+	bse, bok := b.(*SyntaxError)
+	if aok || bok {
+		return aok && bok && *ase == *bse
+	}
 	return a.Error() == b.Error()
 }
 

src/encoding/json/indent.go

@@ -19,7 +19,9 @@ func compact(dst *bytes.Buffer, src []byte, escape bool) error {
 	scan := newScanner()
 	defer freeScanner(scan)
 	start := 0
+	var offset int64
 	for i, c := range src {
+		offset++
 		if escape && (c == '<' || c == '>' || c == '&') {
 			if start < i {
 				dst.Write(src[start:i])
@@ -51,7 +53,7 @@ func compact(dst *bytes.Buffer, src []byte, escape bool) error {
 	}
 	if scan.eof() == scanError {
 		dst.Truncate(origLen)
-		return scan.err
+		return &SyntaxError{msg: scan.errMsg, Offset: offset}
 	}
 	if start < len(src) {
 		dst.Write(src[start:])
@@ -84,8 +86,9 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
 	defer freeScanner(scan)
 	needIndent := false
 	depth := 0
+	var offset int64
 	for _, c := range src {
-		scan.bytes++
+		offset++
 		v := scan.step(scan, c)
 		if v == scanSkipSpace {
 			continue
@@ -137,7 +140,7 @@ func Indent(dst *bytes.Buffer, src []byte, prefix, indent string) error {
 	}
 	if scan.eof() == scanError {
 		dst.Truncate(origLen)
-		return scan.err
+		return &SyntaxError{msg: scan.errMsg, Offset: offset}
 	}
 	return nil
 }

src/encoding/json/scanner.go

@@ -29,14 +29,13 @@ func Valid(data []byte) bool {
 // scan is passed in for use by checkValid to avoid an allocation.
 func checkValid(data []byte, scan *scanner) error {
 	scan.reset()
-	for _, c := range data {
-		scan.bytes++
+	for i, c := range data {
 		if scan.step(scan, c) == scanError {
-			return scan.err
+			return &SyntaxError{msg: scan.errMsg, Offset: int64(i + 1)}
 		}
 	}
 	if scan.eof() == scanError {
-		return scan.err
+		return &SyntaxError{msg: scan.errMsg, Offset: int64(len(data))}
 	}
 	return nil
 }
@@ -75,11 +74,7 @@ type scanner struct {
 	parseState []int
 
 	// Error that happened, if any.
-	err error
-
-	// total bytes consumed, updated by decoder.Decode (and deliberately
-	// not set to zero by scan.reset)
-	bytes int64
+	errMsg string
 }
 
 var scannerPool = sync.Pool{
@@ -90,8 +85,6 @@ var scannerPool = sync.Pool{
 
 func newScanner() *scanner {
 	scan := scannerPool.Get().(*scanner)
-	// scan.reset by design doesn't set bytes to zero
-	scan.bytes = 0
 	scan.reset()
 	return scan
 }
@@ -148,14 +141,14 @@ const maxNestingDepth = 10000
 func (s *scanner) reset() {
 	s.step = stateBeginValue
 	s.parseState = s.parseState[0:0]
-	s.err = nil
+	s.errMsg = ""
 	s.endTop = false
 }
 
 // eof tells the scanner that the end of input has been reached.
 // It returns a scan status just as s.step does.
 func (s *scanner) eof() int {
-	if s.err != nil {
+	if s.errMsg != "" {
 		return scanError
 	}
 	if s.endTop {
@@ -165,8 +158,8 @@ func (s *scanner) eof() int {
 	if s.endTop {
 		return scanEnd
 	}
-	if s.err == nil {
-		s.err = &SyntaxError{"unexpected end of JSON input", s.bytes}
+	if s.errMsg == "" {
+		s.errMsg = "unexpected end of JSON input"
 	}
 	return scanError
 }
@@ -588,7 +581,7 @@ func stateError(s *scanner, c byte) int {
 // error records an error and switches to the error state.
 func (s *scanner) error(c byte, context string) int {
 	s.step = stateError
-	s.err = &SyntaxError{"invalid character " + quoteChar(c) + " " + context, s.bytes}
+	s.errMsg = "invalid character " + quoteChar(c) + " " + context
 	return scanError
 }
 

src/encoding/json/scanner_test.go

@@ -178,18 +178,20 @@ func TestIndentBig(t *testing.T) {
 	}
 }
 
-type indentErrorTest struct {
+type errorTest struct {
 	in  string
 	err error
 }
 
-var indentErrorTests = []indentErrorTest{
+var errorTests = []errorTest{
 	{`{"X": "foo", "Y"}`, &SyntaxError{"invalid character '}' after object key", 17}},
 	{`{"X": "foo" "Y": "bar"}`, &SyntaxError{"invalid character '\"' after object key:value pair", 13}},
+	{``, &SyntaxError{"unexpected end of JSON input", 0}},
+	{`  [  `, &SyntaxError{"unexpected end of JSON input", 5}},
 }
 
 func TestIndentErrors(t *testing.T) {
-	for i, tt := range indentErrorTests {
+	for i, tt := range errorTests {
 		slice := make([]uint8, 0)
 		buf := bytes.NewBuffer(slice)
 		if err := Indent(buf, []uint8(tt.in), "", ""); err != nil {
@@ -201,6 +203,17 @@ func TestIndentErrors(t *testing.T) {
 	}
 }
 
+func TestCompactErrors(t *testing.T) {
+	for i, tt := range errorTests {
+		var buf bytes.Buffer
+		err := Compact(&buf, []byte(tt.in))
+		if !reflect.DeepEqual(err, tt.err) {
+			t.Errorf("#%d: Compact: expected %#v, got: %#v", i, tt.err, err)
+			continue
+		}
+	}
+}
+
 func diff(t *testing.T, a, b []byte) {
 	for i := 0; ; i++ {
 		if i >= len(a) || i >= len(b) || a[i] != b[i] {

src/encoding/json/stream.go

@@ -99,13 +99,8 @@ Input:
 		// Look in the buffer for a new value.
 		for ; scanp < len(dec.buf); scanp++ {
 			c := dec.buf[scanp]
-			dec.scan.bytes++
 			switch dec.scan.step(&dec.scan, c) {
 			case scanEnd:
-				// scanEnd is delayed one byte so we decrement
-				// the scanner bytes count by 1 to ensure that
-				// this value is correct in the next call of Decode.
-				dec.scan.bytes--
 				break Input
 			case scanEndObject, scanEndArray:
 				// scanEnd is delayed one byte.
@@ -116,8 +111,8 @@ Input:
 					break Input
 				}
 			case scanError:
-				dec.err = dec.scan.err
-				return 0, dec.scan.err
+				dec.err = &SyntaxError{msg: dec.scan.errMsg, Offset: dec.scanned + int64(scanp) + 1}
+				return 0, dec.err
 			}
 		}
 

src/encoding/json/stream_test.go

@@ -397,10 +397,14 @@ var tokenStreamCases = []tokenStreamCase{
 	}},
 	{json: `{ "\a" }`, expTokens: []interface{}{
 		Delim('{'),
-		&SyntaxError{"invalid character 'a' in string escape code", 3},
+		&SyntaxError{"invalid character 'a' in string escape code", 5},
 	}},
 	{json: ` \a`, expTokens: []interface{}{
-		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
+		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 2},
+	}},
+	{json: `{ } [] nil`, expTokens: []interface{}{
+		Delim('{'), Delim('}'), Delim('['), Delim(']'),
+		&SyntaxError{"invalid character 'i' in literal null (expecting 'u')", 9},
 	}},
 }