encoding/json: fix byte counter increments when using decoder.Token()

At every iteration of decoder.Token(), scaner.bytes does not increment
when it meets spaces and other non value token.

Signed-off-by: Peng Gao <[email protected]>
Change-Id: Id41b7e22e2c45fffc7fafacd618724f0f214dd2f

Author: ggaaooppeenngg

src/encoding/json/stream.go

@@ -379,6 +379,7 @@ func (dec *Decoder) Token() (Token, error) {
 				return dec.tokenError(c)
 			}
 			dec.scanp++
+			dec.scan.bytes++
 			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
 			dec.tokenState = tokenArrayStart
 			return Delim('['), nil
@@ -388,6 +389,7 @@ func (dec *Decoder) Token() (Token, error) {
 				return dec.tokenError(c)
 			}
 			dec.scanp++
+			dec.scan.bytes++
 			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
 			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
 			dec.tokenValueEnd()
@@ -398,6 +400,7 @@ func (dec *Decoder) Token() (Token, error) {
 				return dec.tokenError(c)
 			}
 			dec.scanp++
+			dec.scan.bytes++
 			dec.tokenStack = append(dec.tokenStack, dec.tokenState)
 			dec.tokenState = tokenObjectStart
 			return Delim('{'), nil
@@ -407,6 +410,7 @@ func (dec *Decoder) Token() (Token, error) {
 				return dec.tokenError(c)
 			}
 			dec.scanp++
+			dec.scan.bytes++
 			dec.tokenState = dec.tokenStack[len(dec.tokenStack)-1]
 			dec.tokenStack = dec.tokenStack[:len(dec.tokenStack)-1]
 			dec.tokenValueEnd()
@@ -417,17 +421,20 @@ func (dec *Decoder) Token() (Token, error) {
 				return dec.tokenError(c)
 			}
 			dec.scanp++
+			dec.scan.bytes++
 			dec.tokenState = tokenObjectValue
 			continue
 
 		case ',':
 			if dec.tokenState == tokenArrayComma {
 				dec.scanp++
+				dec.scan.bytes++
 				dec.tokenState = tokenArrayValue
 				continue
 			}
 			if dec.tokenState == tokenObjectComma {
 				dec.scanp++
+				dec.scan.bytes++
 				dec.tokenState = tokenObjectKey
 				continue
 			}
@@ -493,6 +500,7 @@ func (dec *Decoder) peek() (byte, error) {
 		for i := dec.scanp; i < len(dec.buf); i++ {
 			c := dec.buf[i]
 			if isSpace(c) {
+				dec.scan.bytes++
 				continue
 			}
 			dec.scanp = i

src/encoding/json/stream_test.go

@@ -397,10 +397,14 @@ var tokenStreamCases = []tokenStreamCase{
 	}},
 	{json: `{ "\a" }`, expTokens: []interface{}{
 		Delim('{'),
-		&SyntaxError{"invalid character 'a' in string escape code", 3},
+		&SyntaxError{"invalid character 'a' in string escape code", 5},
 	}},
 	{json: ` \a`, expTokens: []interface{}{
-		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 1},
+		&SyntaxError{"invalid character '\\\\' looking for beginning of value", 2},
+	}},
+	{json: `[$t]`, expTokens: []interface{}{
+		Delim('['),
+		&SyntaxError{"invalid character '$' looking for beginning of value", 2},
 	}},
 }