Report exact position of invalid characters in scanner (#44671)

JoshuaKGoldberg · web-flow · commit 0a9b218b11c0 · 2021-06-21T14:23:14.000-07:00
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -2063,8 +2063,9 @@ namespace ts {
                             pos += charSize(ch);
                             continue;
                         }
-                        error(Diagnostics.Invalid_character);
-                        pos += charSize(ch);
+                        const size = charSize(ch);
+                        error(Diagnostics.Invalid_character, pos, size);
+                        pos += size;
                         return token = SyntaxKind.Unknown;
                 }
             }
diff --git a/tests/baselines/reference/MemberFunctionDeclaration8_es6.errors.txt b/tests/baselines/reference/MemberFunctionDeclaration8_es6.errors.txt
@@ -12,7 +12,7 @@ tests/cases/conformance/es6/memberFunctionDeclarations/MemberFunctionDeclaration
         if (a) ¬ * bar;
             ~
 !!! error TS2304: Cannot find name 'a'.
-               
+               ~
 !!! error TS1127: Invalid character.
                  ~
 !!! error TS1109: Expression expected.
diff --git a/tests/baselines/reference/emitBOM.js b/tests/baselines/reference/emitBOM.js
@@ -8,9 +8,9 @@ tests/cases/compiler/emitBOM.js(1,3): error TS1127: Invalid character.
 
 ==== tests/cases/compiler/emitBOM.js (2 errors) ====
     ï»¿// JS and d.ts output should have a BOM but not the sourcemap
-     
+     ~
 !!! error TS1127: Invalid character.
-      
+      ~
 !!! error TS1127: Invalid character.
     var x;
     //# sourceMappingURL=emitBOM.js.map
diff --git a/tests/baselines/reference/parseErrorInHeritageClause1.errors.txt b/tests/baselines/reference/parseErrorInHeritageClause1.errors.txt
@@ -6,6 +6,6 @@ tests/cases/compiler/parseErrorInHeritageClause1.ts(1,19): error TS1127: Invalid
     class C extends A ¬ {
                     ~
 !!! error TS2304: Cannot find name 'A'.
-                      
+                      ~
 !!! error TS1127: Invalid character.
     }
diff --git a/tests/baselines/reference/parserErrorRecovery_Block2.errors.txt b/tests/baselines/reference/parserErrorRecovery_Block2.errors.txt
@@ -4,7 +4,7 @@ tests/cases/conformance/parser/ecmascript5/ErrorRecovery/Blocks/parserErrorRecov
 ==== tests/cases/conformance/parser/ecmascript5/ErrorRecovery/Blocks/parserErrorRecovery_Block2.ts (1 errors) ====
     function f() {
         ¬
-        
+        ~
 !!! error TS1127: Invalid character.
         return;
     }
diff --git a/tests/baselines/reference/parserErrorRecovery_ClassElement3.errors.txt b/tests/baselines/reference/parserErrorRecovery_ClassElement3.errors.txt
@@ -7,7 +7,7 @@ tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ClassElements/parserErr
 ==== tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ClassElements/parserErrorRecovery_ClassElement3.ts (4 errors) ====
     module M {
        ¬
-       
+       ~
 !!! error TS1127: Invalid character.
        class C {
        }
@@ -16,7 +16,7 @@ tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ClassElements/parserErr
        ~~~~
 !!! error TS1109: Expression expected.
        ¬
-       
+       ~
 !!! error TS1127: Invalid character.
         
 !!! error TS1005: '}' expected.
diff --git a/tests/baselines/reference/parserErrorRecovery_ParameterList4.errors.txt b/tests/baselines/reference/parserErrorRecovery_ParameterList4.errors.txt
@@ -3,6 +3,6 @@ tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ParameterLists/parserEr
 
 ==== tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ParameterLists/parserErrorRecovery_ParameterList4.ts (1 errors) ====
     function f(a,¬) {
-                 
+                 ~
 !!! error TS1127: Invalid character.
     }
diff --git a/tests/baselines/reference/parserSkippedTokens16.errors.txt b/tests/baselines/reference/parserSkippedTokens16.errors.txt
@@ -20,7 +20,7 @@ tests/cases/conformance/parser/ecmascript5/SkippedTokens/parserSkippedTokens16.t
                ~
 !!! error TS1005: ';' expected.
     function Foo      () ¬   { }
-                         
+                         ~
 !!! error TS1127: Invalid character.
     4+:5
       ~
diff --git a/tests/baselines/reference/scannerUnexpectedNullCharacter1.errors.txt b/tests/baselines/reference/scannerUnexpectedNullCharacter1.errors.txt
diff --git a/tests/baselines/reference/unicodeIdentifierName2.errors.txt b/tests/baselines/reference/unicodeIdentifierName2.errors.txt
@@ -6,11 +6,11 @@ tests/cases/compiler/unicodeIdentifierName2.ts(1,26): error TS1127: Invalid char
 
 ==== tests/cases/compiler/unicodeIdentifierName2.ts (4 errors) ====
     var a₁ = "hello"; alert(a₁)
-         
+         ~
 !!! error TS1127: Invalid character.
            ~
 !!! error TS1134: Variable declaration expected.
              ~~~~~~~
 !!! error TS1134: Variable declaration expected.
-                             
+                             ~
 !!! error TS1127: Invalid character.

Original file line number	Diff line number	Diff line change
`@@ -2063,8 +2063,9 @@ namespace ts {`
`2063`	`2063`	`pos += charSize(ch);`
`2064`	`2064`	`continue;`
`2065`	`2065`	`}`
`2066`		`- error(Diagnostics.Invalid_character);`
`2067`		`- pos += charSize(ch);`
	`2066`	`+ const size = charSize(ch);`
	`2067`	`+ error(Diagnostics.Invalid_character, pos, size);`
	`2068`	`+ pos += size;`
`2068`	`2069`	`return token = SyntaxKind.Unknown;`
`2069`	`2070`	`}`
`2070`	`2071`	`}`
Original file line number	Diff line number	Diff line change
`@@ -12,7 +12,7 @@ tests/cases/conformance/es6/memberFunctionDeclarations/MemberFunctionDeclaration`
`12`	`12`	`if (a) ¬ * bar;`
`13`	`13`	`~`
`14`	`14`	`!!! error TS2304: Cannot find name 'a'.`
`15`		`-`
	`15`	`+ ~`
`16`	`16`	`!!! error TS1127: Invalid character.`
`17`	`17`	`~`
`18`	`18`	`!!! error TS1109: Expression expected.`
Original file line number	Diff line number	Diff line change
`@@ -6,6 +6,6 @@ tests/cases/compiler/parseErrorInHeritageClause1.ts(1,19): error TS1127: Invalid`
`6`	`6`	`class C extends A ¬ {`
`7`	`7`	`~`
`8`	`8`	`!!! error TS2304: Cannot find name 'A'.`
`9`		`-`
	`9`	`+ ~`
`10`	`10`	`!!! error TS1127: Invalid character.`
`11`	`11`	`}`
Original file line number	Diff line number	Diff line change
`@@ -4,7 +4,7 @@ tests/cases/conformance/parser/ecmascript5/ErrorRecovery/Blocks/parserErrorRecov`
`4`	`4`	`==== tests/cases/conformance/parser/ecmascript5/ErrorRecovery/Blocks/parserErrorRecovery_Block2.ts (1 errors) ====`
`5`	`5`	`function f() {`
`6`	`6`	`¬`
`7`		`-`
	`7`	`+ ~`
`8`	`8`	`!!! error TS1127: Invalid character.`
`9`	`9`	`return;`
`10`	`10`	`}`
Original file line number	Diff line number	Diff line change
`@@ -3,6 +3,6 @@ tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ParameterLists/parserEr`
`3`	`3`
`4`	`4`	`==== tests/cases/conformance/parser/ecmascript5/ErrorRecovery/ParameterLists/parserErrorRecovery_ParameterList4.ts (1 errors) ====`
`5`	`5`	`function f(a,¬) {`
`6`		`-`
	`6`	`+ ~`
`7`	`7`	`!!! error TS1127: Invalid character.`
`8`	`8`	`}`
Original file line number	Diff line number	Diff line change
`@@ -20,7 +20,7 @@ tests/cases/conformance/parser/ecmascript5/SkippedTokens/parserSkippedTokens16.t`
`20`	`20`	`~`
`21`	`21`	`!!! error TS1005: ';' expected.`
`22`	`22`	`function Foo () ¬ { }`
`23`		`-`
	`23`	`+ ~`
`24`	`24`	`!!! error TS1127: Invalid character.`
`25`	`25`	`4+:5`
`26`	`26`	`~`