Skip to content

Commit 886feda

Browse files
CyrusNajmabadiCyrusNajmabadi
committed
Update the new scanner to follow the new regex scanning rules.
1 parent 7dd30d3 commit 886feda

15 files changed

+60
-55
lines changed

‎src/compiler/diagnosticInformationMap.generated.ts

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -122,6 +122,8 @@ module ts {
122122
let_declarations_can_only_be_declared_inside_a_block: { code: 1157, category: DiagnosticCategory.Error, key: "'let' declarations can only be declared inside a block." },
123123
Invalid_template_literal_expected: { code: 1158, category: DiagnosticCategory.Error, key: "Invalid template literal; expected '}'" },
124124
Tagged_templates_are_only_available_when_targeting_ECMAScript_6_and_higher: { code: 1159, category: DiagnosticCategory.Error, key: "Tagged templates are only available when targeting ECMAScript 6 and higher." },
125+
Unterminated_template_literal: { code: 1160, category: DiagnosticCategory.Error, key: "Unterminated template literal." },
126+
Unterminated_regular_expression_literal: { code: 1161, category: DiagnosticCategory.Error, key: "Unterminated regular expression literal." },
125127
A_object_member_cannot_be_declared_optional: { code: 1160, category: DiagnosticCategory.Error, key: "A object member cannot be declared optional." },
126128
Duplicate_identifier_0: { code: 2300, category: DiagnosticCategory.Error, key: "Duplicate identifier '{0}'." },
127129
Initializer_of_instance_member_variable_0_cannot_reference_identifier_1_declared_in_the_constructor: { code: 2301, category: DiagnosticCategory.Error, key: "Initializer of instance member variable '{0}' cannot reference identifier '{1}' declared in the constructor." },

‎src/compiler/diagnosticMessages.json

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -479,6 +479,14 @@
479479
"category": "Error",
480480
"code": 1159
481481
},
482+
"Unterminated template literal.": {
483+
"category": "Error",
484+
"code": 1160
485+
},
486+
"Unterminated regular expression literal.": {
487+
"category": "Error",
488+
"code": 1161
489+
},
482490

483491
"A object member cannot be declared optional.": {
484492
"category": "Error",

‎src/compiler/scanner.ts

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -553,7 +553,7 @@ module ts {
553553
while (true) {
554554
if (pos >= len) {
555555
result += text.substring(start, pos);
556-
error(Diagnostics.Unexpected_end_of_text);
556+
error(Diagnostics.Unterminated_string_literal);
557557
break;
558558
}
559559
var ch = text.charCodeAt(pos);
@@ -593,7 +593,7 @@ module ts {
593593
while (true) {
594594
if (pos >= len) {
595595
contents += text.substring(start, pos);
596-
error(Diagnostics.Unexpected_end_of_text);
596+
error(Diagnostics.Unterminated_template_literal);
597597
resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
598598
break;
599599
}
@@ -1066,19 +1066,19 @@ module ts {
10661066
var inEscape = false;
10671067
var inCharacterClass = false;
10681068
while (true) {
1069-
// If we've hit EOF without closing off the regex,
1070-
// simply return the token we originally parsed.
1069+
// If we reach the end of a file, or hit a newline, then this is an unterminated
1070+
// regex. Report error and return what we have so far.
10711071
if (p >= len) {
1072-
return token;
1072+
error(Diagnostics.Unterminated_regular_expression_literal)
1073+
break;
10731074
}
10741075

10751076
var ch = text.charCodeAt(p);
1076-
1077-
// Line breaks are not permissible in the middle of a RegExp.
10781077
if (isLineBreak(ch)) {
1079-
return token;
1078+
error(Diagnostics.Unterminated_regular_expression_literal)
1079+
break;
10801080
}
1081-
1081+
10821082
if (inEscape) {
10831083
// Parsing an escape character;
10841084
// reset the flag and just advance to the next char.
@@ -1087,6 +1087,7 @@ module ts {
10871087
else if (ch === CharacterCodes.slash && !inCharacterClass) {
10881088
// A slash within a character class is permissible,
10891089
// but in general it signals the end of the regexp literal.
1090+
p++;
10901091
break;
10911092
}
10921093
else if (ch === CharacterCodes.openBracket) {
@@ -1100,8 +1101,8 @@ module ts {
11001101
}
11011102
p++;
11021103
}
1103-
p++;
1104-
while (isIdentifierPart(text.charCodeAt(p))) {
1104+
1105+
while (p < len && isIdentifierPart(text.charCodeAt(p))) {
11051106
p++;
11061107
}
11071108
pos = p;

‎tests/baselines/reference/noEmitOnError.errors.txt

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,9 @@
1-
tests/cases/compiler/noEmitOnError.ts(2,5): error TS2322: Type 'string' is not assignable to type 'number'.
2-
3-
4-
==== tests/cases/compiler/noEmitOnError.ts (1 errors) ====
5-
6-
var x: number = "";
7-
~
8-
!!! error TS2322: Type 'string' is not assignable to type 'number'.
1+
tests/cases/compiler/noEmitOnError.ts(2,5): error TS2322: Type 'string' is not assignable to type 'number'.
2+
3+
4+
==== tests/cases/compiler/noEmitOnError.ts (1 errors) ====
5+
6+
var x: number = "";
7+
~
8+
!!! error TS2322: Type 'string' is not assignable to type 'number'.
99

‎tests/baselines/reference/parser645086_1.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_1.ts(1,13): error TS1005: ',' expected.
22
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_1.ts(1,14): error TS1134: Variable declaration expected.
3-
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_1.ts(1,15): error TS1109: Expression expected.
3+
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_1.ts(1,15): error TS1161: Unterminated regular expression literal.
44

55

66
==== tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_1.ts (3 errors) ====
@@ -10,4 +10,4 @@ tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_1.ts(1,1
1010
~
1111
!!! error TS1134: Variable declaration expected.
1212

13-
!!! error TS1109: Expression expected.
13+
!!! error TS1161: Unterminated regular expression literal.

‎tests/baselines/reference/parser645086_2.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_2.ts(1,14): error TS1005: ',' expected.
22
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_2.ts(1,15): error TS1134: Variable declaration expected.
3-
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_2.ts(1,16): error TS1109: Expression expected.
3+
tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_2.ts(1,16): error TS1161: Unterminated regular expression literal.
44

55

66
==== tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_2.ts (3 errors) ====
@@ -10,4 +10,4 @@ tests/cases/conformance/parser/ecmascript5/RegressionTests/parser645086_2.ts(1,1
1010
~
1111
!!! error TS1134: Variable declaration expected.
1212

13-
!!! error TS1109: Expression expected.
13+
!!! error TS1161: Unterminated regular expression literal.

‎tests/baselines/reference/parserMissingToken2.errors.txt

Lines changed: 4 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
1-
tests/cases/conformance/parser/ecmascript5/MissingTokens/parserMissingToken2.ts(1,1): error TS1109: Expression expected.
2-
tests/cases/conformance/parser/ecmascript5/MissingTokens/parserMissingToken2.ts(1,3): error TS2304: Cannot find name 'b'.
1+
tests/cases/conformance/parser/ecmascript5/MissingTokens/parserMissingToken2.ts(1,2): error TS1161: Unterminated regular expression literal.
32

43

5-
==== tests/cases/conformance/parser/ecmascript5/MissingTokens/parserMissingToken2.ts (2 errors) ====
4+
==== tests/cases/conformance/parser/ecmascript5/MissingTokens/parserMissingToken2.ts (1 errors) ====
65
/ b;
7-
~
8-
!!! error TS1109: Expression expected.
9-
~
10-
!!! error TS2304: Cannot find name 'b'.
6+
7+
!!! error TS1161: Unterminated regular expression literal.

‎tests/baselines/reference/parserRegularExpressionDivideAmbiguity4.errors.txt

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,13 @@
1-
tests/cases/conformance/parser/ecmascript5/RegularExpressions/parserRegularExpressionDivideAmbiguity4.ts(1,5): error TS1109: Expression expected.
1+
tests/cases/conformance/parser/ecmascript5/RegularExpressions/parserRegularExpressionDivideAmbiguity4.ts(1,6): error TS1161: Unterminated regular expression literal.
2+
tests/cases/conformance/parser/ecmascript5/RegularExpressions/parserRegularExpressionDivideAmbiguity4.ts(1,17): error TS1005: ')' expected.
23
tests/cases/conformance/parser/ecmascript5/RegularExpressions/parserRegularExpressionDivideAmbiguity4.ts(1,1): error TS2304: Cannot find name 'foo'.
3-
tests/cases/conformance/parser/ecmascript5/RegularExpressions/parserRegularExpressionDivideAmbiguity4.ts(1,6): error TS2304: Cannot find name 'notregexp'.
44

55

66
==== tests/cases/conformance/parser/ecmascript5/RegularExpressions/parserRegularExpressionDivideAmbiguity4.ts (3 errors) ====
77
foo(/notregexp);
8-
~
9-
!!! error TS1109: Expression expected.
8+
9+
!!! error TS1161: Unterminated regular expression literal.
10+
11+
!!! error TS1005: ')' expected.
1012
~~~
11-
!!! error TS2304: Cannot find name 'foo'.
12-
~~~~~~~~~
13-
!!! error TS2304: Cannot find name 'notregexp'.
13+
!!! error TS2304: Cannot find name 'foo'.

‎tests/baselines/reference/scannerStringLiterals.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
tests/cases/conformance/scanner/ecmascript5/scannerStringLiterals.ts(10,34): error TS1002: Unterminated string literal.
2-
tests/cases/conformance/scanner/ecmascript5/scannerStringLiterals.ts(11,38): error TS1126: Unexpected end of text.
2+
tests/cases/conformance/scanner/ecmascript5/scannerStringLiterals.ts(11,38): error TS1002: Unterminated string literal.
33

44

55
==== tests/cases/conformance/scanner/ecmascript5/scannerStringLiterals.ts (2 errors) ====
@@ -17,4 +17,4 @@ tests/cases/conformance/scanner/ecmascript5/scannerStringLiterals.ts(11,38): err
1717
!!! error TS1002: Unterminated string literal.
1818
"Should error because of end of file.
1919

20-
!!! error TS1126: Unexpected end of text.
20+
!!! error TS1002: Unterminated string literal.

‎tests/baselines/reference/stringLiteralsErrors.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ tests/cases/compiler/stringLiteralsErrors.ts(22,16): error TS1125: Hexadecimal d
1414
tests/cases/compiler/stringLiteralsErrors.ts(23,17): error TS1125: Hexadecimal digit expected.
1515
tests/cases/compiler/stringLiteralsErrors.ts(24,16): error TS1125: Hexadecimal digit expected.
1616
tests/cases/compiler/stringLiteralsErrors.ts(25,15): error TS1125: Hexadecimal digit expected.
17-
tests/cases/compiler/stringLiteralsErrors.ts(28,14): error TS1126: Unexpected end of text.
17+
tests/cases/compiler/stringLiteralsErrors.ts(28,14): error TS1002: Unterminated string literal.
1818

1919

2020
==== tests/cases/compiler/stringLiteralsErrors.ts (17 errors) ====
@@ -79,4 +79,4 @@ tests/cases/compiler/stringLiteralsErrors.ts(28,14): error TS1126: Unexpected en
7979
// End of file
8080
var es13 = "
8181

82-
!!! error TS1126: Unexpected end of text.
82+
!!! error TS1002: Unterminated string literal.

‎tests/baselines/reference/taggedTemplatesWithIncompleteNoSubstitutionTemplate1.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate1.ts(6,15): error TS1126: Unexpected end of text.
1+
tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate1.ts(6,15): error TS1160: Unterminated template literal.
22

33

44
==== tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate1.ts (1 errors) ====
@@ -9,4 +9,4 @@ tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate1.ts(6,1
99
// Incomplete call, not enough parameters.
1010
f `123qdawdrqw
1111

12-
!!! error TS1126: Unexpected end of text.
12+
!!! error TS1160: Unterminated template literal.

‎tests/baselines/reference/taggedTemplatesWithIncompleteNoSubstitutionTemplate2.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate2.ts(6,4): error TS1126: Unexpected end of text.
1+
tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate2.ts(6,4): error TS1160: Unterminated template literal.
22

33

44
==== tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate2.ts (1 errors) ====
@@ -9,4 +9,4 @@ tests/cases/compiler/taggedTemplatesWithIncompleteNoSubstitutionTemplate2.ts(6,4
99
// Incomplete call, not enough parameters, at EOF.
1010
f `
1111

12-
!!! error TS1126: Unexpected end of text.
12+
!!! error TS1160: Unterminated template literal.

‎tests/baselines/reference/templateStringWithEmbeddedYieldKeyword.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(
22
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(1,15): error TS1005: ';' expected.
33
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(3,26): error TS1158: Invalid template literal; expected '}'
44
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(3,30): error TS1159: Tagged templates are only available when targeting ECMAScript 6 and higher.
5-
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(5,1): error TS1126: Unexpected end of text.
5+
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(5,1): error TS1160: Unterminated template literal.
66
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(1,11): error TS2304: Cannot find name 'gen'.
77
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(3,20): error TS2304: Cannot find name 'yield'.
88
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(3,30): error TS2304: Cannot find name 'def'.
@@ -31,4 +31,4 @@ tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeyword.ts(
3131

3232
!!! error TS1159: Tagged templates are only available when targeting ECMAScript 6 and higher.
3333

34-
!!! error TS1126: Unexpected end of text.
34+
!!! error TS1160: Unterminated template literal.

‎tests/baselines/reference/templateStringWithEmbeddedYieldKeywordES6.errors.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(1,9): error TS1003: Identifier expected.
22
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(1,17): error TS1005: ';' expected.
33
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(3,26): error TS1158: Invalid template literal; expected '}'
4-
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(5,1): error TS1126: Unexpected end of text.
4+
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(5,1): error TS1160: Unterminated template literal.
55
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(1,11): error TS2304: Cannot find name 'gen'.
66
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(3,20): error TS2304: Cannot find name 'yield'.
77
tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.ts(3,30): error TS2304: Cannot find name 'def'.
@@ -26,4 +26,4 @@ tests/cases/conformance/es6/templates/templateStringWithEmbeddedYieldKeywordES6.
2626
}
2727

2828

29-
!!! error TS1126: Unexpected end of text.
29+
!!! error TS1160: Unterminated template literal.

‎tests/baselines/reference/unterminatedRegexAtEndOfSource1.errors.txt

Lines changed: 3 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
1-
tests/cases/compiler/unterminatedRegexAtEndOfSource1.ts(1,9): error TS1109: Expression expected.
2-
tests/cases/compiler/unterminatedRegexAtEndOfSource1.ts(1,10): error TS1109: Expression expected.
1+
tests/cases/compiler/unterminatedRegexAtEndOfSource1.ts(1,10): error TS1161: Unterminated regular expression literal.
32

43

5-
==== tests/cases/compiler/unterminatedRegexAtEndOfSource1.ts (2 errors) ====
4+
==== tests/cases/compiler/unterminatedRegexAtEndOfSource1.ts (1 errors) ====
65
var a = /
7-
~
8-
!!! error TS1109: Expression expected.
96

10-
!!! error TS1109: Expression expected.
7+
!!! error TS1161: Unterminated regular expression literal.

0 commit comments

Comments
 (0)