Skip to content

Commit 7dd30d3

Browse files
CyrusNajmabadiCyrusNajmabadi
committed
Simplify how regexs are incrementally parsed.
Conflicts: tests/Fidelity/incremental/IncrementalParserTests.ts tests/Fidelity/parser/ecmascript5/MissingTokens/MissingToken2.ts.expected tests/Fidelity/parser/ecmascript5/RegressionTests/645086_1.ts.expected tests/Fidelity/parser/ecmascript5/RegressionTests/645086_2.ts.expected tests/Fidelity/parser/ecmascript5/RegularExpressions/RegularExpressionDivideAmbiguity4.ts.expected tests/Fidelity/parser/ecmascript5/SyntaxWalker.generated.ts.expected tests/Fidelity/program.js tests/Fidelity/program.js.map tests/Fidelity/scanner/ecmascript5/StringLiterals.ts.expected tests/Fidelity/test262/suite/ch07/7.3/S7.3_A2.1_T2.js.expected tests/Fidelity/test262/suite/ch07/7.3/S7.3_A2.2_T2.js.expected tests/Fidelity/test262/suite/ch07/7.4/S7.4_A3.js.expected tests/Fidelity/test262/suite/ch07/7.4/S7.4_A4_T1.js.expected tests/Fidelity/test262/suite/ch07/7.4/S7.4_A4_T4.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A1.1_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A1.1_T2.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A1.2_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A1.2_T2.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A3.1_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A3.1_T2.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A3.2_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.4/S7.8.4_A3.2_T2.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A1.2_T2.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A1.3_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A1.3_T3.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A1.5_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A1.5_T3.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A2.2_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A2.3_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A2.3_T3.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A2.5_T1.js.expected tests/Fidelity/test262/suite/ch07/7.8/7.8.5/S7.8.5_A2.5_T3.js.expected tests/Fidelity/test262/suite/ch08/8.4/S8.4_A13_T1.js.expected tests/Fidelity/test262/suite/ch08/8.4/S8.4_A13_T2.js.expected tests/Fidelity/test262/suite/ch08/8.4/S8.4_A14_T1.js.expected tests/Fidelity/test262/suite/ch08/8.4/S8.4_A14_T2.js.expected
1 parent 9f28418 commit 7dd30d3

File tree

5 files changed

+36
-45
lines changed

5 files changed

+36
-45
lines changed

‎src/services/resources/diagnosticCode.generated.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ module TypeScript {
55
warning_TS_0_1: "warning TS{0}: {1}",
66
Unrecognized_escape_sequence: "Unrecognized escape sequence.",
77
Unexpected_character_0: "Unexpected character {0}.",
8-
Missing_close_quote_character: "Missing close quote character.",
8+
Unterminated_string_literal: "Unterminated string literal.",
99
Identifier_expected: "Identifier expected.",
1010
_0_keyword_expected: "'{0}' keyword expected.",
1111
_0_expected: "'{0}' expected.",
@@ -97,6 +97,8 @@ module TypeScript {
9797
Template_literal_cannot_be_used_as_an_element_name: "Template literal cannot be used as an element name.",
9898
Computed_property_names_cannot_be_used_here: "Computed property names cannot be used here.",
9999
yield_expression_must_be_contained_within_a_generator_declaration: "'yield' expression must be contained within a generator declaration.",
100+
Unterminated_regular_expression_literal: "Unterminated regular expression literal.",
101+
Unterminated_template_literal: "Unterminated template literal.",
100102
Duplicate_identifier_0: "Duplicate identifier '{0}'.",
101103
The_name_0_does_not_exist_in_the_current_scope: "The name '{0}' does not exist in the current scope.",
102104
The_name_0_does_not_refer_to_a_value: "The name '{0}' does not refer to a value.",

‎src/services/resources/diagnosticInformationMap.generated.ts

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@ module TypeScript {
66
"warning TS{0}: {1}": { "code": 1, "category": DiagnosticCategory.NoPrefix },
77
"Unrecognized escape sequence.": { "code": 1000, "category": DiagnosticCategory.Error },
88
"Unexpected character {0}.": { "code": 1001, "category": DiagnosticCategory.Error },
9-
"Missing close quote character.": { "code": 1002, "category": DiagnosticCategory.Error },
9+
"Unterminated string literal.": { "code": 1002, "category": DiagnosticCategory.Error },
1010
"Identifier expected.": { "code": 1003, "category": DiagnosticCategory.Error },
1111
"'{0}' keyword expected.": { "code": 1004, "category": DiagnosticCategory.Error },
1212
"'{0}' expected.": { "code": 1005, "category": DiagnosticCategory.Error },
@@ -99,6 +99,8 @@ module TypeScript {
9999
"Template literal cannot be used as an element name.": { "code": 1111, "category": DiagnosticCategory.Error },
100100
"Computed property names cannot be used here.": { "code": 1112, "category": DiagnosticCategory.Error },
101101
"'yield' expression must be contained within a generator declaration.": { "code": 1113, "category": DiagnosticCategory.Error },
102+
"Unterminated regular expression literal.": { "code": 1114, "category": DiagnosticCategory.Error },
103+
"Unterminated template literal.": { "code": 1115, "category": DiagnosticCategory.Error },
102104
"Duplicate identifier '{0}'.": { "code": 2000, "category": DiagnosticCategory.Error },
103105
"The name '{0}' does not exist in the current scope.": { "code": 2001, "category": DiagnosticCategory.Error },
104106
"The name '{0}' does not refer to a value.": { "code": 2002, "category": DiagnosticCategory.Error },

‎src/services/resources/diagnosticMessages.json

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@
1515
"category": "Error",
1616
"code": 1001
1717
},
18-
"Missing close quote character.": {
18+
"Unterminated string literal.": {
1919
"category": "Error",
2020
"code": 1002
2121
},
@@ -383,6 +383,14 @@
383383
"category": "Error",
384384
"code": 1113
385385
},
386+
"Unterminated regular expression literal.": {
387+
"category": "Error",
388+
"code": 1114
389+
},
390+
"Unterminated template literal.": {
391+
"category": "Error",
392+
"code": 1115
393+
},
386394
"Duplicate identifier '{0}'.": {
387395
"category": "Error",
388396
"code": 2000

‎src/services/syntax/parser.ts

Lines changed: 11 additions & 30 deletions
Original file line numberDiff line numberDiff line change
@@ -1373,7 +1373,8 @@ module TypeScript.Parser {
13731373

13741374
function parseFunctionDeclarationWorker(modifiers: ISyntaxToken[], functionKeyword: ISyntaxToken, asteriskToken: ISyntaxToken): FunctionDeclarationSyntax {
13751375
// GeneratorDeclaration[Yield, Default] :
1376-
// function * BindingIdentifier[?Yield](FormalParameters[Yield, GeneratorParameter]) { GeneratorBody[Yield] }
1376+
// function * BindingIdentifier[?Yield](FormalParameters[Yield, GeneratorParameter]) { GeneratorBody[Yield] }
1377+
13771378
var isGenerator = asteriskToken !== undefined;
13781379
return new FunctionDeclarationSyntax(parseNodeData,
13791380
modifiers,
@@ -2167,14 +2168,9 @@ module TypeScript.Parser {
21672168

21682169
case SyntaxKind.SlashToken:
21692170
case SyntaxKind.SlashEqualsToken:
2170-
// Note: if we see a / or /= token then we always consider this an expression. Why?
2171-
// Well, either that / or /= is actually a regular expression, in which case we're
2172-
// definitely an expression. Or, it's actually a divide. In which case, we *still*
2173-
// want to think of ourself as an expression. "But wait", you say. '/' doesn't
2174-
// start an expression. That's true. BUt like the above check for =>, for error
2175-
// tolerance, we will consider ourselves in an expression. We'll then parse out an
2176-
// missing identifier and then will consume the / token naturally as a binary
2177-
// expression.
2171+
// Note: if we see a / or /= token then we always consider this an expression.
2172+
// The / or /= will actually be the start of a regex that we will contextually
2173+
// rescan.
21782174

21792175
// Simple epxressions.
21802176
case SyntaxKind.SuperKeyword:
@@ -2976,15 +2972,9 @@ module TypeScript.Parser {
29762972

29772973
case SyntaxKind.SlashToken:
29782974
case SyntaxKind.SlashEqualsToken:
2979-
// If we see a standalone / or /= and we're expecting a term, then try to reparse
2975+
// If we see a standalone / or /= and we're expecting an expression, then reparse
29802976
// it as a regular expression.
2981-
var result = tryReparseDivideAsRegularExpression();
2982-
2983-
// If we get a result, then use it. Otherwise, create a missing identifier so
2984-
// that parsing can continue. Note: we do this even if 'force' is false. That's
2985-
// because we *do* want to consider a standalone / as an expression that should be
2986-
// returned from tryParseExpression even when 'force' is set to false.
2987-
return result || eatIdentifierToken(DiagnosticCode.Expression_expected);
2977+
return reparseDivideAsRegularExpression();
29882978
}
29892979

29902980
if (!force) {
@@ -2995,7 +2985,7 @@ module TypeScript.Parser {
29952985
return eatIdentifierToken(DiagnosticCode.Expression_expected);
29962986
}
29972987

2998-
function tryReparseDivideAsRegularExpression(): IPrimaryExpressionSyntax {
2988+
function reparseDivideAsRegularExpression(): IPrimaryExpressionSyntax {
29992989
// If we see a / or /= token, then that may actually be the start of a regex in certain
30002990
// contexts.
30012991

@@ -3012,18 +3002,9 @@ module TypeScript.Parser {
30123002
// Debug.assert(SyntaxFacts.isAnyDivideOrRegularExpressionToken(currentToken.kind));
30133003

30143004
var tokenKind = currentToken.kind;
3015-
if (tokenKind === SyntaxKind.SlashToken || tokenKind === SyntaxKind.SlashEqualsToken) {
3016-
// Still came back as a / or /=. This is not a regular expression literal.
3017-
return undefined;
3018-
}
3019-
else if (tokenKind === SyntaxKind.RegularExpressionLiteral) {
3020-
return consumeToken(currentToken);
3021-
}
3022-
else {
3023-
// Something *very* wrong happened. This is an internal parser fault that we need
3024-
// to figure out and fix.
3025-
throw Errors.invalidOperation();
3026-
}
3005+
Debug.assert(tokenKind === SyntaxKind.RegularExpressionLiteral);
3006+
3007+
return consumeToken(currentToken);
30273008
}
30283009

30293010
function parseTypeOfExpression(typeOfKeyword: ISyntaxToken): TypeOfExpressionSyntax {

‎src/services/syntax/scanner.ts

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -281,7 +281,7 @@ module TypeScript.Scanner {
281281
LargeScannerToken.prototype.childCount = 0;
282282

283283
export interface DiagnosticCallback {
284-
(position: number, width: number, key: string, arguments: any[]): void;
284+
(position: number, width: number, key: string, arguments?: any[]): void;
285285
}
286286

287287
interface TokenInfo {
@@ -1008,7 +1008,7 @@ module TypeScript.Scanner {
10081008
while (true) {
10091009
if (index === end) {
10101010
// Hit the end of the file.
1011-
reportDiagnostic(end, 0, DiagnosticCode._0_expected, ["`"]);
1011+
reportDiagnostic(end, 0, DiagnosticCode.Unterminated_template_literal);
10121012
break;
10131013
}
10141014

@@ -1144,10 +1144,7 @@ module TypeScript.Scanner {
11441144
// term, and it sees one of these then it may restart us asking specifically if we could
11451145
// scan out a regex.
11461146
if (allowContextualToken) {
1147-
var result = tryScanRegularExpressionToken();
1148-
if (result !== SyntaxKind.None) {
1149-
return result;
1150-
}
1147+
return scanRegularExpressionToken();
11511148
}
11521149

11531150
if (str.charCodeAt(index) === CharacterCodes.equals) {
@@ -1159,7 +1156,7 @@ module TypeScript.Scanner {
11591156
}
11601157
}
11611158

1162-
function tryScanRegularExpressionToken(): SyntaxKind {
1159+
function scanRegularExpressionToken(): SyntaxKind {
11631160
var startIndex = index;
11641161

11651162
var inEscape = false;
@@ -1168,8 +1165,9 @@ module TypeScript.Scanner {
11681165
var ch = str.charCodeAt(index);
11691166

11701167
if (isNaN(ch) || isNewLineCharacter(ch)) {
1171-
index = startIndex;
1172-
return SyntaxKind.None;
1168+
// Hit the end of line, or end of the file. This is not a legal regex.
1169+
reportDiagnostic(index, 0, DiagnosticCode.Unterminated_regular_expression_literal);
1170+
break;
11731171
}
11741172

11751173
index++;
@@ -1193,7 +1191,7 @@ module TypeScript.Scanner {
11931191
continue;
11941192

11951193
case CharacterCodes.closeBracket:
1196-
// If we ever hit a cloe bracket then we're now no longer in a character
1194+
// If we ever hit a close bracket then we're now no longer in a character
11971195
// class. If we weren't in a character class to begin with, then this has
11981196
// no effect.
11991197
inCharacterClass = false;
@@ -1219,7 +1217,7 @@ module TypeScript.Scanner {
12191217

12201218
// TODO: The grammar says any identifier part is allowed here. Do we need to support
12211219
// \u identifiers here? The existing typescript parser does not.
1222-
while (isIdentifierPartCharacter[str.charCodeAt(index)]) {
1220+
while (index < end && isIdentifierPartCharacter[str.charCodeAt(index)]) {
12231221
index++;
12241222
}
12251223

@@ -1322,7 +1320,7 @@ module TypeScript.Scanner {
13221320
break;
13231321
}
13241322
else if (isNaN(ch) || isNewLineCharacter(ch)) {
1325-
reportDiagnostic(Math.min(index, end), 1, DiagnosticCode.Missing_close_quote_character, undefined);
1323+
reportDiagnostic(index, 0, DiagnosticCode.Unterminated_string_literal);
13261324
break;
13271325
}
13281326
else {

0 commit comments

Comments
 (0)