Skip to content

Optimize parsing #19976

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 8 commits into from
Nov 16, 2017
Merged

Optimize parsing #19976

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
ddf0df9
Introduce TokenFlags enum
ahejlsberg Nov 12, 2017
3411318
Use TokenFlags.PrecedingJSDocComment to guide JSDoc comment processing
ahejlsberg Nov 13, 2017
d143eed
Accept new baselines
ahejlsberg Nov 13, 2017
969f064
Improve lookahead for arrow functions to reduce speculative parsing
ahejlsberg Nov 13, 2017
dbeb682
Merge branch 'master' into optimizeParser
ahejlsberg Nov 14, 2017
787c323
Accept new baselines
ahejlsberg Nov 14, 2017
0bda420
Merge branch 'master' into optimizeParser
ahejlsberg Nov 14, 2017
ee85178
Remove dead code that didn't get removed in #19979
ahejlsberg Nov 14, 2017
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions src/compiler/binder.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1835,7 +1835,7 @@ namespace ts {
}

function checkStrictModeNumericLiteral(node: NumericLiteral) {
if (inStrictMode && node.numericLiteralFlags & NumericLiteralFlags.Octal) {
if (inStrictMode && node.numericLiteralFlags & TokenFlags.Octal) {
file.bindDiagnostics.push(createDiagnosticForNode(node, Diagnostics.Octal_literals_are_not_allowed_in_strict_mode));
}
}
Expand Down Expand Up @@ -3319,7 +3319,7 @@ namespace ts {
break;

case SyntaxKind.NumericLiteral:
if ((<NumericLiteral>node).numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
if ((<NumericLiteral>node).numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
transformFlags |= TransformFlags.AssertES2015;
}
break;
Expand Down
2 changes: 1 addition & 1 deletion src/compiler/checker.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25794,7 +25794,7 @@ namespace ts {

function checkGrammarNumericLiteral(node: NumericLiteral): boolean {
// Grammar checking
if (node.numericLiteralFlags & NumericLiteralFlags.Octal) {
if (node.numericLiteralFlags & TokenFlags.Octal) {
let diagnosticMessage: DiagnosticMessage | undefined;
if (languageVersion >= ScriptTarget.ES5) {
diagnosticMessage = Diagnostics.Octal_literals_are_not_available_when_targeting_ECMAScript_5_and_higher_Use_the_syntax_0;
Expand Down
539 changes: 238 additions & 301 deletions src/compiler/parser.ts
View file
Open in desktop

Large diffs are not rendered by default.

72 changes: 32 additions & 40 deletions src/compiler/scanner.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,7 +27,7 @@ namespace ts {
isReservedWord(): boolean;
isUnterminated(): boolean;
/* @internal */
getNumericLiteralFlags(): NumericLiteralFlags;
getTokenFlags(): TokenFlags;
reScanGreaterToken(): SyntaxKind;
reScanSlashToken(): SyntaxKind;
reScanTemplateToken(): SyntaxKind;
Expand Down Expand Up @@ -814,10 +814,7 @@ namespace ts {

let token: SyntaxKind;
let tokenValue: string;
let precedingLineBreak: boolean;
let hasExtendedUnicodeEscape: boolean;
let tokenIsUnterminated: boolean;
let numericLiteralFlags: NumericLiteralFlags;
let tokenFlags: TokenFlags;

setText(text, start, length);

Expand All @@ -828,12 +825,12 @@ namespace ts {
getTokenPos: () => tokenPos,
getTokenText: () => text.substring(tokenPos, pos),
getTokenValue: () => tokenValue,
hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
hasPrecedingLineBreak: () => precedingLineBreak,
hasExtendedUnicodeEscape: () => (tokenFlags & TokenFlags.ExtendedUnicodeEscape) !== 0,
hasPrecedingLineBreak: () => (tokenFlags & TokenFlags.PrecedingLineBreak) !== 0,
isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
isUnterminated: () => tokenIsUnterminated,
getNumericLiteralFlags: () => numericLiteralFlags,
isUnterminated: () => (tokenFlags & TokenFlags.Unterminated) !== 0,
getTokenFlags: () => tokenFlags,
reScanGreaterToken,
reScanSlashToken,
reScanTemplateToken,
Expand Down Expand Up @@ -870,7 +867,7 @@ namespace ts {
let end = pos;
if (text.charCodeAt(pos) === CharacterCodes.E || text.charCodeAt(pos) === CharacterCodes.e) {
pos++;
numericLiteralFlags = NumericLiteralFlags.Scientific;
tokenFlags |= TokenFlags.Scientific;
if (text.charCodeAt(pos) === CharacterCodes.plus || text.charCodeAt(pos) === CharacterCodes.minus) pos++;
if (isDigit(text.charCodeAt(pos))) {
pos++;
Expand Down Expand Up @@ -942,7 +939,7 @@ namespace ts {
while (true) {
if (pos >= end) {
result += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_string_literal);
break;
}
Expand All @@ -960,7 +957,7 @@ namespace ts {
}
if (isLineBreak(ch)) {
result += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_string_literal);
break;
}
Expand All @@ -984,7 +981,7 @@ namespace ts {
while (true) {
if (pos >= end) {
contents += text.substring(start, pos);
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_template_literal);
resultingToken = startedWithBacktick ? SyntaxKind.NoSubstitutionTemplateLiteral : SyntaxKind.TemplateTail;
break;
Expand Down Expand Up @@ -1070,7 +1067,7 @@ namespace ts {
case CharacterCodes.u:
// '\u{DDDDDDDD}'
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
hasExtendedUnicodeEscape = true;
tokenFlags |= TokenFlags.ExtendedUnicodeEscape;
pos++;
return scanExtendedUnicodeEscape();
}
Expand Down Expand Up @@ -1239,10 +1236,7 @@ namespace ts {

function scan(): SyntaxKind {
startPos = pos;
hasExtendedUnicodeEscape = false;
precedingLineBreak = false;
tokenIsUnterminated = false;
numericLiteralFlags = 0;
tokenFlags = 0;
while (true) {
tokenPos = pos;
if (pos >= end) {
Expand All @@ -1264,7 +1258,7 @@ namespace ts {
switch (ch) {
case CharacterCodes.lineFeed:
case CharacterCodes.carriageReturn:
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
if (skipTrivia) {
pos++;
continue;
Expand Down Expand Up @@ -1395,6 +1389,9 @@ namespace ts {
// Multi-line comment
if (text.charCodeAt(pos + 1) === CharacterCodes.asterisk) {
pos += 2;
if (text.charCodeAt(pos) === CharacterCodes.asterisk && text.charCodeAt(pos + 1) !== CharacterCodes.slash) {
tokenFlags |= TokenFlags.PrecedingJSDocComment;
}

let commentClosed = false;
while (pos < end) {
Expand All @@ -1407,7 +1404,7 @@ namespace ts {
}

if (isLineBreak(ch)) {
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
}
pos++;
}
Expand All @@ -1420,7 +1417,9 @@ namespace ts {
continue;
}
else {
tokenIsUnterminated = !commentClosed;
if (!commentClosed) {
tokenFlags |= TokenFlags.Unterminated;
}
return token = SyntaxKind.MultiLineCommentTrivia;
}
}
Expand All @@ -1441,7 +1440,7 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.HexSpecifier;
tokenFlags |= TokenFlags.HexSpecifier;
return token = SyntaxKind.NumericLiteral;
}
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
Expand All @@ -1452,7 +1451,7 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.BinarySpecifier;
tokenFlags |= TokenFlags.BinarySpecifier;
return token = SyntaxKind.NumericLiteral;
}
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
Expand All @@ -1463,13 +1462,13 @@ namespace ts {
value = 0;
}
tokenValue = "" + value;
numericLiteralFlags = NumericLiteralFlags.OctalSpecifier;
tokenFlags |= TokenFlags.OctalSpecifier;
return token = SyntaxKind.NumericLiteral;
}
// Try to parse as an octal
if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
tokenValue = "" + scanOctalDigits();
numericLiteralFlags = NumericLiteralFlags.Octal;
tokenFlags |= TokenFlags.Octal;
return token = SyntaxKind.NumericLiteral;
}
// This fall-through is a deviation from the EcmaScript grammar. The grammar says that a leading zero
Expand Down Expand Up @@ -1626,7 +1625,7 @@ namespace ts {
continue;
}
else if (isLineBreak(ch)) {
precedingLineBreak = true;
tokenFlags |= TokenFlags.PrecedingLineBreak;
pos++;
continue;
}
Expand Down Expand Up @@ -1669,14 +1668,14 @@ namespace ts {
// If we reach the end of a file, or hit a newline, then this is an unterminated
// regex. Report error and return what we have so far.
if (p >= end) {
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_regular_expression_literal);
break;
}

const ch = text.charCodeAt(p);
if (isLineBreak(ch)) {
tokenIsUnterminated = true;
tokenFlags |= TokenFlags.Unterminated;
error(Diagnostics.Unterminated_regular_expression_literal);
break;
}
Expand Down Expand Up @@ -1894,7 +1893,7 @@ namespace ts {
const saveTokenPos = tokenPos;
const saveToken = token;
const saveTokenValue = tokenValue;
const savePrecedingLineBreak = precedingLineBreak;
const saveTokenFlags = tokenFlags;
const result = callback();

// If our callback returned something 'falsy' or we're just looking ahead,
Expand All @@ -1905,7 +1904,7 @@ namespace ts {
tokenPos = saveTokenPos;
token = saveToken;
tokenValue = saveTokenValue;
precedingLineBreak = savePrecedingLineBreak;
tokenFlags = saveTokenFlags;
}
return result;
}
Expand All @@ -1916,10 +1915,8 @@ namespace ts {
const saveStartPos = startPos;
const saveTokenPos = tokenPos;
const saveToken = token;
const savePrecedingLineBreak = precedingLineBreak;
const saveTokenValue = tokenValue;
const saveHasExtendedUnicodeEscape = hasExtendedUnicodeEscape;
const saveTokenIsUnterminated = tokenIsUnterminated;
const saveTokenFlags = tokenFlags;

setText(text, start, length);
const result = callback();
Expand All @@ -1929,10 +1926,8 @@ namespace ts {
startPos = saveStartPos;
tokenPos = saveTokenPos;
token = saveToken;
precedingLineBreak = savePrecedingLineBreak;
tokenValue = saveTokenValue;
hasExtendedUnicodeEscape = saveHasExtendedUnicodeEscape;
tokenIsUnterminated = saveTokenIsUnterminated;
tokenFlags = saveTokenFlags;

return result;
}
Expand Down Expand Up @@ -1973,11 +1968,8 @@ namespace ts {
startPos = textPos;
tokenPos = textPos;
token = SyntaxKind.Unknown;
precedingLineBreak = false;

tokenValue = undefined;
hasExtendedUnicodeEscape = false;
tokenIsUnterminated = false;
tokenFlags = 0;
}
}
}
2 changes: 1 addition & 1 deletion src/compiler/transformers/es2015.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3633,7 +3633,7 @@ namespace ts {
* @param node A string literal.
*/
function visitNumericLiteral(node: NumericLiteral) {
if (node.numericLiteralFlags & NumericLiteralFlags.BinaryOrOctalSpecifier) {
if (node.numericLiteralFlags & TokenFlags.BinaryOrOctalSpecifier) {
return setTextRange(createNumericLiteral(node.text), node);
}
return node;
Expand Down
19 changes: 12 additions & 7 deletions src/compiler/types.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1451,20 +1451,25 @@ namespace ts {
}

/* @internal */
export const enum NumericLiteralFlags {
export const enum TokenFlags {
None = 0,
Scientific = 1 << 1, // e.g. `10e2`
Octal = 1 << 2, // e.g. `0777`
HexSpecifier = 1 << 3, // e.g. `0x00000000`
BinarySpecifier = 1 << 4, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 5, // e.g. `0o777`
PrecedingLineBreak = 1 << 0,
PrecedingJSDocComment = 1 << 1,
Unterminated = 1 << 2,
ExtendedUnicodeEscape = 1 << 3,
Scientific = 1 << 4, // e.g. `10e2`
Octal = 1 << 5, // e.g. `0777`
HexSpecifier = 1 << 6, // e.g. `0x00000000`
BinarySpecifier = 1 << 7, // e.g. `0b0110010000000000`
OctalSpecifier = 1 << 8, // e.g. `0o777`
BinaryOrOctalSpecifier = BinarySpecifier | OctalSpecifier,
NumericLiteralFlags = Scientific | Octal | HexSpecifier | BinarySpecifier | OctalSpecifier
}

export interface NumericLiteral extends LiteralExpression {
kind: SyntaxKind.NumericLiteral;
/* @internal */
numericLiteralFlags?: NumericLiteralFlags;
numericLiteralFlags?: TokenFlags;
}

export interface TemplateHead extends LiteralLikeNode {
Expand Down
2 changes: 1 addition & 1 deletion tests/baselines/reference/duplicateIdentifierDifferentSpelling.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ var X = { 0b11: '', 3: '' };
//// [duplicateIdentifierDifferentSpelling.js]
var A = /** @class */ (function () {
function A() {
this[0b11] = '';
this[3] = '';
this[3] = '';
}
return A;
Expand Down
37 changes: 0 additions & 37 deletions tests/baselines/reference/parserArrowFunctionExpression6.errors.txt
View file
Open in desktop

This file was deleted.

3 changes: 1 addition & 2 deletions tests/baselines/reference/parserArrowFunctionExpression6.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,6 @@ function foo(q: string, b: number) {

//// [parserArrowFunctionExpression6.js]
function foo(q, b) {
return true ? function (q, , ) { } : ;
;
return true ? (q ? true : false) : (b = q.length, function () { });
}
;
10 changes: 5 additions & 5 deletions tests/baselines/reference/parserArrowFunctionExpression6.symbols
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,11 +5,11 @@ function foo(q: string, b: number) {
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))

return true ? (q ? true : false) : (b = q.length, function() { });
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 22))
> : Symbol((Missing), Decl(parserArrowFunctionExpression6.ts, 1, 29))
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 1, 40))
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 1, 19))
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
>b : Symbol(b, Decl(parserArrowFunctionExpression6.ts, 0, 23))
>q.length : Symbol(String.length, Decl(lib.d.ts, --, --))
>q : Symbol(q, Decl(parserArrowFunctionExpression6.ts, 0, 13))
>length : Symbol(String.length, Decl(lib.d.ts, --, --))

};

Loading