From 8586693fd9e54485d93c7b40a17c9f38689c11ab Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sat, 4 Jul 2020 21:09:47 +0300 Subject: [PATCH 01/13] refactor some logical condisionals --- src/parser.ts | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 0cd9b9243e..a4543307e5 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -981,7 +981,7 @@ export class Parser extends DiagnosticEmitter { } } var range = Range.join(identifier.range, tn.range()); - if ((flags & CommonFlags.DEFINITELY_ASSIGNED) != 0 && initializer !== null) { + if (initializer !== null && (flags & CommonFlags.DEFINITELY_ASSIGNED) != 0) { this.error( DiagnosticCode.A_definite_assignment_assertion_is_not_permitted_in_this_context, range @@ -1852,7 +1852,7 @@ export class Parser extends DiagnosticEmitter { if (!decorators) decorators = new Array(); decorators.push(decorator); } while (tn.skip(Token.AT)); - if (decorators !== null && isInterface) { + if (isInterface && decorators !== null) { this.error( DiagnosticCode.Decorators_are_not_valid_here, Range.join(decorators[0].range, decorators[decorators.length - 1].range) @@ -3246,7 +3246,11 @@ export class Parser extends DiagnosticEmitter { if (!label) return null; if (tn.skip(Token.COLON)) { statements = new Array(); - while (tn.peek() != Token.CASE && tn.nextToken != Token.DEFAULT && tn.nextToken != Token.CLOSEBRACE) { + while ( + tn.peek() != Token.CASE && + tn.nextToken != Token.DEFAULT && + tn.nextToken != Token.CLOSEBRACE + ) { statement = this.parseStatement(tn); if (!statement) return null; statements.push(statement); @@ -3264,7 +3268,11 @@ export class Parser extends DiagnosticEmitter { } else if (tn.skip(Token.DEFAULT)) { if (tn.skip(Token.COLON)) { statements = new Array(); - while (tn.peek() != Token.CASE && tn.nextToken != Token.DEFAULT && tn.nextToken != Token.CLOSEBRACE) { + while ( + tn.peek() != Token.CASE && + tn.nextToken != Token.DEFAULT && + tn.nextToken != Token.CLOSEBRACE + ) { statement = this.parseStatement(tn); if (!statement) return null; statements.push(statement); @@ -3544,8 +3552,7 @@ export class Parser extends DiagnosticEmitter { let typeArguments: TypeNode[] | null = null; let arguments_: Expression[] | null = null; if ( - tn.skip(Token.OPENPAREN) - || + tn.skip(Token.OPENPAREN) || (typeArguments = this.tryParseTypeArgumentsBeforeArguments(tn)) !== null ) { arguments_ = this.parseArguments(tn); @@ -4153,9 +4160,9 @@ export class Parser extends DiagnosticEmitter { ): Expression { var typeArguments: TypeNode[] | null = null; while ( - tn.skip(Token.OPENPAREN) - || - potentiallyGeneric && (typeArguments = this.tryParseTypeArgumentsBeforeArguments(tn)) !== null + tn.skip(Token.OPENPAREN) || + potentiallyGeneric && + (typeArguments = this.tryParseTypeArgumentsBeforeArguments(tn)) !== null ) { let args = this.parseArguments(tn); if (!args) break; From be3442233ef4ab2efc469b511901fa0a77847d08 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sat, 4 Jul 2020 21:17:34 +0300 Subject: [PATCH 02/13] isGetterOrSetter --- src/parser.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index a4543307e5..95832a0a4d 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -2023,11 +2023,12 @@ export class Parser extends DiagnosticEmitter { } } + var isGetterOrSetter = isGetter || isSetter; var name: IdentifierExpression; if (isConstructor) { name = Node.createConstructorExpression(tn.range()); } else { - if (!(isGetter || isSetter) && tn.skip(Token.OPENBRACKET)) { + if (!isGetterOrSetter && tn.skip(Token.OPENBRACKET)) { if (!startPos) startPos = tn.tokenPos; // TODO: also handle symbols, which might have some of these modifiers if (flags & CommonFlags.PUBLIC) { @@ -2091,7 +2092,7 @@ export class Parser extends DiagnosticEmitter { DiagnosticCode.Type_parameters_cannot_appear_on_a_constructor_declaration, tn.range(typeParametersStart, tn.pos) ); // recoverable - } else if (isGetter || isSetter) { + } else if (isGetterOrSetter) { this.error( DiagnosticCode.An_accessor_cannot_have_type_parameters, tn.range(typeParametersStart, tn.pos) @@ -2236,7 +2237,7 @@ export class Parser extends DiagnosticEmitter { name.range ); - } else if (isGetter || isSetter) { + } else if (isGetterOrSetter) { this.error( DiagnosticCode.Function_implementation_is_missing_or_not_immediately_following_the_declaration, name.range From 54f2cdcad82531be24d7f4cea444d79c02028790 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sat, 4 Jul 2020 21:31:36 +0300 Subject: [PATCH 03/13] t -> ty, skipping explicit type decl for ints --- src/parser.ts | 60 +++++++++++++++++++++++++-------------------------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 95832a0a4d..a087d0bab2 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -193,7 +193,7 @@ export class Parser extends DiagnosticEmitter { namespace: NamespaceDeclaration | null = null ): Statement | null { var flags = CommonFlags.NONE; - var startPos: i32 = -1; + var startPos = -1; // check decorators var decorators: DecoratorNode[] | null = null; @@ -209,10 +209,10 @@ export class Parser extends DiagnosticEmitter { } // check modifiers - var exportStart: i32 = 0; - var exportEnd: i32 = 0; - var defaultStart: i32 = 0; - var defaultEnd: i32 = 0; + var exportStart = 0; + var exportEnd = 0; + var defaultStart = 0; + var defaultEnd = 0; if (tn.skip(Token.EXPORT)) { if (startPos < 0) startPos = tn.tokenPos; flags |= CommonFlags.EXPORT; @@ -224,8 +224,8 @@ export class Parser extends DiagnosticEmitter { } } - var declareStart: i32 = 0; - var declareEnd: i32 = 0; + var declareStart = 0; + var declareEnd = 0; var contextIsAmbient = namespace != null && namespace.is(CommonFlags.AMBIENT); if (tn.skip(Token.DECLARE)) { if (contextIsAmbient) { @@ -700,7 +700,7 @@ export class Parser extends DiagnosticEmitter { } else { isSignature = false; // not yet known do { - let paramStart: i32 = -1; + let paramStart = -1; let kind = ParameterKind.DEFAULT; if (tn.skip(Token.DOT_DOT_DOT)) { paramStart = tn.tokenPos; @@ -713,17 +713,17 @@ export class Parser extends DiagnosticEmitter { if (tn.skip(Token.COLON)) { isSignature = true; tn.discard(state); - let t = this.parseType(tn, false); - if (!t) return null; - if (t.kind != NodeKind.NAMEDTYPE) { + let ty = this.parseType(tn, false); + if (!ty) return null; + if (ty.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - t.range + ty.range ); this.tryParseSignatureIsSignature = true; return null; } - thisType = t; + thisType = ty; } else { tn.reset(state); this.tryParseSignatureIsSignature = false; @@ -1154,29 +1154,29 @@ export class Parser extends DiagnosticEmitter { ); let extendsType: NamedTypeNode | null = null; if (tn.skip(Token.EXTENDS)) { - let t = this.parseType(tn); - if (!t) return null; - if (t.kind != NodeKind.NAMEDTYPE) { + let ty = this.parseType(tn); + if (!ty) return null; + if (ty.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - t.range + ty.range ); return null; } - extendsType = t; + extendsType = ty; } let defaultType: NamedTypeNode | null = null; if (tn.skip(Token.EQUALS)) { - let t = this.parseType(tn); - if (!t) return null; - if (t.kind != NodeKind.NAMEDTYPE) { + let ty = this.parseType(tn); + if (!ty) return null; + if (ty.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - t.range + ty.range ); return null; } - defaultType = t; + defaultType = ty; } return Node.createTypeParameter( identifier, @@ -1414,7 +1414,7 @@ export class Parser extends DiagnosticEmitter { } var name = Node.createIdentifierExpression(tn.readIdentifier(), tn.range()); - var signatureStart: i32 = -1; + var signatureStart = -1; var typeParameters: TypeParameterNode[] | null = null; if (tn.skip(Token.LESSTHAN)) { @@ -1940,8 +1940,8 @@ export class Parser extends DiagnosticEmitter { if (parent.flags & CommonFlags.GENERIC) flags |= CommonFlags.GENERIC_CONTEXT; } - var readonlyStart: i32 = 0; - var readonlyEnd: i32 = 0; + var readonlyStart = 0; + var readonlyEnd = 0; if (tn.peek() == Token.READONLY) { let state = tn.mark(); tn.next(); @@ -1960,11 +1960,11 @@ export class Parser extends DiagnosticEmitter { var state = tn.mark(); var isConstructor = false; var isGetter = false; - var getStart: i32 = 0; - var getEnd: i32 = 0; + var getStart = 0; + var getEnd = 0; var isSetter = false; - var setStart: i32 = 0; - var setEnd: i32 = 0; + var setStart = 0; + var setEnd = 0; if (!isInterface) { if (tn.skip(Token.GET)) { if (tn.peek(true, IdentifierHandling.PREFER) == Token.IDENTIFIER && !tn.nextTokenOnNewLine) { From 2b8dc5f71f1fd72cce6f1fcb41a73586889d7bb9 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 00:40:21 +0300 Subject: [PATCH 04/13] t -> ty --- src/parser.ts | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index a087d0bab2..135b38536e 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -1674,16 +1674,16 @@ export class Parser extends DiagnosticEmitter { var extendsType: NamedTypeNode | null = null; if (tn.skip(Token.EXTENDS)) { - let t = this.parseType(tn); - if (!t) return null; - if (t.kind != NodeKind.NAMEDTYPE) { + let ty = this.parseType(tn); + if (!ty) return null; + if (ty.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - t.range + ty.range ); return null; } - extendsType = t; + extendsType = ty; } var implementsTypes: NamedTypeNode[] | null = null; From 75227354e1f43605adcf8ba5c29769c6b861203c Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 19:52:52 +0300 Subject: [PATCH 05/13] use lazy typeArguments init for tryParseTypeArgumentsBeforeArguments --- src/parser.ts | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 135b38536e..63fceec3d0 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -3853,7 +3853,7 @@ export class Parser extends DiagnosticEmitter { var state = tn.mark(); if (!tn.skip(Token.LESSTHAN)) return null; var start = tn.tokenPos; - var typeArguments = new Array(); + var typeArguments: TypeNode[] | null = null; do { if (tn.peek() === Token.GREATERTHAN) { break; @@ -3863,12 +3863,13 @@ export class Parser extends DiagnosticEmitter { tn.reset(state); return null; } - typeArguments.push(type); + if (!typeArguments) typeArguments = [ type ]; + else typeArguments.push(type); } while (tn.skip(Token.COMMA)); if (tn.skip(Token.GREATERTHAN)) { let end = tn.pos; if (tn.skip(Token.OPENPAREN)) { - if (!typeArguments.length) { + if (!typeArguments || !typeArguments.length) { this.error( DiagnosticCode.Type_argument_list_cannot_be_empty, tn.range(start, end) From bd5c0eecdf54bbd7c8c74eb9ed6d79988e710396 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 19:59:23 +0300 Subject: [PATCH 06/13] simplify --- src/parser.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/parser.ts b/src/parser.ts index 63fceec3d0..548ca09285 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -3869,7 +3869,7 @@ export class Parser extends DiagnosticEmitter { if (tn.skip(Token.GREATERTHAN)) { let end = tn.pos; if (tn.skip(Token.OPENPAREN)) { - if (!typeArguments || !typeArguments.length) { + if (!typeArguments) { this.error( DiagnosticCode.Type_argument_list_cannot_be_empty, tn.range(start, end) From df214d61486709010065f72d372e27c8c4d662f9 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 20:08:40 +0300 Subject: [PATCH 07/13] refactor switch with single case to if-branch --- src/tokenizer.ts | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 5dbb891e00..1d6ba8953b 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -970,11 +970,8 @@ export class Tokenizer extends DiagnosticEmitter { var tokenBefore = this.token; var tokenPosBefore = this.tokenPos; var maxCompoundLength = i32.MAX_VALUE; - switch (token) { - case Token.GREATERTHAN: { // where parsing type arguments - maxCompoundLength = 1; - break; - } + if (token == Token.GREATERTHAN) { // where parsing type arguments + maxCompoundLength = 1; } var nextToken: Token; do nextToken = this.unsafeNext(identifierHandling, maxCompoundLength); From f373252be65a9e82e21557416ab7e3100872bdae Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 20:15:05 +0300 Subject: [PATCH 08/13] cache pos for readInteger in tokenizer --- src/tokenizer.ts | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 1d6ba8953b..180dc86d8a 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1220,24 +1220,25 @@ export class Tokenizer extends DiagnosticEmitter { readInteger(): i64 { var text = this.source.text; - if (this.pos + 2 < this.end && text.charCodeAt(this.pos) == CharCode._0) { - switch (text.charCodeAt(this.pos + 1) | 32) { + var pos = this.pos; + if (pos + 2 < this.end && text.charCodeAt(pos) == CharCode._0) { + switch (text.charCodeAt(pos + 1) | 32) { case CharCode.x: { - this.pos += 2; + this.pos = pos + 2; return this.readHexInteger(); } case CharCode.b: { - this.pos += 2; + this.pos = pos + 2; return this.readBinaryInteger(); } case CharCode.o: { - this.pos += 2; + this.pos = pos + 2; return this.readOctalInteger(); } } - if (isOctalDigit(text.charCodeAt(this.pos + 1))) { - let start = this.pos; - ++this.pos; + if (isOctalDigit(text.charCodeAt(pos + 1))) { + let start = pos; + this.pos = pos + 1; let value = this.readOctalInteger(); this.error( DiagnosticCode.Octal_literals_are_not_allowed_in_strict_mode, @@ -1380,7 +1381,7 @@ export class Tokenizer extends DiagnosticEmitter { } else { break; } - ++this.pos; + this.pos = pos + 1; } if (this.pos == start) { this.error( From 9722c43c11170131aa8de3913452d81d255037bd Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 20:45:01 +0300 Subject: [PATCH 09/13] refactor Tokenizer's constructor --- src/diagnostics.ts | 2 +- src/tokenizer.ts | 30 +++++++++++++++--------------- 2 files changed, 16 insertions(+), 16 deletions(-) diff --git a/src/diagnostics.ts b/src/diagnostics.ts index b145714c75..adbb143da7 100644 --- a/src/diagnostics.ts +++ b/src/diagnostics.ts @@ -275,7 +275,7 @@ export abstract class DiagnosticEmitter { /** Initializes this diagnostic emitter. */ protected constructor(diagnostics: DiagnosticMessage[] | null = null) { - if (!diagnostics) diagnostics = new Array(); + if (!diagnostics) diagnostics = []; this.diagnostics = diagnostics; } diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 180dc86d8a..7139268480 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -463,38 +463,38 @@ export class Tokenizer extends DiagnosticEmitter { constructor(source: Source, diagnostics: DiagnosticMessage[] | null = null) { super(diagnostics); - this.source = source; - this.pos = 0; - this.end = source.text.length; if (!diagnostics) diagnostics = []; this.diagnostics = diagnostics; + this.source = source; - var end = this.end; var text = source.text; - + var end = text.length; + var pos = 0; // skip bom if ( - this.pos < end && - text.charCodeAt(this.pos) == CharCode.BYTEORDERMARK + pos < end && + text.charCodeAt(pos) == CharCode.BYTEORDERMARK ) { - ++this.pos; + ++pos; } // skip shebang if ( - this.pos + 1 < end && - text.charCodeAt(this.pos) == CharCode.HASH && - text.charCodeAt(this.pos + 1) == CharCode.EXCLAMATION + pos + 1 < end && + text.charCodeAt(pos) == CharCode.HASH && + text.charCodeAt(pos + 1) == CharCode.EXCLAMATION ) { - this.pos += 2; + pos += 2; while ( - this.pos < end && - text.charCodeAt(this.pos) != CharCode.LINEFEED + pos < end && + text.charCodeAt(pos) != CharCode.LINEFEED ) { - ++this.pos; + ++pos; } // 'next' now starts at lf or eof } + this.pos = pos; + this.end = end; } next(identifierHandling: IdentifierHandling = IdentifierHandling.DEFAULT): Token { From c89ba9da375d721a5342f97f1fb84e683e79e33d Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 21:11:21 +0300 Subject: [PATCH 10/13] cache pos for unsafeNext --- src/tokenizer.ts | 284 ++++++++++++++++++++++++++--------------------- 1 file changed, 155 insertions(+), 129 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 7139268480..5f01892490 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -510,16 +510,17 @@ export class Tokenizer extends DiagnosticEmitter { identifierHandling: IdentifierHandling = IdentifierHandling.DEFAULT, maxTokenLength: i32 = i32.MAX_VALUE ): Token { - var end = this.end; var text = this.source.text; - while (this.pos < end) { - this.tokenPos = this.pos; - let c = text.charCodeAt(this.pos); + var end = this.end; + var pos = this.pos; + while (pos < end) { + this.tokenPos = pos; + let c = text.charCodeAt(pos); switch (c) { case CharCode.CARRIAGERETURN: { if (!( - ++this.pos < end && - text.charCodeAt(this.pos) == CharCode.LINEFEED + ++pos < end && + text.charCodeAt(pos) == CharCode.LINEFEED )) break; // otherwise fall-through } @@ -528,180 +529,190 @@ export class Tokenizer extends DiagnosticEmitter { case CharCode.VERTICALTAB: case CharCode.FORMFEED: case CharCode.SPACE: { - ++this.pos; + ++pos; break; } case CharCode.EXCLAMATION: { - ++this.pos; + ++pos; if ( - maxTokenLength > 1 && this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + maxTokenLength > 1 && pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + ++pos; if ( - maxTokenLength > 2 && this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + maxTokenLength > 2 && pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.EXCLAMATION_EQUALS_EQUALS; } + this.pos = pos; return Token.EXCLAMATION_EQUALS; } + this.pos = pos; return Token.EXCLAMATION; } case CharCode.DOUBLEQUOTE: case CharCode.SINGLEQUOTE: case CharCode.BACKTICK: { // TODO + this.pos = pos; return Token.STRINGLITERAL; // expects a call to readString } case CharCode.PERCENT: { - ++this.pos; + ++pos; if ( - maxTokenLength > 1 && this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + maxTokenLength > 1 && pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.PERCENT_EQUALS; } + this.pos = pos; return Token.PERCENT; } case CharCode.AMPERSAND: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.AMPERSAND) { - ++this.pos; + this.pos = pos + 1; return Token.AMPERSAND_AMPERSAND; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.AMPERSAND_EQUALS; } } + this.pos = pos; return Token.AMPERSAND; } case CharCode.OPENPAREN: { - ++this.pos; + this.pos = pos + 1; return Token.OPENPAREN; } case CharCode.CLOSEPAREN: { - ++this.pos; + this.pos = pos + 1; return Token.CLOSEPAREN; } case CharCode.ASTERISK: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.ASTERISK_EQUALS; } if (chr == CharCode.ASTERISK) { - ++this.pos; + ++pos; if ( - maxTokenLength > 2 && this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + maxTokenLength > 2 && pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.ASTERISK_ASTERISK_EQUALS; } + this.pos = pos; return Token.ASTERISK_ASTERISK; } } + this.pos = pos; return Token.ASTERISK; } case CharCode.PLUS: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.PLUS) { - ++this.pos; + this.pos = pos + 1; return Token.PLUS_PLUS; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.PLUS_EQUALS; } } + this.pos = pos; return Token.PLUS; } case CharCode.COMMA: { - ++this.pos; + this.pos = pos + 1; return Token.COMMA; } case CharCode.MINUS: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.MINUS) { - ++this.pos; + this.pos = pos + 1; return Token.MINUS_MINUS; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.MINUS_EQUALS; } } + this.pos = pos; return Token.MINUS; } case CharCode.DOT: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (isDecimalDigit(chr)) { - --this.pos; + this.pos = pos - 1; return Token.FLOATLITERAL; // expects a call to readFloat } if ( - maxTokenLength > 2 && this.pos + 1 < end && + maxTokenLength > 2 && pos + 1 < end && chr == CharCode.DOT && - text.charCodeAt(this.pos + 1) == CharCode.DOT + text.charCodeAt(pos + 1) == CharCode.DOT ) { - this.pos += 2; + this.pos = pos + 2; return Token.DOT_DOT_DOT; } } + this.pos = pos; return Token.DOT; } case CharCode.SLASH: { - let commentStartPos = this.pos; - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + let commentStartPos = pos; + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.SLASH) { // single-line let commentKind = CommentKind.LINE; if ( - this.pos + 1 < end && - text.charCodeAt(this.pos + 1) == CharCode.SLASH + pos + 1 < end && + text.charCodeAt(pos + 1) == CharCode.SLASH ) { - ++this.pos; + ++pos; commentKind = CommentKind.TRIPLE; } - while (++this.pos < end) { - if (text.charCodeAt(this.pos) == CharCode.LINEFEED) { - ++this.pos; + while (++pos < end) { + if (text.charCodeAt(pos) == CharCode.LINEFEED) { + ++pos; break; } } if (this.onComment) { this.onComment( commentKind, - text.substring(commentStartPos, this.pos), - this.range(commentStartPos, this.pos) + text.substring(commentStartPos, pos), + this.range(commentStartPos, pos) ); } break; } if (chr == CharCode.ASTERISK) { // multi-line let closed = false; - while (++this.pos < end) { - c = text.charCodeAt(this.pos); + while (++pos < end) { + c = text.charCodeAt(pos); if ( c == CharCode.ASTERISK && - this.pos + 1 < end && - text.charCodeAt(this.pos + 1) == CharCode.SLASH + pos + 1 < end && + text.charCodeAt(pos + 1) == CharCode.SLASH ) { - this.pos += 2; + pos += 2; closed = true; break; } @@ -709,22 +720,23 @@ export class Tokenizer extends DiagnosticEmitter { if (!closed) { this.error( DiagnosticCode._0_expected, - this.range(this.pos), "*/" + this.range(pos), "*/" ); } else if (this.onComment) { this.onComment( CommentKind.BLOCK, - text.substring(commentStartPos, this.pos), - this.range(commentStartPos, this.pos) + text.substring(commentStartPos, pos), + this.range(commentStartPos, pos) ); } break; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.SLASH_EQUALS; } } + this.pos = pos; return Token.SLASH; } case CharCode._0: @@ -737,165 +749,175 @@ export class Tokenizer extends DiagnosticEmitter { case CharCode._7: case CharCode._8: case CharCode._9: { + this.pos = pos; return this.testInteger() ? Token.INTEGERLITERAL // expects a call to readInteger : Token.FLOATLITERAL; // expects a call to readFloat } case CharCode.COLON: { - ++this.pos; + this.pos = pos + 1; return Token.COLON; } case CharCode.SEMICOLON: { - ++this.pos; + this.pos = pos + 1; return Token.SEMICOLON; } case CharCode.LESSTHAN: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.LESSTHAN) { - ++this.pos; + ++pos; if ( maxTokenLength > 2 && - this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.LESSTHAN_LESSTHAN_EQUALS; } + this.pos = pos; return Token.LESSTHAN_LESSTHAN; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.LESSTHAN_EQUALS; } } + this.pos = pos; return Token.LESSTHAN; } case CharCode.EQUALS: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.EQUALS) { - ++this.pos; + ++pos; if ( maxTokenLength > 2 && - this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.EQUALS_EQUALS_EQUALS; } + this.pos = pos; return Token.EQUALS_EQUALS; } if (chr == CharCode.GREATERTHAN) { - ++this.pos; + this.pos = pos + 1; return Token.EQUALS_GREATERTHAN; } } + this.pos = pos; return Token.EQUALS; } case CharCode.GREATERTHAN: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.GREATERTHAN) { - ++this.pos; - if (maxTokenLength > 2 && this.pos < end) { - chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 2 && pos < end) { + chr = text.charCodeAt(pos); if (chr == CharCode.GREATERTHAN) { - ++this.pos; + ++pos; if ( - maxTokenLength > 3 && this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + maxTokenLength > 3 && pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.GREATERTHAN_GREATERTHAN_GREATERTHAN_EQUALS; } + this.pos = pos; return Token.GREATERTHAN_GREATERTHAN_GREATERTHAN; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.GREATERTHAN_GREATERTHAN_EQUALS; } } + this.pos = pos; return Token.GREATERTHAN_GREATERTHAN; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.GREATERTHAN_EQUALS; } } + this.pos = pos; return Token.GREATERTHAN; } case CharCode.QUESTION: { - ++this.pos; + this.pos = pos + 1; return Token.QUESTION; } case CharCode.OPENBRACKET: { - ++this.pos; + this.pos = pos + 1; return Token.OPENBRACKET; } case CharCode.CLOSEBRACKET: { - ++this.pos; + this.pos = pos + 1; return Token.CLOSEBRACKET; } case CharCode.CARET: { - ++this.pos; + ++pos; if ( - maxTokenLength > 1 && this.pos < end && - text.charCodeAt(this.pos) == CharCode.EQUALS + maxTokenLength > 1 && pos < end && + text.charCodeAt(pos) == CharCode.EQUALS ) { - ++this.pos; + this.pos = pos + 1; return Token.CARET_EQUALS; } + this.pos = pos; return Token.CARET; } case CharCode.OPENBRACE: { - ++this.pos; + this.pos = pos + 1; return Token.OPENBRACE; } case CharCode.BAR: { - ++this.pos; - if (maxTokenLength > 1 && this.pos < end) { - let chr = text.charCodeAt(this.pos); + ++pos; + if (maxTokenLength > 1 && pos < end) { + let chr = text.charCodeAt(pos); if (chr == CharCode.BAR) { - ++this.pos; + this.pos = pos + 1; return Token.BAR_BAR; } if (chr == CharCode.EQUALS) { - ++this.pos; + this.pos = pos + 1; return Token.BAR_EQUALS; } } + this.pos = pos; return Token.BAR; } case CharCode.CLOSEBRACE: { - ++this.pos; + this.pos = pos + 1; return Token.CLOSEBRACE; } case CharCode.TILDE: { - ++this.pos; + this.pos = pos + 1; return Token.TILDE; } case CharCode.AT: { - ++this.pos; + this.pos = pos + 1; return Token.AT; } default: { if (isIdentifierStart(c)) { if (isKeywordCharacter(c)) { - let posBefore = this.pos; + let posBefore = pos; while ( - ++this.pos < end && - isIdentifierPart(c = text.charCodeAt(this.pos)) + ++pos < end && + isIdentifierPart(c = text.charCodeAt(pos)) ) { if (!isKeywordCharacter(c)) { this.pos = posBefore; return Token.IDENTIFIER; } } - let keywordText = text.substring(posBefore, this.pos); + let keywordText = text.substring(posBefore, pos); let keywordToken = tokenFromKeyword(keywordText); if ( keywordToken !== Token.INVALID && @@ -905,28 +927,32 @@ export class Tokenizer extends DiagnosticEmitter { tokenIsAlsoIdentifier(keywordToken) ) ) { + this.pos = pos; return keywordToken; } - this.pos = posBefore; + this.pos = pos = posBefore; } + this.pos = pos; return Token.IDENTIFIER; // expects a call to readIdentifier } else if (isWhiteSpace(c)) { - ++this.pos; + ++pos; break; } - let start = this.pos++; + let start = pos++; if ( // surrogate pair? - (c & 0xFC00) == 0xD800 && this.pos < this.end && - ((text.charCodeAt(this.pos)) & 0xFC00) == 0xDC00 - ) ++this.pos; + (c & 0xFC00) == 0xD800 && pos < end && + ((text.charCodeAt(pos)) & 0xFC00) == 0xDC00 + ) ++pos; this.error( DiagnosticCode.Invalid_character, - this.range(start, this.pos) + this.range(start, pos) ); + this.pos = pos; return Token.INVALID; } } } + this.pos = pos; return Token.ENDOFFILE; } From 01d8ef6d1bc31e7058956fff2d48cd4fd2becd79 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 21:33:44 +0300 Subject: [PATCH 11/13] more --- src/tokenizer.ts | 28 ++++++++++++++++------------ 1 file changed, 16 insertions(+), 12 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index 5f01892490..ae758256f3 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1063,40 +1063,44 @@ export class Tokenizer extends DiagnosticEmitter { readString(): string { var text = this.source.text; - var quote = text.charCodeAt(this.pos++); - var start = this.pos; var end = this.end; + var pos = this.pos; + var quote = text.charCodeAt(pos++); + var start = pos; var result = ""; while (true) { - if (this.pos >= end) { - result += text.substring(start, this.pos); + if (pos >= end) { + result += text.substring(start, pos); this.error( DiagnosticCode.Unterminated_string_literal, this.range(start - 1, end) ); break; } - let c = text.charCodeAt(this.pos); + let c = text.charCodeAt(pos); if (c == quote) { - result += text.substring(start, this.pos++); + result += text.substring(start, pos++); break; } if (c == CharCode.BACKSLASH) { - result += text.substring(start, this.pos); + result += text.substring(start, pos); + this.pos = pos; // save result += this.readEscapeSequence(); - start = this.pos; + pos = this.pos; // restore + start = pos; continue; } if (isLineBreak(c) && quote != CharCode.BACKTICK) { - result += text.substring(start, this.pos); + result += text.substring(start, pos); this.error( DiagnosticCode.Unterminated_string_literal, - this.range(start - 1, this.pos) + this.range(start - 1, pos) ); break; } - ++this.pos; + ++pos; } + this.pos = pos; return result; } @@ -1224,9 +1228,9 @@ export class Tokenizer extends DiagnosticEmitter { } testInteger(): bool { - var end = this.end; var text = this.source.text; var pos = this.pos; + var end = this.end; if (pos + 1 < end && text.charCodeAt(pos) == CharCode._0) { switch (text.charCodeAt(pos + 2) | 32) { case CharCode.x: From ae9bde338bc85f60e64a70282ad32b1eddcdce4a Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Sun, 5 Jul 2020 21:44:39 +0300 Subject: [PATCH 12/13] more --- src/tokenizer.ts | 116 +++++++++++++++++++++++++---------------------- 1 file changed, 63 insertions(+), 53 deletions(-) diff --git a/src/tokenizer.ts b/src/tokenizer.ts index ae758256f3..bcd9055931 100644 --- a/src/tokenizer.ts +++ b/src/tokenizer.ts @@ -1282,13 +1282,13 @@ export class Tokenizer extends DiagnosticEmitter { readHexInteger(): i64 { var text = this.source.text; - var start = this.pos; + let pos = this.pos; + var end = this.end; + var start = pos; + var sepEnd = start; var value = i64_new(0); var i64_4 = i64_new(4); - var sepEnd = start; - var end = this.end; - while (this.pos < end) { - let pos = this.pos; + while (pos < end) { let c = text.charCodeAt(pos); if (c >= CharCode._0 && c <= CharCode._9) { // value = (value << 4) + c - CharCode._0; @@ -1321,31 +1321,32 @@ export class Tokenizer extends DiagnosticEmitter { } else { break; } - this.pos = pos + 1; + ++pos; } - if (this.pos == start) { + if (pos == start) { this.error( DiagnosticCode.Hexadecimal_digit_expected, this.range(start) ); - } else if (sepEnd == this.pos) { + } else if (sepEnd == pos) { this.error( DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1) ); } + this.pos = pos; return value; } readDecimalInteger(): i64 { var text = this.source.text; - var start = this.pos; + var pos = this.pos; var end = this.end; + var start = pos; + var sepEnd = start; var value = i64_new(0); var i64_10 = i64_new(10); - var sepEnd = start; - while (this.pos < end) { - let pos = this.pos; + while (pos < end) { let c = text.charCodeAt(pos); if (c >= CharCode._0 && c <= CharCode._9) { // value = value * 10 + c - CharCode._0; @@ -1366,31 +1367,32 @@ export class Tokenizer extends DiagnosticEmitter { } else { break; } - this.pos = pos + 1; + ++pos; } - if (this.pos == start) { + if (pos == start) { this.error( DiagnosticCode.Digit_expected, this.range(start) ); - } else if (sepEnd == this.pos) { + } else if (sepEnd == pos) { this.error( DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1) ); } + this.pos = pos; return value; } readOctalInteger(): i64 { var text = this.source.text; - var start = this.pos; + var pos = this.pos; + var end = this.end; + var start = pos; + var sepEnd = start; var value = i64_new(0); var i64_3 = i64_new(3); - var sepEnd = start; - var end = this.end; - while (this.pos < end) { - let pos = this.pos; + while (pos < end) { let c = text.charCodeAt(pos); if (c >= CharCode._0 && c <= CharCode._7) { // value = (value << 3) + c - CharCode._0; @@ -1411,31 +1413,32 @@ export class Tokenizer extends DiagnosticEmitter { } else { break; } - this.pos = pos + 1; + ++pos; } - if (this.pos == start) { + if (pos == start) { this.error( DiagnosticCode.Octal_digit_expected, this.range(start) ); - } else if (sepEnd == this.pos) { + } else if (sepEnd == pos) { this.error( DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1) ); } + this.pos = pos; return value; } readBinaryInteger(): i64 { var text = this.source.text; - var start = this.pos; + var pos = this.pos; + var end = this.end; + var start = pos; + var sepEnd = start; var value = i64_new(0); var i64_1 = i64_new(1); - var sepEnd = start; - var end = this.end; - while (this.pos < end) { - let pos = this.pos; + while (pos < end) { let c = text.charCodeAt(pos); if (c == CharCode._0) { // value = (value << 1); @@ -1459,19 +1462,20 @@ export class Tokenizer extends DiagnosticEmitter { } else { break; } - this.pos = pos + 1; + ++pos; } - if (this.pos == start) { + if (pos == start) { this.error( DiagnosticCode.Binary_digit_expected, this.range(start) ); - } else if (sepEnd == this.pos) { + } else if (sepEnd == pos) { this.error( DiagnosticCode.Numeric_separators_are_not_allowed_here, this.range(sepEnd - 1) ); } + this.pos = pos; return value; } @@ -1491,34 +1495,36 @@ export class Tokenizer extends DiagnosticEmitter { readDecimalFloat(): f64 { // TODO: numeric separators (parseFloat can't handle these) - var start = this.pos; - var end = this.end; var text = this.source.text; - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { - ++this.pos; + var pos = this.pos; + var end = this.end; + var start = pos; + while (pos < end && isDecimalDigit(text.charCodeAt(pos))) { + ++pos; } - if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) { - ++this.pos; - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { - ++this.pos; + if (pos < end && text.charCodeAt(pos) == CharCode.DOT) { + ++pos; + while (pos < end && isDecimalDigit(text.charCodeAt(pos))) { + ++pos; } } - if (this.pos < end) { - let c = text.charCodeAt(this.pos); + if (pos < end) { + let c = text.charCodeAt(pos); if ((c | 32) == CharCode.e) { if ( - ++this.pos < end && - (c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS && - isDecimalDigit(text.charCodeAt(this.pos + 1)) + ++pos < end && + (c = text.charCodeAt(pos)) == CharCode.MINUS || c == CharCode.PLUS && + isDecimalDigit(text.charCodeAt(pos + 1)) ) { - ++this.pos; + ++pos; } - while (this.pos < end && isDecimalDigit(text.charCodeAt(this.pos))) { - ++this.pos; + while (pos < end && isDecimalDigit(text.charCodeAt(pos))) { + ++pos; } } } - return parseFloat(text.substring(start, this.pos)); + this.pos = pos; + return parseFloat(text.substring(start, pos)); } readHexFloat(): f64 { @@ -1527,10 +1533,11 @@ export class Tokenizer extends DiagnosticEmitter { readHexadecimalEscape(remain: i32 = 2): string { var value = 0; - var end = this.end; var text = this.source.text; - while (this.pos < end) { - let c = text.charCodeAt(this.pos++); + var pos = this.pos; + var end = this.end; + while (pos < end) { + let c = text.charCodeAt(pos++); if (c >= CharCode._0 && c <= CharCode._9) { value = (value << 4) + c - CharCode._0; } else if (c >= CharCode.A && c <= CharCode.F) { @@ -1540,8 +1547,9 @@ export class Tokenizer extends DiagnosticEmitter { } else { this.error( DiagnosticCode.Hexadecimal_digit_expected, - this.range(this.pos - 1, this.pos) + this.range(pos - 1, pos) ); + this.pos = pos; return ""; } if (--remain == 0) break; @@ -1549,10 +1557,12 @@ export class Tokenizer extends DiagnosticEmitter { if (remain) { this.error( DiagnosticCode.Unexpected_end_of_text, - this.range(this.pos) + this.range(pos) ); + this.pos = pos; return ""; } + this.pos = pos; return String.fromCharCode(value); } From d351233836c21dbad7c61857938a8f1f1b78a762 Mon Sep 17 00:00:00 2001 From: MaxGraey Date: Thu, 9 Jul 2020 23:38:29 +0300 Subject: [PATCH 13/13] ty -> type --- src/parser.ts | 40 ++++++++++++++++++++-------------------- 1 file changed, 20 insertions(+), 20 deletions(-) diff --git a/src/parser.ts b/src/parser.ts index 548ca09285..a6f1459bd1 100644 --- a/src/parser.ts +++ b/src/parser.ts @@ -713,17 +713,17 @@ export class Parser extends DiagnosticEmitter { if (tn.skip(Token.COLON)) { isSignature = true; tn.discard(state); - let ty = this.parseType(tn, false); - if (!ty) return null; - if (ty.kind != NodeKind.NAMEDTYPE) { + let type = this.parseType(tn, false); + if (!type) return null; + if (type.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - ty.range + type.range ); this.tryParseSignatureIsSignature = true; return null; } - thisType = ty; + thisType = type; } else { tn.reset(state); this.tryParseSignatureIsSignature = false; @@ -1154,29 +1154,29 @@ export class Parser extends DiagnosticEmitter { ); let extendsType: NamedTypeNode | null = null; if (tn.skip(Token.EXTENDS)) { - let ty = this.parseType(tn); - if (!ty) return null; - if (ty.kind != NodeKind.NAMEDTYPE) { + let type = this.parseType(tn); + if (!type) return null; + if (type.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - ty.range + type.range ); return null; } - extendsType = ty; + extendsType = type; } let defaultType: NamedTypeNode | null = null; if (tn.skip(Token.EQUALS)) { - let ty = this.parseType(tn); - if (!ty) return null; - if (ty.kind != NodeKind.NAMEDTYPE) { + let type = this.parseType(tn); + if (!type) return null; + if (type.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - ty.range + type.range ); return null; } - defaultType = ty; + defaultType = type; } return Node.createTypeParameter( identifier, @@ -1674,16 +1674,16 @@ export class Parser extends DiagnosticEmitter { var extendsType: NamedTypeNode | null = null; if (tn.skip(Token.EXTENDS)) { - let ty = this.parseType(tn); - if (!ty) return null; - if (ty.kind != NodeKind.NAMEDTYPE) { + let type = this.parseType(tn); + if (!type) return null; + if (type.kind != NodeKind.NAMEDTYPE) { this.error( DiagnosticCode.Identifier_expected, - ty.range + type.range ); return null; } - extendsType = ty; + extendsType = type; } var implementsTypes: NamedTypeNode[] | null = null;