Skip to content

Commit 50bb499

Browse files
CyrusNajmabadiCyrusNajmabadi
committed
Support passing a start/length when creating a scanner.
1 parent 8ba61f6 commit 50bb499

File tree

3 files changed

+61
-58
lines changed

3 files changed

+61
-58
lines changed

‎src/compiler/parser.ts

+2-3
Original file line numberDiff line numberDiff line change
@@ -5388,9 +5388,8 @@ module ts {
53885388

53895389
// Parses out a JSDoc type expression. The starting position should be right at the open
53905390
// curly in the type expression. Returns 'undefined' if it encounters any errors while parsing.
5391-
export function parseJSDocTypeExpression(content: string, start: number): JSDocType {
5392-
let scanner = createScanner(ScriptTarget.Latest, /*skipTrivia:*/ true, content);
5393-
scanner.setTextPos(start);
5391+
export function parseJSDocTypeExpression(content: string, start?: number, length?: number): JSDocType {
5392+
let scanner = createScanner(ScriptTarget.Latest, /*skipTrivia:*/ true, content, /*onError:*/ undefined, start, length);
53945393

53955394
// Prime the first token for us to start processing.
53965395
let token = nextToken();

‎src/compiler/scanner.ts

+58-53
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,11 @@ module ts {
2323
reScanSlashToken(): SyntaxKind;
2424
reScanTemplateToken(): SyntaxKind;
2525
scan(): SyntaxKind;
26-
setText(text: string): void;
26+
27+
// Sets the text for the scanner to scan. An optional subrange starting point and length
28+
// can be provided to have the scanner only scan a portion of the text.
29+
setText(text: string, start?: number, length?: number): void;
30+
2731
setTextPos(textPos: number): void;
2832
// Invokes the provided callback then unconditionally restores the scanner to the state it
2933
// was in immediately prior to invoking the callback. The result of invoking the callback
@@ -587,9 +591,10 @@ module ts {
587591
ch > CharacterCodes.maxAsciiCharacter && isUnicodeIdentifierPart(ch, languageVersion);
588592
}
589593

590-
export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback): Scanner {
594+
// Creates a scanner over a (possibly unspecified) range of a piece of text.
595+
export function createScanner(languageVersion: ScriptTarget, skipTrivia: boolean, text?: string, onError?: ErrorCallback, start?: number, length?: number): Scanner {
591596
let pos: number; // Current position (end position of text of current token)
592-
let len: number; // Length of text
597+
let end: number; // end of text
593598
let startPos: number; // Start position of whitespace before current token
594599
let tokenPos: number; // Start position of text of current token
595600
let token: SyntaxKind;
@@ -598,6 +603,30 @@ module ts {
598603
let hasExtendedUnicodeEscape: boolean;
599604
let tokenIsUnterminated: boolean;
600605

606+
setText(text, start, length);
607+
608+
return {
609+
getStartPos: () => startPos,
610+
getTextPos: () => pos,
611+
getToken: () => token,
612+
getTokenPos: () => tokenPos,
613+
getTokenText: () => text.substring(tokenPos, pos),
614+
getTokenValue: () => tokenValue,
615+
hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
616+
hasPrecedingLineBreak: () => precedingLineBreak,
617+
isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
618+
isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
619+
isUnterminated: () => tokenIsUnterminated,
620+
reScanGreaterToken,
621+
reScanSlashToken,
622+
reScanTemplateToken,
623+
scan,
624+
setText,
625+
setTextPos,
626+
tryScan,
627+
lookAhead,
628+
};
629+
601630
function error(message: DiagnosticMessage, length?: number): void {
602631
if (onError) {
603632
onError(message, length || 0);
@@ -694,7 +723,7 @@ module ts {
694723
let result = "";
695724
let start = pos;
696725
while (true) {
697-
if (pos >= len) {
726+
if (pos >= end) {
698727
result += text.substring(start, pos);
699728
tokenIsUnterminated = true;
700729
error(Diagnostics.Unterminated_string_literal);
@@ -736,7 +765,7 @@ module ts {
736765
let resultingToken: SyntaxKind;
737766

738767
while (true) {
739-
if (pos >= len) {
768+
if (pos >= end) {
740769
contents += text.substring(start, pos);
741770
tokenIsUnterminated = true;
742771
error(Diagnostics.Unterminated_template_literal);
@@ -755,7 +784,7 @@ module ts {
755784
}
756785

757786
// '${'
758-
if (currChar === CharacterCodes.$ && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) {
787+
if (currChar === CharacterCodes.$ && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.openBrace) {
759788
contents += text.substring(start, pos);
760789
pos += 2;
761790
resultingToken = startedWithBacktick ? SyntaxKind.TemplateHead : SyntaxKind.TemplateMiddle;
@@ -776,7 +805,7 @@ module ts {
776805
contents += text.substring(start, pos);
777806
pos++;
778807

779-
if (pos < len && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
808+
if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
780809
pos++;
781810
}
782811

@@ -796,7 +825,7 @@ module ts {
796825

797826
function scanEscapeSequence(): string {
798827
pos++;
799-
if (pos >= len) {
828+
if (pos >= end) {
800829
error(Diagnostics.Unexpected_end_of_text);
801830
return "";
802831
}
@@ -822,7 +851,7 @@ module ts {
822851
return "\"";
823852
case CharacterCodes.u:
824853
// '\u{DDDDDDDD}'
825-
if (pos < len && text.charCodeAt(pos) === CharacterCodes.openBrace) {
854+
if (pos < end && text.charCodeAt(pos) === CharacterCodes.openBrace) {
826855
hasExtendedUnicodeEscape = true;
827856
pos++;
828857
return scanExtendedUnicodeEscape();
@@ -838,7 +867,7 @@ module ts {
838867
// when encountering a LineContinuation (i.e. a backslash and a line terminator sequence),
839868
// the line terminator is interpreted to be "the empty code unit sequence".
840869
case CharacterCodes.carriageReturn:
841-
if (pos < len && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
870+
if (pos < end && text.charCodeAt(pos) === CharacterCodes.lineFeed) {
842871
pos++;
843872
}
844873
// fall through
@@ -877,7 +906,7 @@ module ts {
877906
isInvalidExtendedEscape = true;
878907
}
879908

880-
if (pos >= len) {
909+
if (pos >= end) {
881910
error(Diagnostics.Unexpected_end_of_text);
882911
isInvalidExtendedEscape = true;
883912
}
@@ -914,7 +943,7 @@ module ts {
914943
// Current character is known to be a backslash. Check for Unicode escape of the form '\uXXXX'
915944
// and return code point value if valid Unicode escape is found. Otherwise return -1.
916945
function peekUnicodeEscape(): number {
917-
if (pos + 5 < len && text.charCodeAt(pos + 1) === CharacterCodes.u) {
946+
if (pos + 5 < end && text.charCodeAt(pos + 1) === CharacterCodes.u) {
918947
let start = pos;
919948
pos += 2;
920949
let value = scanExactNumberOfHexDigits(4);
@@ -927,7 +956,7 @@ module ts {
927956
function scanIdentifierParts(): string {
928957
let result = "";
929958
let start = pos;
930-
while (pos < len) {
959+
while (pos < end) {
931960
let ch = text.charCodeAt(pos);
932961
if (isIdentifierPart(ch)) {
933962
pos++;
@@ -994,7 +1023,7 @@ module ts {
9941023
tokenIsUnterminated = false;
9951024
while (true) {
9961025
tokenPos = pos;
997-
if (pos >= len) {
1026+
if (pos >= end) {
9981027
return token = SyntaxKind.EndOfFileToken;
9991028
}
10001029
let ch = text.charCodeAt(pos);
@@ -1007,7 +1036,7 @@ module ts {
10071036
continue;
10081037
}
10091038
else {
1010-
if (ch === CharacterCodes.carriageReturn && pos + 1 < len && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) {
1039+
if (ch === CharacterCodes.carriageReturn && pos + 1 < end && text.charCodeAt(pos + 1) === CharacterCodes.lineFeed) {
10111040
// consume both CR and LF
10121041
pos += 2;
10131042
}
@@ -1025,7 +1054,7 @@ module ts {
10251054
continue;
10261055
}
10271056
else {
1028-
while (pos < len && isWhiteSpace(text.charCodeAt(pos))) {
1057+
while (pos < end && isWhiteSpace(text.charCodeAt(pos))) {
10291058
pos++;
10301059
}
10311060
return token = SyntaxKind.WhitespaceTrivia;
@@ -1098,7 +1127,7 @@ module ts {
10981127
if (text.charCodeAt(pos + 1) === CharacterCodes.slash) {
10991128
pos += 2;
11001129

1101-
while (pos < len) {
1130+
while (pos < end) {
11021131
if (isLineBreak(text.charCodeAt(pos))) {
11031132
break;
11041133
}
@@ -1118,7 +1147,7 @@ module ts {
11181147
pos += 2;
11191148

11201149
let commentClosed = false;
1121-
while (pos < len) {
1150+
while (pos < end) {
11221151
let ch = text.charCodeAt(pos);
11231152

11241153
if (ch === CharacterCodes.asterisk && text.charCodeAt(pos + 1) === CharacterCodes.slash) {
@@ -1153,7 +1182,7 @@ module ts {
11531182
return pos++, token = SyntaxKind.SlashToken;
11541183

11551184
case CharacterCodes._0:
1156-
if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) {
1185+
if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.X || text.charCodeAt(pos + 1) === CharacterCodes.x)) {
11571186
pos += 2;
11581187
let value = scanMinimumNumberOfHexDigits(1);
11591188
if (value < 0) {
@@ -1163,7 +1192,7 @@ module ts {
11631192
tokenValue = "" + value;
11641193
return token = SyntaxKind.NumericLiteral;
11651194
}
1166-
else if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
1195+
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.B || text.charCodeAt(pos + 1) === CharacterCodes.b)) {
11671196
pos += 2;
11681197
let value = scanBinaryOrOctalDigits(/* base */ 2);
11691198
if (value < 0) {
@@ -1173,7 +1202,7 @@ module ts {
11731202
tokenValue = "" + value;
11741203
return token = SyntaxKind.NumericLiteral;
11751204
}
1176-
else if (pos + 2 < len && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
1205+
else if (pos + 2 < end && (text.charCodeAt(pos + 1) === CharacterCodes.O || text.charCodeAt(pos + 1) === CharacterCodes.o)) {
11771206
pos += 2;
11781207
let value = scanBinaryOrOctalDigits(/* base */ 8);
11791208
if (value < 0) {
@@ -1184,7 +1213,7 @@ module ts {
11841213
return token = SyntaxKind.NumericLiteral;
11851214
}
11861215
// Try to parse as an octal
1187-
if (pos + 1 < len && isOctalDigit(text.charCodeAt(pos + 1))) {
1216+
if (pos + 1 < end && isOctalDigit(text.charCodeAt(pos + 1))) {
11881217
tokenValue = "" + scanOctalDigits();
11891218
return token = SyntaxKind.NumericLiteral;
11901219
}
@@ -1299,7 +1328,7 @@ module ts {
12991328
default:
13001329
if (isIdentifierStart(ch)) {
13011330
pos++;
1302-
while (pos < len && isIdentifierPart(ch = text.charCodeAt(pos))) pos++;
1331+
while (pos < end && isIdentifierPart(ch = text.charCodeAt(pos))) pos++;
13031332
tokenValue = text.substring(tokenPos, pos);
13041333
if (ch === CharacterCodes.backslash) {
13051334
tokenValue += scanIdentifierParts();
@@ -1350,7 +1379,7 @@ module ts {
13501379
while (true) {
13511380
// If we reach the end of a file, or hit a newline, then this is an unterminated
13521381
// regex. Report error and return what we have so far.
1353-
if (p >= len) {
1382+
if (p >= end) {
13541383
tokenIsUnterminated = true;
13551384
error(Diagnostics.Unterminated_regular_expression_literal)
13561385
break;
@@ -1386,7 +1415,7 @@ module ts {
13861415
p++;
13871416
}
13881417

1389-
while (p < len && isIdentifierPart(text.charCodeAt(p))) {
1418+
while (p < end && isIdentifierPart(text.charCodeAt(p))) {
13901419
p++;
13911420
}
13921421
pos = p;
@@ -1435,43 +1464,19 @@ module ts {
14351464
return speculationHelper(callback, /*isLookahead:*/ false);
14361465
}
14371466

1438-
function setText(newText: string) {
1467+
function setText(newText: string, start?: number, length?: number) {
14391468
text = newText || "";
1440-
len = text.length;
1441-
setTextPos(0);
1469+
end = length === undefined ? text.length : start + length;
1470+
setTextPos(start || 0);
14421471
}
14431472

14441473
function setTextPos(textPos: number) {
1474+
Debug.assert(textPos >= 0);
14451475
pos = textPos;
14461476
startPos = textPos;
14471477
tokenPos = textPos;
14481478
token = SyntaxKind.Unknown;
14491479
precedingLineBreak = false;
14501480
}
1451-
1452-
setText(text);
1453-
1454-
1455-
return {
1456-
getStartPos: () => startPos,
1457-
getTextPos: () => pos,
1458-
getToken: () => token,
1459-
getTokenPos: () => tokenPos,
1460-
getTokenText: () => text.substring(tokenPos, pos),
1461-
getTokenValue: () => tokenValue,
1462-
hasExtendedUnicodeEscape: () => hasExtendedUnicodeEscape,
1463-
hasPrecedingLineBreak: () => precedingLineBreak,
1464-
isIdentifier: () => token === SyntaxKind.Identifier || token > SyntaxKind.LastReservedWord,
1465-
isReservedWord: () => token >= SyntaxKind.FirstReservedWord && token <= SyntaxKind.LastReservedWord,
1466-
isUnterminated: () => tokenIsUnterminated,
1467-
reScanGreaterToken,
1468-
reScanSlashToken,
1469-
reScanTemplateToken,
1470-
scan,
1471-
setText,
1472-
setTextPos,
1473-
tryScan,
1474-
lookAhead,
1475-
};
14761481
}
14771482
}

‎src/compiler/utilities.ts

+1-2
Original file line numberDiff line numberDiff line change
@@ -264,8 +264,7 @@ module ts {
264264

265265
/* @internal */
266266
export function getSpanOfTokenAtPosition(sourceFile: SourceFile, pos: number): TextSpan {
267-
let scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.text);
268-
scanner.setTextPos(pos);
267+
let scanner = createScanner(sourceFile.languageVersion, /*skipTrivia*/ true, sourceFile.text, /*onError:*/ undefined, pos);
269268
scanner.scan();
270269
let start = scanner.getTokenPos();
271270
return createTextSpanFromBounds(start, scanner.getTextPos());

0 commit comments

Comments
 (0)