Skip to content

Commit 24f7787

Browse files
author
Andy
committed
Have scanJsDocToken scan keywords (#27162)
* Have scanJsDocToken scan keywords * Update API
1 parent f06bfa3 commit 24f7787

16 files changed

+213
-111
lines changed

‎src/compiler/parser.ts

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6517,7 +6517,7 @@ namespace ts {
65176517
}
65186518
}
65196519

6520-
function skipWhitespaceOrAsterisk(next: () => void): void {
6520+
function skipWhitespaceOrAsterisk(): void {
65216521
if (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
65226522
if (lookAhead(isNextNonwhitespaceTokenEndOfFile)) {
65236523
return; // Don't skip whitespace prior to EoF (or end of comment) - that shouldn't be included in any node's range
@@ -6532,7 +6532,7 @@ namespace ts {
65326532
else if (token() === SyntaxKind.AsteriskToken) {
65336533
precedingLineBreak = false;
65346534
}
6535-
next();
6535+
nextJSDocToken();
65366536
}
65376537
}
65386538

@@ -6542,9 +6542,8 @@ namespace ts {
65426542
atToken.end = scanner.getTextPos();
65436543
nextJSDocToken();
65446544

6545-
// Use 'nextToken' instead of 'nextJsDocToken' so we can parse a type like 'number' in `@enum number`
6546-
const tagName = parseJSDocIdentifierName(/*message*/ undefined, nextToken);
6547-
skipWhitespaceOrAsterisk(nextToken);
6545+
const tagName = parseJSDocIdentifierName(/*message*/ undefined);
6546+
skipWhitespaceOrAsterisk();
65486547

65496548
let tag: JSDocTag | undefined;
65506549
switch (tagName.escapedText) {
@@ -6688,7 +6687,7 @@ namespace ts {
66886687
}
66896688

66906689
function tryParseTypeExpression(): JSDocTypeExpression | undefined {
6691-
skipWhitespaceOrAsterisk(nextJSDocToken);
6690+
skipWhitespaceOrAsterisk();
66926691
return token() === SyntaxKind.OpenBraceToken ? parseJSDocTypeExpression() : undefined;
66936692
}
66946693

@@ -6728,7 +6727,7 @@ namespace ts {
67286727
function parseParameterOrPropertyTag(atToken: AtToken, tagName: Identifier, target: PropertyLikeParse, indent: number): JSDocParameterTag | JSDocPropertyTag {
67296728
let typeExpression = tryParseTypeExpression();
67306729
let isNameFirst = !typeExpression;
6731-
skipWhitespaceOrAsterisk(nextJSDocToken);
6730+
skipWhitespaceOrAsterisk();
67326731

67336732
const { name, isBracketed } = parseBracketNameInPropertyAndParamTag();
67346733
skipWhitespace();
@@ -6862,7 +6861,7 @@ namespace ts {
68626861

68636862
function parseTypedefTag(atToken: AtToken, tagName: Identifier, indent: number): JSDocTypedefTag {
68646863
const typeExpression = tryParseTypeExpression();
6865-
skipWhitespaceOrAsterisk(nextJSDocToken);
6864+
skipWhitespaceOrAsterisk();
68666865

68676866
const typedefTag = <JSDocTypedefTag>createNode(SyntaxKind.JSDocTypedefTag, atToken.pos);
68686867
typedefTag.atToken = atToken;
@@ -7115,7 +7114,7 @@ namespace ts {
71157114
return entity;
71167115
}
71177116

7118-
function parseJSDocIdentifierName(message?: DiagnosticMessage, next: () => void = nextJSDocToken): Identifier {
7117+
function parseJSDocIdentifierName(message?: DiagnosticMessage): Identifier {
71197118
if (!tokenIsIdentifierOrKeyword(token())) {
71207119
return createMissingNode<Identifier>(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ !message, message || Diagnostics.Identifier_expected);
71217120
}
@@ -7126,7 +7125,7 @@ namespace ts {
71267125
result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
71277126
finishNode(result, end);
71287127

7129-
next();
7128+
nextJSDocToken();
71307129
return result;
71317130
}
71327131
}

‎src/compiler/scanner.ts

Lines changed: 86 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -60,81 +60,87 @@ namespace ts {
6060
tryScan<T>(callback: () => T): T;
6161
}
6262

63-
const textToToken = createMapFromTemplate({
64-
"abstract": SyntaxKind.AbstractKeyword,
65-
"any": SyntaxKind.AnyKeyword,
66-
"as": SyntaxKind.AsKeyword,
67-
"boolean": SyntaxKind.BooleanKeyword,
68-
"break": SyntaxKind.BreakKeyword,
69-
"case": SyntaxKind.CaseKeyword,
70-
"catch": SyntaxKind.CatchKeyword,
71-
"class": SyntaxKind.ClassKeyword,
72-
"continue": SyntaxKind.ContinueKeyword,
73-
"const": SyntaxKind.ConstKeyword,
74-
"constructor": SyntaxKind.ConstructorKeyword,
75-
"debugger": SyntaxKind.DebuggerKeyword,
76-
"declare": SyntaxKind.DeclareKeyword,
77-
"default": SyntaxKind.DefaultKeyword,
78-
"delete": SyntaxKind.DeleteKeyword,
79-
"do": SyntaxKind.DoKeyword,
80-
"else": SyntaxKind.ElseKeyword,
81-
"enum": SyntaxKind.EnumKeyword,
82-
"export": SyntaxKind.ExportKeyword,
83-
"extends": SyntaxKind.ExtendsKeyword,
84-
"false": SyntaxKind.FalseKeyword,
85-
"finally": SyntaxKind.FinallyKeyword,
86-
"for": SyntaxKind.ForKeyword,
87-
"from": SyntaxKind.FromKeyword,
88-
"function": SyntaxKind.FunctionKeyword,
89-
"get": SyntaxKind.GetKeyword,
90-
"if": SyntaxKind.IfKeyword,
91-
"implements": SyntaxKind.ImplementsKeyword,
92-
"import": SyntaxKind.ImportKeyword,
93-
"in": SyntaxKind.InKeyword,
94-
"infer": SyntaxKind.InferKeyword,
95-
"instanceof": SyntaxKind.InstanceOfKeyword,
96-
"interface": SyntaxKind.InterfaceKeyword,
97-
"is": SyntaxKind.IsKeyword,
98-
"keyof": SyntaxKind.KeyOfKeyword,
99-
"let": SyntaxKind.LetKeyword,
100-
"module": SyntaxKind.ModuleKeyword,
101-
"namespace": SyntaxKind.NamespaceKeyword,
102-
"never": SyntaxKind.NeverKeyword,
103-
"new": SyntaxKind.NewKeyword,
104-
"null": SyntaxKind.NullKeyword,
105-
"number": SyntaxKind.NumberKeyword,
106-
"object": SyntaxKind.ObjectKeyword,
107-
"package": SyntaxKind.PackageKeyword,
108-
"private": SyntaxKind.PrivateKeyword,
109-
"protected": SyntaxKind.ProtectedKeyword,
110-
"public": SyntaxKind.PublicKeyword,
111-
"readonly": SyntaxKind.ReadonlyKeyword,
112-
"require": SyntaxKind.RequireKeyword,
113-
"global": SyntaxKind.GlobalKeyword,
114-
"return": SyntaxKind.ReturnKeyword,
115-
"set": SyntaxKind.SetKeyword,
116-
"static": SyntaxKind.StaticKeyword,
117-
"string": SyntaxKind.StringKeyword,
118-
"super": SyntaxKind.SuperKeyword,
119-
"switch": SyntaxKind.SwitchKeyword,
120-
"symbol": SyntaxKind.SymbolKeyword,
121-
"this": SyntaxKind.ThisKeyword,
122-
"throw": SyntaxKind.ThrowKeyword,
123-
"true": SyntaxKind.TrueKeyword,
124-
"try": SyntaxKind.TryKeyword,
125-
"type": SyntaxKind.TypeKeyword,
126-
"typeof": SyntaxKind.TypeOfKeyword,
127-
"undefined": SyntaxKind.UndefinedKeyword,
128-
"unique": SyntaxKind.UniqueKeyword,
129-
"unknown": SyntaxKind.UnknownKeyword,
130-
"var": SyntaxKind.VarKeyword,
131-
"void": SyntaxKind.VoidKeyword,
132-
"while": SyntaxKind.WhileKeyword,
133-
"with": SyntaxKind.WithKeyword,
134-
"yield": SyntaxKind.YieldKeyword,
135-
"async": SyntaxKind.AsyncKeyword,
136-
"await": SyntaxKind.AwaitKeyword,
137-
"of": SyntaxKind.OfKeyword,
63+
const textToKeywordObj: MapLike<KeywordSyntaxKind> = {
64+
abstract: SyntaxKind.AbstractKeyword,
65+
any: SyntaxKind.AnyKeyword,
66+
as: SyntaxKind.AsKeyword,
67+
boolean: SyntaxKind.BooleanKeyword,
68+
break: SyntaxKind.BreakKeyword,
69+
case: SyntaxKind.CaseKeyword,
70+
catch: SyntaxKind.CatchKeyword,
71+
class: SyntaxKind.ClassKeyword,
72+
continue: SyntaxKind.ContinueKeyword,
73+
const: SyntaxKind.ConstKeyword,
74+
["" + "constructor"]: SyntaxKind.ConstructorKeyword,
75+
debugger: SyntaxKind.DebuggerKeyword,
76+
declare: SyntaxKind.DeclareKeyword,
77+
default: SyntaxKind.DefaultKeyword,
78+
delete: SyntaxKind.DeleteKeyword,
79+
do: SyntaxKind.DoKeyword,
80+
else: SyntaxKind.ElseKeyword,
81+
enum: SyntaxKind.EnumKeyword,
82+
export: SyntaxKind.ExportKeyword,
83+
extends: SyntaxKind.ExtendsKeyword,
84+
false: SyntaxKind.FalseKeyword,
85+
finally: SyntaxKind.FinallyKeyword,
86+
for: SyntaxKind.ForKeyword,
87+
from: SyntaxKind.FromKeyword,
88+
function: SyntaxKind.FunctionKeyword,
89+
get: SyntaxKind.GetKeyword,
90+
if: SyntaxKind.IfKeyword,
91+
implements: SyntaxKind.ImplementsKeyword,
92+
import: SyntaxKind.ImportKeyword,
93+
in: SyntaxKind.InKeyword,
94+
infer: SyntaxKind.InferKeyword,
95+
instanceof: SyntaxKind.InstanceOfKeyword,
96+
interface: SyntaxKind.InterfaceKeyword,
97+
is: SyntaxKind.IsKeyword,
98+
keyof: SyntaxKind.KeyOfKeyword,
99+
let: SyntaxKind.LetKeyword,
100+
module: SyntaxKind.ModuleKeyword,
101+
namespace: SyntaxKind.NamespaceKeyword,
102+
never: SyntaxKind.NeverKeyword,
103+
new: SyntaxKind.NewKeyword,
104+
null: SyntaxKind.NullKeyword,
105+
number: SyntaxKind.NumberKeyword,
106+
object: SyntaxKind.ObjectKeyword,
107+
package: SyntaxKind.PackageKeyword,
108+
private: SyntaxKind.PrivateKeyword,
109+
protected: SyntaxKind.ProtectedKeyword,
110+
public: SyntaxKind.PublicKeyword,
111+
readonly: SyntaxKind.ReadonlyKeyword,
112+
require: SyntaxKind.RequireKeyword,
113+
global: SyntaxKind.GlobalKeyword,
114+
return: SyntaxKind.ReturnKeyword,
115+
set: SyntaxKind.SetKeyword,
116+
static: SyntaxKind.StaticKeyword,
117+
string: SyntaxKind.StringKeyword,
118+
super: SyntaxKind.SuperKeyword,
119+
switch: SyntaxKind.SwitchKeyword,
120+
symbol: SyntaxKind.SymbolKeyword,
121+
this: SyntaxKind.ThisKeyword,
122+
throw: SyntaxKind.ThrowKeyword,
123+
true: SyntaxKind.TrueKeyword,
124+
try: SyntaxKind.TryKeyword,
125+
type: SyntaxKind.TypeKeyword,
126+
typeof: SyntaxKind.TypeOfKeyword,
127+
undefined: SyntaxKind.UndefinedKeyword,
128+
unique: SyntaxKind.UniqueKeyword,
129+
unknown: SyntaxKind.UnknownKeyword,
130+
var: SyntaxKind.VarKeyword,
131+
void: SyntaxKind.VoidKeyword,
132+
while: SyntaxKind.WhileKeyword,
133+
with: SyntaxKind.WithKeyword,
134+
yield: SyntaxKind.YieldKeyword,
135+
async: SyntaxKind.AsyncKeyword,
136+
await: SyntaxKind.AwaitKeyword,
137+
of: SyntaxKind.OfKeyword,
138+
};
139+
140+
const textToKeyword = createMapFromTemplate(textToKeywordObj);
141+
142+
const textToToken = createMapFromTemplate<SyntaxKind>({
143+
...textToKeywordObj,
138144
"{": SyntaxKind.OpenBraceToken,
139145
"}": SyntaxKind.CloseBraceToken,
140146
"(": SyntaxKind.OpenParenToken,
@@ -1288,15 +1294,15 @@ namespace ts {
12881294
return result;
12891295
}
12901296

1291-
function getIdentifierToken(): SyntaxKind {
1297+
function getIdentifierToken(): SyntaxKind.Identifier | KeywordSyntaxKind {
12921298
// Reserved words are between 2 and 11 characters long and start with a lowercase letter
12931299
const len = tokenValue.length;
12941300
if (len >= 2 && len <= 11) {
12951301
const ch = tokenValue.charCodeAt(0);
12961302
if (ch >= CharacterCodes.a && ch <= CharacterCodes.z) {
1297-
token = textToToken.get(tokenValue)!;
1298-
if (token !== undefined) {
1299-
return token;
1303+
const keyword = textToKeyword.get(tokenValue);
1304+
if (keyword !== undefined) {
1305+
return token = keyword;
13001306
}
13011307
}
13021308
}
@@ -2016,7 +2022,7 @@ namespace ts {
20162022
pos++;
20172023
}
20182024
tokenValue = text.substring(tokenPos, pos);
2019-
return token = SyntaxKind.Identifier;
2025+
return token = getIdentifierToken();
20202026
}
20212027
else {
20222028
return token = SyntaxKind.Unknown;

‎src/compiler/types.ts

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,84 @@ namespace ts {
2424
| SyntaxKind.DotToken
2525
| SyntaxKind.Identifier
2626
| SyntaxKind.NoSubstitutionTemplateLiteral
27-
| SyntaxKind.Unknown;
27+
| SyntaxKind.Unknown
28+
| KeywordSyntaxKind;
29+
30+
export type KeywordSyntaxKind =
31+
| SyntaxKind.AbstractKeyword
32+
| SyntaxKind.AnyKeyword
33+
| SyntaxKind.AsKeyword
34+
| SyntaxKind.BooleanKeyword
35+
| SyntaxKind.BreakKeyword
36+
| SyntaxKind.CaseKeyword
37+
| SyntaxKind.CatchKeyword
38+
| SyntaxKind.ClassKeyword
39+
| SyntaxKind.ContinueKeyword
40+
| SyntaxKind.ConstKeyword
41+
| SyntaxKind.ConstructorKeyword
42+
| SyntaxKind.DebuggerKeyword
43+
| SyntaxKind.DeclareKeyword
44+
| SyntaxKind.DefaultKeyword
45+
| SyntaxKind.DeleteKeyword
46+
| SyntaxKind.DoKeyword
47+
| SyntaxKind.ElseKeyword
48+
| SyntaxKind.EnumKeyword
49+
| SyntaxKind.ExportKeyword
50+
| SyntaxKind.ExtendsKeyword
51+
| SyntaxKind.FalseKeyword
52+
| SyntaxKind.FinallyKeyword
53+
| SyntaxKind.ForKeyword
54+
| SyntaxKind.FromKeyword
55+
| SyntaxKind.FunctionKeyword
56+
| SyntaxKind.GetKeyword
57+
| SyntaxKind.IfKeyword
58+
| SyntaxKind.ImplementsKeyword
59+
| SyntaxKind.ImportKeyword
60+
| SyntaxKind.InKeyword
61+
| SyntaxKind.InferKeyword
62+
| SyntaxKind.InstanceOfKeyword
63+
| SyntaxKind.InterfaceKeyword
64+
| SyntaxKind.IsKeyword
65+
| SyntaxKind.KeyOfKeyword
66+
| SyntaxKind.LetKeyword
67+
| SyntaxKind.ModuleKeyword
68+
| SyntaxKind.NamespaceKeyword
69+
| SyntaxKind.NeverKeyword
70+
| SyntaxKind.NewKeyword
71+
| SyntaxKind.NullKeyword
72+
| SyntaxKind.NumberKeyword
73+
| SyntaxKind.ObjectKeyword
74+
| SyntaxKind.PackageKeyword
75+
| SyntaxKind.PrivateKeyword
76+
| SyntaxKind.ProtectedKeyword
77+
| SyntaxKind.PublicKeyword
78+
| SyntaxKind.ReadonlyKeyword
79+
| SyntaxKind.RequireKeyword
80+
| SyntaxKind.GlobalKeyword
81+
| SyntaxKind.ReturnKeyword
82+
| SyntaxKind.SetKeyword
83+
| SyntaxKind.StaticKeyword
84+
| SyntaxKind.StringKeyword
85+
| SyntaxKind.SuperKeyword
86+
| SyntaxKind.SwitchKeyword
87+
| SyntaxKind.SymbolKeyword
88+
| SyntaxKind.ThisKeyword
89+
| SyntaxKind.ThrowKeyword
90+
| SyntaxKind.TrueKeyword
91+
| SyntaxKind.TryKeyword
92+
| SyntaxKind.TypeKeyword
93+
| SyntaxKind.TypeOfKeyword
94+
| SyntaxKind.UndefinedKeyword
95+
| SyntaxKind.UniqueKeyword
96+
| SyntaxKind.UnknownKeyword
97+
| SyntaxKind.VarKeyword
98+
| SyntaxKind.VoidKeyword
99+
| SyntaxKind.WhileKeyword
100+
| SyntaxKind.WithKeyword
101+
| SyntaxKind.YieldKeyword
102+
| SyntaxKind.AsyncKeyword
103+
| SyntaxKind.AwaitKeyword
104+
| SyntaxKind.OfKeyword;
28105

29106
export type JsxTokenSyntaxKind =
30107
| SyntaxKind.LessThanSlashToken

‎tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.@link tags.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"0": {
77
"kind": "JSDocTag",
88
"pos": 63,
9-
"end": 67,
9+
"end": 68,
1010
"atToken": {
1111
"kind": "AtToken",
1212
"pos": 63,
@@ -22,7 +22,7 @@
2222
},
2323
"length": 1,
2424
"pos": 63,
25-
"end": 67
25+
"end": 68
2626
},
2727
"comment": "{@link first link}\nInside {@link link text} thing"
2828
}

‎tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
"typeParameters": {
2222
"0": {
2323
"kind": "TypeParameter",
24-
"pos": 17,
24+
"pos": 18,
2525
"end": 19,
2626
"name": {
2727
"kind": "Identifier",
@@ -31,7 +31,7 @@
3131
}
3232
},
3333
"length": 1,
34-
"pos": 17,
34+
"pos": 18,
3535
"end": 19
3636
}
3737
},

0 commit comments

Comments
 (0)