Skip to content

Commit dd9b8ca

Browse files
author
Andy
authored
Have scanJsDocToken scan keywords (#27162)
* Have scanJsDocToken scan keywords * Update API
1 parent b85e9e1 commit dd9b8ca

16 files changed

+213
-111
lines changed

‎src/compiler/parser.ts‎

Lines changed: 9 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6512,7 +6512,7 @@ namespace ts {
65126512
}
65136513
}
65146514

6515-
function skipWhitespaceOrAsterisk(next: () => void): void {
6515+
function skipWhitespaceOrAsterisk(): void {
65166516
if (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
65176517
if (lookAhead(isNextNonwhitespaceTokenEndOfFile)) {
65186518
return; // Don't skip whitespace prior to EoF (or end of comment) - that shouldn't be included in any node's range
@@ -6527,7 +6527,7 @@ namespace ts {
65276527
else if (token() === SyntaxKind.AsteriskToken) {
65286528
precedingLineBreak = false;
65296529
}
6530-
next();
6530+
nextJSDocToken();
65316531
}
65326532
}
65336533

@@ -6537,9 +6537,8 @@ namespace ts {
65376537
atToken.end = scanner.getTextPos();
65386538
nextJSDocToken();
65396539

6540-
// Use 'nextToken' instead of 'nextJsDocToken' so we can parse a type like 'number' in `@enum number`
6541-
const tagName = parseJSDocIdentifierName(/*message*/ undefined, nextToken);
6542-
skipWhitespaceOrAsterisk(nextToken);
6540+
const tagName = parseJSDocIdentifierName(/*message*/ undefined);
6541+
skipWhitespaceOrAsterisk();
65436542

65446543
let tag: JSDocTag | undefined;
65456544
switch (tagName.escapedText) {
@@ -6683,7 +6682,7 @@ namespace ts {
66836682
}
66846683

66856684
function tryParseTypeExpression(): JSDocTypeExpression | undefined {
6686-
skipWhitespaceOrAsterisk(nextJSDocToken);
6685+
skipWhitespaceOrAsterisk();
66876686
return token() === SyntaxKind.OpenBraceToken ? parseJSDocTypeExpression() : undefined;
66886687
}
66896688

@@ -6723,7 +6722,7 @@ namespace ts {
67236722
function parseParameterOrPropertyTag(atToken: AtToken, tagName: Identifier, target: PropertyLikeParse, indent: number): JSDocParameterTag | JSDocPropertyTag {
67246723
let typeExpression = tryParseTypeExpression();
67256724
let isNameFirst = !typeExpression;
6726-
skipWhitespaceOrAsterisk(nextJSDocToken);
6725+
skipWhitespaceOrAsterisk();
67276726

67286727
const { name, isBracketed } = parseBracketNameInPropertyAndParamTag();
67296728
skipWhitespace();
@@ -6857,7 +6856,7 @@ namespace ts {
68576856

68586857
function parseTypedefTag(atToken: AtToken, tagName: Identifier, indent: number): JSDocTypedefTag {
68596858
const typeExpression = tryParseTypeExpression();
6860-
skipWhitespaceOrAsterisk(nextJSDocToken);
6859+
skipWhitespaceOrAsterisk();
68616860

68626861
const typedefTag = <JSDocTypedefTag>createNode(SyntaxKind.JSDocTypedefTag, atToken.pos);
68636862
typedefTag.atToken = atToken;
@@ -7110,7 +7109,7 @@ namespace ts {
71107109
return entity;
71117110
}
71127111

7113-
function parseJSDocIdentifierName(message?: DiagnosticMessage, next: () => void = nextJSDocToken): Identifier {
7112+
function parseJSDocIdentifierName(message?: DiagnosticMessage): Identifier {
71147113
if (!tokenIsIdentifierOrKeyword(token())) {
71157114
return createMissingNode<Identifier>(SyntaxKind.Identifier, /*reportAtCurrentPosition*/ !message, message || Diagnostics.Identifier_expected);
71167115
}
@@ -7121,7 +7120,7 @@ namespace ts {
71217120
result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
71227121
finishNode(result, end);
71237122

7124-
next();
7123+
nextJSDocToken();
71257124
return result;
71267125
}
71277126
}

‎src/compiler/scanner.ts‎

Lines changed: 86 additions & 80 deletions
Original file line numberDiff line numberDiff line change
@@ -60,81 +60,87 @@ namespace ts {
6060
tryScan<T>(callback: () => T): T;
6161
}
6262

63-
const textToToken = createMapFromTemplate({
64-
"abstract": SyntaxKind.AbstractKeyword,
65-
"any": SyntaxKind.AnyKeyword,
66-
"as": SyntaxKind.AsKeyword,
67-
"boolean": SyntaxKind.BooleanKeyword,
68-
"break": SyntaxKind.BreakKeyword,
69-
"case": SyntaxKind.CaseKeyword,
70-
"catch": SyntaxKind.CatchKeyword,
71-
"class": SyntaxKind.ClassKeyword,
72-
"continue": SyntaxKind.ContinueKeyword,
73-
"const": SyntaxKind.ConstKeyword,
74-
"constructor": SyntaxKind.ConstructorKeyword,
75-
"debugger": SyntaxKind.DebuggerKeyword,
76-
"declare": SyntaxKind.DeclareKeyword,
77-
"default": SyntaxKind.DefaultKeyword,
78-
"delete": SyntaxKind.DeleteKeyword,
79-
"do": SyntaxKind.DoKeyword,
80-
"else": SyntaxKind.ElseKeyword,
81-
"enum": SyntaxKind.EnumKeyword,
82-
"export": SyntaxKind.ExportKeyword,
83-
"extends": SyntaxKind.ExtendsKeyword,
84-
"false": SyntaxKind.FalseKeyword,
85-
"finally": SyntaxKind.FinallyKeyword,
86-
"for": SyntaxKind.ForKeyword,
87-
"from": SyntaxKind.FromKeyword,
88-
"function": SyntaxKind.FunctionKeyword,
89-
"get": SyntaxKind.GetKeyword,
90-
"if": SyntaxKind.IfKeyword,
91-
"implements": SyntaxKind.ImplementsKeyword,
92-
"import": SyntaxKind.ImportKeyword,
93-
"in": SyntaxKind.InKeyword,
94-
"infer": SyntaxKind.InferKeyword,
95-
"instanceof": SyntaxKind.InstanceOfKeyword,
96-
"interface": SyntaxKind.InterfaceKeyword,
97-
"is": SyntaxKind.IsKeyword,
98-
"keyof": SyntaxKind.KeyOfKeyword,
99-
"let": SyntaxKind.LetKeyword,
100-
"module": SyntaxKind.ModuleKeyword,
101-
"namespace": SyntaxKind.NamespaceKeyword,
102-
"never": SyntaxKind.NeverKeyword,
103-
"new": SyntaxKind.NewKeyword,
104-
"null": SyntaxKind.NullKeyword,
105-
"number": SyntaxKind.NumberKeyword,
106-
"object": SyntaxKind.ObjectKeyword,
107-
"package": SyntaxKind.PackageKeyword,
108-
"private": SyntaxKind.PrivateKeyword,
109-
"protected": SyntaxKind.ProtectedKeyword,
110-
"public": SyntaxKind.PublicKeyword,
111-
"readonly": SyntaxKind.ReadonlyKeyword,
112-
"require": SyntaxKind.RequireKeyword,
113-
"global": SyntaxKind.GlobalKeyword,
114-
"return": SyntaxKind.ReturnKeyword,
115-
"set": SyntaxKind.SetKeyword,
116-
"static": SyntaxKind.StaticKeyword,
117-
"string": SyntaxKind.StringKeyword,
118-
"super": SyntaxKind.SuperKeyword,
119-
"switch": SyntaxKind.SwitchKeyword,
120-
"symbol": SyntaxKind.SymbolKeyword,
121-
"this": SyntaxKind.ThisKeyword,
122-
"throw": SyntaxKind.ThrowKeyword,
123-
"true": SyntaxKind.TrueKeyword,
124-
"try": SyntaxKind.TryKeyword,
125-
"type": SyntaxKind.TypeKeyword,
126-
"typeof": SyntaxKind.TypeOfKeyword,
127-
"undefined": SyntaxKind.UndefinedKeyword,
128-
"unique": SyntaxKind.UniqueKeyword,
129-
"unknown": SyntaxKind.UnknownKeyword,
130-
"var": SyntaxKind.VarKeyword,
131-
"void": SyntaxKind.VoidKeyword,
132-
"while": SyntaxKind.WhileKeyword,
133-
"with": SyntaxKind.WithKeyword,
134-
"yield": SyntaxKind.YieldKeyword,
135-
"async": SyntaxKind.AsyncKeyword,
136-
"await": SyntaxKind.AwaitKeyword,
137-
"of": SyntaxKind.OfKeyword,
63+
const textToKeywordObj: MapLike<KeywordSyntaxKind> = {
64+
abstract: SyntaxKind.AbstractKeyword,
65+
any: SyntaxKind.AnyKeyword,
66+
as: SyntaxKind.AsKeyword,
67+
boolean: SyntaxKind.BooleanKeyword,
68+
break: SyntaxKind.BreakKeyword,
69+
case: SyntaxKind.CaseKeyword,
70+
catch: SyntaxKind.CatchKeyword,
71+
class: SyntaxKind.ClassKeyword,
72+
continue: SyntaxKind.ContinueKeyword,
73+
const: SyntaxKind.ConstKeyword,
74+
["" + "constructor"]: SyntaxKind.ConstructorKeyword,
75+
debugger: SyntaxKind.DebuggerKeyword,
76+
declare: SyntaxKind.DeclareKeyword,
77+
default: SyntaxKind.DefaultKeyword,
78+
delete: SyntaxKind.DeleteKeyword,
79+
do: SyntaxKind.DoKeyword,
80+
else: SyntaxKind.ElseKeyword,
81+
enum: SyntaxKind.EnumKeyword,
82+
export: SyntaxKind.ExportKeyword,
83+
extends: SyntaxKind.ExtendsKeyword,
84+
false: SyntaxKind.FalseKeyword,
85+
finally: SyntaxKind.FinallyKeyword,
86+
for: SyntaxKind.ForKeyword,
87+
from: SyntaxKind.FromKeyword,
88+
function: SyntaxKind.FunctionKeyword,
89+
get: SyntaxKind.GetKeyword,
90+
if: SyntaxKind.IfKeyword,
91+
implements: SyntaxKind.ImplementsKeyword,
92+
import: SyntaxKind.ImportKeyword,
93+
in: SyntaxKind.InKeyword,
94+
infer: SyntaxKind.InferKeyword,
95+
instanceof: SyntaxKind.InstanceOfKeyword,
96+
interface: SyntaxKind.InterfaceKeyword,
97+
is: SyntaxKind.IsKeyword,
98+
keyof: SyntaxKind.KeyOfKeyword,
99+
let: SyntaxKind.LetKeyword,
100+
module: SyntaxKind.ModuleKeyword,
101+
namespace: SyntaxKind.NamespaceKeyword,
102+
never: SyntaxKind.NeverKeyword,
103+
new: SyntaxKind.NewKeyword,
104+
null: SyntaxKind.NullKeyword,
105+
number: SyntaxKind.NumberKeyword,
106+
object: SyntaxKind.ObjectKeyword,
107+
package: SyntaxKind.PackageKeyword,
108+
private: SyntaxKind.PrivateKeyword,
109+
protected: SyntaxKind.ProtectedKeyword,
110+
public: SyntaxKind.PublicKeyword,
111+
readonly: SyntaxKind.ReadonlyKeyword,
112+
require: SyntaxKind.RequireKeyword,
113+
global: SyntaxKind.GlobalKeyword,
114+
return: SyntaxKind.ReturnKeyword,
115+
set: SyntaxKind.SetKeyword,
116+
static: SyntaxKind.StaticKeyword,
117+
string: SyntaxKind.StringKeyword,
118+
super: SyntaxKind.SuperKeyword,
119+
switch: SyntaxKind.SwitchKeyword,
120+
symbol: SyntaxKind.SymbolKeyword,
121+
this: SyntaxKind.ThisKeyword,
122+
throw: SyntaxKind.ThrowKeyword,
123+
true: SyntaxKind.TrueKeyword,
124+
try: SyntaxKind.TryKeyword,
125+
type: SyntaxKind.TypeKeyword,
126+
typeof: SyntaxKind.TypeOfKeyword,
127+
undefined: SyntaxKind.UndefinedKeyword,
128+
unique: SyntaxKind.UniqueKeyword,
129+
unknown: SyntaxKind.UnknownKeyword,
130+
var: SyntaxKind.VarKeyword,
131+
void: SyntaxKind.VoidKeyword,
132+
while: SyntaxKind.WhileKeyword,
133+
with: SyntaxKind.WithKeyword,
134+
yield: SyntaxKind.YieldKeyword,
135+
async: SyntaxKind.AsyncKeyword,
136+
await: SyntaxKind.AwaitKeyword,
137+
of: SyntaxKind.OfKeyword,
138+
};
139+
140+
const textToKeyword = createMapFromTemplate(textToKeywordObj);
141+
142+
const textToToken = createMapFromTemplate<SyntaxKind>({
143+
...textToKeywordObj,
138144
"{": SyntaxKind.OpenBraceToken,
139145
"}": SyntaxKind.CloseBraceToken,
140146
"(": SyntaxKind.OpenParenToken,
@@ -1288,15 +1294,15 @@ namespace ts {
12881294
return result;
12891295
}
12901296

1291-
function getIdentifierToken(): SyntaxKind {
1297+
function getIdentifierToken(): SyntaxKind.Identifier | KeywordSyntaxKind {
12921298
// Reserved words are between 2 and 11 characters long and start with a lowercase letter
12931299
const len = tokenValue.length;
12941300
if (len >= 2 && len <= 11) {
12951301
const ch = tokenValue.charCodeAt(0);
12961302
if (ch >= CharacterCodes.a && ch <= CharacterCodes.z) {
1297-
token = textToToken.get(tokenValue)!;
1298-
if (token !== undefined) {
1299-
return token;
1303+
const keyword = textToKeyword.get(tokenValue);
1304+
if (keyword !== undefined) {
1305+
return token = keyword;
13001306
}
13011307
}
13021308
}
@@ -2016,7 +2022,7 @@ namespace ts {
20162022
pos++;
20172023
}
20182024
tokenValue = text.substring(tokenPos, pos);
2019-
return token = SyntaxKind.Identifier;
2025+
return token = getIdentifierToken();
20202026
}
20212027
else {
20222028
return token = SyntaxKind.Unknown;

‎src/compiler/types.ts‎

Lines changed: 78 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,84 @@ namespace ts {
2424
| SyntaxKind.DotToken
2525
| SyntaxKind.Identifier
2626
| SyntaxKind.NoSubstitutionTemplateLiteral
27-
| SyntaxKind.Unknown;
27+
| SyntaxKind.Unknown
28+
| KeywordSyntaxKind;
29+
30+
export type KeywordSyntaxKind =
31+
| SyntaxKind.AbstractKeyword
32+
| SyntaxKind.AnyKeyword
33+
| SyntaxKind.AsKeyword
34+
| SyntaxKind.BooleanKeyword
35+
| SyntaxKind.BreakKeyword
36+
| SyntaxKind.CaseKeyword
37+
| SyntaxKind.CatchKeyword
38+
| SyntaxKind.ClassKeyword
39+
| SyntaxKind.ContinueKeyword
40+
| SyntaxKind.ConstKeyword
41+
| SyntaxKind.ConstructorKeyword
42+
| SyntaxKind.DebuggerKeyword
43+
| SyntaxKind.DeclareKeyword
44+
| SyntaxKind.DefaultKeyword
45+
| SyntaxKind.DeleteKeyword
46+
| SyntaxKind.DoKeyword
47+
| SyntaxKind.ElseKeyword
48+
| SyntaxKind.EnumKeyword
49+
| SyntaxKind.ExportKeyword
50+
| SyntaxKind.ExtendsKeyword
51+
| SyntaxKind.FalseKeyword
52+
| SyntaxKind.FinallyKeyword
53+
| SyntaxKind.ForKeyword
54+
| SyntaxKind.FromKeyword
55+
| SyntaxKind.FunctionKeyword
56+
| SyntaxKind.GetKeyword
57+
| SyntaxKind.IfKeyword
58+
| SyntaxKind.ImplementsKeyword
59+
| SyntaxKind.ImportKeyword
60+
| SyntaxKind.InKeyword
61+
| SyntaxKind.InferKeyword
62+
| SyntaxKind.InstanceOfKeyword
63+
| SyntaxKind.InterfaceKeyword
64+
| SyntaxKind.IsKeyword
65+
| SyntaxKind.KeyOfKeyword
66+
| SyntaxKind.LetKeyword
67+
| SyntaxKind.ModuleKeyword
68+
| SyntaxKind.NamespaceKeyword
69+
| SyntaxKind.NeverKeyword
70+
| SyntaxKind.NewKeyword
71+
| SyntaxKind.NullKeyword
72+
| SyntaxKind.NumberKeyword
73+
| SyntaxKind.ObjectKeyword
74+
| SyntaxKind.PackageKeyword
75+
| SyntaxKind.PrivateKeyword
76+
| SyntaxKind.ProtectedKeyword
77+
| SyntaxKind.PublicKeyword
78+
| SyntaxKind.ReadonlyKeyword
79+
| SyntaxKind.RequireKeyword
80+
| SyntaxKind.GlobalKeyword
81+
| SyntaxKind.ReturnKeyword
82+
| SyntaxKind.SetKeyword
83+
| SyntaxKind.StaticKeyword
84+
| SyntaxKind.StringKeyword
85+
| SyntaxKind.SuperKeyword
86+
| SyntaxKind.SwitchKeyword
87+
| SyntaxKind.SymbolKeyword
88+
| SyntaxKind.ThisKeyword
89+
| SyntaxKind.ThrowKeyword
90+
| SyntaxKind.TrueKeyword
91+
| SyntaxKind.TryKeyword
92+
| SyntaxKind.TypeKeyword
93+
| SyntaxKind.TypeOfKeyword
94+
| SyntaxKind.UndefinedKeyword
95+
| SyntaxKind.UniqueKeyword
96+
| SyntaxKind.UnknownKeyword
97+
| SyntaxKind.VarKeyword
98+
| SyntaxKind.VoidKeyword
99+
| SyntaxKind.WhileKeyword
100+
| SyntaxKind.WithKeyword
101+
| SyntaxKind.YieldKeyword
102+
| SyntaxKind.AsyncKeyword
103+
| SyntaxKind.AwaitKeyword
104+
| SyntaxKind.OfKeyword;
28105

29106
export type JsxTokenSyntaxKind =
30107
| SyntaxKind.LessThanSlashToken

‎tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.@link tags.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,7 +6,7 @@
66
"0": {
77
"kind": "JSDocTag",
88
"pos": 63,
9-
"end": 67,
9+
"end": 68,
1010
"atToken": {
1111
"kind": "AtToken",
1212
"pos": 63,
@@ -22,7 +22,7 @@
2222
},
2323
"length": 1,
2424
"pos": 63,
25-
"end": 67
25+
"end": 68
2626
},
2727
"comment": "{@link first link}\nInside {@link link text} thing"
2828
}

‎tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
"typeParameters": {
2222
"0": {
2323
"kind": "TypeParameter",
24-
"pos": 17,
24+
"pos": 18,
2525
"end": 19,
2626
"name": {
2727
"kind": "Identifier",
@@ -31,7 +31,7 @@
3131
}
3232
},
3333
"length": 1,
34-
"pos": 17,
34+
"pos": 18,
3535
"end": 19
3636
}
3737
},

0 commit comments

Comments
 (0)