Set startPos at EOF in jsdoc token scanner so node end positions for nodes terminated at EoF are right (#24184)

weswigham · web-flow · commit d82d35c7f5eb · 2018-05-17T15:16:18.000-07:00
* Set startPos at EOF in jsdoc token scanner to node end positions for nodes terminated at EoF are right

* More complete nonwhitespace token check, fix syntactica jsdoc classifier

* Use loop and no nested lookahead

* Do thigns unrelated to the bug in the test

* Fix typo move return

* Patch up typedef end pos

* Fix indentation, make end pos target more obvious
diff --git a/src/compiler/parser.ts b/src/compiler/parser.ts
@@ -6481,7 +6481,25 @@ namespace ts {
                     return finishNode(result, end);
                 }
 
+                function isNextNonwhitespaceTokenEndOfFile(): boolean {
+                    // We must use infinite lookahead, as there could be any number of newlines :(
+                    while (true) {
+                        nextJSDocToken();
+                        if (token() === SyntaxKind.EndOfFileToken) {
+                            return true;
+                        }
+                        if (!(token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia)) {
+                            return false;
+                        }
+                    }
+                }
+
                 function skipWhitespace(): void {
+                    if (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
+                        if (lookAhead(isNextNonwhitespaceTokenEndOfFile)) {
+                            return; // Don't skip whitespace prior to EoF (or end of comment) - that shouldn't be included in any node's range
+                        }
+                    }
                     while (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
                         nextJSDocToken();
                     }
@@ -6802,6 +6820,7 @@ namespace ts {
                     typedefTag.comment = parseTagComments(indent);
 
                     typedefTag.typeExpression = typeExpression;
+                    let end: number;
                     if (!typeExpression || isObjectOrObjectArrayTypeReference(typeExpression.type)) {
                         let child: JSDocTypeTag | JSDocPropertyTag | false;
                         let jsdocTypeLiteral: JSDocTypeLiteral;
@@ -6830,10 +6849,12 @@ namespace ts {
                             typedefTag.typeExpression = childTypeTag && childTypeTag.typeExpression && !isObjectOrObjectArrayTypeReference(childTypeTag.typeExpression.type) ?
                                 childTypeTag.typeExpression :
                                 finishNode(jsdocTypeLiteral);
+                            end = typedefTag.typeExpression.end;
                         }
                     }
 
-                    return finishNode(typedefTag);
+                    // Only include the characters between the name end and the next token if a comment was actually parsed out - otherwise it's just whitespace
+                    return finishNode(typedefTag, end || typedefTag.comment !== undefined ? scanner.getStartPos() : (typedefTag.fullName || typedefTag.typeExpression || typedefTag.tagName).end);
                 }
 
                 function parseJSDocTypeNameWithNamespace(nested?: boolean) {
@@ -7075,7 +7096,7 @@ namespace ts {
                     const pos = scanner.getTokenPos();
                     const end = scanner.getTextPos();
                     const result = <Identifier>createNode(SyntaxKind.Identifier, pos);
-                    result.escapedText = escapeLeadingUnderscores(content.substring(pos, end));
+                    result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
                     finishNode(result, end);
 
                     nextJSDocToken();
diff --git a/src/compiler/scanner.ts b/src/compiler/scanner.ts
@@ -1928,13 +1928,11 @@ namespace ts {
         }
 
         function scanJSDocToken(): JsDocSyntaxKind {
+            startPos = tokenPos = pos;
             if (pos >= end) {
                 return token = SyntaxKind.EndOfFileToken;
             }
 
-            startPos = pos;
-            tokenPos = pos;
-
             const ch = text.charCodeAt(pos);
             pos++;
             switch (ch) {
diff --git a/src/services/classifier.ts b/src/services/classifier.ts
@@ -706,16 +706,17 @@ namespace ts {
                             break;
                         case SyntaxKind.JSDocTemplateTag:
                             processJSDocTemplateTag(<JSDocTemplateTag>tag);
+                            pos = tag.end;
                             break;
                         case SyntaxKind.JSDocTypeTag:
                             processElement((<JSDocTypeTag>tag).typeExpression);
+                            pos = tag.end;
                             break;
                         case SyntaxKind.JSDocReturnTag:
                             processElement((<JSDocReturnTag>tag).typeExpression);
+                            pos = tag.end;
                             break;
                     }
-
-                    pos = tag.end;
                 }
             }
 
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.Nested @param tags.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.Nested @param tags.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 6,
-            "end": 63,
+            "end": 64,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 6,
@@ -21,11 +21,11 @@
             "typeExpression": {
                 "kind": "JSDocTypeExpression",
                 "pos": 34,
-                "end": 63,
+                "end": 64,
                 "type": {
                     "kind": "JSDocTypeLiteral",
                     "pos": 34,
-                    "end": 63,
+                    "end": 64,
                     "jsDocPropertyTags": [
                         {
                             "kind": "JSDocParameterTag",
@@ -88,6 +88,6 @@
         },
         "length": 1,
         "pos": 6,
-        "end": 63
+        "end": 64
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argSynonymForParamTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argSynonymForParamTag.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 40,
+            "end": 42,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 40
+        "end": 42
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argumentSynonymForParamTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argumentSynonymForParamTag.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 45,
+            "end": 47,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 45
+        "end": 47
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.less-than and greater-than characters.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.less-than and greater-than characters.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 7,
-            "end": 58,
+            "end": 59,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 7,
@@ -30,6 +30,6 @@
         },
         "length": 1,
         "pos": 7,
-        "end": 58
+        "end": 59
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocReturnTag",
             "pos": 8,
-            "end": 16,
+            "end": 15,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -21,6 +21,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 16
+        "end": 15
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.oneParamTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.oneParamTag.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 30,
+            "end": 32,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -39,6 +39,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 30
+        "end": 32
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTag1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTag1.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 55,
+            "end": 57,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 55
+        "end": 57
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName1.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName1.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 57,
+            "end": 59,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 57
+        "end": 59
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagBracketedName2.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 62,
+            "end": 64,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 62
+        "end": 64
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTagNameThenType2.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 42,
+            "end": 44,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 42
+        "end": 44
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramWithoutType.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramWithoutType.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 19,
+            "end": 21,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -29,6 +29,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 19
+        "end": 21
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocTemplateTag",
             "pos": 8,
-            "end": 20,
+            "end": 19,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -22,7 +22,7 @@
                 "0": {
                     "kind": "TypeParameter",
                     "pos": 18,
-                    "end": 20,
+                    "end": 19,
                     "name": {
                         "kind": "Identifier",
                         "pos": 18,
@@ -32,11 +32,11 @@
                 },
                 "length": 1,
                 "pos": 18,
-                "end": 20
+                "end": 19
             }
         },
         "length": 1,
         "pos": 8,
-        "end": 20
+        "end": 19
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag2.json
@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocTemplateTag",
             "pos": 8,
-            "end": 22,
+            "end": 21,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -33,7 +33,7 @@
                 "1": {
                     "kind": "TypeParameter",
                     "pos": 20,
-                    "end": 22,
+                    "end": 21,
                     "name": {
                         "kind": "Identifier",
                         "pos": 20,
@@ -43,11 +43,11 @@
                 },
                 "length": 2,
                 "pos": 18,
-                "end": 22
+                "end": 21
             }
         },
         "length": 1,
         "pos": 8,
-        "end": 22
+        "end": 21
     }
 }
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag3.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag3.json
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag4.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag4.json
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag5.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.templateTag5.json
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTag2.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTag2.json
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTagOnSameLine.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.twoParamTagOnSameLine.json
diff --git a/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typedefTagWithChildrenTags.json b/tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.typedefTagWithChildrenTags.json
diff --git a/tests/cases/fourslash/incrementalJsDocAdjustsLengthsRight.ts b/tests/cases/fourslash/incrementalJsDocAdjustsLengthsRight.ts

Original file line number	Diff line number	Diff line change
`@@ -1928,13 +1928,11 @@ namespace ts {`
`1928`	`1928`	`}`
`1929`	`1929`
`1930`	`1930`	`function scanJSDocToken(): JsDocSyntaxKind {`
	`1931`	`+ startPos = tokenPos = pos;`
`1931`	`1932`	`if (pos >= end) {`
`1932`	`1933`	`return token = SyntaxKind.EndOfFileToken;`
`1933`	`1934`	`}`
`1934`	`1935`
`1935`		`- startPos = pos;`
`1936`		`- tokenPos = pos;`
`1937`		`-`
`1938`	`1936`	`const ch = text.charCodeAt(pos);`
`1939`	`1937`	`pos++;`
`1940`	`1938`	`switch (ch) {`
Original file line number	Diff line number	Diff line change
`@@ -706,16 +706,17 @@ namespace ts {`
`706`	`706`	`break;`
`707`	`707`	`case SyntaxKind.JSDocTemplateTag:`
`708`	`708`	`processJSDocTemplateTag(<JSDocTemplateTag>tag);`
	`709`	`+ pos = tag.end;`
`709`	`710`	`break;`
`710`	`711`	`case SyntaxKind.JSDocTypeTag:`
`711`	`712`	`processElement((<JSDocTypeTag>tag).typeExpression);`
	`713`	`+ pos = tag.end;`
`712`	`714`	`break;`
`713`	`715`	`case SyntaxKind.JSDocReturnTag:`
`714`	`716`	`processElement((<JSDocReturnTag>tag).typeExpression);`
	`717`	`+ pos = tag.end;`
`715`	`718`	`break;`
`716`	`719`	`}`
`717`		`-`
`718`		`- pos = tag.end;`
`719`	`720`	`}`
`720`	`721`	`}`
`721`	`722`