Set startPos at EOF in jsdoc token scanner so node end positions for nodes terminated at EoF are right (#24184)

* Set startPos at EOF in jsdoc token scanner to node end positions for nodes terminated at EoF are right

* More complete nonwhitespace token check, fix syntactica jsdoc classifier

* Use loop and no nested lookahead

* Do thigns unrelated to the bug in the test

* Fix typo move return

* Patch up typedef end pos

* Fix indentation, make end pos target more obvious

Author: weswigham

src/compiler/parser.ts

@@ -6481,7 +6481,25 @@ namespace ts {
                     return finishNode(result, end);
                 }
 
+                function isNextNonwhitespaceTokenEndOfFile(): boolean {
+                    // We must use infinite lookahead, as there could be any number of newlines :(
+                    while (true) {
+                        nextJSDocToken();
+                        if (token() === SyntaxKind.EndOfFileToken) {
+                            return true;
+                        }
+                        if (!(token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia)) {
+                            return false;
+                        }
+                    }
+                }
+
                 function skipWhitespace(): void {
+                    if (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
+                        if (lookAhead(isNextNonwhitespaceTokenEndOfFile)) {
+                            return; // Don't skip whitespace prior to EoF (or end of comment) - that shouldn't be included in any node's range
+                        }
+                    }
                     while (token() === SyntaxKind.WhitespaceTrivia || token() === SyntaxKind.NewLineTrivia) {
                         nextJSDocToken();
                     }
@@ -6802,6 +6820,7 @@ namespace ts {
                     typedefTag.comment = parseTagComments(indent);
 
                     typedefTag.typeExpression = typeExpression;
+                    let end: number;
                     if (!typeExpression || isObjectOrObjectArrayTypeReference(typeExpression.type)) {
                         let child: JSDocTypeTag | JSDocPropertyTag | false;
                         let jsdocTypeLiteral: JSDocTypeLiteral;
@@ -6830,10 +6849,12 @@ namespace ts {
                             typedefTag.typeExpression = childTypeTag && childTypeTag.typeExpression && !isObjectOrObjectArrayTypeReference(childTypeTag.typeExpression.type) ?
                                 childTypeTag.typeExpression :
                                 finishNode(jsdocTypeLiteral);
+                            end = typedefTag.typeExpression.end;
                         }
                     }
 
-                    return finishNode(typedefTag);
+                    // Only include the characters between the name end and the next token if a comment was actually parsed out - otherwise it's just whitespace
+                    return finishNode(typedefTag, end || typedefTag.comment !== undefined ? scanner.getStartPos() : (typedefTag.fullName || typedefTag.typeExpression || typedefTag.tagName).end);
                 }
 
                 function parseJSDocTypeNameWithNamespace(nested?: boolean) {
@@ -7075,7 +7096,7 @@ namespace ts {
                     const pos = scanner.getTokenPos();
                     const end = scanner.getTextPos();
                     const result = <Identifier>createNode(SyntaxKind.Identifier, pos);
-                    result.escapedText = escapeLeadingUnderscores(content.substring(pos, end));
+                    result.escapedText = escapeLeadingUnderscores(scanner.getTokenText());
                     finishNode(result, end);
 
                     nextJSDocToken();

src/compiler/scanner.ts

@@ -1928,13 +1928,11 @@ namespace ts {
         }
 
         function scanJSDocToken(): JsDocSyntaxKind {
+            startPos = tokenPos = pos;
             if (pos >= end) {
                 return token = SyntaxKind.EndOfFileToken;
             }
 
-            startPos = pos;
-            tokenPos = pos;
-
             const ch = text.charCodeAt(pos);
             pos++;
             switch (ch) {

src/services/classifier.ts

@@ -706,16 +706,17 @@ namespace ts {
                             break;
                         case SyntaxKind.JSDocTemplateTag:
                             processJSDocTemplateTag(<JSDocTemplateTag>tag);
+                            pos = tag.end;
                             break;
                         case SyntaxKind.JSDocTypeTag:
                             processElement((<JSDocTypeTag>tag).typeExpression);
+                            pos = tag.end;
                             break;
                         case SyntaxKind.JSDocReturnTag:
                             processElement((<JSDocReturnTag>tag).typeExpression);
+                            pos = tag.end;
                             break;
                     }
-
-                    pos = tag.end;
                 }
             }
 

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.Nested @param tags.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 6,
-            "end": 63,
+            "end": 64,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 6,
@@ -21,11 +21,11 @@
             "typeExpression": {
                 "kind": "JSDocTypeExpression",
                 "pos": 34,
-                "end": 63,
+                "end": 64,
                 "type": {
                     "kind": "JSDocTypeLiteral",
                     "pos": 34,
-                    "end": 63,
+                    "end": 64,
                     "jsDocPropertyTags": [
                         {
                             "kind": "JSDocParameterTag",
@@ -88,6 +88,6 @@
         },
         "length": 1,
         "pos": 6,
-        "end": 63
+        "end": 64
     }
 }

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argSynonymForParamTag.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 40,
+            "end": 42,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 40
+        "end": 42
     }
 }

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.argumentSynonymForParamTag.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 45,
+            "end": 47,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 45
+        "end": 47
     }
 }

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.less-than and greater-than characters.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 7,
-            "end": 58,
+            "end": 59,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 7,
@@ -30,6 +30,6 @@
         },
         "length": 1,
         "pos": 7,
-        "end": 58
+        "end": 59
     }
 }

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.noReturnType.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocReturnTag",
             "pos": 8,
-            "end": 16,
+            "end": 15,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -21,6 +21,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 16
+        "end": 15
     }
 }

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.oneParamTag.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 30,
+            "end": 32,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -39,6 +39,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 30
+        "end": 32
     }
 }

tests/baselines/reference/JSDocParsing/DocComments.parsesCorrectly.paramTag1.json

@@ -6,7 +6,7 @@
         "0": {
             "kind": "JSDocParameterTag",
             "pos": 8,
-            "end": 55,
+            "end": 57,
             "atToken": {
                 "kind": "AtToken",
                 "pos": 8,
@@ -40,6 +40,6 @@
         },
         "length": 1,
         "pos": 8,
-        "end": 55
+        "end": 57
     }
 }