TypeParser - support comments at EOL with //

Author: shmax

src/Ast/Attribute.php

@@ -13,4 +13,6 @@ final class Attribute
 
 	public const ORIGINAL_NODE = 'originalNode';
 
+	public const COMMENTS = 'comments';
+
 }

src/Ast/Comment.php

@@ -0,0 +1,28 @@
+<?php declare(strict_types = 1);
+
+namespace PHPStan\PhpDocParser\Ast;
+
+use function trim;
+
+class Comment
+{
+
+	public string $text;
+
+	public int $startLine;
+
+	public int $startIndex;
+
+	public function __construct(string $text, int $startLine = -1, int $startIndex = -1)
+	{
+		$this->text = $text;
+		$this->startLine = $startLine;
+		$this->startIndex = $startIndex;
+	}
+
+	public function getReformattedText(): ?string
+	{
+		return trim($this->text);
+	}
+
+}

src/Lexer/Lexer.php

@@ -51,6 +51,8 @@ class Lexer
 	public const TOKEN_NEGATED = 35;
 	public const TOKEN_ARROW = 36;
 
+	public const TOKEN_COMMENT = 37;
+
 	public const TOKEN_LABELS = [
 		self::TOKEN_REFERENCE => '\'&\'',
 		self::TOKEN_UNION => '\'|\'',
@@ -66,6 +68,7 @@ class Lexer
 		self::TOKEN_OPEN_CURLY_BRACKET => '\'{\'',
 		self::TOKEN_CLOSE_CURLY_BRACKET => '\'}\'',
 		self::TOKEN_COMMA => '\',\'',
+		self::TOKEN_COMMENT => '\'//\'',
 		self::TOKEN_COLON => '\':\'',
 		self::TOKEN_VARIADIC => '\'...\'',
 		self::TOKEN_DOUBLE_COLON => '\'::\'',
@@ -160,6 +163,7 @@ private function generateRegexp(): string
 			self::TOKEN_CLOSE_CURLY_BRACKET => '\\}',
 
 			self::TOKEN_COMMA => ',',
+			self::TOKEN_COMMENT => '((?<![:/])\/\/[^\n]*)',
 			self::TOKEN_VARIADIC => '\\.\\.\\.',
 			self::TOKEN_DOUBLE_COLON => '::',
 			self::TOKEN_DOUBLE_ARROW => '=>',

src/Parser/TokenIterator.php

@@ -3,6 +3,7 @@
 namespace PHPStan\PhpDocParser\Parser;
 
 use LogicException;
+use PHPStan\PhpDocParser\Ast\Comment;
 use PHPStan\PhpDocParser\Lexer\Lexer;
 use function array_pop;
 use function assert;
@@ -19,11 +20,16 @@ class TokenIterator
 
 	private int $index;
 
+	/** @var array<Comment> */
+	private array $comments = [];
+
 	/** @var int[] */
 	private array $savePoints = [];
 
 	/** @var list<int> */
-	private array $skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS];
+	private array $skippedTokenTypes = [
+		Lexer::TOKEN_HORIZONTAL_WS,
+		Lexer::TOKEN_COMMENT];
 
 	private ?string $newline = null;
 
@@ -152,8 +158,7 @@ public function consumeTokenType(int $tokenType): void
 			}
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 	}
 
 
@@ -166,8 +171,7 @@ public function consumeTokenValue(int $tokenType, string $tokenValue): void
 			$this->throwError($tokenType, $tokenValue);
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 	}
 
 
@@ -178,12 +182,20 @@ public function tryConsumeTokenValue(string $tokenValue): bool
 			return false;
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 
 		return true;
 	}
 
+	/**
+	 * @return Comment[]
+	 */
+	public function flushComments(): array
+	{
+		$res = $this->comments;
+		$this->comments = [];
+		return $res;
+	}
 
 	/** @phpstan-impure */
 	public function tryConsumeTokenType(int $tokenType): bool
@@ -198,8 +210,7 @@ public function tryConsumeTokenType(int $tokenType): bool
 			}
 		}
 
-		$this->index++;
-		$this->skipIrrelevantTokens();
+		$this->next();
 
 		return true;
 	}
@@ -267,6 +278,11 @@ private function skipIrrelevantTokens(): void
 			if (!isset($this->tokens[$this->index + 1])) {
 				break;
 			}
+
+			if ($this->currentTokenType() === Lexer::TOKEN_COMMENT) {
+				$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
+			}
+
 			$this->index++;
 		}
 	}
@@ -310,7 +326,6 @@ public function rollback(): void
 		$this->index = $index;
 	}
 
-
 	/**
 	 * @throws ParserException
 	 */

src/Parser/TypeParser.php

@@ -82,6 +82,7 @@ private function enrichTypeOnUnionOrIntersection(TokenIterator $tokens, Ast\Type
 	 * @internal
 	 * @template T of Ast\Node
 	 * @param T $type
+	 *
 	 * @return T
 	 */
 	public function enrichWithAttributes(TokenIterator $tokens, Ast\Node $type, int $startLine, int $startIndex): Ast\Node
@@ -91,6 +92,11 @@ public function enrichWithAttributes(TokenIterator $tokens, Ast\Node $type, int
 			$type->setAttribute(Ast\Attribute::END_LINE, $tokens->currentTokenLine());
 		}
 
+		$comments = $tokens->flushComments();
+		if ($this->config->useCommentsAttributes) {
+			$type->setAttribute(Ast\Attribute::COMMENTS, $comments);
+		}
+
 		if ($this->config->useIndexAttributes) {
 			$type->setAttribute(Ast\Attribute::START_INDEX, $startIndex);
 			$type->setAttribute(Ast\Attribute::END_INDEX, $tokens->endIndexOfLastRelevantToken());
@@ -429,6 +435,7 @@ public function isHtml(TokenIterator $tokens): bool
 	public function parseGeneric(TokenIterator $tokens, Ast\Type\IdentifierTypeNode $baseType): Ast\Type\GenericTypeNode
 	{
 		$tokens->consumeTokenType(Lexer::TOKEN_OPEN_ANGLE_BRACKET);
+		$tokens->skipNewLineTokens();
 
 		$startLine = $baseType->getAttribute(Ast\Attribute::START_LINE);
 		$startIndex = $baseType->getAttribute(Ast\Attribute::START_INDEX);
@@ -859,6 +866,8 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
 		$sealed = true;
 		$unsealedType = null;
 
+		$done = false;
+
 		do {
 			$tokens->skipNewLineTokens();
 
@@ -884,9 +893,17 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
 			}
 
 			$items[] = $this->parseArrayShapeItem($tokens);
-
 			$tokens->skipNewLineTokens();
-		} while ($tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA));
+			if (!$tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)) {
+				$done = true;
+			}
+			if ($tokens->currentTokenType() !== Lexer::TOKEN_COMMENT) {
+				continue;
+			}
+
+			$tokens->next();
+
+		} while (!$done);
 
 		$tokens->skipNewLineTokens();
 		$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_CURLY_BRACKET);
@@ -904,12 +921,17 @@ private function parseArrayShapeItem(TokenIterator $tokens): Ast\Type\ArrayShape
 	{
 		$startLine = $tokens->currentTokenLine();
 		$startIndex = $tokens->currentTokenIndex();
+
+		// parse any comments above the item
+		$tokens->skipNewLineTokens();
+
 		try {
 			$tokens->pushSavePoint();
 			$key = $this->parseArrayShapeKey($tokens);
 			$optional = $tokens->tryConsumeTokenType(Lexer::TOKEN_NULLABLE);
 			$tokens->consumeTokenType(Lexer::TOKEN_COLON);
 			$value = $this->parse($tokens);
+
 			$tokens->dropSavePoint();
 
 			return $this->enrichWithAttributes(
@@ -1056,12 +1078,19 @@ private function parseObjectShapeItem(TokenIterator $tokens): Ast\Type\ObjectSha
 		$startLine = $tokens->currentTokenLine();
 		$startIndex = $tokens->currentTokenIndex();
 
+		$tokens->skipNewLineTokens();
+
 		$key = $this->parseObjectShapeKey($tokens);
 		$optional = $tokens->tryConsumeTokenType(Lexer::TOKEN_NULLABLE);
 		$tokens->consumeTokenType(Lexer::TOKEN_COLON);
 		$value = $this->parse($tokens);
 
-		return $this->enrichWithAttributes($tokens, new Ast\Type\ObjectShapeItemNode($key, $optional, $value), $startLine, $startIndex);
+		return $this->enrichWithAttributes(
+			$tokens,
+			new Ast\Type\ObjectShapeItemNode($key, $optional, $value),
+			$startLine,
+			$startIndex,
+		);
 	}
 
 	/**

src/ParserConfig.php

@@ -9,13 +9,16 @@ class ParserConfig
 
 	public bool $useIndexAttributes;
 
+	public bool $useCommentsAttributes;
+
 	/**
-	 * @param array{lines?: bool, indexes?: bool} $usedAttributes
+	 * @param array{lines?: bool, indexes?: bool, comments?: bool} $usedAttributes
 	 */
 	public function __construct(array $usedAttributes)
 	{
 		$this->useLinesAttributes = $usedAttributes['lines'] ?? false;
 		$this->useIndexAttributes = $usedAttributes['indexes'] ?? false;
+		$this->useCommentsAttributes = $usedAttributes['comments'] ?? false;
 	}
 
 }