Skip to content

Commit 341932c

Browse files
ondrejmirtesondrejmirtes
committed
Do not make TOKEN_COMMENT skipped token type
1 parent 47840f1 commit 341932c

File tree

2 files changed

+68
-49
lines changed

2 files changed

+68
-49
lines changed

‎src/Parser/TokenIterator.php

+27-8
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,7 @@ class TokenIterator
2727
private array $savePoints = [];
2828

2929
/** @var list<int> */
30-
private array $skippedTokenTypes = [
31-
Lexer::TOKEN_HORIZONTAL_WS,
32-
Lexer::TOKEN_COMMENT];
30+
private array $skippedTokenTypes = [Lexer::TOKEN_HORIZONTAL_WS];
3331

3432
private ?string $newline = null;
3533

@@ -216,7 +214,9 @@ public function tryConsumeTokenType(int $tokenType): bool
216214
}
217215

218216

219-
/** @phpstan-impure */
217+
/**
218+
* @deprecated Use skipNewLineTokensAndConsumeComments instead (when parsing a type)
219+
*/
220220
public function skipNewLineTokens(): void
221221
{
222222
if (!$this->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
@@ -229,6 +229,29 @@ public function skipNewLineTokens(): void
229229
}
230230

231231

232+
public function skipNewLineTokensAndConsumeComments(): void
233+
{
234+
if ($this->currentTokenType() === Lexer::TOKEN_COMMENT) {
235+
$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
236+
$this->next();
237+
}
238+
239+
if (!$this->isCurrentTokenType(Lexer::TOKEN_PHPDOC_EOL)) {
240+
return;
241+
}
242+
243+
do {
244+
$foundNewLine = $this->tryConsumeTokenType(Lexer::TOKEN_PHPDOC_EOL);
245+
if ($this->currentTokenType() !== Lexer::TOKEN_COMMENT) {
246+
continue;
247+
}
248+
249+
$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
250+
$this->next();
251+
} while ($foundNewLine === true);
252+
}
253+
254+
232255
private function detectNewline(): void
233256
{
234257
$value = $this->currentTokenValue();
@@ -279,10 +302,6 @@ private function skipIrrelevantTokens(): void
279302
break;
280303
}
281304

282-
if ($this->currentTokenType() === Lexer::TOKEN_COMMENT) {
283-
$this->comments[] = new Comment($this->currentTokenValue(), $this->currentTokenLine(), $this->currentTokenIndex());
284-
}
285-
286305
$this->index++;
287306
}
288307
}

‎src/Parser/TypeParser.php

+41-41
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ public function parse(TokenIterator $tokens): Ast\Type\TypeNode
4141
$type = $this->parseAtomic($tokens);
4242

4343
$tokens->pushSavePoint();
44-
$tokens->skipNewLineTokens();
44+
$tokens->skipNewLineTokensAndConsumeComments();
4545

4646
try {
4747
$enrichedType = $this->enrichTypeOnUnionOrIntersection($tokens, $type);
@@ -123,7 +123,7 @@ private function subParse(TokenIterator $tokens): Ast\Type\TypeNode
123123
if ($tokens->isCurrentTokenValue('is')) {
124124
$type = $this->parseConditional($tokens, $type);
125125
} else {
126-
$tokens->skipNewLineTokens();
126+
$tokens->skipNewLineTokensAndConsumeComments();
127127

128128
if ($tokens->isCurrentTokenType(Lexer::TOKEN_UNION)) {
129129
$type = $this->subParseUnion($tokens, $type);
@@ -145,9 +145,9 @@ private function parseAtomic(TokenIterator $tokens): Ast\Type\TypeNode
145145
$startIndex = $tokens->currentTokenIndex();
146146

147147
if ($tokens->tryConsumeTokenType(Lexer::TOKEN_OPEN_PARENTHESES)) {
148-
$tokens->skipNewLineTokens();
148+
$tokens->skipNewLineTokensAndConsumeComments();
149149
$type = $this->subParse($tokens);
150-
$tokens->skipNewLineTokens();
150+
$tokens->skipNewLineTokensAndConsumeComments();
151151

152152
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_PARENTHESES);
153153

@@ -289,9 +289,9 @@ private function subParseUnion(TokenIterator $tokens, Ast\Type\TypeNode $type):
289289
$types = [$type];
290290

291291
while ($tokens->tryConsumeTokenType(Lexer::TOKEN_UNION)) {
292-
$tokens->skipNewLineTokens();
292+
$tokens->skipNewLineTokensAndConsumeComments();
293293
$types[] = $this->parseAtomic($tokens);
294-
$tokens->skipNewLineTokens();
294+
$tokens->skipNewLineTokensAndConsumeComments();
295295
}
296296

297297
return new Ast\Type\UnionTypeNode($types);
@@ -317,9 +317,9 @@ private function subParseIntersection(TokenIterator $tokens, Ast\Type\TypeNode $
317317
$types = [$type];
318318

319319
while ($tokens->tryConsumeTokenType(Lexer::TOKEN_INTERSECTION)) {
320-
$tokens->skipNewLineTokens();
320+
$tokens->skipNewLineTokensAndConsumeComments();
321321
$types[] = $this->parseAtomic($tokens);
322-
$tokens->skipNewLineTokens();
322+
$tokens->skipNewLineTokensAndConsumeComments();
323323
}
324324

325325
return new Ast\Type\IntersectionTypeNode($types);
@@ -339,15 +339,15 @@ private function parseConditional(TokenIterator $tokens, Ast\Type\TypeNode $subj
339339

340340
$targetType = $this->parse($tokens);
341341

342-
$tokens->skipNewLineTokens();
342+
$tokens->skipNewLineTokensAndConsumeComments();
343343
$tokens->consumeTokenType(Lexer::TOKEN_NULLABLE);
344-
$tokens->skipNewLineTokens();
344+
$tokens->skipNewLineTokensAndConsumeComments();
345345

346346
$ifType = $this->parse($tokens);
347347

348-
$tokens->skipNewLineTokens();
348+
$tokens->skipNewLineTokensAndConsumeComments();
349349
$tokens->consumeTokenType(Lexer::TOKEN_COLON);
350-
$tokens->skipNewLineTokens();
350+
$tokens->skipNewLineTokensAndConsumeComments();
351351

352352
$elseType = $this->subParse($tokens);
353353

@@ -368,15 +368,15 @@ private function parseConditionalForParameter(TokenIterator $tokens, string $par
368368

369369
$targetType = $this->parse($tokens);
370370

371-
$tokens->skipNewLineTokens();
371+
$tokens->skipNewLineTokensAndConsumeComments();
372372
$tokens->consumeTokenType(Lexer::TOKEN_NULLABLE);
373-
$tokens->skipNewLineTokens();
373+
$tokens->skipNewLineTokensAndConsumeComments();
374374

375375
$ifType = $this->parse($tokens);
376376

377-
$tokens->skipNewLineTokens();
377+
$tokens->skipNewLineTokensAndConsumeComments();
378378
$tokens->consumeTokenType(Lexer::TOKEN_COLON);
379-
$tokens->skipNewLineTokens();
379+
$tokens->skipNewLineTokensAndConsumeComments();
380380

381381
$elseType = $this->subParse($tokens);
382382

@@ -435,7 +435,7 @@ public function isHtml(TokenIterator $tokens): bool
435435
public function parseGeneric(TokenIterator $tokens, Ast\Type\IdentifierTypeNode $baseType): Ast\Type\GenericTypeNode
436436
{
437437
$tokens->consumeTokenType(Lexer::TOKEN_OPEN_ANGLE_BRACKET);
438-
$tokens->skipNewLineTokens();
438+
$tokens->skipNewLineTokensAndConsumeComments();
439439

440440
$startLine = $baseType->getAttribute(Ast\Attribute::START_LINE);
441441
$startIndex = $baseType->getAttribute(Ast\Attribute::START_INDEX);
@@ -447,7 +447,7 @@ public function parseGeneric(TokenIterator $tokens, Ast\Type\IdentifierTypeNode
447447
$isFirst
448448
|| $tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)
449449
) {
450-
$tokens->skipNewLineTokens();
450+
$tokens->skipNewLineTokensAndConsumeComments();
451451

452452
// trailing comma case
453453
if (!$isFirst && $tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_ANGLE_BRACKET)) {
@@ -456,7 +456,7 @@ public function parseGeneric(TokenIterator $tokens, Ast\Type\IdentifierTypeNode
456456
$isFirst = false;
457457

458458
[$genericTypes[], $variances[]] = $this->parseGenericTypeArgument($tokens);
459-
$tokens->skipNewLineTokens();
459+
$tokens->skipNewLineTokensAndConsumeComments();
460460
}
461461

462462
$type = new Ast\Type\GenericTypeNode($baseType, $genericTypes, $variances);
@@ -547,19 +547,19 @@ private function parseCallable(TokenIterator $tokens, Ast\Type\IdentifierTypeNod
547547
: [];
548548

549549
$tokens->consumeTokenType(Lexer::TOKEN_OPEN_PARENTHESES);
550-
$tokens->skipNewLineTokens();
550+
$tokens->skipNewLineTokensAndConsumeComments();
551551

552552
$parameters = [];
553553
if (!$tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PARENTHESES)) {
554554
$parameters[] = $this->parseCallableParameter($tokens);
555-
$tokens->skipNewLineTokens();
555+
$tokens->skipNewLineTokensAndConsumeComments();
556556
while ($tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)) {
557-
$tokens->skipNewLineTokens();
557+
$tokens->skipNewLineTokensAndConsumeComments();
558558
if ($tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_PARENTHESES)) {
559559
break;
560560
}
561561
$parameters[] = $this->parseCallableParameter($tokens);
562-
$tokens->skipNewLineTokens();
562+
$tokens->skipNewLineTokensAndConsumeComments();
563563
}
564564
}
565565

@@ -587,7 +587,7 @@ private function parseCallableTemplates(TokenIterator $tokens): array
587587

588588
$isFirst = true;
589589
while ($isFirst || $tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)) {
590-
$tokens->skipNewLineTokens();
590+
$tokens->skipNewLineTokensAndConsumeComments();
591591

592592
// trailing comma case
593593
if (!$isFirst && $tokens->isCurrentTokenType(Lexer::TOKEN_CLOSE_ANGLE_BRACKET)) {
@@ -596,7 +596,7 @@ private function parseCallableTemplates(TokenIterator $tokens): array
596596
$isFirst = false;
597597

598598
$templates[] = $this->parseCallableTemplateArgument($tokens);
599-
$tokens->skipNewLineTokens();
599+
$tokens->skipNewLineTokensAndConsumeComments();
600600
}
601601

602602
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_ANGLE_BRACKET);
@@ -869,7 +869,7 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
869869
$done = false;
870870

871871
do {
872-
$tokens->skipNewLineTokens();
872+
$tokens->skipNewLineTokensAndConsumeComments();
873873

874874
if ($tokens->tryConsumeTokenType(Lexer::TOKEN_CLOSE_CURLY_BRACKET)) {
875875
return Ast\Type\ArrayShapeNode::createSealed($items, $kind);
@@ -878,22 +878,22 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
878878
if ($tokens->tryConsumeTokenType(Lexer::TOKEN_VARIADIC)) {
879879
$sealed = false;
880880

881-
$tokens->skipNewLineTokens();
881+
$tokens->skipNewLineTokensAndConsumeComments();
882882
if ($tokens->isCurrentTokenType(Lexer::TOKEN_OPEN_ANGLE_BRACKET)) {
883883
if ($kind === Ast\Type\ArrayShapeNode::KIND_ARRAY) {
884884
$unsealedType = $this->parseArrayShapeUnsealedType($tokens);
885885
} else {
886886
$unsealedType = $this->parseListShapeUnsealedType($tokens);
887887
}
888-
$tokens->skipNewLineTokens();
888+
$tokens->skipNewLineTokensAndConsumeComments();
889889
}
890890

891891
$tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA);
892892
break;
893893
}
894894

895895
$items[] = $this->parseArrayShapeItem($tokens);
896-
$tokens->skipNewLineTokens();
896+
$tokens->skipNewLineTokensAndConsumeComments();
897897
if (!$tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)) {
898898
$done = true;
899899
}
@@ -905,7 +905,7 @@ private function parseArrayShape(TokenIterator $tokens, Ast\Type\TypeNode $type,
905905

906906
} while (!$done);
907907

908-
$tokens->skipNewLineTokens();
908+
$tokens->skipNewLineTokensAndConsumeComments();
909909
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_CURLY_BRACKET);
910910

911911
if ($sealed) {
@@ -923,7 +923,7 @@ private function parseArrayShapeItem(TokenIterator $tokens): Ast\Type\ArrayShape
923923
$startIndex = $tokens->currentTokenIndex();
924924

925925
// parse any comments above the item
926-
$tokens->skipNewLineTokens();
926+
$tokens->skipNewLineTokensAndConsumeComments();
927927

928928
try {
929929
$tokens->pushSavePoint();
@@ -997,18 +997,18 @@ private function parseArrayShapeUnsealedType(TokenIterator $tokens): Ast\Type\Ar
997997
$startIndex = $tokens->currentTokenIndex();
998998

999999
$tokens->consumeTokenType(Lexer::TOKEN_OPEN_ANGLE_BRACKET);
1000-
$tokens->skipNewLineTokens();
1000+
$tokens->skipNewLineTokensAndConsumeComments();
10011001

10021002
$valueType = $this->parse($tokens);
1003-
$tokens->skipNewLineTokens();
1003+
$tokens->skipNewLineTokensAndConsumeComments();
10041004

10051005
$keyType = null;
10061006
if ($tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA)) {
1007-
$tokens->skipNewLineTokens();
1007+
$tokens->skipNewLineTokensAndConsumeComments();
10081008

10091009
$keyType = $valueType;
10101010
$valueType = $this->parse($tokens);
1011-
$tokens->skipNewLineTokens();
1011+
$tokens->skipNewLineTokensAndConsumeComments();
10121012
}
10131013

10141014
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_ANGLE_BRACKET);
@@ -1030,10 +1030,10 @@ private function parseListShapeUnsealedType(TokenIterator $tokens): Ast\Type\Arr
10301030
$startIndex = $tokens->currentTokenIndex();
10311031

10321032
$tokens->consumeTokenType(Lexer::TOKEN_OPEN_ANGLE_BRACKET);
1033-
$tokens->skipNewLineTokens();
1033+
$tokens->skipNewLineTokensAndConsumeComments();
10341034

10351035
$valueType = $this->parse($tokens);
1036-
$tokens->skipNewLineTokens();
1036+
$tokens->skipNewLineTokensAndConsumeComments();
10371037

10381038
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_ANGLE_BRACKET);
10391039

@@ -1055,18 +1055,18 @@ private function parseObjectShape(TokenIterator $tokens): Ast\Type\ObjectShapeNo
10551055
$items = [];
10561056

10571057
do {
1058-
$tokens->skipNewLineTokens();
1058+
$tokens->skipNewLineTokensAndConsumeComments();
10591059

10601060
if ($tokens->tryConsumeTokenType(Lexer::TOKEN_CLOSE_CURLY_BRACKET)) {
10611061
return new Ast\Type\ObjectShapeNode($items);
10621062
}
10631063

10641064
$items[] = $this->parseObjectShapeItem($tokens);
10651065

1066-
$tokens->skipNewLineTokens();
1066+
$tokens->skipNewLineTokensAndConsumeComments();
10671067
} while ($tokens->tryConsumeTokenType(Lexer::TOKEN_COMMA));
10681068

1069-
$tokens->skipNewLineTokens();
1069+
$tokens->skipNewLineTokensAndConsumeComments();
10701070
$tokens->consumeTokenType(Lexer::TOKEN_CLOSE_CURLY_BRACKET);
10711071

10721072
return new Ast\Type\ObjectShapeNode($items);
@@ -1078,7 +1078,7 @@ private function parseObjectShapeItem(TokenIterator $tokens): Ast\Type\ObjectSha
10781078
$startLine = $tokens->currentTokenLine();
10791079
$startIndex = $tokens->currentTokenIndex();
10801080

1081-
$tokens->skipNewLineTokens();
1081+
$tokens->skipNewLineTokensAndConsumeComments();
10821082

10831083
$key = $this->parseObjectShapeKey($tokens);
10841084
$optional = $tokens->tryConsumeTokenType(Lexer::TOKEN_NULLABLE);

0 commit comments

Comments
 (0)