diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php index 304344c957..d674bc2fe2 100644 --- a/src/Tokenizers/PHP.php +++ b/src/Tokenizers/PHP.php @@ -569,12 +569,12 @@ protected function tokenize($string) } /* - PHP 8 tokenizes a new line after a slash comment to the next whitespace token. + PHP 8 tokenizes a new line after a slash and hash comment to the next whitespace token. */ if (PHP_VERSION_ID >= 80000 && $tokenIsArray === true - && ($token[0] === T_COMMENT && strpos($token[1], '//') === 0) + && ($token[0] === T_COMMENT && (strpos($token[1], '//') === 0 || strpos($token[1], '#') === 0)) && isset($tokens[($stackPtr + 1)]) === true && is_array($tokens[($stackPtr + 1)]) === true && $tokens[($stackPtr + 1)][0] === T_WHITESPACE diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc index 6d92b9cfd3..3bf013c66b 100644 --- a/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc +++ b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc @@ -111,9 +111,29 @@ $prop = 123; /** Comment */ * @tag Comment */ +/* testSingleLineHashComment */ +# Comment + +/* testSingleLineHashCommentTrailing */ +echo 'a'; # Comment + +/* testMultiLineHashComment */ +# Comment1 +# Comment2 +# Comment3 + +/* testMultiLineHashCommentWithIndent */ + # Comment1 + # Comment2 + # Comment3 + /* testSingleLineSlashCommentNoNewLineAtEnd */ // Slash ?> + 'T_WHITESPACE', 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineHashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineHashCommentTrailing */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineHashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineHashCommentWithIndent */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment1 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment2 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', ], ], @@ -933,6 +1021,20 @@ public function dataCommentTokenization() [ 'type' => 'T_CLOSE_TAG', 'content' => '?> +', + ], + ], + ], + [ + '/* testSingleLineHashCommentNoNewLineAtEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Hash ', + ], + [ + 'type' => 'T_CLOSE_TAG', + 'content' => '?> ', ], ], diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc index 8ca0003924..6ca7026d4b 100644 --- a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc +++ b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc @@ -39,5 +39,25 @@ echo 'a'; // Comment // Slash ?> + 'T_CLOSE_TAG', 'content' => '?> +', + ], + ], + ], + [ + '/* testSingleLineHashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineHashCommentTrailing */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineHashComment */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment1 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment2 +', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testMultiLineHashCommentWithIndent */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment1 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment2 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' ', + ], + [ + 'type' => 'T_COMMENT', + 'content' => '# Comment3 +', + ], + [ + 'type' => 'T_WHITESPACE', + 'content' => ' +', + ], + ], + ], + [ + '/* testSingleLineHashCommentNoNewLineAtEnd */', + [ + [ + 'type' => 'T_COMMENT', + 'content' => '# Hash ', + ], + [ + 'type' => 'T_CLOSE_TAG', + 'content' => '?> ', ], ],