From 0be1542de351c55cbdfdd2bd028366e4801520da Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Mon, 31 Aug 2020 20:40:17 +0200
Subject: [PATCH] PHP 8.0 | Stabilize comment tokenization for hash comments

Follow up on 3027 which handled this for slash-style comments.

Hash-style comments were not addressed in the earlier change. This oversight has now been fixed.

Includes additional unit tests.

Fixes 3069
---
 src/Tokenizers/PHP.php                        |   4 +-
 .../Tokenizer/StableCommentWhitespaceTest.inc |  20 ++++
 .../Tokenizer/StableCommentWhitespaceTest.php | 102 ++++++++++++++++++
 .../StableCommentWhitespaceWinTest.inc        |  20 ++++
 .../StableCommentWhitespaceWinTest.php        | 102 ++++++++++++++++++
 5 files changed, 246 insertions(+), 2 deletions(-)

diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
index 6135e957cf..8f52ca8fc6 100644
--- a/src/Tokenizers/PHP.php
+++ b/src/Tokenizers/PHP.php
@@ -567,12 +567,12 @@ protected function tokenize($string)
             }
 
             /*
-                PHP 8 tokenizes a new line after a slash comment to the next whitespace token.
+                PHP 8 tokenizes a new line after a slash and hash comment to the next whitespace token.
             */
 
             if (PHP_VERSION_ID >= 80000
                 && $tokenIsArray === true
-                && ($token[0] === T_COMMENT && strpos($token[1], '//') === 0)
+                && ($token[0] === T_COMMENT && (strpos($token[1], '//') === 0 || strpos($token[1], '#') === 0))
                 && isset($tokens[($stackPtr + 1)]) === true
                 && is_array($tokens[($stackPtr + 1)]) === true
                 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc
index 6d92b9cfd3..3bf013c66b 100644
--- a/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc
+++ b/tests/Core/Tokenizer/StableCommentWhitespaceTest.inc
@@ -111,9 +111,29 @@ $prop = 123; /** Comment */
  * @tag Comment
  */
 
+/* testSingleLineHashComment */
+# Comment
+
+/* testSingleLineHashCommentTrailing */
+echo 'a'; # Comment
+
+/* testMultiLineHashComment */
+# Comment1
+# Comment2
+# Comment3
+
+/* testMultiLineHashCommentWithIndent */
+    # Comment1
+    # Comment2
+    # Comment3
+
 /* testSingleLineSlashCommentNoNewLineAtEnd */
 // Slash ?>
 <?php
 
+/* testSingleLineHashCommentNoNewLineAtEnd */
+# Hash ?>
+<?php
+
 /* testCommentAtEndOfFile */
 /* Comment
\ No newline at end of file
diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceTest.php b/tests/Core/Tokenizer/StableCommentWhitespaceTest.php
index 01c7440b6d..389bb74e69 100644
--- a/tests/Core/Tokenizer/StableCommentWhitespaceTest.php
+++ b/tests/Core/Tokenizer/StableCommentWhitespaceTest.php
@@ -919,6 +919,94 @@ public function dataCommentTokenization()
                     [
                         'type'    => 'T_WHITESPACE',
                         'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testSingleLineHashComment */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testSingleLineHashCommentTrailing */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testMultiLineHashComment */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment1
+',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment2
+',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment3
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testMultiLineHashCommentWithIndent */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment1
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment2
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment3
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
 ',
                     ],
                 ],
@@ -933,6 +1021,20 @@ public function dataCommentTokenization()
                     [
                         'type'    => 'T_CLOSE_TAG',
                         'content' => '?>
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testSingleLineHashCommentNoNewLineAtEnd */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Hash ',
+                    ],
+                    [
+                        'type'    => 'T_CLOSE_TAG',
+                        'content' => '?>
 ',
                     ],
                 ],
diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc
index 8ca0003924..6ca7026d4b 100644
--- a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc
+++ b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.inc
@@ -39,5 +39,25 @@ echo 'a'; // Comment
 // Slash ?>
 <?php
 
+/* testSingleLineHashComment */
+# Comment
+
+/* testSingleLineHashCommentTrailing */
+echo 'a'; # Comment
+
+/* testMultiLineHashComment */
+# Comment1
+# Comment2
+# Comment3
+
+/* testMultiLineHashCommentWithIndent */
+    # Comment1
+    # Comment2
+    # Comment3
+
+/* testSingleLineHashCommentNoNewLineAtEnd */
+# Hash ?>
+<?php
+
 /* testCommentAtEndOfFile */
 /* Comment
\ No newline at end of file
diff --git a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php
index df45baa417..9ab66e9c7a 100644
--- a/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php
+++ b/tests/Core/Tokenizer/StableCommentWhitespaceWinTest.php
@@ -244,6 +244,108 @@ public function dataCommentTokenization()
                     [
                         'type'    => 'T_CLOSE_TAG',
                         'content' => '?>
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testSingleLineHashComment */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testSingleLineHashCommentTrailing */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testMultiLineHashComment */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment1
+',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment2
+',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment3
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testMultiLineHashCommentWithIndent */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment1
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment2
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Comment3
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            [
+                '/* testSingleLineHashCommentNoNewLineAtEnd */',
+                [
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '# Hash ',
+                    ],
+                    [
+                        'type'    => 'T_CLOSE_TAG',
+                        'content' => '?>
 ',
                     ],
                 ],