From 62cda4011c8242947a92401a7ea9f7fd6c65b27a Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Thu, 24 Oct 2024 22:02:58 +0200
Subject: [PATCH 1/8] PHP 8.3 | Tokenizer/PHP: support "yield from" with
 comments

As discussed in and discovered via issue 529:
* Prior to PHP 8.3, only whitespace was allowed between the `yield` and `from` keywords. A comment between the `yield` and `from` keywords in a `yield from` expression would result in a parse error.
* As of PHP 8.3, this is no longer a parse error and both whitespace as well as comments are allowed between the `yield` and `from` and the complete expression is tokenized in PHP itself as one `T_YIELD_FROM` token. See: https://3v4l.org/2SI2Q#veol
    In the context of PHPCS this is problematic as comments should always have their own token to allow sniffs to examine them.
    Additionally, such comments may contain PHPCS ignore annotations, which, when not tokenized as a separate token, will not be respected.

This commit adds support for this change in PHP 8.3 to PHP_CodeSniffer. It does contain an, albeit small, BC-break, due to the BC-break created by PHP.

Previously in PHPCS:
* A single line `yield from` expression would always tokenize as `T_YIELD_FROM`, independently of the type and amount of whitespace between the keywords.
* A multi-line `yield` [new line]+ `from` expression would tokenize as multiple `T_YIELD_FROM` tokens, one for each line.
* A `yield from` expression with a comment between the keywords was not supported.
    In PHP < 8.3, this meant that this would tokenize as `T_YIELD`, [`T_WHITESPACE`|T_COMMENT`]+, `T_STRING` (`from`).
    As of PHP 8.3, this was tokenized as one or more `T_YIELD_FROM` tokens (depending on single/multi-line) with the comment being tokenized as `T_YIELD_FROM` as well.

This commit changes this as follows:
* Single line `yield from` expression with only whitespace between the keywords: **no change**, this will still tokenize as a single `T_YIELD_FROM` token.
* Multi-line `yield` [new line]+ `from` expressions and `yield from` expressions with a comment (both single line as well as multi-line) will now consistently be tokenized as `T_YIELD_FROM` (`yield`), [`T_WHITESPACE`|T_COMMENT`]+, `T_YIELD_FROM` (`from`).

In practice, this means that:
* Whitespace and comments between the keywords can now be examined and handled by relevant sniffs, which are likely to give more accurate results (fewer false negatives, like for tab indentation of a `from` keyword on a separate line).
* The tokenization used by PHPCS is now consistent again for all supported PHP versions.
* The PHP 8.3 change is now supported.

It does mean that sniffs which explicitly handle multi-token `yield from` expressions, will need to be updated.

In my opinion, adding this change in a minor is justified as:
1. The PHP 8.3 change can not be supported otherwise.
2. The impact is expected to be minimal anyhow as there are not many sniffs which specifically look for and handle `T_YIELD_FROM` tokens and those sniffs within PHPCS itself will be updated/adjusted in the same release.

Also, the (negative) impact on _end-users_ of this BC-break is also expected to be minimal as a scan of the top 2000 projects listed on Packagist shows that in those project no multi-line/multi-token `yield from` expressions are used in the source code, which means that even when sniff code is not updated (yet) for the change in tokenization, the chances of an end-user getting incorrect results because of this are very slim as the code affected is just not written as multi-line/with comment that often.

Includes tests.

Fixes 529

Refs:
* squizlabs/PHP_CodeSniffer 1524 (original polyfill code)
* php/php-src 10125
* php/php-src 14926
* https://externals.io/message/124462

---

Information for standards maintainers

The "yield from" _keyword_ could previously already consist of multiple T_YIELD_FROM tokens if the "keyword" was spread over multiple lines.
Now, the tokens between the actual keywords will be tokenized as `T_WHITESPACE` and comment tokens.

To find the last token for a `T_YIELD_FROM` "keyword", change old code like this:
```php
$yieldFromEnd = $stackPtr;
if (preg_match('`yield\s+from`', $tokens[$stackPtr]['content']) !== 1) {
    for ($yieldFromEnd = ($stackPtr + 1); $tokens[$yieldFromEnd]['code'] === T_YIELD_FROM; $yieldFromEnd++);
    --$yieldFromEnd;
}
```
to
```php
$yieldFromEnd = $stackPtr;
if (strtolower(trim($tokens[$stackPtr]['content'])) === 'yield') {
    for ($i = ($stackPtr + 1); $i < $phpcsFile->numTokens; $i++) {
        if ($tokens[$i]['code'] === T_YIELD_FROM && strtolower(trim($tokens[$i]['content'])) === 'from') {
            $yieldFromEnd = $i;
            break;
        }

        if (isset(Tokens::$emptyTokens[$tokens[$i]['code']]) === false && $tokens[$i]['code'] !== T_YIELD_FROM) {
            // Shouldn't be possible. Just to be on the safe side.
            break;
        }
    }
}
```

The above presumes that `$stackPtr` is set to a `T_YIELD_FROM` token.
Also note that the second code snippet is largely cross-version compatible. It will work with older PHPCS versions with code compatible with PHP < 8.3 and will work on PHPCS 3.11.0+ for code compatible with all supported PHP versions.
---
 src/Tokenizers/PHP.php                  | 140 ++++++++++++++++-
 tests/Core/Tokenizers/PHP/YieldTest.inc |  25 +++
 tests/Core/Tokenizers/PHP/YieldTest.php | 193 +++++++++++++++++++++++-
 3 files changed, 348 insertions(+), 10 deletions(-)

diff --git a/src/Tokenizers/PHP.php b/src/Tokenizers/PHP.php
index 42969782b7..9c6c11e4c3 100644
--- a/src/Tokenizers/PHP.php
+++ b/src/Tokenizers/PHP.php
@@ -1514,7 +1514,7 @@ protected function tokenize($string)
             }//end if
 
             /*
-                Before PHP 7.0, the "yield from" was tokenized as
+                Before PHP 7.0, "yield from" was tokenized as
                 T_YIELD, T_WHITESPACE and T_STRING. So look for
                 and change this token in earlier versions.
             */
@@ -1525,12 +1525,16 @@ protected function tokenize($string)
                 && isset($tokens[($stackPtr + 1)]) === true
                 && isset($tokens[($stackPtr + 2)]) === true
                 && $tokens[($stackPtr + 1)][0] === T_WHITESPACE
+                && strpos($tokens[($stackPtr + 1)][1], $this->eolChar) === false
                 && $tokens[($stackPtr + 2)][0] === T_STRING
                 && strtolower($tokens[($stackPtr + 2)][1]) === 'from'
             ) {
-                // Could be multi-line, so adjust the token stack.
-                $token[0]  = T_YIELD_FROM;
-                $token[1] .= $tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1];
+                // Single-line "yield from" with only whitespace between.
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_YIELD_FROM,
+                    'type'    => 'T_YIELD_FROM',
+                    'content' => $token[1].$tokens[($stackPtr + 1)][1].$tokens[($stackPtr + 2)][1],
+                ];
 
                 if (PHP_CODESNIFFER_VERBOSITY > 1) {
                     for ($i = ($stackPtr + 1); $i <= ($stackPtr + 2); $i++) {
@@ -1540,9 +1544,131 @@ protected function tokenize($string)
                     }
                 }
 
-                $tokens[($stackPtr + 1)] = null;
-                $tokens[($stackPtr + 2)] = null;
-            }
+                $newStackPtr++;
+                $stackPtr += 2;
+
+                continue;
+            } else if (PHP_VERSION_ID < 80300
+                && $tokenIsArray === true
+                && $token[0] === T_STRING
+                && strtolower($token[1]) === 'from'
+                && $finalTokens[$lastNotEmptyToken]['code'] === T_YIELD
+            ) {
+                /*
+                    Before PHP 8.3, if there was a comment between the "yield" and "from" keywords,
+                    it was tokenized as T_YIELD, T_WHITESPACE, T_COMMENT... and T_STRING.
+                    We want to keep the tokenization of the tokens between, but need to change the
+                    `T_YIELD` and `T_STRING` (from) keywords to `T_YIELD_FROM.
+                */
+
+                $finalTokens[$lastNotEmptyToken]['code'] = T_YIELD_FROM;
+                $finalTokens[$lastNotEmptyToken]['type'] = 'T_YIELD_FROM';
+
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_YIELD_FROM,
+                    'type'    => 'T_YIELD_FROM',
+                    'content' => $token[1],
+                ];
+                $newStackPtr++;
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    echo "\t\t* token $lastNotEmptyToken (new stack) changed into T_YIELD_FROM; was: T_YIELD".PHP_EOL;
+                    echo "\t\t* token $stackPtr changed into T_YIELD_FROM; was: T_STRING".PHP_EOL;
+                }
+
+                continue;
+            } else if (PHP_VERSION_ID >= 70000
+                && $tokenIsArray === true
+                && $token[0] === T_YIELD_FROM
+                && strpos($token[1], $this->eolChar) !== false
+                && preg_match('`^yield\s+from$`i', $token[1]) === 1
+            ) {
+                /*
+                    In PHP 7.0+, a multi-line "yield from" (without comment) tokenizes as a single
+                    T_YIELD_FROM token, but we want to split it and tokenize the whitespace
+                    separately for consistency.
+                */
+
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_YIELD_FROM,
+                    'type'    => 'T_YIELD_FROM',
+                    'content' => substr($token[1], 0, 5),
+                ];
+                $newStackPtr++;
+
+                $tokenLines = explode($this->eolChar, substr($token[1], 5, -4));
+                $numLines   = count($tokenLines);
+                $newToken   = [
+                    'type'    => 'T_WHITESPACE',
+                    'code'    => T_WHITESPACE,
+                    'content' => '',
+                ];
+
+                foreach ($tokenLines as $i => $line) {
+                    $newToken['content'] = $line;
+                    if ($i === ($numLines - 1)) {
+                        if ($line === '') {
+                            break;
+                        }
+                    } else {
+                        $newToken['content'] .= $this->eolChar;
+                    }
+
+                    $finalTokens[$newStackPtr] = $newToken;
+                    $newStackPtr++;
+                }
+
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_YIELD_FROM,
+                    'type'    => 'T_YIELD_FROM',
+                    'content' => substr($token[1], -4),
+                ];
+                $newStackPtr++;
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    echo "\t\t* token $stackPtr split into 'yield', one or more whitespace tokens and 'from'".PHP_EOL;
+                }
+
+                continue;
+            } else if (PHP_VERSION_ID >= 80300
+                && $tokenIsArray === true
+                && $token[0] === T_YIELD_FROM
+                && preg_match('`^yield[ \t]+from$`i', $token[1]) !== 1
+                && stripos($token[1], 'yield') === 0
+            ) {
+                /*
+                    Since PHP 8.3, "yield from" allows for comments and will
+                    swallow the comment in the `T_YIELD_FROM` token.
+                    We need to split this up to allow for sniffs handling comments.
+                */
+
+                $finalTokens[$newStackPtr] = [
+                    'code'    => T_YIELD_FROM,
+                    'type'    => 'T_YIELD_FROM',
+                    'content' => substr($token[1], 0, 5),
+                ];
+                $newStackPtr++;
+
+                $yieldFromSubtokens = @token_get_all("<?php\n".substr($token[1], 5, -4));
+                // Remove the PHP open tag token.
+                array_shift($yieldFromSubtokens);
+                // Add the "from" keyword.
+                $yieldFromSubtokens[] = [
+                    0 => T_YIELD_FROM,
+                    1 => substr($token[1], -4),
+                ];
+
+                // Inject the new tokens into the token stack.
+                array_splice($tokens, ($stackPtr + 1), 0, $yieldFromSubtokens);
+                $numTokens = count($tokens);
+
+                if (PHP_CODESNIFFER_VERBOSITY > 1) {
+                    echo "\t\t* token $stackPtr split into parts (yield from with comment)".PHP_EOL;
+                }
+
+                unset($yieldFromSubtokens);
+                continue;
+            }//end if
 
             /*
                 Before PHP 5.6, the ... operator was tokenized as three
diff --git a/tests/Core/Tokenizers/PHP/YieldTest.inc b/tests/Core/Tokenizers/PHP/YieldTest.inc
index fdac3c71eb..3130b84600 100644
--- a/tests/Core/Tokenizers/PHP/YieldTest.inc
+++ b/tests/Core/Tokenizers/PHP/YieldTest.inc
@@ -22,6 +22,31 @@ function generator()
 
     FROM
     gen2();
+
+    /* testYieldFromSplitByComment */
+    yield /* comment */ from gen2();
+
+    /* testYieldFromWithTrailingComment */
+    yield // comment
+    from gen2();
+
+    /* testYieldFromWithTrailingAnnotation */
+    yield // phpcs:ignore Stnd.Cat.Sniff -- for reasons.
+    from gen2();
+
+    /* testYieldFromSplitByNewLineAndComments */
+    yield
+    /* comment line 1
+       line 2 */
+    // another comment
+    from
+    gen2();
+
+    /* testYieldFromSplitByNewLineAndAnnotation */
+    YIELD
+    // @phpcs:disable Stnd.Cat.Sniff -- for reasons.
+    From
+    gen2();
 }
 
 /* testYieldUsedAsClassName */
diff --git a/tests/Core/Tokenizers/PHP/YieldTest.php b/tests/Core/Tokenizers/PHP/YieldTest.php
index 23a01ce5a1..301c39b7b8 100644
--- a/tests/Core/Tokenizers/PHP/YieldTest.php
+++ b/tests/Core/Tokenizers/PHP/YieldTest.php
@@ -181,22 +181,209 @@ public function testYieldFromKeywordMultiToken($testMarker, $expectedTokens)
     public static function dataYieldFromKeywordMultiToken()
     {
         return [
-            'yield from with new line' => [
+            'yield from with new line'                => [
                 'testMarker'     => '/* testYieldFromSplitByNewLines */',
                 'expectedTokens' => [
                     [
                         'type'    => 'T_YIELD_FROM',
-                        'content' => 'yield
+                        'content' => 'yield',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'FROM',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            'yield from with comment'                 => [
+                'testMarker'     => '/* testYieldFromSplitByComment */',
+                'expectedTokens' => [
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'yield',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '/* comment */',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'from',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                ],
+            ],
+            'yield from with trailing comment'        => [
+                'testMarker'     => '/* testYieldFromWithTrailingComment */',
+                'expectedTokens' => [
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'yield',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '// comment
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'from',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                ],
+            ],
+            'yield from with trailing annotation'     => [
+                'testMarker'     => '/* testYieldFromWithTrailingAnnotation */',
+                'expectedTokens' => [
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'yield',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                    [
+                        'type'    => 'T_PHPCS_IGNORE',
+                        'content' => '// phpcs:ignore Stnd.Cat.Sniff -- for reasons.
 ',
                     ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'from',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => ' ',
+                    ],
+                ],
+            ],
+            'yield from with new line and comment'    => [
+                'testMarker'     => '/* testYieldFromSplitByNewLineAndComments */',
+                'expectedTokens' => [
                     [
                         'type'    => 'T_YIELD_FROM',
+                        'content' => 'yield',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
                         'content' => '
 ',
                     ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '/* comment line 1
+',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '       line 2 */',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_COMMENT',
+                        'content' => '// another comment
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'from',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                ],
+            ],
+            'yield from with new line and annotation' => [
+                'testMarker'     => '/* testYieldFromSplitByNewLineAndAnnotation */',
+                'expectedTokens' => [
+                    [
+                        'type'    => 'T_YIELD_FROM',
+                        'content' => 'YIELD',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
+                    [
+                        'type'    => 'T_PHPCS_DISABLE',
+                        'content' => '// @phpcs:disable Stnd.Cat.Sniff -- for reasons.
+',
+                    ],
+                    [
+                        'type'    => 'T_WHITESPACE',
+                        'content' => '    ',
+                    ],
                     [
                         'type'    => 'T_YIELD_FROM',
-                        'content' => '    FROM',
+                        'content' => 'From',
                     ],
                     [
                         'type'    => 'T_WHITESPACE',

From b7683026c3c39e796f4b92e86c89afa6fe26dcf3 Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Tue, 22 Oct 2024 02:10:26 +0200
Subject: [PATCH 2/8] Tokenizer: apply tab replacement to "yield from"

The `yield from` keyword(s) can be separated by spaces, tabs, new lines and since PHP 8.3, even comments.

The PHPCS `Tokenizer` previously did not execute tab replacement on this token leading to unexpected `'content'` and incorrect `'length'` values in the `File::$tokens` array, which in turn could lead to incorrect sniff results and incorrect fixes.

Previously, this affected all `T_YIELD_FROM` tokens. After the change to support PHP 8.3 "yield ... comment... from" syntax, this now only affects single line `yield from` expressions where the keywords are separated by a tab or a mix of tabs and spaces.

All the same, consistency is key, so this commit adds the `T_YIELD_FROM` token to the array of tokens for which to do tab replacement, which should make them more consistent with the rest of PHPCS.

Includes unit tests safeguarding this change.
---
 src/Tokenizers/Tokenizer.php                  |   1 +
 tests/Core/Tokenizers/PHP/YieldTest.php       |   6 +-
 .../CreatePositionMapYieldFromTest.inc        |  15 +++
 .../CreatePositionMapYieldFromTest.php        | 100 ++++++++++++++++++
 4 files changed, 121 insertions(+), 1 deletion(-)
 create mode 100644 tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc
 create mode 100644 tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.php

diff --git a/src/Tokenizers/Tokenizer.php b/src/Tokenizers/Tokenizer.php
index aa706fbfa9..8c90cd7c19 100644
--- a/src/Tokenizers/Tokenizer.php
+++ b/src/Tokenizers/Tokenizer.php
@@ -200,6 +200,7 @@ private function createPositionMap()
             T_END_HEREDOC              => true,
             T_END_NOWDOC               => true,
             T_INLINE_HTML              => true,
+            T_YIELD_FROM               => true,
         ];
 
         $this->numTokens = count($this->tokens);
diff --git a/tests/Core/Tokenizers/PHP/YieldTest.php b/tests/Core/Tokenizers/PHP/YieldTest.php
index 301c39b7b8..efb12096cf 100644
--- a/tests/Core/Tokenizers/PHP/YieldTest.php
+++ b/tests/Core/Tokenizers/PHP/YieldTest.php
@@ -100,7 +100,11 @@ public function testYieldFromKeywordSingleToken($testMarker, $expectedContent)
         $this->assertSame(T_YIELD_FROM, $tokenArray['code'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (code)');
         $this->assertSame('T_YIELD_FROM', $tokenArray['type'], 'Token tokenized as '.$tokenArray['type'].', not T_YIELD_FROM (type)');
 
-        $this->assertSame($expectedContent, $tokenArray['content'], 'Token content does not match expectation');
+        if (isset($tokenArray['orig_content']) === true) {
+            $this->assertSame($expectedContent, $tokenArray['orig_content'], 'Token (orig) content does not match expectation');
+        } else {
+            $this->assertSame($expectedContent, $tokenArray['content'], 'Token content does not match expectation');
+        }
 
     }//end testYieldFromKeywordSingleToken()
 
diff --git a/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc b/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc
new file mode 100644
index 0000000000..59365dd5ce
--- /dev/null
+++ b/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.inc
@@ -0,0 +1,15 @@
+<?php
+
+function myGenerator() {
+    /* testYieldFromHasSingleSpace */
+    yield from gen2();
+
+    /* testYieldFromHasMultiSpace */
+    yield     from gen2();
+
+    /* testYieldFromHasTabs */
+    yield		from gen2();
+
+    /* testYieldFromMixedTabsSpaces */
+    Yield	   	 	From gen2();
+}
diff --git a/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.php b/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.php
new file mode 100644
index 0000000000..a6d919859b
--- /dev/null
+++ b/tests/Core/Tokenizers/Tokenizer/CreatePositionMapYieldFromTest.php
@@ -0,0 +1,100 @@
+<?php
+/**
+ * Tests the tokenization of "yield from" tokens.
+ *
+ * @author    Juliette Reinders Folmer <phpcs_nospam@adviesenzo.nl>
+ * @copyright 2024 PHPCSStandards and contributors
+ * @license   https://github.com/PHPCSStandards/PHP_CodeSniffer/blob/master/licence.txt BSD Licence
+ */
+
+namespace PHP_CodeSniffer\Tests\Core\Tokenizers\Tokenizer;
+
+use PHP_CodeSniffer\Tests\Core\Tokenizers\AbstractTokenizerTestCase;
+
+/**
+ * Yield from token test.
+ *
+ * @covers PHP_CodeSniffer\Tokenizers\Tokenizer::createPositionMap
+ */
+final class CreatePositionMapYieldFromTest extends AbstractTokenizerTestCase
+{
+
+
+    /**
+     * Verify that spaces/tabs in "yield from" tokens get the tab replacement treatment.
+     *
+     * @param string                         $testMarker The comment prefacing the target token.
+     * @param array<string, int|string|null> $expected   Expectations for the token array.
+     * @param string                         $content    Optional. The test token content to search for.
+     *                                                   Defaults to null.
+     *
+     * @dataProvider dataYieldFromTabReplacement
+     *
+     * @return void
+     */
+    public function testYieldFromTabReplacement($testMarker, $expected, $content=null)
+    {
+        $tokens = $this->phpcsFile->getTokens();
+        $target = $this->getTargetToken($testMarker, [T_YIELD_FROM], $content);
+
+        foreach ($expected as $key => $value) {
+            if ($key === 'orig_content' && $value === null) {
+                $this->assertArrayNotHasKey($key, $tokens[$target], "Unexpected 'orig_content' key found in the token array.");
+                continue;
+            }
+
+            $this->assertArrayHasKey($key, $tokens[$target], "Key $key not found in the token array.");
+            $this->assertSame($value, $tokens[$target][$key], "Value for key $key does not match expectation.");
+        }
+
+    }//end testYieldFromTabReplacement()
+
+
+    /**
+     * Data provider.
+     *
+     * @see testYieldFromTabReplacement()
+     *
+     * @return array<string, array<string, string|array<string, int|string|null>>>
+     */
+    public static function dataYieldFromTabReplacement()
+    {
+        return [
+            'Yield from, single line, single space'           => [
+                'testMarker' => '/* testYieldFromHasSingleSpace */',
+                'expected'   => [
+                    'length'       => 10,
+                    'content'      => 'yield from',
+                    'orig_content' => null,
+                ],
+            ],
+            'Yield from, single line, multiple spaces'        => [
+                'testMarker' => '/* testYieldFromHasMultiSpace */',
+                'expected'   => [
+                    'length'       => 14,
+                    'content'      => 'yield     from',
+                    'orig_content' => null,
+                ],
+            ],
+            'Yield from, single line, has tabs'               => [
+                'testMarker' => '/* testYieldFromHasTabs */',
+                'expected'   => [
+                    'length'       => 16,
+                    'content'      => 'yield       from',
+                    'orig_content' => 'yield		from',
+                ],
+            ],
+            'Yield from, single line, mix of tabs and spaces' => [
+                'testMarker' => '/* testYieldFromMixedTabsSpaces */',
+                'expected'   => [
+                    'length'       => 20,
+                    'content'      => 'Yield           From',
+                    'orig_content' => 'Yield	   	 	From',
+                ],
+            ],
+        ];
+
+    }//end dataYieldFromTabReplacement()
+
+
+}//end class

From 08e3d9618b91a26cb86b85aa0417c02c9ef7dc50 Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Tue, 22 Oct 2024 02:10:26 +0200
Subject: [PATCH 3/8] Generic/LanguageConstructSpacing: update for changed
 "yield from" tokenization

This commit updates the `Generic.WhiteSpace.LanguageConstructSpacing` sniff to recognize that a multi-token `yield from` may now have whitespace and comment tokens between the keywords.
---
 .../LanguageConstructSpacingSniff.php         | 44 +++++++++++--------
 1 file changed, 25 insertions(+), 19 deletions(-)

diff --git a/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php b/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php
index d4e913c1ad..f8876ff17e 100644
--- a/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php
+++ b/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php
@@ -80,20 +80,30 @@ public function process(File $phpcsFile, $stackPtr)
         if ($tokens[$stackPtr]['code'] === T_YIELD_FROM
             && strtolower($content) !== 'yield from'
         ) {
-            if ($tokens[($stackPtr - 1)]['code'] === T_YIELD_FROM) {
-                // A multi-line statement that has already been processed.
-                return;
-            }
+            $found        = $content;
+            $hasComment   = false;
+            $yieldFromEnd = $stackPtr;
+
+            // Handle potentially multi-line/multi-token "yield from" expressions.
+            if (preg_match('`yield\s+from`i', $content) !== 1) {
+                for ($i = ($stackPtr + 1); $i < $phpcsFile->numTokens; $i++) {
+                    if (isset(Tokens::$emptyTokens[$tokens[$i]['code']]) === false
+                        && $tokens[$i]['code'] !== T_YIELD_FROM
+                    ) {
+                        break;
+                    }
 
-            $found = $content;
-            if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) {
-                // This yield from statement is split over multiple lines.
-                $i = ($stackPtr + 1);
-                do {
                     $found .= $tokens[$i]['content'];
-                    $i++;
-                } while ($tokens[$i]['code'] === T_YIELD_FROM);
-            }
+
+                    if ($tokens[$i]['code'] === T_YIELD_FROM
+                        && strtolower(trim($tokens[$i]['content'])) === 'from'
+                    ) {
+                        break;
+                    }
+                }
+
+                $yieldFromEnd = $i;
+            }//end if
 
             $error = 'Language constructs must be followed by a single space; expected 1 space between YIELD FROM found "%s"';
             $data  = [Common::prepareForOutput($found)];
@@ -104,18 +114,14 @@ public function process(File $phpcsFile, $stackPtr)
                 $phpcsFile->fixer->beginChangeset();
                 $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
 
-                if ($tokens[($stackPtr + 1)]['code'] === T_YIELD_FROM) {
-                    $i = ($stackPtr + 1);
-                    do {
-                        $phpcsFile->fixer->replaceToken($i, '');
-                        $i++;
-                    } while ($tokens[$i]['code'] === T_YIELD_FROM);
+                for ($i = ($stackPtr + 1); $i <= $yieldFromEnd; $i++) {
+                    $phpcsFile->fixer->replaceToken($i, '');
                 }
 
                 $phpcsFile->fixer->endChangeset();
             }
 
-            return;
+            return ($yieldFromEnd + 1);
         }//end if
 
         if ($tokens[($stackPtr + 1)]['code'] === T_WHITESPACE) {

From 007267cd9fd828e274defc6f4843a53bde9c35ed Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Thu, 24 Oct 2024 07:31:14 +0200
Subject: [PATCH 4/8] Generic/LanguageConstructSpacing: don't auto-fix 'yield
 from' with comment

If there is a comment between the `yield` and the `from` keywords in a `yield from` expression, the sniff should report this, but should not auto-fix it, as it cannot be determined where the comment should be moved to.

Thic commit updates the sniff to throw a non-autofixable error with error code `IncorrectYieldFromWithComment` for that situation.

Includes unit tests.
---
 .../LanguageConstructSpacingSniff.php         | 31 ++++++++++++-------
 .../LanguageConstructSpacingUnitTest.1.inc    |  9 ++++++
 ...nguageConstructSpacingUnitTest.1.inc.fixed |  9 ++++++
 .../LanguageConstructSpacingUnitTest.php      |  3 ++
 4 files changed, 41 insertions(+), 11 deletions(-)

diff --git a/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php b/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php
index f8876ff17e..6cb5f92f0f 100644
--- a/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php
+++ b/src/Standards/Generic/Sniffs/WhiteSpace/LanguageConstructSpacingSniff.php
@@ -93,6 +93,10 @@ public function process(File $phpcsFile, $stackPtr)
                         break;
                     }
 
+                    if (isset(Tokens::$commentTokens[$tokens[$i]['code']]) === true) {
+                        $hasComment = true;
+                    }
+
                     $found .= $tokens[$i]['content'];
 
                     if ($tokens[$i]['code'] === T_YIELD_FROM
@@ -107,19 +111,24 @@ public function process(File $phpcsFile, $stackPtr)
 
             $error = 'Language constructs must be followed by a single space; expected 1 space between YIELD FROM found "%s"';
             $data  = [Common::prepareForOutput($found)];
-            $fix   = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data);
-            if ($fix === true) {
-                preg_match('/yield/i', $found, $yield);
-                preg_match('/from/i', $found, $from);
-                $phpcsFile->fixer->beginChangeset();
-                $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
 
-                for ($i = ($stackPtr + 1); $i <= $yieldFromEnd; $i++) {
-                    $phpcsFile->fixer->replaceToken($i, '');
-                }
+            if ($hasComment === true) {
+                $phpcsFile->addError($error, $stackPtr, 'IncorrectYieldFromWithComment', $data);
+            } else {
+                $fix = $phpcsFile->addFixableError($error, $stackPtr, 'IncorrectYieldFrom', $data);
+                if ($fix === true) {
+                    preg_match('/yield/i', $found, $yield);
+                    preg_match('/from/i', $found, $from);
+                    $phpcsFile->fixer->beginChangeset();
+                    $phpcsFile->fixer->replaceToken($stackPtr, $yield[0].' '.$from[0]);
 
-                $phpcsFile->fixer->endChangeset();
-            }
+                    for ($i = ($stackPtr + 1); $i <= $yieldFromEnd; $i++) {
+                        $phpcsFile->fixer->replaceToken($i, '');
+                    }
+
+                    $phpcsFile->fixer->endChangeset();
+                }
+            }//end if
 
             return ($yieldFromEnd + 1);
         }//end if
diff --git a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc
index 1847778d09..8d4acfe0be 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc
@@ -89,3 +89,12 @@ $newLine;
 // The following line must have a single space at the end (after return)
 return 
 $spaceAndNewLine;
+
+// Related to issue #529. These should not be auto-fixed as we don't know what to do with the comment.
+yield    /*comment*/    from $test();
+yield
+    # comment
+    from $test();
+yield
+    // phpcs:ignore Stnd.Category.SniffName
+    from $test();
diff --git a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed
index 4f5d3cec2f..6f83483a4d 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.1.inc.fixed
@@ -83,3 +83,12 @@ return $newLine;
 
 // The following line must have a single space at the end (after return)
 return $spaceAndNewLine;
+
+// Related to issue #529. These should not be auto-fixed as we don't know what to do with the comment.
+yield    /*comment*/    from $test();
+yield
+    # comment
+    from $test();
+yield
+    // phpcs:ignore Stnd.Category.SniffName
+    from $test();
diff --git a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php
index adac2c843f..10c60b5743 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php
+++ b/src/Standards/Generic/Tests/WhiteSpace/LanguageConstructSpacingUnitTest.php
@@ -71,6 +71,9 @@ public function getErrorList($testFile='')
                 85 => 1,
                 86 => 1,
                 90 => 1,
+                94 => 1,
+                95 => 1,
+                98 => 1,
             ];
 
         default:

From a504083109de938bfb83919c732447c3715c5eb8 Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Wed, 23 Oct 2024 04:10:37 +0200
Subject: [PATCH 5/8] Generic/LowerCaseKeyword: add extra tests with "yield
 from"

... to safeguard that a "yield from" keyword split into multiple tokens is still handled correctly.
---
 .../Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc         | 7 +++++++
 .../Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed   | 7 +++++++
 .../Generic/Tests/PHP/LowerCaseKeywordUnitTest.php         | 3 +++
 3 files changed, 17 insertions(+)

diff --git a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc
index a50ac22853..10d3ed6933 100644
--- a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc
+++ b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc
@@ -53,5 +53,12 @@ if (ISSET($a) && !Empty($a)) { UnSeT($a); }
 eval('foo');
 eVaL('foo');
 
+$c = function() {
+    Yield /*comment*/ From fun();
+    YIELD
+    /*comment*/
+    FROM fun();
+}
+
 __HALT_COMPILER(); // An exception due to phar support.
 function
diff --git a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed
index 5e15765146..547f72fca9 100644
--- a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed
+++ b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.inc.fixed
@@ -53,5 +53,12 @@ if (isset($a) && !empty($a)) { unset($a); }
 eval('foo');
 eval('foo');
 
+$c = function() {
+    yield /*comment*/ from fun();
+    yield
+    /*comment*/
+    from fun();
+}
+
 __HALT_COMPILER(); // An exception due to phar support.
 function
diff --git a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php
index 6d337504f4..31bfad6df6 100644
--- a/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php
+++ b/src/Standards/Generic/Tests/PHP/LowerCaseKeywordUnitTest.php
@@ -52,6 +52,9 @@ public function getErrorList()
             48 => 1,
             52 => 3,
             54 => 1,
+            57 => 2,
+            58 => 1,
+            60 => 1,
         ];
 
     }//end getErrorList()

From 8da7f11444e2ea69ce0c5fb7b3a5386738c265b4 Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Wed, 23 Oct 2024 04:24:13 +0200
Subject: [PATCH 6/8] Generic/ScopeIndent: add extra test with multi-token
 "yield from"

This confirms that the Tokenizer changes related to `yield from` tokenization which were made in a previous commit also fix issue squizlabs/PHP_CodeSniffer 3808, as reported upstream.

Closes squizlabs/PHP_CodeSniffer 3808

Includes moving the `phpcs:set` directives in test case file 3 to inside the PHP tags to make the tests reproducable when doing a plain `phpcs` run against the file.
---
 .../Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc |  6 ++++++
 .../Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed   |  6 ++++++
 .../Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc |  6 ++++++
 .../Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed   |  6 ++++++
 .../Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc | 10 ++++++++--
 .../Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed   | 10 ++++++++--
 .../Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php   |  9 +++++----
 7 files changed, 45 insertions(+), 8 deletions(-)

diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc
index bab866e00a..74c5c0728f 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc
@@ -1614,6 +1614,12 @@ match (true) {
     ]
 };
 
+// Issue squizlabs/PHP_CodeSniffer#3808
+function test() {
+    yield
+        from [ 3, 4 ];
+}
+
 /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
 ?>
 
diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed
index dbbfa71c40..414ea6f7f1 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.1.inc.fixed
@@ -1614,6 +1614,12 @@ match (true) {
     ]
 };
 
+// Issue squizlabs/PHP_CodeSniffer#3808
+function test() {
+    yield
+        from [ 3, 4 ];
+}
+
 /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
 ?>
 
diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc
index de344f9f6e..c30e5b8ddd 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc
@@ -1614,6 +1614,12 @@ match (true) {
 	]
 };
 
+// Issue squizlabs/PHP_CodeSniffer#3808
+function test() {
+	yield
+		from [ 3, 4 ];
+}
+
 /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
 ?>
 
diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed
index 3cf7fb61a6..4660f75888 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.2.inc.fixed
@@ -1614,6 +1614,12 @@ match (true) {
 	]
 };
 
+// Issue squizlabs/PHP_CodeSniffer#3808
+function test() {
+	yield
+		from [ 3, 4 ];
+}
+
 /* ADD NEW TESTS ABOVE THIS LINE AND MAKE SURE THAT THE 1 (space-based) AND 2 (tab-based) FILES ARE IN SYNC! */
 ?>
 
diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc
index 55d1a06ab8..dd095617c2 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc
@@ -1,6 +1,6 @@
-phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false
-phpcs:set Generic.WhiteSpace.ScopeIndent exact true
 <?php
+// phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false
+// phpcs:set Generic.WhiteSpace.ScopeIndent exact true
 function test()
 {
      echo 'test';
@@ -26,3 +26,9 @@ if ($foo) {
 $this->foo()
     ->bar()
     ->baz();
+
+// Issue squizlabs/PHP_CodeSniffer#3808
+function test() {
+    yield
+        from [ 3, 4 ];
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed
index e9ae5ff392..aaa0b1c819 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.3.inc.fixed
@@ -1,6 +1,6 @@
-phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false
-phpcs:set Generic.WhiteSpace.ScopeIndent exact true
 <?php
+// phpcs:set Generic.WhiteSpace.ScopeIndent tabIndent false
+// phpcs:set Generic.WhiteSpace.ScopeIndent exact true
 function test()
 {
     echo 'test';
@@ -26,3 +26,9 @@ if ($foo) {
 $this->foo()
     ->bar()
     ->baz();
+
+// Issue squizlabs/PHP_CodeSniffer#3808
+function test() {
+    yield
+    from [ 3, 4 ];
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php
index bb1a5d0ed4..fc9f9a8b92 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php
+++ b/src/Standards/Generic/Tests/WhiteSpace/ScopeIndentUnitTest.php
@@ -84,6 +84,7 @@ public function getErrorList($testFile='')
                 6  => 1,
                 7  => 1,
                 10 => 1,
+                33 => 1,
             ];
         }
 
@@ -192,10 +193,10 @@ public function getErrorList($testFile='')
             1527 => 1,
             1529 => 1,
             1530 => 1,
-            1625 => 1,
-            1626 => 1,
-            1627 => 1,
-            1628 => 1,
+            1631 => 1,
+            1632 => 1,
+            1633 => 1,
+            1634 => 1,
         ];
 
     }//end getErrorList()

From 7349295bfd25352fd21369311bac4684a11bf58e Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Thu, 24 Oct 2024 08:04:12 +0200
Subject: [PATCH 7/8] Generic/DisallowTabIndent: improve handling of `yield
 from`

This commit on the one hand confirms (via a test) that indentation for a multi-line `yield from` expression is now handled correctly by the sniff.

On the other hand, this commit enables checking for tabs being used for inline "alignment" between the `yield` and the `from` keyword.
This will now also be handled (and auto-fixed) by this sniff. Includes test.
---
 .../WhiteSpace/DisallowTabIndentSniff.php     |  1 +
 .../DisallowTabIndentUnitTest.1.inc           |  9 +++
 .../DisallowTabIndentUnitTest.1.inc.fixed     |  9 +++
 .../WhiteSpace/DisallowTabIndentUnitTest.php  | 76 ++++++++++---------
 4 files changed, 58 insertions(+), 37 deletions(-)

diff --git a/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php b/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php
index ecf19ca53f..b33a58b461 100644
--- a/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php
+++ b/src/Standards/Generic/Sniffs/WhiteSpace/DisallowTabIndentSniff.php
@@ -78,6 +78,7 @@ public function process(File $phpcsFile, $stackPtr)
             T_COMMENT                => true,
             T_END_HEREDOC            => true,
             T_END_NOWDOC             => true,
+            T_YIELD_FROM             => true,
         ];
 
         for ($i = 0; $i < $phpcsFile->numTokens; $i++) {
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc
index cf61177e88..74fa505119 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc
@@ -91,3 +91,12 @@ $var = "$hello	$there";
 
 	 Another line.
 	 */
+
+// A `yield from` can be single-line and multiline and may contain a tab in the whitespace between the keywords.
+function myGenerator() {
+    yield		from gen1();
+
+    yield
+	from
+    gen2();
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed
index 8154179c98..3425921201 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.1.inc.fixed
@@ -91,3 +91,12 @@ $var = "$hello	$there";
 
      Another line.
      */
+
+// A `yield from` can be single-line and multiline and may contain a tab in the whitespace between the keywords.
+function myGenerator() {
+    yield       from gen1();
+
+    yield
+    from
+    gen2();
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php
index 7a5f15e9f5..9618e55d86 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowTabIndentUnitTest.php
@@ -50,43 +50,45 @@ public function getErrorList($testFile='')
         switch ($testFile) {
         case 'DisallowTabIndentUnitTest.1.inc':
             return [
-                5  => 2,
-                9  => 1,
-                15 => 1,
-                20 => 2,
-                21 => 1,
-                22 => 2,
-                23 => 1,
-                24 => 2,
-                31 => 1,
-                32 => 2,
-                33 => 2,
-                41 => 1,
-                42 => 1,
-                43 => 1,
-                44 => 1,
-                45 => 1,
-                46 => 1,
-                47 => 1,
-                48 => 1,
-                54 => 1,
-                55 => 1,
-                56 => 1,
-                57 => 1,
-                58 => 1,
-                59 => 1,
-                79 => 1,
-                80 => 1,
-                81 => 1,
-                82 => 1,
-                83 => 1,
-                85 => 1,
-                86 => 1,
-                87 => 1,
-                89 => 1,
-                90 => 1,
-                92 => 1,
-                93 => 1,
+                5   => 2,
+                9   => 1,
+                15  => 1,
+                20  => 2,
+                21  => 1,
+                22  => 2,
+                23  => 1,
+                24  => 2,
+                31  => 1,
+                32  => 2,
+                33  => 2,
+                41  => 1,
+                42  => 1,
+                43  => 1,
+                44  => 1,
+                45  => 1,
+                46  => 1,
+                47  => 1,
+                48  => 1,
+                54  => 1,
+                55  => 1,
+                56  => 1,
+                57  => 1,
+                58  => 1,
+                59  => 1,
+                79  => 1,
+                80  => 1,
+                81  => 1,
+                82  => 1,
+                83  => 1,
+                85  => 1,
+                86  => 1,
+                87  => 1,
+                89  => 1,
+                90  => 1,
+                92  => 1,
+                93  => 1,
+                97  => 1,
+                100 => 1,
             ];
 
         case 'DisallowTabIndentUnitTest.2.inc':

From fc6d98a4eb364f262ffe7eb56c68b2949330bd14 Mon Sep 17 00:00:00 2001
From: jrfnl <jrfnl@users.noreply.github.com>
Date: Thu, 4 Jul 2024 23:44:36 +0200
Subject: [PATCH 8/8] Generic/DisallowSpaceIndent: add test with multi-line
 `yield from`

As the indentation whitespace in a multi-line `yield from` expression is now tokenized as `T_WHITESPACE`, instead of being tokenized as part of the (second) `T_YIELD_FROM` token, space indentation of such a line will now be handled correctly by the sniff.

This commit just adds a test to confirm and safeguard this.
---
 .../Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc     | 7 +++++++
 .../WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed     | 7 +++++++
 .../Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc     | 7 +++++++
 .../WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed     | 7 +++++++
 .../Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php       | 1 +
 5 files changed, 29 insertions(+)

diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc
index 1826b585b2..b19a58d874 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc
@@ -116,3 +116,10 @@ $x = 1;
 
      Another line.
      */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+    from
+	gen2();
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed
index 298b3771a5..ca037ee923 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.1.inc.fixed
@@ -116,3 +116,10 @@ $x = 1;
 
 	 Another line.
 	 */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+	from
+	gen2();
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc
index 1826b585b2..b19a58d874 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc
@@ -116,3 +116,10 @@ $x = 1;
 
      Another line.
      */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+    from
+	gen2();
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed
index 298b3771a5..ca037ee923 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.2.inc.fixed
@@ -116,3 +116,10 @@ $x = 1;
 
 	 Another line.
 	 */
+
+// A `yield from` can be multiline and may contain spaces in the indentation whitespace between the keywords.
+function myGenerator() {
+	yield
+	from
+	gen2();
+}
diff --git a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php
index 4a27f8839b..21e89c646f 100644
--- a/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php
+++ b/src/Standards/Generic/Tests/WhiteSpace/DisallowSpaceIndentUnitTest.php
@@ -89,6 +89,7 @@ public function getErrorList($testFile='')
                 115 => 1,
                 117 => 1,
                 118 => 1,
+                123 => 1,
             ];
 
         case 'DisallowSpaceIndentUnitTest.3.inc':