Merge pull request #79 from greg0ire/token-class

doctrine · Dec 9, 2022 · c63c80d · c63c80d
2 parents 0cbb56c + a56013d
commit c63c80d
Show file tree

Hide file tree

Showing 10 changed files with 241 additions and 103 deletions.
diff --git a/UPGRADE.md b/UPGRADE.md
@@ -0,0 +1,14 @@
+Note about upgrading: Doctrine uses static and runtime mechanisms to raise
+awareness about deprecated code.
+
+- Use of `@deprecated` docblock that is detected by IDEs (like PHPStorm) or
+  Static Analysis tools (like Psalm, phpstan)
+- Use of our low-overhead runtime deprecation API, details:
+  https://github.com/doctrine/deprecations/
+
+# Upgrade to 2.0.0
+
+`AbstractLexer::glimpse()` and `AbstractLexer::peek()` now return
+instances of `Doctrine\Common\Lexer\Token`, which is an array-like class
+Using it as an array is deprecated in favor of using properties of that class.
+Using `count()` on it is deprecated with no replacement.
diff --git a/composer.json b/composer.json
@@ -26,7 +26,8 @@
     ],
     "homepage": "https://www.doctrine-project.org/projects/lexer.html",
     "require": {
-        "php": "^7.1 || ^8.0"
+        "php": "^7.1 || ^8.0",
+        "doctrine/deprecations": "^1.0"
     },
     "require-dev": {
         "doctrine/coding-standard": "^9 || ^10",

diff --git a/docs/en/dql-parser.rst b/docs/en/dql-parser.rst
@@ -254,7 +254,7 @@ Lexer implementation:
         {
             $this->lexer->moveNext();
 
-            switch ($this->lexer->lookahead['type']) {
+            switch ($this->lexer->lookahead->type) {
                 case Lexer::T_SELECT:
                     $statement = $this->SelectStatement();
                     break;

diff --git a/docs/en/simple-parser-example.rst b/docs/en/simple-parser-example.rst
@@ -70,8 +70,8 @@ Use ``CharacterTypeLexer`` to extract an array of upper case characters:
 
                 $this->lexer->moveNext();
 
-                if ($this->lexer->token['type'] === CharacterTypeLexer::T_UPPER) {
-                    $upperCaseChars[] = $this->lexer->token['value'];
+                if ($this->lexer->token->isA(CharacterTypeLexer::T_UPPER)) {
+                    $upperCaseChars[] = $this->lexer->token->value;
                 }
             }
 

diff --git a/lib/Doctrine/Common/Lexer/AbstractLexer.php b/lib/Doctrine/Common/Lexer/AbstractLexer.php
@@ -7,7 +7,6 @@
 use ReflectionClass;
 
 use function implode;
-use function in_array;
 use function preg_split;
 use function sprintf;
 use function substr;
@@ -19,7 +18,7 @@
 /**
  * Base class for writing simple lexers, i.e. for creating small DSLs.
  *
- * @psalm-type Token = array{value: int|string, type:string|int|null, position:int}
+ * @template T of string|int
  */
 abstract class AbstractLexer
 {
@@ -33,14 +32,7 @@ abstract class AbstractLexer
     /**
      * Array of scanned tokens.
      *
-     * Each token is an associative array containing three items:
-     *  - 'value'    : the string value of the token in the input string
-     *  - 'type'     : the type of the token (identifier, numeric, string, input
-     *                 parameter, none)
-     *  - 'position' : the position of the token in the input string
-     *
-     * @var mixed[][]
-     * @psalm-var list<Token>
+     * @var list<Token<T>>
      */
     private $tokens = [];
 
@@ -62,15 +54,15 @@ abstract class AbstractLexer
      * The next token in the input.
      *
      * @var mixed[]|null
-     * @psalm-var Token|null
+     * @psalm-var Token<T>|null
      */
     public $lookahead;
 
     /**
      * The last matched/seen token.
      *
      * @var mixed[]|null
-     * @psalm-var Token|null
+     * @psalm-var Token<T>|null
      */
     public $token;
 
@@ -150,25 +142,25 @@ public function getInputUntilPosition($position)
     /**
      * Checks whether a given token matches the current lookahead.
      *
-     * @param int|string $type
+     * @param T $type
      *
      * @return bool
      */
     public function isNextToken($type)
     {
-        return $this->lookahead !== null && $this->lookahead['type'] === $type;
+        return $this->lookahead !== null && $this->lookahead->isA($type);
     }
 
     /**
      * Checks whether any of the given tokens matches the current lookahead.
      *
-     * @param list<int|string> $types
+     * @param list<T> $types
      *
      * @return bool
      */
     public function isNextTokenAny(array $types)
     {
-        return $this->lookahead !== null && in_array($this->lookahead['type'], $types, true);
+        return $this->lookahead !== null && $this->lookahead->isA(...$types);
     }
 
     /**
@@ -195,7 +187,7 @@ public function moveNext()
      */
     public function skipUntil($type)
     {
-        while ($this->lookahead !== null && $this->lookahead['type'] !== $type) {
+        while ($this->lookahead !== null && ! $this->lookahead->isA($type)) {
             $this->moveNext();
         }
     }
@@ -217,7 +209,7 @@ public function isA($value, $token)
      * Moves the lookahead token forward.
      *
      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
-     * @psalm-return Token|null
+     * @psalm-return Token<T>|null
      */
     public function peek()
     {
@@ -232,7 +224,7 @@ public function peek()
      * Peeks at the next token, returns it and immediately resets the peek.
      *
      * @return mixed[]|null The next token or NULL if there are no more tokens ahead.
-     * @psalm-return Token|null
+     * @psalm-return Token<T>|null
      */
     public function glimpse()
     {
@@ -272,11 +264,11 @@ protected function scan($input)
             // Must remain before 'value' assignment since it can change content
             $type = $this->getType($match[0]);
 
-            $this->tokens[] = [
-                'value' => $match[0],
-                'type'  => $type,
-                'position' => $match[1],
-            ];
+            $this->tokens[] = new Token(
+                $match[0],
+                $type,
+                $match[1]
+            );
         }
     }
 
@@ -331,7 +323,7 @@ abstract protected function getNonCatchablePatterns();
      *
      * @param string $value
      *
-     * @return int|string|null
+     * @return T|null
      */
     abstract protected function getType(&$value);
 }
diff --git a/lib/Doctrine/Common/Lexer/Token.php b/lib/Doctrine/Common/Lexer/Token.php
@@ -0,0 +1,124 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Doctrine\Common\Lexer;
+
+use ArrayAccess;
+use Doctrine\Deprecations\Deprecation;
+use ReturnTypeWillChange;
+
+use function in_array;
+
+/**
+ * @template T of string|int
+ * @implements ArrayAccess<string,mixed>
+ */
+final class Token implements ArrayAccess
+{
+    /**
+     * The string value of the token in the input string
+     *
+     * @readonly
+     * @var string|int
+     */
+    public $value;
+
+    /**
+     * The type of the token (identifier, numeric, string, input parameter, none)
+     *
+     * @readonly
+     * @var T|null
+     */
+    public $type;
+
+    /**
+     * The position of the token in the input string
+     *
+     * @readonly
+     * @var int
+     */
+    public $position;
+
+    /**
+     * @param string|int $value
+     * @param T|null     $type
+     */
+    public function __construct($value, $type, int $position)
+    {
+        $this->value    = $value;
+        $this->type     = $type;
+        $this->position = $position;
+    }
+
+    /** @param T ...$types */
+    public function isA(...$types): bool
+    {
+        return in_array($this->type, $types, true);
+    }
+
+    /**
+     * @deprecated Use the value, type or position property instead
+     * {@inheritDoc}
+     */
+    public function offsetExists($offset): bool
+    {
+        Deprecation::trigger(
+            'doctrine/lexer',
+            'https://github.com/doctrine/lexer/pull/79',
+            'Accessing %s properties via ArrayAccess is deprecated, use the value, type or position property instead',
+            self::class
+        );
+
+        return in_array($offset, ['value', 'type', 'position'], true);
+    }
+
+    /**
+     * @deprecated Use the value, type or position property instead
+     * {@inheritDoc}
+     */
+    #[ReturnTypeWillChange]
+    public function offsetGet($offset)
+    {
+        Deprecation::trigger(
+            'doctrine/lexer',
+            'https://github.com/doctrine/lexer/pull/79',
+            'Accessing %s properties via ArrayAccess is deprecated, use the value, type or position property instead',
+            self::class
+        );
+
+        return $this->$offset;
+    }
+
+    /**
+     * @deprecated no replacement planned
+     * {@inheritDoc}
+     */
+    public function offsetSet($offset, $value): void
+    {
+        Deprecation::trigger(
+            'doctrine/lexer',
+            'https://github.com/doctrine/lexer/pull/79',
+            'Setting %s properties via ArrayAccess is deprecated',
+            self::class
+        );
+
+        $this->$offset = $value;
+    }
+
+    /**
+     * @deprecated no replacement planned
+     * {@inheritDoc}
+     */
+    public function offsetUnset($offset): void
+    {
+        Deprecation::trigger(
+            'doctrine/lexer',
+            'https://github.com/doctrine/lexer/pull/79',
+            'Setting %s properties via ArrayAccess is deprecated',
+            self::class
+        );
+
+        $this->$offset = null;
+    }
+}