Refactor TokenList

phpstan-baseline.neon

@@ -1,10 +1,5 @@
 parameters:
 	ignoreErrors:
-		-
-			message: "#^Cannot assign new offset to array\\<PhpMyAdmin\\\\SqlParser\\\\Token\\>\\|string\\.$#"
-			count: 3
-			path: src/Components/AlterOperation.php
-
 		-
 			message: "#^Property PhpMyAdmin\\\\SqlParser\\\\Components\\\\AlterOperation\\:\\:\\$options \\(PhpMyAdmin\\\\SqlParser\\\\Components\\\\OptionsArray\\) does not accept PhpMyAdmin\\\\SqlParser\\\\Components\\\\OptionsArray\\|null\\.$#"
 			count: 1
@@ -15,11 +10,6 @@ parameters:
 			count: 1
 			path: src/Components/AlterOperation.php
 
-		-
-			message: "#^array\\<PhpMyAdmin\\\\SqlParser\\\\Token\\>\\|string does not accept PhpMyAdmin\\\\SqlParser\\\\Token\\.$#"
-			count: 3
-			path: src/Components/AlterOperation.php
-
 		-
 			message: "#^Cannot access an offset on array\\<int\\<0, max\\>, mixed\\>\\|static\\(PhpMyAdmin\\\\SqlParser\\\\Components\\\\ArrayObj\\)\\.$#"
 			count: 1
@@ -500,11 +490,6 @@ parameters:
 			count: 2
 			path: src/Statements/CreateStatement.php
 
-		-
-			message: "#^Cannot assign new offset to array\\<PhpMyAdmin\\\\SqlParser\\\\Token\\>\\|string\\.$#"
-			count: 4
-			path: src/Statements/CreateStatement.php
-
 		-
 			message: "#^Cannot call method build\\(\\) on PhpMyAdmin\\\\SqlParser\\\\Components\\\\DataType\\|null\\.$#"
 			count: 1
@@ -550,11 +535,6 @@ parameters:
 			count: 1
 			path: src/Statements/CreateStatement.php
 
-		-
-			message: "#^array\\<PhpMyAdmin\\\\SqlParser\\\\Token\\>\\|string does not accept PhpMyAdmin\\\\SqlParser\\\\Token\\.$#"
-			count: 4
-			path: src/Statements/CreateStatement.php
-
 		-
 			message: "#^Argument of an invalid type array\\<PhpMyAdmin\\\\SqlParser\\\\Components\\\\Expression\\>\\|null supplied for foreach, only iterables are supported\\.$#"
 			count: 1
@@ -830,26 +810,6 @@ parameters:
 			count: 2
 			path: src/Utils/Table.php
 
-		-
-			message: "#^Binary operation \"&\" between int\\|string and int results in an error\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
-		-
-			message: "#^Cannot cast mixed to string\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
-		-
-			message: "#^Parameter \\#1 \\$string1 of function strcasecmp expects string, int\\|string given\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
-		-
-			message: "#^Parameter \\#2 \\$pattern of static method PhpMyAdmin\\\\SqlParser\\\\Utils\\\\Tokens\\:\\:match\\(\\) expects array\\<string, int\\|string\\|null\\>, PhpMyAdmin\\\\SqlParser\\\\Token given\\.$#"
-			count: 1
-			path: src/Utils/Tokens.php
-
 		-
 			message: "#^Cannot call method __toString\\(\\) on PhpMyAdmin\\\\SqlParser\\\\Components\\\\Expression\\|null\\.$#"
 			count: 2
@@ -960,11 +920,6 @@ parameters:
 			count: 1
 			path: tests/Utils/TableTest.php
 
-		-
-			message: "#^Parameter \\#2 \\$find of static method PhpMyAdmin\\\\SqlParser\\\\Utils\\\\Tokens\\:\\:replaceTokens\\(\\) expects array\\<PhpMyAdmin\\\\SqlParser\\\\Token\\>, array\\<array\\<string, string\\>\\> given\\.$#"
-			count: 1
-			path: tests/Utils/TokensTest.php
-
 		-
 			message: "#^Expression \"\\$str1\\[\\$i\\]\" on a separate line does not do anything\\.$#"
 			count: 1

psalm-baseline.xml

@@ -1060,11 +1060,6 @@
       <code><![CDATA[$this->withers]]></code>
     </PossiblyNullArrayOffset>
   </file>
-  <file src="src/TokensList.php">
-    <RedundantConditionGivenDocblockType>
-      <code>is_array($list)</code>
-    </RedundantConditionGivenDocblockType>
-  </file>
   <file src="src/Tools/ContextGenerator.php">
     <PossiblyNullArgument>
       <code><![CDATA[$options['keywords']]]></code>
@@ -1325,46 +1320,6 @@
       <code>has</code>
     </PossiblyNullReference>
   </file>
-  <file src="src/Utils/Tokens.php">
-    <InvalidArgument>
-      <code>$find[$k]</code>
-    </InvalidArgument>
-    <MixedArgument>
-      <code><![CDATA[$list->tokens[$j]]]></code>
-    </MixedArgument>
-    <MixedArgumentTypeCoercion>
-      <code>$newList</code>
-      <code>$newList</code>
-    </MixedArgumentTypeCoercion>
-    <MixedArrayAccess>
-      <code><![CDATA[$list->tokens[$i]]]></code>
-      <code><![CDATA[$list->tokens[$j]]]></code>
-      <code><![CDATA[$list->tokens[$j]]]></code>
-    </MixedArrayAccess>
-    <MixedAssignment>
-      <code>$newList[]</code>
-      <code>$newList[]</code>
-    </MixedAssignment>
-    <MixedPropertyFetch>
-      <code><![CDATA[$list->tokens[$i]->type]]></code>
-      <code><![CDATA[$list->tokens[$j]->type]]></code>
-    </MixedPropertyFetch>
-    <PossiblyInvalidArgument>
-      <code>$list</code>
-      <code><![CDATA[$pattern['value_str']]]></code>
-    </PossiblyInvalidArgument>
-    <PossiblyInvalidOperand>
-      <code><![CDATA[$pattern['flags']]]></code>
-    </PossiblyInvalidOperand>
-    <PossiblyInvalidPropertyFetch>
-      <code><![CDATA[$list->count]]></code>
-      <code><![CDATA[$list->tokens]]></code>
-    </PossiblyInvalidPropertyFetch>
-    <UndefinedPropertyFetch>
-      <code><![CDATA[$list->count]]></code>
-      <code><![CDATA[$list->tokens]]></code>
-    </UndefinedPropertyFetch>
-  </file>
   <file src="tests/Builder/AlterStatementTest.php">
     <PossiblyUnusedMethod>
       <code>provideBuilderForRenameColumn</code>
@@ -1803,15 +1758,6 @@
       <code>getForeignKeysProvider</code>
     </PossiblyUnusedMethod>
   </file>
-  <file src="tests/Utils/TokensTest.php">
-    <InvalidArgument>
-      <code>$find</code>
-    </InvalidArgument>
-    <PossiblyUnusedMethod>
-      <code>matchProvider</code>
-      <code>replaceTokensProvider</code>
-    </PossiblyUnusedMethod>
-  </file>
   <file src="tests/benchmarks/UtfStringBench.php">
     <MissingConstructor>
       <code>$testContents</code>

src/Components/AlterOperation.php

@@ -256,13 +256,6 @@ final class AlterOperation implements Component
      */
     public $partitions;
 
-    /**
-     * Unparsed tokens.
-     *
-     * @var Token[]|string
-     */
-    public $unknown = [];
-
     /**
      * @param OptionsArray               $options    options of alter operation
      * @param Expression|string|null     $field      altered field
@@ -273,7 +266,7 @@ public function __construct(
         $options = null,
         $field = null,
         $partitions = null,
-        $unknown = []
+        public array $unknown = []
     ) {
         $this->partitions = $partitions;
         $this->options = $options;
@@ -525,7 +518,7 @@ public function build(): string
             $ret .= $this->field . ' ';
         }
 
-        $ret .= $afterFieldsOptions . TokensList::build($this->unknown);
+        $ret .= $afterFieldsOptions . TokensList::buildFromArray($this->unknown);
 
         if (isset($this->partitions)) {
             $ret .= PartitionDefinition::buildAll($this->partitions);

src/Lexer.php

@@ -144,21 +144,6 @@ class Lexer extends Core
      */
     public $delimiterLen;
 
-    /**
-     * Gets the tokens list parsed by a new instance of a lexer.
-     *
-     * @param string|UtfString $str       the query to be lexed
-     * @param bool             $strict    whether strict mode should be
-     *                                    enabled or not
-     * @param string           $delimiter the delimiter to be used
-     */
-    public static function getTokens($str, $strict = false, $delimiter = null): TokensList
-    {
-        $lexer = new self($str, $strict, $delimiter);
-
-        return $lexer->list;
-    }
-
     /**
      * @param string|UtfString $str       the query to be lexed
      * @param bool             $strict    whether strict mode should be

src/Statements/CreateStatement.php

@@ -390,9 +390,9 @@ class CreateStatement extends Statement
      * For views, it is the select statement that creates the view.
      * Used by `CREATE FUNCTION`, `CREATE PROCEDURE` and `CREATE VIEW`.
      *
-     * @var Token[]|string
+     * @var Token[]
      */
-    public $body = [];
+    public array $body = [];
 
     public function build(): string
     {
@@ -476,7 +476,7 @@ public function build(): string
                 . $this->options->build() . ' '
                 . $this->name->build() . ' '
                 . $fields . ' AS ' . $builtStatement
-                . (! empty($this->body) ? TokensList::build($this->body) : '') . ' '
+                . TokensList::buildFromArray($this->body) . ' '
                 . ($this->entityOptions?->build() ?? '');
         }
 
@@ -486,7 +486,7 @@ public function build(): string
                 . $this->name->build() . ' '
                 . $this->entityOptions->build() . ' '
                 . 'ON ' . $this->table->build() . ' '
-                . 'FOR EACH ROW ' . TokensList::build($this->body);
+                . 'FOR EACH ROW ' . TokensList::buildFromArray($this->body);
         }
 
         if ($this->options->has('PROCEDURE') || $this->options->has('FUNCTION')) {
@@ -500,13 +500,13 @@ public function build(): string
                 . $this->name->build() . ' '
                 . ParameterDefinition::buildAll($this->parameters) . ' '
                 . $tmp . ' ' . $this->entityOptions->build() . ' '
-                . TokensList::build($this->body);
+                . TokensList::buildFromArray($this->body);
         }
 
         return 'CREATE '
             . $this->options->build() . ' '
             . $this->name->build() . ' '
-            . TokensList::build($this->body);
+            . TokensList::buildFromArray($this->body);
     }
 
     /**

src/Statements/WithStatement.php

@@ -330,6 +330,6 @@ private function getSubTokenList(TokensList $list): ParserException|TokensList
 
         $length = $list->idx - $idx;
 
-        return new TokensList(array_slice($list->tokens, $idx, $length), $length);
+        return new TokensList(array_slice($list->tokens, $idx, $length));
     }
 }

src/TokensList.php

@@ -9,7 +9,6 @@
 use function count;
 use function in_array;
 use function is_array;
-use function is_string;
 
 /**
  * Defines an array of tokens and utility functions to iterate through it.
@@ -20,13 +19,6 @@
  */
 class TokensList implements ArrayAccess
 {
-    /**
-     * The array of tokens.
-     *
-     * @var Token[]
-     */
-    public $tokens = [];
-
     /**
      * The count of tokens.
      *
@@ -42,39 +34,31 @@ class TokensList implements ArrayAccess
     public $idx = 0;
 
     /**
-     * @param Token[] $tokens the initial array of tokens
-     * @param int     $count  the count of tokens in the initial array
+     * @param Token[] $tokens The array of tokens.
      */
-    public function __construct(array $tokens = [], $count = -1)
+    public function __construct(public array $tokens = [])
     {
-        if ($tokens === []) {
-            return;
-        }
+        $this->count = count($tokens);
+    }
 
-        $this->tokens = $tokens;
-        $this->count = $count === -1 ? count($tokens) : $count;
+    /**
+     * Builds an array of tokens by merging their raw value.
+     */
+    public function build(): string
+    {
+        return static::buildFromArray($this->tokens);
     }
 
     /**
      * Builds an array of tokens by merging their raw value.
      *
-     * @param string|Token[]|TokensList $list the tokens to be built
+     * @param Token[] $list the tokens to be built
      */
-    public static function build($list): string
+    public static function buildFromArray(array $list): string
     {
-        if (is_string($list)) {
-            return $list;
-        }
-
-        if ($list instanceof self) {
-            $list = $list->tokens;
-        }
-
         $ret = '';
-        if (is_array($list)) {
-            foreach ($list as $tok) {
-                $ret .= $tok->token;
-            }
+        foreach ($list as $token) {
+            $ret .= $token->token;
         }
 
         return $ret;