Skip to content

Introduce the TokenType enum #507

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
MauricioFauth merged 1 commit into from
Sep 26, 2023
Merged

Introduce the TokenType enum #507

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
2 changes: 1 addition & 1 deletion composer.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,7 @@
"phpunit/phpunit": "^10.0",
"psalm/plugin-phpunit": "^0.18.4",
"vimeo/psalm": "^5.7",
"zumba/json-serializer": "^3.0"
"zumba/json-serializer": "^3.2"
},
"conflict": {
"phpmyadmin/motranslator": "<5.2"
Expand Down
2 changes: 1 addition & 1 deletion psalm-baseline.xml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1260,7 +1260,7 @@
<code>$i</code>
</PossiblyNullOperand>
<PossiblyNullPropertyFetch>
<code><![CDATA[$lexer->list->getNextOfType(Token::TYPE_KEYWORD)->keyword]]></code>
<code><![CDATA[$lexer->list->getNextOfType(TokenType::Keyword)->keyword]]></code>
<code><![CDATA[$statement->into->dest]]></code>
</PossiblyNullPropertyFetch>
<PossiblyNullReference>
Expand Down
25 changes: 13 additions & 12 deletions src/Components/AlterOperation.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
use PhpMyAdmin\SqlParser\Parser;
use PhpMyAdmin\SqlParser\Token;
use PhpMyAdmin\SqlParser\TokensList;
use PhpMyAdmin\SqlParser\TokenType;

use function array_key_exists;
use function in_array;
Expand Down Expand Up @@ -321,17 +322,17 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$token = $list->tokens[$list->idx];

// End of statement.
if ($token->type === Token::TYPE_DELIMITER) {
if ($token->type === TokenType::Delimiter) {
break;
}

// Skipping comments.
if ($token->type === Token::TYPE_COMMENT) {
if ($token->type === TokenType::Comment) {
continue;
}

// Skipping whitespaces.
if ($token->type === Token::TYPE_WHITESPACE) {
if ($token->type === TokenType::Whitespace) {
if ($state === 2) {
// When parsing the unknown part, the whitespaces are
// included to not break anything.
Expand All @@ -347,7 +348,7 @@ public static function parse(Parser $parser, TokensList $list, array $options =
// body in the unknown tokens list, as they define their own statements.
if ($ret->options->has('AS') || $ret->options->has('DO')) {
for (; $list->idx < $list->count; ++$list->idx) {
if ($list->tokens[$list->idx]->type === Token::TYPE_DELIMITER) {
if ($list->tokens[$list->idx]->type === TokenType::Delimiter) {
break;
}

Expand Down Expand Up @@ -392,22 +393,22 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$arrayKey = $token->token;
}

if ($token->type === Token::TYPE_OPERATOR) {
if ($token->type === TokenType::Operator) {
if ($token->value === '(') {
++$brackets;
} elseif ($token->value === ')') {
--$brackets;
} elseif (($token->value === ',') && ($brackets === 0)) {
break;
}
} elseif (! self::checkIfTokenQuotedSymbol($token) && $token->type !== Token::TYPE_STRING) {
} elseif (! self::checkIfTokenQuotedSymbol($token) && $token->type !== TokenType::String) {
if (isset(Parser::STATEMENT_PARSERS[$arrayKey]) && Parser::STATEMENT_PARSERS[$arrayKey] !== '') {
$list->idx++; // Ignore the current token
$nextToken = $list->getNext();

if ($token->value === 'SET' && $nextToken !== null && $nextToken->value === '(') {
// To avoid adding the tokens between the SET() parentheses to the unknown tokens
$list->getNextOfTypeAndValue(Token::TYPE_OPERATOR, ')');
$list->getNextOfTypeAndValue(TokenType::Operator, ')');
} elseif ($token->value === 'SET' && $nextToken !== null && $nextToken->value === 'DEFAULT') {
// to avoid adding the `DEFAULT` token to the unknown tokens.
++$list->idx;
Expand Down Expand Up @@ -438,12 +439,12 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$list->idx++; // Ignore the current token
$nextToken = $list->getNext();
if (
($token->type === Token::TYPE_KEYWORD)
($token->type === TokenType::Keyword)
&& (($token->keyword === 'PARTITION BY')
|| ($token->keyword === 'PARTITION' && $nextToken && $nextToken->value !== '('))
) {
$partitionState = 1;
} elseif (($token->type === Token::TYPE_KEYWORD) && ($token->keyword === 'PARTITION')) {
} elseif (($token->type === TokenType::Keyword) && ($token->keyword === 'PARTITION')) {
$partitionState = 2;
}

Expand All @@ -466,15 +467,15 @@ public static function parse(Parser $parser, TokensList $list, array $options =
}

if (
$token->type === Token::TYPE_OPERATOR
$token->type === TokenType::Operator
&& $token->value === '('
&& $nextToken
&& $nextToken->keyword === 'PARTITION'
) {
$partitionState = 2;
--$list->idx; // Current idx is on "(". We need a step back for ArrayObj::parse incoming.
} else {
$ret->field .= $token->type === Token::TYPE_WHITESPACE ? ' ' : $token->token;
$ret->field .= $token->type === TokenType::Whitespace ? ' ' : $token->token;
}
} elseif ($partitionState === 2) {
$ret->partitions = ArrayObj::parse(
Expand Down Expand Up @@ -552,7 +553,7 @@ private static function checkIfColumnDefinitionKeyword($tokenValue): bool
*/
private static function checkIfTokenQuotedSymbol($token): bool
{
return $token->type === Token::TYPE_SYMBOL && $token->flags === Token::FLAG_SYMBOL_BACKTICK;
return $token->type === TokenType::Symbol && $token->flags === Token::FLAG_SYMBOL_BACKTICK;
}

public function __toString(): string
Expand Down
7 changes: 4 additions & 3 deletions src/Components/Array2d.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
use PhpMyAdmin\SqlParser\Parser;
use PhpMyAdmin\SqlParser\Token;
use PhpMyAdmin\SqlParser\TokensList;
use PhpMyAdmin\SqlParser\TokenType;
use PhpMyAdmin\SqlParser\Translator;
use RuntimeException;

Expand Down Expand Up @@ -58,17 +59,17 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$token = $list->tokens[$list->idx];

// End of statement.
if ($token->type === Token::TYPE_DELIMITER) {
if ($token->type === TokenType::Delimiter) {
break;
}

// Skipping whitespaces and comments.
if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
if (($token->type === TokenType::Whitespace) || ($token->type === TokenType::Comment)) {
continue;
}

// No keyword is expected.
if (($token->type === Token::TYPE_KEYWORD) && ($token->flags & Token::FLAG_KEYWORD_RESERVED)) {
if (($token->type === TokenType::Keyword) && ($token->flags & Token::FLAG_KEYWORD_RESERVED)) {
break;
}

Expand Down
10 changes: 5 additions & 5 deletions src/Components/ArrayObj.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,8 +6,8 @@

use PhpMyAdmin\SqlParser\Component;
use PhpMyAdmin\SqlParser\Parser;
use PhpMyAdmin\SqlParser\Token;
use PhpMyAdmin\SqlParser\TokensList;
use PhpMyAdmin\SqlParser\TokenType;

use function implode;
use function strlen;
Expand Down Expand Up @@ -88,7 +88,7 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$token = $list->tokens[$list->idx];

// End of statement.
if ($token->type === Token::TYPE_DELIMITER) {
if ($token->type === TokenType::Delimiter) {
if ($brackets > 0) {
$parser->error('A closing bracket was expected.', $token);
}
Expand All @@ -97,18 +97,18 @@ public static function parse(Parser $parser, TokensList $list, array $options =
}

// Skipping whitespaces and comments.
if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
if (($token->type === TokenType::Whitespace) || ($token->type === TokenType::Comment)) {
$lastRaw .= $token->token;
$lastValue = trim($lastValue) . ' ';
continue;
}

if (($brackets === 0) && (($token->type !== Token::TYPE_OPERATOR) || ($token->value !== '('))) {
if (($brackets === 0) && (($token->type !== TokenType::Operator) || ($token->value !== '('))) {
$parser->error('An opening bracket was expected.', $token);
break;
}

if ($token->type === Token::TYPE_OPERATOR) {
if ($token->type === TokenType::Operator) {
if ($token->value === '(') {
if (++$brackets === 1) { // 1 is the base level.
continue;
Expand Down
29 changes: 15 additions & 14 deletions src/Components/CaseExpression.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
use PhpMyAdmin\SqlParser\Parser;
use PhpMyAdmin\SqlParser\Token;
use PhpMyAdmin\SqlParser\TokensList;
use PhpMyAdmin\SqlParser\TokenType;

use function count;

Expand Down Expand Up @@ -98,12 +99,12 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$token = $list->tokens[$list->idx];

// Skipping whitespaces and comments.
if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
if (($token->type === TokenType::Whitespace) || ($token->type === TokenType::Comment)) {
continue;
}

if ($state === 0) {
if ($token->type === Token::TYPE_KEYWORD) {
if ($token->type === TokenType::Keyword) {
switch ($token->keyword) {
case 'WHEN':
++$list->idx; // Skip 'WHEN'
Expand Down Expand Up @@ -132,7 +133,7 @@ public static function parse(Parser $parser, TokensList $list, array $options =
}
} elseif ($state === 1) {
if ($type === 0) {
if ($token->type === Token::TYPE_KEYWORD) {
if ($token->type === TokenType::Keyword) {
switch ($token->keyword) {
case 'WHEN':
++$list->idx; // Skip 'WHEN'
Expand All @@ -154,23 +155,23 @@ public static function parse(Parser $parser, TokensList $list, array $options =
break 2;
}
}
} elseif ($token->type === Token::TYPE_KEYWORD && $token->keyword === 'THEN') {
} elseif ($token->type === TokenType::Keyword && $token->keyword === 'THEN') {
++$list->idx; // Skip 'THEN'
$newResult = Expression::parse($parser, $list);
$state = 0;
$ret->results[] = $newResult;
} elseif ($token->type === Token::TYPE_KEYWORD) {
} elseif ($token->type === TokenType::Keyword) {
$parser->error('Unexpected keyword.', $token);
break;
}
} elseif ($state === 2) {
if ($type === 0) {
if ($token->type === Token::TYPE_KEYWORD && $token->keyword === 'THEN') {
if ($token->type === TokenType::Keyword && $token->keyword === 'THEN') {
++$list->idx; // Skip 'THEN'
$newResult = Expression::parse($parser, $list);
$ret->results[] = $newResult;
$state = 1;
} elseif ($token->type === Token::TYPE_KEYWORD) {
} elseif ($token->type === TokenType::Keyword) {
$parser->error('Unexpected keyword.', $token);
break;
}
Expand All @@ -187,17 +188,17 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$token = $list->tokens[$list->idx];

// End of statement.
if ($token->type === Token::TYPE_DELIMITER) {
if ($token->type === TokenType::Delimiter) {
break;
}

// Skipping whitespaces and comments.
if (($token->type === Token::TYPE_WHITESPACE) || ($token->type === Token::TYPE_COMMENT)) {
if (($token->type === TokenType::Whitespace) || ($token->type === TokenType::Comment)) {
continue;
}

// Handle optional AS keyword before alias
if ($token->type === Token::TYPE_KEYWORD && $token->keyword === 'AS') {
if ($token->type === TokenType::Keyword && $token->keyword === 'AS') {
if ($asFound || ! empty($ret->alias)) {
$parser->error('Potential duplicate alias of CASE expression.', $token);
break;
Expand All @@ -209,7 +210,7 @@ public static function parse(Parser $parser, TokensList $list, array $options =

if (
$asFound
&& $token->type === Token::TYPE_KEYWORD
&& $token->type === TokenType::Keyword
&& ($token->flags & Token::FLAG_KEYWORD_RESERVED || $token->flags & Token::FLAG_KEYWORD_FUNCTION)
) {
$parser->error('An alias expected after AS but got ' . $token->value, $token);
Expand All @@ -219,9 +220,9 @@ public static function parse(Parser $parser, TokensList $list, array $options =

if (
$asFound
|| $token->type === Token::TYPE_STRING
|| ($token->type === Token::TYPE_SYMBOL && ! $token->flags & Token::FLAG_SYMBOL_VARIABLE)
|| $token->type === Token::TYPE_NONE
|| $token->type === TokenType::String
|| ($token->type === TokenType::Symbol && ! $token->flags & Token::FLAG_SYMBOL_VARIABLE)
|| $token->type === TokenType::None
) {
// An alias is expected (the keyword `AS` was previously found).
if (! empty($ret->alias)) {
Expand Down
19 changes: 10 additions & 9 deletions src/Components/Condition.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
use PhpMyAdmin\SqlParser\Parser;
use PhpMyAdmin\SqlParser\Token;
use PhpMyAdmin\SqlParser\TokensList;
use PhpMyAdmin\SqlParser\TokenType;

use function implode;
use function in_array;
Expand Down Expand Up @@ -122,18 +123,18 @@ public static function parse(Parser $parser, TokensList $list, array $options =
$token = $list->tokens[$list->idx];

// End of statement.
if ($token->type === Token::TYPE_DELIMITER) {
if ($token->type === TokenType::Delimiter) {
break;
}

// Skipping whitespaces and comments.
if ($token->type === Token::TYPE_COMMENT) {
if ($token->type === TokenType::Comment) {
continue;
}

// Replacing all whitespaces (new lines, tabs, etc.) with a single
// space character.
if ($token->type === Token::TYPE_WHITESPACE) {
if ($token->type === TokenType::Whitespace) {
$expr->expr .= ' ';
continue;
}
Expand Down Expand Up @@ -162,7 +163,7 @@ public static function parse(Parser $parser, TokensList $list, array $options =
}

if (
($token->type === Token::TYPE_KEYWORD)
($token->type === TokenType::Keyword)
&& ($token->flags & Token::FLAG_KEYWORD_RESERVED)
&& ! ($token->flags & Token::FLAG_KEYWORD_FUNCTION)
) {
Expand All @@ -175,7 +176,7 @@ public static function parse(Parser $parser, TokensList $list, array $options =
}
}

if ($token->type === Token::TYPE_OPERATOR) {
if ($token->type === TokenType::Operator) {
if ($token->value === '(') {
++$brackets;
} elseif ($token->value === ')') {
Expand All @@ -189,11 +190,11 @@ public static function parse(Parser $parser, TokensList $list, array $options =

$expr->expr .= $token->token;
if (
($token->type !== Token::TYPE_NONE)
&& (($token->type !== Token::TYPE_KEYWORD)
($token->type !== TokenType::None)
&& (($token->type !== TokenType::Keyword)
|| ($token->flags & Token::FLAG_KEYWORD_RESERVED))
&& ($token->type !== Token::TYPE_STRING)
&& ($token->type !== Token::TYPE_SYMBOL)
&& ($token->type !== TokenType::String)
&& ($token->type !== TokenType::Symbol)
) {
continue;
}
Expand Down
Loading