first commit

vendor/nikic/php-parser/lib/PhpParser/Lexer/Emulative.php

@@ -0,0 +1,226 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer;
+
+use PhpParser\Error;
+use PhpParser\ErrorHandler;
+use PhpParser\Lexer;
+use PhpParser\Lexer\TokenEmulator\AttributeEmulator;
+use PhpParser\Lexer\TokenEmulator\EnumTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\CoaleseEqualTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ExplicitOctalEmulator;
+use PhpParser\Lexer\TokenEmulator\FlexibleDocStringEmulator;
+use PhpParser\Lexer\TokenEmulator\FnTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\MatchTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NullsafeTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\NumericLiteralSeparatorEmulator;
+use PhpParser\Lexer\TokenEmulator\ReadonlyFunctionTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ReadonlyTokenEmulator;
+use PhpParser\Lexer\TokenEmulator\ReverseEmulator;
+use PhpParser\Lexer\TokenEmulator\TokenEmulator;
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+class Emulative extends Lexer {
+    /** @var array{int, string, string}[] Patches used to reverse changes introduced in the code */
+    private array $patches = [];
+
+    /** @var list<TokenEmulator> */
+    private array $emulators = [];
+
+    private PhpVersion $targetPhpVersion;
+
+    private PhpVersion $hostPhpVersion;
+
+    /**
+     * @param PhpVersion|null $phpVersion PHP version to emulate. Defaults to newest supported.
+     */
+    public function __construct(?PhpVersion $phpVersion = null) {
+        $this->targetPhpVersion = $phpVersion ?? PhpVersion::getNewestSupported();
+        $this->hostPhpVersion = PhpVersion::getHostVersion();
+
+        $emulators = [
+            new MatchTokenEmulator(),
+            new NullsafeTokenEmulator(),
+            new AttributeEmulator(),
+            new EnumTokenEmulator(),
+            new ReadonlyTokenEmulator(),
+            new ExplicitOctalEmulator(),
+            new ReadonlyFunctionTokenEmulator(),
+        ];
+
+        // Collect emulators that are relevant for the PHP version we're running
+        // and the PHP version we're targeting for emulation.
+        foreach ($emulators as $emulator) {
+            $emulatorPhpVersion = $emulator->getPhpVersion();
+            if ($this->isForwardEmulationNeeded($emulatorPhpVersion)) {
+                $this->emulators[] = $emulator;
+            } elseif ($this->isReverseEmulationNeeded($emulatorPhpVersion)) {
+                $this->emulators[] = new ReverseEmulator($emulator);
+            }
+        }
+    }
+
+    public function tokenize(string $code, ?ErrorHandler $errorHandler = null): array {
+        $emulators = array_filter($this->emulators, function ($emulator) use ($code) {
+            return $emulator->isEmulationNeeded($code);
+        });
+
+        if (empty($emulators)) {
+            // Nothing to emulate, yay
+            return parent::tokenize($code, $errorHandler);
+        }
+
+        if ($errorHandler === null) {
+            $errorHandler = new ErrorHandler\Throwing();
+        }
+
+        $this->patches = [];
+        foreach ($emulators as $emulator) {
+            $code = $emulator->preprocessCode($code, $this->patches);
+        }
+
+        $collector = new ErrorHandler\Collecting();
+        $tokens = parent::tokenize($code, $collector);
+        $this->sortPatches();
+        $tokens = $this->fixupTokens($tokens);
+
+        $errors = $collector->getErrors();
+        if (!empty($errors)) {
+            $this->fixupErrors($errors);
+            foreach ($errors as $error) {
+                $errorHandler->handleError($error);
+            }
+        }
+
+        foreach ($emulators as $emulator) {
+            $tokens = $emulator->emulate($code, $tokens);
+        }
+
+        return $tokens;
+    }
+
+    private function isForwardEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
+        return $this->hostPhpVersion->older($emulatorPhpVersion)
+            && $this->targetPhpVersion->newerOrEqual($emulatorPhpVersion);
+    }
+
+    private function isReverseEmulationNeeded(PhpVersion $emulatorPhpVersion): bool {
+        return $this->hostPhpVersion->newerOrEqual($emulatorPhpVersion)
+            && $this->targetPhpVersion->older($emulatorPhpVersion);
+    }
+
+    private function sortPatches(): void {
+        // Patches may be contributed by different emulators.
+        // Make sure they are sorted by increasing patch position.
+        usort($this->patches, function ($p1, $p2) {
+            return $p1[0] <=> $p2[0];
+        });
+    }
+
+    /**
+     * @param list<Token> $tokens
+     * @return list<Token>
+     */
+    private function fixupTokens(array $tokens): array {
+        if (\count($this->patches) === 0) {
+            return $tokens;
+        }
+
+        // Load first patch
+        $patchIdx = 0;
+        list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+
+        // We use a manual loop over the tokens, because we modify the array on the fly
+        $posDelta = 0;
+        $lineDelta = 0;
+        for ($i = 0, $c = \count($tokens); $i < $c; $i++) {
+            $token = $tokens[$i];
+            $pos = $token->pos;
+            $token->pos += $posDelta;
+            $token->line += $lineDelta;
+            $localPosDelta = 0;
+            $len = \strlen($token->text);
+            while ($patchPos >= $pos && $patchPos < $pos + $len) {
+                $patchTextLen = \strlen($patchText);
+                if ($patchType === 'remove') {
+                    if ($patchPos === $pos && $patchTextLen === $len) {
+                        // Remove token entirely
+                        array_splice($tokens, $i, 1, []);
+                        $i--;
+                        $c--;
+                    } else {
+                        // Remove from token string
+                        $token->text = substr_replace(
+                            $token->text, '', $patchPos - $pos + $localPosDelta, $patchTextLen
+                        );
+                        $localPosDelta -= $patchTextLen;
+                    }
+                    $lineDelta -= \substr_count($patchText, "\n");
+                } elseif ($patchType === 'add') {
+                    // Insert into the token string
+                    $token->text = substr_replace(
+                        $token->text, $patchText, $patchPos - $pos + $localPosDelta, 0
+                    );
+                    $localPosDelta += $patchTextLen;
+                    $lineDelta += \substr_count($patchText, "\n");
+                } elseif ($patchType === 'replace') {
+                    // Replace inside the token string
+                    $token->text = substr_replace(
+                        $token->text, $patchText, $patchPos - $pos + $localPosDelta, $patchTextLen
+                    );
+                } else {
+                    assert(false);
+                }
+
+                // Fetch the next patch
+                $patchIdx++;
+                if ($patchIdx >= \count($this->patches)) {
+                    // No more patches. However, we still need to adjust position.
+                    $patchPos = \PHP_INT_MAX;
+                    break;
+                }
+
+                list($patchPos, $patchType, $patchText) = $this->patches[$patchIdx];
+            }
+
+            $posDelta += $localPosDelta;
+        }
+        return $tokens;
+    }
+
+    /**
+     * Fixup line and position information in errors.
+     *
+     * @param Error[] $errors
+     */
+    private function fixupErrors(array $errors): void {
+        foreach ($errors as $error) {
+            $attrs = $error->getAttributes();
+
+            $posDelta = 0;
+            $lineDelta = 0;
+            foreach ($this->patches as $patch) {
+                list($patchPos, $patchType, $patchText) = $patch;
+                if ($patchPos >= $attrs['startFilePos']) {
+                    // No longer relevant
+                    break;
+                }
+
+                if ($patchType === 'add') {
+                    $posDelta += strlen($patchText);
+                    $lineDelta += substr_count($patchText, "\n");
+                } elseif ($patchType === 'remove') {
+                    $posDelta -= strlen($patchText);
+                    $lineDelta -= substr_count($patchText, "\n");
+                }
+            }
+
+            $attrs['startFilePos'] += $posDelta;
+            $attrs['endFilePos'] += $posDelta;
+            $attrs['startLine'] += $lineDelta;
+            $attrs['endLine'] += $lineDelta;
+            $error->setAttributes($attrs);
+        }
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/AttributeEmulator.php

@@ -0,0 +1,49 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+final class AttributeEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 0);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '#[') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way.
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if ($token->text === '#' && isset($tokens[$i + 1]) && $tokens[$i + 1]->text === '[') {
+                array_splice($tokens, $i, 2, [
+                    new Token(\T_ATTRIBUTE, '#[', $token->line, $token->pos),
+                ]);
+                $c--;
+                continue;
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // TODO
+        return $tokens;
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        $pos = 0;
+        while (false !== $pos = strpos($code, '#[', $pos)) {
+            // Replace #[ with %[
+            $code[$pos] = '%';
+            $patches[] = [$pos, 'replace', '#'];
+            $pos += 2;
+        }
+        return $code;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/EnumTokenEmulator.php

@@ -0,0 +1,26 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class EnumTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 1);
+    }
+
+    public function getKeywordString(): string {
+        return 'enum';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_ENUM;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool {
+        return parent::isKeywordContext($tokens, $pos)
+            && isset($tokens[$pos + 2])
+            && $tokens[$pos + 1]->id === \T_WHITESPACE
+            && $tokens[$pos + 2]->id === \T_STRING;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/ExplicitOctalEmulator.php

@@ -0,0 +1,45 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+class ExplicitOctalEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 1);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '0o') !== false || strpos($code, '0O') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if ($token->id == \T_LNUMBER && $token->text === '0' &&
+                isset($tokens[$i + 1]) && $tokens[$i + 1]->id == \T_STRING &&
+                preg_match('/[oO][0-7]+(?:_[0-7]+)*/', $tokens[$i + 1]->text)
+            ) {
+                $tokenKind = $this->resolveIntegerOrFloatToken($tokens[$i + 1]->text);
+                array_splice($tokens, $i, 2, [
+                    new Token($tokenKind, '0' . $tokens[$i + 1]->text, $token->line, $token->pos),
+                ]);
+                $c--;
+            }
+        }
+        return $tokens;
+    }
+
+    private function resolveIntegerOrFloatToken(string $str): int {
+        $str = substr($str, 1);
+        $str = str_replace('_', '', $str);
+        $num = octdec($str);
+        return is_float($num) ? \T_DNUMBER : \T_LNUMBER;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Explicit octals were not legal code previously, don't bother.
+        return $tokens;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/KeywordEmulator.php

@@ -0,0 +1,56 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\Token;
+
+abstract class KeywordEmulator extends TokenEmulator {
+    abstract public function getKeywordString(): string;
+    abstract public function getKeywordToken(): int;
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos(strtolower($code), $this->getKeywordString()) !== false;
+    }
+
+    /** @param Token[] $tokens */
+    protected function isKeywordContext(array $tokens, int $pos): bool {
+        $previousNonSpaceToken = $this->getPreviousNonSpaceToken($tokens, $pos);
+        return $previousNonSpaceToken === null || $previousNonSpaceToken->id !== \T_OBJECT_OPERATOR;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        $keywordString = $this->getKeywordString();
+        foreach ($tokens as $i => $token) {
+            if ($token->id === T_STRING && strtolower($token->text) === $keywordString
+                    && $this->isKeywordContext($tokens, $i)) {
+                $token->id = $this->getKeywordToken();
+            }
+        }
+
+        return $tokens;
+    }
+
+    /** @param Token[] $tokens */
+    private function getPreviousNonSpaceToken(array $tokens, int $start): ?Token {
+        for ($i = $start - 1; $i >= 0; --$i) {
+            if ($tokens[$i]->id === T_WHITESPACE) {
+                continue;
+            }
+
+            return $tokens[$i];
+        }
+
+        return null;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        $keywordToken = $this->getKeywordToken();
+        foreach ($tokens as $token) {
+            if ($token->id === $keywordToken) {
+                $token->id = \T_STRING;
+            }
+        }
+
+        return $tokens;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/MatchTokenEmulator.php

@@ -0,0 +1,19 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class MatchTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 0);
+    }
+
+    public function getKeywordString(): string {
+        return 'match';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_MATCH;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/NullsafeTokenEmulator.php

@@ -0,0 +1,60 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+final class NullsafeTokenEmulator extends TokenEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 0);
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return strpos($code, '?->') !== false;
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        // We need to manually iterate and manage a count because we'll change
+        // the tokens array on the way
+        for ($i = 0, $c = count($tokens); $i < $c; ++$i) {
+            $token = $tokens[$i];
+            if ($token->text === '?' && isset($tokens[$i + 1]) && $tokens[$i + 1]->id === \T_OBJECT_OPERATOR) {
+                array_splice($tokens, $i, 2, [
+                    new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
+                ]);
+                $c--;
+                continue;
+            }
+
+            // Handle ?-> inside encapsed string.
+            if ($token->id === \T_ENCAPSED_AND_WHITESPACE && isset($tokens[$i - 1])
+                && $tokens[$i - 1]->id === \T_VARIABLE
+                && preg_match('/^\?->([a-zA-Z_\x80-\xff][a-zA-Z0-9_\x80-\xff]*)/', $token->text, $matches)
+            ) {
+                $replacement = [
+                    new Token(\T_NULLSAFE_OBJECT_OPERATOR, '?->', $token->line, $token->pos),
+                    new Token(\T_STRING, $matches[1], $token->line, $token->pos + 3),
+                ];
+                $matchLen = \strlen($matches[0]);
+                if ($matchLen !== \strlen($token->text)) {
+                    $replacement[] = new Token(
+                        \T_ENCAPSED_AND_WHITESPACE,
+                        \substr($token->text, $matchLen),
+                        $token->line, $token->pos + $matchLen
+                    );
+                }
+                array_splice($tokens, $i, 1, $replacement);
+                $c += \count($replacement) - 1;
+                continue;
+            }
+        }
+
+        return $tokens;
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // ?-> was not valid code previously, don't bother.
+        return $tokens;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/ReadonlyFunctionTokenEmulator.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+/*
+ * In PHP 8.1, "readonly(" was special cased in the lexer in order to support functions with
+ * name readonly. In PHP 8.2, this may conflict with readonly properties having a DNF type. For
+ * this reason, PHP 8.2 instead treats this as T_READONLY and then handles it specially in the
+ * parser. This emulator only exists to handle this special case, which is skipped by the
+ * PHP 8.1 ReadonlyTokenEmulator.
+ */
+class ReadonlyFunctionTokenEmulator extends KeywordEmulator {
+    public function getKeywordString(): string {
+        return 'readonly';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_READONLY;
+    }
+
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 2);
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        // Don't bother
+        return $tokens;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/ReadonlyTokenEmulator.php

@@ -0,0 +1,31 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+final class ReadonlyTokenEmulator extends KeywordEmulator {
+    public function getPhpVersion(): PhpVersion {
+        return PhpVersion::fromComponents(8, 1);
+    }
+
+    public function getKeywordString(): string {
+        return 'readonly';
+    }
+
+    public function getKeywordToken(): int {
+        return \T_READONLY;
+    }
+
+    protected function isKeywordContext(array $tokens, int $pos): bool {
+        if (!parent::isKeywordContext($tokens, $pos)) {
+            return false;
+        }
+        // Support "function readonly("
+        return !(isset($tokens[$pos + 1]) &&
+                 ($tokens[$pos + 1]->text === '(' ||
+                  ($tokens[$pos + 1]->id === \T_WHITESPACE &&
+                   isset($tokens[$pos + 2]) &&
+                   $tokens[$pos + 2]->text === '(')));
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/ReverseEmulator.php

@@ -0,0 +1,37 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+
+/**
+ * Reverses emulation direction of the inner emulator.
+ */
+final class ReverseEmulator extends TokenEmulator {
+    /** @var TokenEmulator Inner emulator */
+    private TokenEmulator $emulator;
+
+    public function __construct(TokenEmulator $emulator) {
+        $this->emulator = $emulator;
+    }
+
+    public function getPhpVersion(): PhpVersion {
+        return $this->emulator->getPhpVersion();
+    }
+
+    public function isEmulationNeeded(string $code): bool {
+        return $this->emulator->isEmulationNeeded($code);
+    }
+
+    public function emulate(string $code, array $tokens): array {
+        return $this->emulator->reverseEmulate($code, $tokens);
+    }
+
+    public function reverseEmulate(string $code, array $tokens): array {
+        return $this->emulator->emulate($code, $tokens);
+    }
+
+    public function preprocessCode(string $code, array &$patches): string {
+        return $code;
+    }
+}

vendor/nikic/php-parser/lib/PhpParser/Lexer/TokenEmulator/TokenEmulator.php

@@ -0,0 +1,30 @@
+<?php declare(strict_types=1);
+
+namespace PhpParser\Lexer\TokenEmulator;
+
+use PhpParser\PhpVersion;
+use PhpParser\Token;
+
+/** @internal */
+abstract class TokenEmulator {
+    abstract public function getPhpVersion(): PhpVersion;
+
+    abstract public function isEmulationNeeded(string $code): bool;
+
+    /**
+     * @param Token[] $tokens Original tokens
+     * @return Token[] Modified Tokens
+     */
+    abstract public function emulate(string $code, array $tokens): array;
+
+    /**
+     * @param Token[] $tokens Original tokens
+     * @return Token[] Modified Tokens
+     */
+    abstract public function reverseEmulate(string $code, array $tokens): array;
+
+    /** @param array{int, string, string}[] $patches */
+    public function preprocessCode(string $code, array &$patches): string {
+        return $code;
+    }
+}