<?php

declare (strict_types=1);
namespace PhpParser;

require __DIR__ . '/compatibility_tokens.php';
class Lexer
{
    /**
     * Tokenize the provided source code.
     *
     * The token array is in the same format as provided by the PhpToken::tokenize() method in
     * PHP 8.0. The tokens are instances of PhpParser\Token, to abstract over a polyfill
     * implementation in earlier PHP version.
     *
     * The token array is terminated by a sentinel token with token ID 0.
     * The token array does not discard any tokens (i.e. whitespace and comments are included).
     * The token position attributes are against this token array.
     *
     * @param string $code The source code to tokenize.
     * @param ErrorHandler|null $errorHandler Error handler to use for lexing errors. Defaults to
     *                                        ErrorHandler\Throwing.
     * @return Token[] Tokens
     */
    public function tokenize(string $code, ?\PhpParser\ErrorHandler $errorHandler = null): array
    {
        if (null === $errorHandler) {
            $errorHandler = new \PhpParser\ErrorHandler\Throwing();
        }
        $scream = ini_set('xdebug.scream', '0');
        $tokens = @\PhpParser\Token::tokenize($code);
        $this->postprocessTokens($tokens, $errorHandler);
        if (\false !== $scream) {
            ini_set('xdebug.scream', $scream);
        }
        return $tokens;
    }
    private function handleInvalidCharacter(\PhpParser\Token $token, \PhpParser\ErrorHandler $errorHandler): void
    {
        $chr = $token->text;
        if ($chr === "\x00") {
            // PHP cuts error message after null byte, so need special case
            $errorMsg = 'Unexpected null byte';
        } else {
            $errorMsg = sprintf('Unexpected character "%s" (ASCII %d)', $chr, ord($chr));
        }
        $errorHandler->handleError(new \PhpParser\Error($errorMsg, ['startLine' => $token->line, 'endLine' => $token->line, 'startFilePos' => $token->pos, 'endFilePos' => $token->pos]));
    }
    private function isUnterminatedComment(\PhpParser\Token $token): bool
    {
        return $token->is([\T_COMMENT, \T_DOC_COMMENT]) && substr($token->text, 0, 2) === '/*' && substr($token->text, -2) !== '*/';
    }
    /**
     * @param list<Token> $tokens
     */
    protected function postprocessTokens(array &$tokens, \PhpParser\ErrorHandler $errorHandler): void
    {
        // This function reports errors (bad characters and unterminated comments) in the token
        // array, and performs certain canonicalizations:
        //  * Use PHP 8.1 T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG and
        //    T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG tokens used to disambiguate intersection types.
        //  * Add a sentinel token with ID 0.
        $numTokens = \count($tokens);
        if ($numTokens === 0) {
            // Empty input edge case: Just add the sentinel token.
            $tokens[] = new \PhpParser\Token(0, "\x00", 1, 0);
            return;
        }
        for ($i = 0; $i < $numTokens; $i++) {
            $token = $tokens[$i];
            if ($token->id === \T_BAD_CHARACTER) {
                $this->handleInvalidCharacter($token, $errorHandler);
            }
            if ($token->id === \ord('&')) {
                $next = $i + 1;
                while (isset($tokens[$next]) && $tokens[$next]->id === \T_WHITESPACE) {
                    $next++;
                }
                $followedByVarOrVarArg = isset($tokens[$next]) && $tokens[$next]->is([\T_VARIABLE, \T_ELLIPSIS]);
                $token->id = $followedByVarOrVarArg ? \T_AMPERSAND_FOLLOWED_BY_VAR_OR_VARARG : \T_AMPERSAND_NOT_FOLLOWED_BY_VAR_OR_VARARG;
            }
        }
        // Check for unterminated comment
        $lastToken = $tokens[$numTokens - 1];
        if ($this->isUnterminatedComment($lastToken)) {
            $errorHandler->handleError(new \PhpParser\Error('Unterminated comment', ['startLine' => $lastToken->line, 'endLine' => $lastToken->getEndLine(), 'startFilePos' => $lastToken->pos, 'endFilePos' => $lastToken->getEndPos()]));
        }
        // Add sentinel token.
        $tokens[] = new \PhpParser\Token(0, "\x00", $lastToken->getEndLine(), $lastToken->getEndPos());
    }
}
