$type, 'typeName' => token_name($type), 'char' => $char, 'line' => $currentLine, ]; if ($char === "\n") { $line[] = $current; $tokens[$lineNum] = $line; $lineNum++; $line = []; } // Only return the first line of a multi-line token for this line array if ($char !== "\n" && strpos($char, "\n") !== FALSE) { $chars = explode("\n", $char); $current['original'] = [ 'string' => $char, 'lines' => $chars, ]; $current['char'] = array_shift($chars); // Add new lines for additional newline characters $nextLine = $currentLine; foreach ($chars as $char) { $nextLine++; if ( ! array_key_exists($nextLine, $tokens)) { $tokens[$nextLine] = []; } $tokens[$nextLine][] = [ 'type' => -1, 'typeName' => 'RAW', 'char' => $char, ]; } } if ($currentLine !== $lineNum) { $existing = $tokens[$lineNum] ?? []; $tokens[$lineNum] = array_merge($existing, $line); $lineNum = $currentLine; $line = []; } $line[] = $current; } else if (is_string($t)) { // Simple characters, usually delimiters or single character operators $line[] = [ 'type' => -1, 'typeName' => 'RAW', 'char' => tabs_to_spaces($t), ]; } } $tokens[$lineNum] = array_merge($tokens[$lineNum] ?? [], $line); ksort($tokens); return $tokens; } public static function getFileTokens(string $filename): array { $code = file_get_contents($filename); if ($code === FALSE) { return []; } return self::getTokens($code); } }