diff --git a/src/Editor.php b/src/Editor.php index 063dff5..5e7f980 100644 --- a/src/Editor.php +++ b/src/Editor.php @@ -6,6 +6,7 @@ use Aviat\Kilo\Enum\{ Key, Highlight, }; +use Aviat\Kilo\Tokens\PHP; /** * // Don't highlight this! @@ -128,7 +129,7 @@ class Editor { // Pre-tokenize the file if ($this->syntax->filetype === 'PHP') { - $this->syntax->tokens = get_php_tokens(file_get_contents($this->filename)); + $this->syntax->tokens = PHP::getFileTokens($this->filename); } // Update the syntax highlighting for all the rows of the file @@ -929,7 +930,7 @@ class Editor { private function refreshPHPSyntax(): void { - $this->syntax->tokens = get_php_tokens($this->rowsToString()); + $this->syntax->tokens = PHP::getTokens($this->rowsToString()); for ($i = 0; $i < $this->numRows; $i++) { $this->rows[$i]->updateSyntax(); diff --git a/src/Tokens/PHP.php b/src/Tokens/PHP.php new file mode 100644 index 0000000..a791a0b --- /dev/null +++ b/src/Tokens/PHP.php @@ -0,0 +1,112 @@ + $type, + 'typeName' => token_name($type), + 'char' => $char, + 'line' => $currentLine, + ]; + + if ($char === "\n") + { + $line[] = $current; + $tokens[$lineNum] = $line; + $lineNum++; + $line = []; + } + + // Only return the first line of a multi-line token for this line array + if ($char !== "\n" && strpos($char, "\n") !== FALSE) + { + $chars = explode("\n", $char); + $current['original'] = [ + 'string' => $char, + 'lines' => $chars, + ]; + $current['char'] = array_shift($chars); + + // Add new lines for additional newline characters + $nextLine = $currentLine; + foreach ($chars as $char) + { + $nextLine++; + + if ( ! array_key_exists($nextLine, $tokens)) + { + $tokens[$nextLine] = []; + } + + $tokens[$nextLine][] = [ + 'type' => -1, + 'typeName' => 'RAW', + 'char' => $char, + ]; + } + } + + if ($currentLine !== $lineNum) + { + $existing = $tokens[$lineNum] ?? []; + $tokens[$lineNum] = array_merge($existing, $line); + + $lineNum = $currentLine; + $line = []; + } + + $line[] = $current; + } + else if (is_string($t)) + { + // Simple characters, usually delimiters or single character operators + $line[] = [ + 'type' => -1, + 'typeName' => 'RAW', + 'char' => tabs_to_spaces($t), + ]; + } + } + + $tokens[$lineNum] = array_merge($tokens[$lineNum] ?? [], $line); + + ksort($tokens); + + return $tokens; + } + + public static function getFileTokens(string $filename): array + { + $code = file_get_contents($filename); + + if ($code === FALSE) + { + return []; + } + + return self::getTokens($code); + } +} \ No newline at end of file diff --git a/src/functions.php b/src/functions.php index cd9a7b3..18d8540 100644 --- a/src/functions.php +++ b/src/functions.php @@ -46,7 +46,9 @@ function get_window_size() if ($res === -1 || $ws->ws_col === 0) { - return get_cursor_position(); + // Return a default screen size until get_cursor_position function works + return [25, 80]; + // return get_cursor_position(); } return [$ws->ws_row, $ws->ws_col]; @@ -286,99 +288,6 @@ function tabs_to_spaces(string $str, ?int $number = KILO_TAB_STOP): string return str_replace("\t", str_repeat(' ', $number), $str); } -/** - * Use 'token_get_all' to get the tokens for a file, - * organized by row number - * - * @param string $code - * @return array - */ -function get_php_tokens(string $code): array -{ - $rawTokens = token_get_all($code); - $tokens = []; - $lineNum = 1; - $line = []; - foreach($rawTokens as $t) - { - if (is_array($t)) - { - [$type, $rawChar, $currentLine] = $t; - $char = tabs_to_spaces($rawChar); - - $current = [ - 'type' => $type, - 'typeName' => token_name($type), - 'char' => $char, - 'line' => $currentLine, - ]; - - if ($char === "\n") - { - $line[] = $current; - $tokens[$lineNum] = $line; - $lineNum++; - $line = []; - } - - // Only return the first line of a multi-line token for this line array - if ($char !== "\n" && strpos($char, "\n") !== FALSE) - { - $chars = explode("\n", $char); - $current['original'] = [ - 'string' => $char, - 'lines' => $chars, - ]; - $current['char'] = array_shift($chars); - - // Add new lines for additional newline characters - $nextLine = $currentLine; - foreach ($chars as $char) - { - $nextLine++; - - if ( ! array_key_exists($nextLine, $tokens)) - { - $tokens[$nextLine] = []; - } - - $tokens[$nextLine][] = [ - 'type' => -1, - 'typeName' => 'RAW', - 'char' => $char, - ]; - } - } - - if ($currentLine !== $lineNum) - { - $existing = $tokens[$lineNum] ?? []; - $tokens[$lineNum] = array_merge($existing, $line); - - $lineNum = $currentLine; - $line = []; - } - - $line[] = $current; - } - else if (is_string($t)) - { - // Simple characters, usually delimiters or single character operators - $line[] = [ - 'type' => -1, - 'typeName' => 'RAW', - 'char' => tabs_to_spaces($t), - ]; - } - } - - $tokens[$lineNum] = array_merge($tokens[$lineNum] ?? [], $line); - - ksort($tokens); - - return $tokens; -} - /** * Generate/Get the syntax highlighting objects * diff --git a/tests/FunctionTest.php b/tests/FunctionTest.php index 5dbf55e..0ee4ce1 100644 --- a/tests/FunctionTest.php +++ b/tests/FunctionTest.php @@ -2,48 +2,23 @@ namespace Aviat\Kilo\Tests; -use function Aviat\Kilo\get_php_tokens; use PHPUnit\Framework\TestCase; +use function Aviat\Kilo\get_window_size; +use function Aviat\Kilo\is_ascii; + class FunctionTest extends TestCase { - public function testGetPhpTokensLineAlignment(): void + public function test_get_window_size(): void { - $file = file_get_contents(realpath(__DIR__ . '/../test.php')); - $tokens = get_php_tokens($file); + [$rows, $cols] = get_window_size(); + $this->assertGreaterThan(0, $rows); + $this->assertGreaterThan(0, $cols); + } - $this->assertNotEmpty($file); - - $lines = explode("\n", $file); - array_unshift($lines, ''); - - $misplacedTokens = []; - - foreach ($tokens as $index => $lineTokens) - { - if (empty($lineTokens)) - { - $this->assertNotEmpty(trim($lines[$index]), 'Token is empty for non-empty line'); - } - - foreach ($lineTokens as $token) - { - // don't compare whitespace-only tokens - if (empty(trim($token['char']))) - { - continue; - } - - $this->assertIsArray($token, 'All outputted tokens should be arrays'); - - // Make sure the matched string for the token is on the correct line - if (strpos($lines[$index], trim($token['char'])) === FALSE) - { - $token['misplaced_line'] = $index; - $misplacedTokens[] = $token; - } - } - } - - $this->assertEmpty($misplacedTokens, 'Not all tokens are on the correct lines: ' . print_r($misplacedTokens, TRUE)); + public function test_is_ascii(): void + { + $this->assertFalse(is_ascii('©')); + $this->assertFalse(is_ascii("\x80")); + $this->assertTrue(is_ascii('a')); } } \ No newline at end of file diff --git a/tests/Tokens/PHPTest.php b/tests/Tokens/PHPTest.php new file mode 100644 index 0000000..4ea3703 --- /dev/null +++ b/tests/Tokens/PHPTest.php @@ -0,0 +1,49 @@ +assertNotEmpty($file); + + $lines = explode("\n", $file); + array_unshift($lines, ''); + + $misplacedTokens = []; + + foreach ($tokens as $index => $lineTokens) + { + if (empty($lineTokens)) + { + $this->assertNotEmpty(trim($lines[$index]), 'Token is empty for non-empty line'); + } + + foreach ($lineTokens as $token) + { + // don't compare whitespace-only tokens + if (empty(trim($token['char']))) + { + continue; + } + + $this->assertIsArray($token, 'All outputted tokens should be arrays'); + + // Make sure the matched string for the token is on the correct line + if (strpos($lines[$index], trim($token['char'])) === FALSE) + { + $token['misplaced_line'] = $index; + $misplacedTokens[] = $token; + } + } + } + + $this->assertEmpty($misplacedTokens, 'Not all tokens are on the correct lines: ' . print_r($misplacedTokens, TRUE)); + } +} \ No newline at end of file