timw4mail/php-kilo
1
0
Fork 0
Code Issues Pull Requests Releases Activity

Refactor PHP token function into class, more tests

Browse Source
This commit is contained in:
Timothy Warren 2019-11-14 12:14:02 -05:00
parent de0b6bda43
commit 519a193a8d
5 changed files with 180 additions and 134 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
src/Editor.php
View File

@ -6,6 +6,7 @@ use Aviat\Kilo\Enum\{
Key,
Highlight,
};
use Aviat\Kilo\Tokens\PHP;
/**
* // Don't highlight this!
@ -128,7 +129,7 @@ class Editor {
// Pre-tokenize the file
if ($this->syntax->filetype === 'PHP')
{
$this->syntax->tokens = get_php_tokens(file_get_contents($this->filename));
$this->syntax->tokens = PHP::getFileTokens($this->filename);
}
// Update the syntax highlighting for all the rows of the file
@ -929,7 +930,7 @@ class Editor {
private function refreshPHPSyntax(): void
{
$this->syntax->tokens = get_php_tokens($this->rowsToString());
$this->syntax->tokens = PHP::getTokens($this->rowsToString());
for ($i = 0; $i < $this->numRows; $i++)
{
$this->rows[$i]->updateSyntax();

112
src/Tokens/PHP.php Normal file
View File

@ -0,0 +1,112 @@
<?php declare(strict_types=1);
namespace Aviat\Kilo\Tokens;
use function Aviat\Kilo\tabs_to_spaces;
class PHP {
/**
* Use 'token_get_all' to get the tokens for a file,
* organized by row number
*
* @param string $code
* @return array
*/
public static function getTokens(string $code): array
{
$rawTokens = token_get_all($code);
$tokens = [];
$lineNum = 1;
$line = [];
foreach($rawTokens as $t)
{
if (is_array($t))
{
[$type, $rawChar, $currentLine] = $t;
$char = tabs_to_spaces($rawChar);
$current = [
'type' => $type,
'typeName' => token_name($type),
'char' => $char,
'line' => $currentLine,
];
if ($char === "\n")
{
$line[] = $current;
$tokens[$lineNum] = $line;
$lineNum++;
$line = [];
}
// Only return the first line of a multi-line token for this line array
if ($char !== "\n" && strpos($char, "\n") !== FALSE)
{
$chars = explode("\n", $char);
$current['original'] = [
'string' => $char,
'lines' => $chars,
];
$current['char'] = array_shift($chars);
// Add new lines for additional newline characters
$nextLine = $currentLine;
foreach ($chars as $char)
{
$nextLine++;
if ( ! array_key_exists($nextLine, $tokens))
{
$tokens[$nextLine] = [];
}
$tokens[$nextLine][] = [
'type' => -1,
'typeName' => 'RAW',
'char' => $char,
];
}
}
if ($currentLine !== $lineNum)
{
$existing = $tokens[$lineNum] ?? [];
$tokens[$lineNum] = array_merge($existing, $line);
$lineNum = $currentLine;
$line = [];
}
$line[] = $current;
}
else if (is_string($t))
{
// Simple characters, usually delimiters or single character operators
$line[] = [
'type' => -1,
'typeName' => 'RAW',
'char' => tabs_to_spaces($t),
];
}
}
$tokens[$lineNum] = array_merge($tokens[$lineNum] ?? [], $line);
ksort($tokens);
return $tokens;
}
public static function getFileTokens(string $filename): array
{
$code = file_get_contents($filename);
if ($code === FALSE)
{
return [];
}
return self::getTokens($code);
}
}

97
src/functions.php
View File

@ -46,7 +46,9 @@ function get_window_size()
if ($res === -1 || $ws->ws_col === 0)
{
return get_cursor_position();
// Return a default screen size until get_cursor_position function works
return [25, 80];
// return get_cursor_position();
}
return [$ws->ws_row, $ws->ws_col];
@ -286,99 +288,6 @@ function tabs_to_spaces(string $str, ?int $number = KILO_TAB_STOP): string
return str_replace("\t", str_repeat(' ', $number), $str);
}
/**
* Use 'token_get_all' to get the tokens for a file,
* organized by row number
*
* @param string $code
* @return array
*/
function get_php_tokens(string $code): array
{
$rawTokens = token_get_all($code);
$tokens = [];
$lineNum = 1;
$line = [];
foreach($rawTokens as $t)
{
if (is_array($t))
{
[$type, $rawChar, $currentLine] = $t;
$char = tabs_to_spaces($rawChar);
$current = [
'type' => $type,
'typeName' => token_name($type),
'char' => $char,
'line' => $currentLine,
];
if ($char === "\n")
{
$line[] = $current;
$tokens[$lineNum] = $line;
$lineNum++;
$line = [];
}
// Only return the first line of a multi-line token for this line array
if ($char !== "\n" && strpos($char, "\n") !== FALSE)
{
$chars = explode("\n", $char);
$current['original'] = [
'string' => $char,
'lines' => $chars,
];
$current['char'] = array_shift($chars);
// Add new lines for additional newline characters
$nextLine = $currentLine;
foreach ($chars as $char)
{
$nextLine++;
if ( ! array_key_exists($nextLine, $tokens))
{
$tokens[$nextLine] = [];
}
$tokens[$nextLine][] = [
'type' => -1,
'typeName' => 'RAW',
'char' => $char,
];
}
}
if ($currentLine !== $lineNum)
{
$existing = $tokens[$lineNum] ?? [];
$tokens[$lineNum] = array_merge($existing, $line);
$lineNum = $currentLine;
$line = [];
}
$line[] = $current;
}
else if (is_string($t))
{
// Simple characters, usually delimiters or single character operators
$line[] = [
'type' => -1,
'typeName' => 'RAW',
'char' => tabs_to_spaces($t),
];
}
}
$tokens[$lineNum] = array_merge($tokens[$lineNum] ?? [], $line);
ksort($tokens);
return $tokens;
}
/**
* Generate/Get the syntax highlighting objects
*

49
tests/FunctionTest.php
View File

@ -2,48 +2,23 @@
namespace Aviat\Kilo\Tests;
use function Aviat\Kilo\get_php_tokens;
use PHPUnit\Framework\TestCase;
use function Aviat\Kilo\get_window_size;
use function Aviat\Kilo\is_ascii;
class FunctionTest extends TestCase {
public function testGetPhpTokensLineAlignment(): void
public function test_get_window_size(): void
{
$file = file_get_contents(realpath(__DIR__ . '/../test.php'));
$tokens = get_php_tokens($file);
[$rows, $cols] = get_window_size();
$this->assertGreaterThan(0, $rows);
$this->assertGreaterThan(0, $cols);
}
$this->assertNotEmpty($file);
$lines = explode("\n", $file);
array_unshift($lines, '');
$misplacedTokens = [];
foreach ($tokens as $index => $lineTokens)
public function test_is_ascii(): void
{
if (empty($lineTokens))
{
$this->assertNotEmpty(trim($lines[$index]), 'Token is empty for non-empty line');
}
foreach ($lineTokens as $token)
{
// don't compare whitespace-only tokens
if (empty(trim($token['char'])))
{
continue;
}
$this->assertIsArray($token, 'All outputted tokens should be arrays');
// Make sure the matched string for the token is on the correct line
if (strpos($lines[$index], trim($token['char'])) === FALSE)
{
$token['misplaced_line'] = $index;
$misplacedTokens[] = $token;
}
}
}
$this->assertEmpty($misplacedTokens, 'Not all tokens are on the correct lines: ' . print_r($misplacedTokens, TRUE));
$this->assertFalse(is_ascii('©'));
$this->assertFalse(is_ascii("\x80"));
$this->assertTrue(is_ascii('a'));
}
}

49
tests/Tokens/PHPTest.php Normal file
View File

@ -0,0 +1,49 @@
<?php declare(strict_types=1);
namespace Aviat\Kilo\Tests\Tokens;
use Aviat\Kilo\Tokens\PHP;
use PHPUnit\Framework\TestCase;
class PHPTest extends TestCase {
public function testGetTokens(): void
{
$file = file_get_contents(realpath(__DIR__ . '/../../test.php'));
$tokens = PHP::getTokens($file);
$this->assertNotEmpty($file);
$lines = explode("\n", $file);
array_unshift($lines, '');
$misplacedTokens = [];
foreach ($tokens as $index => $lineTokens)
{
if (empty($lineTokens))
{
$this->assertNotEmpty(trim($lines[$index]), 'Token is empty for non-empty line');
}
foreach ($lineTokens as $token)
{
// don't compare whitespace-only tokens
if (empty(trim($token['char'])))
{
continue;
}
$this->assertIsArray($token, 'All outputted tokens should be arrays');
// Make sure the matched string for the token is on the correct line
if (strpos($lines[$index], trim($token['char'])) === FALSE)
{
$token['misplaced_line'] = $index;
$misplacedTokens[] = $token;
}
}
}
$this->assertEmpty($misplacedTokens, 'Not all tokens are on the correct lines: ' . print_r($misplacedTokens, TRUE));
}
}