First unit test

composer.json

@@ -17,8 +17,8 @@
         "phpunit/phpunit": "^8"
     },
 	"scripts": {
-		"coverage": "phpdbg -qrr -- vendor/bin/phpunit",
+		"coverage": "phpdbg -qrr -- vendor/bin/phpunit tests",
-		"test": "vendor/bin/phpunit"
+		"test": "vendor/bin/phpunit tests"
 	},
 	"require": {
 		"ext-ffi": "*"

src/Row.php

@@ -159,6 +159,7 @@ class Row {
 		"'" => Highlight::DELIMITER,
 
 		// Single character operators
+		'?' => Highlight::OPERATOR,
 		',' => Highlight::OPERATOR,
 		';' => Highlight::OPERATOR,
 		':' => Highlight::OPERATOR,
@@ -445,10 +446,14 @@ class Row {
 	{
 		$rowNum = $this->idx + 1;
 
-		if (
+		$hasTokens = isset($this->parent->syntax->tokens);
-			( ! isset($this->parent->syntax->tokens))
+		$hasRow = array_key_exists($rowNum, $this->parent->syntax->tokens);
-			|| ( ! array_key_exists($rowNum, $this->parent->syntax->tokens))
+
-			|| $this->idx > $this->parent->numRows)
+		if ( ! (
+			isset($this->parent->syntax->tokens) &&
+			array_key_exists($rowNum, $this->parent->syntax->tokens) &&
+			$this->idx < $this->parent->numRows
+		))
 		{
 			return;
 		}
@@ -461,28 +466,6 @@ class Row {
 		// multiples of the same tokens can be effectively matched
 		$offset = 0;
 
-		// There will be no tokens if a line is empty, or if in a multi-line comment
-		if (empty($tokens))
-		{
-			if ($inComment && $this->rsize > 0)
-			{
-				while ($offset < $this->rsize)
-				{
-					if (substr($this->render, $offset, 2) === '*/')
-					{
-						array_replace_range($this->hl, $offset, 2, Highlight::ML_COMMENT);
-						break;
-					}
-
-					$this->hl[$offset] = Highlight::ML_COMMENT;
-					$offset++;
-					continue;
-				}
-			}
-
-			return;
-		}
-
 		foreach ($tokens as $token)
 		{
 			if ($offset >= $this->rsize)
@@ -535,17 +518,23 @@ class Row {
 				// Start of multi-line comment
 				$start = strpos($this->render, '/*', $offset);
 				$end = strpos($this->render, '*/', $offset);
-				if ($start !== FALSE && $end !== FALSE)
+				$hasStart = $start !== FALSE;
+				$hasEnd = $end !== FALSE;
+
+				if ($hasStart)
 				{
-					$len = $end - $start + 2;
+					if ($hasEnd)
-					array_replace_range($this->hl, $start, $len, Highlight::ML_COMMENT);
+					{
-					$inComment = FALSE;
+						$len = $end - $start + 2;
-				}
+						array_replace_range($this->hl, $start, $len, Highlight::ML_COMMENT);
-				if ($start !== FALSE && $end === FALSE)
+						$inComment = FALSE;
-				{
+					}
-					$inComment = TRUE;
+					else
-					array_replace_range($this->hl, $start, $charLen - $offset, Highlight::ML_COMMENT);
+					{
-					$offset = $start + $charLen - $offset;
+						$inComment = TRUE;
+						array_replace_range($this->hl, $start, $charLen - $offset, Highlight::ML_COMMENT);
+						$offset = $start + $charLen - $offset;
+					}
 				}
 
 				if ($inComment)

src/functions.php

@@ -240,6 +240,13 @@ function array_replace_range(array &$array, int $offset, int $length, $value):vo
 	} */
 }
 
+function str_contains(string $haystack, string $str, ?int $offset = NULL): bool
+{
+	return ($offset !== NULL)
+		? strpos($haystack, $str, $offset) !== FALSE
+		: strpos($haystack, $str) !== FALSE;
+}
+
 /**
  * Get the ASCII color escape number for the specified syntax type
  *

test.php

@@ -46,6 +46,8 @@ class FooBar extends Foo implements Ifoo {
  */
 $foobar = new FooBar();
 
+$baz = ['a' => 'b'];
+
 // C++ style comment
 $x = 3;
 
@@ -56,6 +58,11 @@ $y = [
 	3
 ];
 
+// Multi-line ternary statement
+$q = ($x !== 2)
+	? 'yes'
+	: 'no';
+
 /*
 Heredoc
 */$z = $x + $y;

tests/FunctionTest.php

@@ -2,8 +2,48 @@
 
 namespace Aviat\Kilo\Tests;
 
+use function Aviat\Kilo\get_php_tokens;
 use PHPUnit\Framework\TestCase;
 
 class FunctionTest extends TestCase {
+	public function testGetPhpTokensLineAlignment(): void
+	{
+		$file = file_get_contents(realpath(__DIR__ . '/../test.php'));
+		$tokens = get_php_tokens($file);
 
+		$this->assertNotEmpty($file);
+
+		$lines = explode("\n", $file);
+		array_unshift($lines, '');
+
+		$misplacedTokens = [];
+
+		foreach ($tokens as $index => $lineTokens)
+		{
+			if (empty($lineTokens))
+			{
+				$this->assertNotEmpty(trim($lines[$index]), 'Token is empty for non-empty line');
+			}
+
+			foreach ($lineTokens as $token)
+			{
+				// don't compare whitespace-only tokens
+				if (empty(trim($token['char'])))
+				{
+					continue;
+				}
+
+				$this->assertIsArray($token, 'All outputted tokens should be arrays');
+
+				// Make sure the matched string for the token is on the correct line
+				if (strpos($lines[$index], trim($token['char'])) === FALSE)
+				{
+					$token['misplaced_line'] = $index;
+					$misplacedTokens[] = $token;
+				}
+			}
+		}
+
+		$this->assertEmpty($misplacedTokens, 'Not all tokens are on the correct lines: ' . print_r($misplacedTokens, TRUE));
+	}
 }