More progress on PHP syntax highlighting

src/Editor.php

@@ -43,8 +43,6 @@ class Editor {
 		[$this->screenRows, $this->screenCols] = get_window_size();
 
 		$this->screenRows -= 2;
-
-		// print_r($this); die();
 	}
 
 	public function __get(string $name)
@@ -122,6 +120,12 @@ class Editor {
 				) {
 					$this->syntax = $syntax;
 
+					// Pre-tokenize the file
+					if ($this->syntax->filetype === 'PHP')
+					{
+						$this->syntax->tokens = get_php_tokens(file_get_contents($this->filename));
+					}
+
 					// Update the syntax highlighting for all the rows of the file
 					for ($i = 0; $i < $this->numRows; $i++)
 					{
@@ -173,33 +177,30 @@ class Editor {
 		return $cx;
 	}
 
-	protected function insertRow(int $at, string $s): void
+	protected function insertRow(int $at, string $s, bool $updateSyntax = TRUE): void
 	{
 		if ($at < 0 || $at > $this->numRows)
 		{
 			return;
 		}
 
-		// Re-tokenize the file
-		if ($this->syntax->filetype === 'PHP')
-		{
-			$this->syntax->tokens = get_php_tokens($this->filename);
-		}
-
 		$row = Row::new($this, $s, $at);
-		// Update other row indices
-		for ($i = $at + 1; $i <= $this->numRows; $i++)
-		{
-			$this->rows[$i]->idx++;
-		}
-
 
 		if ($at === $this->numRows)
 		{
+			$row->idx = $this->numRows;
 			$this->rows[] = $row;
 		}
 		else
 		{
+			// Update other row indices
+			$numRows = $this->numRows;
+			$i = $at + 1;
+			for (; $i <= $numRows; $i++)
+			{
+				$this->rows[$i]->idx++;
+			}
+
 			$this->rows = [
 				...array_slice($this->rows, 0, $at),
 				$row,
@@ -207,7 +208,15 @@ class Editor {
 			];
 		}
 
-		$this->rows[$at]->update();
+		// Re-tokenize the file
+		if ($updateSyntax && $this->syntax->filetype === 'PHP')
+		{
+			$this->refreshPHPSyntax();
+		}
+		else
+		{
+			$this->rows[$at]->update();
+		}
 
 		$this->dirty++;
 	}
@@ -219,12 +228,6 @@ class Editor {
 			return;
 		}
 
-		// Re-tokenize the file
-		if ($this->syntax->filetype === 'PHP')
-		{
-			$this->syntax->tokens = get_php_tokens($this->filename);
-		}
-
 		// Remove the row
 		unset($this->rows[$at]);
 
@@ -235,6 +238,12 @@ class Editor {
 			$this->rows[$i]->idx--;
 		}
 
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->refreshPHPSyntax();
+		}
+
 		$this->dirty++;
 	}
 
@@ -244,28 +253,23 @@ class Editor {
 
 	protected function insertChar(string $c): void
 	{
-		// Re-tokenize the file
-		if ($this->syntax->filetype === 'PHP')
-		{
-			$this->syntax->tokens = get_php_tokens($this->filename);
-		}
-
 		if ($this->cursorY === $this->numRows)
 		{
 			$this->insertRow($this->numRows, '');
 		}
 		$this->rows[$this->cursorY]->insertChar($this->cursorX, $c);
+
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->refreshPHPSyntax();
+		}
+
 		$this->cursorX++;
 	}
 
 	protected function insertNewline(): void
 	{
-		// Re-tokenize the file
-		if ($this->syntax->filetype === 'PHP')
-		{
-			$this->syntax->tokens = get_php_tokens($this->filename);
-		}
-
 		if ($this->cursorX === 0)
 		{
 			$this->insertRow($this->cursorY, '');
@@ -293,12 +297,6 @@ class Editor {
 			return;
 		}
 
-		// Re-tokenize the file
-		if ($this->syntax->filetype === 'PHP')
-		{
-			$this->syntax->tokens = get_php_tokens($this->filename);
-		}
-
 		$row = $this->rows[$this->cursorY];
 		if ($this->cursorX > 0)
 		{
@@ -312,6 +310,12 @@ class Editor {
 			$this->deleteRow($this->cursorY);
 			$this->cursorY--;
 		}
+
+		// Re-tokenize the file
+		if ($this->syntax->filetype === 'PHP')
+		{
+			$this->syntax->tokens = get_php_tokens($this->rowsToString());
+		}
 	}
 
 	// ------------------------------------------------------------------------
@@ -336,12 +340,6 @@ class Editor {
 
 		$this->selectSyntaxHighlight();
 
-		// Pre-tokenize the file
-		if ($this->syntax->filetype === 'PHP')
-		{
-			$this->syntax->tokens = get_php_tokens(file_get_contents($this->filename));
-		}
-
 		// #TODO gracefully handle issues with loading a file
 		$handle = fopen($filename, 'rb');
 		if ($handle === FALSE)
@@ -356,7 +354,7 @@ class Editor {
 		while (($line = fgets($handle)) !== FALSE)
 		{
 			// Remove line endings when reading the file
-			$this->insertRow($this->numRows, rtrim($line));
+			$this->insertRow($this->numRows, rtrim($line), FALSE);
 		}
 
 		fclose($handle);
@@ -921,4 +919,13 @@ class Editor {
 			$this->moveCursor($c === Key::PAGE_UP ? Key::ARROW_UP : Key::ARROW_DOWN);
 		}
 	}
+
+	private function refreshPHPSyntax(): void
+	{
+		$this->syntax->tokens = get_php_tokens($this->rowsToString());
+		foreach($this->rows as $row)
+		{
+			$row->updateSyntax();
+		}
+	}
 }

src/Row.php

@@ -38,6 +38,7 @@ class Row {
 
 		// Simple variables
 		T_VARIABLE => Highlight::VARIABLE,
+		T_STRING_VARNAME => Highlight::VARIABLE,
 
 		// Operators
 		T_AND_EQUAL => Highlight::OPERATOR,
@@ -423,14 +424,9 @@ class Row {
 
 	protected function updateSyntaxPHP():void
 	{
+		// The index for the tokens should exist
 		$tokens = $this->parent->syntax->tokens[$this->idx + 1];
 
-		// The line is probably just empty
-		if ($tokens === NULL)
-		{
-			return;
-		}
-
 		// $inComment = ($this->idx > 0 && $this->parent->rows[$this->idx - 1]->hlOpenComment);
 
 		// Keep track of where you are in the line, so that

src/functions.php

@@ -331,6 +331,7 @@ function get_php_tokens(string $code): array
 		if ( ! is_array($token))
 		{
 			$line[] = [
+				'type' => -1,
 				'typeName' => 'RAW',
 				'char' => $token,
 			];
@@ -349,6 +350,14 @@ function get_php_tokens(string $code): array
 		if ($currentLine !== $lineNum)
 		{
 			$tokens[$lineNum] = $line;
+
+			// Make sure to insert empty arrays for empty lines
+			// So the array of tokens isn't sparse
+			for ($i = $lineNum; $i < $currentLine; $i++)
+			{
+				$tokens[$i] = [];
+			}
+
 			$lineNum = $currentLine;
 			$line = [];
 		}