Refactor keyword highlighting to handle more edge cases

char/char.go

@@ -1,4 +1,4 @@
-package key
+package char
 
 import (
 	"strings"
@@ -10,9 +10,12 @@ import (
 // ----------------------------------------------------------------------------
 
 const (
-	Backspace = '\x7f'
-	Esc       = '\x1b'
-	Enter     = '\r'
+	Backspace   = '\x7f'
+	Backslash   = '\\'
+	Esc         = '\x1b'
+	Enter       = '\r'
+	SingleQuote = '\''
+	DoubleQuote = '"'
 )
 
 // IsAscii Is this an ASCII character?
@@ -44,3 +47,13 @@ func Ctrl(char rune) rune {
 func IsSeparator(char rune) bool {
 	return unicode.IsSpace(char) || strings.ContainsRune(",.()+-/*=~%<>[];", char)
 }
+
+// IsDigit is a simple wrapper around built-in unicode.IsDigit
+func IsDigit(char rune) bool {
+	return unicode.IsDigit(char)
+}
+
+// IsNumeric checks whether the character is a digit, or a numeric separator
+func IsNumeric(char rune) bool {
+	return IsDigit(char) || char == '.'
+}

char/char_test.go

@@ -0,0 +1,103 @@
+package char
+
+import "testing"
+
+type isA struct {
+	arg1     rune
+	expected bool
+}
+
+func TestIsAscii(t *testing.T) {
+	// (╯°□°）╯︵ ┻━┻
+	var isAsciiTest = []isA{
+		{'┻', false},
+		{'$', true},
+		{'︵', false},
+		{0x7f, true},
+	}
+
+	for _, test := range isAsciiTest {
+		if output := IsAscii(test.arg1); output != test.expected {
+			t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
+		}
+	}
+}
+
+func TestIsCtrl(t *testing.T) {
+	var isCtrlTest = []isA{
+		{0x78, false},
+		{0x7f, true},
+		{0x02, true},
+		{0x98, false},
+		{'a', false},
+	}
+
+	for _, test := range isCtrlTest {
+		if output := IsCtrl(test.arg1); output != test.expected {
+			t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
+		}
+	}
+}
+
+func TestCtrl(t *testing.T) {
+	type ctrlTest struct {
+		arg1, expected rune
+	}
+
+	var ctrlTests = []ctrlTest{
+		{'A', '\x01'},
+		{'B', '\x02'},
+		{'Z', '\x1a'},
+		{'#', 3},
+		{'┻', 0},
+		{'😿', 0},
+	}
+
+	for _, test := range ctrlTests {
+		if output := Ctrl(test.arg1); output != test.expected {
+			t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
+		}
+	}
+}
+
+func TestIsSeparator(t *testing.T) {
+	for _, r := range ",.()+-/*=~%<>[] \t" {
+		if !IsSeparator(r) {
+			t.Errorf("Expected %q to be a syntax separator", r)
+		}
+	}
+}
+
+func TestIsDigit(t *testing.T) {
+	var digitTests = []isA{
+		{'0', true},
+		{'1', true},
+		{'.', false},
+		{'A', false},
+		{'😿', false},
+		{'$', false},
+	}
+
+	for _, test := range digitTests {
+		if output := IsDigit(test.arg1); output != test.expected {
+			t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
+		}
+	}
+}
+
+func TestIsNumeric(t *testing.T) {
+	var numericTests = []isA{
+		{'0', true},
+		{'1', true},
+		{'.', true},
+		{'A', false},
+		{'😿', false},
+		{'$', false},
+	}
+
+	for _, test := range numericTests {
+		if output := IsNumeric(test.arg1); output != test.expected {
+			t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
+		}
+	}
+}

editor/Editor.go

@@ -4,9 +4,9 @@ package editor
 import (
 	"fmt"
 	"time"
+	"timshome.page/gilo/char"
 	"timshome.page/gilo/editor/document"
 	"timshome.page/gilo/internal/gilo"
-	"timshome.page/gilo/key"
 	"timshome.page/gilo/terminal"
 )
 
@@ -104,7 +104,7 @@ func (e *Editor) prompt(prompt string, callback func(string, string)) string {
 		}
 
 		ch, _ := terminal.ReadKey()
-		if ch == key.Enter {
+		if ch == char.Enter {
 			if buf.Len() != 0 {
 				e.SetStatusMessage("")
 				if callback != nil {
@@ -113,15 +113,15 @@ func (e *Editor) prompt(prompt string, callback func(string, string)) string {
 
 				return buf.ToString()
 			}
-		} else if key.IsAscii(ch) && !key.IsCtrl(ch) {
+		} else if char.IsAscii(ch) && !char.IsCtrl(ch) {
 			buf.AppendRune(ch)
-		} else if ch == key.Backspace || ch == key.Ctrl('h') {
+		} else if ch == char.Backspace || ch == char.Ctrl('h') {
 			buf.Truncate(buf.Len() - 1)
-		} else if ch == key.Esc {
+		} else if ch == char.Esc {
 			k := parseEscapeSequence()
 			if k == keyDelete {
 				buf.Truncate(buf.Len() - 1)
-			} else if k == string(key.Esc) {
+			} else if k == string(char.Esc) {
 				e.SetStatusMessage("")
 				if callback != nil {
 					callback(buf.ToString(), k)

editor/document/row.go

@@ -2,10 +2,9 @@ package document
 
 import (
 	"strings"
+	"timshome.page/gilo/char"
 	"timshome.page/gilo/editor/highlight"
 	"timshome.page/gilo/internal/gilo"
-	"timshome.page/gilo/key"
-	"unicode"
 )
 
 type Row struct {
@@ -115,6 +114,7 @@ func (r *Row) update() {
 }
 
 // updateSyntax is the equivalent of editorUpdateSyntax in kilo
+// this is basically the core syntax highlighting algorithm
 func (r *Row) updateSyntax() {
 	i := 0
 	s := r.parent.Syntax
@@ -129,6 +129,7 @@ func (r *Row) updateSyntax() {
 	}
 
 	renderStr := string(r.render)
+	renderLen := r.RenderSize()
 	keywords1 := s.Keywords1
 	keywords2 := s.Keywords2
 
@@ -151,7 +152,7 @@ func (r *Row) updateSyntax() {
 
 		// Single line comments
 		if inString == '0' && scsIndex == i {
-			for j := scsIndex; j < r.RenderSize(); j++ {
+			for j := scsIndex; j < renderLen; j++ {
 				r.Hl[j] = highlight.Comment
 			}
 			break
@@ -160,7 +161,7 @@ func (r *Row) updateSyntax() {
 		// String literals
 		if s.Flags&highlight.DoStrings == highlight.DoStrings {
 			// At the start of a string literal
-			if inString == '0' && (ch == '"' || ch == '\'') {
+			if inString == '0' && (ch == char.DoubleQuote || ch == char.SingleQuote) {
 				inString = ch
 				r.Hl[i] = highlight.String
 				i++
@@ -172,7 +173,7 @@ func (r *Row) updateSyntax() {
 				r.Hl[i] = highlight.String
 
 				// Handle when a quote is escaped inside a string
-				if ch == '\\' && ip1 < r.RenderSize() {
+				if ch == char.Backslash && ip1 < renderLen {
 					r.Hl[ip1] = highlight.String
 					i += 2
 					continue
@@ -192,8 +193,8 @@ func (r *Row) updateSyntax() {
 
 		// Numeric literals
 		if s.Flags&highlight.DoNumbers == highlight.DoNumbers {
-			if (unicode.IsDigit(ch) && (prevSep || prevHl == highlight.Number)) ||
-				(ch == '.' && prevHl == highlight.Number) {
+			if (char.IsDigit(ch) && prevSep) ||
+				(char.IsNumeric(ch) && prevHl == highlight.Number) {
 				r.Hl[i] = highlight.Number
 				i += 1
 				prevSep = false
@@ -203,42 +204,39 @@ func (r *Row) updateSyntax() {
 
 		// Keywords
 		if prevSep {
-			renderLen := r.RenderSize()
+			matched := false
 
-			for _, word := range keywords1 {
-				wordLen := len(word)
-				nextInd := i + wordLen
-				if nextInd >= renderLen || renderStr[i:nextInd] != word {
-					continue
+		keywordLoop:
+			for n, list := range [][]string{keywords1, keywords2} {
+				hlType := highlight.Keyword1
+				if n == 1 {
+					hlType = highlight.Keyword2
 				}
 
-				if renderStr[i:renderLen] == word || key.IsSeparator(r.render[nextInd]) {
-					for k := i; k < nextInd; k++ {
-						r.Hl[k] = highlight.Keyword1
+				for _, word := range list {
+					wordLen := len(word)
+					nextInd := i + wordLen
+					endMatch := renderStr[i:renderLen] == word
+					goodMatch := nextInd < renderLen && renderStr[i:nextInd] == word
+
+					if endMatch || (goodMatch && char.IsSeparator(r.render[nextInd])) {
+						for k := i; k < nextInd; k++ {
+							r.Hl[k] = hlType
+						}
+						i += wordLen
+						matched = true
+						break keywordLoop
 					}
-					i += wordLen
-					break
 				}
 			}
 
-			for _, word := range keywords2 {
-				wordLen := len(word)
-				nextInd := i + wordLen
-				if nextInd >= renderLen || renderStr[i:nextInd] != word {
-					continue
-				}
-
-				if renderStr[i:renderLen] == word || key.IsSeparator(r.render[nextInd]) {
-					for k := i; k < nextInd; k++ {
-						r.Hl[k] = highlight.Keyword2
-					}
-					i += wordLen
-					break
-				}
+			if matched {
+				prevSep = false
+				continue
 			}
 		}
 
-		prevSep = key.IsSeparator(ch)
+		prevSep = char.IsSeparator(ch)
 		i++
 	}
 }

editor/input.go

@@ -1,9 +1,9 @@
 package editor
 
 import (
+	"timshome.page/gilo/char"
 	"timshome.page/gilo/editor/document"
 	"timshome.page/gilo/internal/gilo"
-	"timshome.page/gilo/key"
 	"timshome.page/gilo/terminal"
 )
 
@@ -28,7 +28,7 @@ const (
  */
 func (e *Editor) processKeypressChar(ch rune) bool {
 	switch ch {
-	case key.Ctrl('q'):
+	case char.Ctrl('q'):
 		if e.doc.IsDirty() && e.quitTimes > 0 {
 			e.SetStatusMessage("WARNING!!! File has unsaved changes. Press Ctrl-Q %d more times to quit.", e.quitTimes)
 			e.quitTimes -= 1
@@ -41,21 +41,21 @@ func (e *Editor) processKeypressChar(ch rune) bool {
 
 		return false
 
-	case key.Ctrl('s'):
+	case char.Ctrl('s'):
 		e.save()
 
-	case key.Ctrl('f'):
+	case char.Ctrl('f'):
 		e.find()
 
-	case key.Enter:
+	case char.Enter:
 		e.doc.InsertNewline(e.cursor)
 		e.cursor.Y += 1
 		e.cursor.X = 0
 
-	case key.Backspace, key.Ctrl('h'):
+	case char.Backspace, char.Ctrl('h'):
 		e.delChar()
 
-	case key.Esc, key.Ctrl('l'):
+	case char.Esc, char.Ctrl('l'):
 		// Modifier keys that return ANSI escape sequences
 		str := parseEscapeSequence()
 
@@ -190,7 +190,7 @@ func parseEscapeSequence() string {
 	for i := 0; i < 2; i++ {
 		ch, size := terminal.ReadKey()
 		if size == 0 {
-			return string(key.Esc)
+			return string(char.Esc)
 		}
 		runes[i] = ch
 
@@ -204,7 +204,7 @@ func parseEscapeSequence() string {
 		}
 	}
 
-	return string(key.Esc)
+	return string(char.Esc)
 }
 
 func escSeqToKey(seq []rune) string {
@@ -255,5 +255,5 @@ func escSeqToKey(seq []rune) string {
 		}
 	}
 
-	return string(key.Esc)
+	return string(char.Esc)
 }

editor/input_test.go

@@ -2,8 +2,8 @@ package editor
 
 import (
 	"testing"
+	"timshome.page/gilo/char"
 	gilo2 "timshome.page/gilo/internal/gilo"
-	"timshome.page/gilo/key"
 )
 
 type moveCursor struct {
@@ -13,7 +13,7 @@ type moveCursor struct {
 }
 
 var cursorTests = []moveCursor{
-	{[]string{string(key.Esc)}, false, gilo2.DefaultPoint()},
+	{[]string{string(char.Esc)}, false, gilo2.DefaultPoint()},
 	{[]string{keyRight}, true, gilo2.NewPoint(1, 0)},
 	{[]string{keyDown, keyEnd}, true, gilo2.NewPoint(14, 1)},
 	{[]string{keyEnd, keyHome}, true, gilo2.DefaultPoint()},
@@ -63,8 +63,8 @@ var seqTests = []seqTest{
 	{"[6~", keyPageDown},
 	{"[7~", keyHome},
 	{"[8~", keyEnd},
-	{"OQ", string(key.Esc)},
-	{"XZ", string(key.Esc)},
+	{"OQ", string(char.Esc)},
+	{"XZ", string(char.Esc)},
 }
 
 func TestEscToKey(t *testing.T) {

editor/search.go

@@ -1,9 +1,9 @@
 package editor
 
 import (
+	"timshome.page/gilo/char"
 	"timshome.page/gilo/editor/highlight"
 	"timshome.page/gilo/internal/gilo"
-	"timshome.page/gilo/key"
 )
 
 type search struct {
@@ -55,7 +55,7 @@ func (e *Editor) findCallback(query string, ch string) {
 		e.search.savedhlLine = -1
 	}
 
-	if ch == string(key.Enter) || ch == string(key.Esc) {
+	if ch == string(char.Enter) || ch == string(char.Esc) {
 		e.search.lastMatch = -1
 		e.search.direction = 1
 		return

key/key_test.go

@@ -1,70 +0,0 @@
-package key
-
-import "testing"
-
-type isRune struct {
-	arg1     rune
-	expected bool
-}
-
-// (╯°□°）╯︵ ┻━┻
-var isAsciiTest = []isRune{
-	{'┻', false},
-	{'$', true},
-	{'︵', false},
-	{0x7f, true},
-}
-
-func TestIsAscii(t *testing.T) {
-	for _, test := range isAsciiTest {
-		if output := IsAscii(test.arg1); output != test.expected {
-			t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
-		}
-	}
-}
-
-var isCtrlTest = []isRune{
-	{0x78, false},
-	{0x7f, true},
-	{0x02, true},
-	{0x98, false},
-	{'a', false},
-}
-
-func TestIsCtrl(t *testing.T) {
-	for _, test := range isCtrlTest {
-		if output := IsCtrl(test.arg1); output != test.expected {
-			t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
-		}
-	}
-}
-
-type ctrlTest struct {
-	arg1, expected rune
-}
-
-var ctrlTests = []ctrlTest{
-	{'A', '\x01'},
-	{'B', '\x02'},
-	{'Z', '\x1a'},
-	{'#', 3},
-	{'┻', 0},
-	{'😿', 0},
-}
-
-func TestCtrl(t *testing.T) {
-	for _, test := range ctrlTests {
-		if output := Ctrl(test.arg1); output != test.expected {
-			t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
-		}
-	}
-}
-
-func TestIsSeparator(t *testing.T) {
-	separators := ",.()+-/*=~%<>[] \t"
-	for _, r := range separators {
-		if !IsSeparator(r) {
-			t.Errorf("Expected %q to be a syntax separator", r)
-		}
-	}
-}