Refactor keyword highlighting to handle more edge cases
All checks were successful
timw4mail/gilo/pipeline/head This commit looks good

This commit is contained in:
Timothy Warren 2023-10-06 11:30:21 -04:00
parent 5ff459b6ad
commit ceed34c634
8 changed files with 173 additions and 129 deletions

View File

@ -1,4 +1,4 @@
package key package char
import ( import (
"strings" "strings"
@ -11,8 +11,11 @@ import (
const ( const (
Backspace = '\x7f' Backspace = '\x7f'
Backslash = '\\'
Esc = '\x1b' Esc = '\x1b'
Enter = '\r' Enter = '\r'
SingleQuote = '\''
DoubleQuote = '"'
) )
// IsAscii Is this an ASCII character? // IsAscii Is this an ASCII character?
@ -44,3 +47,13 @@ func Ctrl(char rune) rune {
func IsSeparator(char rune) bool { func IsSeparator(char rune) bool {
return unicode.IsSpace(char) || strings.ContainsRune(",.()+-/*=~%<>[];", char) return unicode.IsSpace(char) || strings.ContainsRune(",.()+-/*=~%<>[];", char)
} }
// IsDigit is a simple wrapper around built-in unicode.IsDigit
func IsDigit(char rune) bool {
return unicode.IsDigit(char)
}
// IsNumeric checks whether the character is a digit, or a numeric separator
func IsNumeric(char rune) bool {
return IsDigit(char) || char == '.'
}

103
char/char_test.go Normal file
View File

@ -0,0 +1,103 @@
package char
import "testing"
type isA struct {
arg1 rune
expected bool
}
func TestIsAscii(t *testing.T) {
// (╯°□°)╯︵ ┻━┻
var isAsciiTest = []isA{
{'┻', false},
{'$', true},
{'︵', false},
{0x7f, true},
}
for _, test := range isAsciiTest {
if output := IsAscii(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsCtrl(t *testing.T) {
var isCtrlTest = []isA{
{0x78, false},
{0x7f, true},
{0x02, true},
{0x98, false},
{'a', false},
}
for _, test := range isCtrlTest {
if output := IsCtrl(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestCtrl(t *testing.T) {
type ctrlTest struct {
arg1, expected rune
}
var ctrlTests = []ctrlTest{
{'A', '\x01'},
{'B', '\x02'},
{'Z', '\x1a'},
{'#', 3},
{'┻', 0},
{'😿', 0},
}
for _, test := range ctrlTests {
if output := Ctrl(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsSeparator(t *testing.T) {
for _, r := range ",.()+-/*=~%<>[] \t" {
if !IsSeparator(r) {
t.Errorf("Expected %q to be a syntax separator", r)
}
}
}
func TestIsDigit(t *testing.T) {
var digitTests = []isA{
{'0', true},
{'1', true},
{'.', false},
{'A', false},
{'😿', false},
{'$', false},
}
for _, test := range digitTests {
if output := IsDigit(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsNumeric(t *testing.T) {
var numericTests = []isA{
{'0', true},
{'1', true},
{'.', true},
{'A', false},
{'😿', false},
{'$', false},
}
for _, test := range numericTests {
if output := IsNumeric(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}

View File

@ -4,9 +4,9 @@ package editor
import ( import (
"fmt" "fmt"
"time" "time"
"timshome.page/gilo/char"
"timshome.page/gilo/editor/document" "timshome.page/gilo/editor/document"
"timshome.page/gilo/internal/gilo" "timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
"timshome.page/gilo/terminal" "timshome.page/gilo/terminal"
) )
@ -104,7 +104,7 @@ func (e *Editor) prompt(prompt string, callback func(string, string)) string {
} }
ch, _ := terminal.ReadKey() ch, _ := terminal.ReadKey()
if ch == key.Enter { if ch == char.Enter {
if buf.Len() != 0 { if buf.Len() != 0 {
e.SetStatusMessage("") e.SetStatusMessage("")
if callback != nil { if callback != nil {
@ -113,15 +113,15 @@ func (e *Editor) prompt(prompt string, callback func(string, string)) string {
return buf.ToString() return buf.ToString()
} }
} else if key.IsAscii(ch) && !key.IsCtrl(ch) { } else if char.IsAscii(ch) && !char.IsCtrl(ch) {
buf.AppendRune(ch) buf.AppendRune(ch)
} else if ch == key.Backspace || ch == key.Ctrl('h') { } else if ch == char.Backspace || ch == char.Ctrl('h') {
buf.Truncate(buf.Len() - 1) buf.Truncate(buf.Len() - 1)
} else if ch == key.Esc { } else if ch == char.Esc {
k := parseEscapeSequence() k := parseEscapeSequence()
if k == keyDelete { if k == keyDelete {
buf.Truncate(buf.Len() - 1) buf.Truncate(buf.Len() - 1)
} else if k == string(key.Esc) { } else if k == string(char.Esc) {
e.SetStatusMessage("") e.SetStatusMessage("")
if callback != nil { if callback != nil {
callback(buf.ToString(), k) callback(buf.ToString(), k)

View File

@ -2,10 +2,9 @@ package document
import ( import (
"strings" "strings"
"timshome.page/gilo/char"
"timshome.page/gilo/editor/highlight" "timshome.page/gilo/editor/highlight"
"timshome.page/gilo/internal/gilo" "timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
"unicode"
) )
type Row struct { type Row struct {
@ -115,6 +114,7 @@ func (r *Row) update() {
} }
// updateSyntax is the equivalent of editorUpdateSyntax in kilo // updateSyntax is the equivalent of editorUpdateSyntax in kilo
// this is basically the core syntax highlighting algorithm
func (r *Row) updateSyntax() { func (r *Row) updateSyntax() {
i := 0 i := 0
s := r.parent.Syntax s := r.parent.Syntax
@ -129,6 +129,7 @@ func (r *Row) updateSyntax() {
} }
renderStr := string(r.render) renderStr := string(r.render)
renderLen := r.RenderSize()
keywords1 := s.Keywords1 keywords1 := s.Keywords1
keywords2 := s.Keywords2 keywords2 := s.Keywords2
@ -151,7 +152,7 @@ func (r *Row) updateSyntax() {
// Single line comments // Single line comments
if inString == '0' && scsIndex == i { if inString == '0' && scsIndex == i {
for j := scsIndex; j < r.RenderSize(); j++ { for j := scsIndex; j < renderLen; j++ {
r.Hl[j] = highlight.Comment r.Hl[j] = highlight.Comment
} }
break break
@ -160,7 +161,7 @@ func (r *Row) updateSyntax() {
// String literals // String literals
if s.Flags&highlight.DoStrings == highlight.DoStrings { if s.Flags&highlight.DoStrings == highlight.DoStrings {
// At the start of a string literal // At the start of a string literal
if inString == '0' && (ch == '"' || ch == '\'') { if inString == '0' && (ch == char.DoubleQuote || ch == char.SingleQuote) {
inString = ch inString = ch
r.Hl[i] = highlight.String r.Hl[i] = highlight.String
i++ i++
@ -172,7 +173,7 @@ func (r *Row) updateSyntax() {
r.Hl[i] = highlight.String r.Hl[i] = highlight.String
// Handle when a quote is escaped inside a string // Handle when a quote is escaped inside a string
if ch == '\\' && ip1 < r.RenderSize() { if ch == char.Backslash && ip1 < renderLen {
r.Hl[ip1] = highlight.String r.Hl[ip1] = highlight.String
i += 2 i += 2
continue continue
@ -192,8 +193,8 @@ func (r *Row) updateSyntax() {
// Numeric literals // Numeric literals
if s.Flags&highlight.DoNumbers == highlight.DoNumbers { if s.Flags&highlight.DoNumbers == highlight.DoNumbers {
if (unicode.IsDigit(ch) && (prevSep || prevHl == highlight.Number)) || if (char.IsDigit(ch) && prevSep) ||
(ch == '.' && prevHl == highlight.Number) { (char.IsNumeric(ch) && prevHl == highlight.Number) {
r.Hl[i] = highlight.Number r.Hl[i] = highlight.Number
i += 1 i += 1
prevSep = false prevSep = false
@ -203,42 +204,39 @@ func (r *Row) updateSyntax() {
// Keywords // Keywords
if prevSep { if prevSep {
renderLen := r.RenderSize() matched := false
for _, word := range keywords1 { keywordLoop:
for n, list := range [][]string{keywords1, keywords2} {
hlType := highlight.Keyword1
if n == 1 {
hlType = highlight.Keyword2
}
for _, word := range list {
wordLen := len(word) wordLen := len(word)
nextInd := i + wordLen nextInd := i + wordLen
if nextInd >= renderLen || renderStr[i:nextInd] != word { endMatch := renderStr[i:renderLen] == word
continue goodMatch := nextInd < renderLen && renderStr[i:nextInd] == word
}
if renderStr[i:renderLen] == word || key.IsSeparator(r.render[nextInd]) { if endMatch || (goodMatch && char.IsSeparator(r.render[nextInd])) {
for k := i; k < nextInd; k++ { for k := i; k < nextInd; k++ {
r.Hl[k] = highlight.Keyword1 r.Hl[k] = hlType
} }
i += wordLen i += wordLen
break matched = true
break keywordLoop
}
} }
} }
for _, word := range keywords2 { if matched {
wordLen := len(word) prevSep = false
nextInd := i + wordLen
if nextInd >= renderLen || renderStr[i:nextInd] != word {
continue continue
} }
if renderStr[i:renderLen] == word || key.IsSeparator(r.render[nextInd]) {
for k := i; k < nextInd; k++ {
r.Hl[k] = highlight.Keyword2
}
i += wordLen
break
}
}
} }
prevSep = key.IsSeparator(ch) prevSep = char.IsSeparator(ch)
i++ i++
} }
} }

View File

@ -1,9 +1,9 @@
package editor package editor
import ( import (
"timshome.page/gilo/char"
"timshome.page/gilo/editor/document" "timshome.page/gilo/editor/document"
"timshome.page/gilo/internal/gilo" "timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
"timshome.page/gilo/terminal" "timshome.page/gilo/terminal"
) )
@ -28,7 +28,7 @@ const (
*/ */
func (e *Editor) processKeypressChar(ch rune) bool { func (e *Editor) processKeypressChar(ch rune) bool {
switch ch { switch ch {
case key.Ctrl('q'): case char.Ctrl('q'):
if e.doc.IsDirty() && e.quitTimes > 0 { if e.doc.IsDirty() && e.quitTimes > 0 {
e.SetStatusMessage("WARNING!!! File has unsaved changes. Press Ctrl-Q %d more times to quit.", e.quitTimes) e.SetStatusMessage("WARNING!!! File has unsaved changes. Press Ctrl-Q %d more times to quit.", e.quitTimes)
e.quitTimes -= 1 e.quitTimes -= 1
@ -41,21 +41,21 @@ func (e *Editor) processKeypressChar(ch rune) bool {
return false return false
case key.Ctrl('s'): case char.Ctrl('s'):
e.save() e.save()
case key.Ctrl('f'): case char.Ctrl('f'):
e.find() e.find()
case key.Enter: case char.Enter:
e.doc.InsertNewline(e.cursor) e.doc.InsertNewline(e.cursor)
e.cursor.Y += 1 e.cursor.Y += 1
e.cursor.X = 0 e.cursor.X = 0
case key.Backspace, key.Ctrl('h'): case char.Backspace, char.Ctrl('h'):
e.delChar() e.delChar()
case key.Esc, key.Ctrl('l'): case char.Esc, char.Ctrl('l'):
// Modifier keys that return ANSI escape sequences // Modifier keys that return ANSI escape sequences
str := parseEscapeSequence() str := parseEscapeSequence()
@ -190,7 +190,7 @@ func parseEscapeSequence() string {
for i := 0; i < 2; i++ { for i := 0; i < 2; i++ {
ch, size := terminal.ReadKey() ch, size := terminal.ReadKey()
if size == 0 { if size == 0 {
return string(key.Esc) return string(char.Esc)
} }
runes[i] = ch runes[i] = ch
@ -204,7 +204,7 @@ func parseEscapeSequence() string {
} }
} }
return string(key.Esc) return string(char.Esc)
} }
func escSeqToKey(seq []rune) string { func escSeqToKey(seq []rune) string {
@ -255,5 +255,5 @@ func escSeqToKey(seq []rune) string {
} }
} }
return string(key.Esc) return string(char.Esc)
} }

View File

@ -2,8 +2,8 @@ package editor
import ( import (
"testing" "testing"
"timshome.page/gilo/char"
gilo2 "timshome.page/gilo/internal/gilo" gilo2 "timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
) )
type moveCursor struct { type moveCursor struct {
@ -13,7 +13,7 @@ type moveCursor struct {
} }
var cursorTests = []moveCursor{ var cursorTests = []moveCursor{
{[]string{string(key.Esc)}, false, gilo2.DefaultPoint()}, {[]string{string(char.Esc)}, false, gilo2.DefaultPoint()},
{[]string{keyRight}, true, gilo2.NewPoint(1, 0)}, {[]string{keyRight}, true, gilo2.NewPoint(1, 0)},
{[]string{keyDown, keyEnd}, true, gilo2.NewPoint(14, 1)}, {[]string{keyDown, keyEnd}, true, gilo2.NewPoint(14, 1)},
{[]string{keyEnd, keyHome}, true, gilo2.DefaultPoint()}, {[]string{keyEnd, keyHome}, true, gilo2.DefaultPoint()},
@ -63,8 +63,8 @@ var seqTests = []seqTest{
{"[6~", keyPageDown}, {"[6~", keyPageDown},
{"[7~", keyHome}, {"[7~", keyHome},
{"[8~", keyEnd}, {"[8~", keyEnd},
{"OQ", string(key.Esc)}, {"OQ", string(char.Esc)},
{"XZ", string(key.Esc)}, {"XZ", string(char.Esc)},
} }
func TestEscToKey(t *testing.T) { func TestEscToKey(t *testing.T) {

View File

@ -1,9 +1,9 @@
package editor package editor
import ( import (
"timshome.page/gilo/char"
"timshome.page/gilo/editor/highlight" "timshome.page/gilo/editor/highlight"
"timshome.page/gilo/internal/gilo" "timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
) )
type search struct { type search struct {
@ -55,7 +55,7 @@ func (e *Editor) findCallback(query string, ch string) {
e.search.savedhlLine = -1 e.search.savedhlLine = -1
} }
if ch == string(key.Enter) || ch == string(key.Esc) { if ch == string(char.Enter) || ch == string(char.Esc) {
e.search.lastMatch = -1 e.search.lastMatch = -1
e.search.direction = 1 e.search.direction = 1
return return

View File

@ -1,70 +0,0 @@
package key
import "testing"
type isRune struct {
arg1 rune
expected bool
}
// (╯°□°)╯︵ ┻━┻
var isAsciiTest = []isRune{
{'┻', false},
{'$', true},
{'︵', false},
{0x7f, true},
}
func TestIsAscii(t *testing.T) {
for _, test := range isAsciiTest {
if output := IsAscii(test.arg1); output != test.expected {
t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
}
}
}
var isCtrlTest = []isRune{
{0x78, false},
{0x7f, true},
{0x02, true},
{0x98, false},
{'a', false},
}
func TestIsCtrl(t *testing.T) {
for _, test := range isCtrlTest {
if output := IsCtrl(test.arg1); output != test.expected {
t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
}
}
}
type ctrlTest struct {
arg1, expected rune
}
var ctrlTests = []ctrlTest{
{'A', '\x01'},
{'B', '\x02'},
{'Z', '\x1a'},
{'#', 3},
{'┻', 0},
{'😿', 0},
}
func TestCtrl(t *testing.T) {
for _, test := range ctrlTests {
if output := Ctrl(test.arg1); output != test.expected {
t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsSeparator(t *testing.T) {
separators := ",.()+-/*=~%<>[] \t"
for _, r := range separators {
if !IsSeparator(r) {
t.Errorf("Expected %q to be a syntax separator", r)
}
}
}