Refactor keyword highlighting to handle more edge cases
All checks were successful
timw4mail/gilo/pipeline/head This commit looks good

This commit is contained in:
Timothy Warren 2023-10-06 11:30:21 -04:00
parent 5ff459b6ad
commit ceed34c634
8 changed files with 173 additions and 129 deletions

View File

@ -1,4 +1,4 @@
package key
package char
import (
"strings"
@ -11,8 +11,11 @@ import (
const (
Backspace = '\x7f'
Backslash = '\\'
Esc = '\x1b'
Enter = '\r'
SingleQuote = '\''
DoubleQuote = '"'
)
// IsAscii Is this an ASCII character?
@ -44,3 +47,13 @@ func Ctrl(char rune) rune {
func IsSeparator(char rune) bool {
return unicode.IsSpace(char) || strings.ContainsRune(",.()+-/*=~%<>[];", char)
}
// IsDigit is a simple wrapper around built-in unicode.IsDigit
func IsDigit(char rune) bool {
return unicode.IsDigit(char)
}
// IsNumeric checks whether the character is a digit, or a numeric separator
func IsNumeric(char rune) bool {
return IsDigit(char) || char == '.'
}

103
char/char_test.go Normal file
View File

@ -0,0 +1,103 @@
package char
import "testing"
type isA struct {
arg1 rune
expected bool
}
func TestIsAscii(t *testing.T) {
// (╯°□°)╯︵ ┻━┻
var isAsciiTest = []isA{
{'┻', false},
{'$', true},
{'︵', false},
{0x7f, true},
}
for _, test := range isAsciiTest {
if output := IsAscii(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsCtrl(t *testing.T) {
var isCtrlTest = []isA{
{0x78, false},
{0x7f, true},
{0x02, true},
{0x98, false},
{'a', false},
}
for _, test := range isCtrlTest {
if output := IsCtrl(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestCtrl(t *testing.T) {
type ctrlTest struct {
arg1, expected rune
}
var ctrlTests = []ctrlTest{
{'A', '\x01'},
{'B', '\x02'},
{'Z', '\x1a'},
{'#', 3},
{'┻', 0},
{'😿', 0},
}
for _, test := range ctrlTests {
if output := Ctrl(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsSeparator(t *testing.T) {
for _, r := range ",.()+-/*=~%<>[] \t" {
if !IsSeparator(r) {
t.Errorf("Expected %q to be a syntax separator", r)
}
}
}
func TestIsDigit(t *testing.T) {
var digitTests = []isA{
{'0', true},
{'1', true},
{'.', false},
{'A', false},
{'😿', false},
{'$', false},
}
for _, test := range digitTests {
if output := IsDigit(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsNumeric(t *testing.T) {
var numericTests = []isA{
{'0', true},
{'1', true},
{'.', true},
{'A', false},
{'😿', false},
{'$', false},
}
for _, test := range numericTests {
if output := IsNumeric(test.arg1); output != test.expected {
t.Errorf("Output '%v' not equal to expected '%v' for input %q", output, test.expected, test.arg1)
}
}
}

View File

@ -4,9 +4,9 @@ package editor
import (
"fmt"
"time"
"timshome.page/gilo/char"
"timshome.page/gilo/editor/document"
"timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
"timshome.page/gilo/terminal"
)
@ -104,7 +104,7 @@ func (e *Editor) prompt(prompt string, callback func(string, string)) string {
}
ch, _ := terminal.ReadKey()
if ch == key.Enter {
if ch == char.Enter {
if buf.Len() != 0 {
e.SetStatusMessage("")
if callback != nil {
@ -113,15 +113,15 @@ func (e *Editor) prompt(prompt string, callback func(string, string)) string {
return buf.ToString()
}
} else if key.IsAscii(ch) && !key.IsCtrl(ch) {
} else if char.IsAscii(ch) && !char.IsCtrl(ch) {
buf.AppendRune(ch)
} else if ch == key.Backspace || ch == key.Ctrl('h') {
} else if ch == char.Backspace || ch == char.Ctrl('h') {
buf.Truncate(buf.Len() - 1)
} else if ch == key.Esc {
} else if ch == char.Esc {
k := parseEscapeSequence()
if k == keyDelete {
buf.Truncate(buf.Len() - 1)
} else if k == string(key.Esc) {
} else if k == string(char.Esc) {
e.SetStatusMessage("")
if callback != nil {
callback(buf.ToString(), k)

View File

@ -2,10 +2,9 @@ package document
import (
"strings"
"timshome.page/gilo/char"
"timshome.page/gilo/editor/highlight"
"timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
"unicode"
)
type Row struct {
@ -115,6 +114,7 @@ func (r *Row) update() {
}
// updateSyntax is the equivalent of editorUpdateSyntax in kilo
// this is basically the core syntax highlighting algorithm
func (r *Row) updateSyntax() {
i := 0
s := r.parent.Syntax
@ -129,6 +129,7 @@ func (r *Row) updateSyntax() {
}
renderStr := string(r.render)
renderLen := r.RenderSize()
keywords1 := s.Keywords1
keywords2 := s.Keywords2
@ -151,7 +152,7 @@ func (r *Row) updateSyntax() {
// Single line comments
if inString == '0' && scsIndex == i {
for j := scsIndex; j < r.RenderSize(); j++ {
for j := scsIndex; j < renderLen; j++ {
r.Hl[j] = highlight.Comment
}
break
@ -160,7 +161,7 @@ func (r *Row) updateSyntax() {
// String literals
if s.Flags&highlight.DoStrings == highlight.DoStrings {
// At the start of a string literal
if inString == '0' && (ch == '"' || ch == '\'') {
if inString == '0' && (ch == char.DoubleQuote || ch == char.SingleQuote) {
inString = ch
r.Hl[i] = highlight.String
i++
@ -172,7 +173,7 @@ func (r *Row) updateSyntax() {
r.Hl[i] = highlight.String
// Handle when a quote is escaped inside a string
if ch == '\\' && ip1 < r.RenderSize() {
if ch == char.Backslash && ip1 < renderLen {
r.Hl[ip1] = highlight.String
i += 2
continue
@ -192,8 +193,8 @@ func (r *Row) updateSyntax() {
// Numeric literals
if s.Flags&highlight.DoNumbers == highlight.DoNumbers {
if (unicode.IsDigit(ch) && (prevSep || prevHl == highlight.Number)) ||
(ch == '.' && prevHl == highlight.Number) {
if (char.IsDigit(ch) && prevSep) ||
(char.IsNumeric(ch) && prevHl == highlight.Number) {
r.Hl[i] = highlight.Number
i += 1
prevSep = false
@ -203,42 +204,39 @@ func (r *Row) updateSyntax() {
// Keywords
if prevSep {
renderLen := r.RenderSize()
matched := false
for _, word := range keywords1 {
keywordLoop:
for n, list := range [][]string{keywords1, keywords2} {
hlType := highlight.Keyword1
if n == 1 {
hlType = highlight.Keyword2
}
for _, word := range list {
wordLen := len(word)
nextInd := i + wordLen
if nextInd >= renderLen || renderStr[i:nextInd] != word {
continue
}
endMatch := renderStr[i:renderLen] == word
goodMatch := nextInd < renderLen && renderStr[i:nextInd] == word
if renderStr[i:renderLen] == word || key.IsSeparator(r.render[nextInd]) {
if endMatch || (goodMatch && char.IsSeparator(r.render[nextInd])) {
for k := i; k < nextInd; k++ {
r.Hl[k] = highlight.Keyword1
r.Hl[k] = hlType
}
i += wordLen
break
matched = true
break keywordLoop
}
}
}
for _, word := range keywords2 {
wordLen := len(word)
nextInd := i + wordLen
if nextInd >= renderLen || renderStr[i:nextInd] != word {
if matched {
prevSep = false
continue
}
if renderStr[i:renderLen] == word || key.IsSeparator(r.render[nextInd]) {
for k := i; k < nextInd; k++ {
r.Hl[k] = highlight.Keyword2
}
i += wordLen
break
}
}
}
prevSep = key.IsSeparator(ch)
prevSep = char.IsSeparator(ch)
i++
}
}

View File

@ -1,9 +1,9 @@
package editor
import (
"timshome.page/gilo/char"
"timshome.page/gilo/editor/document"
"timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
"timshome.page/gilo/terminal"
)
@ -28,7 +28,7 @@ const (
*/
func (e *Editor) processKeypressChar(ch rune) bool {
switch ch {
case key.Ctrl('q'):
case char.Ctrl('q'):
if e.doc.IsDirty() && e.quitTimes > 0 {
e.SetStatusMessage("WARNING!!! File has unsaved changes. Press Ctrl-Q %d more times to quit.", e.quitTimes)
e.quitTimes -= 1
@ -41,21 +41,21 @@ func (e *Editor) processKeypressChar(ch rune) bool {
return false
case key.Ctrl('s'):
case char.Ctrl('s'):
e.save()
case key.Ctrl('f'):
case char.Ctrl('f'):
e.find()
case key.Enter:
case char.Enter:
e.doc.InsertNewline(e.cursor)
e.cursor.Y += 1
e.cursor.X = 0
case key.Backspace, key.Ctrl('h'):
case char.Backspace, char.Ctrl('h'):
e.delChar()
case key.Esc, key.Ctrl('l'):
case char.Esc, char.Ctrl('l'):
// Modifier keys that return ANSI escape sequences
str := parseEscapeSequence()
@ -190,7 +190,7 @@ func parseEscapeSequence() string {
for i := 0; i < 2; i++ {
ch, size := terminal.ReadKey()
if size == 0 {
return string(key.Esc)
return string(char.Esc)
}
runes[i] = ch
@ -204,7 +204,7 @@ func parseEscapeSequence() string {
}
}
return string(key.Esc)
return string(char.Esc)
}
func escSeqToKey(seq []rune) string {
@ -255,5 +255,5 @@ func escSeqToKey(seq []rune) string {
}
}
return string(key.Esc)
return string(char.Esc)
}

View File

@ -2,8 +2,8 @@ package editor
import (
"testing"
"timshome.page/gilo/char"
gilo2 "timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
)
type moveCursor struct {
@ -13,7 +13,7 @@ type moveCursor struct {
}
var cursorTests = []moveCursor{
{[]string{string(key.Esc)}, false, gilo2.DefaultPoint()},
{[]string{string(char.Esc)}, false, gilo2.DefaultPoint()},
{[]string{keyRight}, true, gilo2.NewPoint(1, 0)},
{[]string{keyDown, keyEnd}, true, gilo2.NewPoint(14, 1)},
{[]string{keyEnd, keyHome}, true, gilo2.DefaultPoint()},
@ -63,8 +63,8 @@ var seqTests = []seqTest{
{"[6~", keyPageDown},
{"[7~", keyHome},
{"[8~", keyEnd},
{"OQ", string(key.Esc)},
{"XZ", string(key.Esc)},
{"OQ", string(char.Esc)},
{"XZ", string(char.Esc)},
}
func TestEscToKey(t *testing.T) {

View File

@ -1,9 +1,9 @@
package editor
import (
"timshome.page/gilo/char"
"timshome.page/gilo/editor/highlight"
"timshome.page/gilo/internal/gilo"
"timshome.page/gilo/key"
)
type search struct {
@ -55,7 +55,7 @@ func (e *Editor) findCallback(query string, ch string) {
e.search.savedhlLine = -1
}
if ch == string(key.Enter) || ch == string(key.Esc) {
if ch == string(char.Enter) || ch == string(char.Esc) {
e.search.lastMatch = -1
e.search.direction = 1
return

View File

@ -1,70 +0,0 @@
package key
import "testing"
type isRune struct {
arg1 rune
expected bool
}
// (╯°□°)╯︵ ┻━┻
var isAsciiTest = []isRune{
{'┻', false},
{'$', true},
{'︵', false},
{0x7f, true},
}
func TestIsAscii(t *testing.T) {
for _, test := range isAsciiTest {
if output := IsAscii(test.arg1); output != test.expected {
t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
}
}
}
var isCtrlTest = []isRune{
{0x78, false},
{0x7f, true},
{0x02, true},
{0x98, false},
{'a', false},
}
func TestIsCtrl(t *testing.T) {
for _, test := range isCtrlTest {
if output := IsCtrl(test.arg1); output != test.expected {
t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
}
}
}
type ctrlTest struct {
arg1, expected rune
}
var ctrlTests = []ctrlTest{
{'A', '\x01'},
{'B', '\x02'},
{'Z', '\x1a'},
{'#', 3},
{'┻', 0},
{'😿', 0},
}
func TestCtrl(t *testing.T) {
for _, test := range ctrlTests {
if output := Ctrl(test.arg1); output != test.expected {
t.Errorf("Output %v not equal to expected %v for input %q", output, test.expected, test.arg1)
}
}
}
func TestIsSeparator(t *testing.T) {
separators := ",.()+-/*=~%<>[] \t"
for _, r := range separators {
if !IsSeparator(r) {
t.Errorf("Expected %q to be a syntax separator", r)
}
}
}