tree sitter syntax highlighting

memory-refactor
Patrick Cleaveliln 2025-07-18 03:57:56 +00:00
parent 9f9ddaa198
commit 86b2dcfbfb
10 changed files with 488 additions and 14 deletions

View File

@ -1,6 +1,6 @@
export RUSTFLAGS=-C target-feature=-avx2
all: editor
all: bin/libtree-sitter.a editor
editor: grep src/**/*.odin
mkdir -p bin
@ -10,4 +10,18 @@ grep:
cargo build --manifest-path "src/pkg/grep_lib/Cargo.toml"
test: src/**/*.odin
odin test src/tests/ -all-packages -debug -out:bin/test_runner
odin test src/tests/ -all-packages -debug -out:bin/test_runner
TS_DIR := third_party/tree-sitter/lib
TS_SRC := $(wildcard $(TS_DIR)/src/*.c)
TS_OBJ := $(TS_SRC:.c=.o)
TS_ARFLAGS := rcs
CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE
override CFLAGS += -I$(TS_DIR)/src -I$(TS_DIR)/src/wasm -I$(TS_DIR)/include
override CFLAGS += -o bin/
bin/libtree-sitter.a: $(TS_OBJ)
$(AR) $(TS_ARFLAGS) $@ $^ --output bin/

View File

@ -10,6 +10,7 @@ import "core:slice"
import "base:runtime"
import "core:strings"
import ts "../tree_sitter"
import "../theme"
ScrollDir :: enum {
@ -46,9 +47,10 @@ FileBuffer :: struct {
extension: string,
top_line: int,
// cursor: Cursor,
selection: Maybe(Selection),
tree: ts.State,
history: FileHistory,
glyphs: GlyphBuffer,
@ -640,8 +642,8 @@ swap_selections :: proc(selection: Selection) -> (swapped: Selection) {
// TODO: don't access PieceTableIndex directly
is_selection_inverted :: proc(selection: Selection) -> bool {
return selection.start.index.chunk_index > selection.end.index.chunk_index ||
(selection.start.index.chunk_index == selection.end.index.chunk_index
&& selection.start.index.char_index > selection.end.index.char_index)
(selection.start.index.chunk_index == selection.end.index.chunk_index &&
selection.start.index.char_index > selection.end.index.char_index)
}
selection_length :: proc(buffer: ^FileBuffer, selection: Selection) -> int {
@ -707,7 +709,7 @@ new_file_buffer :: proc(allocator: mem.Allocator, file_path: string, base_dir: s
width := 256;
height := 256;
fmt.eprintln("file path", fi.fullpath[4:]);
fmt.eprintln("file path", fi.fullpath[:]);
buffer := FileBuffer {
allocator = allocator,
@ -717,18 +719,46 @@ new_file_buffer :: proc(allocator: mem.Allocator, file_path: string, base_dir: s
// file_path = fi.fullpath[4:],
extension = extension,
// TODO: derive language type from extension
tree = ts.make_state(.Odin),
history = make_history(original_content),
glyphs = make_glyph_buffer(width, height),
input_buffer = make([dynamic]u8, 0, 1024),
};
ts.parse_buffer(&buffer.tree, tree_sitter_file_buffer_input(&buffer))
return buffer, error();
} else {
return FileBuffer{}, error(ErrorType.FileIOError, fmt.aprintf("failed to read from file"));
}
}
tree_sitter_file_buffer_input :: proc(buffer: ^FileBuffer) -> ts.Input {
read :: proc "c" (payload: rawptr, byte_index: u32, position: ts.Point, bytes_read: ^u32) -> ^u8 {
context = runtime.default_context()
buffer := transmute(^FileBuffer)payload
if iter, ok := new_piece_table_iter_from_byte_offset(&buffer.history.piece_table, int(byte_index)); ok {
bytes := iter.t.chunks[iter.index.chunk_index][iter.index.char_index:]
bytes_read^ = u32(len(bytes))
return raw_data(bytes)
} else {
bytes_read^ = 0
return nil
}
}
return ts.Input {
payload = buffer,
read = read,
encoding = .UTF8,
}
}
save_buffer_to_disk :: proc(state: ^State, buffer: ^FileBuffer) -> (error: os.Error) {
fd := os.open(buffer.file_path, flags = os.O_WRONLY | os.O_TRUNC | os.O_CREATE) or_return;
defer os.close(fd);
@ -760,6 +790,7 @@ next_buffer :: proc(state: ^State, prev_buffer: ^int) -> int {
// TODO: replace this with arena for the file buffer
free_file_buffer :: proc(buffer: ^FileBuffer) {
ts.delete_state(&buffer.tree)
free_history(&buffer.history)
delete(buffer.glyphs.buffer)
delete(buffer.input_buffer)

View File

@ -2,6 +2,7 @@ package core
import "core:fmt"
import ts "../tree_sitter"
import "../theme"
GlyphBuffer :: struct {
@ -27,7 +28,21 @@ make_glyph_buffer :: proc(width, height: int, allocator := context.allocator) ->
update_glyph_buffer_from_file_buffer :: proc(buffer: ^FileBuffer) {
for &glyph in buffer.glyphs.buffer {
glyph = Glyph{};
glyph = Glyph {}
glyph.color = .Foreground
}
outer: for highlight in buffer.tree.highlights {
for line in highlight.start.row..=highlight.end.row {
if int(line) < buffer.top_line { continue; }
screen_line := int(line) - buffer.top_line
if screen_line >= buffer.glyphs.height { break outer; }
for col in highlight.start.column..<highlight.end.column {
buffer.glyphs.buffer[int(col) + screen_line * buffer.glyphs.width].color = highlight.color;
}
}
}
begin := buffer.top_line;
@ -71,7 +86,7 @@ update_glyph_buffer_from_file_buffer :: proc(buffer: ^FileBuffer) {
}
if rendered_line >= begin && rendered_col < buffer.glyphs.width {
buffer.glyphs.buffer[rendered_col + screen_line * buffer.glyphs.width] = Glyph { codepoint = character, color = .Foreground };
buffer.glyphs.buffer[rendered_col + screen_line * buffer.glyphs.width].codepoint = character
}
rendered_col += 1;

View File

@ -71,6 +71,28 @@ new_piece_table_iter_from_index :: proc(t: ^PieceTable, index: PieceTableIndex)
}
}
new_piece_table_iter_from_byte_offset :: proc(t: ^PieceTable, byte_offset: int) -> (iter: PieceTableIter, ok: bool) {
bytes := 0
for chunk, chunk_index in t.chunks {
if bytes + len(chunk) > byte_offset {
char_index := byte_offset - bytes
return PieceTableIter {
t = t,
index = PieceTableIndex {
chunk_index = chunk_index,
char_index = char_index,
}
}, true
} else {
bytes += len(chunk)
}
}
return
}
new_piece_table_index_from_end :: proc(t: ^PieceTable) -> PieceTableIndex {
chunk_index := len(t.chunks)-1
char_index := len(t.chunks[chunk_index])-1

View File

@ -18,6 +18,7 @@ import "core"
import "panels"
import "theme"
import "ui"
import ts "tree_sitter"
State :: core.State;
FileBuffer :: core.FileBuffer;
@ -85,7 +86,7 @@ draw :: proc(state: ^State) {
ui.draw(new_ui, state)
// TODO: figure out when to not show the input help menu
if state.mode != .Insert { // && state.current_input_map != &state.input_map.mode[state.mode] {
if false && state.mode != .Insert { // && state.current_input_map != &state.input_map.mode[state.mode] {
longest_description := 0;
for key, action in state.current_input_map.key_actions {
if len(action.description) > longest_description {
@ -200,6 +201,8 @@ ui_file_buffer :: proc(s: ^ui.State, buffer: ^FileBuffer) {
}
main :: proc() {
ts.set_allocator()
_command_arena: mem.Arena
mem.arena_init(&_command_arena, make([]u8, 1024*1024));

View File

@ -9,6 +9,7 @@ import "core:log"
import "vendor:sdl2"
import ts "../tree_sitter"
import "../core"
import "../input"
import "../util"
@ -65,6 +66,15 @@ register_default_leader_actions :: proc(input_map: ^core.InputActions) {
core.register_key_action(input_map, .R, proc(state: ^core.State) {
open_grep_panel(state)
}, "Grep Workspace")
core.register_key_action(input_map, .K, proc(state: ^core.State) {
buffer := core.current_buffer(state)
ts.update_cursor(&buffer.tree, buffer.history.cursor.line, buffer.history.cursor.col)
ts.print_node_type(&buffer.tree)
core.reset_input_map(state)
}, "View Symbol")
}
register_default_panel_actions :: proc(input_map: ^core.InputActions) {
@ -282,6 +292,11 @@ make_file_buffer_panel :: proc(buffer_index: int) -> core.Panel {
return &state.buffers[panel_state.buffer_index], true
},
drop = proc(state: ^core.State, panel_state: ^core.PanelState) {
if panel_state, ok := &panel_state.(core.FileBufferPanel); ok {
core.free_file_buffer(&state.buffers[panel_state.buffer_index])
}
},
render_proc = proc(state: ^core.State, panel_state: ^core.PanelState) -> (ok: bool) {
panel_state := panel_state.(core.FileBufferPanel) or_return;
s := transmute(^ui.State)state.ui
@ -426,6 +441,7 @@ make_grep_panel :: proc(state: ^core.State) -> core.Panel {
},
drop = proc(state: ^core.State, panel_state: ^core.PanelState) {
if panel_state, ok := &panel_state.(core.GrepPanel); ok {
// core.free_file_buffer(&state.buffers[panel_state.buffer])
delete(panel_state.query_arena.data, state.ctx.allocator)
}
},

367
src/tree_sitter/ts.odin Normal file
View File

@ -0,0 +1,367 @@
package tree_sitter
import "base:runtime"
import "core:strings"
import "core:fmt"
import "core:log"
import "core:os"
import "core:mem"
import "../theme"
foreign import ts "../../bin/libtree-sitter.a"
@(default_calling_convention = "c", link_prefix="ts_")
foreign ts {
parser_new :: proc() -> Parser ---
parser_delete :: proc(parser: Parser) -> Parser ---
parser_set_language :: proc(parser: Parser, language: Language) -> bool ---
parser_set_logger :: proc(parser: Parser, logger: TSLogger) ---
parser_print_dot_graphs :: proc(parser: Parser, fd: int) ---
parser_parse :: proc(parser: Parser, old_tree: Tree, input: Input) -> Tree ---
parser_parse_string :: proc(parser: Parser, old_tree: Tree, source: []u8, len: u32) -> Tree ---
tree_root_node :: proc(tree: Tree) -> Node ---
tree_delete :: proc(tree: Tree) ---
tree_cursor_new :: proc(node: Node) -> TreeCursor ---
tree_cursor_reset :: proc(tree: ^TreeCursor, node: Node) ---
tree_cursor_delete :: proc(tree: ^TreeCursor) ---
tree_cursor_current_node :: proc(tree: ^TreeCursor) -> Node ---
tree_cursor_current_field_name :: proc(tree: ^TreeCursor) -> cstring ---
tree_cursor_goto_first_child :: proc(self: ^TreeCursor) -> bool ---
tree_cursor_goto_first_child_for_point :: proc(self: ^TreeCursor, goal_point: Point) -> u64 ---
node_start_point :: proc(self: Node) -> Point ---
node_end_point :: proc(self: Node) -> Point ---
node_type :: proc(self: Node) -> cstring ---
node_named_child :: proc(self: Node, child_index: u32) -> Node ---
node_child_count :: proc(self: Node) -> u32 ---
node_is_null :: proc(self: Node) -> bool ---
node_string :: proc(self: Node) -> cstring ---
query_new :: proc(language: Language, source: []u8, source_len: u32, error_offset: ^u32, error_type: ^QueryError) -> Query ---
query_delete :: proc(query: Query) ---
query_cursor_new :: proc() -> QueryCursor ---
query_cursor_exec :: proc(cursor: QueryCursor, query: Query, node: Node) ---
query_cursor_next_match :: proc(cursor: QueryCursor, match: ^QueryMatch) -> bool ---
query_capture_name_for_id :: proc(query: Query, index: u32, length: ^u32) -> ^u8 ---
@(link_name = "ts_set_allocator")
ts_set_allocator :: proc(new_malloc: MallocProc, new_calloc: CAllocProc, new_realloc: ReAllocProc, new_free: FreeProc) ---
}
TS_ALLOCATOR: mem.Allocator
MallocProc :: proc "c" (size: uint) -> rawptr
CAllocProc :: proc "c" (num: uint, size: uint) -> rawptr
ReAllocProc :: proc "c" (ptr: rawptr, size: uint) -> rawptr
FreeProc :: proc "c" (ptr: rawptr)
set_allocator :: proc(allocator := context.allocator) {
TS_ALLOCATOR = allocator
new_malloc :: proc "c" (size: uint) -> rawptr {
context = runtime.default_context()
data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Alloc, int(size), runtime.DEFAULT_ALIGNMENT, nil, 0)
return raw_data(data)
}
new_calloc :: proc "c" (num: uint, size: uint) -> rawptr {
context = runtime.default_context()
data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Alloc, int(num * size), runtime.DEFAULT_ALIGNMENT, nil, 0)
return raw_data(data)
}
new_realloc :: proc "c" (old_ptr: rawptr, size: uint) -> rawptr {
context = runtime.default_context()
data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Resize, int(size), runtime.DEFAULT_ALIGNMENT, old_ptr, 0)
return raw_data(data)
}
new_free :: proc "c" (ptr: rawptr) {
context = runtime.default_context()
TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Free, 0, runtime.DEFAULT_ALIGNMENT, ptr, 0)
}
}
foreign import ts_odin "../../bin/libtree-sitter-odin.a"
foreign ts_odin {
tree_sitter_odin :: proc "c" () -> Language ---
}
foreign import ts_json "../../bin/libtree-sitter-json.a"
foreign ts_json {
tree_sitter_json :: proc "c" () -> Language ---
}
State :: struct {
parser: Parser,
language: Language,
tree: Tree,
cursor: TreeCursor,
highlights: [dynamic]Highlight,
}
Highlight :: struct {
start: Point,
end: Point,
color: theme.PaletteColor,
}
LanguageType :: enum {
Json,
Odin,
}
TestStuff :: struct {
start: Point,
end: Point,
}
Parser :: distinct rawptr
Language :: distinct rawptr
Query :: distinct rawptr
QueryCursor :: distinct rawptr
QueryError :: enum {
None = 0,
Syntax,
NodeType,
Field,
Capture,
}
QueryCapture :: struct {
node: Node,
index: u32,
}
QueryMatch :: struct {
id: u32,
pattern_index: u16,
capture_count: u16,
captures: [^]QueryCapture,
}
DecodeFunction :: proc "c" (text: []u8, length: u32, code_point: ^u32) -> u32
Input :: struct {
payload: rawptr,
read: proc "c" (payload: rawptr, byte_index: u32, position: Point, bytes_read: ^u32) -> ^u8,
encoding: InputEncoding,
decode: DecodeFunction,
}
InputEncoding :: enum {
UTF8 = 0,
UTF16LE,
UTF16BE,
Custom,
}
Tree :: distinct rawptr
TreeCursor :: struct {
tree: rawptr,
id: rawptr,
ctx: [3]u32,
}
Node :: struct {
ctx: [4]u32,
id: rawptr,
tree: Tree,
}
Point :: struct {
row: u32,
column: u32
}
TSLogType :: enum { Parse, Lex }
TSLogger :: struct {
log: proc "c" (payload: rawptr, log_type: TSLogType, msg: cstring),
payload: rawptr,
}
log_callback :: proc "c" (payload: rawptr, log_type: TSLogType, msg: cstring) {
context = runtime.default_context()
fmt.printf("Tree-sitter log: %s", msg)
}
make_state :: proc(type: LanguageType, allocator := context.allocator) -> State {
context.allocator = allocator
parser := parser_new()
parser_set_logger(parser, TSLogger{log = log_callback, payload = nil})
language: Language
switch (type) {
case .Odin: language = tree_sitter_odin()
case .Json: language = tree_sitter_json()
}
if !parser_set_language(parser, language) {
log.errorf("failed to set language to '%v'", type)
return State {}
}
return State {
parser = parser,
language = language
}
}
delete_state :: proc(state: ^State) {
tree_cursor_delete(&state.cursor)
tree_delete(state.tree)
parser_delete(state.parser)
}
parse_buffer :: proc(state: ^State, input: Input) {
old_tree := state.tree
if old_tree != nil {
defer tree_delete(old_tree)
}
state.tree = parser_parse(state.parser, nil, input)
if state.tree == nil {
log.error("failed to parse buffer")
return
}
state.cursor = tree_cursor_new(tree_root_node(state.tree))
load_highlights(state)
}
update_cursor :: proc(state: ^State, line: int, col: int) {
assert(state.tree != nil)
root_node := tree_root_node(state.tree)
tree_cursor_reset(&state.cursor, root_node)
node := tree_cursor_current_node(&state.cursor)
for node_child_count(node) > 1 {
if tree_cursor_goto_first_child_for_point(&state.cursor, Point {
row = u32(line),
column = u32(col),
}) < 0 {
break
}
node = tree_cursor_current_node(&state.cursor)
}
}
load_highlights :: proc(state: ^State) {
// TODO: have this be language specific
capture_to_color := make(map[string]theme.PaletteColor, allocator = context.temp_allocator)
capture_to_color["include"] = .Red
capture_to_color["keyword.function"] = .Red
capture_to_color["storageclass"] = .Red
capture_to_color["keyword.operator"] = .Purple
capture_to_color["keyword"] = .Blue
capture_to_color["repeat"] = .Blue
capture_to_color["conditional"] = .Blue
capture_to_color["function"] = .Blue
capture_to_color["type.decl"] = .BrightBlue
capture_to_color["field"] = .BrightYellow
capture_to_color["type.builtin"] = .Aqua
capture_to_color["function.call"] = .Green
capture_to_color["string"] = .Green
capture_to_color["comment"] = .Gray
fd, err := os.open("../tree-sitter-odin/queries/highlights.scm")
if err != nil {
log.errorf("failed to open file: errno=%x", err)
return
}
defer os.close(fd);
if highlight_query, success := os.read_entire_file_from_handle(fd); success {
error_offset: u32
error_type: QueryError
query := query_new(state.language, highlight_query, u32(len(highlight_query)), &error_offset, &error_type)
defer query_delete(query)
if error_type != .None {
log.errorf("got error: '%v'", error_type)
return
}
cursor := query_cursor_new()
query_cursor_exec(cursor, query, tree_root_node(state.tree))
if state.highlights != nil {
clear(&state.highlights)
} else {
state.highlights = make([dynamic]Highlight)
}
match: QueryMatch
for query_cursor_next_match(cursor, &match) {
for i in 0..<match.capture_count {
cap := &match.captures[i]
start := node_start_point(cap.node)
end := node_end_point(cap.node)
length: u32
name := query_capture_name_for_id(query, cap.index, &length)
node_type := string(node_type(cap.node))
capture_name := strings.string_from_ptr(name, int(length))
if color, ok := capture_to_color[capture_name]; ok {
append(&state.highlights, Highlight { start = start, end = end, color = color })
}
// if color, ok := capture_to_color[node_type]; ok {
// append(&state.highlights, Highlight { start = start, end = end, color = color })
// }
}
}
}
}
print_node_type :: proc(state: ^State) {
current_node := tree_cursor_current_node(&state.cursor)
if node_is_null(current_node) {
log.error("Current node is null after goto_first_child")
return
}
node_type_str := node_type(current_node)
fmt.println("\n")
log.infof("Current node type: %s", node_type_str)
name := tree_cursor_current_field_name(&state.cursor)
if name == nil {
log.info("No field name for current node")
} else {
log.infof("Field name: %s", name)
}
start_point := node_start_point(current_node)
end_point := node_end_point(current_node)
log.infof("Node position: (%d:%d) to (%d:%d)",
start_point.row+1, start_point.column+1,
end_point.row+1, end_point.column+1)
}

1
third_party/tree-sitter/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
lib/src/*.o

View File

@ -57,7 +57,7 @@ override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
all: libtree-sitter.a
libtree-sitter.a: $(OBJ)
$(AR) $(ARFLAGS) $@ $^
$(AR) $(ARFLAGS) $@ $^
# libtree-sitter.$(SOEXT): $(OBJ)
# $(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@

13
todo.md
View File

@ -1,13 +1,14 @@
# Bugs
- Fix crash when cursor is over a new-line
- Memory Leak
- Fix jumping forward a word jumping past consecutive brackets
- Odd scrolling behavior on small screen heights
- Closing the only panel crashes
- Scrolling past end/beginning of results panics
# Visual QOL
- Split grep search results into a table to avoid funky unaligned text
# Planned Features
- Use grouped lifetimes exclusively for memory allocation/freeing
- [ ] Highlight which panel is currently active
- [ ] Persist end of line cursor position
- Testing Harness
@ -29,8 +30,12 @@
- [ ] Go-to Definition/
- [ ] Find references
- Re-implement lost features from Plugins
- [ ] Syntax Highlighting
- [ ] Integrate tree-sitter
- [ ] Integrate tree-sitter
- [x] Syntax Highlighting
- [ ] Auto Setup Parsers
- [ ] Download parser
- [ ] Compile/"Install"
- [ ] Auto-indent?
- [ ] Bootleg Telescope
- [ ] Grepping Files
- [x] Query across project