diff --git a/Makefile b/Makefile index 866b113..ebd5741 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ export RUSTFLAGS=-C target-feature=-avx2 -all: editor +all: bin/libtree-sitter.a editor editor: grep src/**/*.odin mkdir -p bin @@ -10,4 +10,18 @@ grep: cargo build --manifest-path "src/pkg/grep_lib/Cargo.toml" test: src/**/*.odin - odin test src/tests/ -all-packages -debug -out:bin/test_runner \ No newline at end of file + odin test src/tests/ -all-packages -debug -out:bin/test_runner + +TS_DIR := third_party/tree-sitter/lib +TS_SRC := $(wildcard $(TS_DIR)/src/*.c) +TS_OBJ := $(TS_SRC:.c=.o) + +TS_ARFLAGS := rcs +CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types +override CFLAGS += -std=c11 -fPIC -fvisibility=hidden +override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE +override CFLAGS += -I$(TS_DIR)/src -I$(TS_DIR)/src/wasm -I$(TS_DIR)/include +override CFLAGS += -o bin/ + +bin/libtree-sitter.a: $(TS_OBJ) + $(AR) $(TS_ARFLAGS) $@ $^ --output bin/ diff --git a/src/core/file_buffer.odin b/src/core/file_buffer.odin index 62cd765..c79df99 100644 --- a/src/core/file_buffer.odin +++ b/src/core/file_buffer.odin @@ -10,6 +10,7 @@ import "core:slice" import "base:runtime" import "core:strings" +import ts "../tree_sitter" import "../theme" ScrollDir :: enum { @@ -46,9 +47,10 @@ FileBuffer :: struct { extension: string, top_line: int, - // cursor: Cursor, selection: Maybe(Selection), + tree: ts.State, + history: FileHistory, glyphs: GlyphBuffer, @@ -640,8 +642,8 @@ swap_selections :: proc(selection: Selection) -> (swapped: Selection) { // TODO: don't access PieceTableIndex directly is_selection_inverted :: proc(selection: Selection) -> bool { return selection.start.index.chunk_index > selection.end.index.chunk_index || - (selection.start.index.chunk_index == selection.end.index.chunk_index - && selection.start.index.char_index > selection.end.index.char_index) + (selection.start.index.chunk_index == selection.end.index.chunk_index && + selection.start.index.char_index > selection.end.index.char_index) } selection_length :: proc(buffer: ^FileBuffer, selection: Selection) -> int { @@ -707,7 +709,7 @@ new_file_buffer :: proc(allocator: mem.Allocator, file_path: string, base_dir: s width := 256; height := 256; - fmt.eprintln("file path", fi.fullpath[4:]); + fmt.eprintln("file path", fi.fullpath[:]); buffer := FileBuffer { allocator = allocator, @@ -717,18 +719,46 @@ new_file_buffer :: proc(allocator: mem.Allocator, file_path: string, base_dir: s // file_path = fi.fullpath[4:], extension = extension, + // TODO: derive language type from extension + tree = ts.make_state(.Odin), history = make_history(original_content), glyphs = make_glyph_buffer(width, height), input_buffer = make([dynamic]u8, 0, 1024), }; + ts.parse_buffer(&buffer.tree, tree_sitter_file_buffer_input(&buffer)) + return buffer, error(); } else { return FileBuffer{}, error(ErrorType.FileIOError, fmt.aprintf("failed to read from file")); } } +tree_sitter_file_buffer_input :: proc(buffer: ^FileBuffer) -> ts.Input { + read :: proc "c" (payload: rawptr, byte_index: u32, position: ts.Point, bytes_read: ^u32) -> ^u8 { + context = runtime.default_context() + + buffer := transmute(^FileBuffer)payload + + if iter, ok := new_piece_table_iter_from_byte_offset(&buffer.history.piece_table, int(byte_index)); ok { + bytes := iter.t.chunks[iter.index.chunk_index][iter.index.char_index:] + bytes_read^ = u32(len(bytes)) + + return raw_data(bytes) + } else { + bytes_read^ = 0 + return nil + } + } + + return ts.Input { + payload = buffer, + read = read, + encoding = .UTF8, + } +} + save_buffer_to_disk :: proc(state: ^State, buffer: ^FileBuffer) -> (error: os.Error) { fd := os.open(buffer.file_path, flags = os.O_WRONLY | os.O_TRUNC | os.O_CREATE) or_return; defer os.close(fd); @@ -760,6 +790,7 @@ next_buffer :: proc(state: ^State, prev_buffer: ^int) -> int { // TODO: replace this with arena for the file buffer free_file_buffer :: proc(buffer: ^FileBuffer) { + ts.delete_state(&buffer.tree) free_history(&buffer.history) delete(buffer.glyphs.buffer) delete(buffer.input_buffer) diff --git a/src/core/glyph_buffer.odin b/src/core/glyph_buffer.odin index c2b9d5d..aae2d71 100644 --- a/src/core/glyph_buffer.odin +++ b/src/core/glyph_buffer.odin @@ -2,6 +2,7 @@ package core import "core:fmt" +import ts "../tree_sitter" import "../theme" GlyphBuffer :: struct { @@ -27,7 +28,21 @@ make_glyph_buffer :: proc(width, height: int, allocator := context.allocator) -> update_glyph_buffer_from_file_buffer :: proc(buffer: ^FileBuffer) { for &glyph in buffer.glyphs.buffer { - glyph = Glyph{}; + glyph = Glyph {} + glyph.color = .Foreground + } + + outer: for highlight in buffer.tree.highlights { + for line in highlight.start.row..=highlight.end.row { + if int(line) < buffer.top_line { continue; } + + screen_line := int(line) - buffer.top_line + if screen_line >= buffer.glyphs.height { break outer; } + + for col in highlight.start.column..= begin && rendered_col < buffer.glyphs.width { - buffer.glyphs.buffer[rendered_col + screen_line * buffer.glyphs.width] = Glyph { codepoint = character, color = .Foreground }; + buffer.glyphs.buffer[rendered_col + screen_line * buffer.glyphs.width].codepoint = character } rendered_col += 1; diff --git a/src/core/piece_table.odin b/src/core/piece_table.odin index cac5f08..e087f84 100644 --- a/src/core/piece_table.odin +++ b/src/core/piece_table.odin @@ -71,6 +71,28 @@ new_piece_table_iter_from_index :: proc(t: ^PieceTable, index: PieceTableIndex) } } +new_piece_table_iter_from_byte_offset :: proc(t: ^PieceTable, byte_offset: int) -> (iter: PieceTableIter, ok: bool) { + bytes := 0 + + for chunk, chunk_index in t.chunks { + if bytes + len(chunk) > byte_offset { + char_index := byte_offset - bytes + + return PieceTableIter { + t = t, + index = PieceTableIndex { + chunk_index = chunk_index, + char_index = char_index, + } + }, true + } else { + bytes += len(chunk) + } + } + + return +} + new_piece_table_index_from_end :: proc(t: ^PieceTable) -> PieceTableIndex { chunk_index := len(t.chunks)-1 char_index := len(t.chunks[chunk_index])-1 diff --git a/src/main.odin b/src/main.odin index 1d65b9a..cde68a5 100644 --- a/src/main.odin +++ b/src/main.odin @@ -18,6 +18,7 @@ import "core" import "panels" import "theme" import "ui" +import ts "tree_sitter" State :: core.State; FileBuffer :: core.FileBuffer; @@ -85,7 +86,7 @@ draw :: proc(state: ^State) { ui.draw(new_ui, state) // TODO: figure out when to not show the input help menu - if state.mode != .Insert { // && state.current_input_map != &state.input_map.mode[state.mode] { + if false && state.mode != .Insert { // && state.current_input_map != &state.input_map.mode[state.mode] { longest_description := 0; for key, action in state.current_input_map.key_actions { if len(action.description) > longest_description { @@ -200,6 +201,8 @@ ui_file_buffer :: proc(s: ^ui.State, buffer: ^FileBuffer) { } main :: proc() { + ts.set_allocator() + _command_arena: mem.Arena mem.arena_init(&_command_arena, make([]u8, 1024*1024)); diff --git a/src/panels/panels.odin b/src/panels/panels.odin index 7496940..40e2abd 100644 --- a/src/panels/panels.odin +++ b/src/panels/panels.odin @@ -9,6 +9,7 @@ import "core:log" import "vendor:sdl2" +import ts "../tree_sitter" import "../core" import "../input" import "../util" @@ -65,6 +66,15 @@ register_default_leader_actions :: proc(input_map: ^core.InputActions) { core.register_key_action(input_map, .R, proc(state: ^core.State) { open_grep_panel(state) }, "Grep Workspace") + + core.register_key_action(input_map, .K, proc(state: ^core.State) { + buffer := core.current_buffer(state) + ts.update_cursor(&buffer.tree, buffer.history.cursor.line, buffer.history.cursor.col) + + ts.print_node_type(&buffer.tree) + + core.reset_input_map(state) + }, "View Symbol") } register_default_panel_actions :: proc(input_map: ^core.InputActions) { @@ -282,6 +292,11 @@ make_file_buffer_panel :: proc(buffer_index: int) -> core.Panel { return &state.buffers[panel_state.buffer_index], true }, + drop = proc(state: ^core.State, panel_state: ^core.PanelState) { + if panel_state, ok := &panel_state.(core.FileBufferPanel); ok { + core.free_file_buffer(&state.buffers[panel_state.buffer_index]) + } + }, render_proc = proc(state: ^core.State, panel_state: ^core.PanelState) -> (ok: bool) { panel_state := panel_state.(core.FileBufferPanel) or_return; s := transmute(^ui.State)state.ui @@ -426,6 +441,7 @@ make_grep_panel :: proc(state: ^core.State) -> core.Panel { }, drop = proc(state: ^core.State, panel_state: ^core.PanelState) { if panel_state, ok := &panel_state.(core.GrepPanel); ok { + // core.free_file_buffer(&state.buffers[panel_state.buffer]) delete(panel_state.query_arena.data, state.ctx.allocator) } }, diff --git a/src/tree_sitter/ts.odin b/src/tree_sitter/ts.odin new file mode 100644 index 0000000..31155ef --- /dev/null +++ b/src/tree_sitter/ts.odin @@ -0,0 +1,367 @@ +package tree_sitter + +import "base:runtime" +import "core:strings" +import "core:fmt" +import "core:log" +import "core:os" +import "core:mem" + +import "../theme" + +foreign import ts "../../bin/libtree-sitter.a" +@(default_calling_convention = "c", link_prefix="ts_") +foreign ts { + parser_new :: proc() -> Parser --- + parser_delete :: proc(parser: Parser) -> Parser --- + + parser_set_language :: proc(parser: Parser, language: Language) -> bool --- + parser_set_logger :: proc(parser: Parser, logger: TSLogger) --- + parser_print_dot_graphs :: proc(parser: Parser, fd: int) --- + + parser_parse :: proc(parser: Parser, old_tree: Tree, input: Input) -> Tree --- + parser_parse_string :: proc(parser: Parser, old_tree: Tree, source: []u8, len: u32) -> Tree --- + + tree_root_node :: proc(tree: Tree) -> Node --- + tree_delete :: proc(tree: Tree) --- + + tree_cursor_new :: proc(node: Node) -> TreeCursor --- + tree_cursor_reset :: proc(tree: ^TreeCursor, node: Node) --- + tree_cursor_delete :: proc(tree: ^TreeCursor) --- + tree_cursor_current_node :: proc(tree: ^TreeCursor) -> Node --- + tree_cursor_current_field_name :: proc(tree: ^TreeCursor) -> cstring --- + + tree_cursor_goto_first_child :: proc(self: ^TreeCursor) -> bool --- + tree_cursor_goto_first_child_for_point :: proc(self: ^TreeCursor, goal_point: Point) -> u64 --- + + node_start_point :: proc(self: Node) -> Point --- + node_end_point :: proc(self: Node) -> Point --- + node_type :: proc(self: Node) -> cstring --- + node_named_child :: proc(self: Node, child_index: u32) -> Node --- + node_child_count :: proc(self: Node) -> u32 --- + node_is_null :: proc(self: Node) -> bool --- + node_string :: proc(self: Node) -> cstring --- + + query_new :: proc(language: Language, source: []u8, source_len: u32, error_offset: ^u32, error_type: ^QueryError) -> Query --- + query_delete :: proc(query: Query) --- + query_cursor_new :: proc() -> QueryCursor --- + query_cursor_exec :: proc(cursor: QueryCursor, query: Query, node: Node) --- + query_cursor_next_match :: proc(cursor: QueryCursor, match: ^QueryMatch) -> bool --- + + query_capture_name_for_id :: proc(query: Query, index: u32, length: ^u32) -> ^u8 --- + + @(link_name = "ts_set_allocator") + ts_set_allocator :: proc(new_malloc: MallocProc, new_calloc: CAllocProc, new_realloc: ReAllocProc, new_free: FreeProc) --- +} + +TS_ALLOCATOR: mem.Allocator + +MallocProc :: proc "c" (size: uint) -> rawptr +CAllocProc :: proc "c" (num: uint, size: uint) -> rawptr +ReAllocProc :: proc "c" (ptr: rawptr, size: uint) -> rawptr +FreeProc :: proc "c" (ptr: rawptr) + +set_allocator :: proc(allocator := context.allocator) { + TS_ALLOCATOR = allocator + + new_malloc :: proc "c" (size: uint) -> rawptr { + context = runtime.default_context() + + data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Alloc, int(size), runtime.DEFAULT_ALIGNMENT, nil, 0) + return raw_data(data) + } + + new_calloc :: proc "c" (num: uint, size: uint) -> rawptr { + context = runtime.default_context() + + data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Alloc, int(num * size), runtime.DEFAULT_ALIGNMENT, nil, 0) + return raw_data(data) + } + + new_realloc :: proc "c" (old_ptr: rawptr, size: uint) -> rawptr { + context = runtime.default_context() + + data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Resize, int(size), runtime.DEFAULT_ALIGNMENT, old_ptr, 0) + return raw_data(data) + } + + new_free :: proc "c" (ptr: rawptr) { + context = runtime.default_context() + + TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Free, 0, runtime.DEFAULT_ALIGNMENT, ptr, 0) + } +} + +foreign import ts_odin "../../bin/libtree-sitter-odin.a" +foreign ts_odin { + tree_sitter_odin :: proc "c" () -> Language --- +} + +foreign import ts_json "../../bin/libtree-sitter-json.a" +foreign ts_json { + tree_sitter_json :: proc "c" () -> Language --- +} + +State :: struct { + parser: Parser, + language: Language, + + tree: Tree, + cursor: TreeCursor, + + highlights: [dynamic]Highlight, +} + +Highlight :: struct { + start: Point, + end: Point, + color: theme.PaletteColor, +} + +LanguageType :: enum { + Json, + Odin, +} + +TestStuff :: struct { + start: Point, + end: Point, +} + +Parser :: distinct rawptr +Language :: distinct rawptr + +Query :: distinct rawptr +QueryCursor :: distinct rawptr + +QueryError :: enum { + None = 0, + Syntax, + NodeType, + Field, + Capture, +} + +QueryCapture :: struct { + node: Node, + index: u32, +} + +QueryMatch :: struct { + id: u32, + pattern_index: u16, + capture_count: u16, + captures: [^]QueryCapture, +} + +DecodeFunction :: proc "c" (text: []u8, length: u32, code_point: ^u32) -> u32 +Input :: struct { + payload: rawptr, + read: proc "c" (payload: rawptr, byte_index: u32, position: Point, bytes_read: ^u32) -> ^u8, + encoding: InputEncoding, + decode: DecodeFunction, +} + +InputEncoding :: enum { + UTF8 = 0, + UTF16LE, + UTF16BE, + Custom, +} + +Tree :: distinct rawptr + +TreeCursor :: struct { + tree: rawptr, + id: rawptr, + ctx: [3]u32, +} + +Node :: struct { + ctx: [4]u32, + id: rawptr, + tree: Tree, +} + +Point :: struct { + row: u32, + column: u32 +} + +TSLogType :: enum { Parse, Lex } +TSLogger :: struct { + log: proc "c" (payload: rawptr, log_type: TSLogType, msg: cstring), + payload: rawptr, +} +log_callback :: proc "c" (payload: rawptr, log_type: TSLogType, msg: cstring) { + context = runtime.default_context() + fmt.printf("Tree-sitter log: %s", msg) +} + +make_state :: proc(type: LanguageType, allocator := context.allocator) -> State { + context.allocator = allocator + + parser := parser_new() + parser_set_logger(parser, TSLogger{log = log_callback, payload = nil}) + + language: Language + + switch (type) { + case .Odin: language = tree_sitter_odin() + case .Json: language = tree_sitter_json() + } + + if !parser_set_language(parser, language) { + log.errorf("failed to set language to '%v'", type) + return State {} + } + + return State { + parser = parser, + language = language + } +} + +delete_state :: proc(state: ^State) { + tree_cursor_delete(&state.cursor) + tree_delete(state.tree) + parser_delete(state.parser) +} + +parse_buffer :: proc(state: ^State, input: Input) { + old_tree := state.tree + if old_tree != nil { + defer tree_delete(old_tree) + } + + state.tree = parser_parse(state.parser, nil, input) + + if state.tree == nil { + log.error("failed to parse buffer") + return + } + + state.cursor = tree_cursor_new(tree_root_node(state.tree)) + load_highlights(state) +} + +update_cursor :: proc(state: ^State, line: int, col: int) { + assert(state.tree != nil) + + root_node := tree_root_node(state.tree) + tree_cursor_reset(&state.cursor, root_node) + + node := tree_cursor_current_node(&state.cursor) + for node_child_count(node) > 1 { + if tree_cursor_goto_first_child_for_point(&state.cursor, Point { + row = u32(line), + column = u32(col), + }) < 0 { + break + } + + node = tree_cursor_current_node(&state.cursor) + } +} + +load_highlights :: proc(state: ^State) { + // TODO: have this be language specific + capture_to_color := make(map[string]theme.PaletteColor, allocator = context.temp_allocator) + capture_to_color["include"] = .Red + capture_to_color["keyword.function"] = .Red + capture_to_color["storageclass"] = .Red + + capture_to_color["keyword.operator"] = .Purple + + capture_to_color["keyword"] = .Blue + capture_to_color["repeat"] = .Blue + capture_to_color["conditional"] = .Blue + capture_to_color["function"] = .Blue + + capture_to_color["type.decl"] = .BrightBlue + capture_to_color["field"] = .BrightYellow + + capture_to_color["type.builtin"] = .Aqua + + capture_to_color["function.call"] = .Green + capture_to_color["string"] = .Green + + capture_to_color["comment"] = .Gray + + fd, err := os.open("../tree-sitter-odin/queries/highlights.scm") + if err != nil { + log.errorf("failed to open file: errno=%x", err) + return + } + defer os.close(fd); + + if highlight_query, success := os.read_entire_file_from_handle(fd); success { + error_offset: u32 + error_type: QueryError + + query := query_new(state.language, highlight_query, u32(len(highlight_query)), &error_offset, &error_type) + defer query_delete(query) + + if error_type != .None { + log.errorf("got error: '%v'", error_type) + return + } + + cursor := query_cursor_new() + query_cursor_exec(cursor, query, tree_root_node(state.tree)) + + if state.highlights != nil { + clear(&state.highlights) + } else { + state.highlights = make([dynamic]Highlight) + } + + match: QueryMatch + for query_cursor_next_match(cursor, &match) { + for i in 0..