add rust syntax

.devcontainer/devcontainer.json

@@ -1,19 +0,0 @@
-{
-    "image": "mcr.microsoft.com/devcontainers/cpp:ubuntu",
-    "customizations": {
-        "vscode": {
-            "extensions": [
-                "danielgavin.ols"
-            ]
-        }
-    },
-    "features": {
-        "./odin-feature": {
-            "version": "latest"
-        },
-        "ghcr.io/devcontainers/features/rust:1": {
-            "version": "1.88.0",
-            "profile": "default"
-        }
-    }
-}

.devcontainer/odin-feature/devcontainer-feature.json

@@ -1,11 +0,0 @@
-{
-    "name": "Odin Compiler",
-    "id": "odin",
-    "version": "1.0.0",
-    "installsAfter": [
-        "ghcr.io/devcontainers/features/common-utils"
-    ],
-    "containerEnv": {
-        "ODIN_ROOT": "/usr/local/share/odin"
-    }
-}

.devcontainer/odin-feature/install.sh

@@ -1,19 +0,0 @@
-#!/bin/sh
-set -e
-
-git clone --branch "dev-2024-12" "https://github.com/odin-lang/Odin.git"
-
-cd Odin
-
-mkdir -p /usr/local/share/odin
-cp -r ./base /usr/local/share/odin/
-cp -r ./core /usr/local/share/odin/
-cp -r ./vendor /usr/local/share/odin/
-
-make release
-
-cp ./odin /usr/local/bin/
-
-echo "Installing SDL2 & SDL2_ttf"
-apt update
-apt --yes --force-yes install libsdl2-dev libsdl2-ttf-dev

Makefile

@@ -1,6 +1,6 @@
 export RUSTFLAGS=-C target-feature=-avx2
 
-all: editor
+all: bin/libtree-sitter.a editor
 
 editor: grep src/**/*.odin
 	mkdir -p bin
@@ -10,4 +10,18 @@ grep:
 	cargo build --manifest-path "src/pkg/grep_lib/Cargo.toml"
 
 test: src/**/*.odin
-	odin test src/tests/ -all-packages -debug -out:bin/test_runner
+	odin test src/tests/ -all-packages -debug -out:bin/test_runner
+
+TS_DIR := third_party/tree-sitter/lib
+TS_SRC := $(wildcard $(TS_DIR)/src/*.c)
+TS_OBJ := $(TS_SRC:.c=.o)
+
+TS_ARFLAGS := rcs
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE
+override CFLAGS += -I$(TS_DIR)/src -I$(TS_DIR)/src/wasm -I$(TS_DIR)/include
+override CFLAGS += -o bin/
+
+bin/libtree-sitter.a: $(TS_OBJ)
+	$(AR) $(TS_ARFLAGS) $@ $^ --output bin/

src/core/core.odin

@@ -38,9 +38,6 @@ State :: struct {
     source_font_height: int,
     line_number_padding: int,
 
-    current_buffer: int,
-    buffers: [dynamic]FileBuffer,
-
     // TODO: make more than one register to plop stuff into
     yank_register: Register,
 
@@ -77,33 +74,44 @@ EditorCommandArgument :: union #no_nil {
     i32
 }
 
-PanelRenderProc :: proc(state: ^State, panel_state: ^PanelState) -> (ok: bool)
-PanelBufferProc :: proc(state: ^State, panel_state: ^PanelState) -> (buffer: ^FileBuffer, ok: bool)
-PanelBufferInputProc :: proc(state: ^State, panel_state: ^PanelState)
-PanelDropProc :: proc(state: ^State, panel_state: ^PanelState)
 Panel :: struct {
-    is_floating: bool,
+    using vtable: Panel_VTable,
-    panel_state: PanelState,
+    arena: mem.Arena,
+    allocator: mem.Allocator,
+
+    id: int,
+    type: PanelType,
     input_map: InputMap,
-    buffer_proc: PanelBufferProc,
+    is_floating: bool,
-    on_buffer_input_proc: PanelBufferInputProc,
-    render_proc: PanelRenderProc,
-    drop: PanelDropProc,
 }
 
-PanelState :: union {
+Panel_VTable :: struct {
+    create:          proc(panel: ^Panel, state: ^State), 
+    drop:            proc(panel: ^Panel, state: ^State),
+
+    on_buffer_input: proc(panel: ^Panel, state: ^State),
+    buffer:          proc(panel: ^Panel, state: ^State) -> (buffer: ^FileBuffer, ok: bool),
+    render:          proc(panel: ^Panel, state: ^State) -> (ok: bool),
+}
+
+PanelType :: union {
     FileBufferPanel,
     GrepPanel,
 }
 
 FileBufferPanel :: struct {
-    buffer_index: int,
+    buffer: FileBuffer,
+    viewed_symbol: Maybe(string),
+
+    // only used for initialization
+    file_path: string,
+    line, col: int,
 }
 
 GrepPanel :: struct {
     query_arena: mem.Arena,
     query_region: mem.Arena_Temp_Memory,
-    buffer: int,
+    buffer: FileBuffer,
     selected_result: int,
     search_query: string,
     query_results: []GrepQueryResult,
@@ -118,80 +126,67 @@ GrepQueryResult :: struct {
 }
 
 current_buffer :: proc(state: ^State) -> ^FileBuffer {
-    if current_panel, ok := util.get(&state.panels, state.current_panel.? or_else -1).?; ok {
+    if current_panel, ok := state.current_panel.?; ok {
-        if current_panel.buffer_proc != nil {
+        if panel, ok := util.get(&state.panels, current_panel).?; ok {
-            if panel_buffer, ok := current_panel.buffer_proc(state, &current_panel.panel_state); ok && panel_buffer != nil {
+            buffer, _ := panel->buffer(state)
-                return panel_buffer
+            return buffer
-            }
         }
     }
 
-    if state.current_buffer == -2 {
+    return nil
-        return &state.log_buffer;
-    }
-
-    if len(state.buffers) < 1 {
-        return nil
-    }
-
-    return &state.buffers[state.current_buffer];
 }
 
-yank_whole_line :: proc(state: ^State) {
+yank_whole_line :: proc(state: ^State, buffer: ^FileBuffer) {
+    context.allocator = buffer.allocator
+
     if state.yank_register.data != nil {
         delete(state.yank_register.data)
         state.yank_register.data = nil
     }
 
-    if buffer := current_buffer(state); buffer != nil {
+    selection := new_selection(buffer, buffer.history.cursor)
-        selection := new_selection(buffer, buffer.history.cursor)
+    length := selection_length(buffer, selection)
-        length := selection_length(buffer, selection)
 
-        state.yank_register.whole_line = true
+    state.yank_register.whole_line = true
-        state.yank_register.data = make([]u8, length)
+    state.yank_register.data = make([]u8, length)
 
-        it := new_file_buffer_iter_with_cursor(buffer, selection.start)
+    it := new_file_buffer_iter_with_cursor(buffer, selection.start)
 
-        index := 0
+    index := 0
-        for !it.hit_end && index < length {
+    for !it.hit_end && index < length {
-            state.yank_register.data[index] = get_character_at_iter(it) 
+        state.yank_register.data[index] = get_character_at_iter(it) 
 
-            iterate_file_buffer(&it)
+        iterate_file_buffer(&it)
-            index += 1
+        index += 1
-        }
     }
 }
 
-yank_selection :: proc(state: ^State) {
+yank_selection :: proc(state: ^State, buffer: ^FileBuffer) {
     if state.yank_register.data != nil {
         delete(state.yank_register.data)
         state.yank_register.data = nil
     }
 
-    if buffer := current_buffer(state); buffer != nil && buffer.selection != nil {
+    selection := swap_selections(buffer.selection.?)
-        selection := swap_selections(buffer.selection.?)
+    length := selection_length(buffer, selection)
-        length := selection_length(buffer, selection)
 
-        state.yank_register.whole_line = false
+    state.yank_register.whole_line = false
-        state.yank_register.data = make([]u8, length)
+    state.yank_register.data = make([]u8, length)
 
-        it := new_file_buffer_iter_with_cursor(buffer, selection.start)
+    it := new_file_buffer_iter_with_cursor(buffer, selection.start)
 
-        index := 0
+    index := 0
-        for !it.hit_end && index < length {
+    for !it.hit_end && index < length {
-            state.yank_register.data[index] = get_character_at_iter(it) 
+        state.yank_register.data[index] = get_character_at_iter(it) 
 
-            iterate_file_buffer(&it)
+        iterate_file_buffer(&it)
-            index += 1
+        index += 1
-        }
     }
 }
 
-paste_register :: proc(state: ^State, register: Register) {
+paste_register :: proc(state: ^State, register: Register, buffer: ^FileBuffer) {
-    if buffer := current_buffer(state); buffer != nil && register.data != nil {
+    insert_content(buffer, register.data)
-        insert_content(buffer, register.data)
+    move_cursor_left(buffer)
-        move_cursor_left(buffer)
-    }
 }
 
 reset_input_map_from_state_mode :: proc(state: ^State) {
@@ -204,29 +199,25 @@ reset_input_map_from_mode :: proc(state: ^State, mode: Mode) {
 }
 reset_input_map :: proc{reset_input_map_from_mode, reset_input_map_from_state_mode}
 
-buffer_from_index :: proc(state: ^State, buffer_index: int) -> ^FileBuffer {
+InputMap :: struct {
-    if buffer_index == -2 {
+    mode: map[Mode]InputActions,
-        return &state.log_buffer;
-    }
-
-    return &state.buffers[buffer_index];
 }
 
-EditorAction :: proc(state: ^State);
 InputGroup :: union {EditorAction, InputActions}
+EditorAction :: proc(state: ^State, user_data: rawptr);
+InputActions :: struct {
+    key_actions: map[Key]Action,
+    ctrl_key_actions: map[Key]Action,
+    show_help: bool,
+}
 Action :: struct {
     action: InputGroup,
     description: string,
 }
-InputMap :: struct {
-    mode: map[Mode]InputActions,
-}
-InputActions :: struct {
-    key_actions: map[Key]Action,
-    ctrl_key_actions: map[Key]Action,
-}
 
-new_input_map :: proc() -> InputMap {
+new_input_map :: proc(show_help: bool = false, allocator := context.allocator) -> InputMap {
+    context.allocator = allocator
+
     input_map := InputMap {
         mode = make(map[Mode]InputActions),
     }
@@ -238,13 +229,19 @@ new_input_map :: proc() -> InputMap {
         }
     }
 
+    normal_actions := &input_map.mode[.Normal]
+    normal_actions.show_help = show_help
+
     return input_map;
 }
 
-new_input_actions :: proc() -> InputActions {
+new_input_actions :: proc(show_help: bool = false, allocator := context.allocator) -> InputActions {
+    context.allocator = allocator
+
     input_actions := InputActions {
         key_actions = make(map[Key]Action),
         ctrl_key_actions = make(map[Key]Action),
+        show_help = show_help,
     }
 
     return input_actions;
@@ -377,7 +374,8 @@ run_command :: proc(state: ^State, group: string, name: string) {
         for cmd in cmds {
             if cmd.name == name {
                 log.info("Running command", group, name);
-                cmd.action(state);
+                // TODO: rework command system
+                // cmd.action(state);
                 return;
             }
         }

src/core/file_buffer.odin

@@ -10,6 +10,7 @@ import "core:slice"
 import "base:runtime"
 import "core:strings"
 
+import ts "../tree_sitter"
 import "../theme"
 
 ScrollDir :: enum {
@@ -45,16 +46,24 @@ FileBuffer :: struct {
     file_path: string,
     extension: string,
 
+    flags: BufferFlagSet,
+    last_col: int,
     top_line: int,
-    // cursor: Cursor,
     selection: Maybe(Selection),
 
+    tree: ts.State,
+
     history: FileHistory,
     glyphs: GlyphBuffer,
 
     input_buffer: [dynamic]u8,
 }
 
+BufferFlagSet :: bit_set[BufferFlags]
+BufferFlags :: enum {
+    UnsavedChanges,
+}
+
 FileBufferIter :: struct {
     cursor: Cursor,
     buffer: ^FileBuffer,
@@ -415,6 +424,8 @@ move_cursor_start_of_line :: proc(buffer: ^FileBuffer, cursor: Maybe(^Cursor) =
 
         cursor.?^ = it.cursor;
     }
+
+    buffer.last_col = cursor.?.col
 }
 
 move_cursor_end_of_line :: proc(buffer: ^FileBuffer, stop_at_end: bool = true, cursor: Maybe(^Cursor) = nil) {
@@ -424,6 +435,8 @@ move_cursor_end_of_line :: proc(buffer: ^FileBuffer, stop_at_end: bool = true, c
         cursor = &buffer.history.cursor;
     }
 
+    buffer.last_col = cursor.?.col
+
     it := new_file_buffer_iter_with_cursor(buffer, cursor.?^);
     line_length := file_buffer_line_length(buffer, it.cursor.index);
     if stop_at_end {
@@ -439,6 +452,8 @@ move_cursor_end_of_line :: proc(buffer: ^FileBuffer, stop_at_end: bool = true, c
 
         cursor.?^ = it.cursor;
     }
+
+    buffer.last_col = cursor.?.col
 }
 
 move_cursor_up :: proc(buffer: ^FileBuffer, amount: int = 1, cursor: Maybe(^Cursor) = nil) {
@@ -472,6 +487,12 @@ move_cursor_up :: proc(buffer: ^FileBuffer, amount: int = 1, cursor: Maybe(^Curs
         cursor.?^ = it.cursor;
     }
 
+    if cursor.?.col < buffer.last_col && file_buffer_line_length(buffer, cursor.?.index)-1 >= cursor.?.col {
+        last_col := buffer.last_col
+        move_cursor_right(buffer, amt = buffer.last_col - cursor.?.col, cursor = cursor)
+        buffer.last_col = last_col
+    }
+
     update_file_buffer_scroll(buffer, cursor);
 }
 
@@ -505,6 +526,13 @@ move_cursor_down :: proc(buffer: ^FileBuffer, amount: int = 1, cursor: Maybe(^Cu
     }
 
     cursor.?^ = it.cursor;
+
+    if cursor.?.col < buffer.last_col && file_buffer_line_length(buffer, cursor.?.index)-1 >= cursor.?.col {
+        last_col := buffer.last_col
+        move_cursor_right(buffer, amt = buffer.last_col - cursor.?.col, cursor = cursor)
+        buffer.last_col = last_col
+    }
+
     update_file_buffer_scroll(buffer, cursor);
 }
 
@@ -515,6 +543,8 @@ move_cursor_left :: proc(buffer: ^FileBuffer, cursor: Maybe(^Cursor) = nil) {
         cursor = &buffer.history.cursor;
     }
 
+    buffer.last_col = cursor.?.col
+
     if cursor.?.col > 0 {
         it := new_file_buffer_iter_with_cursor(buffer, cursor.?^);
         iterate_file_buffer_reverse(&it);
@@ -529,6 +559,8 @@ move_cursor_right :: proc(buffer: ^FileBuffer, stop_at_end: bool = true, amt: in
         cursor = &buffer.history.cursor;
     }
 
+    buffer.last_col = cursor.?.col
+
     it := new_file_buffer_iter_with_cursor(buffer, cursor.?^);
     line_length := file_buffer_line_length(buffer, it.cursor.index);
 
@@ -551,6 +583,8 @@ move_cursor_forward_start_of_word :: proc(buffer: ^FileBuffer, cursor: Maybe(^Cu
     iterate_file_buffer_until(&it, until_start_of_word);
     cursor.?^ = it.cursor;
 
+    buffer.last_col = cursor.?.col
+
     update_file_buffer_scroll(buffer, cursor);
 }
 
@@ -565,6 +599,8 @@ move_cursor_forward_end_of_word :: proc(buffer: ^FileBuffer, cursor: Maybe(^Curs
     iterate_file_buffer_until(&it, until_end_of_word);
     cursor.?^ = it.cursor;
 
+    buffer.last_col = cursor.?.col
+
     update_file_buffer_scroll(buffer, cursor);
 }
 
@@ -580,6 +616,8 @@ move_cursor_backward_start_of_word :: proc(buffer: ^FileBuffer, cursor: Maybe(^C
     //iterate_file_buffer_until(&it, until_non_whitespace);
     cursor.?^ = it.cursor;
 
+    buffer.last_col = cursor.?.col
+
     update_file_buffer_scroll(buffer, cursor);
 }
 
@@ -594,6 +632,8 @@ move_cursor_backward_end_of_word :: proc(buffer: ^FileBuffer, cursor: Maybe(^Cur
     iterate_file_buffer_until_reverse(&it, until_start_of_word);
     cursor.?^ = it.cursor;
 
+    buffer.last_col = cursor.?.col
+
     update_file_buffer_scroll(buffer, cursor);
 }
 
@@ -640,8 +680,8 @@ swap_selections :: proc(selection: Selection) -> (swapped: Selection) {
 // TODO: don't access PieceTableIndex directly
 is_selection_inverted :: proc(selection: Selection) -> bool {
     return selection.start.index.chunk_index > selection.end.index.chunk_index ||
-        (selection.start.index.chunk_index == selection.end.index.chunk_index
+        (selection.start.index.chunk_index == selection.end.index.chunk_index &&
-            && selection.start.index.char_index > selection.end.index.char_index)
+            selection.start.index.char_index > selection.end.index.char_index)
 }
 
 selection_length :: proc(buffer: ^FileBuffer, selection: Selection) -> int {
@@ -660,7 +700,7 @@ selection_length :: proc(buffer: ^FileBuffer, selection: Selection) -> int {
     return length
 }
 
-new_virtual_file_buffer :: proc(allocator: mem.Allocator) -> FileBuffer {
+new_virtual_file_buffer :: proc(allocator := context.allocator) -> FileBuffer {
     context.allocator = allocator;
     width := 256;
     height := 256;
@@ -703,11 +743,25 @@ new_file_buffer :: proc(allocator: mem.Allocator, file_path: string, base_dir: s
 
     extension := filepath.ext(fi.fullpath);
 
+    file_type: ts.LanguageType = .None
+    if extension == ".odin" {
+        file_type = .Odin
+    } else if extension == ".rs" {
+        file_type = .Rust
+    } else if extension == ".json" {
+        file_type = .Json
+    }
+
     if original_content, success := os.read_entire_file_from_handle(fd); success {
+        defer delete(original_content)
+
+        content := make([]u8, len(original_content))
+        copy_slice(content, original_content)
+
         width := 256;
         height := 256;
 
-        fmt.eprintln("file path", fi.fullpath[4:]);
+        fmt.eprintln("file path", fi.fullpath[:]);
 
         buffer := FileBuffer {
             allocator = allocator,
@@ -717,18 +771,46 @@ new_file_buffer :: proc(allocator: mem.Allocator, file_path: string, base_dir: s
             // file_path = fi.fullpath[4:],
             extension = extension,
 
-            history = make_history(original_content),
+            // TODO: derive language type from extension
+            tree = ts.make_state(file_type),
+            history = make_history(content),
 
             glyphs = make_glyph_buffer(width, height),
             input_buffer = make([dynamic]u8, 0, 1024),
         };
 
+        ts.parse_buffer(&buffer.tree, tree_sitter_file_buffer_input(&buffer))
+
         return buffer, error();
     } else {
         return FileBuffer{}, error(ErrorType.FileIOError, fmt.aprintf("failed to read from file"));
     }
 }
 
+tree_sitter_file_buffer_input :: proc(buffer: ^FileBuffer) -> ts.Input {
+    read :: proc "c" (payload: rawptr, byte_index: u32, position: ts.Point, bytes_read: ^u32) -> ^u8 {
+        context = runtime.default_context()
+
+        buffer := transmute(^FileBuffer)payload
+
+        if iter, ok := new_piece_table_iter_from_byte_offset(&buffer.history.piece_table, int(byte_index)); ok {
+            bytes := iter.t.chunks[iter.index.chunk_index][iter.index.char_index:]
+            bytes_read^ = u32(len(bytes))
+
+            return raw_data(bytes)
+        } else {
+            bytes_read^ = 0
+            return nil
+        }
+    }
+
+    return ts.Input {
+        payload = buffer,
+        read = read,
+        encoding = .UTF8,
+    }
+}
+
 save_buffer_to_disk :: proc(state: ^State, buffer: ^FileBuffer) -> (error: os.Error) {
     fd := os.open(buffer.file_path, flags = os.O_WRONLY | os.O_TRUNC | os.O_CREATE) or_return;
     defer os.close(fd);
@@ -742,24 +824,14 @@ save_buffer_to_disk :: proc(state: ^State, buffer: ^FileBuffer) -> (error: os.Er
     os.flush(fd)
 
     log.infof("written %v bytes", offset)
+    buffer.flags -= { .UnsavedChanges }
     
     return
 }
 
-next_buffer :: proc(state: ^State, prev_buffer: ^int) -> int {
-    index := prev_buffer^;
-
-    if prev_buffer^ >= len(state.buffers)-1 {
-        prev_buffer^ = -1;
-    } else {
-        prev_buffer^ += 1;
-    }
-
-    return index;
-}
-
 // TODO: replace this with arena for the file buffer
 free_file_buffer :: proc(buffer: ^FileBuffer) {
+    ts.delete_state(&buffer.tree)
     free_history(&buffer.history)
     delete(buffer.glyphs.buffer)
     delete(buffer.input_buffer)
@@ -800,10 +872,11 @@ color_character :: proc(buffer: ^FileBuffer, start: Cursor, end: Cursor, palette
     }
 }
 
-draw_file_buffer :: proc(state: ^State, buffer: ^FileBuffer, x: int, y: int, show_line_numbers: bool = true) {
+draw_file_buffer :: proc(state: ^State, buffer: ^FileBuffer, x, y, w, h: int, show_line_numbers: bool = true, show_cursor: bool = true) {
-    update_glyph_buffer(buffer);
+    glyph_width := math.min(256, int(w / state.source_font_width));
+    glyph_height := math.min(256, int(h / state.source_font_height)) + 1;
 
-    // TODO: syntax highlighting
+    update_glyph_buffer(buffer, glyph_width, glyph_height);
 
     padding := 0;
     if show_line_numbers {
@@ -817,43 +890,44 @@ draw_file_buffer :: proc(state: ^State, buffer: ^FileBuffer, x: int, y: int, sho
     cursor_y -= begin * state.source_font_height;
 
     // draw cursor
-    if state.mode == .Normal || current_buffer(state) != buffer {
+    if show_cursor {
-        draw_rect(state, cursor_x, cursor_y, state.source_font_width, state.source_font_height, .Background4);
+        if state.mode == .Normal {
-    } else if state.mode == .Visual {
+            draw_rect(state, cursor_x, cursor_y, state.source_font_width, state.source_font_height, .Background4);
-        start_sel_x := x + padding + buffer.selection.?.start.col * state.source_font_width;
+        } else if state.mode == .Visual && buffer.selection != nil {
-        start_sel_y := y + buffer.selection.?.start.line * state.source_font_height;
+            start_sel_x := x + padding + buffer.selection.?.start.col * state.source_font_width;
+            start_sel_y := y + buffer.selection.?.start.line * state.source_font_height;
 
-        end_sel_x := x + padding + buffer.selection.?.end.col * state.source_font_width;
+            end_sel_x := x + padding + buffer.selection.?.end.col * state.source_font_width;
-        end_sel_y := y + buffer.selection.?.end.line * state.source_font_height;
+            end_sel_y := y + buffer.selection.?.end.line * state.source_font_height;
 
-        start_sel_y -= begin * state.source_font_height;
+            start_sel_y -= begin * state.source_font_height;
-        end_sel_y -= begin * state.source_font_height;
+            end_sel_y -= begin * state.source_font_height;
 
-        draw_rect(state, start_sel_x, start_sel_y, state.source_font_width, state.source_font_height, .Green);
+            draw_rect(state, start_sel_x, start_sel_y, state.source_font_width, state.source_font_height, .Green);
-        draw_rect(state, end_sel_x, end_sel_y, state.source_font_width, state.source_font_height, .Blue);
+            draw_rect(state, end_sel_x, end_sel_y, state.source_font_width, state.source_font_height, .Blue);
-    }
+        } else if state.mode == .Insert {
-    else if state.mode == .Insert {
+            draw_rect(state, cursor_x, cursor_y, state.source_font_width, state.source_font_height, .Green);
-        draw_rect(state, cursor_x, cursor_y, state.source_font_width, state.source_font_height, .Green);
 
-        num_line_break := 0;
+            num_line_break := 0;
-        line_length := 0;
+            line_length := 0;
-        for c in buffer.input_buffer {
+            for c in buffer.input_buffer {
-            if c == '\n' {
+                if c == '\n' {
-                num_line_break += 1;
+                    num_line_break += 1;
-                line_length = 0;
+                    line_length = 0;
-            } else {
+                } else {
-                line_length += 1;
+                    line_length += 1;
+                }
             }
-        }
 
-        if num_line_break > 0 {
+            if num_line_break > 0 {
-            cursor_x = x + padding + line_length * state.source_font_width;
+                cursor_x = x + padding + line_length * state.source_font_width;
-            cursor_y = cursor_y + num_line_break * state.source_font_height;
+                cursor_y = cursor_y + num_line_break * state.source_font_height;
-        } else {
+            } else {
-            cursor_x += line_length * state.source_font_width;
+                cursor_x += line_length * state.source_font_width;
-        }
+            }
 
-        draw_rect(state, cursor_x, cursor_y, state.source_font_width, state.source_font_height, .Blue);
+            draw_rect(state, cursor_x, cursor_y, state.source_font_width, state.source_font_height, .Blue);
+        }
     }
 
     // TODO: replace with glyph_buffer.draw_glyph_buffer
@@ -877,9 +951,8 @@ draw_file_buffer :: proc(state: ^State, buffer: ^FileBuffer, x: int, y: int, sho
 
         // NOTE: this requires transparent background color because it renders after the text
         // and its after the text because the line length needs to be calculated
-        if state.mode == .Visual && current_buffer(state) == buffer {
+        if state.mode == .Visual && buffer.selection != nil {
             selection := swap_selections(buffer.selection.?)
-            // selection := buffer.selection.?
 
             sel_x := x + padding;
             width: int
@@ -911,22 +984,17 @@ update_file_buffer_scroll :: proc(buffer: ^FileBuffer, cursor: Maybe(^Cursor) =
         cursor = &buffer.history.cursor;
     }
 
-    if cursor.?.line > (buffer.top_line + buffer.glyphs.height - 5) {
+    if buffer.glyphs.height < 5 {
+        buffer.top_line = cursor.?.line
+    } else if cursor.?.line > (buffer.top_line + buffer.glyphs.height - 5) {
         buffer.top_line = math.max(cursor.?.line - buffer.glyphs.height + 5, 0);
     } else if cursor.?.line < (buffer.top_line + 5) {
         buffer.top_line = math.max(cursor.?.line - 5, 0);
     }
-
-    // if buffer.history.cursor.line > (buffer.top_line + buffer.glyphs.height - 5) {
-    //     buffer.top_line = math.max(buffer.history.cursor.line - buffer.glyphs.height + 5, 0);
-    // } else if buffer.history.cursor.line < (buffer.top_line + 5) {
-    //     buffer.top_line = math.max(buffer.history.cursor.line - 5, 0);
-    // }
 }
 
 // TODO: don't mangle cursor
 scroll_file_buffer :: proc(buffer: ^FileBuffer, dir: ScrollDir, cursor: Maybe(^Cursor) = nil) {
-
     switch dir {
         case .Up:
         {
@@ -943,6 +1011,7 @@ insert_content :: proc(buffer: ^FileBuffer, to_be_inserted: []u8, append_to_end:
     if len(to_be_inserted) == 0 {
         return;
     }
+    buffer.flags += { .UnsavedChanges }
 
     index := buffer.history.cursor.index if !append_to_end else new_piece_table_index_from_end(buffer_piece_table(buffer))
 
@@ -952,12 +1021,16 @@ insert_content :: proc(buffer: ^FileBuffer, to_be_inserted: []u8, append_to_end:
         update_file_buffer_index_from_cursor(buffer);
         move_cursor_right(buffer, false, amt = len(to_be_inserted) - 1);
     }
+
+    ts.parse_buffer(&buffer.tree, tree_sitter_file_buffer_input(buffer))
 }
 
 delete_content_from_buffer_cursor :: proc(buffer: ^FileBuffer, amount: int) {
     if amount <= len(buffer.input_buffer) {
         runtime.resize(&buffer.input_buffer, len(buffer.input_buffer)-amount);
     } else {
+        buffer.flags += { .UnsavedChanges }
+
         amount := amount - len(buffer.input_buffer);
         runtime.clear(&buffer.input_buffer);
 
@@ -970,13 +1043,19 @@ delete_content_from_buffer_cursor :: proc(buffer: ^FileBuffer, amount: int) {
         buffer.history.cursor.line = it.cursor.line
         buffer.history.cursor.col = it.cursor.col
     }
+
+    ts.parse_buffer(&buffer.tree, tree_sitter_file_buffer_input(buffer))
 }
 
 delete_content_from_selection :: proc(buffer: ^FileBuffer, selection: ^Selection) {
+    buffer.flags += { .UnsavedChanges }
+
     selection^ = swap_selections(selection^)
     delete_text_in_span(buffer_piece_table(buffer), &selection.start.index, &selection.end.index)
 
     buffer.history.cursor.index = selection.start.index
+
+    ts.parse_buffer(&buffer.tree, tree_sitter_file_buffer_input(buffer))
 }
 
 delete_content :: proc{delete_content_from_buffer_cursor, delete_content_from_selection};

src/core/gfx.odin

@@ -107,6 +107,13 @@ free_font_atlas :: proc(font_atlas: FontAtlas) {
     }
 }
 
+draw_line :: proc(state: ^State, x1,y1,x2,y2: int, color: theme.PaletteColor) {
+    color := theme.get_palette_color(color)
+
+    sdl2.SetRenderDrawColor(state.sdl_renderer, color.r, color.g, color.b, color.a)
+    sdl2.RenderDrawLine(state.sdl_renderer, i32(x1), i32(y1), i32(x2), i32(y2))
+}
+
 draw_rect_outline :: proc(state: ^State, x,y,w,h: int, color: theme.PaletteColor) {
     color := theme.get_palette_color(color);
 

src/core/glyph_buffer.odin

@@ -2,6 +2,7 @@ package core
 
 import "core:fmt"
 
+import ts "../tree_sitter"
 import "../theme"
 
 GlyphBuffer :: struct {
@@ -25,9 +26,27 @@ make_glyph_buffer :: proc(width, height: int, allocator := context.allocator) ->
     }
 }
 
-update_glyph_buffer_from_file_buffer :: proc(buffer: ^FileBuffer) {
+update_glyph_buffer_from_file_buffer :: proc(buffer: ^FileBuffer, width, height: int) {
+    // TODO: limit to 256
+    buffer.glyphs.width = width
+    buffer.glyphs.height = height
+
     for &glyph in buffer.glyphs.buffer {
-        glyph = Glyph{};
+        glyph = Glyph {}
+        glyph.color = .Foreground
+    }
+
+    outer: for highlight in buffer.tree.highlights {
+        for line in highlight.start.row..=highlight.end.row {
+            if int(line) < buffer.top_line { continue; }
+
+            screen_line := int(line) - buffer.top_line
+            if screen_line >= buffer.glyphs.height { break outer; }
+
+            for col in highlight.start.column..<highlight.end.column {
+                buffer.glyphs.buffer[int(col) + screen_line * buffer.glyphs.width].color = highlight.color;
+            }
+        }
     }
 
     begin := buffer.top_line;
@@ -71,7 +90,7 @@ update_glyph_buffer_from_file_buffer :: proc(buffer: ^FileBuffer) {
         }
 
         if rendered_line >= begin && rendered_col < buffer.glyphs.width {
-            buffer.glyphs.buffer[rendered_col + screen_line * buffer.glyphs.width] = Glyph { codepoint = character, color = .Foreground };
+            buffer.glyphs.buffer[rendered_col + screen_line * buffer.glyphs.width].codepoint = character
         }
 
         rendered_col += 1;

src/core/history.odin

@@ -1,8 +1,11 @@
 package core
 
 import "core:log"
+import "core:mem"
 
 FileHistory :: struct {
+    allocator: mem.Allocator,
+
     piece_table: PieceTable,
     cursor: Cursor,
 
@@ -20,6 +23,7 @@ make_history_with_data :: proc(initial_data: []u8, starting_capacity: int = 1024
     context.allocator = allocator
 
     return FileHistory {
+        allocator = allocator,
         piece_table = make_piece_table(initial_data, starting_capacity = starting_capacity),
         snapshots = make([]Snapshot, starting_capacity),
         next = 0,
@@ -31,6 +35,7 @@ make_history_empty :: proc(starting_capacity: int = 1024, allocator := context.a
     context.allocator = allocator
 
     return FileHistory {
+        allocator = allocator,
         piece_table = make_piece_table(starting_capacity = starting_capacity),
         snapshots = make([]Snapshot, starting_capacity),
         next = 0,
@@ -54,6 +59,8 @@ free_history :: proc(history: ^FileHistory) {
 }
 
 push_new_snapshot :: proc(history: ^FileHistory) {
+    context.allocator = history.allocator
+
     if history.snapshots[history.next].chunks != nil {
         delete(history.snapshots[history.next].chunks)
     }
@@ -65,6 +72,8 @@ push_new_snapshot :: proc(history: ^FileHistory) {
 }
 
 pop_snapshot :: proc(history: ^FileHistory, make_new_snapshot: bool = false) {
+    context.allocator = history.allocator
+
     new_next, _ := next_indexes(history, backward = true)
     if new_next == history.next do return
 
@@ -81,6 +90,8 @@ pop_snapshot :: proc(history: ^FileHistory, make_new_snapshot: bool = false) {
 }
 
 recover_snapshot :: proc(history: ^FileHistory) {
+    context.allocator = history.allocator
+
     new_next, _ := next_indexes(history)
     if history.snapshots[new_next].chunks == nil do return
     history.next = new_next

src/core/piece_table.odin

@@ -71,6 +71,28 @@ new_piece_table_iter_from_index :: proc(t: ^PieceTable, index: PieceTableIndex)
     }
 }
 
+new_piece_table_iter_from_byte_offset :: proc(t: ^PieceTable, byte_offset: int) -> (iter: PieceTableIter, ok: bool) {
+    bytes := 0
+
+    for chunk, chunk_index in t.chunks {
+        if bytes + len(chunk) > byte_offset {
+            char_index := byte_offset - bytes
+
+            return PieceTableIter {
+                t = t,
+                index = PieceTableIndex {
+                    chunk_index = chunk_index,
+                    char_index = char_index,
+                }
+            }, true
+        } else {
+            bytes += len(chunk)
+        }
+    }
+
+    return
+}
+
 new_piece_table_index_from_end :: proc(t: ^PieceTable) -> PieceTableIndex {
     chunk_index := len(t.chunks)-1
     char_index := len(t.chunks[chunk_index])-1

src/input/input.odin

@@ -1,286 +0,0 @@
-package input
-
-import "base:runtime"
-import "core:log"
-
-import "vendor:sdl2"
-
-import "../core"
-import "../util"
-
-State :: core.State
-
-register_default_go_actions :: proc(input_map: ^core.InputActions) {
-    core.register_key_action(input_map, .H, proc(state: ^State) {
-        core.move_cursor_start_of_line(core.current_buffer(state));
-        core.reset_input_map(state)
-    }, "move to beginning of line");
-    core.register_key_action(input_map, .L, proc(state: ^State) {
-        core.move_cursor_end_of_line(core.current_buffer(state));
-        core.reset_input_map(state)
-    }, "move to end of line");
-}
-
-register_default_input_actions :: proc(input_map: ^core.InputActions) {
-    // Cursor Movement
-    {
-        core.register_key_action(input_map, .W, proc(state: ^State) {
-            core.move_cursor_forward_start_of_word(core.current_buffer(state));
-        }, "move forward one word");
-        core.register_key_action(input_map, .E, proc(state: ^State) {
-            core.move_cursor_forward_end_of_word(core.current_buffer(state));
-        }, "move forward to end of word");
-
-        core.register_key_action(input_map, .B, proc(state: ^State) {
-            core.move_cursor_backward_start_of_word(core.current_buffer(state));
-        }, "move backward one word");
-
-        core.register_key_action(input_map, .K, proc(state: ^State) {
-            core.move_cursor_up(core.current_buffer(state));
-        }, "move up one line");
-        core.register_key_action(input_map, .J, proc(state: ^State) {
-            core.move_cursor_down(core.current_buffer(state));
-        }, "move down one line");
-        core.register_key_action(input_map, .H, proc(state: ^State) {
-            core.move_cursor_left(core.current_buffer(state));
-        }, "move left one char");
-        core.register_key_action(input_map, .L, proc(state: ^State) {
-            core.move_cursor_right(core.current_buffer(state));
-        }, "move right one char");
-
-        core.register_ctrl_key_action(input_map, .U, proc(state: ^State) {
-            core.scroll_file_buffer(core.current_buffer(state), .Up);
-        }, "scroll buffer up");
-        core.register_ctrl_key_action(input_map, .D, proc(state: ^State) {
-            core.scroll_file_buffer(core.current_buffer(state), .Down);
-        }, "scroll buffer up");
-    }
-
-    // Scale font size
-    {
-        core.register_ctrl_key_action(input_map, .MINUS, proc(state: ^State) {
-            if state.source_font_height > 16 {
-                state.source_font_height -= 2;
-                state.source_font_width = state.source_font_height / 2;
-
-                state.font_atlas = core.gen_font_atlas(state, core.HardcodedFontPath);
-            }
-            log.debug(state.source_font_height);
-        }, "increase font size");
-        core.register_ctrl_key_action(input_map, .EQUAL, proc(state: ^State) {
-            state.source_font_height += 2;
-            state.source_font_width = state.source_font_height / 2;
-
-            state.font_atlas = core.gen_font_atlas(state, core.HardcodedFontPath);
-        }, "decrease font size");
-    }
-
-    // Save file
-    core.register_ctrl_key_action(input_map, .S, proc(state: ^State) {
-        if err := core.save_buffer_to_disk(state, core.current_buffer(state)); err != nil {
-            log.errorf("failed to save buffer to disk: %v", err)
-        }
-    }, "Save file")
-
-    core.register_key_action(input_map, .G, core.new_input_actions(), "Go commands");
-    register_default_go_actions(&(&input_map.key_actions[.G]).action.(core.InputActions));
-
-    core.register_key_action(input_map, .V, proc(state: ^State) {
-        state.mode = .Visual;
-        core.reset_input_map(state)
-
-        core.current_buffer(state).selection = core.new_selection(core.current_buffer(state).history.cursor);
-    }, "enter visual mode");
-
-}
-
-register_default_visual_actions :: proc(input_map: ^core.InputActions) {
-    core.register_key_action(input_map, .ESCAPE, proc(state: ^State) {
-        state.mode = .Normal;
-        core.reset_input_map(state)
-
-        core.current_buffer(state).selection = nil;
-        core.update_file_buffer_scroll(core.current_buffer(state))
-    }, "exit visual mode");
-
-    // Cursor Movement
-    {
-        core.register_key_action(input_map, .W, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_forward_start_of_word(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move forward one word");
-        core.register_key_action(input_map, .E, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_forward_end_of_word(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move forward to end of word");
-
-        core.register_key_action(input_map, .B, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_backward_start_of_word(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move backward one word");
-
-        core.register_key_action(input_map, .K, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_up(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move up one line");
-        core.register_key_action(input_map, .J, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_down(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move down one line");
-        core.register_key_action(input_map, .H, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_left(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move left one char");
-        core.register_key_action(input_map, .L, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.move_cursor_right(core.current_buffer(state), cursor = &sel_cur.end);
-        }, "move right one char");
-
-        core.register_ctrl_key_action(input_map, .U, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.scroll_file_buffer(core.current_buffer(state), .Up, cursor = &sel_cur.end);
-        }, "scroll buffer up");
-        core.register_ctrl_key_action(input_map, .D, proc(state: ^State) {
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.scroll_file_buffer(core.current_buffer(state), .Down, cursor = &sel_cur.end);
-        }, "scroll buffer up");
-    }
-
-    // Text Modification
-    {
-        core.register_key_action(input_map, .D, proc(state: ^State) {
-            core.push_new_snapshot(&core.current_buffer(state).history)
-
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.delete_content(core.current_buffer(state), sel_cur);
-            core.current_buffer(state).selection = nil;
-            core.update_file_buffer_scroll(core.current_buffer(state))
-
-            state.mode = .Normal
-            core.reset_input_map(state)
-        }, "delete selection");
-
-        core.register_key_action(input_map, .C, proc(state: ^State) {
-            core.push_new_snapshot(&core.current_buffer(state).history)
-
-            sel_cur := &(core.current_buffer(state).selection.?);
-
-            core.delete_content(core.current_buffer(state), sel_cur);
-            core.current_buffer(state).selection = nil;
-            core.update_file_buffer_scroll(core.current_buffer(state))
-
-            state.mode = .Insert
-            core.reset_input_map(state, core.Mode.Normal)
-            sdl2.StartTextInput();
-        }, "change selection");
-    }
-
-    // Copy-Paste
-    {
-        core.register_key_action(input_map, .Y, proc(state: ^State) {
-            core.yank_selection(state)
-
-            state.mode = .Normal;
-            core.reset_input_map(state)
-
-            core.current_buffer(state).selection = nil;
-            core.update_file_buffer_scroll(core.current_buffer(state))
-        }, "Yank Line");
-
-        core.register_key_action(input_map, .P, proc(state: ^State) {
-            core.push_new_snapshot(&core.current_buffer(state).history)
-
-            if state.yank_register.whole_line {
-                core.insert_content(core.current_buffer(state), []u8{'\n'});
-                core.paste_register(state, state.yank_register)
-                core.insert_content(core.current_buffer(state), []u8{'\n'});
-            } else {
-                core.paste_register(state, state.yank_register)
-            }
-
-            core.reset_input_map(state)
-        }, "Paste");
-    }
-}
-
-register_default_text_input_actions :: proc(input_map: ^core.InputActions) {
-    core.register_key_action(input_map, .I, proc(state: ^State) {
-        core.push_new_snapshot(&core.current_buffer(state).history)
-
-        state.mode = .Insert;
-        sdl2.StartTextInput();
-    }, "enter insert mode");
-    core.register_key_action(input_map, .A, proc(state: ^State) {
-        core.push_new_snapshot(&core.current_buffer(state).history)
-
-        core.move_cursor_right(core.current_buffer(state), false);
-        state.mode = .Insert;
-        sdl2.StartTextInput();
-    }, "enter insert mode after character (append)");
-
-    core.register_key_action(input_map, .U, proc(state: ^State) {
-        core.pop_snapshot(&core.current_buffer(state).history, true)
-    }, "Undo");
-
-    core.register_ctrl_key_action(input_map, .R, proc(state: ^State) {
-        core.recover_snapshot(&core.current_buffer(state).history)
-    }, "Redo");
-
-    // TODO: add shift+o to insert newline above current one
-
-    core.register_key_action(input_map, .O, proc(state: ^State) {
-        core.push_new_snapshot(&core.current_buffer(state).history)
-
-        if buffer := core.current_buffer(state); buffer != nil {
-            core.move_cursor_end_of_line(buffer, false);
-            runtime.clear(&buffer.input_buffer)
-
-            append(&buffer.input_buffer, '\n')
-
-            state.mode = .Insert;
-
-            sdl2.StartTextInput();
-        }
-    }, "insert mode on newline");
-
-    // Copy-Paste
-    {
-        {
-            yank_actions := core.new_input_actions()
-            defer core.register_key_action(input_map, .Y, yank_actions)
-
-            core.register_key_action(&yank_actions, .Y, proc(state: ^State) {
-                core.yank_whole_line(state)
-
-                core.reset_input_map(state)
-            }, "Yank Line");
-        }
-
-        core.register_key_action(input_map, .P, proc(state: ^State) {
-            core.push_new_snapshot(&core.current_buffer(state).history)
-
-            if state.yank_register.whole_line {
-                core.move_cursor_end_of_line(core.current_buffer(state), false);
-                core.insert_content(core.current_buffer(state), []u8{'\n'});
-                core.move_cursor_right(core.current_buffer(state), false);
-            } else {
-                core.move_cursor_right(core.current_buffer(state))
-            }
-            core.paste_register(state, state.yank_register)
-            core.move_cursor_start_of_line(core.current_buffer(state))
-
-            core.reset_input_map(state)
-        }, "Paste");
-    }
-
-}

src/main.odin

@@ -18,13 +18,11 @@ import "core"
 import "panels"
 import "theme"
 import "ui"
+import ts "tree_sitter"
 
 State :: core.State;
 FileBuffer :: core.FileBuffer;
 
-// TODO: should probably go into state
-scratch: mem.Scratch;
-scratch_alloc: runtime.Allocator;
 state := core.State {};
 
 do_normal_mode :: proc(state: ^State, buffer: ^FileBuffer) {
@@ -53,11 +51,6 @@ ui_font_height :: proc() -> i32 {
 }
 
 draw :: proc(state: ^State) {
-    if buffer := core.current_buffer(state); buffer != nil {
-        buffer.glyphs.height = math.min(256, int((state.screen_height - state.source_font_height*2) / state.source_font_height)) + 1;
-        buffer.glyphs.width = math.min(256, int((state.screen_width - state.source_font_width) / state.source_font_width));
-    }
-
     render_color := theme.get_palette_color(.Background);
     sdl2.SetRenderDrawColor(state.sdl_renderer, render_color.r, render_color.g, render_color.b, render_color.a);
     sdl2.RenderClear(state.sdl_renderer);
@@ -66,74 +59,79 @@ draw :: proc(state: ^State) {
     new_ui.max_size.x = state.screen_width
     new_ui.max_size.y = state.screen_height
 
-    ui.open_element(new_ui, nil, {
+    ui.open_element(new_ui, nil,
-        dir = .LeftToRight,
+        {
-        kind = {ui.Grow{}, ui.Grow{}},
+            dir = .LeftToRight,
-    })
+            kind = {ui.Grow{}, ui.Grow{}},
+        },
+        style = {
+            background_color = .Background1
+        }
+    )
     { 
+        floating_panels := [16]int{}
+        num_floating := 0
+
         for i in 0..<len(state.panels.data) {
             if panel, ok := util.get(&state.panels, i).?; ok {
-                if panel.render_proc != nil {
+                if panel.render != nil {
-                    panel.render_proc(state, &panel.panel_state)
+                    if panel.is_floating {
+                        if num_floating < len(floating_panels) {
+                            floating_panels[num_floating] = i
+                            num_floating += 1
+                        }
+                    } else {
+                        panel->render(state)
+                    }
                 }
             }
         }
+
+        for i in 0..<num_floating {
+            panel_id := floating_panels[i]
+
+            if panel, ok := util.get(&state.panels, panel_id).?; ok {
+                panel->render(state)
+            }
+        }
+
+        {
+            if state.mode != .Insert && state.current_input_map.show_help {
+                @(thread_local)
+                layout: ui.UI_Layout
+
+                ui.open_element(new_ui, nil,
+                    {
+                        dir = .TopToBottom,
+                        kind = {ui.Fit{}, ui.Fit{}},
+                        pos = {state.screen_width - layout.size.x - state.source_font_width, state.screen_height - layout.size.y - state.source_font_height},
+                        floating = true,
+                    },
+                    style = {
+                        border = {.Left, .Right, .Top, .Bottom},
+                        border_color = .Background4,
+                        background_color = .Background3
+                    }
+                )
+                {
+                    for key, action in state.current_input_map.key_actions {
+                        ui.open_element(new_ui, fmt.tprintf("%s - %s", key, action.description), {})
+                        ui.close_element(new_ui)
+                    }
+                    for key, action in state.current_input_map.ctrl_key_actions {
+                        ui.open_element(new_ui, fmt.tprintf("<C>-%s - %s", key, action.description), {})
+                        ui.close_element(new_ui)
+                    }
+                }
+                layout = ui.close_element(new_ui)
+            }
+        }
     }
     ui.close_element(new_ui)
 
-    ui.compute_layout_2(new_ui)
+    ui.compute_layout(new_ui)
     ui.draw(new_ui, state)
 
-    // TODO: figure out when to not show the input help menu
-    if state.mode != .Insert { // && state.current_input_map != &state.input_map.mode[state.mode] {
-        longest_description := 0;
-        for key, action in state.current_input_map.key_actions {
-            if len(action.description) > longest_description {
-                longest_description = len(action.description);
-            }
-        }
-        for key, action in state.current_input_map.ctrl_key_actions {
-            if len(action.description) > longest_description {
-                longest_description = len(action.description);
-            }
-        }
-        longest_description += 8;
-
-        helper_height := state.source_font_height * (len(state.current_input_map.key_actions) + len(state.current_input_map.ctrl_key_actions));
-        offset_from_bottom := state.source_font_height * 2;
-
-        core.draw_rect(
-            state,
-            state.screen_width - longest_description * state.source_font_width,
-            state.screen_height - helper_height - offset_from_bottom,
-            longest_description*state.source_font_width,
-            helper_height,
-            .Background2
-        );
-
-        index := 0;
-        for key, action in state.current_input_map.key_actions {
-            core.draw_text(
-                state,
-                fmt.tprintf("%s - %s", key, action.description),
-                state.screen_width - longest_description * state.source_font_width,
-                state.screen_height - helper_height + index * state.source_font_height - offset_from_bottom
-            );
-
-            index += 1;
-        }
-        for key, action in state.current_input_map.ctrl_key_actions {
-            core.draw_text(
-                state,
-                fmt.tprintf("<C>-%s - %s", key, action.description),
-                state.screen_width - longest_description * state.source_font_width,
-                state.screen_height - helper_height + index * state.source_font_height - offset_from_bottom
-            );
-
-            index += 1;
-        }
-    }
-
     sdl2.RenderPresent(state.sdl_renderer);
 }
 
@@ -164,42 +162,9 @@ expose_event_watcher :: proc "c" (state: rawptr, event: ^sdl2.Event) -> i32 {
     return 0;
 }
 
-ui_file_buffer :: proc(s: ^ui.State, buffer: ^FileBuffer) {
-    draw_func := proc(state: ^State, e: ui.UI_Element, user_data: rawptr) {
-        buffer := transmute(^FileBuffer)user_data;
-        if buffer != nil {
-            buffer.glyphs.width = e.layout.size.x / state.source_font_width;
-            buffer.glyphs.height = e.layout.size.y / state.source_font_height + 1;
-
-            core.draw_file_buffer(state, buffer, e.layout.pos.x, e.layout.pos.y);
-        }
-    };
-
-    relative_file_path, _ := filepath.rel(state.directory, buffer.file_path, context.temp_allocator)
-
-    ui.open_element(s, nil, {
-        dir = .TopToBottom,
-        kind = {ui.Grow{}, ui.Grow{}},
-    })
-    {
-        ui.open_element(s, ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)buffer}, {
-            kind = {ui.Grow{}, ui.Grow{}}
-        })
-        ui.close_element(s)
-
-        ui.open_element(s, nil, {
-            kind = {ui.Grow{}, ui.Exact(state.source_font_height)}
-        })
-        {
-            ui.open_element(s, fmt.tprintf("%s", state.mode), {})
-            ui.close_element(s)
-        }
-        ui.close_element(s)
-    }
-    ui.close_element(s)
-}
-
 main :: proc() {
+    ts.set_allocator() 
+
     _command_arena: mem.Arena
     mem.arena_init(&_command_arena, make([]u8, 1024*1024));
 
@@ -227,10 +192,6 @@ main :: proc() {
         prev_elements = make([]ui.UI_Element, 8192),
     }
 
-    // TODO: don't use this
-    mem.scratch_allocator_init(&scratch, 1024*1024);
-    scratch_alloc = mem.scratch_allocator(&scratch);
-
     core.reset_input_map(&state)
 
     // core.register_editor_command(
@@ -273,55 +234,52 @@ main :: proc() {
     //     }
     // )
 
-    core.register_editor_command(
+    // core.register_editor_command(
-        &state.commands,
+    //     &state.commands,
-        "nl.spacegirl.editor.core",
+    //     "nl.spacegirl.editor.core",
-        "New Scratch Buffer",
+    //     "New Scratch Buffer",
-        "Opens a new scratch buffer",
+    //     "Opens a new scratch buffer",
-        proc(state: ^State) {
+    //     proc(state: ^State) {
-            buffer := core.new_virtual_file_buffer(context.allocator);
+    //         buffer := core.new_virtual_file_buffer(context.allocator);
-            util.append_static_list(&state.panels, panels.make_file_buffer_panel(len(state.buffers)))
+    //         util.append_static_list(&state.panels, panels.make_file_buffer_panel(len(state.buffers)))
-            runtime.append(&state.buffers, buffer);
+    //         runtime.append(&state.buffers, buffer);
-        }
+    //     }
-    )
+    // )
-    core.register_editor_command(
+    // core.register_editor_command(
-        &state.commands,
+    //     &state.commands,
-        "nl.spacegirl.editor.core",
+    //     "nl.spacegirl.editor.core",
-        "Open File",
+    //     "Open File",
-        "Opens a file in a new buffer",
+    //     "Opens a file in a new buffer",
-        proc(state: ^State) {
+    //     proc(state: ^State) {
-            log.info("open file args:");
+    //         log.info("open file args:");
 
-            Args :: struct {
+    //         Args :: struct {
-                file_path: string
+    //             file_path: string
-            }
+    //         }
 
-            if args, ok := core.attempt_read_command_args(Args, state.command_args[:]); ok {
+    //         if args, ok := core.attempt_read_command_args(Args, state.command_args[:]); ok {
-                log.info("attempting to open file", args.file_path)
+    //             log.info("attempting to open file", args.file_path)
 
-                panels.open_file_buffer_in_new_panel(state, args.file_path, 0, 0)
+    //             panels.open_file_buffer_in_new_panel(state, args.file_path, 0, 0)
-            }
+    //         }
-        }
+    //     }
-    )
+    // )
-    core.register_editor_command(
+    // core.register_editor_command(
-        &state.commands,
+    //     &state.commands,
-        "nl.spacegirl.editor.core",
+    //     "nl.spacegirl.editor.core",
-        "Quit",
+    //     "Quit",
-        "Quits the application",
+    //     "Quits the application",
-        proc(state: ^State) {
+    //     proc(state: ^State) {
-            state.should_close = true
+    //         state.should_close = true
-        }
+    //     }
-    )
+    // )
 
     if len(os.args) > 1 {
         for arg in os.args[1:] {
-            panels.open_file_buffer_in_new_panel(&state, arg, 0, 0)
+            panels.open(&state, panels.make_file_buffer_panel(arg))
         }
     } else {
-        buffer := core.new_virtual_file_buffer(context.allocator);
+        panels.open(&state, panels.make_file_buffer_panel(""))
-
-        panels.open(&state, panels.make_file_buffer_panel(len(state.buffers)))
-        runtime.append(&state.buffers, buffer);
     }
 
     if sdl2.Init({.VIDEO}) < 0 {
@@ -423,12 +381,14 @@ main :: proc() {
                 }
 
                 run_key_action := proc(state: ^core.State, control_key_pressed: bool, key: core.Key) -> bool {
-                    if state.current_input_map != nil {
+                    if current_panel, ok := state.current_panel.?; ok {
+                        panel := util.get(&state.panels, current_panel).?
+
                         if control_key_pressed {
                             if action, exists := state.current_input_map.ctrl_key_actions[key]; exists {
                                 switch value in action.action {
                                     case core.EditorAction:
-                                        value(state);
+                                        value(state, panel);
                                         return true;
                                     case core.InputActions:
                                         state.current_input_map = &(&state.current_input_map.ctrl_key_actions[key]).action.(core.InputActions)
@@ -439,7 +399,7 @@ main :: proc() {
                             if action, exists := state.current_input_map.key_actions[key]; exists {
                                 switch value in action.action {
                                     case core.EditorAction:
-                                        value(state);
+                                        value(state, panel);
                                         return true;
                                     case core.InputActions:
                                         state.current_input_map = &(&state.current_input_map.key_actions[key]).action.(core.InputActions)
@@ -516,8 +476,8 @@ main :: proc() {
                             }
 
                             if current_panel, ok := state.current_panel.?; ok {
-                                if panel, ok := util.get(&state.panels, current_panel).?; ok && panel.on_buffer_input_proc != nil {
+                                if panel, ok := util.get(&state.panels, current_panel).?; ok && panel.on_buffer_input != nil {
-                                    panel.on_buffer_input_proc(&state, &panel.panel_state)
+                                    panel->on_buffer_input(&state)
                                 }
                             }
                         }

src/panels/file_buffer.odin

@@ -0,0 +1,527 @@
+package panels
+
+import "base:runtime"
+import "core:log"
+import "core:fmt"
+import "core:path/filepath"
+
+import "vendor:sdl2"
+
+import ts "../tree_sitter"
+import "../core"
+import "../ui"
+
+make_file_buffer_panel :: proc(file_path: string, line: int = 0, col: int = 0) -> core.Panel {
+    return core.Panel {
+        type = core.FileBufferPanel {
+            file_path = file_path,
+            line = line,
+            col = col,
+        },
+        drop = proc(panel: ^core.Panel, state: ^core.State) {
+            panel_state := &panel.type.(core.FileBufferPanel)
+
+            ts.delete_state(&panel_state.buffer.tree)
+        },
+        create = proc(panel: ^core.Panel, state: ^core.State) {
+            context.allocator = panel.allocator
+
+            panel_state := &panel.type.(core.FileBufferPanel)
+            panel.input_map = core.new_input_map()
+
+            if len(panel_state.file_path) == 0 {
+                panel_state.buffer = core.new_virtual_file_buffer(panel.allocator)
+            } else {
+                buffer, err := core.new_file_buffer(panel.allocator, panel_state.file_path, state.directory)
+                if err.type != .None {
+                    log.error("Failed to create file buffer:", err);
+                    return;
+                }
+
+                buffer.history.cursor.line = panel_state.line
+                buffer.history.cursor.col = panel_state.col
+                buffer.top_line = buffer.history.cursor.line
+                core.update_file_buffer_index_from_cursor(&buffer)
+
+                panel_state.buffer = buffer
+            }
+
+            leader_actions := core.new_input_actions(show_help = true)
+            register_default_leader_actions(&leader_actions);
+            file_buffer_leader_actions(&leader_actions);
+            core.register_key_action(&panel.input_map.mode[.Normal], .SPACE, leader_actions, "leader commands");
+
+            panel_actions := core.new_input_actions(show_help = true)
+            register_default_panel_actions(&panel_actions)
+            core.register_ctrl_key_action(&panel.input_map.mode[.Normal], .W, panel_actions, "Panel Navigation") 
+
+            file_buffer_input_actions(&panel.input_map.mode[.Normal]);
+            file_buffer_visual_actions(&panel.input_map.mode[.Visual]);
+            file_buffer_text_input_actions(&panel.input_map.mode[.Normal]);
+        },
+        buffer = proc(panel: ^core.Panel, state: ^core.State) -> (buffer: ^core.FileBuffer, ok: bool) {
+            panel_state := &panel.type.(core.FileBufferPanel)
+
+            return &panel_state.buffer, true
+        },
+        render = proc(panel: ^core.Panel, state: ^core.State) -> (ok: bool) {
+            panel_state := &panel.type.(core.FileBufferPanel)
+
+            s := transmute(^ui.State)state.ui
+            render_file_buffer(state, s, &panel_state.buffer)
+
+            if viewed_symbol, ok := panel_state.viewed_symbol.?; ok {
+                ui.open_element(s, nil,
+                    {
+                        dir = .TopToBottom,
+                        kind = {ui.Fit{}, ui.Fit{}},
+                        floating = true, 
+                    },
+                    style = {
+                        background_color = .Background2,
+                    },
+                )
+                {
+                    ui.open_element(s, viewed_symbol, {})
+                    ui.close_element(s)
+                }
+                ui.close_element(s)
+            }
+
+            return true
+        }
+    }
+}
+
+render_file_buffer :: proc(state: ^core.State, s: ^ui.State, buffer: ^core.FileBuffer) {
+    draw_func := proc(state: ^core.State, e: ui.UI_Element, user_data: rawptr) {
+        buffer := transmute(^core.FileBuffer)user_data;
+        if buffer != nil {
+            core.draw_file_buffer(state, buffer, e.layout.pos.x, e.layout.pos.y, e.layout.size.x, e.layout.size.y);
+        }
+    };
+
+    relative_file_path, _ := filepath.rel(state.directory, buffer.file_path, context.temp_allocator)
+
+    ui.open_element(s, nil,
+        {
+            dir = .TopToBottom,
+            kind = {ui.Grow{}, ui.Grow{}},
+        },
+    )
+    {
+        ui.open_element(s,
+            ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)buffer},
+            {
+                kind = {ui.Grow{}, ui.Grow{}}
+            },
+        )
+        ui.close_element(s)
+
+        ui.open_element(s, nil, {
+            kind = {ui.Grow{}, ui.Exact(state.source_font_height)}
+            },
+            style = {
+                border = {.Left, .Right, .Top, .Bottom},
+                border_color = .Background4,
+                background_color = .Background1,
+            }
+        )
+        {
+            ui.open_element(s, fmt.tprintf("%s", state.mode), {})
+            ui.close_element(s)
+
+            if .UnsavedChanges in buffer.flags {
+                ui.open_element(s, "[Unsaved Changes]", {})
+                ui.close_element(s)
+            }
+
+            ui.open_element(s, nil, { kind = {ui.Grow{}, ui.Grow{}}})
+            ui.close_element(s)
+
+            it := core.new_file_buffer_iter_with_cursor(buffer, buffer.history.cursor)
+            ui.open_element(
+                s,
+                fmt.tprintf(
+                    "%v:%v - Slice %v:%v - Char: %v - Last Col: %v",
+                    buffer.history.cursor.line + 1,
+                    buffer.history.cursor.col + 1,
+                    buffer.history.cursor.index.chunk_index,
+                    buffer.history.cursor.index.char_index,
+                    core.get_character_at_iter(it),
+                    buffer.last_col,
+                ),
+                {}
+            )
+            ui.close_element(s)
+        }
+        ui.close_element(s)
+    }
+    ui.close_element(s)
+}
+
+file_buffer_leader_actions :: proc(input_map: ^core.InputActions) {
+    core.register_key_action(input_map, .K, proc(state: ^core.State, user_data: rawptr) {
+        panel := transmute(^core.Panel)user_data
+        panel_state := &panel.type.(core.FileBufferPanel)
+        buffer := &panel_state.buffer
+
+        ts.update_cursor(&buffer.tree, buffer.history.cursor.line, buffer.history.cursor.col)
+        panel_state.viewed_symbol = ts.print_node_type(&buffer.tree)
+
+        core.reset_input_map(state)
+    }, "View Symbol")
+}
+
+file_buffer_go_actions :: proc(input_map: ^core.InputActions) {
+    core.register_key_action(input_map, .H, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.move_cursor_start_of_line(buffer);
+        core.reset_input_map(state)
+    }, "move to beginning of line");
+    core.register_key_action(input_map, .L, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.move_cursor_end_of_line(buffer);
+        core.reset_input_map(state)
+    }, "move to end of line");
+}
+
+file_buffer_input_actions :: proc(input_map: ^core.InputActions) {
+    // Cursor Movement
+    {
+        core.register_key_action(input_map, .W, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_forward_start_of_word(buffer);
+        }, "move forward one word");
+        core.register_key_action(input_map, .E, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_forward_end_of_word(buffer);
+        }, "move forward to end of word");
+
+        core.register_key_action(input_map, .B, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_backward_start_of_word(buffer);
+        }, "move backward one word");
+
+        core.register_key_action(input_map, .K, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_up(buffer);
+        }, "move up one line");
+        core.register_key_action(input_map, .J, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_down(buffer);
+        }, "move down one line");
+        core.register_key_action(input_map, .H, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_left(buffer);
+        }, "move left one char");
+        core.register_key_action(input_map, .L, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.move_cursor_right(buffer);
+        }, "move right one char");
+
+        core.register_ctrl_key_action(input_map, .U, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.scroll_file_buffer(buffer, .Up);
+        }, "scroll buffer up");
+        core.register_ctrl_key_action(input_map, .D, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.scroll_file_buffer(buffer, .Down);
+        }, "scroll buffer up");
+    }
+
+    // Scale font size
+    {
+        core.register_ctrl_key_action(input_map, .MINUS, proc(state: ^core.State, user_data: rawptr) {
+            if state.source_font_height > 16 {
+                state.source_font_height -= 2;
+                state.source_font_width = state.source_font_height / 2;
+
+                state.font_atlas = core.gen_font_atlas(state, core.HardcodedFontPath);
+            }
+            log.debug(state.source_font_height);
+        }, "increase font size");
+        core.register_ctrl_key_action(input_map, .EQUAL, proc(state: ^core.State, user_data: rawptr) {
+            state.source_font_height += 2;
+            state.source_font_width = state.source_font_height / 2;
+
+            state.font_atlas = core.gen_font_atlas(state, core.HardcodedFontPath);
+        }, "decrease font size");
+    }
+
+    // Save file
+    core.register_ctrl_key_action(input_map, .S, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        if err := core.save_buffer_to_disk(state, buffer); err != nil {
+            log.errorf("failed to save buffer to disk: %v", err)
+        }
+    }, "Save file")
+
+    go_actions := core.new_input_actions(show_help = true)
+    file_buffer_go_actions(&go_actions);
+    core.register_key_action(input_map, .G, go_actions, "Go commands");
+
+    core.register_key_action(input_map, .V, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        state.mode = .Visual;
+        core.reset_input_map(state)
+
+        buffer.selection = core.new_selection(buffer.history.cursor);
+    }, "enter visual mode");
+
+    core.register_key_action(input_map, .ESCAPE, proc(state: ^core.State, user_data: rawptr) {
+        panel := transmute(^core.Panel)user_data
+        panel_state := &panel.type.(core.FileBufferPanel)
+        buffer := &panel_state.buffer
+
+        panel_state.viewed_symbol = nil
+    });
+}
+
+file_buffer_visual_actions :: proc(input_map: ^core.InputActions) {
+    core.register_key_action(input_map, .ESCAPE, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        state.mode = .Normal;
+        core.reset_input_map(state)
+
+        buffer.selection = nil;
+        core.update_file_buffer_scroll(buffer)
+    }, "exit visual mode");
+
+    // Cursor Movement
+    {
+        core.register_key_action(input_map, .W, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_forward_start_of_word(buffer, cursor = &sel_cur.end);
+        }, "move forward one word");
+        core.register_key_action(input_map, .E, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_forward_end_of_word(buffer, cursor = &sel_cur.end);
+        }, "move forward to end of word");
+
+        core.register_key_action(input_map, .B, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_backward_start_of_word(buffer, cursor = &sel_cur.end);
+        }, "move backward one word");
+
+        core.register_key_action(input_map, .K, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_up(buffer, cursor = &sel_cur.end);
+        }, "move up one line");
+        core.register_key_action(input_map, .J, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_down(buffer, cursor = &sel_cur.end);
+        }, "move down one line");
+        core.register_key_action(input_map, .H, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_left(buffer, cursor = &sel_cur.end);
+        }, "move left one char");
+        core.register_key_action(input_map, .L, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.move_cursor_right(buffer, cursor = &sel_cur.end);
+        }, "move right one char");
+
+        core.register_ctrl_key_action(input_map, .U, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.scroll_file_buffer(buffer, .Up, cursor = &sel_cur.end);
+        }, "scroll buffer up");
+        core.register_ctrl_key_action(input_map, .D, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            sel_cur := &(buffer.selection.?);
+
+            core.scroll_file_buffer(buffer, .Down, cursor = &sel_cur.end);
+        }, "scroll buffer up");
+    }
+
+    // Text Modification
+    {
+        core.register_key_action(input_map, .D, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.push_new_snapshot(&buffer.history)
+
+            sel_cur := &(buffer.selection.?);
+
+            core.delete_content(buffer, sel_cur);
+            buffer.selection = nil;
+            core.update_file_buffer_scroll(buffer)
+
+            state.mode = .Normal
+            core.reset_input_map(state)
+        }, "delete selection");
+
+        core.register_key_action(input_map, .C, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.push_new_snapshot(&buffer.history)
+
+            sel_cur := &(buffer.selection.?);
+
+            core.delete_content(buffer, sel_cur);
+            buffer.selection = nil;
+            core.update_file_buffer_scroll(buffer)
+
+            state.mode = .Insert
+            core.reset_input_map(state, core.Mode.Normal)
+            sdl2.StartTextInput();
+        }, "change selection");
+    }
+
+    // Copy-Paste
+    {
+        core.register_key_action(input_map, .Y, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.yank_selection(state, buffer)
+
+            state.mode = .Normal;
+            core.reset_input_map(state)
+
+            buffer.selection = nil;
+            core.update_file_buffer_scroll(buffer)
+        }, "Yank Line");
+
+        core.register_key_action(input_map, .P, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.push_new_snapshot(&buffer.history)
+
+            if state.yank_register.whole_line {
+                core.insert_content(buffer, []u8{'\n'});
+                core.paste_register(state, state.yank_register, buffer)
+                core.insert_content(buffer, []u8{'\n'});
+            } else {
+                core.paste_register(state, state.yank_register, buffer)
+            }
+
+            core.reset_input_map(state)
+        }, "Paste");
+    }
+}
+
+file_buffer_text_input_actions :: proc(input_map: ^core.InputActions) {
+    core.register_key_action(input_map, .I, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.push_new_snapshot(&buffer.history)
+
+        state.mode = .Insert;
+        sdl2.StartTextInput();
+    }, "enter insert mode");
+    core.register_key_action(input_map, .A, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.push_new_snapshot(&buffer.history)
+
+        core.move_cursor_right(buffer, false);
+        state.mode = .Insert;
+        sdl2.StartTextInput();
+    }, "enter insert mode after character (append)");
+
+    core.register_key_action(input_map, .U, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.pop_snapshot(&buffer.history, true)
+        ts.parse_buffer(&buffer.tree, core.tree_sitter_file_buffer_input(buffer))
+    }, "Undo");
+
+    core.register_ctrl_key_action(input_map, .R, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.recover_snapshot(&buffer.history)
+        ts.parse_buffer(&buffer.tree, core.tree_sitter_file_buffer_input(buffer))
+    }, "Redo");
+
+    // TODO: add shift+o to insert newline above current one
+
+    core.register_key_action(input_map, .O, proc(state: ^core.State, user_data: rawptr) {
+        buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+        core.push_new_snapshot(&buffer.history)
+
+        if buffer := buffer; buffer != nil {
+            core.move_cursor_end_of_line(buffer, false);
+            runtime.clear(&buffer.input_buffer)
+
+            append(&buffer.input_buffer, '\n')
+
+            state.mode = .Insert;
+
+            sdl2.StartTextInput();
+        }
+    }, "insert mode on newline");
+
+    // Copy-Paste
+    {
+        {
+            yank_actions := core.new_input_actions(show_help = true)
+            defer core.register_key_action(input_map, .Y, yank_actions)
+
+            core.register_key_action(&yank_actions, .Y, proc(state: ^core.State, user_data: rawptr) {
+                buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+                core.yank_whole_line(state, buffer)
+
+                core.reset_input_map(state)
+            }, "Yank Line");
+        }
+
+        core.register_key_action(input_map, .P, proc(state: ^core.State, user_data: rawptr) {
+            buffer := &(&(transmute(^core.Panel)user_data).type.(core.FileBufferPanel)).buffer
+
+            core.push_new_snapshot(&buffer.history)
+
+            if state.yank_register.whole_line {
+                core.move_cursor_end_of_line(buffer, false);
+                core.insert_content(buffer, []u8{'\n'});
+                core.move_cursor_right(buffer, false);
+            } else {
+                core.move_cursor_right(buffer)
+            }
+            core.paste_register(state, state.yank_register, buffer)
+            core.move_cursor_start_of_line(buffer)
+
+            core.reset_input_map(state)
+        }, "Paste");
+    }
+
+}

src/panels/grep.odin

@@ -0,0 +1,326 @@
+package panels
+
+import "base:runtime"
+import "core:mem"
+import "core:fmt"
+import "core:strings"
+import "core:log"
+
+import "vendor:sdl2"
+
+import ts "../tree_sitter"
+import "../core"
+import "../util"
+import "../ui"
+
+open_grep_panel :: proc(state: ^core.State) {
+    open(state, make_grep_panel())
+
+    state.mode = .Insert
+    sdl2.StartTextInput()
+}
+
+make_grep_panel :: proc() -> core.Panel {
+    run_query :: proc(panel_state: ^core.GrepPanel, query: string, directory: string) {
+        if panel_state.query_region.arena != nil {
+            mem.end_arena_temp_memory(panel_state.query_region)
+        }
+        panel_state.query_region = mem.begin_arena_temp_memory(&panel_state.query_arena)
+
+        context.allocator = mem.arena_allocator(&panel_state.query_arena)
+
+        rs_results := grep(
+            strings.clone_to_cstring(query),
+            strings.clone_to_cstring(directory)
+        );
+
+        panel_state.query_results = rs_grep_as_results(&rs_results)
+
+        panel_state.selected_result = 0
+        if len(panel_state.query_results) > 0 {
+            core.update_glyph_buffer_from_bytes(
+                &panel_state.glyphs,
+                transmute([]u8)panel_state.query_results[panel_state.selected_result].file_context,
+                panel_state.query_results[panel_state.selected_result].line,
+            )
+        }
+    }
+
+    return core.Panel {
+        type = core.GrepPanel {},
+        is_floating = true,
+        drop = proc(panel: ^core.Panel, state: ^core.State) {
+            panel_state := &panel.type.(core.GrepPanel)
+
+            ts.delete_state(&panel_state.buffer.tree)
+        },
+        create = proc(panel: ^core.Panel, state: ^core.State) {
+            context.allocator = panel.allocator
+
+            panel_state := &panel.type.(core.GrepPanel)
+
+            arena_bytes, err := make([]u8, 1024*1024*2)
+            if err != nil {
+                log.errorf("failed to allocate arena for grep panel: '%v'", err)
+                return
+            }
+            mem.arena_init(&panel_state.query_arena, arena_bytes)
+
+            panel.input_map = core.new_input_map(show_help = true)
+            panel_state.glyphs = core.make_glyph_buffer(256,256)
+            panel_state.buffer = core.new_virtual_file_buffer()
+
+            panel_actions := core.new_input_actions(show_help = true)
+            register_default_panel_actions(&panel_actions)
+            core.register_ctrl_key_action(&panel.input_map.mode[.Normal], .W, panel_actions, "Panel Navigation") 
+
+            core.register_key_action(&panel.input_map.mode[.Normal], .ENTER, proc(state: ^core.State, user_data: rawptr) {
+                this_panel := transmute(^core.Panel)user_data
+
+                if panel_state, ok := &this_panel.type.(core.GrepPanel); ok {
+                    if panel_state.query_results != nil {
+                        selected_result := &panel_state.query_results[panel_state.selected_result]
+
+                        if panel_id, ok := open(state, make_file_buffer_panel(selected_result.file_path, selected_result.line, selected_result.col)); ok {
+                            close(state, this_panel.id)
+
+                            state.current_panel = panel_id
+                        } else {
+                            log.error("failed to open file buffer in new panel")
+                        }
+                    }
+                }
+            }, "Open File");
+            core.register_key_action(&panel.input_map.mode[.Normal], .I, proc(state: ^core.State, user_data: rawptr) {
+                state.mode = .Insert;
+                sdl2.StartTextInput();
+            }, "enter insert mode");
+            core.register_key_action(&panel.input_map.mode[.Normal], .K, proc(state: ^core.State, user_data: rawptr) {
+                this_panel := transmute(^core.Panel)user_data
+
+                if panel_state, ok := &this_panel.type.(core.GrepPanel); ok {
+                    // TODO: bounds checking
+                    panel_state.selected_result -= 1
+
+                    core.update_glyph_buffer_from_bytes(
+                        &panel_state.glyphs,
+                        transmute([]u8)panel_state.query_results[panel_state.selected_result].file_context,
+                        panel_state.query_results[panel_state.selected_result].line,
+                    )
+                }
+            }, "move selection up");
+            core.register_key_action(&panel.input_map.mode[.Normal], .J, proc(state: ^core.State, user_data: rawptr) {
+                this_panel := transmute(^core.Panel)user_data
+
+                if panel_state, ok := &this_panel.type.(core.GrepPanel); ok {
+                    // TODO: bounds checking
+                    panel_state.selected_result += 1
+
+                    core.update_glyph_buffer_from_bytes(
+                        &panel_state.glyphs,
+                        transmute([]u8)panel_state.query_results[panel_state.selected_result].file_context,
+                        panel_state.query_results[panel_state.selected_result].line,
+                    )
+                }
+            }, "move selection down");
+
+            core.register_key_action(&panel.input_map.mode[.Insert], .ESCAPE, proc(state: ^core.State, user_data: rawptr) {
+                state.mode = .Normal;
+                sdl2.StopTextInput();
+            }, "exit insert mode");
+            core.register_key_action(&panel.input_map.mode[.Normal], .ESCAPE, proc(state: ^core.State, user_data: rawptr) {
+                this_panel := transmute(^core.Panel)user_data
+                close(state, this_panel.id)
+            }, "close panel");
+        },
+        buffer = proc(panel: ^core.Panel, state: ^core.State) -> (buffer: ^core.FileBuffer, ok: bool) {
+            if panel_state, ok := &panel.type.(core.GrepPanel); ok {
+                return &panel_state.buffer, true
+            }
+
+            return
+        },
+        on_buffer_input = proc(panel: ^core.Panel, state: ^core.State) {
+            if panel_state, ok := &panel.type.(core.GrepPanel); ok {
+                run_query(panel_state, string(panel_state.buffer.input_buffer[:]), state.directory)
+            }
+        },
+        render = proc(panel: ^core.Panel, state: ^core.State) -> (ok: bool) {
+            if panel_state, ok := &panel.type.(core.GrepPanel); ok {
+                s := transmute(^ui.State)state.ui
+
+                ui.open_element(s, nil,
+                    {
+                        dir = .TopToBottom,
+                        kind = {ui.Grow{}, ui.Grow{}},
+                        floating = true, 
+                    },
+                    style = {
+                        background_color = .Background1,
+                    },
+                )
+                {
+                    // query results and file contents side-by-side
+                    ui.open_element(s, nil, {
+                        dir = .LeftToRight,
+                        kind = {ui.Grow{}, ui.Grow{}}
+                    })
+                    {
+                        if panel_state.query_results != nil {
+                            // query results
+                            query_result_container := ui.open_element(s, nil,
+                                {
+                                    dir = .TopToBottom,
+                                    kind = {ui.Grow{}, ui.Grow{}}
+                                },
+                                style = {
+                                    border = {.Right},
+                                    border_color = .Background4
+                                }
+                            )
+                            {
+                                container_height := query_result_container.layout.size.y
+                                max_results := container_height / 16
+
+                                for result, i in panel_state.query_results {
+                                    if i > max_results {
+                                        break
+                                    }
+
+                                    ui.open_element(s, nil, {
+                                        dir = .LeftToRight,
+                                        kind = {ui.Fit{}, ui.Fit{}},
+                                    })
+                                    {
+                                        defer ui.close_element(s)
+
+                                        ui.open_element(s, fmt.tprintf("%v:%v: ", result.line, result.col), {})
+                                        ui.close_element(s)
+
+
+                                        style := ui.UI_Style{}
+
+                                        if panel_state.selected_result == i {
+                                            style.background_color = .Background2
+                                        }
+
+                                        ui.open_element(s, result.file_path[len(state.directory):], {}, style)
+                                        ui.close_element(s)
+                                    }
+                                }
+                            }
+                            ui.close_element(s)
+
+                            // file contents
+                            selected_result := &panel_state.query_results[panel_state.selected_result]
+
+                            core.update_glyph_buffer_from_bytes(
+                                &panel_state.glyphs,
+                                transmute([]u8)selected_result.file_context,
+                                selected_result.line,
+                            )
+                            render_glyph_buffer(state, s, &panel_state.glyphs)
+                        }
+                    }
+                    ui.close_element(s)
+
+                    // text input
+                    ui.open_element(s, nil,
+                        {
+                            kind = {ui.Grow{}, ui.Exact(state.source_font_height)}
+                        },
+                        style = {
+                            background_color = .Background2
+                        }
+                    )
+                    { 
+                        defer ui.close_element(s)
+
+                        render_raw_buffer(state, s, &panel_state.buffer)
+                    }
+                }
+                ui.close_element(s)
+
+                return true
+            }
+
+            return false
+        }
+    }
+}
+
+foreign import grep_lib "../pkg/grep_lib/target/debug/libgrep.a"
+@(default_calling_convention = "c")
+foreign grep_lib {
+	grep :: proc (pattern: cstring, directory: cstring) -> RS_GrepResults ---
+    free_grep_results :: proc(results: RS_GrepResults) ---
+}
+
+RS_GrepResults :: struct {
+    results: [^]RS_GrepResult,
+    len: u32,
+}
+RS_GrepResult :: struct {
+    line_number: u64,
+    column: u64,
+
+    text_len: u32,
+    path_len: u32,
+
+    text: [^]u8,
+    path: [^]u8,
+}
+
+@(private)
+rs_grep_as_results :: proc(results: ^RS_GrepResults, allocator := context.allocator) -> []core.GrepQueryResult {
+    context.allocator = allocator
+
+    query_results := make([]core.GrepQueryResult, results.len)
+
+    for i in 0..<results.len {
+        r := results.results[i]
+
+        query_results[i] = core.GrepQueryResult {
+            file_context = strings.clone_from_ptr(r.text, int(r.text_len)) or_continue,
+            file_path = strings.clone_from_ptr(r.path, int(r.path_len)) or_continue,
+            line = int(r.line_number) - 1,
+            col = int(r.column) - 1,
+        }
+    }
+
+    free_grep_results(results^)
+    return query_results
+}
+
+render_raw_buffer :: proc(state: ^core.State, s: ^ui.State, buffer: ^core.FileBuffer) {
+    draw_func := proc(state: ^core.State, e: ui.UI_Element, user_data: rawptr) {
+        buffer := transmute(^core.FileBuffer)user_data;
+        if buffer != nil {
+            core.draw_file_buffer(state, buffer, e.layout.pos.x, e.layout.pos.y, e.layout.size.x, e.layout.size.y, false);
+        }
+    };
+
+    ui.open_element(s, ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)buffer}, {
+        kind = {ui.Grow{}, ui.Grow{}}
+    })
+    ui.close_element(s)
+    
+}
+
+render_glyph_buffer :: proc(state: ^core.State, s: ^ui.State, glyphs: ^core.GlyphBuffer) {
+    draw_func := proc(state: ^core.State, e: ui.UI_Element, user_data: rawptr) {
+        glyphs := transmute(^core.GlyphBuffer)user_data;
+        if glyphs != nil {
+            glyphs.width = e.layout.size.x / state.source_font_width;
+            glyphs.height = e.layout.size.y / state.source_font_height + 1;
+
+            core.draw_glyph_buffer(state, glyphs, e.layout.pos.x, e.layout.pos.y, 0, true);
+        }
+    };
+
+    ui.open_element(s, ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)glyphs}, {
+        kind = {ui.Grow{}, ui.Grow{}}
+    })
+    ui.close_element(s)
+    
+}

src/panels/panels.odin

@@ -2,73 +2,29 @@ package panels
 
 import "base:runtime"
 import "core:mem"
-import "core:path/filepath"
 import "core:fmt"
 import "core:strings"
 import "core:log"
 
 import "vendor:sdl2"
 
+import ts "../tree_sitter"
 import "../core"
-import "../input"
 import "../util"
 import "../ui"
 
-foreign import grep_lib "../pkg/grep_lib/target/debug/libgrep.a"
-@(default_calling_convention = "c")
-foreign grep_lib {
-	grep :: proc (pattern: cstring, directory: cstring) -> RS_GrepResults ---
-    free_grep_results :: proc(results: RS_GrepResults) ---
-}
-
-RS_GrepResults :: struct {
-    results: [^]RS_GrepResult,
-    len: u32,
-}
-RS_GrepResult :: struct {
-    line_number: u64,
-    column: u64,
-
-    text_len: u32,
-    path_len: u32,
-
-    text: [^]u8,
-    path: [^]u8,
-}
-
-@(private)
-rs_grep_as_results :: proc(results: ^RS_GrepResults, allocator := context.allocator) -> []core.GrepQueryResult {
-    context.allocator = allocator
-
-    query_results := make([]core.GrepQueryResult, results.len)
-
-    for i in 0..<results.len {
-        r := results.results[i]
-
-        query_results[i] = core.GrepQueryResult {
-            file_context = strings.clone_from_ptr(r.text, int(r.text_len)) or_continue,
-            file_path = strings.clone_from_ptr(r.path, int(r.path_len)) or_continue,
-            line = int(r.line_number) - 1,
-            col = int(r.column) - 1,
-        }
-    }
-
-    return query_results
-}
-
-// NOTE: odd that this is here, but I don't feel like thinking of a better dep-tree to fix it
 register_default_leader_actions :: proc(input_map: ^core.InputActions) {
-    core.register_key_action(input_map, .Q, proc(state: ^core.State) {
+    core.register_key_action(input_map, .Q, proc(state: ^core.State, user_data: rawptr) {
         core.reset_input_map(state)
     }, "close this help");
 
-    core.register_key_action(input_map, .R, proc(state: ^core.State) {
+    core.register_key_action(input_map, .R, proc(state: ^core.State, user_data: rawptr) {
-        open(state, make_grep_panel(state))
+        open_grep_panel(state)
     }, "Grep Workspace")
 }
 
 register_default_panel_actions :: proc(input_map: ^core.InputActions) {
-    core.register_key_action(input_map, .H, proc(state: ^core.State) {
+    core.register_key_action(input_map, .H, proc(state: ^core.State, user_data: rawptr) {
         if current_panel, ok := state.current_panel.?; ok {
             if prev, ok := util.get_prev(&state.panels, current_panel).?; ok {
                 state.current_panel = prev
@@ -77,11 +33,7 @@ register_default_panel_actions :: proc(input_map: ^core.InputActions) {
 
         core.reset_input_map(state)
     }, "focus panel to the left");
-    core.register_key_action(input_map, .L, proc(state: ^core.State) {
+    core.register_key_action(input_map, .L, proc(state: ^core.State, user_data: rawptr) {
-        if state.current_buffer < len(state.buffers)-1  {
-            state.current_buffer += 1
-        }
-
         if current_panel, ok := state.current_panel.?; ok {
             if next, ok := util.get_next(&state.panels, current_panel).?; ok {
                 state.current_panel = next
@@ -91,7 +43,7 @@ register_default_panel_actions :: proc(input_map: ^core.InputActions) {
         core.reset_input_map(state)
     }, "focus panel to the right");
 
-    core.register_key_action(input_map, .Q, proc(state: ^core.State) {
+    core.register_key_action(input_map, .Q, proc(state: ^core.State, user_data: rawptr) {
         if current_panel, ok := state.current_panel.?; ok {
             close(state, current_panel) 
         }
@@ -100,9 +52,22 @@ register_default_panel_actions :: proc(input_map: ^core.InputActions) {
 
 
 open :: proc(state: ^core.State, panel: core.Panel, make_active: bool = true) -> (panel_id: int, ok: bool) {
-    if panel_id, ok := util.append_static_list(&state.panels, panel).?; ok && make_active {
+    if panel_id, panel, ok := util.append_static_list(&state.panels, panel); ok && make_active {
+        panel.id = panel_id
         state.current_panel = panel_id
 
+        arena_bytes, err := make([]u8, 1024*1024*8)
+        if err != nil {
+            log.errorf("failed to allocate memory for panel: '%v'", err)
+            util.delete(&state.panels, panel_id)
+            return 
+        }
+
+        mem.arena_init(&panel.arena, arena_bytes)
+        panel.allocator = mem.arena_allocator(&panel.arena)
+
+        panel->create(state)
+
         core.reset_input_map(state)
 
         return panel_id, true
@@ -114,9 +79,11 @@ open :: proc(state: ^core.State, panel: core.Panel, make_active: bool = true) ->
 close :: proc(state: ^core.State, panel_id: int) {
     if panel, ok := util.get(&state.panels, panel_id).?; ok {
         if panel.drop != nil {
-            panel.drop(state, &panel.panel_state)
+            panel->drop(state)
         }
 
+        mem.free(raw_data(panel.arena.data))
+
         util.delete(&state.panels, panel_id)
 
         // TODO: keep track of the last active panel instead of focusing back to the first one
@@ -127,363 +94,3 @@ close :: proc(state: ^core.State, panel_id: int) {
         core.reset_input_map(state)
     }
 }
-
-open_file_buffer_in_new_panel :: proc(state: ^core.State, file_path: string, line, col: int) -> (panel_id, buffer_index: int, ok: bool) {
-    buffer, err := core.new_file_buffer(context.allocator, file_path, state.directory);
-    if err.type != .None {
-        log.error("Failed to create file buffer:", err);
-        return;
-    }
-
-    buffer.history.cursor.line = line
-    buffer.history.cursor.col = col
-    buffer.top_line = buffer.history.cursor.line
-    core.update_file_buffer_index_from_cursor(&buffer)
-
-    buffer_index = len(state.buffers)
-    runtime.append(&state.buffers, buffer);
-
-    if panel_id, ok := open(state, make_file_buffer_panel(buffer_index)); ok {
-        return panel_id, buffer_index, true
-    }
-
-    return -1, -1, false
-}
-
-render_file_buffer :: proc(state: ^core.State, s: ^ui.State, buffer: ^core.FileBuffer) {
-    draw_func := proc(state: ^core.State, e: ui.UI_Element, user_data: rawptr) {
-        buffer := transmute(^core.FileBuffer)user_data;
-        if buffer != nil {
-            buffer.glyphs.width = e.layout.size.x / state.source_font_width;
-            buffer.glyphs.height = e.layout.size.y / state.source_font_height + 1;
-
-            core.draw_file_buffer(state, buffer, e.layout.pos.x, e.layout.pos.y);
-        }
-    };
-
-    relative_file_path, _ := filepath.rel(state.directory, buffer.file_path, context.temp_allocator)
-
-    ui.open_element(s, nil, {
-        dir = .TopToBottom,
-        kind = {ui.Grow{}, ui.Grow{}},
-    })
-    {
-        ui.open_element(s, ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)buffer}, {
-            kind = {ui.Grow{}, ui.Grow{}}
-        })
-        ui.close_element(s)
-
-        ui.open_element(s, nil, {
-            kind = {ui.Grow{}, ui.Exact(state.source_font_height)}
-        })
-        {
-            ui.open_element(s, fmt.tprintf("%s", state.mode), {})
-            ui.close_element(s)
-
-            ui.open_element(s, nil, { kind = {ui.Grow{}, ui.Grow{}}})
-            ui.close_element(s)
-
-            it := core.new_file_buffer_iter_with_cursor(buffer, buffer.history.cursor)
-            ui.open_element(
-                s,
-                fmt.tprintf(
-                    "%v:%v - Slice %v:%v - Char: %v",
-                    buffer.history.cursor.line + 1,
-                    buffer.history.cursor.col + 1,
-                    buffer.history.cursor.index.chunk_index,
-                    buffer.history.cursor.index.char_index,
-                    core.get_character_at_iter(it)
-                ),
-                {}
-            )
-            ui.close_element(s)
-        }
-        ui.close_element(s)
-    }
-    ui.close_element(s)
-}
-
-render_raw_buffer :: proc(state: ^core.State, s: ^ui.State, buffer: ^core.FileBuffer) {
-    draw_func := proc(state: ^core.State, e: ui.UI_Element, user_data: rawptr) {
-        buffer := transmute(^core.FileBuffer)user_data;
-        if buffer != nil {
-            buffer.glyphs.width = e.layout.size.x / state.source_font_width;
-            buffer.glyphs.height = e.layout.size.y / state.source_font_height + 1;
-
-            core.draw_file_buffer(state, buffer, e.layout.pos.x, e.layout.pos.y, false);
-        }
-    };
-
-    ui.open_element(s, ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)buffer}, {
-        kind = {ui.Grow{}, ui.Grow{}}
-    })
-    ui.close_element(s)
-    
-}
-
-render_glyph_buffer :: proc(state: ^core.State, s: ^ui.State, glyphs: ^core.GlyphBuffer) {
-    draw_func := proc(state: ^core.State, e: ui.UI_Element, user_data: rawptr) {
-        glyphs := transmute(^core.GlyphBuffer)user_data;
-        if glyphs != nil {
-            glyphs.width = e.layout.size.x / state.source_font_width;
-            glyphs.height = e.layout.size.y / state.source_font_height + 1;
-
-            core.draw_glyph_buffer(state, glyphs, e.layout.pos.x, e.layout.pos.y, 0, true);
-        }
-    };
-
-    ui.open_element(s, ui.UI_Element_Kind_Custom{fn = draw_func, user_data = transmute(rawptr)glyphs}, {
-        kind = {ui.Grow{}, ui.Grow{}}
-    })
-    ui.close_element(s)
-    
-}
-
-make_file_buffer_panel :: proc(buffer_index: int) -> core.Panel {
-    input_map := core.new_input_map()
-
-    leader_actions := core.new_input_actions()
-    register_default_leader_actions(&leader_actions);
-    core.register_key_action(&input_map.mode[.Normal], .SPACE, leader_actions, "leader commands");
-
-    core.register_ctrl_key_action(&input_map.mode[.Normal], .W, core.new_input_actions(), "Panel Navigation") 
-    register_default_panel_actions(&(&input_map.mode[.Normal].ctrl_key_actions[.W]).action.(core.InputActions))
-
-
-    input.register_default_input_actions(&input_map.mode[.Normal]);
-    input.register_default_visual_actions(&input_map.mode[.Visual]);
-    input.register_default_text_input_actions(&input_map.mode[.Normal]);
-
-    return core.Panel {
-        panel_state = core.FileBufferPanel { buffer_index = buffer_index },
-        input_map = input_map,
-        buffer_proc = proc(state: ^core.State, panel_state: ^core.PanelState) -> (buffer: ^core.FileBuffer, ok: bool) {
-            panel_state := panel_state.(core.FileBufferPanel) or_return;
-
-            return &state.buffers[panel_state.buffer_index], true
-        },
-        render_proc = proc(state: ^core.State, panel_state: ^core.PanelState) -> (ok: bool) {
-            panel_state := panel_state.(core.FileBufferPanel) or_return;
-            s := transmute(^ui.State)state.ui
-            buffer := &state.buffers[panel_state.buffer_index]
-
-            render_file_buffer(state, s, buffer)
-
-            return true
-        }
-    }
-}
-
-make_grep_panel :: proc(state: ^core.State) -> core.Panel {
-    query_arena: mem.Arena
-    mem.arena_init(&query_arena, make([]u8, 1024*1024*2, state.ctx.allocator))
-
-    glyphs := core.make_glyph_buffer(256,256, allocator = mem.arena_allocator(&query_arena))
-
-    input_map := core.new_input_map()
-    grep_input_buffer := core.new_virtual_file_buffer(context.allocator)
-    runtime.append(&state.buffers, grep_input_buffer)
-
-    run_query :: proc(panel_state: ^core.GrepPanel, query: string, directory: string) {
-        if panel_state.query_region.arena != nil {
-            mem.end_arena_temp_memory(panel_state.query_region)
-        }
-        panel_state.query_region = mem.begin_arena_temp_memory(&panel_state.query_arena)
-
-        context.allocator = mem.arena_allocator(&panel_state.query_arena)
-
-        rs_results := grep(
-            strings.clone_to_cstring(query, allocator = context.temp_allocator),
-            strings.clone_to_cstring(directory, allocator = context.temp_allocator)
-        );
-
-        panel_state.query_results = rs_grep_as_results(&rs_results)
-        free_grep_results(rs_results)
-
-        panel_state.selected_result = 0
-        if len(panel_state.query_results) > 0 {
-            core.update_glyph_buffer_from_bytes(
-                &panel_state.glyphs,
-                transmute([]u8)panel_state.query_results[panel_state.selected_result].file_context,
-                panel_state.query_results[panel_state.selected_result].line,
-            )
-        }
-    }
-
-    core.register_key_action(&input_map.mode[.Normal], .ENTER, proc(state: ^core.State) {
-        if current_panel, ok := util.get(&state.panels, state.current_panel.? or_else -1).?; ok {
-            this_panel := state.current_panel.?
-
-            if panel_state, ok := &current_panel.panel_state.(core.GrepPanel); ok {
-                if panel_state.query_results != nil {
-                    selected_result := &panel_state.query_results[panel_state.selected_result]
-
-                    if panel_id, buffer, ok := open_file_buffer_in_new_panel(state, selected_result.file_path, selected_result.line, selected_result.col); ok {
-                        close(state, this_panel)
-
-                        state.current_panel = panel_id
-                        state.current_buffer = buffer
-                    } else {
-                        log.error("failed to open file buffer in new panel")
-                    }
-                }
-            }
-        }
-    }, "Open File");
-    core.register_key_action(&input_map.mode[.Normal], .I, proc(state: ^core.State) {
-        state.mode = .Insert;
-        sdl2.StartTextInput();
-    }, "enter insert mode");
-    core.register_key_action(&input_map.mode[.Normal], .K, proc(state: ^core.State) {
-        // NOTE: this is really jank, should probably update the input
-        // action stuff to allow panels to be passed into these handlers
-        if current_panel, ok := util.get(&state.panels, state.current_panel.? or_else -1).?; ok {
-            if panel_state, ok := &current_panel.panel_state.(core.GrepPanel); ok {
-                // TODO: bounds checking
-                panel_state.selected_result -= 1
-
-                core.update_glyph_buffer_from_bytes(
-                    &panel_state.glyphs,
-                    transmute([]u8)panel_state.query_results[panel_state.selected_result].file_context,
-                    panel_state.query_results[panel_state.selected_result].line,
-                )
-            }
-        }
-    }, "move selection up");
-    core.register_key_action(&input_map.mode[.Normal], .J, proc(state: ^core.State) {
-        // NOTE: this is really jank, should probably update the input
-        // action stuff to allow panels to be passed into these handlers
-        if current_panel, ok := util.get(&state.panels, state.current_panel.? or_else -1).?; ok {
-            if panel_state, ok := &current_panel.panel_state.(core.GrepPanel); ok {
-                // TODO: bounds checking
-                panel_state.selected_result += 1
-
-                core.update_glyph_buffer_from_bytes(
-                    &panel_state.glyphs,
-                    transmute([]u8)panel_state.query_results[panel_state.selected_result].file_context,
-                    panel_state.query_results[panel_state.selected_result].line,
-                )
-            }
-        }
-    }, "move selection down");
-
-    core.register_key_action(&input_map.mode[.Insert], .ESCAPE, proc(state: ^core.State) {
-        state.mode = .Normal;
-        sdl2.StopTextInput();
-    }, "exit insert mode");
-    core.register_key_action(&input_map.mode[.Normal], .ESCAPE, proc(state: ^core.State) {
-        if state.current_panel != nil {
-            close(state, state.current_panel.?)
-        }
-    }, "close panel");
-
-
-    return core.Panel {
-        panel_state = core.GrepPanel {
-            query_arena = query_arena,
-            buffer = len(state.buffers)-1,
-            query_results = nil,
-            glyphs = glyphs,
-        },
-        input_map = input_map,
-        buffer_proc = proc(state: ^core.State, panel_state: ^core.PanelState) -> (buffer: ^core.FileBuffer, ok: bool) {
-            panel_state := panel_state.(core.GrepPanel) or_return;
-
-            return &state.buffers[panel_state.buffer], true
-        },
-        on_buffer_input_proc = proc(state: ^core.State, panel_state: ^core.PanelState) {
-            if panel_state, ok := &panel_state.(core.GrepPanel); ok {
-                buffer := &state.buffers[panel_state.buffer]
-                run_query(panel_state, string(buffer.input_buffer[:]), state.directory)
-            }
-        },
-        drop = proc(state: ^core.State, panel_state: ^core.PanelState) {
-            if panel_state, ok := &panel_state.(core.GrepPanel); ok {
-                delete(panel_state.query_arena.data, state.ctx.allocator)
-            }
-        },
-        render_proc = proc(state: ^core.State, panel_state: ^core.PanelState) -> (ok: bool) {
-            if panel_state, ok := &panel_state.(core.GrepPanel); ok {
-                s := transmute(^ui.State)state.ui
-
-                ui.open_element(s, nil, {
-                    dir = .TopToBottom,
-                    kind = {ui.Grow{}, ui.Grow{}}
-                })
-                {
-                    // query results and file contents side-by-side
-                    ui.open_element(s, nil, {
-                        dir = .LeftToRight,
-                        kind = {ui.Grow{}, ui.Grow{}}
-                    })
-                    {
-                        if panel_state.query_results != nil {
-                            // query results
-                            query_result_container := ui.open_element(s, nil, {
-                                dir = .TopToBottom,
-                                kind = {ui.Grow{}, ui.Grow{}}
-                            })
-                            {
-                                container_height := query_result_container.layout.size.y
-                                max_results := container_height / 16
-
-                                for result, i in panel_state.query_results {
-                                    if i > max_results {
-                                        break
-                                    }
-
-                                    ui.open_element(s, nil, {
-                                        dir = .LeftToRight,
-                                        kind = {ui.Fit{}, ui.Fit{}},
-                                    })
-                                    {
-                                        defer ui.close_element(s)
-
-                                        ui.open_element(s, fmt.tprintf("%v:%v: ", result.line, result.col), {})
-                                        ui.close_element(s)
-
-                                        // TODO: when styling is implemented, make this look better
-                                        if panel_state.selected_result == i {
-                                            ui.open_element(s, fmt.tprintf("%s <--", result.file_path), {})
-                                            ui.close_element(s)
-                                        } else {
-                                            ui.open_element(s, result.file_path, {})
-                                            ui.close_element(s)
-                                        }
-                                    }
-                                }
-                            }
-                            ui.close_element(s)
-
-                            // file contents
-                            selected_result := &panel_state.query_results[panel_state.selected_result]
-
-                            core.update_glyph_buffer_from_bytes(
-                                &panel_state.glyphs,
-                                transmute([]u8)selected_result.file_context,
-                                selected_result.line,
-                            )
-                            render_glyph_buffer(state, s, &panel_state.glyphs)
-                        }
-                    }
-                    ui.close_element(s)
-
-                    // text input
-                    ui.open_element(s, nil, {
-                        kind = {ui.Grow{}, ui.Exact(state.source_font_height)}
-                    })
-                    { 
-                        defer ui.close_element(s)
-
-                        render_raw_buffer(state, s, &state.buffers[panel_state.buffer])
-                    }
-                }
-                ui.close_element(s)
-
-                return true
-            }
-
-            return false
-        }
-    }
-}

src/pkg/grep_lib/src/lib.rs

@@ -139,17 +139,17 @@ impl From<Match> for GrepResult {
             // this won't totally bite me later
             text_len: value.text.len() as u32,
             path_len: value.path.len() as u32,
-            text: Box::leak(value.text.into_boxed_slice()).as_ptr(),
+            text: Box::into_raw(value.text.into_boxed_slice()) as _,
-            path: Box::leak(value.path.into_bytes().into_boxed_slice()).as_ptr(),
+            path: Box::into_raw(value.path.into_bytes().into_boxed_slice()) as _,
         }
     }
 }
 impl From<GrepResult> for Match {
     fn from(value: GrepResult) -> Self {
         unsafe {
-            let text = std::slice::from_raw_parts(value.text, value.text_len as usize).to_vec();
+            let text = Box::from_raw(std::slice::from_raw_parts_mut(value.text as *mut _, value.text_len as usize)).to_vec();
 
-            let path = std::slice::from_raw_parts(value.path, value.path_len as usize).to_vec();
+            let path = Box::from_raw(std::slice::from_raw_parts_mut(value.path as *mut _, value.path_len as usize)).to_vec();
             let path = String::from_utf8_unchecked(path);
 
             Self {
@@ -188,17 +188,26 @@ extern "C" fn grep(
         .into_iter()
         .map(|sink| sink.matches.into_iter())
         .flatten()
-        .map(Into::into)
+        .map(|v| GrepResult::from(v))
         .collect::<Vec<_>>()
         .into_boxed_slice();
 
     let len = boxed.len() as u32;
 
     GrepResults {
-        results: Box::leak(boxed).as_ptr(),
+        results: Box::into_raw(boxed) as _,
         len,
     }
 }
 
 #[unsafe(no_mangle)]
-extern "C" fn free_grep_results(_: GrepResults) { }
+extern "C" fn free_grep_results(results: GrepResults) {
+    unsafe {
+        let mut array = std::slice::from_raw_parts_mut(results.results as *mut GrepResult, results.len as usize);
+        let array = Box::from_raw(array);
+
+        for v in array {
+            let _ = Match::from(v);
+        }
+    }
+}

src/tests/tests.odin

@@ -26,6 +26,10 @@ new_test_editor :: proc() -> core.State {
     return state
 }
 
+delete_editor :: proc(e: ^core.State) {
+    util.delete(&e.panels)
+}
+
 buffer_to_string :: proc(buffer: ^core.FileBuffer) -> string {
     if buffer == nil {
         log.error("nil buffer")
@@ -84,9 +88,7 @@ input_text :: proc(text: string) -> ArtificialTextInput {
 }
 
 setup_empty_buffer :: proc(state: ^core.State) {
-    buffer := core.new_virtual_file_buffer(context.allocator);
+    panels.open(state, panels.make_file_buffer_panel(""))
-    panels.open(state, panels.make_file_buffer_panel(len(state.buffers)))
-    runtime.append(&state.buffers, buffer);
 
     core.reset_input_map(state)
 }
@@ -135,8 +137,12 @@ expect_cursor_index :: proc(t: ^testing.T, cursor: core.Cursor, chunk_index, cha
 insert_from_empty_no_newlines :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!"
     expected_text := fmt.aprintf("%v\n", inputted_text)
@@ -146,6 +152,8 @@ insert_from_empty_no_newlines :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 0, 12)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -153,8 +161,12 @@ insert_from_empty_no_newlines :: proc(t: ^testing.T) {
 insert_from_empty_with_newline :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!\nThis is a new line"
     expected_text := fmt.aprintf("%v\n", inputted_text)
@@ -164,6 +176,8 @@ insert_from_empty_with_newline :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 0, 31)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -171,8 +185,12 @@ insert_from_empty_with_newline :: proc(t: ^testing.T) {
 insert_in_between_text :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!"
     expected_text := "Hello, beautiful world!\n"
@@ -188,6 +206,8 @@ insert_in_between_text :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 1, 9)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -195,8 +215,12 @@ insert_in_between_text :: proc(t: ^testing.T) {
 insert_before_slice_at_beginning_of_file :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!"
     expected_text := "Well, Hello, beautiful world!\n"
@@ -215,6 +239,8 @@ insert_before_slice_at_beginning_of_file :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 0, 5)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -222,8 +248,12 @@ insert_before_slice_at_beginning_of_file :: proc(t: ^testing.T) {
 insert_before_slice :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!"
     expected_text := "Hello, beautiful rich world!\n"
@@ -243,6 +273,8 @@ insert_before_slice :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 2, 4)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -250,8 +282,12 @@ insert_before_slice :: proc(t: ^testing.T) {
 delete_last_content_slice_beginning_of_file :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "Hello, world!")
 
@@ -283,8 +319,12 @@ delete_last_content_slice_beginning_of_file :: proc(t: ^testing.T) {
 delete_in_slice :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!"
     expected_text := "Hello, beautiful h world!\n"
@@ -310,6 +350,8 @@ delete_in_slice :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 3, 0)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -317,8 +359,12 @@ delete_in_slice :: proc(t: ^testing.T) {
 delete_across_slices :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     inputted_text := "Hello, world!"
     expected_text := "Hello, beautiful world!\n"
@@ -352,6 +398,8 @@ delete_across_slices :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 2, 0)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -359,10 +407,14 @@ delete_across_slices :: proc(t: ^testing.T) {
 move_down_next_line_has_shorter_length :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "012345678\n0")
 
@@ -383,10 +435,14 @@ move_down_next_line_has_shorter_length :: proc(t: ^testing.T) {
 move_down_on_last_line :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "012345678")
 
@@ -402,8 +458,12 @@ move_down_on_last_line :: proc(t: ^testing.T) {
 move_left_at_beginning_of_file :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "01234")
     // Move cursor from --------^
@@ -425,10 +485,14 @@ move_left_at_beginning_of_file :: proc(t: ^testing.T) {
 move_right_at_end_of_file :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "01234")
 
@@ -447,10 +511,14 @@ move_right_at_end_of_file :: proc(t: ^testing.T) {
 move_to_end_of_line_from_end :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "01234\n01234")
 
@@ -468,10 +536,14 @@ move_to_end_of_line_from_end :: proc(t: ^testing.T) {
 move_to_end_of_line_from_middle :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "01234\n01234")
 
@@ -492,10 +564,14 @@ move_to_end_of_line_from_middle :: proc(t: ^testing.T) {
 move_to_beginning_of_line_from_middle :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "01234\n01234")
 
@@ -516,10 +592,14 @@ move_to_beginning_of_line_from_middle :: proc(t: ^testing.T) {
 move_to_beginning_of_line_from_start :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
     is_ctrl_pressed := false
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "01234\n01234")
 
@@ -540,8 +620,12 @@ move_to_beginning_of_line_from_start :: proc(t: ^testing.T) {
 append_end_of_line :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     run_text_insertion(&e, "hello")
 
@@ -562,8 +646,12 @@ append_end_of_line :: proc(t: ^testing.T) {
 insert_line_under_current :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     initial_text := "Hello, world!\nThis is a new line"
     run_text_insertion(&e, initial_text)
@@ -590,6 +678,8 @@ insert_line_under_current :: proc(t: ^testing.T) {
     expect_cursor_index(t, buffer.history.cursor, 1, 23)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
@@ -597,8 +687,12 @@ insert_line_under_current :: proc(t: ^testing.T) {
 yank_and_paste_whole_line :: proc(t: ^testing.T) {
     e := new_test_editor()
     setup_empty_buffer(&e)
+    defer {
+        panels.close(&e, 0)
+        delete_editor(&e)
+    }
 
-    buffer := &e.buffers[0]
+    buffer := core.current_buffer(&e)
 
     initial_text := "Hello, world!\nThis is a new line"
     run_text_insertion(&e, initial_text)
@@ -617,22 +711,22 @@ yank_and_paste_whole_line :: proc(t: ^testing.T) {
     expect_line_col(t, buffer.history.cursor, 1, 0)
 
     contents := buffer_to_string(core.current_buffer(&e))
+    defer delete(contents)
+
     testing.expectf(t, contents == expected_text, "got '%v', expected '%v'", contents, expected_text)
 }
 
 run_editor_frame :: proc(state: ^core.State, input: ArtificialInput, is_ctrl_pressed: ^bool) {
-    log.infof("running input: %v", input)
-
     {
         run_key_action := proc(state: ^core.State, control_key_pressed: bool, key: core.Key) -> bool {
-            log.info("key_action")
+            if current_panel, ok := state.current_panel.?; ok {
+                panel := util.get(&state.panels, current_panel).?
 
-            if state.current_input_map != nil {
                 if control_key_pressed {
                     if action, exists := state.current_input_map.ctrl_key_actions[key]; exists {
                         switch value in action.action {
                             case core.EditorAction:
-                                value(state);
+                                value(state, panel);
                                 return true;
                             case core.InputActions:
                                 state.current_input_map = &(&state.current_input_map.ctrl_key_actions[key]).action.(core.InputActions)
@@ -643,7 +737,7 @@ run_editor_frame :: proc(state: ^core.State, input: ArtificialInput, is_ctrl_pre
                     if action, exists := state.current_input_map.key_actions[key]; exists {
                         switch value in action.action {
                             case core.EditorAction:
-                                value(state);
+                                value(state, panel);
                                 return true;
                             case core.InputActions:
                                 state.current_input_map = &(&state.current_input_map.key_actions[key]).action.(core.InputActions)
@@ -651,8 +745,6 @@ run_editor_frame :: proc(state: ^core.State, input: ArtificialInput, is_ctrl_pre
                         }
                     }
                 }
-            } else {
-                log.info("current_input_map is null")
             }
 
             return false
@@ -661,8 +753,6 @@ run_editor_frame :: proc(state: ^core.State, input: ArtificialInput, is_ctrl_pre
         switch state.mode {
             case .Visual: fallthrough
             case .Normal: {
-                log.info("it's normal/visual mode")
-
                 if key, ok := input.(ArtificialKey); ok {
                     if key.is_down {
                         if key.key == .LCTRL {
@@ -678,8 +768,6 @@ run_editor_frame :: proc(state: ^core.State, input: ArtificialInput, is_ctrl_pre
                 }
             }
             case .Insert: {
-                log.info("it's insert mode")
-
                 buffer := core.current_buffer(state);
 
                 if key, ok := input.(ArtificialKey); ok {
@@ -710,24 +798,20 @@ run_editor_frame :: proc(state: ^core.State, input: ArtificialInput, is_ctrl_pre
                     }
                 }
 
-                log.info("before text input")
                 if text_input, ok := input.(ArtificialTextInput); ok {
-                    log.infof("attempting to append '%v' to buffer", text_input)
-
                     for char in text_input.text {
                         if char < 1 {
                             break;
                         }
 
                         if char == '\n' || (char >= 32 && char <= 125 && len(buffer.input_buffer) < 1024-1) {
-                            log.infof("appening '%v' to buffer", char)
                             append(&buffer.input_buffer, u8(char));
                         }
                     }
 
                     if current_panel, ok := state.current_panel.?; ok {
-                        if panel, ok := util.get(&state.panels, current_panel).?; ok && panel.on_buffer_input_proc != nil {
+                        if panel, ok := util.get(&state.panels, current_panel).?; ok && panel.on_buffer_input != nil {
-                            panel.on_buffer_input_proc(state, &panel.panel_state)
+                            panel->on_buffer_input(state)
                         }
                     }
                 }

src/theme/theme.odin

@@ -1,6 +1,7 @@
 package theme
 
 PaletteColor :: enum {
+    None,
     Background,
     Foreground,
 
@@ -33,6 +34,7 @@ PaletteColor :: enum {
 
 // Its the gruvbox dark theme <https://github.com/morhetz/gruvbox>
 palette := []u32 {
+    0x00000000,
     0x282828ff,
     0xebdbb2ff,
 
@@ -65,6 +67,7 @@ palette := []u32 {
 
 
 light_palette := []u32 {
+    0x00000000,
     0xfbf1c7ff,
     0x3c3836ff,
 

src/tree_sitter/ts.odin

@@ -0,0 +1,445 @@
+package tree_sitter
+
+import "base:runtime"
+import "core:strings"
+import "core:fmt"
+import "core:log"
+import "core:os"
+import "core:mem"
+
+import "../theme"
+
+foreign import ts "../../bin/libtree-sitter.a"
+@(default_calling_convention = "c", link_prefix="ts_")
+foreign ts {
+    parser_new :: proc() -> Parser ---
+    parser_delete :: proc(parser: Parser) -> Parser ---
+
+    parser_set_language :: proc(parser: Parser, language: Language) -> bool ---
+    parser_set_logger :: proc(parser: Parser, logger: TSLogger) ---
+    parser_print_dot_graphs :: proc(parser: Parser, fd: int) ---
+
+    parser_parse :: proc(parser: Parser, old_tree: Tree, input: Input) -> Tree ---
+    parser_parse_string :: proc(parser: Parser, old_tree: Tree, source: []u8, len: u32) -> Tree ---
+
+    tree_root_node :: proc(tree: Tree) -> Node ---
+    tree_delete :: proc(tree: Tree) ---
+
+    tree_cursor_new :: proc(node: Node) -> TreeCursor ---
+    tree_cursor_reset :: proc(tree: ^TreeCursor, node: Node) ---
+    tree_cursor_delete :: proc(tree: ^TreeCursor) ---
+    tree_cursor_current_node :: proc(tree: ^TreeCursor) -> Node ---
+    tree_cursor_current_field_name :: proc(tree: ^TreeCursor) -> cstring ---
+
+    tree_cursor_goto_first_child :: proc(self: ^TreeCursor) -> bool ---
+    tree_cursor_goto_first_child_for_point :: proc(self: ^TreeCursor, goal_point: Point) -> u64 ---
+
+    node_start_point :: proc(self: Node) -> Point ---
+    node_end_point :: proc(self: Node) -> Point ---
+    node_type :: proc(self: Node) -> cstring ---
+    node_named_child :: proc(self: Node, child_index: u32) -> Node ---
+    node_child_count :: proc(self: Node) -> u32 ---
+    node_is_null :: proc(self: Node) -> bool ---
+    node_string :: proc(self: Node) -> cstring ---
+
+    query_new :: proc(language: Language, source: ^u8, source_len: u32, error_offset: ^u32, error_type: ^QueryError) -> Query ---
+    query_delete :: proc(query: Query) ---
+    query_cursor_new :: proc() -> QueryCursor ---
+    query_cursor_exec :: proc(cursor: QueryCursor, query: Query, node: Node) ---
+    query_cursor_next_match :: proc(cursor: QueryCursor, match: ^QueryMatch) -> bool ---
+
+    query_capture_name_for_id :: proc(query: Query, index: u32, length: ^u32) -> ^u8 ---
+
+    @(link_name = "ts_set_allocator")
+    ts_set_allocator :: proc(new_malloc: MallocProc, new_calloc: CAllocProc, new_realloc: ReAllocProc, new_free: FreeProc) ---
+}
+
+TS_ALLOCATOR: mem.Allocator
+
+MallocProc :: proc "c" (size: uint) -> rawptr
+CAllocProc :: proc "c" (num: uint, size: uint) -> rawptr
+ReAllocProc :: proc "c" (ptr: rawptr, size: uint) -> rawptr
+FreeProc :: proc "c" (ptr: rawptr)
+
+set_allocator :: proc(allocator := context.allocator) {
+    TS_ALLOCATOR = allocator
+
+    new_malloc :: proc "c" (size: uint) -> rawptr {
+        context = runtime.default_context() 
+
+        data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Alloc, int(size), runtime.DEFAULT_ALIGNMENT, nil, 0)
+        return raw_data(data)
+    }
+
+    new_calloc :: proc "c" (num: uint, size: uint) -> rawptr {
+        context = runtime.default_context() 
+
+        data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Alloc, int(num * size), runtime.DEFAULT_ALIGNMENT, nil, 0)
+        return raw_data(data)
+    }
+
+    new_realloc :: proc "c" (old_ptr: rawptr, size: uint) -> rawptr {
+        context = runtime.default_context() 
+
+        data, _ := TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Resize, int(size), runtime.DEFAULT_ALIGNMENT, old_ptr, 0)
+        return raw_data(data)
+    }
+
+    new_free :: proc "c" (ptr: rawptr) {
+        context = runtime.default_context() 
+
+        TS_ALLOCATOR.procedure(TS_ALLOCATOR.data, .Free, 0, runtime.DEFAULT_ALIGNMENT, ptr, 0)
+    }
+}
+
+foreign import ts_odin "../../bin/libtree-sitter-odin.a"
+foreign ts_odin {
+    tree_sitter_odin :: proc "c" () -> Language ---
+}
+
+foreign import ts_rust "../../bin/libtree-sitter-rust.a"
+foreign ts_rust {
+    tree_sitter_rust :: proc "c" () -> Language ---
+}
+
+foreign import ts_json "../../bin/libtree-sitter-json.a"
+foreign ts_json {
+    tree_sitter_json :: proc "c" () -> Language ---
+}
+
+State :: struct {
+    parser: Parser,
+    language: Language,
+    language_type: LanguageType,
+
+    tree: Tree,
+    cursor: TreeCursor,
+
+    highlights: [dynamic]Highlight,
+}
+
+Highlight :: struct {
+    start: Point,
+    end: Point,
+    color: theme.PaletteColor,
+}
+
+LanguageType :: enum {
+    None,
+    Json,
+    Odin,
+    Rust,
+}
+
+TestStuff :: struct {
+    start: Point,
+    end: Point,
+}
+
+Parser :: distinct rawptr
+Language :: distinct rawptr
+
+Query :: distinct rawptr
+QueryCursor :: distinct rawptr
+
+QueryError :: enum {
+    None = 0,
+    Syntax,
+    NodeType,
+    Field,
+    Capture,
+}
+
+QueryCapture :: struct {
+    node: Node,
+    index: u32,
+}
+
+QueryMatch :: struct {
+    id: u32,
+    pattern_index: u16,
+    capture_count: u16,
+    captures: [^]QueryCapture,
+}
+
+DecodeFunction :: proc "c" (text: []u8, length: u32, code_point: ^u32) -> u32
+Input :: struct {
+    payload: rawptr,
+    read: proc "c" (payload: rawptr, byte_index: u32, position: Point, bytes_read: ^u32) -> ^u8,
+    encoding: InputEncoding,
+    decode: DecodeFunction,
+}
+
+InputEncoding :: enum {
+    UTF8 = 0,
+    UTF16LE,
+    UTF16BE,
+    Custom,
+}
+
+Tree :: distinct rawptr
+
+TreeCursor :: struct {
+    tree: rawptr,
+    id: rawptr,
+    ctx: [3]u32,
+}
+
+Node :: struct {
+    ctx: [4]u32,
+    id: rawptr,
+    tree: Tree,
+}
+
+Point :: struct {
+    row: u32,
+    column: u32
+}
+
+TSLogType :: enum { Parse, Lex }
+TSLogger :: struct {
+    log: proc "c" (payload: rawptr, log_type: TSLogType, msg: cstring),
+    payload: rawptr,
+}
+log_callback :: proc "c" (payload: rawptr, log_type: TSLogType, msg: cstring) {
+    context = runtime.default_context()
+    fmt.printf("Tree-sitter log: %s", msg)
+}
+
+make_state :: proc(type: LanguageType, allocator := context.allocator) -> State {
+    context.allocator = allocator
+
+    parser := parser_new()
+    parser_set_logger(parser, TSLogger{log = log_callback, payload = nil})
+
+    language: Language
+
+    switch (type) {
+        case .None: {}
+        case .Odin: language = tree_sitter_odin()
+        case .Rust: language = tree_sitter_rust()
+        case .Json: language = tree_sitter_json()
+    }
+
+    if language != nil && !parser_set_language(parser, language) {
+        log.errorf("failed to set language to '%v'", type)
+        return State {}
+    }
+
+    return State {
+        parser = parser,
+        language = language,
+        language_type = type,
+    }
+}
+
+delete_state :: proc(state: ^State) {
+    delete(state.highlights)
+    tree_cursor_delete(&state.cursor)
+    tree_delete(state.tree)
+    parser_delete(state.parser)
+}
+
+parse_buffer :: proc(state: ^State, input: Input) {
+    if state.parser == nil || state.language_type == .None {
+        return
+    }
+
+    old_tree := state.tree
+    if old_tree != nil {
+        defer tree_delete(old_tree)
+    }
+
+    state.tree = parser_parse(state.parser, nil, input)
+
+    if state.tree == nil {
+        log.error("failed to parse buffer")
+        return
+    }
+
+    state.cursor = tree_cursor_new(tree_root_node(state.tree))
+
+    if state.language_type != .None {
+        load_highlights(state)
+    }
+}
+
+update_cursor :: proc(state: ^State, line: int, col: int) {
+    if state.tree == nil {
+        return
+    }
+
+    root_node := tree_root_node(state.tree)
+    tree_cursor_reset(&state.cursor, root_node)
+
+    node := tree_cursor_current_node(&state.cursor)
+    for node_child_count(node) > 1 {
+        if tree_cursor_goto_first_child_for_point(&state.cursor, Point {
+            row = u32(line),
+            column = u32(col),
+        }) < 0 {
+            break
+        }
+
+        node = tree_cursor_current_node(&state.cursor)
+    }
+}
+
+load_highlights :: proc(state: ^State) {
+    // TODO: have this be language specific
+    capture_to_color := make(map[string]theme.PaletteColor, allocator = context.temp_allocator)
+
+    switch state.language_type {
+        case .None: {}
+
+        case .Json: {
+            capture_to_color["string"] = .Green
+            capture_to_color["constant.builtin"] = .Yellow
+            capture_to_color["field"] = .Blue
+            capture_to_color["number"] = .Foreground
+        }
+        case .Odin: {
+            capture_to_color["include"] = .Red
+            capture_to_color["keyword.function"] = .Red
+            capture_to_color["keyword.return"] = .Red
+            capture_to_color["storageclass"] = .Red
+
+            capture_to_color["keyword.operator"] = .Purple
+
+            capture_to_color["keyword"] = .Blue
+            capture_to_color["repeat"] = .Blue
+            capture_to_color["conditional"] = .Blue
+            capture_to_color["function"] = .Blue
+
+            capture_to_color["type.decl"] = .BrightBlue
+            capture_to_color["field"] = .BrightYellow
+
+            capture_to_color["type.builtin"] = .Aqua
+
+            capture_to_color["function.call"] = .Green
+            capture_to_color["string"] = .Green
+
+            capture_to_color["comment"] = .Gray
+
+        }
+        case .Rust: {
+            capture_to_color["string"] = .Green
+            capture_to_color["comment"] = .Gray
+
+            capture_to_color["type"] = .BrightBlue
+
+            capture_to_color["type.builtin"] = .Aqua
+
+            capture_to_color["label"] = .Red
+            capture_to_color["variable.builtin"] = .Red
+
+            capture_to_color["keyword"] = .Blue
+            capture_to_color["function"] = .Blue
+            capture_to_color["function.method"] = .Blue
+
+            capture_to_color["property"] = .BrightYellow
+            capture_to_color["constructor"] = .Yellow
+            capture_to_color["attribute"] = .BrightGray
+
+            capture_to_color["constant.builtin"] = .Purple
+        }
+    }
+
+    path: string
+    switch state.language_type {
+        case .None: {}
+
+        case .Json: {
+            path = "../tree-sitter-json/queries/highlights.scm"
+        }
+        case .Odin: {
+            path = "../tree-sitter-odin/queries/highlights.scm"
+        }
+        case .Rust: {
+            path = "../tree-sitter-rust/queries/highlights.scm"
+        }
+    }
+
+    fd, err := os.open(path)
+    if err != nil {
+        log.errorf("failed to open file: errno=%x", err)
+        return
+    }
+    defer os.close(fd);
+
+    if highlight_query, success := os.read_entire_file_from_handle(fd); success {
+        error_offset: u32 = 0
+        error_type: QueryError = .None
+
+        query := query_new(state.language, raw_data(highlight_query), u32(len(highlight_query)), &error_offset, &error_type)
+        defer query_delete(query)
+
+        if error_type != .None {
+            log.errorf("got error: '%v'", error_type)
+            return
+        }
+
+        cursor := query_cursor_new()
+        query_cursor_exec(cursor, query, tree_root_node(state.tree))
+
+        if state.highlights != nil {
+            clear(&state.highlights)
+        } else {
+            state.highlights = make([dynamic]Highlight)
+        }
+
+        match: QueryMatch
+        for query_cursor_next_match(cursor, &match) {
+            for i in 0..<match.capture_count {
+                cap := &match.captures[i]
+                start := node_start_point(cap.node)
+                end := node_end_point(cap.node)
+
+                length: u32
+                name := query_capture_name_for_id(query, cap.index, &length)
+
+                node_type := string(node_type(cap.node))
+                capture_name := strings.string_from_ptr(name, int(length))
+
+                if color, ok := capture_to_color[capture_name]; ok {
+                    append(&state.highlights, Highlight { start = start, end = end, color = color })
+                }
+
+                // if color, ok := capture_to_color[node_type]; ok {
+                //     append(&state.highlights, Highlight { start = start, end = end, color = color })
+                // }
+            }
+        }
+    }
+}
+
+print_node_type :: proc(state: ^State) -> Maybe(string) {
+    if state.tree == nil {
+        return nil
+    }
+
+    current_node := tree_cursor_current_node(&state.cursor)
+    if node_is_null(current_node) {
+        log.error("Current node is null after goto_first_child")
+        return nil
+    }
+
+    node_type_str := node_type(current_node)
+    fmt.println("\n")
+    log.infof("Current node type: %s", node_type_str)
+
+    name := tree_cursor_current_field_name(&state.cursor)
+    if name == nil {
+        log.info("No field name for current node")
+    } else {
+        log.infof("Field name: %s", name)
+    }
+
+    start_point := node_start_point(current_node)
+    end_point := node_end_point(current_node)
+    log.infof("Node position: (%d:%d) to (%d:%d)", 
+        start_point.row+1, start_point.column+1, 
+        end_point.row+1, end_point.column+1)
+
+    return string(node_type_str)
+}

src/ui/ui.odin

@@ -26,6 +26,7 @@ UI_Element :: struct {
 
     kind: UI_Element_Kind,
     layout: UI_Layout,
+    style: UI_Style,
 }
 
 UI_Element_Kind :: union {
@@ -47,6 +48,8 @@ UI_Layout :: struct {
     kind: [2]UI_Size_Kind,
     size: [2]int,
     pos: [2]int,
+
+    floating: bool,
 }
 
 UI_Size_Kind :: union {
@@ -59,6 +62,15 @@ Exact :: distinct i32
 Grow :: struct {}
 Fit :: struct {}
 
+UI_Style :: struct {
+    border: UI_Border_Set,
+
+    border_color: theme.PaletteColor,
+    background_color: theme.PaletteColor,
+}
+UI_Border_Set :: bit_set[UI_Border]
+UI_Border :: enum{Left, Right, Top, Bottom}
+
 UI_Direction :: enum {
     LeftToRight,
     RightToLeft,
@@ -66,13 +78,17 @@ UI_Direction :: enum {
     BottomToTop,
 }
 
-open_element :: proc(state: ^State, kind: UI_Element_Kind, layout: UI_Layout) -> UI_Element {
+open_element :: proc(state: ^State, kind: UI_Element_Kind, layout: UI_Layout, style: UI_Style = {}) -> UI_Element {
     e := UI_Element {
         kind = kind,
         layout = layout,
+        style = style,
+    }
+    
+    if !e.layout.floating {
+        e.layout.pos = state.curr_elements[state.num_curr].layout.pos
+        e.layout.size = state.curr_elements[state.num_curr].layout.size
     }
-    e.layout.pos = state.curr_elements[state.num_curr].layout.pos
-    e.layout.size = state.curr_elements[state.num_curr].layout.size
 
     if parent, ok := state.current_open_element.?; ok {
         e.parent = parent
@@ -108,7 +124,7 @@ close_element :: proc(state: ^State, loc := #caller_location) -> UI_Layout {
                 switch v in e.kind {
                     case UI_Element_Kind_Text: {
                         // FIXME: properly use font size
-                        e.layout.size.x = len(v) * 10
+                        e.layout.size.x = len(v) * 12
                     }
                     case UI_Element_Kind_Image: {
                         // TODO
@@ -119,9 +135,9 @@ close_element :: proc(state: ^State, loc := #caller_location) -> UI_Layout {
 
             case Exact: { e.layout.size.x = int(v) }
             case Fit: {
-                child_index := e.first
+                it := e.first
-                for child_index != nil {
+                for child in iterate_siblings(state, &it) {
-                    child := &state.curr_elements[child_index.?]
+                    if child.layout.floating { continue }
 
                     switch e.layout.dir {
                         case .RightToLeft: fallthrough
@@ -134,8 +150,6 @@ close_element :: proc(state: ^State, loc := #caller_location) -> UI_Layout {
                             e.layout.size.x = math.max(e.layout.size.x, child.layout.size.x)
                         }
                     }
-
-                    child_index = child.next
                 }
             }
             case Grow: {
@@ -162,9 +176,9 @@ close_element :: proc(state: ^State, loc := #caller_location) -> UI_Layout {
 
             case Exact: { e.layout.size.y = int(v) }
             case Fit: {
-                child_index := e.first
+                it := e.first
-                for child_index != nil {
+                for child in iterate_siblings(state, &it) {
-                    child := &state.curr_elements[child_index.?]
+                    if child.layout.floating { continue }
 
                     switch e.layout.dir {
                         case .RightToLeft: fallthrough
@@ -177,11 +191,13 @@ close_element :: proc(state: ^State, loc := #caller_location) -> UI_Layout {
                             e.layout.size.y += child.layout.size.y
                         }
                     }
-
-                    child_index = child.next
                 }
             }
-            case Grow: { /* Done in the Grow pass */ }
+            case Grow: {
+                if _, ok := e.parent.?; !ok {
+                    e.layout.size = state.max_size
+                }
+            }
         }
 
         state.current_open_element = e.parent
@@ -195,18 +211,64 @@ close_element :: proc(state: ^State, loc := #caller_location) -> UI_Layout {
 }
 
 @(private)
-non_fit_parent_size :: proc(state: ^State, index: int, axis: int) -> [2]int {
+iterate_siblings :: proc(state: ^State, sibling: ^Maybe(int)) -> (e: ^UI_Element, index: int, cond: bool) {
+    if sibling == nil || sibling^ == nil {
+        cond = false
+        return
+    }
+
+    e = &state.curr_elements[sibling.?]
+    index = sibling.?
+    cond = true
+
+    sibling^ = e.next
+    
+    return
+}
+
+@(private)
+iterate_siblings_reverse :: proc(state: ^State, sibling: ^Maybe(int)) -> (e: ^UI_Element, index: int, cond: bool) {
+    if sibling == nil || sibling^ == nil {
+        cond = false
+        return
+    }
+
+    e = &state.curr_elements[sibling.?]
+    index = sibling.?
+    cond = true
+
+    sibling^ = e.prev
+    
+    return
+}
+
+@(private)
+non_fit_parent_size :: proc(state: ^State, index: int, axis: int) -> int {
     if _, ok := state.curr_elements[index].layout.kind[axis].(Fit); ok {
-        if parent_index, ok := state.curr_elements[index].parent.?; ok {
+        if parent_index, ok := state.curr_elements[index].parent.?; ok && !state.curr_elements[index].layout.floating {
             return non_fit_parent_size(state, parent_index, axis)
         } else {
-            return state.max_size
+            return state.max_size[axis]
         }
+    } else if state.curr_elements[index].layout.floating {
+        return state.max_size[axis]
     } else {
-        return state.curr_elements[index].layout.size
+        return state.curr_elements[index].layout.size[axis]
     }
 }
 
+@(private)
+prev_non_floating :: proc(state: ^State, index: Maybe(int)) -> Maybe(int) {
+    it := index
+    for sibling, index in iterate_siblings_reverse(state, &it) {
+        if sibling.layout.floating { continue }
+
+        return index
+    }
+
+    return nil
+}
+
 @(private)
 grow_children :: proc(state: ^State, index: int) {
     e := &state.curr_elements[index]
@@ -217,9 +279,14 @@ grow_children :: proc(state: ^State, index: int) {
     children_size: [2]int
     num_growing: [2]int
 
-    child_index := e.first
+    has_floating := false
-    for child_index != nil {
+
-        child := &state.curr_elements[child_index.?]
+    it := e.first
+    for child in iterate_siblings(state, &it) {
+        if child.layout.floating {
+            has_floating = true
+            continue
+        }
 
         if _, ok := child.layout.kind.x.(Grow); ok {
             num_growing.x += 1
@@ -247,37 +314,33 @@ grow_children :: proc(state: ^State, index: int) {
                 }
             }
         }
-
-        child_index = child.next
     }
 
     if num_growing.x > 0 || num_growing.y > 0 {
-        remaining_size := [2]int{ x_e.x, y_e.y } - children_size
+        remaining_size := [2]int{ x_e, y_e } - children_size
         to_grow: [2]int
         to_grow.x = 0 if num_growing.x < 1 else remaining_size.x/num_growing.x
         to_grow.y = 0 if num_growing.y < 1 else remaining_size.y/num_growing.y
 
-        child_index := e.first
+        it := e.first
-        for child_index != nil {
+        for child, child_index in iterate_siblings(state, &it) {
-            child := &state.curr_elements[child_index.?]
-
             switch e.layout.dir {
                 case .RightToLeft: fallthrough
                 case .LeftToRight: {
                     if _, ok := child.layout.kind.x.(Grow); ok {
-                        child.layout.size.x = to_grow.x
+                        child.layout.size.x = state.max_size.x if child.layout.floating else to_grow.x
                     }
                     if _, ok := child.layout.kind.y.(Grow); ok {
-                        child.layout.size.y = remaining_size.y
+                        child.layout.size.y = state.max_size.y if child.layout.floating else y_e
                     }
                 }
                 case .BottomToTop: fallthrough
                 case .TopToBottom: {
                     if _, ok := child.layout.kind.x.(Grow); ok {
-                        child.layout.size.x = remaining_size.x
+                        child.layout.size.x = state.max_size.x if child.layout.floating else x_e
                     }
                     if _, ok := child.layout.kind.y.(Grow); ok {
-                        child.layout.size.y = to_grow.y
+                        child.layout.size.y = state.max_size.y if child.layout.floating else to_grow.y
                     }
                 }
             }
@@ -285,25 +348,23 @@ grow_children :: proc(state: ^State, index: int) {
             _, x_growing := child.layout.kind.x.(Grow)
             _, y_growing := child.layout.kind.y.(Grow)
 
-            if x_growing || y_growing {
+            if x_growing || y_growing || child.layout.floating {
-                grow_children(state, child_index.?)
+                grow_children(state, child_index)
             }
-
-            child_index = child.next
         }
     }
 }
 
-compute_layout_2 :: proc(state: ^State) {
+compute_layout :: proc(state: ^State) {
     grow_children(state, 0)
 
     for i in 0..<state.num_curr {
         e := &state.curr_elements[i]
 
-        if parent_index, ok := e.parent.?; ok {
+        if parent_index, ok := e.parent.?; ok && !e.layout.floating {
             parent := &state.curr_elements[parent_index]
 
-            if prev_index, ok := e.prev.?; ok {
+            if prev_index, ok := prev_non_floating(state, e.prev).?; ok {
                 prev := &state.curr_elements[prev_index]
 
                 switch parent.layout.dir {
@@ -354,24 +415,17 @@ draw :: proc(state: ^State, core_state: ^core.State) {
     for i in 0..<state.num_curr {
         e := &state.curr_elements[i]
 
+        core.draw_rect(
+            core_state,
+            e.layout.pos.x,
+            e.layout.pos.y,
+            e.layout.size.x,
+            e.layout.size.y,
+            e.style.background_color,
+        );
+
         switch v in e.kind {
             case nil: {
-                core.draw_rect(
-                    core_state,
-                    e.layout.pos.x,
-                    e.layout.pos.y,
-                    e.layout.size.x,
-                    e.layout.size.y,
-                    .Background1
-                );
-                core.draw_rect_outline(
-                    core_state,
-                    e.layout.pos.x,
-                    e.layout.pos.y,
-                    e.layout.size.x,
-                    e.layout.size.y,
-                    .Background4
-                );
             }
             case UI_Element_Kind_Text: {
                 core.draw_text(core_state, string(v), e.layout.pos.x, e.layout.pos.y);
@@ -383,6 +437,47 @@ draw :: proc(state: ^State, core_state: ^core.State) {
                 v.fn(core_state, e^, v.user_data) 
             }
         }
+
+        if .Left in e.style.border {
+            core.draw_line(
+                core_state,
+                e.layout.pos.x,
+                e.layout.pos.y,
+                e.layout.pos.x,
+                e.layout.pos.y + e.layout.size.y,
+                e.style.border_color,
+            )
+        }
+        if .Right in e.style.border {
+            core.draw_line(
+                core_state,
+                e.layout.pos.x + e.layout.size.x,
+                e.layout.pos.y,
+                e.layout.pos.x + e.layout.size.x,
+                e.layout.pos.y + e.layout.size.y,
+                e.style.border_color,
+            )
+        }
+        if .Top in e.style.border {
+            core.draw_line(
+                core_state,
+                e.layout.pos.x,
+                e.layout.pos.y,
+                e.layout.pos.x + e.layout.size.x,
+                e.layout.pos.y,
+                e.style.border_color,
+            )
+        }
+        if .Bottom in e.style.border {
+            core.draw_line(
+                core_state,
+                e.layout.pos.x,
+                e.layout.pos.y + e.layout.size.y,
+                e.layout.pos.x + e.layout.size.x,
+                e.layout.pos.y + e.layout.size.y,
+                e.style.border_color,
+            )
+        }
     }
 
     state.num_curr = 0

src/util/list.odin

@@ -11,17 +11,17 @@ StaticListSlot :: struct($T: typeid) {
     data: T,
 }
 
-append_static_list :: proc(list: ^StaticList($T), value: T) -> Maybe(int) {
+append_static_list :: proc(list: ^StaticList($T), value: T) -> (id: int, item: ^T, ok: bool) {
     for i in 0..<len(list.data) {
         if !list.data[i].active {
             list.data[i].active = true 
             list.data[i].data = value
 
-            return i
+            return i, &list.data[i].data, true
         }
     }
 
-    return nil
+    return
 }
 append :: proc{append_static_list}
 
@@ -89,4 +89,8 @@ delete_static_list_elem :: proc(list: ^StaticList($T), index: int) {
     }
 }
 
-delete :: proc{delete_static_list_elem}
+delete_static_list :: proc(list: ^StaticList($T)) {
+    runtime.delete(list.data)
+}
+
+delete :: proc{delete_static_list_elem, delete_static_list}

third_party/tree-sitter/.gitignore

@@ -0,0 +1 @@
+lib/src/*.o

third_party/tree-sitter/Makefile

@@ -0,0 +1,137 @@
+# VERSION := 0.26.0
+# DESCRIPTION := An incremental parsing system for programming tools
+# HOMEPAGE_URL := https://tree-sitter.github.io/tree-sitter/
+
+# install directory layout
+# PREFIX ?= /usr/local
+# INCLUDEDIR ?= $(PREFIX)/include
+# LIBDIR ?= $(PREFIX)/lib
+# BINDIR ?= $(PREFIX)/bin
+# PCLIBDIR ?= $(LIBDIR)/pkgconfig
+
+# collect sources
+# ifneq ($(AMALGAMATED),1)
+# 	SRC := $(wildcard src/*.c)
+# 	# do not double-include amalgamation
+# 	SRC := $(filter-out src/lib.c,$(SRC))
+# else
+# 	# use amalgamated build
+# 	SRC := lib/src/lib.c
+# endif
+
+SRC := lib/src/lib.c
+OBJ := $(SRC:.c=.o)
+
+# define default flags, and override to append mandatory flags
+ARFLAGS := rcs
+CFLAGS ?= -O3 -Wall -Wextra -Wshadow -Wpedantic -Werror=incompatible-pointer-types
+override CFLAGS += -std=c11 -fPIC -fvisibility=hidden
+override CFLAGS += -D_POSIX_C_SOURCE=200112L -D_DEFAULT_SOURCE
+override CFLAGS += -Ilib/src -Ilib/src/wasm -Ilib/include
+
+# ABI versioning
+# SONAME_MAJOR := $(word 1,$(subst ., ,$(VERSION)))
+# SONAME_MINOR := $(word 2,$(subst ., ,$(VERSION)))
+
+# OS-specific bits
+# MACHINE := $(shell $(CC) -dumpmachine)
+
+# ifneq ($(findstring darwin,$(MACHINE)),)
+# 	SOEXT = dylib
+# 	SOEXTVER_MAJOR = $(SONAME_MAJOR).$(SOEXT)
+# 	SOEXTVER = $(SONAME_MAJOR).$(SONAME_MINOR).$(SOEXT)
+# 	LINKSHARED += -dynamiclib -Wl,-install_name,$(LIBDIR)/libtree-sitter.$(SOEXTVER)
+# else ifneq ($(findstring mingw32,$(MACHINE)),)
+# 	SOEXT = dll
+# 	LINKSHARED += -s -shared -Wl,--out-implib,libtree-sitter.dll.a
+# else
+# 	SOEXT = so
+# 	SOEXTVER_MAJOR = $(SOEXT).$(SONAME_MAJOR)
+# 	SOEXTVER = $(SOEXT).$(SONAME_MAJOR).$(SONAME_MINOR)
+# 	LINKSHARED += -shared -Wl,-soname,libtree-sitter.$(SOEXTVER)
+# ifneq ($(filter $(shell uname),FreeBSD NetBSD DragonFly),)
+# 	PCLIBDIR := $(PREFIX)/libdata/pkgconfig
+# endif
+# endif
+
+all: libtree-sitter.a
+
+libtree-sitter.a: $(OBJ)
+	$(AR) $(ARFLAGS) $@ $^
+
+# libtree-sitter.$(SOEXT): $(OBJ)
+# 	$(CC) $(LDFLAGS) $(LINKSHARED) $^ $(LDLIBS) -o $@
+# ifneq ($(STRIP),)
+# 	$(STRIP) $@
+# endif
+
+# ifneq ($(findstring mingw32,$(MACHINE)),)
+# libtree-sitter.dll.a: libtree-sitter.$(SOEXT)
+# endif
+
+# tree-sitter.pc: lib/tree-sitter.pc.in
+# 	sed -e 's|@PROJECT_VERSION@|$(VERSION)|' \
+# 		-e 's|@CMAKE_INSTALL_LIBDIR@|$(LIBDIR:$(PREFIX)/%=%)|' \
+# 		-e 's|@CMAKE_INSTALL_INCLUDEDIR@|$(INCLUDEDIR:$(PREFIX)/%=%)|' \
+# 		-e 's|@PROJECT_DESCRIPTION@|$(DESCRIPTION)|' \
+# 		-e 's|@PROJECT_HOMEPAGE_URL@|$(HOMEPAGE_URL)|' \
+# 		-e 's|@CMAKE_INSTALL_PREFIX@|$(PREFIX)|' $< > $@
+
+clean:
+	$(RM) $(OBJ) tree-sitter.pc libtree-sitter.a libtree-sitter.$(SOEXT) libtree-stitter.dll.a
+
+# install: all
+# 	install -d '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter '$(DESTDIR)$(PCLIBDIR)' '$(DESTDIR)$(LIBDIR)'
+# 	install -m644 lib/include/tree_sitter/api.h '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h
+# 	install -m644 tree-sitter.pc '$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
+# 	install -m644 libtree-sitter.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a
+# ifneq ($(findstring mingw32,$(MACHINE)),)
+# 	install -d '$(DESTDIR)$(BINDIR)'
+# 	install -m755 libtree-sitter.dll '$(DESTDIR)$(BINDIR)'/libtree-sitter.dll
+# 	install -m755 libtree-sitter.dll.a '$(DESTDIR)$(LIBDIR)'/libtree-sitter.dll.a
+# else
+# 	install -m755 libtree-sitter.$(SOEXT) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER)
+# 	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER) libtree-sitter.$(SOEXTVER_MAJOR)
+# 	cd '$(DESTDIR)$(LIBDIR)' && ln -sf libtree-sitter.$(SOEXTVER_MAJOR) libtree-sitter.$(SOEXT)
+# endif
+
+# uninstall:
+# 	$(RM) '$(DESTDIR)$(LIBDIR)'/libtree-sitter.a \
+# 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER) \
+# 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXTVER_MAJOR) \
+# 		'$(DESTDIR)$(LIBDIR)'/libtree-sitter.$(SOEXT) \
+# 		'$(DESTDIR)$(INCLUDEDIR)'/tree_sitter/api.h \
+# 		'$(DESTDIR)$(PCLIBDIR)'/tree-sitter.pc
+# 	rmdir '$(DESTDIR)$(INCLUDEDIR)'/tree_sitter
+
+# .PHONY: all install uninstall clean
+
+
+# ##### Dev targets #####
+
+# test:
+# 	cargo xtask fetch-fixtures
+# 	cargo xtask generate-fixtures
+# 	cargo xtask test
+
+# test-wasm:
+# 	cargo xtask generate-fixtures --wasm
+# 	cargo xtask test-wasm
+
+# lint:
+# 	cargo update --workspace --locked --quiet
+# 	cargo check --workspace --all-targets
+# 	cargo fmt --all --check
+# 	cargo clippy --workspace --all-targets -- -D warnings
+
+# lint-web:
+# 	npm --prefix lib/binding_web ci
+# 	npm --prefix lib/binding_web run lint
+
+# format:
+# 	cargo fmt --all
+
+# changelog:
+# 	@git-cliff --config .github/cliff.toml --prepend CHANGELOG.md --latest --github-token $(shell gh auth token)
+
+# .PHONY: test test-wasm lint format changelog

third_party/tree-sitter/lib/LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2018-2024 Max Brunsfeld
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

third_party/tree-sitter/lib/src/alloc.c

@@ -0,0 +1,48 @@
+#include "alloc.h"
+#include "tree_sitter/api.h"
+#include <stdlib.h>
+
+static void *ts_malloc_default(size_t size) {
+  void *result = malloc(size);
+  if (size > 0 && !result) {
+    fprintf(stderr, "tree-sitter failed to allocate %zu bytes", size);
+    abort();
+  }
+  return result;
+}
+
+static void *ts_calloc_default(size_t count, size_t size) {
+  void *result = calloc(count, size);
+  if (count > 0 && !result) {
+    fprintf(stderr, "tree-sitter failed to allocate %zu bytes", count * size);
+    abort();
+  }
+  return result;
+}
+
+static void *ts_realloc_default(void *buffer, size_t size) {
+  void *result = realloc(buffer, size);
+  if (size > 0 && !result) {
+    fprintf(stderr, "tree-sitter failed to reallocate %zu bytes", size);
+    abort();
+  }
+  return result;
+}
+
+// Allow clients to override allocation functions dynamically
+TS_PUBLIC void *(*ts_current_malloc)(size_t) = ts_malloc_default;
+TS_PUBLIC void *(*ts_current_calloc)(size_t, size_t) = ts_calloc_default;
+TS_PUBLIC void *(*ts_current_realloc)(void *, size_t) = ts_realloc_default;
+TS_PUBLIC void (*ts_current_free)(void *) = free;
+
+void ts_set_allocator(
+  void *(*new_malloc)(size_t size),
+  void *(*new_calloc)(size_t count, size_t size),
+  void *(*new_realloc)(void *ptr, size_t size),
+  void (*new_free)(void *ptr)
+) {
+  ts_current_malloc = new_malloc ? new_malloc : ts_malloc_default;
+  ts_current_calloc = new_calloc ? new_calloc : ts_calloc_default;
+  ts_current_realloc = new_realloc ? new_realloc : ts_realloc_default;
+  ts_current_free = new_free ? new_free : free;
+}

third_party/tree-sitter/lib/src/alloc.h

@@ -0,0 +1,41 @@
+#ifndef TREE_SITTER_ALLOC_H_
+#define TREE_SITTER_ALLOC_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#if defined(TREE_SITTER_HIDDEN_SYMBOLS) || defined(_WIN32)
+#define TS_PUBLIC
+#else
+#define TS_PUBLIC __attribute__((visibility("default")))
+#endif
+
+TS_PUBLIC extern void *(*ts_current_malloc)(size_t size);
+TS_PUBLIC extern void *(*ts_current_calloc)(size_t count, size_t size);
+TS_PUBLIC extern void *(*ts_current_realloc)(void *ptr, size_t size);
+TS_PUBLIC extern void (*ts_current_free)(void *ptr);
+
+// Allow clients to override allocation functions
+#ifndef ts_malloc
+#define ts_malloc  ts_current_malloc
+#endif
+#ifndef ts_calloc
+#define ts_calloc  ts_current_calloc
+#endif
+#ifndef ts_realloc
+#define ts_realloc ts_current_realloc
+#endif
+#ifndef ts_free
+#define ts_free    ts_current_free
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TREE_SITTER_ALLOC_H_

third_party/tree-sitter/lib/src/array.h

@@ -0,0 +1,291 @@
+#ifndef TREE_SITTER_ARRAY_H_
+#define TREE_SITTER_ARRAY_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./alloc.h"
+#include "./ts_assert.h"
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+
+#ifdef _MSC_VER
+#pragma warning(push)
+#pragma warning(disable : 4101)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic push
+#pragma GCC diagnostic ignored "-Wunused-variable"
+#endif
+
+#define Array(T)       \
+  struct {             \
+    T *contents;       \
+    uint32_t size;     \
+    uint32_t capacity; \
+  }
+
+/// Initialize an array.
+#define array_init(self) \
+  ((self)->size = 0, (self)->capacity = 0, (self)->contents = NULL)
+
+/// Create an empty array.
+#define array_new() \
+  { NULL, 0, 0 }
+
+/// Get a pointer to the element at a given `index` in the array.
+#define array_get(self, _index) \
+  (ts_assert((uint32_t)(_index) < (self)->size), &(self)->contents[_index])
+
+/// Get a pointer to the first element in the array.
+#define array_front(self) array_get(self, 0)
+
+/// Get a pointer to the last element in the array.
+#define array_back(self) array_get(self, (self)->size - 1)
+
+/// Clear the array, setting its size to zero. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_clear(self) ((self)->size = 0)
+
+/// Reserve `new_capacity` elements of space in the array. If `new_capacity` is
+/// less than the array's current capacity, this function has no effect.
+#define array_reserve(self, new_capacity) \
+  _array__reserve((Array *)(self), array_elem_size(self), new_capacity)
+
+/// Free any memory allocated for this array. Note that this does not free any
+/// memory allocated for the array's contents.
+#define array_delete(self) _array__delete((Array *)(self))
+
+/// Push a new `element` onto the end of the array.
+#define array_push(self, element)                            \
+  (_array__grow((Array *)(self), 1, array_elem_size(self)), \
+   (self)->contents[(self)->size++] = (element))
+
+/// Increase the array's size by `count` elements.
+/// New elements are zero-initialized.
+#define array_grow_by(self, count) \
+  do { \
+    if ((count) == 0) break; \
+    _array__grow((Array *)(self), count, array_elem_size(self)); \
+    memset((self)->contents + (self)->size, 0, (count) * array_elem_size(self)); \
+    (self)->size += (count); \
+  } while (0)
+
+/// Append all elements from one array to the end of another.
+#define array_push_all(self, other)                                       \
+  array_extend((self), (other)->size, (other)->contents)
+
+/// Append `count` elements to the end of the array, reading their values from the
+/// `contents` pointer.
+#define array_extend(self, count, contents)                    \
+  _array__splice(                                               \
+    (Array *)(self), array_elem_size(self), (self)->size, \
+    0, count,  contents                                        \
+  )
+
+/// Remove `old_count` elements from the array starting at the given `index`. At
+/// the same index, insert `new_count` new elements, reading their values from the
+/// `new_contents` pointer.
+#define array_splice(self, _index, old_count, new_count, new_contents)  \
+  _array__splice(                                                       \
+    (Array *)(self), array_elem_size(self), _index,                \
+    old_count, new_count, new_contents                                 \
+  )
+
+/// Insert one `element` into the array at the given `index`.
+#define array_insert(self, _index, element) \
+  _array__splice((Array *)(self), array_elem_size(self), _index, 0, 1, &(element))
+
+/// Remove one element from the array at the given `index`.
+#define array_erase(self, _index) \
+  _array__erase((Array *)(self), array_elem_size(self), _index)
+
+/// Pop the last element off the array, returning the element by value.
+#define array_pop(self) ((self)->contents[--(self)->size])
+
+/// Assign the contents of one array to another, reallocating if necessary.
+#define array_assign(self, other) \
+  _array__assign((Array *)(self), (const Array *)(other), array_elem_size(self))
+
+/// Swap one array with another
+#define array_swap(self, other) \
+  _array__swap((Array *)(self), (Array *)(other))
+
+/// Get the size of the array contents
+#define array_elem_size(self) (sizeof *(self)->contents)
+
+/// Search a sorted array for a given `needle` value, using the given `compare`
+/// callback to determine the order.
+///
+/// If an existing element is found to be equal to `needle`, then the `index`
+/// out-parameter is set to the existing value's index, and the `exists`
+/// out-parameter is set to true. Otherwise, `index` is set to an index where
+/// `needle` should be inserted in order to preserve the sorting, and `exists`
+/// is set to false.
+#define array_search_sorted_with(self, compare, needle, _index, _exists) \
+  _array__search_sorted(self, 0, compare, , needle, _index, _exists)
+
+/// Search a sorted array for a given `needle` value, using integer comparisons
+/// of a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_with`.
+#define array_search_sorted_by(self, field, needle, _index, _exists) \
+  _array__search_sorted(self, 0, _compare_int, field, needle, _index, _exists)
+
+/// Insert a given `value` into a sorted array, using the given `compare`
+/// callback to determine the order.
+#define array_insert_sorted_with(self, compare, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_with(self, compare, &(value), &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+/// Insert a given `value` into a sorted array, using integer comparisons of
+/// a given struct field (specified with a leading dot) to determine the order.
+///
+/// See also `array_search_sorted_by`.
+#define array_insert_sorted_by(self, field, value) \
+  do { \
+    unsigned _index, _exists; \
+    array_search_sorted_by(self, field, (value) field, &_index, &_exists); \
+    if (!_exists) array_insert(self, _index, value); \
+  } while (0)
+
+// Private
+
+typedef Array(void) Array;
+
+/// This is not what you're looking for, see `array_delete`.
+static inline void _array__delete(Array *self) {
+  if (self->contents) {
+    ts_free(self->contents);
+    self->contents = NULL;
+    self->size = 0;
+    self->capacity = 0;
+  }
+}
+
+/// This is not what you're looking for, see `array_erase`.
+static inline void _array__erase(Array *self, size_t element_size,
+                                uint32_t index) {
+  ts_assert(index < self->size);
+  char *contents = (char *)self->contents;
+  memmove(contents + index * element_size, contents + (index + 1) * element_size,
+          (self->size - index - 1) * element_size);
+  self->size--;
+}
+
+/// This is not what you're looking for, see `array_reserve`.
+static inline void _array__reserve(Array *self, size_t element_size, uint32_t new_capacity) {
+  if (new_capacity > self->capacity) {
+    if (self->contents) {
+      self->contents = ts_realloc(self->contents, new_capacity * element_size);
+    } else {
+      self->contents = ts_malloc(new_capacity * element_size);
+    }
+    self->capacity = new_capacity;
+  }
+}
+
+/// This is not what you're looking for, see `array_assign`.
+static inline void _array__assign(Array *self, const Array *other, size_t element_size) {
+  _array__reserve(self, element_size, other->size);
+  self->size = other->size;
+  memcpy(self->contents, other->contents, self->size * element_size);
+}
+
+/// This is not what you're looking for, see `array_swap`.
+static inline void _array__swap(Array *self, Array *other) {
+  Array swap = *other;
+  *other = *self;
+  *self = swap;
+}
+
+/// This is not what you're looking for, see `array_push` or `array_grow_by`.
+static inline void _array__grow(Array *self, uint32_t count, size_t element_size) {
+  uint32_t new_size = self->size + count;
+  if (new_size > self->capacity) {
+    uint32_t new_capacity = self->capacity * 2;
+    if (new_capacity < 8) new_capacity = 8;
+    if (new_capacity < new_size) new_capacity = new_size;
+    _array__reserve(self, element_size, new_capacity);
+  }
+}
+
+/// This is not what you're looking for, see `array_splice`.
+static inline void _array__splice(Array *self, size_t element_size,
+                                 uint32_t index, uint32_t old_count,
+                                 uint32_t new_count, const void *elements) {
+  uint32_t new_size = self->size + new_count - old_count;
+  uint32_t old_end = index + old_count;
+  uint32_t new_end = index + new_count;
+  ts_assert(old_end <= self->size);
+
+  _array__reserve(self, element_size, new_size);
+
+  char *contents = (char *)self->contents;
+  if (self->size > old_end) {
+    memmove(
+      contents + new_end * element_size,
+      contents + old_end * element_size,
+      (self->size - old_end) * element_size
+    );
+  }
+  if (new_count > 0) {
+    if (elements) {
+      memcpy(
+        (contents + index * element_size),
+        elements,
+        new_count * element_size
+      );
+    } else {
+      memset(
+        (contents + index * element_size),
+        0,
+        new_count * element_size
+      );
+    }
+  }
+  self->size += new_count - old_count;
+}
+
+/// A binary search routine, based on Rust's `std::slice::binary_search_by`.
+/// This is not what you're looking for, see `array_search_sorted_with` or `array_search_sorted_by`.
+#define _array__search_sorted(self, start, compare, suffix, needle, _index, _exists) \
+  do { \
+    *(_index) = start; \
+    *(_exists) = false; \
+    uint32_t size = (self)->size - *(_index); \
+    if (size == 0) break; \
+    int comparison; \
+    while (size > 1) { \
+      uint32_t half_size = size / 2; \
+      uint32_t mid_index = *(_index) + half_size; \
+      comparison = compare(&((self)->contents[mid_index] suffix), (needle)); \
+      if (comparison <= 0) *(_index) = mid_index; \
+      size -= half_size; \
+    } \
+    comparison = compare(&((self)->contents[*(_index)] suffix), (needle)); \
+    if (comparison == 0) *(_exists) = true; \
+    else if (comparison < 0) *(_index) += 1; \
+  } while (0)
+
+/// Helper macro for the `_sorted_by` routines below. This takes the left (existing)
+/// parameter by reference in order to work with the generic sorting function above.
+#define _compare_int(a, b) ((int)*(a) - (int)(b))
+
+#ifdef _MSC_VER
+#pragma warning(pop)
+#elif defined(__GNUC__) || defined(__clang__)
+#pragma GCC diagnostic pop
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_ARRAY_H_

third_party/tree-sitter/lib/src/atomic.h

@@ -0,0 +1,68 @@
+#ifndef TREE_SITTER_ATOMIC_H_
+#define TREE_SITTER_ATOMIC_H_
+
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#ifdef __TINYC__
+
+static inline size_t atomic_load(const volatile size_t *p) {
+  return *p;
+}
+
+static inline uint32_t atomic_inc(volatile uint32_t *p) {
+  *p += 1;
+  return *p;
+}
+
+static inline uint32_t atomic_dec(volatile uint32_t *p) {
+  *p-= 1;
+  return *p;
+}
+
+#elif defined(_WIN32)
+
+#include <windows.h>
+
+static inline size_t atomic_load(const volatile size_t *p) {
+  return *p;
+}
+
+static inline uint32_t atomic_inc(volatile uint32_t *p) {
+  return InterlockedIncrement((long volatile *)p);
+}
+
+static inline uint32_t atomic_dec(volatile uint32_t *p) {
+  return InterlockedDecrement((long volatile *)p);
+}
+
+#else
+
+static inline size_t atomic_load(const volatile size_t *p) {
+#ifdef __ATOMIC_RELAXED
+  return __atomic_load_n(p, __ATOMIC_RELAXED);
+#else
+  return __sync_fetch_and_add((volatile size_t *)p, 0);
+#endif
+}
+
+static inline uint32_t atomic_inc(volatile uint32_t *p) {
+  #ifdef __ATOMIC_RELAXED
+    return __atomic_add_fetch(p, 1U, __ATOMIC_SEQ_CST);
+  #else
+    return __sync_add_and_fetch(p, 1U);
+  #endif
+}
+
+static inline uint32_t atomic_dec(volatile uint32_t *p) {
+  #ifdef __ATOMIC_RELAXED
+    return __atomic_sub_fetch(p, 1U, __ATOMIC_SEQ_CST);
+  #else
+    return __sync_sub_and_fetch(p, 1U);
+  #endif
+}
+
+#endif
+
+#endif  // TREE_SITTER_ATOMIC_H_

third_party/tree-sitter/lib/src/clock.h

@@ -0,0 +1,146 @@
+#ifndef TREE_SITTER_CLOCK_H_
+#define TREE_SITTER_CLOCK_H_
+
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef uint64_t TSDuration;
+
+#ifdef _WIN32
+
+// Windows:
+// * Represent a time as a performance counter value.
+// * Represent a duration as a number of performance counter ticks.
+
+#include <windows.h>
+typedef uint64_t TSClock;
+
+static inline TSDuration duration_from_micros(uint64_t micros) {
+  LARGE_INTEGER frequency;
+  QueryPerformanceFrequency(&frequency);
+  return micros * (uint64_t)frequency.QuadPart / 1000000;
+}
+
+static inline uint64_t duration_to_micros(TSDuration self) {
+  LARGE_INTEGER frequency;
+  QueryPerformanceFrequency(&frequency);
+  return self * 1000000 / (uint64_t)frequency.QuadPart;
+}
+
+static inline TSClock clock_null(void) {
+  return 0;
+}
+
+static inline TSClock clock_now(void) {
+  LARGE_INTEGER result;
+  QueryPerformanceCounter(&result);
+  return (uint64_t)result.QuadPart;
+}
+
+static inline TSClock clock_after(TSClock base, TSDuration duration) {
+  return base + duration;
+}
+
+static inline bool clock_is_null(TSClock self) {
+  return !self;
+}
+
+static inline bool clock_is_gt(TSClock self, TSClock other) {
+  return self > other;
+}
+
+#elif defined(CLOCK_MONOTONIC)
+
+// POSIX with monotonic clock support (Linux, macOS)
+// * Represent a time as a monotonic (seconds, nanoseconds) pair.
+// * Represent a duration as a number of microseconds.
+//
+// On these platforms, parse timeouts will correspond accurately to
+// real time, regardless of what other processes are running.
+
+#include <time.h>
+typedef struct timespec TSClock;
+
+static inline TSDuration duration_from_micros(uint64_t micros) {
+  return micros;
+}
+
+static inline uint64_t duration_to_micros(TSDuration self) {
+  return self;
+}
+
+static inline TSClock clock_now(void) {
+  TSClock result;
+  clock_gettime(CLOCK_MONOTONIC, &result);
+  return result;
+}
+
+static inline TSClock clock_null(void) {
+  return (TSClock) {0, 0};
+}
+
+static inline TSClock clock_after(TSClock base, TSDuration duration) {
+  TSClock result = base;
+  result.tv_sec += duration / 1000000;
+  result.tv_nsec += (duration % 1000000) * 1000;
+  if (result.tv_nsec >= 1000000000) {
+    result.tv_nsec -= 1000000000;
+    ++(result.tv_sec);
+  }
+  return result;
+}
+
+static inline bool clock_is_null(TSClock self) {
+  return !self.tv_sec && !self.tv_nsec;
+}
+
+static inline bool clock_is_gt(TSClock self, TSClock other) {
+  if (self.tv_sec > other.tv_sec) return true;
+  if (self.tv_sec < other.tv_sec) return false;
+  return self.tv_nsec > other.tv_nsec;
+}
+
+#else
+
+// POSIX without monotonic clock support
+// * Represent a time as a process clock value.
+// * Represent a duration as a number of process clock ticks.
+//
+// On these platforms, parse timeouts may be affected by other processes,
+// which is not ideal, but is better than using a non-monotonic time API
+// like `gettimeofday`.
+
+#include <time.h>
+typedef uint64_t TSClock;
+
+static inline TSDuration duration_from_micros(uint64_t micros) {
+  return micros * (uint64_t)CLOCKS_PER_SEC / 1000000;
+}
+
+static inline uint64_t duration_to_micros(TSDuration self) {
+  return self * 1000000 / (uint64_t)CLOCKS_PER_SEC;
+}
+
+static inline TSClock clock_null(void) {
+  return 0;
+}
+
+static inline TSClock clock_now(void) {
+  return (uint64_t)clock();
+}
+
+static inline TSClock clock_after(TSClock base, TSDuration duration) {
+  return base + duration;
+}
+
+static inline bool clock_is_null(TSClock self) {
+  return !self;
+}
+
+static inline bool clock_is_gt(TSClock self, TSClock other) {
+  return self > other;
+}
+
+#endif
+
+#endif  // TREE_SITTER_CLOCK_H_

third_party/tree-sitter/lib/src/error_costs.h

@@ -0,0 +1,11 @@
+#ifndef TREE_SITTER_ERROR_COSTS_H_
+#define TREE_SITTER_ERROR_COSTS_H_
+
+#define ERROR_STATE 0
+#define ERROR_COST_PER_RECOVERY 500
+#define ERROR_COST_PER_MISSING_TREE 110
+#define ERROR_COST_PER_SKIPPED_TREE 100
+#define ERROR_COST_PER_SKIPPED_LINE 30
+#define ERROR_COST_PER_SKIPPED_CHAR 1
+
+#endif

third_party/tree-sitter/lib/src/get_changed_ranges.c

@@ -0,0 +1,523 @@
+#include "./get_changed_ranges.h"
+#include "./subtree.h"
+#include "./language.h"
+#include "./error_costs.h"
+#include "./tree_cursor.h"
+#include "./ts_assert.h"
+
+// #define DEBUG_GET_CHANGED_RANGES
+
+static void ts_range_array_add(
+  TSRangeArray *self,
+  Length start,
+  Length end
+) {
+  if (self->size > 0) {
+    TSRange *last_range = array_back(self);
+    if (start.bytes <= last_range->end_byte) {
+      last_range->end_byte = end.bytes;
+      last_range->end_point = end.extent;
+      return;
+    }
+  }
+
+  if (start.bytes < end.bytes) {
+    TSRange range = { start.extent, end.extent, start.bytes, end.bytes };
+    array_push(self, range);
+  }
+}
+
+bool ts_range_array_intersects(
+  const TSRangeArray *self,
+  unsigned start_index,
+  uint32_t start_byte,
+  uint32_t end_byte
+) {
+  for (unsigned i = start_index; i < self->size; i++) {
+    TSRange *range = array_get(self, i);
+    if (range->end_byte > start_byte) {
+      if (range->start_byte >= end_byte) break;
+      return true;
+    }
+  }
+  return false;
+}
+
+void ts_range_array_get_changed_ranges(
+  const TSRange *old_ranges, unsigned old_range_count,
+  const TSRange *new_ranges, unsigned new_range_count,
+  TSRangeArray *differences
+) {
+  unsigned new_index = 0;
+  unsigned old_index = 0;
+  Length current_position = length_zero();
+  bool in_old_range = false;
+  bool in_new_range = false;
+
+  while (old_index < old_range_count || new_index < new_range_count) {
+    const TSRange *old_range = &old_ranges[old_index];
+    const TSRange *new_range = &new_ranges[new_index];
+
+    Length next_old_position;
+    if (in_old_range) {
+      next_old_position = (Length) {old_range->end_byte, old_range->end_point};
+    } else if (old_index < old_range_count) {
+      next_old_position = (Length) {old_range->start_byte, old_range->start_point};
+    } else {
+      next_old_position = LENGTH_MAX;
+    }
+
+    Length next_new_position;
+    if (in_new_range) {
+      next_new_position = (Length) {new_range->end_byte, new_range->end_point};
+    } else if (new_index < new_range_count) {
+      next_new_position = (Length) {new_range->start_byte, new_range->start_point};
+    } else {
+      next_new_position = LENGTH_MAX;
+    }
+
+    if (next_old_position.bytes < next_new_position.bytes) {
+      if (in_old_range != in_new_range) {
+        ts_range_array_add(differences, current_position, next_old_position);
+      }
+      if (in_old_range) old_index++;
+      current_position = next_old_position;
+      in_old_range = !in_old_range;
+    } else if (next_new_position.bytes < next_old_position.bytes) {
+      if (in_old_range != in_new_range) {
+        ts_range_array_add(differences, current_position, next_new_position);
+      }
+      if (in_new_range) new_index++;
+      current_position = next_new_position;
+      in_new_range = !in_new_range;
+    } else {
+      if (in_old_range != in_new_range) {
+        ts_range_array_add(differences, current_position, next_new_position);
+      }
+      if (in_old_range) old_index++;
+      if (in_new_range) new_index++;
+      in_old_range = !in_old_range;
+      in_new_range = !in_new_range;
+      current_position = next_new_position;
+    }
+  }
+}
+
+typedef struct {
+  TreeCursor cursor;
+  const TSLanguage *language;
+  unsigned visible_depth;
+  bool in_padding;
+  Subtree prev_external_token;
+} Iterator;
+
+static Iterator iterator_new(
+  TreeCursor *cursor,
+  const Subtree *tree,
+  const TSLanguage *language
+) {
+  array_clear(&cursor->stack);
+  array_push(&cursor->stack, ((TreeCursorEntry) {
+    .subtree = tree,
+    .position = length_zero(),
+    .child_index = 0,
+    .structural_child_index = 0,
+  }));
+  return (Iterator) {
+    .cursor = *cursor,
+    .language = language,
+    .visible_depth = 1,
+    .in_padding = false,
+    .prev_external_token = NULL_SUBTREE,
+  };
+}
+
+static bool iterator_done(Iterator *self) {
+  return self->cursor.stack.size == 0;
+}
+
+static Length iterator_start_position(Iterator *self) {
+  TreeCursorEntry entry = *array_back(&self->cursor.stack);
+  if (self->in_padding) {
+    return entry.position;
+  } else {
+    return length_add(entry.position, ts_subtree_padding(*entry.subtree));
+  }
+}
+
+static Length iterator_end_position(Iterator *self) {
+  TreeCursorEntry entry = *array_back(&self->cursor.stack);
+  Length result = length_add(entry.position, ts_subtree_padding(*entry.subtree));
+  if (self->in_padding) {
+    return result;
+  } else {
+    return length_add(result, ts_subtree_size(*entry.subtree));
+  }
+}
+
+static bool iterator_tree_is_visible(const Iterator *self) {
+  TreeCursorEntry entry = *array_back(&self->cursor.stack);
+  if (ts_subtree_visible(*entry.subtree)) return true;
+  if (self->cursor.stack.size > 1) {
+    Subtree parent = *array_get(&self->cursor.stack, self->cursor.stack.size - 2)->subtree;
+    return ts_language_alias_at(
+      self->language,
+      parent.ptr->production_id,
+      entry.structural_child_index
+    ) != 0;
+  }
+  return false;
+}
+
+static void iterator_get_visible_state(
+  const Iterator *self,
+  Subtree *tree,
+  TSSymbol *alias_symbol,
+  uint32_t *start_byte
+) {
+  uint32_t i = self->cursor.stack.size - 1;
+
+  if (self->in_padding) {
+    if (i == 0) return;
+    i--;
+  }
+
+  for (; i + 1 > 0; i--) {
+    TreeCursorEntry entry = *array_get(&self->cursor.stack, i);
+
+    if (i > 0) {
+      const Subtree *parent = array_get(&self->cursor.stack, i - 1)->subtree;
+      *alias_symbol = ts_language_alias_at(
+        self->language,
+        parent->ptr->production_id,
+        entry.structural_child_index
+      );
+    }
+
+    if (ts_subtree_visible(*entry.subtree) || *alias_symbol) {
+      *tree = *entry.subtree;
+      *start_byte = entry.position.bytes;
+      break;
+    }
+  }
+}
+
+static void iterator_ascend(Iterator *self) {
+  if (iterator_done(self)) return;
+  if (iterator_tree_is_visible(self) && !self->in_padding) self->visible_depth--;
+  if (array_back(&self->cursor.stack)->child_index > 0) self->in_padding = false;
+  self->cursor.stack.size--;
+}
+
+static bool iterator_descend(Iterator *self, uint32_t goal_position) {
+  if (self->in_padding) return false;
+
+  bool did_descend = false;
+  do {
+    did_descend = false;
+    TreeCursorEntry entry = *array_back(&self->cursor.stack);
+    Length position = entry.position;
+    uint32_t structural_child_index = 0;
+    for (uint32_t i = 0, n = ts_subtree_child_count(*entry.subtree); i < n; i++) {
+      const Subtree *child = &ts_subtree_children(*entry.subtree)[i];
+      Length child_left = length_add(position, ts_subtree_padding(*child));
+      Length child_right = length_add(child_left, ts_subtree_size(*child));
+
+      if (child_right.bytes > goal_position) {
+        array_push(&self->cursor.stack, ((TreeCursorEntry) {
+          .subtree = child,
+          .position = position,
+          .child_index = i,
+          .structural_child_index = structural_child_index,
+        }));
+
+        if (iterator_tree_is_visible(self)) {
+          if (child_left.bytes > goal_position) {
+            self->in_padding = true;
+          } else {
+            self->visible_depth++;
+          }
+          return true;
+        }
+
+        did_descend = true;
+        break;
+      }
+
+      position = child_right;
+      if (!ts_subtree_extra(*child)) structural_child_index++;
+      Subtree last_external_token = ts_subtree_last_external_token(*child);
+      if (last_external_token.ptr) {
+        self->prev_external_token = last_external_token;
+      }
+    }
+  } while (did_descend);
+
+  return false;
+}
+
+static void iterator_advance(Iterator *self) {
+  if (self->in_padding) {
+    self->in_padding = false;
+    if (iterator_tree_is_visible(self)) {
+      self->visible_depth++;
+    } else {
+      iterator_descend(self, 0);
+    }
+    return;
+  }
+
+  for (;;) {
+    if (iterator_tree_is_visible(self)) self->visible_depth--;
+    TreeCursorEntry entry = array_pop(&self->cursor.stack);
+    if (iterator_done(self)) return;
+
+    const Subtree *parent = array_back(&self->cursor.stack)->subtree;
+    uint32_t child_index = entry.child_index + 1;
+    Subtree last_external_token = ts_subtree_last_external_token(*entry.subtree);
+    if (last_external_token.ptr) {
+      self->prev_external_token = last_external_token;
+    }
+    if (ts_subtree_child_count(*parent) > child_index) {
+      Length position = length_add(entry.position, ts_subtree_total_size(*entry.subtree));
+      uint32_t structural_child_index = entry.structural_child_index;
+      if (!ts_subtree_extra(*entry.subtree)) structural_child_index++;
+      const Subtree *next_child = &ts_subtree_children(*parent)[child_index];
+
+      array_push(&self->cursor.stack, ((TreeCursorEntry) {
+        .subtree = next_child,
+        .position = position,
+        .child_index = child_index,
+        .structural_child_index = structural_child_index,
+      }));
+
+      if (iterator_tree_is_visible(self)) {
+        if (ts_subtree_padding(*next_child).bytes > 0) {
+          self->in_padding = true;
+        } else {
+          self->visible_depth++;
+        }
+      } else {
+        iterator_descend(self, 0);
+      }
+      break;
+    }
+  }
+}
+
+typedef enum {
+  IteratorDiffers,
+  IteratorMayDiffer,
+  IteratorMatches,
+} IteratorComparison;
+
+static IteratorComparison iterator_compare(
+  const Iterator *old_iter,
+  const Iterator *new_iter
+) {
+  Subtree old_tree = NULL_SUBTREE;
+  Subtree new_tree = NULL_SUBTREE;
+  uint32_t old_start = 0;
+  uint32_t new_start = 0;
+  TSSymbol old_alias_symbol = 0;
+  TSSymbol new_alias_symbol = 0;
+  iterator_get_visible_state(old_iter, &old_tree, &old_alias_symbol, &old_start);
+  iterator_get_visible_state(new_iter, &new_tree, &new_alias_symbol, &new_start);
+  TSSymbol old_symbol = ts_subtree_symbol(old_tree);
+  TSSymbol new_symbol = ts_subtree_symbol(new_tree);
+
+  if (!old_tree.ptr && !new_tree.ptr) return IteratorMatches;
+  if (!old_tree.ptr || !new_tree.ptr) return IteratorDiffers;
+  if (old_alias_symbol != new_alias_symbol || old_symbol != new_symbol) return IteratorDiffers;
+
+  uint32_t old_size = ts_subtree_size(old_tree).bytes;
+  uint32_t new_size = ts_subtree_size(new_tree).bytes;
+  TSStateId old_state = ts_subtree_parse_state(old_tree);
+  TSStateId new_state = ts_subtree_parse_state(new_tree);
+  bool old_has_external_tokens = ts_subtree_has_external_tokens(old_tree);
+  bool new_has_external_tokens = ts_subtree_has_external_tokens(new_tree);
+  uint32_t old_error_cost = ts_subtree_error_cost(old_tree);
+  uint32_t new_error_cost = ts_subtree_error_cost(new_tree);
+
+  if (
+    old_start != new_start ||
+    old_symbol == ts_builtin_sym_error ||
+    old_size != new_size ||
+    old_state == TS_TREE_STATE_NONE ||
+    new_state == TS_TREE_STATE_NONE ||
+    ((old_state == ERROR_STATE) != (new_state == ERROR_STATE)) ||
+    old_error_cost != new_error_cost ||
+    old_has_external_tokens != new_has_external_tokens ||
+    ts_subtree_has_changes(old_tree) ||
+    (
+      old_has_external_tokens &&
+      !ts_subtree_external_scanner_state_eq(old_iter->prev_external_token, new_iter->prev_external_token)
+    )
+  ) {
+    return IteratorMayDiffer;
+  }
+
+  return IteratorMatches;
+}
+
+#ifdef DEBUG_GET_CHANGED_RANGES
+static inline void iterator_print_state(Iterator *self) {
+  TreeCursorEntry entry = *array_back(&self->cursor.stack);
+  TSPoint start = iterator_start_position(self).extent;
+  TSPoint end = iterator_end_position(self).extent;
+  const char *name = ts_language_symbol_name(self->language, ts_subtree_symbol(*entry.subtree));
+  printf(
+    "(%-25s %s\t depth:%u [%u, %u] - [%u, %u])",
+    name, self->in_padding ? "(p)" : "   ",
+    self->visible_depth,
+    start.row, start.column,
+    end.row, end.column
+  );
+}
+#endif
+
+unsigned ts_subtree_get_changed_ranges(
+  const Subtree *old_tree, const Subtree *new_tree,
+  TreeCursor *cursor1, TreeCursor *cursor2,
+  const TSLanguage *language,
+  const TSRangeArray *included_range_differences,
+  TSRange **ranges
+) {
+  TSRangeArray results = array_new();
+
+  Iterator old_iter = iterator_new(cursor1, old_tree, language);
+  Iterator new_iter = iterator_new(cursor2, new_tree, language);
+
+  unsigned included_range_difference_index = 0;
+
+  Length position = iterator_start_position(&old_iter);
+  Length next_position = iterator_start_position(&new_iter);
+  if (position.bytes < next_position.bytes) {
+    ts_range_array_add(&results, position, next_position);
+    position = next_position;
+  } else if (position.bytes > next_position.bytes) {
+    ts_range_array_add(&results, next_position, position);
+    next_position = position;
+  }
+
+  do {
+    #ifdef DEBUG_GET_CHANGED_RANGES
+    printf("At [%-2u, %-2u] Compare ", position.extent.row, position.extent.column);
+    iterator_print_state(&old_iter);
+    printf("\tvs\t");
+    iterator_print_state(&new_iter);
+    puts("");
+    #endif
+
+    // Compare the old and new subtrees.
+    IteratorComparison comparison = iterator_compare(&old_iter, &new_iter);
+
+    // Even if the two subtrees appear to be identical, they could differ
+    // internally if they contain a range of text that was previously
+    // excluded from the parse, and is now included, or vice-versa.
+    if (comparison == IteratorMatches && ts_range_array_intersects(
+      included_range_differences,
+      included_range_difference_index,
+      position.bytes,
+      iterator_end_position(&old_iter).bytes
+    )) {
+      comparison = IteratorMayDiffer;
+    }
+
+    bool is_changed = false;
+    switch (comparison) {
+      // If the subtrees are definitely identical, move to the end
+      // of both subtrees.
+      case IteratorMatches:
+        next_position = iterator_end_position(&old_iter);
+        break;
+
+      // If the subtrees might differ internally, descend into both
+      // subtrees, finding the first child that spans the current position.
+      case IteratorMayDiffer:
+        if (iterator_descend(&old_iter, position.bytes)) {
+          if (!iterator_descend(&new_iter, position.bytes)) {
+            is_changed = true;
+            next_position = iterator_end_position(&old_iter);
+          }
+        } else if (iterator_descend(&new_iter, position.bytes)) {
+          is_changed = true;
+          next_position = iterator_end_position(&new_iter);
+        } else {
+          next_position = length_min(
+            iterator_end_position(&old_iter),
+            iterator_end_position(&new_iter)
+          );
+        }
+        break;
+
+      // If the subtrees are different, record a change and then move
+      // to the end of both subtrees.
+      case IteratorDiffers:
+        is_changed = true;
+        next_position = length_min(
+          iterator_end_position(&old_iter),
+          iterator_end_position(&new_iter)
+        );
+        break;
+    }
+
+    // Ensure that both iterators are caught up to the current position.
+    while (
+      !iterator_done(&old_iter) &&
+      iterator_end_position(&old_iter).bytes <= next_position.bytes
+    ) iterator_advance(&old_iter);
+    while (
+      !iterator_done(&new_iter) &&
+      iterator_end_position(&new_iter).bytes <= next_position.bytes
+    ) iterator_advance(&new_iter);
+
+    // Ensure that both iterators are at the same depth in the tree.
+    while (old_iter.visible_depth > new_iter.visible_depth) {
+      iterator_ascend(&old_iter);
+    }
+    while (new_iter.visible_depth > old_iter.visible_depth) {
+      iterator_ascend(&new_iter);
+    }
+
+    if (is_changed) {
+      #ifdef DEBUG_GET_CHANGED_RANGES
+      printf(
+        "  change: [[%u, %u] - [%u, %u]]\n",
+        position.extent.row + 1, position.extent.column,
+        next_position.extent.row + 1, next_position.extent.column
+      );
+      #endif
+
+      ts_range_array_add(&results, position, next_position);
+    }
+
+    position = next_position;
+
+    // Keep track of the current position in the included range differences
+    // array in order to avoid scanning the entire array on each iteration.
+    while (included_range_difference_index < included_range_differences->size) {
+      const TSRange *range = array_get(included_range_differences,
+        included_range_difference_index
+      );
+      if (range->end_byte <= position.bytes) {
+        included_range_difference_index++;
+      } else {
+        break;
+      }
+    }
+  } while (!iterator_done(&old_iter) && !iterator_done(&new_iter));
+
+  Length old_size = ts_subtree_total_size(*old_tree);
+  Length new_size = ts_subtree_total_size(*new_tree);
+  if (old_size.bytes < new_size.bytes) {
+    ts_range_array_add(&results, old_size, new_size);
+  } else if (new_size.bytes < old_size.bytes) {
+    ts_range_array_add(&results, new_size, old_size);
+  }
+
+  *cursor1 = old_iter.cursor;
+  *cursor2 = new_iter.cursor;
+  *ranges = results.contents;
+  return results.size;
+}

third_party/tree-sitter/lib/src/get_changed_ranges.h

@@ -0,0 +1,36 @@
+#ifndef TREE_SITTER_GET_CHANGED_RANGES_H_
+#define TREE_SITTER_GET_CHANGED_RANGES_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./tree_cursor.h"
+#include "./subtree.h"
+
+typedef Array(TSRange) TSRangeArray;
+
+void ts_range_array_get_changed_ranges(
+  const TSRange *old_ranges, unsigned old_range_count,
+  const TSRange *new_ranges, unsigned new_range_count,
+  TSRangeArray *differences
+);
+
+bool ts_range_array_intersects(
+  const TSRangeArray *self, unsigned start_index,
+  uint32_t start_byte, uint32_t end_byte
+);
+
+unsigned ts_subtree_get_changed_ranges(
+  const Subtree *old_tree, const Subtree *new_tree,
+  TreeCursor *cursor1, TreeCursor *cursor2,
+  const TSLanguage *language,
+  const TSRangeArray *included_range_differences,
+  TSRange **ranges
+);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_GET_CHANGED_RANGES_H_

third_party/tree-sitter/lib/src/host.h

@@ -0,0 +1,21 @@
+
+// Determine endian and pointer size based on known defines.
+// TS_BIG_ENDIAN and TS_PTR_SIZE can be set as -D compiler arguments
+// to override this.
+
+#if !defined(TS_BIG_ENDIAN)
+#if (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__) \
+  || (defined( __APPLE_CC__) && (defined(__ppc__) || defined(__ppc64__)))
+#define TS_BIG_ENDIAN 1
+#else
+#define TS_BIG_ENDIAN 0
+#endif
+#endif
+
+#if !defined(TS_PTR_SIZE)
+#if UINTPTR_MAX == 0xFFFFFFFF
+#define TS_PTR_SIZE 32
+#else
+#define TS_PTR_SIZE 64
+#endif
+#endif

third_party/tree-sitter/lib/src/language.c

@@ -0,0 +1,293 @@
+#include "./language.h"
+#include "./wasm_store.h"
+#include "tree_sitter/api.h"
+#include <string.h>
+
+const TSLanguage *ts_language_copy(const TSLanguage *self) {
+  if (self && ts_language_is_wasm(self)) {
+    ts_wasm_language_retain(self);
+  }
+  return self;
+}
+
+void ts_language_delete(const TSLanguage *self) {
+  if (self && ts_language_is_wasm(self)) {
+    ts_wasm_language_release(self);
+  }
+}
+
+uint32_t ts_language_symbol_count(const TSLanguage *self) {
+  return self->symbol_count + self->alias_count;
+}
+
+uint32_t ts_language_state_count(const TSLanguage *self) {
+  return self->state_count;
+}
+
+const TSSymbol *ts_language_supertypes(const TSLanguage *self, uint32_t *length) {
+  if (self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS) {
+    *length = self->supertype_count;
+    return self->supertype_symbols;
+  } else {
+    *length = 0;
+    return NULL;
+  }
+}
+
+const TSSymbol *ts_language_subtypes(
+  const TSLanguage *self,
+  TSSymbol supertype,
+  uint32_t *length
+) {
+  if (self->abi_version < LANGUAGE_VERSION_WITH_RESERVED_WORDS || !ts_language_symbol_metadata(self, supertype).supertype) {
+    *length = 0;
+    return NULL;
+  }
+
+  TSMapSlice slice = self->supertype_map_slices[supertype];
+  *length = slice.length;
+  return &self->supertype_map_entries[slice.index];
+}
+
+uint32_t ts_language_version(const TSLanguage *self) {
+  return self->abi_version;
+}
+
+uint32_t ts_language_abi_version(const TSLanguage *self) {
+  return self->abi_version;
+}
+
+const TSLanguageMetadata *ts_language_metadata(const TSLanguage *self) {
+    return self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS ? &self->metadata : NULL;
+}
+
+const char *ts_language_name(const TSLanguage *self) {
+  return self->abi_version >= LANGUAGE_VERSION_WITH_RESERVED_WORDS ? self->name : NULL;
+}
+
+uint32_t ts_language_field_count(const TSLanguage *self) {
+  return self->field_count;
+}
+
+void ts_language_table_entry(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol,
+  TableEntry *result
+) {
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
+    result->action_count = 0;
+    result->is_reusable = false;
+    result->actions = NULL;
+  } else {
+    ts_assert(symbol < self->token_count);
+    uint32_t action_index = ts_language_lookup(self, state, symbol);
+    const TSParseActionEntry *entry = &self->parse_actions[action_index];
+    result->action_count = entry->entry.count;
+    result->is_reusable = entry->entry.reusable;
+    result->actions = (const TSParseAction *)(entry + 1);
+  }
+}
+
+TSLexerMode ts_language_lex_mode_for_state(
+   const TSLanguage *self,
+   TSStateId state
+) {
+  if (self->abi_version < 15) {
+    TSLexMode mode = ((const TSLexMode *)self->lex_modes)[state];
+    return (TSLexerMode) {
+      .lex_state = mode.lex_state,
+      .external_lex_state = mode.external_lex_state,
+      .reserved_word_set_id = 0,
+    };
+  } else {
+    return self->lex_modes[state];
+  }
+}
+
+bool ts_language_is_reserved_word(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol
+) {
+  TSLexerMode lex_mode = ts_language_lex_mode_for_state(self, state);
+  if (lex_mode.reserved_word_set_id > 0) {
+    unsigned start = lex_mode.reserved_word_set_id * self->max_reserved_word_set_size;
+    unsigned end = start + self->max_reserved_word_set_size;
+    for (unsigned i = start; i < end; i++) {
+      if (self->reserved_words[i] == symbol) return true;
+      if (self->reserved_words[i] == 0) break;
+    }
+  }
+  return false;
+}
+
+TSSymbolMetadata ts_language_symbol_metadata(
+  const TSLanguage *self,
+  TSSymbol symbol
+) {
+  if (symbol == ts_builtin_sym_error)  {
+    return (TSSymbolMetadata) {.visible = true, .named = true};
+  } else if (symbol == ts_builtin_sym_error_repeat) {
+    return (TSSymbolMetadata) {.visible = false, .named = false};
+  } else {
+    return self->symbol_metadata[symbol];
+  }
+}
+
+TSSymbol ts_language_public_symbol(
+  const TSLanguage *self,
+  TSSymbol symbol
+) {
+  if (symbol == ts_builtin_sym_error) return symbol;
+  return self->public_symbol_map[symbol];
+}
+
+TSStateId ts_language_next_state(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol
+) {
+  if (symbol == ts_builtin_sym_error || symbol == ts_builtin_sym_error_repeat) {
+    return 0;
+  } else if (symbol < self->token_count) {
+    uint32_t count;
+    const TSParseAction *actions = ts_language_actions(self, state, symbol, &count);
+    if (count > 0) {
+      TSParseAction action = actions[count - 1];
+      if (action.type == TSParseActionTypeShift) {
+        return action.shift.extra ? state : action.shift.state;
+      }
+    }
+    return 0;
+  } else {
+    return ts_language_lookup(self, state, symbol);
+  }
+}
+
+const char *ts_language_symbol_name(
+  const TSLanguage *self,
+  TSSymbol symbol
+) {
+  if (symbol == ts_builtin_sym_error) {
+    return "ERROR";
+  } else if (symbol == ts_builtin_sym_error_repeat) {
+    return "_ERROR";
+  } else if (symbol < ts_language_symbol_count(self)) {
+    return self->symbol_names[symbol];
+  } else {
+    return NULL;
+  }
+}
+
+TSSymbol ts_language_symbol_for_name(
+  const TSLanguage *self,
+  const char *string,
+  uint32_t length,
+  bool is_named
+) {
+  if (!strncmp(string, "ERROR", length)) return ts_builtin_sym_error;
+  uint16_t count = (uint16_t)ts_language_symbol_count(self);
+  for (TSSymbol i = 0; i < count; i++) {
+    TSSymbolMetadata metadata = ts_language_symbol_metadata(self, i);
+    if ((!metadata.visible && !metadata.supertype) || metadata.named != is_named) continue;
+    const char *symbol_name = self->symbol_names[i];
+    if (!strncmp(symbol_name, string, length) && !symbol_name[length]) {
+      return self->public_symbol_map[i];
+    }
+  }
+  return 0;
+}
+
+TSSymbolType ts_language_symbol_type(
+  const TSLanguage *self,
+  TSSymbol symbol
+) {
+  TSSymbolMetadata metadata = ts_language_symbol_metadata(self, symbol);
+  if (metadata.named && metadata.visible) {
+    return TSSymbolTypeRegular;
+  } else if (metadata.visible) {
+    return TSSymbolTypeAnonymous;
+  } else if (metadata.supertype) {
+    return TSSymbolTypeSupertype;
+  } else {
+    return TSSymbolTypeAuxiliary;
+  }
+}
+
+const char *ts_language_field_name_for_id(
+  const TSLanguage *self,
+  TSFieldId id
+) {
+  uint32_t count = ts_language_field_count(self);
+  if (count && id <= count) {
+    return self->field_names[id];
+  } else {
+    return NULL;
+  }
+}
+
+TSFieldId ts_language_field_id_for_name(
+  const TSLanguage *self,
+  const char *name,
+  uint32_t name_length
+) {
+  uint16_t count = (uint16_t)ts_language_field_count(self);
+  for (TSSymbol i = 1; i < count + 1; i++) {
+    switch (strncmp(name, self->field_names[i], name_length)) {
+      case 0:
+        if (self->field_names[i][name_length] == 0) return i;
+        break;
+      case -1:
+        return 0;
+      default:
+        break;
+    }
+  }
+  return 0;
+}
+
+TSLookaheadIterator *ts_lookahead_iterator_new(const TSLanguage *self, TSStateId state) {
+  if (state >= self->state_count) return NULL;
+  LookaheadIterator *iterator = ts_malloc(sizeof(LookaheadIterator));
+  *iterator = ts_language_lookaheads(self, state);
+  return (TSLookaheadIterator *)iterator;
+}
+
+void ts_lookahead_iterator_delete(TSLookaheadIterator *self) {
+  ts_free(self);
+}
+
+bool ts_lookahead_iterator_reset_state(TSLookaheadIterator * self, TSStateId state) {
+  LookaheadIterator *iterator = (LookaheadIterator *)self;
+  if (state >= iterator->language->state_count) return false;
+  *iterator = ts_language_lookaheads(iterator->language, state);
+  return true;
+}
+
+const TSLanguage *ts_lookahead_iterator_language(const TSLookaheadIterator *self) {
+  const LookaheadIterator *iterator = (const LookaheadIterator *)self;
+  return iterator->language;
+}
+
+bool ts_lookahead_iterator_reset(TSLookaheadIterator *self, const TSLanguage *language, TSStateId state) {
+  if (state >= language->state_count) return false;
+  LookaheadIterator *iterator = (LookaheadIterator *)self;
+  *iterator = ts_language_lookaheads(language, state);
+  return true;
+}
+
+bool ts_lookahead_iterator_next(TSLookaheadIterator *self) {
+  LookaheadIterator *iterator = (LookaheadIterator *)self;
+  return ts_lookahead_iterator__next(iterator);
+}
+
+TSSymbol ts_lookahead_iterator_current_symbol(const TSLookaheadIterator *self) {
+  const LookaheadIterator *iterator = (const LookaheadIterator *)self;
+  return iterator->symbol;
+}
+
+const char *ts_lookahead_iterator_current_symbol_name(const TSLookaheadIterator *self) {
+  const LookaheadIterator *iterator = (const LookaheadIterator *)self;
+  return ts_language_symbol_name(iterator->language, iterator->symbol);
+}

third_party/tree-sitter/lib/src/language.h

@@ -0,0 +1,293 @@
+#ifndef TREE_SITTER_LANGUAGE_H_
+#define TREE_SITTER_LANGUAGE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./subtree.h"
+#include "./parser.h"
+
+#define ts_builtin_sym_error_repeat (ts_builtin_sym_error - 1)
+
+#define LANGUAGE_VERSION_WITH_RESERVED_WORDS 15
+#define LANGUAGE_VERSION_WITH_PRIMARY_STATES 14
+
+typedef struct {
+  const TSParseAction *actions;
+  uint32_t action_count;
+  bool is_reusable;
+} TableEntry;
+
+typedef struct {
+  const TSLanguage *language;
+  const uint16_t *data;
+  const uint16_t *group_end;
+  TSStateId state;
+  uint16_t table_value;
+  uint16_t section_index;
+  uint16_t group_count;
+  bool is_small_state;
+
+  const TSParseAction *actions;
+  TSSymbol symbol;
+  TSStateId next_state;
+  uint16_t action_count;
+} LookaheadIterator;
+
+void ts_language_table_entry(const TSLanguage *self, TSStateId state, TSSymbol symbol, TableEntry *result);
+TSLexerMode ts_language_lex_mode_for_state(const TSLanguage *self, TSStateId state);
+bool ts_language_is_reserved_word(const TSLanguage *self, TSStateId state, TSSymbol symbol);
+TSSymbolMetadata ts_language_symbol_metadata(const TSLanguage *self, TSSymbol symbol);
+TSSymbol ts_language_public_symbol(const TSLanguage *self, TSSymbol symbol);
+
+static inline const TSParseAction *ts_language_actions(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol,
+  uint32_t *count
+) {
+  TableEntry entry;
+  ts_language_table_entry(self, state, symbol, &entry);
+  *count = entry.action_count;
+  return entry.actions;
+}
+
+static inline bool ts_language_has_reduce_action(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol
+) {
+  TableEntry entry;
+  ts_language_table_entry(self, state, symbol, &entry);
+  return entry.action_count > 0 && entry.actions[0].type == TSParseActionTypeReduce;
+}
+
+// Lookup the table value for a given symbol and state.
+//
+// For non-terminal symbols, the table value represents a successor state.
+// For terminal symbols, it represents an index in the actions table.
+// For 'large' parse states, this is a direct lookup. For 'small' parse
+// states, this requires searching through the symbol groups to find
+// the given symbol.
+static inline uint16_t ts_language_lookup(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol
+) {
+  if (state >= self->large_state_count) {
+    uint32_t index = self->small_parse_table_map[state - self->large_state_count];
+    const uint16_t *data = &self->small_parse_table[index];
+    uint16_t group_count = *(data++);
+    for (unsigned i = 0; i < group_count; i++) {
+      uint16_t section_value = *(data++);
+      uint16_t symbol_count = *(data++);
+      for (unsigned j = 0; j < symbol_count; j++) {
+        if (*(data++) == symbol) return section_value;
+      }
+    }
+    return 0;
+  } else {
+    return self->parse_table[state * self->symbol_count + symbol];
+  }
+}
+
+static inline bool ts_language_has_actions(
+  const TSLanguage *self,
+  TSStateId state,
+  TSSymbol symbol
+) {
+  return ts_language_lookup(self, state, symbol) != 0;
+}
+
+// Iterate over all of the symbols that are valid in the given state.
+//
+// For 'large' parse states, this just requires iterating through
+// all possible symbols and checking the parse table for each one.
+// For 'small' parse states, this exploits the structure of the
+// table to only visit the valid symbols.
+static inline LookaheadIterator ts_language_lookaheads(
+  const TSLanguage *self,
+  TSStateId state
+) {
+  bool is_small_state = state >= self->large_state_count;
+  const uint16_t *data;
+  const uint16_t *group_end = NULL;
+  uint16_t group_count = 0;
+  if (is_small_state) {
+    uint32_t index = self->small_parse_table_map[state - self->large_state_count];
+    data = &self->small_parse_table[index];
+    group_end = data + 1;
+    group_count = *data;
+  } else {
+    data = &self->parse_table[state * self->symbol_count] - 1;
+  }
+  return (LookaheadIterator) {
+    .language = self,
+    .data = data,
+    .group_end = group_end,
+    .group_count = group_count,
+    .is_small_state = is_small_state,
+    .symbol = UINT16_MAX,
+    .next_state = 0,
+  };
+}
+
+static inline bool ts_lookahead_iterator__next(LookaheadIterator *self) {
+  // For small parse states, valid symbols are listed explicitly,
+  // grouped by their value. There's no need to look up the actions
+  // again until moving to the next group.
+  if (self->is_small_state) {
+    self->data++;
+    if (self->data == self->group_end) {
+      if (self->group_count == 0) return false;
+      self->group_count--;
+      self->table_value = *(self->data++);
+      unsigned symbol_count = *(self->data++);
+      self->group_end = self->data + symbol_count;
+      self->symbol = *self->data;
+    } else {
+      self->symbol = *self->data;
+      return true;
+    }
+  }
+
+  // For large parse states, iterate through every symbol until one
+  // is found that has valid actions.
+  else {
+    do {
+      self->data++;
+      self->symbol++;
+      if (self->symbol >= self->language->symbol_count) return false;
+      self->table_value = *self->data;
+    } while (!self->table_value);
+  }
+
+  // Depending on if the symbols is terminal or non-terminal, the table value either
+  // represents a list of actions or a successor state.
+  if (self->symbol < self->language->token_count) {
+    const TSParseActionEntry *entry = &self->language->parse_actions[self->table_value];
+    self->action_count = entry->entry.count;
+    self->actions = (const TSParseAction *)(entry + 1);
+    self->next_state = 0;
+  } else {
+    self->action_count = 0;
+    self->next_state = self->table_value;
+  }
+  return true;
+}
+
+// Whether the state is a "primary state". If this returns false, it indicates that there exists
+// another state that behaves identically to this one with respect to query analysis.
+static inline bool ts_language_state_is_primary(
+  const TSLanguage *self,
+  TSStateId state
+) {
+  if (self->abi_version >= LANGUAGE_VERSION_WITH_PRIMARY_STATES) {
+    return state == self->primary_state_ids[state];
+  } else {
+    return true;
+  }
+}
+
+static inline const bool *ts_language_enabled_external_tokens(
+  const TSLanguage *self,
+  unsigned external_scanner_state
+) {
+  if (external_scanner_state == 0) {
+    return NULL;
+  } else {
+    return self->external_scanner.states + self->external_token_count * external_scanner_state;
+  }
+}
+
+static inline const TSSymbol *ts_language_alias_sequence(
+  const TSLanguage *self,
+  uint32_t production_id
+) {
+  return production_id ?
+    &self->alias_sequences[production_id * self->max_alias_sequence_length] :
+    NULL;
+}
+
+static inline TSSymbol ts_language_alias_at(
+  const TSLanguage *self,
+  uint32_t production_id,
+  uint32_t child_index
+) {
+  return production_id ?
+    self->alias_sequences[production_id * self->max_alias_sequence_length + child_index] :
+    0;
+}
+
+static inline void ts_language_field_map(
+  const TSLanguage *self,
+  uint32_t production_id,
+  const TSFieldMapEntry **start,
+  const TSFieldMapEntry **end
+) {
+  if (self->field_count == 0) {
+    *start = NULL;
+    *end = NULL;
+    return;
+  }
+
+  TSMapSlice slice = self->field_map_slices[production_id];
+  *start = &self->field_map_entries[slice.index];
+  *end = &self->field_map_entries[slice.index] + slice.length;
+}
+
+static inline void ts_language_aliases_for_symbol(
+  const TSLanguage *self,
+  TSSymbol original_symbol,
+  const TSSymbol **start,
+  const TSSymbol **end
+) {
+  *start = &self->public_symbol_map[original_symbol];
+  *end = *start + 1;
+
+  unsigned idx = 0;
+  for (;;) {
+    TSSymbol symbol = self->alias_map[idx++];
+    if (symbol == 0 || symbol > original_symbol) break;
+    uint16_t count = self->alias_map[idx++];
+    if (symbol == original_symbol) {
+      *start = &self->alias_map[idx];
+      *end = &self->alias_map[idx + count];
+      break;
+    }
+    idx += count;
+  }
+}
+
+static inline void ts_language_write_symbol_as_dot_string(
+  const TSLanguage *self,
+  FILE *f,
+  TSSymbol symbol
+) {
+  const char *name = ts_language_symbol_name(self, symbol);
+  for (const char *chr = name; *chr; chr++) {
+    switch (*chr) {
+      case '"':
+      case '\\':
+        fputc('\\', f);
+        fputc(*chr, f);
+        break;
+      case '\n':
+        fputs("\\n", f);
+        break;
+      case '\t':
+        fputs("\\t", f);
+        break;
+      default:
+        fputc(*chr, f);
+        break;
+    }
+  }
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_LANGUAGE_H_

third_party/tree-sitter/lib/src/length.h

@@ -0,0 +1,52 @@
+#ifndef TREE_SITTER_LENGTH_H_
+#define TREE_SITTER_LENGTH_H_
+
+#include <stdlib.h>
+#include <stdbool.h>
+#include "./point.h"
+#include "tree_sitter/api.h"
+
+typedef struct {
+  uint32_t bytes;
+  TSPoint extent;
+} Length;
+
+static const Length LENGTH_UNDEFINED = {0, {0, 1}};
+static const Length LENGTH_MAX = {UINT32_MAX, {UINT32_MAX, UINT32_MAX}};
+
+static inline bool length_is_undefined(Length length) {
+  return length.bytes == 0 && length.extent.column != 0;
+}
+
+static inline Length length_min(Length len1, Length len2) {
+  return (len1.bytes < len2.bytes) ? len1 : len2;
+}
+
+static inline Length length_add(Length len1, Length len2) {
+  Length result;
+  result.bytes = len1.bytes + len2.bytes;
+  result.extent = point_add(len1.extent, len2.extent);
+  return result;
+}
+
+static inline Length length_sub(Length len1, Length len2) {
+  Length result;
+  result.bytes = (len1.bytes >= len2.bytes) ? len1.bytes - len2.bytes : 0;
+  result.extent = point_sub(len1.extent, len2.extent);
+  return result;
+}
+
+static inline Length length_zero(void) {
+  Length result = {0, {0, 0}};
+  return result;
+}
+
+static inline Length length_saturating_sub(Length len1, Length len2) {
+  if (len1.bytes > len2.bytes) {
+    return length_sub(len1, len2);
+  } else {
+    return length_zero();
+  }
+}
+
+#endif

third_party/tree-sitter/lib/src/lexer.c

@@ -0,0 +1,483 @@
+#include "./length.h"
+#include "./lexer.h"
+#include "./unicode.h"
+
+#include "tree_sitter/api.h"
+
+#include <stdarg.h>
+#include <stdio.h>
+
+#define LOG(message, character)              \
+  if (self->logger.log) {                    \
+    snprintf(                                \
+      self->debug_buffer,                    \
+      TREE_SITTER_SERIALIZATION_BUFFER_SIZE, \
+      32 <= character && character < 127 ?   \
+        message " character:'%c'" :          \
+        message " character:%d",             \
+      character                              \
+    );                                       \
+    self->logger.log(                        \
+      self->logger.payload,                  \
+      TSLogTypeLex,                          \
+      self->debug_buffer                     \
+    );                                       \
+  }
+
+static const int32_t BYTE_ORDER_MARK = 0xFEFF;
+
+static const TSRange DEFAULT_RANGE = {
+  .start_point = {
+    .row = 0,
+    .column = 0,
+  },
+  .end_point = {
+    .row = UINT32_MAX,
+    .column = UINT32_MAX,
+  },
+  .start_byte = 0,
+  .end_byte = UINT32_MAX
+};
+
+/**
+ * Sets the column data to the given value and marks it valid.
+ * @param self The lexer state.
+ * @param val The new value of the column data.
+ */
+static void ts_lexer__set_column_data(Lexer *self, uint32_t val) {
+  self->column_data.valid = true;
+  self->column_data.value = val;
+}
+
+/**
+ * Increments the value of the column data; no-op if invalid.
+ * @param self The lexer state.
+ */
+static void ts_lexer__increment_column_data(Lexer *self) {
+  if (self->column_data.valid) {
+    self->column_data.value++;
+  }
+}
+
+/**
+ * Marks the column data as invalid.
+ * @param self The lexer state.
+ */
+static void ts_lexer__invalidate_column_data(Lexer *self) {
+  self->column_data.valid = false;
+  self->column_data.value = 0;
+}
+
+// Check if the lexer has reached EOF. This state is stored
+// by setting the lexer's `current_included_range_index` such that
+// it has consumed all of its available ranges.
+static bool ts_lexer__eof(const TSLexer *_self) {
+  Lexer *self = (Lexer *)_self;
+  return self->current_included_range_index == self->included_range_count;
+}
+
+// Clear the currently stored chunk of source code, because the lexer's
+// position has changed.
+static void ts_lexer__clear_chunk(Lexer *self) {
+  self->chunk = NULL;
+  self->chunk_size = 0;
+  self->chunk_start = 0;
+}
+
+// Call the lexer's input callback to obtain a new chunk of source code
+// for the current position.
+static void ts_lexer__get_chunk(Lexer *self) {
+  self->chunk_start = self->current_position.bytes;
+  self->chunk = self->input.read(
+    self->input.payload,
+    self->current_position.bytes,
+    self->current_position.extent,
+    &self->chunk_size
+  );
+  if (!self->chunk_size) {
+    self->current_included_range_index = self->included_range_count;
+    self->chunk = NULL;
+  }
+}
+
+// Decode the next unicode character in the current chunk of source code.
+// This assumes that the lexer has already retrieved a chunk of source
+// code that spans the current position.
+static void ts_lexer__get_lookahead(Lexer *self) {
+  uint32_t position_in_chunk = self->current_position.bytes - self->chunk_start;
+  uint32_t size = self->chunk_size - position_in_chunk;
+
+  if (size == 0) {
+    self->lookahead_size = 1;
+    self->data.lookahead = '\0';
+    return;
+  }
+
+  const uint8_t *chunk = (const uint8_t *)self->chunk + position_in_chunk;
+  DecodeFunction decode =
+    self->input.encoding == TSInputEncodingUTF8    ? ts_decode_utf8     :
+    self->input.encoding == TSInputEncodingUTF16LE ? ts_decode_utf16_le :
+    self->input.encoding == TSInputEncodingUTF16BE ? ts_decode_utf16_be : self->input.decode;
+
+  self->lookahead_size = decode(chunk, size, &self->data.lookahead);
+
+  // If this chunk ended in the middle of a multi-byte character,
+  // try again with a fresh chunk.
+  if (self->data.lookahead == TS_DECODE_ERROR && size < 4) {
+    ts_lexer__get_chunk(self);
+    chunk = (const uint8_t *)self->chunk;
+    size = self->chunk_size;
+    self->lookahead_size = decode(chunk, size, &self->data.lookahead);
+  }
+
+  if (self->data.lookahead == TS_DECODE_ERROR) {
+    self->lookahead_size = 1;
+  }
+}
+
+static void ts_lexer_goto(Lexer *self, Length position) {
+  if (position.bytes != self->current_position.bytes) {
+    ts_lexer__invalidate_column_data(self);
+  }
+
+  self->current_position = position;
+
+  // Move to the first valid position at or after the given position.
+  bool found_included_range = false;
+  for (unsigned i = 0; i < self->included_range_count; i++) {
+    TSRange *included_range = &self->included_ranges[i];
+    if (
+      included_range->end_byte > self->current_position.bytes &&
+      included_range->end_byte > included_range->start_byte
+    ) {
+      if (included_range->start_byte >= self->current_position.bytes) {
+        self->current_position = (Length) {
+          .bytes = included_range->start_byte,
+          .extent = included_range->start_point,
+        };
+      }
+
+      self->current_included_range_index = i;
+      found_included_range = true;
+      break;
+    }
+  }
+
+  if (found_included_range) {
+    // If the current position is outside of the current chunk of text,
+    // then clear out the current chunk of text.
+    if (self->chunk && (
+      self->current_position.bytes < self->chunk_start ||
+      self->current_position.bytes >= self->chunk_start + self->chunk_size
+    )) {
+      ts_lexer__clear_chunk(self);
+    }
+
+    self->lookahead_size = 0;
+    self->data.lookahead = '\0';
+  }
+
+  // If the given position is beyond any of included ranges, move to the EOF
+  // state - past the end of the included ranges.
+  else {
+    self->current_included_range_index = self->included_range_count;
+    TSRange *last_included_range = &self->included_ranges[self->included_range_count - 1];
+    self->current_position = (Length) {
+      .bytes = last_included_range->end_byte,
+      .extent = last_included_range->end_point,
+    };
+    ts_lexer__clear_chunk(self);
+    self->lookahead_size = 1;
+    self->data.lookahead = '\0';
+  }
+}
+
+/**
+ * Actually advances the lexer. Does not log anything.
+ * @param self The lexer state.
+ * @param skip Whether to mark the consumed codepoint as whitespace.
+ */
+static void ts_lexer__do_advance(Lexer *self, bool skip) {
+  if (self->lookahead_size) {
+    if (self->data.lookahead == '\n') {
+      self->current_position.extent.row++;
+      self->current_position.extent.column = 0;
+      ts_lexer__set_column_data(self, 0);
+    } else {
+      bool is_bom = self->current_position.bytes == 0 && 
+        self->data.lookahead == BYTE_ORDER_MARK;
+      if (!is_bom) ts_lexer__increment_column_data(self);
+      self->current_position.extent.column += self->lookahead_size;
+    }
+    self->current_position.bytes += self->lookahead_size;
+  }
+
+  const TSRange *current_range = &self->included_ranges[self->current_included_range_index];
+  while (
+    self->current_position.bytes >= current_range->end_byte ||
+    current_range->end_byte == current_range->start_byte
+  ) {
+    if (self->current_included_range_index < self->included_range_count) {
+      self->current_included_range_index++;
+    }
+    if (self->current_included_range_index < self->included_range_count) {
+      current_range++;
+      self->current_position = (Length) {
+        current_range->start_byte,
+        current_range->start_point,
+      };
+    } else {
+      current_range = NULL;
+      break;
+    }
+  }
+
+  if (skip) self->token_start_position = self->current_position;
+
+  if (current_range) {
+    if (
+      self->current_position.bytes < self->chunk_start ||
+      self->current_position.bytes >= self->chunk_start + self->chunk_size
+    ) {
+      ts_lexer__get_chunk(self);
+    }
+    ts_lexer__get_lookahead(self);
+  } else {
+    ts_lexer__clear_chunk(self);
+    self->data.lookahead = '\0';
+    self->lookahead_size = 1;
+  }
+}
+
+// Advance to the next character in the source code, retrieving a new
+// chunk of source code if needed.
+static void ts_lexer__advance(TSLexer *_self, bool skip) {
+  Lexer *self = (Lexer *)_self;
+  if (!self->chunk) return;
+
+  if (skip) {
+    LOG("skip", self->data.lookahead)
+  } else {
+    LOG("consume", self->data.lookahead)
+  }
+
+  ts_lexer__do_advance(self, skip);
+}
+
+// Mark that a token match has completed. This can be called multiple
+// times if a longer match is found later.
+static void ts_lexer__mark_end(TSLexer *_self) {
+  Lexer *self = (Lexer *)_self;
+  if (!ts_lexer__eof(&self->data)) {
+    // If the lexer is right at the beginning of included range,
+    // then the token should be considered to end at the *end* of the
+    // previous included range, rather than here.
+    TSRange *current_included_range = &self->included_ranges[
+      self->current_included_range_index
+    ];
+    if (
+      self->current_included_range_index > 0 &&
+      self->current_position.bytes == current_included_range->start_byte
+    ) {
+      TSRange *previous_included_range = current_included_range - 1;
+      self->token_end_position = (Length) {
+        previous_included_range->end_byte,
+        previous_included_range->end_point,
+      };
+      return;
+    }
+  }
+  self->token_end_position = self->current_position;
+}
+
+static uint32_t ts_lexer__get_column(TSLexer *_self) {
+  Lexer *self = (Lexer *)_self;
+
+  self->did_get_column = true;
+
+  if (!self->column_data.valid) {
+    // Record current position
+    uint32_t goal_byte = self->current_position.bytes;
+
+    // Back up to the beginning of the line
+    Length start_of_col = {
+      self->current_position.bytes - self->current_position.extent.column,
+      {self->current_position.extent.row, 0},
+    };
+    ts_lexer_goto(self, start_of_col);
+    ts_lexer__set_column_data(self, 0);
+    ts_lexer__get_chunk(self);
+
+    if (!ts_lexer__eof(_self)) {
+      ts_lexer__get_lookahead(self);
+
+      // Advance to the recorded position
+      while (self->current_position.bytes < goal_byte && !ts_lexer__eof(_self) && self->chunk) {
+        ts_lexer__do_advance(self, false);
+        if (ts_lexer__eof(_self)) break;
+      }
+    }
+  }
+
+  return self->column_data.value;
+}
+
+// Is the lexer at a boundary between two disjoint included ranges of
+// source code? This is exposed as an API because some languages' external
+// scanners need to perform custom actions at these boundaries.
+static bool ts_lexer__is_at_included_range_start(const TSLexer *_self) {
+  const Lexer *self = (const Lexer *)_self;
+  if (self->current_included_range_index < self->included_range_count) {
+    TSRange *current_range = &self->included_ranges[self->current_included_range_index];
+    return self->current_position.bytes == current_range->start_byte;
+  } else {
+    return false;
+  }
+}
+
+static void ts_lexer__log(const TSLexer *_self, const char *fmt, ...) {
+  Lexer *self = (Lexer *)_self;
+  va_list args;
+  va_start(args, fmt);
+  if (self->logger.log) {
+    vsnprintf(self->debug_buffer, TREE_SITTER_SERIALIZATION_BUFFER_SIZE, fmt, args);
+    self->logger.log(self->logger.payload, TSLogTypeLex, self->debug_buffer);
+  }
+  va_end(args);
+}
+
+void ts_lexer_init(Lexer *self) {
+  *self = (Lexer) {
+    .data = {
+      // The lexer's methods are stored as struct fields so that generated
+      // parsers can call them without needing to be linked against this
+      // library.
+      .advance = ts_lexer__advance,
+      .mark_end = ts_lexer__mark_end,
+      .get_column = ts_lexer__get_column,
+      .is_at_included_range_start = ts_lexer__is_at_included_range_start,
+      .eof = ts_lexer__eof,
+      .log = ts_lexer__log,
+      .lookahead = 0,
+      .result_symbol = 0,
+    },
+    .chunk = NULL,
+    .chunk_size = 0,
+    .chunk_start = 0,
+    .current_position = {0, {0, 0}},
+    .logger = {
+      .payload = NULL,
+      .log = NULL
+    },
+    .included_ranges = NULL,
+    .included_range_count = 0,
+    .current_included_range_index = 0,
+    .did_get_column = false,
+    .column_data = {
+      .valid = false,
+      .value = 0
+    }
+  };
+  ts_lexer_set_included_ranges(self, NULL, 0);
+}
+
+void ts_lexer_delete(Lexer *self) {
+  ts_free(self->included_ranges);
+}
+
+void ts_lexer_set_input(Lexer *self, TSInput input) {
+  self->input = input;
+  ts_lexer__clear_chunk(self);
+  ts_lexer_goto(self, self->current_position);
+}
+
+// Move the lexer to the given position. This doesn't do any work
+// if the parser is already at the given position.
+void ts_lexer_reset(Lexer *self, Length position) {
+  if (position.bytes != self->current_position.bytes) {
+    ts_lexer_goto(self, position);
+  }
+}
+
+void ts_lexer_start(Lexer *self) {
+  self->token_start_position = self->current_position;
+  self->token_end_position = LENGTH_UNDEFINED;
+  self->data.result_symbol = 0;
+  self->did_get_column = false;
+  if (!ts_lexer__eof(&self->data)) {
+    if (!self->chunk_size) ts_lexer__get_chunk(self);
+    if (!self->lookahead_size) ts_lexer__get_lookahead(self);
+    if (self->current_position.bytes == 0) {
+      if (self->data.lookahead == BYTE_ORDER_MARK) {
+        ts_lexer__advance(&self->data, true);
+      }
+      ts_lexer__set_column_data(self, 0);
+    }
+  }
+}
+
+void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte) {
+  if (length_is_undefined(self->token_end_position)) {
+    ts_lexer__mark_end(&self->data);
+  }
+
+  // If the token ended at an included range boundary, then its end position
+  // will have been reset to the end of the preceding range. Reset the start
+  // position to match.
+  if (self->token_end_position.bytes < self->token_start_position.bytes) {
+    self->token_start_position = self->token_end_position;
+  }
+
+  uint32_t current_lookahead_end_byte = self->current_position.bytes + 1;
+
+  // In order to determine that a byte sequence is invalid UTF8 or UTF16,
+  // the character decoding algorithm may have looked at the following byte.
+  // Therefore, the next byte *after* the current (invalid) character
+  // affects the interpretation of the current character.
+  if (self->data.lookahead == TS_DECODE_ERROR) {
+    current_lookahead_end_byte += 4; // the maximum number of bytes read to identify an invalid code point
+  }
+
+  if (current_lookahead_end_byte > *lookahead_end_byte) {
+    *lookahead_end_byte = current_lookahead_end_byte;
+  }
+}
+
+void ts_lexer_mark_end(Lexer *self) {
+  ts_lexer__mark_end(&self->data);
+}
+
+bool ts_lexer_set_included_ranges(
+  Lexer *self,
+  const TSRange *ranges,
+  uint32_t count
+) {
+  if (count == 0 || !ranges) {
+    ranges = &DEFAULT_RANGE;
+    count = 1;
+  } else {
+    uint32_t previous_byte = 0;
+    for (unsigned i = 0; i < count; i++) {
+      const TSRange *range = &ranges[i];
+      if (
+        range->start_byte < previous_byte ||
+        range->end_byte < range->start_byte
+      ) return false;
+      previous_byte = range->end_byte;
+    }
+  }
+
+  size_t size = count * sizeof(TSRange);
+  self->included_ranges = ts_realloc(self->included_ranges, size);
+  memcpy(self->included_ranges, ranges, size);
+  self->included_range_count = count;
+  ts_lexer_goto(self, self->current_position);
+  return true;
+}
+
+TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count) {
+  *count = self->included_range_count;
+  return self->included_ranges;
+}
+
+#undef LOG

third_party/tree-sitter/lib/src/lexer.h

@@ -0,0 +1,54 @@
+#ifndef TREE_SITTER_LEXER_H_
+#define TREE_SITTER_LEXER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./length.h"
+#include "./subtree.h"
+#include "tree_sitter/api.h"
+#include "./parser.h"
+
+typedef struct {
+  uint32_t value;
+  bool valid;
+} ColumnData;
+
+typedef struct {
+  TSLexer data;
+  Length current_position;
+  Length token_start_position;
+  Length token_end_position;
+
+  TSRange *included_ranges;
+  const char *chunk;
+  TSInput input;
+  TSLogger logger;
+
+  uint32_t included_range_count;
+  uint32_t current_included_range_index;
+  uint32_t chunk_start;
+  uint32_t chunk_size;
+  uint32_t lookahead_size;
+  bool did_get_column;
+  ColumnData column_data;
+
+  char debug_buffer[TREE_SITTER_SERIALIZATION_BUFFER_SIZE];
+} Lexer;
+
+void ts_lexer_init(Lexer *self);
+void ts_lexer_delete(Lexer *self);
+void ts_lexer_set_input(Lexer *self, TSInput input);
+void ts_lexer_reset(Lexer *self, Length position);
+void ts_lexer_start(Lexer *self);
+void ts_lexer_finish(Lexer *self, uint32_t *lookahead_end_byte);
+void ts_lexer_mark_end(Lexer *self);
+bool ts_lexer_set_included_ranges(Lexer *self, const TSRange *ranges, uint32_t count);
+TSRange *ts_lexer_included_ranges(const Lexer *self, uint32_t *count);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_LEXER_H_

third_party/tree-sitter/lib/src/lib.c

@@ -0,0 +1,12 @@
+#include "./alloc.c"
+#include "./get_changed_ranges.c"
+#include "./language.c"
+#include "./lexer.c"
+#include "./node.c"
+#include "./parser.c"
+#include "./query.c"
+#include "./stack.c"
+#include "./subtree.c"
+#include "./tree_cursor.c"
+#include "./tree.c"
+#include "./wasm_store.c"

third_party/tree-sitter/lib/src/node.c

@@ -0,0 +1,875 @@
+#include <stdbool.h>
+#include "./point.h"
+#include "./subtree.h"
+#include "./tree.h"
+#include "./language.h"
+
+typedef struct {
+  Subtree parent;
+  const TSTree *tree;
+  Length position;
+  uint32_t child_index;
+  uint32_t structural_child_index;
+  const TSSymbol *alias_sequence;
+} NodeChildIterator;
+
+static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous);
+
+// TSNode - constructors
+
+TSNode ts_node_new(
+  const TSTree *tree,
+  const Subtree *subtree,
+  Length position,
+  TSSymbol alias
+) {
+  return (TSNode) {
+    {position.bytes, position.extent.row, position.extent.column, alias},
+    subtree,
+    tree,
+  };
+}
+
+static inline TSNode ts_node__null(void) {
+  return ts_node_new(NULL, NULL, length_zero(), 0);
+}
+
+// TSNode - accessors
+
+uint32_t ts_node_start_byte(TSNode self) {
+  return self.context[0];
+}
+
+TSPoint ts_node_start_point(TSNode self) {
+  return (TSPoint) {self.context[1], self.context[2]};
+}
+
+static inline uint32_t ts_node__alias(const TSNode *self) {
+  return self->context[3];
+}
+
+static inline Subtree ts_node__subtree(TSNode self) {
+  return *(const Subtree *)self.id;
+}
+
+// NodeChildIterator
+
+static inline NodeChildIterator ts_node_iterate_children(const TSNode *node) {
+  Subtree subtree = ts_node__subtree(*node);
+  if (ts_subtree_child_count(subtree) == 0) {
+    return (NodeChildIterator) {NULL_SUBTREE, node->tree, length_zero(), 0, 0, NULL};
+  }
+  const TSSymbol *alias_sequence = ts_language_alias_sequence(
+    node->tree->language,
+    subtree.ptr->production_id
+  );
+  return (NodeChildIterator) {
+    .tree = node->tree,
+    .parent = subtree,
+    .position = {ts_node_start_byte(*node), ts_node_start_point(*node)},
+    .child_index = 0,
+    .structural_child_index = 0,
+    .alias_sequence = alias_sequence,
+  };
+}
+
+static inline bool ts_node_child_iterator_done(NodeChildIterator *self) {
+  return self->child_index == self->parent.ptr->child_count;
+}
+
+static inline bool ts_node_child_iterator_next(
+  NodeChildIterator *self,
+  TSNode *result
+) {
+  if (!self->parent.ptr || ts_node_child_iterator_done(self)) return false;
+  const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
+  TSSymbol alias_symbol = 0;
+  if (!ts_subtree_extra(*child)) {
+    if (self->alias_sequence) {
+      alias_symbol = self->alias_sequence[self->structural_child_index];
+    }
+    self->structural_child_index++;
+  }
+  if (self->child_index > 0) {
+    self->position = length_add(self->position, ts_subtree_padding(*child));
+  }
+  *result = ts_node_new(
+    self->tree,
+    child,
+    self->position,
+    alias_symbol
+  );
+  self->position = length_add(self->position, ts_subtree_size(*child));
+  self->child_index++;
+  return true;
+}
+
+// TSNode - private
+
+static inline bool ts_node__is_relevant(TSNode self, bool include_anonymous) {
+  Subtree tree = ts_node__subtree(self);
+  if (include_anonymous) {
+    return ts_subtree_visible(tree) || ts_node__alias(&self);
+  } else {
+    TSSymbol alias = ts_node__alias(&self);
+    if (alias) {
+      return ts_language_symbol_metadata(self.tree->language, alias).named;
+    } else {
+      return ts_subtree_visible(tree) && ts_subtree_named(tree);
+    }
+  }
+}
+
+static inline uint32_t ts_node__relevant_child_count(
+  TSNode self,
+  bool include_anonymous
+) {
+  Subtree tree = ts_node__subtree(self);
+  if (ts_subtree_child_count(tree) > 0) {
+    if (include_anonymous) {
+      return tree.ptr->visible_child_count;
+    } else {
+      return tree.ptr->named_child_count;
+    }
+  } else {
+    return 0;
+  }
+}
+
+static inline TSNode ts_node__child(
+  TSNode self,
+  uint32_t child_index,
+  bool include_anonymous
+) {
+  TSNode result = self;
+  bool did_descend = true;
+
+  while (did_descend) {
+    did_descend = false;
+
+    TSNode child;
+    uint32_t index = 0;
+    NodeChildIterator iterator = ts_node_iterate_children(&result);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      if (ts_node__is_relevant(child, include_anonymous)) {
+        if (index == child_index) {
+          return child;
+        }
+        index++;
+      } else {
+        uint32_t grandchild_index = child_index - index;
+        uint32_t grandchild_count = ts_node__relevant_child_count(child, include_anonymous);
+        if (grandchild_index < grandchild_count) {
+          did_descend = true;
+          result = child;
+          child_index = grandchild_index;
+          break;
+        }
+        index += grandchild_count;
+      }
+    }
+  }
+
+  return ts_node__null();
+}
+
+static bool ts_subtree_has_trailing_empty_descendant(
+  Subtree self,
+  Subtree other
+) {
+  for (unsigned i = ts_subtree_child_count(self) - 1; i + 1 > 0; i--) {
+    Subtree child = ts_subtree_children(self)[i];
+    if (ts_subtree_total_bytes(child) > 0) break;
+    if (child.ptr == other.ptr || ts_subtree_has_trailing_empty_descendant(child, other)) {
+      return true;
+    }
+  }
+  return false;
+}
+
+static inline TSNode ts_node__prev_sibling(TSNode self, bool include_anonymous) {
+  Subtree self_subtree = ts_node__subtree(self);
+  bool self_is_empty = ts_subtree_total_bytes(self_subtree) == 0;
+  uint32_t target_end_byte = ts_node_end_byte(self);
+
+  TSNode node = ts_node_parent(self);
+  TSNode earlier_node = ts_node__null();
+  bool earlier_node_is_relevant = false;
+
+  while (!ts_node_is_null(node)) {
+    TSNode earlier_child = ts_node__null();
+    bool earlier_child_is_relevant = false;
+    bool found_child_containing_target = false;
+
+    TSNode child;
+    NodeChildIterator iterator = ts_node_iterate_children(&node);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      if (child.id == self.id) break;
+      if (iterator.position.bytes > target_end_byte) {
+        found_child_containing_target = true;
+        break;
+      }
+
+      if (iterator.position.bytes == target_end_byte &&
+          (!self_is_empty ||
+           ts_subtree_has_trailing_empty_descendant(ts_node__subtree(child), self_subtree))) {
+        found_child_containing_target = true;
+        break;
+      }
+
+      if (ts_node__is_relevant(child, include_anonymous)) {
+        earlier_child = child;
+        earlier_child_is_relevant = true;
+      } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) {
+        earlier_child = child;
+        earlier_child_is_relevant = false;
+      }
+    }
+
+    if (found_child_containing_target) {
+      if (!ts_node_is_null(earlier_child)) {
+        earlier_node = earlier_child;
+        earlier_node_is_relevant = earlier_child_is_relevant;
+      }
+      node = child;
+    } else if (earlier_child_is_relevant) {
+      return earlier_child;
+    } else if (!ts_node_is_null(earlier_child)) {
+      node = earlier_child;
+    } else if (earlier_node_is_relevant) {
+      return earlier_node;
+    } else {
+      node = earlier_node;
+      earlier_node = ts_node__null();
+      earlier_node_is_relevant = false;
+    }
+  }
+
+  return ts_node__null();
+}
+
+static inline TSNode ts_node__next_sibling(TSNode self, bool include_anonymous) {
+  uint32_t target_end_byte = ts_node_end_byte(self);
+
+  TSNode node = ts_node_parent(self);
+  TSNode later_node = ts_node__null();
+  bool later_node_is_relevant = false;
+
+  while (!ts_node_is_null(node)) {
+    TSNode later_child = ts_node__null();
+    bool later_child_is_relevant = false;
+    TSNode child_containing_target = ts_node__null();
+
+    TSNode child;
+    NodeChildIterator iterator = ts_node_iterate_children(&node);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      if (iterator.position.bytes <= target_end_byte) continue;
+      uint32_t start_byte = ts_node_start_byte(self);
+      uint32_t child_start_byte = ts_node_start_byte(child);
+
+      bool is_empty = start_byte == target_end_byte;
+      bool contains_target = is_empty ?
+        child_start_byte < start_byte :
+        child_start_byte <= start_byte;
+
+      if (contains_target) {
+        if (ts_node__subtree(child).ptr != ts_node__subtree(self).ptr) {
+          child_containing_target = child;
+        }
+      } else if (ts_node__is_relevant(child, include_anonymous)) {
+        later_child = child;
+        later_child_is_relevant = true;
+        break;
+      } else if (ts_node__relevant_child_count(child, include_anonymous) > 0) {
+        later_child = child;
+        later_child_is_relevant = false;
+        break;
+      }
+    }
+
+    if (!ts_node_is_null(child_containing_target)) {
+      if (!ts_node_is_null(later_child)) {
+        later_node = later_child;
+        later_node_is_relevant = later_child_is_relevant;
+      }
+      node = child_containing_target;
+    } else if (later_child_is_relevant) {
+      return later_child;
+    } else if (!ts_node_is_null(later_child)) {
+      node = later_child;
+    } else if (later_node_is_relevant) {
+      return later_node;
+    } else {
+      node = later_node;
+    }
+  }
+
+  return ts_node__null();
+}
+
+static inline TSNode ts_node__first_child_for_byte(
+  TSNode self,
+  uint32_t goal,
+  bool include_anonymous
+) {
+  TSNode node = self;
+  bool did_descend = true;
+
+  NodeChildIterator last_iterator;
+  bool has_last_iterator = false;
+
+  while (did_descend) {
+    did_descend = false;
+
+    TSNode child;
+    NodeChildIterator iterator = ts_node_iterate_children(&node);
+  loop:
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      if (ts_node_end_byte(child) > goal) {
+        if (ts_node__is_relevant(child, include_anonymous)) {
+          return child;
+        } else if (ts_node_child_count(child) > 0) {
+          if (iterator.child_index < ts_subtree_child_count(ts_node__subtree(child))) {
+            last_iterator = iterator;
+            has_last_iterator = true;
+          }
+          did_descend = true;
+          node = child;
+          break;
+        }
+      }
+    }
+
+    if (!did_descend && has_last_iterator) {
+      iterator = last_iterator;
+      has_last_iterator = false;
+      goto loop;
+    }
+  }
+
+  return ts_node__null();
+}
+
+static inline TSNode ts_node__descendant_for_byte_range(
+  TSNode self,
+  uint32_t range_start,
+  uint32_t range_end,
+  bool include_anonymous
+) {
+  if (range_start > range_end) {
+    return ts_node__null();
+  }
+  TSNode node = self;
+  TSNode last_visible_node = self;
+
+  bool did_descend = true;
+  while (did_descend) {
+    did_descend = false;
+
+    TSNode child;
+    NodeChildIterator iterator = ts_node_iterate_children(&node);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      uint32_t node_end = iterator.position.bytes;
+
+      // The end of this node must extend far enough forward to touch
+      // the end of the range
+      if (node_end < range_end) continue;
+
+      // ...and exceed the start of the range, unless the node itself is
+      // empty, in which case it must at least be equal to the start of the range.
+      bool is_empty = ts_node_start_byte(child) == node_end;
+      if (is_empty ? node_end < range_start : node_end <= range_start) continue;
+
+      // The start of this node must extend far enough backward to
+      // touch the start of the range.
+      if (range_start < ts_node_start_byte(child)) break;
+
+      node = child;
+      if (ts_node__is_relevant(node, include_anonymous)) {
+        last_visible_node = node;
+      }
+      did_descend = true;
+      break;
+    }
+  }
+
+  return last_visible_node;
+}
+
+static inline TSNode ts_node__descendant_for_point_range(
+  TSNode self,
+  TSPoint range_start,
+  TSPoint range_end,
+  bool include_anonymous
+) {
+  if (point_gt(range_start, range_end)) {
+    return ts_node__null();
+  }
+  TSNode node = self;
+  TSNode last_visible_node = self;
+
+  bool did_descend = true;
+  while (did_descend) {
+    did_descend = false;
+
+    TSNode child;
+    NodeChildIterator iterator = ts_node_iterate_children(&node);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      TSPoint node_end = iterator.position.extent;
+
+      // The end of this node must extend far enough forward to touch
+      // the end of the range
+      if (point_lt(node_end, range_end)) continue;
+
+      // ...and exceed the start of the range, unless the node itself is
+      // empty, in which case it must at least be equal to the start of the range.
+      bool is_empty =  point_eq(ts_node_start_point(child), node_end);
+      if (is_empty ? point_lt(node_end, range_start) : point_lte(node_end, range_start)) {
+        continue;
+      }
+
+      // The start of this node must extend far enough backward to
+      // touch the start of the range.
+      if (point_lt(range_start, ts_node_start_point(child))) break;
+
+      node = child;
+      if (ts_node__is_relevant(node, include_anonymous)) {
+        last_visible_node = node;
+      }
+      did_descend = true;
+      break;
+    }
+  }
+
+  return last_visible_node;
+}
+
+// TSNode - public
+
+uint32_t ts_node_end_byte(TSNode self) {
+  return ts_node_start_byte(self) + ts_subtree_size(ts_node__subtree(self)).bytes;
+}
+
+TSPoint ts_node_end_point(TSNode self) {
+  return point_add(ts_node_start_point(self), ts_subtree_size(ts_node__subtree(self)).extent);
+}
+
+TSSymbol ts_node_symbol(TSNode self) {
+  TSSymbol symbol = ts_node__alias(&self);
+  if (!symbol) symbol = ts_subtree_symbol(ts_node__subtree(self));
+  return ts_language_public_symbol(self.tree->language, symbol);
+}
+
+const char *ts_node_type(TSNode self) {
+  TSSymbol symbol = ts_node__alias(&self);
+  if (!symbol) symbol = ts_subtree_symbol(ts_node__subtree(self));
+  return ts_language_symbol_name(self.tree->language, symbol);
+}
+
+const TSLanguage *ts_node_language(TSNode self) {
+  return self.tree->language;
+}
+
+TSSymbol ts_node_grammar_symbol(TSNode self) {
+  return ts_subtree_symbol(ts_node__subtree(self));
+}
+
+const char *ts_node_grammar_type(TSNode self) {
+  TSSymbol symbol = ts_subtree_symbol(ts_node__subtree(self));
+  return ts_language_symbol_name(self.tree->language, symbol);
+}
+
+char *ts_node_string(TSNode self) {
+  TSSymbol alias_symbol = ts_node__alias(&self);
+  return ts_subtree_string(
+    ts_node__subtree(self),
+    alias_symbol,
+    ts_language_symbol_metadata(self.tree->language, alias_symbol).visible,
+    self.tree->language,
+    false
+  );
+}
+
+bool ts_node_eq(TSNode self, TSNode other) {
+  return self.tree == other.tree && self.id == other.id;
+}
+
+bool ts_node_is_null(TSNode self) {
+  return self.id == 0;
+}
+
+bool ts_node_is_extra(TSNode self) {
+  return ts_subtree_extra(ts_node__subtree(self));
+}
+
+bool ts_node_is_named(TSNode self) {
+  TSSymbol alias = ts_node__alias(&self);
+  return alias
+    ? ts_language_symbol_metadata(self.tree->language, alias).named
+    : ts_subtree_named(ts_node__subtree(self));
+}
+
+bool ts_node_is_missing(TSNode self) {
+  return ts_subtree_missing(ts_node__subtree(self));
+}
+
+bool ts_node_has_changes(TSNode self) {
+  return ts_subtree_has_changes(ts_node__subtree(self));
+}
+
+bool ts_node_has_error(TSNode self) {
+  return ts_subtree_error_cost(ts_node__subtree(self)) > 0;
+}
+
+bool ts_node_is_error(TSNode self) {
+  TSSymbol symbol = ts_node_symbol(self);
+  return symbol == ts_builtin_sym_error;
+}
+
+uint32_t ts_node_descendant_count(TSNode self) {
+  return ts_subtree_visible_descendant_count(ts_node__subtree(self)) + 1;
+}
+
+TSStateId ts_node_parse_state(TSNode self) {
+  return ts_subtree_parse_state(ts_node__subtree(self));
+}
+
+TSStateId ts_node_next_parse_state(TSNode self) {
+  const TSLanguage *language = self.tree->language;
+  uint16_t state = ts_node_parse_state(self);
+  if (state == TS_TREE_STATE_NONE) {
+    return TS_TREE_STATE_NONE;
+  }
+  uint16_t symbol = ts_node_grammar_symbol(self);
+  return ts_language_next_state(language, state, symbol);
+}
+
+TSNode ts_node_parent(TSNode self) {
+  TSNode node = ts_tree_root_node(self.tree);
+  if (node.id == self.id) return ts_node__null();
+
+  while (true) {
+    TSNode next_node = ts_node_child_with_descendant(node, self);
+    if (next_node.id == self.id || ts_node_is_null(next_node)) break;
+    node = next_node;
+  }
+
+  return node;
+}
+
+TSNode ts_node_child_with_descendant(TSNode self, TSNode descendant) {
+  uint32_t start_byte = ts_node_start_byte(descendant);
+  uint32_t end_byte = ts_node_end_byte(descendant);
+  bool is_empty = start_byte == end_byte;
+
+  do {
+    NodeChildIterator iter = ts_node_iterate_children(&self);
+    do {
+      if (
+        !ts_node_child_iterator_next(&iter, &self)
+        || ts_node_start_byte(self) > start_byte
+      ) {
+        return ts_node__null();
+      }
+      if (self.id == descendant.id) {
+        return self;
+      }
+
+      // If the descendant is empty, and the end byte is within `self`,
+      // we check whether `self` contains it or not.
+      if (is_empty && iter.position.bytes >= end_byte && ts_node_child_count(self) > 0) {
+        TSNode child = ts_node_child_with_descendant(self, descendant);
+        // If the child is not null, return self if it's relevant, else return the child
+        if (!ts_node_is_null(child)) {
+          return ts_node__is_relevant(self, true) ? self : child;
+        }
+      }
+    } while ((is_empty ? iter.position.bytes <= end_byte : iter.position.bytes < end_byte) || ts_node_child_count(self) == 0);
+  } while (!ts_node__is_relevant(self, true));
+
+  return self;
+}
+
+TSNode ts_node_child(TSNode self, uint32_t child_index) {
+  return ts_node__child(self, child_index, true);
+}
+
+TSNode ts_node_named_child(TSNode self, uint32_t child_index) {
+  return ts_node__child(self, child_index, false);
+}
+
+TSNode ts_node_child_by_field_id(TSNode self, TSFieldId field_id) {
+recur:
+  if (!field_id || ts_node_child_count(self) == 0) return ts_node__null();
+
+  const TSFieldMapEntry *field_map, *field_map_end;
+  ts_language_field_map(
+    self.tree->language,
+    ts_node__subtree(self).ptr->production_id,
+    &field_map,
+    &field_map_end
+  );
+  if (field_map == field_map_end) return ts_node__null();
+
+  // The field mappings are sorted by their field id. Scan all
+  // the mappings to find the ones for the given field id.
+  while (field_map->field_id < field_id) {
+    field_map++;
+    if (field_map == field_map_end) return ts_node__null();
+  }
+  while (field_map_end[-1].field_id > field_id) {
+    field_map_end--;
+    if (field_map == field_map_end) return ts_node__null();
+  }
+
+  TSNode child;
+  NodeChildIterator iterator = ts_node_iterate_children(&self);
+  while (ts_node_child_iterator_next(&iterator, &child)) {
+    if (!ts_subtree_extra(ts_node__subtree(child))) {
+      uint32_t index = iterator.structural_child_index - 1;
+      if (index < field_map->child_index) continue;
+
+      // Hidden nodes' fields are "inherited" by their visible parent.
+      if (field_map->inherited) {
+
+        // If this is the *last* possible child node for this field,
+        // then perform a tail call to avoid recursion.
+        if (field_map + 1 == field_map_end) {
+          self = child;
+          goto recur;
+        }
+
+        // Otherwise, descend into this child, but if it doesn't contain
+        // the field, continue searching subsequent children.
+        else {
+          TSNode result = ts_node_child_by_field_id(child, field_id);
+          if (result.id) return result;
+          field_map++;
+          if (field_map == field_map_end) return ts_node__null();
+        }
+      }
+
+      else if (ts_node__is_relevant(child, true)) {
+        return child;
+      }
+
+      // If the field refers to a hidden node with visible children,
+      // return the first visible child.
+      else if (ts_node_child_count(child) > 0 ) {
+        return ts_node_child(child, 0);
+      }
+
+      // Otherwise, continue searching subsequent children.
+      else {
+        field_map++;
+        if (field_map == field_map_end) return ts_node__null();
+      }
+    }
+  }
+
+  return ts_node__null();
+}
+
+static inline const char *ts_node__field_name_from_language(TSNode self, uint32_t structural_child_index) {
+    const TSFieldMapEntry *field_map, *field_map_end;
+    ts_language_field_map(
+      self.tree->language,
+      ts_node__subtree(self).ptr->production_id,
+      &field_map,
+      &field_map_end
+    );
+    for (; field_map != field_map_end; field_map++) {
+      if (!field_map->inherited && field_map->child_index == structural_child_index) {
+        return self.tree->language->field_names[field_map->field_id];
+      }
+    }
+    return NULL;
+}
+
+const char *ts_node_field_name_for_child(TSNode self, uint32_t child_index) {
+  TSNode result = self;
+  bool did_descend = true;
+  const char *inherited_field_name = NULL;
+
+  while (did_descend) {
+    did_descend = false;
+
+    TSNode child;
+    uint32_t index = 0;
+    NodeChildIterator iterator = ts_node_iterate_children(&result);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      if (ts_node__is_relevant(child, true)) {
+        if (index == child_index) {
+          if (ts_node_is_extra(child)) {
+            return NULL;
+          }
+          const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
+          if (field_name) return field_name;
+          return inherited_field_name;
+        }
+        index++;
+      } else {
+        uint32_t grandchild_index = child_index - index;
+        uint32_t grandchild_count = ts_node__relevant_child_count(child, true);
+        if (grandchild_index < grandchild_count) {
+          const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
+          if (field_name) inherited_field_name = field_name;
+
+          did_descend = true;
+          result = child;
+          child_index = grandchild_index;
+          break;
+        }
+        index += grandchild_count;
+      }
+    }
+  }
+
+  return NULL;
+}
+
+const char *ts_node_field_name_for_named_child(TSNode self, uint32_t named_child_index) {
+  TSNode result = self;
+  bool did_descend = true;
+  const char *inherited_field_name = NULL;
+
+  while (did_descend) {
+    did_descend = false;
+
+    TSNode child;
+    uint32_t index = 0;
+    NodeChildIterator iterator = ts_node_iterate_children(&result);
+    while (ts_node_child_iterator_next(&iterator, &child)) {
+      if (ts_node__is_relevant(child, false)) {
+        if (index == named_child_index) {
+          if (ts_node_is_extra(child)) {
+            return NULL;
+          }
+          const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
+          if (field_name) return field_name;
+          return inherited_field_name;
+        }
+        index++;
+      } else {
+        uint32_t named_grandchild_index = named_child_index - index;
+        uint32_t grandchild_count = ts_node__relevant_child_count(child, false);
+        if (named_grandchild_index < grandchild_count) {
+          const char *field_name = ts_node__field_name_from_language(result, iterator.structural_child_index - 1);
+          if (field_name) inherited_field_name = field_name;
+
+          did_descend = true;
+          result = child;
+          named_child_index = named_grandchild_index;
+          break;
+        }
+        index += grandchild_count;
+      }
+    }
+  }
+
+  return NULL;
+}
+
+TSNode ts_node_child_by_field_name(
+  TSNode self,
+  const char *name,
+  uint32_t name_length
+) {
+  TSFieldId field_id = ts_language_field_id_for_name(
+    self.tree->language,
+    name,
+    name_length
+  );
+  return ts_node_child_by_field_id(self, field_id);
+}
+
+uint32_t ts_node_child_count(TSNode self) {
+  Subtree tree = ts_node__subtree(self);
+  if (ts_subtree_child_count(tree) > 0) {
+    return tree.ptr->visible_child_count;
+  } else {
+    return 0;
+  }
+}
+
+uint32_t ts_node_named_child_count(TSNode self) {
+  Subtree tree = ts_node__subtree(self);
+  if (ts_subtree_child_count(tree) > 0) {
+    return tree.ptr->named_child_count;
+  } else {
+    return 0;
+  }
+}
+
+TSNode ts_node_next_sibling(TSNode self) {
+  return ts_node__next_sibling(self, true);
+}
+
+TSNode ts_node_next_named_sibling(TSNode self) {
+  return ts_node__next_sibling(self, false);
+}
+
+TSNode ts_node_prev_sibling(TSNode self) {
+  return ts_node__prev_sibling(self, true);
+}
+
+TSNode ts_node_prev_named_sibling(TSNode self) {
+  return ts_node__prev_sibling(self, false);
+}
+
+TSNode ts_node_first_child_for_byte(TSNode self, uint32_t byte) {
+  return ts_node__first_child_for_byte(self, byte, true);
+}
+
+TSNode ts_node_first_named_child_for_byte(TSNode self, uint32_t byte) {
+  return ts_node__first_child_for_byte(self, byte, false);
+}
+
+TSNode ts_node_descendant_for_byte_range(
+  TSNode self,
+  uint32_t start,
+  uint32_t end
+) {
+  return ts_node__descendant_for_byte_range(self, start, end, true);
+}
+
+TSNode ts_node_named_descendant_for_byte_range(
+  TSNode self,
+  uint32_t start,
+  uint32_t end
+) {
+  return ts_node__descendant_for_byte_range(self, start, end, false);
+}
+
+TSNode ts_node_descendant_for_point_range(
+  TSNode self,
+  TSPoint start,
+  TSPoint end
+) {
+  return ts_node__descendant_for_point_range(self, start, end, true);
+}
+
+TSNode ts_node_named_descendant_for_point_range(
+  TSNode self,
+  TSPoint start,
+  TSPoint end
+) {
+  return ts_node__descendant_for_point_range(self, start, end, false);
+}
+
+void ts_node_edit(TSNode *self, const TSInputEdit *edit) {
+  uint32_t start_byte = ts_node_start_byte(*self);
+  TSPoint start_point = ts_node_start_point(*self);
+
+  if (start_byte >= edit->old_end_byte) {
+    start_byte = edit->new_end_byte + (start_byte - edit->old_end_byte);
+    start_point = point_add(edit->new_end_point, point_sub(start_point, edit->old_end_point));
+  } else if (start_byte > edit->start_byte) {
+    start_byte = edit->new_end_byte;
+    start_point = edit->new_end_point;
+  }
+
+  self->context[0] = start_byte;
+  self->context[1] = start_point.row;
+  self->context[2] = start_point.column;
+}

third_party/tree-sitter/lib/src/parser.h

@@ -0,0 +1,286 @@
+#ifndef TREE_SITTER_PARSER_H_
+#define TREE_SITTER_PARSER_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <stdbool.h>
+#include <stdint.h>
+#include <stdlib.h>
+
+#define ts_builtin_sym_error ((TSSymbol)-1)
+#define ts_builtin_sym_end 0
+#define TREE_SITTER_SERIALIZATION_BUFFER_SIZE 1024
+
+#ifndef TREE_SITTER_API_H_
+typedef uint16_t TSStateId;
+typedef uint16_t TSSymbol;
+typedef uint16_t TSFieldId;
+typedef struct TSLanguage TSLanguage;
+typedef struct TSLanguageMetadata {
+  uint8_t major_version;
+  uint8_t minor_version;
+  uint8_t patch_version;
+} TSLanguageMetadata;
+#endif
+
+typedef struct {
+  TSFieldId field_id;
+  uint8_t child_index;
+  bool inherited;
+} TSFieldMapEntry;
+
+// Used to index the field and supertype maps.
+typedef struct {
+  uint16_t index;
+  uint16_t length;
+} TSMapSlice;
+
+typedef struct {
+  bool visible;
+  bool named;
+  bool supertype;
+} TSSymbolMetadata;
+
+typedef struct TSLexer TSLexer;
+
+struct TSLexer {
+  int32_t lookahead;
+  TSSymbol result_symbol;
+  void (*advance)(TSLexer *, bool);
+  void (*mark_end)(TSLexer *);
+  uint32_t (*get_column)(TSLexer *);
+  bool (*is_at_included_range_start)(const TSLexer *);
+  bool (*eof)(const TSLexer *);
+  void (*log)(const TSLexer *, const char *, ...);
+};
+
+typedef enum {
+  TSParseActionTypeShift,
+  TSParseActionTypeReduce,
+  TSParseActionTypeAccept,
+  TSParseActionTypeRecover,
+} TSParseActionType;
+
+typedef union {
+  struct {
+    uint8_t type;
+    TSStateId state;
+    bool extra;
+    bool repetition;
+  } shift;
+  struct {
+    uint8_t type;
+    uint8_t child_count;
+    TSSymbol symbol;
+    int16_t dynamic_precedence;
+    uint16_t production_id;
+  } reduce;
+  uint8_t type;
+} TSParseAction;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+} TSLexMode;
+
+typedef struct {
+  uint16_t lex_state;
+  uint16_t external_lex_state;
+  uint16_t reserved_word_set_id;
+} TSLexerMode;
+
+typedef union {
+  TSParseAction action;
+  struct {
+    uint8_t count;
+    bool reusable;
+  } entry;
+} TSParseActionEntry;
+
+typedef struct {
+  int32_t start;
+  int32_t end;
+} TSCharacterRange;
+
+struct TSLanguage {
+  uint32_t abi_version;
+  uint32_t symbol_count;
+  uint32_t alias_count;
+  uint32_t token_count;
+  uint32_t external_token_count;
+  uint32_t state_count;
+  uint32_t large_state_count;
+  uint32_t production_id_count;
+  uint32_t field_count;
+  uint16_t max_alias_sequence_length;
+  const uint16_t *parse_table;
+  const uint16_t *small_parse_table;
+  const uint32_t *small_parse_table_map;
+  const TSParseActionEntry *parse_actions;
+  const char * const *symbol_names;
+  const char * const *field_names;
+  const TSMapSlice *field_map_slices;
+  const TSFieldMapEntry *field_map_entries;
+  const TSSymbolMetadata *symbol_metadata;
+  const TSSymbol *public_symbol_map;
+  const uint16_t *alias_map;
+  const TSSymbol *alias_sequences;
+  const TSLexerMode *lex_modes;
+  bool (*lex_fn)(TSLexer *, TSStateId);
+  bool (*keyword_lex_fn)(TSLexer *, TSStateId);
+  TSSymbol keyword_capture_token;
+  struct {
+    const bool *states;
+    const TSSymbol *symbol_map;
+    void *(*create)(void);
+    void (*destroy)(void *);
+    bool (*scan)(void *, TSLexer *, const bool *symbol_whitelist);
+    unsigned (*serialize)(void *, char *);
+    void (*deserialize)(void *, const char *, unsigned);
+  } external_scanner;
+  const TSStateId *primary_state_ids;
+  const char *name;
+  const TSSymbol *reserved_words;
+  uint16_t max_reserved_word_set_size;
+  uint32_t supertype_count;
+  const TSSymbol *supertype_symbols;
+  const TSMapSlice *supertype_map_slices;
+  const TSSymbol *supertype_map_entries;
+  TSLanguageMetadata metadata;
+};
+
+static inline bool set_contains(const TSCharacterRange *ranges, uint32_t len, int32_t lookahead) {
+  uint32_t index = 0;
+  uint32_t size = len - index;
+  while (size > 1) {
+    uint32_t half_size = size / 2;
+    uint32_t mid_index = index + half_size;
+    const TSCharacterRange *range = &ranges[mid_index];
+    if (lookahead >= range->start && lookahead <= range->end) {
+      return true;
+    } else if (lookahead > range->end) {
+      index = mid_index;
+    }
+    size -= half_size;
+  }
+  const TSCharacterRange *range = &ranges[index];
+  return (lookahead >= range->start && lookahead <= range->end);
+}
+
+/*
+ *  Lexer Macros
+ */
+
+#ifdef _MSC_VER
+#define UNUSED __pragma(warning(suppress : 4101))
+#else
+#define UNUSED __attribute__((unused))
+#endif
+
+#define START_LEXER()           \
+  bool result = false;          \
+  bool skip = false;            \
+  UNUSED                        \
+  bool eof = false;             \
+  int32_t lookahead;            \
+  goto start;                   \
+  next_state:                   \
+  lexer->advance(lexer, skip);  \
+  start:                        \
+  skip = false;                 \
+  lookahead = lexer->lookahead;
+
+#define ADVANCE(state_value) \
+  {                          \
+    state = state_value;     \
+    goto next_state;         \
+  }
+
+#define ADVANCE_MAP(...)                                              \
+  {                                                                   \
+    static const uint16_t map[] = { __VA_ARGS__ };                    \
+    for (uint32_t i = 0; i < sizeof(map) / sizeof(map[0]); i += 2) {  \
+      if (map[i] == lookahead) {                                      \
+        state = map[i + 1];                                           \
+        goto next_state;                                              \
+      }                                                               \
+    }                                                                 \
+  }
+
+#define SKIP(state_value) \
+  {                       \
+    skip = true;          \
+    state = state_value;  \
+    goto next_state;      \
+  }
+
+#define ACCEPT_TOKEN(symbol_value)     \
+  result = true;                       \
+  lexer->result_symbol = symbol_value; \
+  lexer->mark_end(lexer);
+
+#define END_STATE() return result;
+
+/*
+ *  Parse Table Macros
+ */
+
+#define SMALL_STATE(id) ((id) - LARGE_STATE_COUNT)
+
+#define STATE(id) id
+
+#define ACTIONS(id) id
+
+#define SHIFT(state_value)            \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value)          \
+    }                                 \
+  }}
+
+#define SHIFT_REPEAT(state_value)     \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .state = (state_value),         \
+      .repetition = true              \
+    }                                 \
+  }}
+
+#define SHIFT_EXTRA()                 \
+  {{                                  \
+    .shift = {                        \
+      .type = TSParseActionTypeShift, \
+      .extra = true                   \
+    }                                 \
+  }}
+
+#define REDUCE(symbol_name, children, precedence, prod_id) \
+  {{                                                       \
+    .reduce = {                                            \
+      .type = TSParseActionTypeReduce,                     \
+      .symbol = symbol_name,                               \
+      .child_count = children,                             \
+      .dynamic_precedence = precedence,                    \
+      .production_id = prod_id                             \
+    },                                                     \
+  }}
+
+#define RECOVER()                    \
+  {{                                 \
+    .type = TSParseActionTypeRecover \
+  }}
+
+#define ACCEPT_INPUT()              \
+  {{                                \
+    .type = TSParseActionTypeAccept \
+  }}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSER_H_

third_party/tree-sitter/lib/src/point.h

@@ -0,0 +1,48 @@
+#ifndef TREE_SITTER_POINT_H_
+#define TREE_SITTER_POINT_H_
+
+#include "tree_sitter/api.h"
+
+#define POINT_ZERO ((TSPoint) {0, 0})
+#define POINT_MAX ((TSPoint) {UINT32_MAX, UINT32_MAX})
+
+static inline TSPoint point__new(unsigned row, unsigned column) {
+  TSPoint result = {row, column};
+  return result;
+}
+
+static inline TSPoint point_add(TSPoint a, TSPoint b) {
+  if (b.row > 0)
+    return point__new(a.row + b.row, b.column);
+  else
+    return point__new(a.row, a.column + b.column);
+}
+
+static inline TSPoint point_sub(TSPoint a, TSPoint b) {
+  if (a.row > b.row)
+    return point__new(a.row - b.row, a.column);
+  else
+    return point__new(0, (a.column >= b.column) ? a.column - b.column : 0);
+}
+
+static inline bool point_lte(TSPoint a, TSPoint b) {
+  return (a.row < b.row) || (a.row == b.row && a.column <= b.column);
+}
+
+static inline bool point_lt(TSPoint a, TSPoint b) {
+  return (a.row < b.row) || (a.row == b.row && a.column < b.column);
+}
+
+static inline bool point_gt(TSPoint a, TSPoint b) {
+  return (a.row > b.row) || (a.row == b.row && a.column > b.column);
+}
+
+static inline bool point_gte(TSPoint a, TSPoint b) {
+  return (a.row > b.row) || (a.row == b.row && a.column >= b.column);
+}
+
+static inline bool point_eq(TSPoint a, TSPoint b) {
+  return a.row == b.row && a.column == b.column;
+}
+
+#endif

third_party/tree-sitter/lib/src/portable/endian.h

@@ -0,0 +1,240 @@
+// "License": Public Domain
+// I, Mathias Panzenböck, place this file hereby into the public domain. Use it at your own risk for whatever you like.
+// In case there are jurisdictions that don't support putting things in the public domain you can also consider it to
+// be "dual licensed" under the BSD, MIT and Apache licenses, if you want to. This code is trivial anyway. Consider it
+// an example on how to get the endian conversion functions on different platforms.
+
+// updates from https://github.com/mikepb/endian.h/issues/4
+
+#ifndef ENDIAN_H
+#define ENDIAN_H
+
+#if (defined(_WIN16) || defined(_WIN32) || defined(_WIN64)) && !defined(__WINDOWS__)
+
+#    define __WINDOWS__
+
+#endif
+
+#if defined(HAVE_ENDIAN_H) || \
+    defined(__linux__) || \
+    defined(__GNU__) || \
+    defined(__illumos__) || \
+    defined(__NetBSD__) || \
+    defined(__OpenBSD__) || \
+    defined(__CYGWIN__) || \
+    defined(__MSYS__) || \
+    defined(__EMSCRIPTEN__) || \
+    defined(__wasi__) || \
+    defined(__wasm__)
+
+#if defined(__NetBSD__)
+#define _NETBSD_SOURCE 1
+#endif
+
+# include <endian.h>
+
+#elif defined(HAVE_SYS_ENDIAN_H) || \
+    defined(__FreeBSD__) || \
+    defined(__DragonFly__)
+
+# include <sys/endian.h>
+
+#elif defined(__APPLE__)
+#    define __BYTE_ORDER    BYTE_ORDER
+#    define __BIG_ENDIAN    BIG_ENDIAN
+#    define __LITTLE_ENDIAN LITTLE_ENDIAN
+#    define __PDP_ENDIAN    PDP_ENDIAN
+
+#    if !defined(_POSIX_C_SOURCE)
+#        include <libkern/OSByteOrder.h>
+
+#        define htobe16(x) OSSwapHostToBigInt16(x)
+#        define htole16(x) OSSwapHostToLittleInt16(x)
+#        define be16toh(x) OSSwapBigToHostInt16(x)
+#        define le16toh(x) OSSwapLittleToHostInt16(x)
+
+#        define htobe32(x) OSSwapHostToBigInt32(x)
+#        define htole32(x) OSSwapHostToLittleInt32(x)
+#        define be32toh(x) OSSwapBigToHostInt32(x)
+#        define le32toh(x) OSSwapLittleToHostInt32(x)
+
+#        define htobe64(x) OSSwapHostToBigInt64(x)
+#        define htole64(x) OSSwapHostToLittleInt64(x)
+#        define be64toh(x) OSSwapBigToHostInt64(x)
+#        define le64toh(x) OSSwapLittleToHostInt64(x)
+#    else
+#        if BYTE_ORDER == LITTLE_ENDIAN
+#            define htobe16(x) __builtin_bswap16(x)
+#            define htole16(x) (x)
+#            define be16toh(x) __builtin_bswap16(x)
+#            define le16toh(x) (x)
+
+#            define htobe32(x) __builtin_bswap32(x)
+#            define htole32(x) (x)
+#            define be32toh(x) __builtin_bswap32(x)
+#            define le32toh(x) (x)
+
+#            define htobe64(x) __builtin_bswap64(x)
+#            define htole64(x) (x)
+#            define be64toh(x) __builtin_bswap64(x)
+#            define le64toh(x) (x)
+#        elif BYTE_ORDER == BIG_ENDIAN
+#            define htobe16(x) (x)
+#            define htole16(x) __builtin_bswap16(x)
+#            define be16toh(x) (x)
+#            define le16toh(x) __builtin_bswap16(x)
+
+#            define htobe32(x) (x)
+#            define htole32(x) __builtin_bswap32(x)
+#            define be32toh(x) (x)
+#            define le32toh(x) __builtin_bswap32(x)
+
+#            define htobe64(x) (x)
+#            define htole64(x) __builtin_bswap64(x)
+#            define be64toh(x) (x)
+#            define le64toh(x) __builtin_bswap64(x)
+#        else
+#            error byte order not supported
+#        endif
+#    endif
+
+#elif defined(__WINDOWS__)
+
+#    if defined(_MSC_VER) && !defined(__clang__)
+#        include <stdlib.h>
+#        define B_SWAP_16(x) _byteswap_ushort(x)
+#        define B_SWAP_32(x) _byteswap_ulong(x)
+#        define B_SWAP_64(x) _byteswap_uint64(x)
+#    else
+#        define B_SWAP_16(x) __builtin_bswap16(x)
+#        define B_SWAP_32(x) __builtin_bswap32(x)
+#        define B_SWAP_64(x) __builtin_bswap64(x)
+#    endif
+
+# if defined(__MINGW32__) || defined(HAVE_SYS_PARAM_H)
+#   include <sys/param.h>
+# endif
+
+#    ifndef BIG_ENDIAN
+#        ifdef __BIG_ENDIAN
+#            define BIG_ENDIAN __BIG_ENDIAN
+#        elif defined(__ORDER_BIG_ENDIAN__)
+#            define BIG_ENDIAN __ORDER_BIG_ENDIAN__
+#        else
+#            define BIG_ENDIAN 4321
+#        endif
+#    endif
+
+#    ifndef LITTLE_ENDIAN
+#        ifdef __LITTLE_ENDIAN
+#            define LITTLE_ENDIAN __LITTLE_ENDIAN
+#        elif defined(__ORDER_LITTLE_ENDIAN__)
+#            define LITTLE_ENDIAN __ORDER_LITTLE_ENDIAN__
+#        else
+#            define LITTLE_ENDIAN 1234
+#        endif
+#    endif
+
+#    ifndef BYTE_ORDER
+#        ifdef __BYTE_ORDER
+#            define BYTE_ORDER __BYTE_ORDER
+#        elif defined(__BYTE_ORDER__)
+#            define BYTE_ORDER __BYTE_ORDER__
+#        else
+             /* assume LE on Windows if nothing was defined */
+#            define BYTE_ORDER LITTLE_ENDIAN
+#        endif
+#    endif
+
+#    if BYTE_ORDER == LITTLE_ENDIAN
+
+#        define htobe16(x) B_SWAP_16(x)
+#        define htole16(x) (x)
+#        define be16toh(x) B_SWAP_16(x)
+#        define le16toh(x) (x)
+
+#        define htobe32(x) B_SWAP_32(x)
+#        define htole32(x) (x)
+#        define be32toh(x) B_SWAP_32(x)
+#        define le32toh(x) (x)
+
+#        define htobe64(x) B_SWAP_64(x)
+#        define htole64(x) (x)
+#        define be64toh(x) B_SWAP_64(x)
+#        define le64toh(x) (x)
+
+#    elif BYTE_ORDER == BIG_ENDIAN
+
+#        define htobe16(x) (x)
+#        define htole16(x) B_SWAP_16(x)
+#        define be16toh(x) (x)
+#        define le16toh(x) B_SWAP_16(x)
+
+#        define htobe32(x) (x)
+#        define htole32(x) B_SWAP_32(x)
+#        define be32toh(x) (x)
+#        define le32toh(x) B_SWAP_32(x)
+
+#        define htobe64(x) (x)
+#        define htole64(x) B_SWAP_64(x)
+#        define be64toh(x) (x)
+#        define le64toh(x) B_SWAP_64(x)
+
+#    else
+
+#        error byte order not supported
+
+#    endif
+
+#elif defined(__QNXNTO__)
+
+#    include <gulliver.h>
+
+#    define __LITTLE_ENDIAN 1234
+#    define __BIG_ENDIAN    4321
+#    define __PDP_ENDIAN    3412
+
+#    if defined(__BIGENDIAN__)
+
+#        define __BYTE_ORDER __BIG_ENDIAN
+
+#        define htobe16(x) (x)
+#        define htobe32(x) (x)
+#        define htobe64(x) (x)
+
+#        define htole16(x) ENDIAN_SWAP16(x)
+#        define htole32(x) ENDIAN_SWAP32(x)
+#        define htole64(x) ENDIAN_SWAP64(x)
+
+#    elif defined(__LITTLEENDIAN__)
+
+#        define __BYTE_ORDER __LITTLE_ENDIAN
+
+#        define htole16(x) (x)
+#        define htole32(x) (x)
+#        define htole64(x) (x)
+
+#        define htobe16(x) ENDIAN_SWAP16(x)
+#        define htobe32(x) ENDIAN_SWAP32(x)
+#        define htobe64(x) ENDIAN_SWAP64(x)
+
+#    else
+
+#        error byte order not supported
+
+#    endif
+
+#    define be16toh(x) ENDIAN_BE16(x)
+#    define be32toh(x) ENDIAN_BE32(x)
+#    define be64toh(x) ENDIAN_BE64(x)
+#    define le16toh(x) ENDIAN_LE16(x)
+#    define le32toh(x) ENDIAN_LE32(x)
+#    define le64toh(x) ENDIAN_LE64(x)
+
+#else
+
+#    error platform not supported
+
+#endif
+
+#endif

third_party/tree-sitter/lib/src/reduce_action.h

@@ -0,0 +1,34 @@
+#ifndef TREE_SITTER_REDUCE_ACTION_H_
+#define TREE_SITTER_REDUCE_ACTION_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./array.h"
+#include "tree_sitter/api.h"
+
+typedef struct {
+  uint32_t count;
+  TSSymbol symbol;
+  int dynamic_precedence;
+  unsigned short production_id;
+} ReduceAction;
+
+typedef Array(ReduceAction) ReduceActionSet;
+
+static inline void ts_reduce_action_set_add(ReduceActionSet *self,
+                                            ReduceAction new_action) {
+  for (uint32_t i = 0; i < self->size; i++) {
+    ReduceAction action = self->contents[i];
+    if (action.symbol == new_action.symbol && action.count == new_action.count)
+      return;
+  }
+  array_push(self, new_action);
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_REDUCE_ACTION_H_

third_party/tree-sitter/lib/src/reusable_node.h

@@ -0,0 +1,95 @@
+#include "./subtree.h"
+
+typedef struct {
+  Subtree tree;
+  uint32_t child_index;
+  uint32_t byte_offset;
+} StackEntry;
+
+typedef struct {
+  Array(StackEntry) stack;
+  Subtree last_external_token;
+} ReusableNode;
+
+static inline ReusableNode reusable_node_new(void) {
+  return (ReusableNode) {array_new(), NULL_SUBTREE};
+}
+
+static inline void reusable_node_clear(ReusableNode *self) {
+  array_clear(&self->stack);
+  self->last_external_token = NULL_SUBTREE;
+}
+
+static inline Subtree reusable_node_tree(ReusableNode *self) {
+  return self->stack.size > 0
+    ? self->stack.contents[self->stack.size - 1].tree
+    : NULL_SUBTREE;
+}
+
+static inline uint32_t reusable_node_byte_offset(ReusableNode *self) {
+  return self->stack.size > 0
+    ? self->stack.contents[self->stack.size - 1].byte_offset
+    : UINT32_MAX;
+}
+
+static inline void reusable_node_delete(ReusableNode *self) {
+  array_delete(&self->stack);
+}
+
+static inline void reusable_node_advance(ReusableNode *self) {
+  StackEntry last_entry = *array_back(&self->stack);
+  uint32_t byte_offset = last_entry.byte_offset + ts_subtree_total_bytes(last_entry.tree);
+  if (ts_subtree_has_external_tokens(last_entry.tree)) {
+    self->last_external_token = ts_subtree_last_external_token(last_entry.tree);
+  }
+
+  Subtree tree;
+  uint32_t next_index;
+  do {
+    StackEntry popped_entry = array_pop(&self->stack);
+    next_index = popped_entry.child_index + 1;
+    if (self->stack.size == 0) return;
+    tree = array_back(&self->stack)->tree;
+  } while (ts_subtree_child_count(tree) <= next_index);
+
+  array_push(&self->stack, ((StackEntry) {
+    .tree = ts_subtree_children(tree)[next_index],
+    .child_index = next_index,
+    .byte_offset = byte_offset,
+  }));
+}
+
+static inline bool reusable_node_descend(ReusableNode *self) {
+  StackEntry last_entry = *array_back(&self->stack);
+  if (ts_subtree_child_count(last_entry.tree) > 0) {
+    array_push(&self->stack, ((StackEntry) {
+      .tree = ts_subtree_children(last_entry.tree)[0],
+      .child_index = 0,
+      .byte_offset = last_entry.byte_offset,
+    }));
+    return true;
+  } else {
+    return false;
+  }
+}
+
+static inline void reusable_node_advance_past_leaf(ReusableNode *self) {
+  while (reusable_node_descend(self)) {}
+  reusable_node_advance(self);
+}
+
+static inline void reusable_node_reset(ReusableNode *self, Subtree tree) {
+  reusable_node_clear(self);
+  array_push(&self->stack, ((StackEntry) {
+    .tree = tree,
+    .child_index = 0,
+    .byte_offset = 0,
+  }));
+
+  // Never reuse the root node, because it has a non-standard internal structure
+  // due to transformations that are applied when it is accepted: adding the EOF
+  // child and any extra children.
+  if (!reusable_node_descend(self)) {
+    reusable_node_clear(self);
+  }
+}

third_party/tree-sitter/lib/src/stack.c

@@ -0,0 +1,912 @@
+#include "./alloc.h"
+#include "./language.h"
+#include "./subtree.h"
+#include "./array.h"
+#include "./stack.h"
+#include "./length.h"
+#include <assert.h>
+#include <inttypes.h>
+#include <stdio.h>
+
+#define MAX_LINK_COUNT 8
+#define MAX_NODE_POOL_SIZE 50
+#define MAX_ITERATOR_COUNT 64
+
+#if defined _WIN32 && !defined __GNUC__
+#define forceinline __forceinline
+#else
+#define forceinline static inline __attribute__((always_inline))
+#endif
+
+typedef struct StackNode StackNode;
+
+typedef struct {
+  StackNode *node;
+  Subtree subtree;
+  bool is_pending;
+} StackLink;
+
+struct StackNode {
+  TSStateId state;
+  Length position;
+  StackLink links[MAX_LINK_COUNT];
+  short unsigned int link_count;
+  uint32_t ref_count;
+  unsigned error_cost;
+  unsigned node_count;
+  int dynamic_precedence;
+};
+
+typedef struct {
+  StackNode *node;
+  SubtreeArray subtrees;
+  uint32_t subtree_count;
+  bool is_pending;
+} StackIterator;
+
+typedef Array(StackNode *) StackNodeArray;
+
+typedef enum {
+  StackStatusActive,
+  StackStatusPaused,
+  StackStatusHalted,
+} StackStatus;
+
+typedef struct {
+  StackNode *node;
+  StackSummary *summary;
+  unsigned node_count_at_last_error;
+  Subtree last_external_token;
+  Subtree lookahead_when_paused;
+  StackStatus status;
+} StackHead;
+
+struct Stack {
+  Array(StackHead) heads;
+  StackSliceArray slices;
+  Array(StackIterator) iterators;
+  StackNodeArray node_pool;
+  StackNode *base_node;
+  SubtreePool *subtree_pool;
+};
+
+typedef unsigned StackAction;
+enum {
+  StackActionNone,
+  StackActionStop = 1,
+  StackActionPop = 2,
+};
+
+typedef StackAction (*StackCallback)(void *, const StackIterator *);
+
+static void stack_node_retain(StackNode *self) {
+  if (!self)
+    return;
+  ts_assert(self->ref_count > 0);
+  self->ref_count++;
+  ts_assert(self->ref_count != 0);
+}
+
+static void stack_node_release(
+  StackNode *self,
+  StackNodeArray *pool,
+  SubtreePool *subtree_pool
+) {
+recur:
+  ts_assert(self->ref_count != 0);
+  self->ref_count--;
+  if (self->ref_count > 0) return;
+
+  StackNode *first_predecessor = NULL;
+  if (self->link_count > 0) {
+    for (unsigned i = self->link_count - 1; i > 0; i--) {
+      StackLink link = self->links[i];
+      if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree);
+      stack_node_release(link.node, pool, subtree_pool);
+    }
+    StackLink link = self->links[0];
+    if (link.subtree.ptr) ts_subtree_release(subtree_pool, link.subtree);
+    first_predecessor = self->links[0].node;
+  }
+
+  if (pool->size < MAX_NODE_POOL_SIZE) {
+    array_push(pool, self);
+  } else {
+    ts_free(self);
+  }
+
+  if (first_predecessor) {
+    self = first_predecessor;
+    goto recur;
+  }
+}
+
+/// Get the number of nodes in the subtree, for the purpose of measuring
+/// how much progress has been made by a given version of the stack.
+static uint32_t stack__subtree_node_count(Subtree subtree) {
+  uint32_t count = ts_subtree_visible_descendant_count(subtree);
+  if (ts_subtree_visible(subtree)) count++;
+
+  // Count intermediate error nodes even though they are not visible,
+  // because a stack version's node count is used to check whether it
+  // has made any progress since the last time it encountered an error.
+  if (ts_subtree_symbol(subtree) == ts_builtin_sym_error_repeat) count++;
+
+  return count;
+}
+
+static StackNode *stack_node_new(
+  StackNode *previous_node,
+  Subtree subtree,
+  bool is_pending,
+  TSStateId state,
+  StackNodeArray *pool
+) {
+  StackNode *node = pool->size > 0
+    ? array_pop(pool)
+    : ts_malloc(sizeof(StackNode));
+  *node = (StackNode) {
+    .ref_count = 1,
+    .link_count = 0,
+    .state = state
+  };
+
+  if (previous_node) {
+    node->link_count = 1;
+    node->links[0] = (StackLink) {
+      .node = previous_node,
+      .subtree = subtree,
+      .is_pending = is_pending,
+    };
+
+    node->position = previous_node->position;
+    node->error_cost = previous_node->error_cost;
+    node->dynamic_precedence = previous_node->dynamic_precedence;
+    node->node_count = previous_node->node_count;
+
+    if (subtree.ptr) {
+      node->error_cost += ts_subtree_error_cost(subtree);
+      node->position = length_add(node->position, ts_subtree_total_size(subtree));
+      node->node_count += stack__subtree_node_count(subtree);
+      node->dynamic_precedence += ts_subtree_dynamic_precedence(subtree);
+    }
+  } else {
+    node->position = length_zero();
+    node->error_cost = 0;
+  }
+
+  return node;
+}
+
+static bool stack__subtree_is_equivalent(Subtree left, Subtree right) {
+  if (left.ptr == right.ptr) return true;
+  if (!left.ptr || !right.ptr) return false;
+
+  // Symbols must match
+  if (ts_subtree_symbol(left) != ts_subtree_symbol(right)) return false;
+
+  // If both have errors, don't bother keeping both.
+  if (ts_subtree_error_cost(left) > 0 && ts_subtree_error_cost(right) > 0) return true;
+
+  return (
+    ts_subtree_padding(left).bytes == ts_subtree_padding(right).bytes &&
+    ts_subtree_size(left).bytes == ts_subtree_size(right).bytes &&
+    ts_subtree_child_count(left) == ts_subtree_child_count(right) &&
+    ts_subtree_extra(left) == ts_subtree_extra(right) &&
+    ts_subtree_external_scanner_state_eq(left, right)
+  );
+}
+
+static void stack_node_add_link(
+  StackNode *self,
+  StackLink link,
+  SubtreePool *subtree_pool
+) {
+  if (link.node == self) return;
+
+  for (int i = 0; i < self->link_count; i++) {
+    StackLink *existing_link = &self->links[i];
+    if (stack__subtree_is_equivalent(existing_link->subtree, link.subtree)) {
+      // In general, we preserve ambiguities until they are removed from the stack
+      // during a pop operation where multiple paths lead to the same node. But in
+      // the special case where two links directly connect the same pair of nodes,
+      // we can safely remove the ambiguity ahead of time without changing behavior.
+      if (existing_link->node == link.node) {
+        if (
+          ts_subtree_dynamic_precedence(link.subtree) >
+          ts_subtree_dynamic_precedence(existing_link->subtree)
+        ) {
+          ts_subtree_retain(link.subtree);
+          ts_subtree_release(subtree_pool, existing_link->subtree);
+          existing_link->subtree = link.subtree;
+          self->dynamic_precedence =
+            link.node->dynamic_precedence + ts_subtree_dynamic_precedence(link.subtree);
+        }
+        return;
+      }
+
+      // If the previous nodes are mergeable, merge them recursively.
+      if (
+        existing_link->node->state == link.node->state &&
+        existing_link->node->position.bytes == link.node->position.bytes &&
+        existing_link->node->error_cost == link.node->error_cost
+      ) {
+        for (int j = 0; j < link.node->link_count; j++) {
+          stack_node_add_link(existing_link->node, link.node->links[j], subtree_pool);
+        }
+        int32_t dynamic_precedence = link.node->dynamic_precedence;
+        if (link.subtree.ptr) {
+          dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
+        }
+        if (dynamic_precedence > self->dynamic_precedence) {
+          self->dynamic_precedence = dynamic_precedence;
+        }
+        return;
+      }
+    }
+  }
+
+  if (self->link_count == MAX_LINK_COUNT) return;
+
+  stack_node_retain(link.node);
+  unsigned node_count = link.node->node_count;
+  int dynamic_precedence = link.node->dynamic_precedence;
+  self->links[self->link_count++] = link;
+
+  if (link.subtree.ptr) {
+    ts_subtree_retain(link.subtree);
+    node_count += stack__subtree_node_count(link.subtree);
+    dynamic_precedence += ts_subtree_dynamic_precedence(link.subtree);
+  }
+
+  if (node_count > self->node_count) self->node_count = node_count;
+  if (dynamic_precedence > self->dynamic_precedence) self->dynamic_precedence = dynamic_precedence;
+}
+
+static void stack_head_delete(
+  StackHead *self,
+  StackNodeArray *pool,
+  SubtreePool *subtree_pool
+) {
+  if (self->node) {
+    if (self->last_external_token.ptr) {
+      ts_subtree_release(subtree_pool, self->last_external_token);
+    }
+    if (self->lookahead_when_paused.ptr) {
+      ts_subtree_release(subtree_pool, self->lookahead_when_paused);
+    }
+    if (self->summary) {
+      array_delete(self->summary);
+      ts_free(self->summary);
+    }
+    stack_node_release(self->node, pool, subtree_pool);
+  }
+}
+
+static StackVersion ts_stack__add_version(
+  Stack *self,
+  StackVersion original_version,
+  StackNode *node
+) {
+  StackHead head = {
+    .node = node,
+    .node_count_at_last_error = array_get(&self->heads, original_version)->node_count_at_last_error,
+    .last_external_token = array_get(&self->heads, original_version)->last_external_token,
+    .status = StackStatusActive,
+    .lookahead_when_paused = NULL_SUBTREE,
+  };
+  array_push(&self->heads, head);
+  stack_node_retain(node);
+  if (head.last_external_token.ptr) ts_subtree_retain(head.last_external_token);
+  return (StackVersion)(self->heads.size - 1);
+}
+
+static void ts_stack__add_slice(
+  Stack *self,
+  StackVersion original_version,
+  StackNode *node,
+  SubtreeArray *subtrees
+) {
+  for (uint32_t i = self->slices.size - 1; i + 1 > 0; i--) {
+    StackVersion version = array_get(&self->slices, i)->version;
+    if (array_get(&self->heads, version)->node == node) {
+      StackSlice slice = {*subtrees, version};
+      array_insert(&self->slices, i + 1, slice);
+      return;
+    }
+  }
+
+  StackVersion version = ts_stack__add_version(self, original_version, node);
+  StackSlice slice = { *subtrees, version };
+  array_push(&self->slices, slice);
+}
+
+static StackSliceArray stack__iter(
+  Stack *self,
+  StackVersion version,
+  StackCallback callback,
+  void *payload,
+  int goal_subtree_count
+) {
+  array_clear(&self->slices);
+  array_clear(&self->iterators);
+
+  StackHead *head = array_get(&self->heads, version);
+  StackIterator new_iterator = {
+    .node = head->node,
+    .subtrees = array_new(),
+    .subtree_count = 0,
+    .is_pending = true,
+  };
+
+  bool include_subtrees = false;
+  if (goal_subtree_count >= 0) {
+    include_subtrees = true;
+    array_reserve(&new_iterator.subtrees, (uint32_t)ts_subtree_alloc_size(goal_subtree_count) / sizeof(Subtree));
+  }
+
+  array_push(&self->iterators, new_iterator);
+
+  while (self->iterators.size > 0) {
+    for (uint32_t i = 0, size = self->iterators.size; i < size; i++) {
+      StackIterator *iterator = array_get(&self->iterators, i);
+      StackNode *node = iterator->node;
+
+      StackAction action = callback(payload, iterator);
+      bool should_pop = action & StackActionPop;
+      bool should_stop = action & StackActionStop || node->link_count == 0;
+
+      if (should_pop) {
+        SubtreeArray subtrees = iterator->subtrees;
+        if (!should_stop) {
+          ts_subtree_array_copy(subtrees, &subtrees);
+        }
+        ts_subtree_array_reverse(&subtrees);
+        ts_stack__add_slice(
+          self,
+          version,
+          node,
+          &subtrees
+        );
+      }
+
+      if (should_stop) {
+        if (!should_pop) {
+          ts_subtree_array_delete(self->subtree_pool, &iterator->subtrees);
+        }
+        array_erase(&self->iterators, i);
+        i--, size--;
+        continue;
+      }
+
+      for (uint32_t j = 1; j <= node->link_count; j++) {
+        StackIterator *next_iterator;
+        StackLink link;
+        if (j == node->link_count) {
+          link = node->links[0];
+          next_iterator = array_get(&self->iterators, i);
+        } else {
+          if (self->iterators.size >= MAX_ITERATOR_COUNT) continue;
+          link = node->links[j];
+          StackIterator current_iterator = *array_get(&self->iterators, i);
+          array_push(&self->iterators, current_iterator);
+          next_iterator = array_back(&self->iterators);
+          ts_subtree_array_copy(next_iterator->subtrees, &next_iterator->subtrees);
+        }
+
+        next_iterator->node = link.node;
+        if (link.subtree.ptr) {
+          if (include_subtrees) {
+            array_push(&next_iterator->subtrees, link.subtree);
+            ts_subtree_retain(link.subtree);
+          }
+
+          if (!ts_subtree_extra(link.subtree)) {
+            next_iterator->subtree_count++;
+            if (!link.is_pending) {
+              next_iterator->is_pending = false;
+            }
+          }
+        } else {
+          next_iterator->subtree_count++;
+          next_iterator->is_pending = false;
+        }
+      }
+    }
+  }
+
+  return self->slices;
+}
+
+Stack *ts_stack_new(SubtreePool *subtree_pool) {
+  Stack *self = ts_calloc(1, sizeof(Stack));
+
+  array_init(&self->heads);
+  array_init(&self->slices);
+  array_init(&self->iterators);
+  array_init(&self->node_pool);
+  array_reserve(&self->heads, 4);
+  array_reserve(&self->slices, 4);
+  array_reserve(&self->iterators, 4);
+  array_reserve(&self->node_pool, MAX_NODE_POOL_SIZE);
+
+  self->subtree_pool = subtree_pool;
+  self->base_node = stack_node_new(NULL, NULL_SUBTREE, false, 1, &self->node_pool);
+  ts_stack_clear(self);
+
+  return self;
+}
+
+void ts_stack_delete(Stack *self) {
+  if (self->slices.contents)
+    array_delete(&self->slices);
+  if (self->iterators.contents)
+    array_delete(&self->iterators);
+  stack_node_release(self->base_node, &self->node_pool, self->subtree_pool);
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool);
+  }
+  array_clear(&self->heads);
+  if (self->node_pool.contents) {
+    for (uint32_t i = 0; i < self->node_pool.size; i++)
+      ts_free(*array_get(&self->node_pool, i));
+    array_delete(&self->node_pool);
+  }
+  array_delete(&self->heads);
+  ts_free(self);
+}
+
+uint32_t ts_stack_version_count(const Stack *self) {
+  return self->heads.size;
+}
+
+uint32_t ts_stack_halted_version_count(Stack *self) {
+  uint32_t count = 0;
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    StackHead *head = array_get(&self->heads, i);
+    if (head->status == StackStatusHalted) {
+      count++;
+    }
+  }
+  return count;
+}
+
+TSStateId ts_stack_state(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->node->state;
+}
+
+Length ts_stack_position(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->node->position;
+}
+
+Subtree ts_stack_last_external_token(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->last_external_token;
+}
+
+void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token) {
+  StackHead *head = array_get(&self->heads, version);
+  if (token.ptr) ts_subtree_retain(token);
+  if (head->last_external_token.ptr) ts_subtree_release(self->subtree_pool, head->last_external_token);
+  head->last_external_token = token;
+}
+
+unsigned ts_stack_error_cost(const Stack *self, StackVersion version) {
+  StackHead *head = array_get(&self->heads, version);
+  unsigned result = head->node->error_cost;
+  if (
+    head->status == StackStatusPaused ||
+    (head->node->state == ERROR_STATE && !head->node->links[0].subtree.ptr)) {
+    result += ERROR_COST_PER_RECOVERY;
+  }
+  return result;
+}
+
+unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version) {
+  StackHead *head = array_get(&self->heads, version);
+  if (head->node->node_count < head->node_count_at_last_error) {
+    head->node_count_at_last_error = head->node->node_count;
+  }
+  return head->node->node_count - head->node_count_at_last_error;
+}
+
+void ts_stack_push(
+  Stack *self,
+  StackVersion version,
+  Subtree subtree,
+  bool pending,
+  TSStateId state
+) {
+  StackHead *head = array_get(&self->heads, version);
+  StackNode *new_node = stack_node_new(head->node, subtree, pending, state, &self->node_pool);
+  if (!subtree.ptr) head->node_count_at_last_error = new_node->node_count;
+  head->node = new_node;
+}
+
+forceinline StackAction pop_count_callback(void *payload, const StackIterator *iterator) {
+  unsigned *goal_subtree_count = payload;
+  if (iterator->subtree_count == *goal_subtree_count) {
+    return StackActionPop | StackActionStop;
+  } else {
+    return StackActionNone;
+  }
+}
+
+StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count) {
+  return stack__iter(self, version, pop_count_callback, &count, (int)count);
+}
+
+
+forceinline StackAction pop_pending_callback(void *payload, const StackIterator *iterator) {
+  (void)payload;
+  if (iterator->subtree_count >= 1) {
+    if (iterator->is_pending) {
+      return StackActionPop | StackActionStop;
+    } else {
+      return StackActionStop;
+    }
+  } else {
+    return StackActionNone;
+  }
+}
+
+StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version) {
+  StackSliceArray pop = stack__iter(self, version, pop_pending_callback, NULL, 0);
+  if (pop.size > 0) {
+    ts_stack_renumber_version(self, array_get(&pop, 0)->version, version);
+    array_get(&pop, 0)->version = version;
+  }
+  return pop;
+}
+
+forceinline StackAction pop_error_callback(void *payload, const StackIterator *iterator) {
+  if (iterator->subtrees.size > 0) {
+    bool *found_error = payload;
+    if (!*found_error && ts_subtree_is_error(*array_get(&iterator->subtrees, 0))) {
+      *found_error = true;
+      return StackActionPop | StackActionStop;
+    } else {
+      return StackActionStop;
+    }
+  } else {
+    return StackActionNone;
+  }
+}
+
+SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version) {
+  StackNode *node = array_get(&self->heads, version)->node;
+  for (unsigned i = 0; i < node->link_count; i++) {
+    if (node->links[i].subtree.ptr && ts_subtree_is_error(node->links[i].subtree)) {
+      bool found_error = false;
+      StackSliceArray pop = stack__iter(self, version, pop_error_callback, &found_error, 1);
+      if (pop.size > 0) {
+        ts_assert(pop.size == 1);
+        ts_stack_renumber_version(self, array_get(&pop, 0)->version, version);
+        return array_get(&pop, 0)->subtrees;
+      }
+      break;
+    }
+  }
+  return (SubtreeArray) {.size = 0};
+}
+
+forceinline StackAction pop_all_callback(void *payload, const StackIterator *iterator) {
+  (void)payload;
+  return iterator->node->link_count == 0 ? StackActionPop : StackActionNone;
+}
+
+StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version) {
+  return stack__iter(self, version, pop_all_callback, NULL, 0);
+}
+
+typedef struct {
+  StackSummary *summary;
+  unsigned max_depth;
+} SummarizeStackSession;
+
+forceinline StackAction summarize_stack_callback(void *payload, const StackIterator *iterator) {
+  SummarizeStackSession *session = payload;
+  TSStateId state = iterator->node->state;
+  unsigned depth = iterator->subtree_count;
+  if (depth > session->max_depth) return StackActionStop;
+  for (unsigned i = session->summary->size - 1; i + 1 > 0; i--) {
+    StackSummaryEntry entry = *array_get(session->summary, i);
+    if (entry.depth < depth) break;
+    if (entry.depth == depth && entry.state == state) return StackActionNone;
+  }
+  array_push(session->summary, ((StackSummaryEntry) {
+    .position = iterator->node->position,
+    .depth = depth,
+    .state = state,
+  }));
+  return StackActionNone;
+}
+
+void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth) {
+  SummarizeStackSession session = {
+    .summary = ts_malloc(sizeof(StackSummary)),
+    .max_depth = max_depth
+  };
+  array_init(session.summary);
+  stack__iter(self, version, summarize_stack_callback, &session, -1);
+  StackHead *head = array_get(&self->heads, version);
+  if (head->summary) {
+    array_delete(head->summary);
+    ts_free(head->summary);
+  }
+  head->summary = session.summary;
+}
+
+StackSummary *ts_stack_get_summary(Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->summary;
+}
+
+int ts_stack_dynamic_precedence(Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->node->dynamic_precedence;
+}
+
+bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version) {
+  const StackHead *head = array_get(&self->heads, version);
+  const StackNode *node = head->node;
+  if (node->error_cost == 0) return true;
+  while (node) {
+    if (node->link_count > 0) {
+      Subtree subtree = node->links[0].subtree;
+      if (subtree.ptr) {
+        if (ts_subtree_total_bytes(subtree) > 0) {
+          return true;
+        } else if (
+          node->node_count > head->node_count_at_last_error &&
+          ts_subtree_error_cost(subtree) == 0
+        ) {
+          node = node->links[0].node;
+          continue;
+        }
+      }
+    }
+    break;
+  }
+  return false;
+}
+
+void ts_stack_remove_version(Stack *self, StackVersion version) {
+  stack_head_delete(array_get(&self->heads, version), &self->node_pool, self->subtree_pool);
+  array_erase(&self->heads, version);
+}
+
+void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2) {
+  if (v1 == v2) return;
+  ts_assert(v2 < v1);
+  ts_assert((uint32_t)v1 < self->heads.size);
+  StackHead *source_head = array_get(&self->heads, v1);
+  StackHead *target_head = array_get(&self->heads, v2);
+  if (target_head->summary && !source_head->summary) {
+    source_head->summary = target_head->summary;
+    target_head->summary = NULL;
+  }
+  stack_head_delete(target_head, &self->node_pool, self->subtree_pool);
+  *target_head = *source_head;
+  array_erase(&self->heads, v1);
+}
+
+void ts_stack_swap_versions(Stack *self, StackVersion v1, StackVersion v2) {
+  StackHead temporary_head = *array_get(&self->heads, v1);
+  *array_get(&self->heads, v1) = *array_get(&self->heads, v2);
+  *array_get(&self->heads, v2) = temporary_head;
+}
+
+StackVersion ts_stack_copy_version(Stack *self, StackVersion version) {
+  ts_assert(version < self->heads.size);
+  StackHead version_head = *array_get(&self->heads, version);
+  array_push(&self->heads, version_head);
+  StackHead *head = array_back(&self->heads);
+  stack_node_retain(head->node);
+  if (head->last_external_token.ptr) ts_subtree_retain(head->last_external_token);
+  head->summary = NULL;
+  return self->heads.size - 1;
+}
+
+bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2) {
+  if (!ts_stack_can_merge(self, version1, version2)) return false;
+  StackHead *head1 = array_get(&self->heads, version1);
+  StackHead *head2 = array_get(&self->heads, version2);
+  for (uint32_t i = 0; i < head2->node->link_count; i++) {
+    stack_node_add_link(head1->node, head2->node->links[i], self->subtree_pool);
+  }
+  if (head1->node->state == ERROR_STATE) {
+    head1->node_count_at_last_error = head1->node->node_count;
+  }
+  ts_stack_remove_version(self, version2);
+  return true;
+}
+
+bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2) {
+  StackHead *head1 = array_get(&self->heads, version1);
+  StackHead *head2 = array_get(&self->heads, version2);
+  return
+    head1->status == StackStatusActive &&
+    head2->status == StackStatusActive &&
+    head1->node->state == head2->node->state &&
+    head1->node->position.bytes == head2->node->position.bytes &&
+    head1->node->error_cost == head2->node->error_cost &&
+    ts_subtree_external_scanner_state_eq(head1->last_external_token, head2->last_external_token);
+}
+
+void ts_stack_halt(Stack *self, StackVersion version) {
+  array_get(&self->heads, version)->status = StackStatusHalted;
+}
+
+void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead) {
+  StackHead *head = array_get(&self->heads, version);
+  head->status = StackStatusPaused;
+  head->lookahead_when_paused = lookahead;
+  head->node_count_at_last_error = head->node->node_count;
+}
+
+bool ts_stack_is_active(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->status == StackStatusActive;
+}
+
+bool ts_stack_is_halted(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->status == StackStatusHalted;
+}
+
+bool ts_stack_is_paused(const Stack *self, StackVersion version) {
+  return array_get(&self->heads, version)->status == StackStatusPaused;
+}
+
+Subtree ts_stack_resume(Stack *self, StackVersion version) {
+  StackHead *head = array_get(&self->heads, version);
+  ts_assert(head->status == StackStatusPaused);
+  Subtree result = head->lookahead_when_paused;
+  head->status = StackStatusActive;
+  head->lookahead_when_paused = NULL_SUBTREE;
+  return result;
+}
+
+void ts_stack_clear(Stack *self) {
+  stack_node_retain(self->base_node);
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    stack_head_delete(array_get(&self->heads, i), &self->node_pool, self->subtree_pool);
+  }
+  array_clear(&self->heads);
+  array_push(&self->heads, ((StackHead) {
+    .node = self->base_node,
+    .status = StackStatusActive,
+    .last_external_token = NULL_SUBTREE,
+    .lookahead_when_paused = NULL_SUBTREE,
+  }));
+}
+
+bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f) {
+  array_reserve(&self->iterators, 32);
+  if (!f) f = stderr;
+
+  fprintf(f, "digraph stack {\n");
+  fprintf(f, "rankdir=\"RL\";\n");
+  fprintf(f, "edge [arrowhead=none]\n");
+
+  Array(StackNode *) visited_nodes = array_new();
+
+  array_clear(&self->iterators);
+  for (uint32_t i = 0; i < self->heads.size; i++) {
+    StackHead *head = array_get(&self->heads, i);
+    if (head->status == StackStatusHalted) continue;
+
+    fprintf(f, "node_head_%u [shape=none, label=\"\"]\n", i);
+    fprintf(f, "node_head_%u -> node_%p [", i, (void *)head->node);
+
+    if (head->status == StackStatusPaused) {
+      fprintf(f, "color=red ");
+    }
+    fprintf(f,
+      "label=%u, fontcolor=blue, weight=10000, labeltooltip=\"node_count: %u\nerror_cost: %u",
+      i,
+      ts_stack_node_count_since_error(self, i),
+      ts_stack_error_cost(self, i)
+    );
+
+    if (head->summary) {
+      fprintf(f, "\nsummary:");
+      for (uint32_t j = 0; j < head->summary->size; j++) fprintf(f, " %u", array_get(head->summary, j)->state);
+    }
+
+    if (head->last_external_token.ptr) {
+      const ExternalScannerState *state = &head->last_external_token.ptr->external_scanner_state;
+      const char *data = ts_external_scanner_state_data(state);
+      fprintf(f, "\nexternal_scanner_state:");
+      for (uint32_t j = 0; j < state->length; j++) fprintf(f, " %2X", data[j]);
+    }
+
+    fprintf(f, "\"]\n");
+    array_push(&self->iterators, ((StackIterator) {
+      .node = head->node
+    }));
+  }
+
+  bool all_iterators_done = false;
+  while (!all_iterators_done) {
+    all_iterators_done = true;
+
+    for (uint32_t i = 0; i < self->iterators.size; i++) {
+      StackIterator iterator = *array_get(&self->iterators, i);
+      StackNode *node = iterator.node;
+
+      for (uint32_t j = 0; j < visited_nodes.size; j++) {
+        if (*array_get(&visited_nodes, j) == node) {
+          node = NULL;
+          break;
+        }
+      }
+
+      if (!node) continue;
+      all_iterators_done = false;
+
+      fprintf(f, "node_%p [", (void *)node);
+      if (node->state == ERROR_STATE) {
+        fprintf(f, "label=\"?\"");
+      } else if (
+        node->link_count == 1 &&
+        node->links[0].subtree.ptr &&
+        ts_subtree_extra(node->links[0].subtree)
+      ) {
+        fprintf(f, "shape=point margin=0 label=\"\"");
+      } else {
+        fprintf(f, "label=\"%d\"", node->state);
+      }
+
+      fprintf(
+        f,
+        " tooltip=\"position: %u,%u\nnode_count:%u\nerror_cost: %u\ndynamic_precedence: %d\"];\n",
+        node->position.extent.row + 1,
+        node->position.extent.column,
+        node->node_count,
+        node->error_cost,
+        node->dynamic_precedence
+      );
+
+      for (int j = 0; j < node->link_count; j++) {
+        StackLink link = node->links[j];
+        fprintf(f, "node_%p -> node_%p [", (void *)node, (void *)link.node);
+        if (link.is_pending) fprintf(f, "style=dashed ");
+        if (link.subtree.ptr && ts_subtree_extra(link.subtree)) fprintf(f, "fontcolor=gray ");
+
+        if (!link.subtree.ptr) {
+          fprintf(f, "color=red");
+        } else {
+          fprintf(f, "label=\"");
+          bool quoted = ts_subtree_visible(link.subtree) && !ts_subtree_named(link.subtree);
+          if (quoted) fprintf(f, "'");
+          ts_language_write_symbol_as_dot_string(language, f, ts_subtree_symbol(link.subtree));
+          if (quoted) fprintf(f, "'");
+          fprintf(f, "\"");
+          fprintf(
+            f,
+            "labeltooltip=\"error_cost: %u\ndynamic_precedence: %" PRId32 "\"",
+            ts_subtree_error_cost(link.subtree),
+            ts_subtree_dynamic_precedence(link.subtree)
+          );
+        }
+
+        fprintf(f, "];\n");
+
+        StackIterator *next_iterator;
+        if (j == 0) {
+          next_iterator = array_get(&self->iterators, i);
+        } else {
+          array_push(&self->iterators, iterator);
+          next_iterator = array_back(&self->iterators);
+        }
+        next_iterator->node = link.node;
+      }
+
+      array_push(&visited_nodes, node);
+    }
+  }
+
+  fprintf(f, "}\n");
+
+  array_delete(&visited_nodes);
+  return true;
+}
+
+#undef forceinline

third_party/tree-sitter/lib/src/stack.h

@@ -0,0 +1,133 @@
+#ifndef TREE_SITTER_PARSE_STACK_H_
+#define TREE_SITTER_PARSE_STACK_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "./array.h"
+#include "./subtree.h"
+#include <stdio.h>
+
+typedef struct Stack Stack;
+
+typedef unsigned StackVersion;
+#define STACK_VERSION_NONE ((StackVersion)-1)
+
+typedef struct {
+  SubtreeArray subtrees;
+  StackVersion version;
+} StackSlice;
+typedef Array(StackSlice) StackSliceArray;
+
+typedef struct {
+  Length position;
+  unsigned depth;
+  TSStateId state;
+} StackSummaryEntry;
+typedef Array(StackSummaryEntry) StackSummary;
+
+// Create a stack.
+Stack *ts_stack_new(SubtreePool *subtree_pool);
+
+// Release the memory reserved for a given stack.
+void ts_stack_delete(Stack *self);
+
+// Get the stack's current number of versions.
+uint32_t ts_stack_version_count(const Stack *self);
+
+// Get the stack's current number of halted versions.
+uint32_t ts_stack_halted_version_count(Stack *self);
+
+// Get the state at the top of the given version of the stack. If the stack is
+// empty, this returns the initial state, 0.
+TSStateId ts_stack_state(const Stack *self, StackVersion version);
+
+// Get the last external token associated with a given version of the stack.
+Subtree ts_stack_last_external_token(const Stack *self, StackVersion version);
+
+// Set the last external token associated with a given version of the stack.
+void ts_stack_set_last_external_token(Stack *self, StackVersion version, Subtree token);
+
+// Get the position of the given version of the stack within the document.
+Length ts_stack_position(const Stack *, StackVersion);
+
+// Push a tree and state onto the given version of the stack.
+//
+// This transfers ownership of the tree to the Stack. Callers that
+// need to retain ownership of the tree for their own purposes should
+// first retain the tree.
+void ts_stack_push(Stack *self, StackVersion version, Subtree subtree, bool pending, TSStateId state);
+
+// Pop the given number of entries from the given version of the stack. This
+// operation can increase the number of stack versions by revealing multiple
+// versions which had previously been merged. It returns an array that
+// specifies the index of each revealed version and the trees that were
+// removed from that version.
+StackSliceArray ts_stack_pop_count(Stack *self, StackVersion version, uint32_t count);
+
+// Remove an error at the top of the given version of the stack.
+SubtreeArray ts_stack_pop_error(Stack *self, StackVersion version);
+
+// Remove any pending trees from the top of the given version of the stack.
+StackSliceArray ts_stack_pop_pending(Stack *self, StackVersion version);
+
+// Remove all trees from the given version of the stack.
+StackSliceArray ts_stack_pop_all(Stack *self, StackVersion version);
+
+// Get the maximum number of tree nodes reachable from this version of the stack
+// since the last error was detected.
+unsigned ts_stack_node_count_since_error(const Stack *self, StackVersion version);
+
+int ts_stack_dynamic_precedence(Stack *self, StackVersion version);
+
+bool ts_stack_has_advanced_since_error(const Stack *self, StackVersion version);
+
+// Compute a summary of all the parse states near the top of the given
+// version of the stack and store the summary for later retrieval.
+void ts_stack_record_summary(Stack *self, StackVersion version, unsigned max_depth);
+
+// Retrieve a summary of all the parse states near the top of the
+// given version of the stack.
+StackSummary *ts_stack_get_summary(Stack *self, StackVersion version);
+
+// Get the total cost of all errors on the given version of the stack.
+unsigned ts_stack_error_cost(const Stack *self, StackVersion version);
+
+// Merge the given two stack versions if possible, returning true
+// if they were successfully merged and false otherwise.
+bool ts_stack_merge(Stack *self, StackVersion version1, StackVersion version2);
+
+// Determine whether the given two stack versions can be merged.
+bool ts_stack_can_merge(Stack *self, StackVersion version1, StackVersion version2);
+
+Subtree ts_stack_resume(Stack *self, StackVersion version);
+
+void ts_stack_pause(Stack *self, StackVersion version, Subtree lookahead);
+
+void ts_stack_halt(Stack *self, StackVersion version);
+
+bool ts_stack_is_active(const Stack *self, StackVersion version);
+
+bool ts_stack_is_paused(const Stack *self, StackVersion version);
+
+bool ts_stack_is_halted(const Stack *self, StackVersion version);
+
+void ts_stack_renumber_version(Stack *self, StackVersion v1, StackVersion v2);
+
+void ts_stack_swap_versions(Stack *, StackVersion v1, StackVersion v2);
+
+StackVersion ts_stack_copy_version(Stack *self, StackVersion version);
+
+// Remove the given version from the stack.
+void ts_stack_remove_version(Stack *self, StackVersion version);
+
+void ts_stack_clear(Stack *self);
+
+bool ts_stack_print_dot_graph(Stack *self, const TSLanguage *language, FILE *f);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_PARSE_STACK_H_

third_party/tree-sitter/lib/src/subtree.h

@@ -0,0 +1,399 @@
+#ifndef TREE_SITTER_SUBTREE_H_
+#define TREE_SITTER_SUBTREE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <limits.h>
+#include <stdbool.h>
+#include <stdio.h>
+#include "./length.h"
+#include "./array.h"
+#include "./error_costs.h"
+#include "./host.h"
+#include "tree_sitter/api.h"
+#include "./parser.h"
+
+#define TS_TREE_STATE_NONE USHRT_MAX
+#define NULL_SUBTREE ((Subtree) {.ptr = NULL})
+
+// The serialized state of an external scanner.
+//
+// Every time an external token subtree is created after a call to an
+// external scanner, the scanner's `serialize` function is called to
+// retrieve a serialized copy of its state. The bytes are then copied
+// onto the subtree itself so that the scanner's state can later be
+// restored using its `deserialize` function.
+//
+// Small byte arrays are stored inline, and long ones are allocated
+// separately on the heap.
+typedef struct {
+  union {
+    char *long_data;
+    char short_data[24];
+  };
+  uint32_t length;
+} ExternalScannerState;
+
+// A compact representation of a subtree.
+//
+// This representation is used for small leaf nodes that are not
+// errors, and were not created by an external scanner.
+//
+// The idea behind the layout of this struct is that the `is_inline`
+// bit will fall exactly into the same location as the least significant
+// bit of the pointer in `Subtree` or `MutableSubtree`, respectively.
+// Because of alignment, for any valid pointer this will be 0, giving
+// us the opportunity to make use of this bit to signify whether to use
+// the pointer or the inline struct.
+typedef struct SubtreeInlineData SubtreeInlineData;
+
+#define SUBTREE_BITS    \
+  bool visible : 1;     \
+  bool named : 1;       \
+  bool extra : 1;       \
+  bool has_changes : 1; \
+  bool is_missing : 1;  \
+  bool is_keyword : 1;
+
+#define SUBTREE_SIZE           \
+  uint8_t padding_columns;     \
+  uint8_t padding_rows : 4;    \
+  uint8_t lookahead_bytes : 4; \
+  uint8_t padding_bytes;       \
+  uint8_t size_bytes;
+
+#if TS_BIG_ENDIAN
+#if TS_PTR_SIZE == 32
+
+struct SubtreeInlineData {
+  uint16_t parse_state;
+  uint8_t symbol;
+  SUBTREE_BITS
+  bool unused : 1;
+  bool is_inline : 1;
+  SUBTREE_SIZE
+};
+
+#else
+
+struct SubtreeInlineData {
+  SUBTREE_SIZE
+  uint16_t parse_state;
+  uint8_t symbol;
+  SUBTREE_BITS
+  bool unused : 1;
+  bool is_inline : 1;
+};
+
+#endif
+#else
+
+struct SubtreeInlineData {
+  bool is_inline : 1;
+  SUBTREE_BITS
+  uint8_t symbol;
+  uint16_t parse_state;
+  SUBTREE_SIZE
+};
+
+#endif
+
+#undef SUBTREE_BITS
+#undef SUBTREE_SIZE
+
+// A heap-allocated representation of a subtree.
+//
+// This representation is used for parent nodes, external tokens,
+// errors, and other leaf nodes whose data is too large to fit into
+// the inline representation.
+typedef struct {
+  volatile uint32_t ref_count;
+  Length padding;
+  Length size;
+  uint32_t lookahead_bytes;
+  uint32_t error_cost;
+  uint32_t child_count;
+  TSSymbol symbol;
+  TSStateId parse_state;
+
+  bool visible : 1;
+  bool named : 1;
+  bool extra : 1;
+  bool fragile_left : 1;
+  bool fragile_right : 1;
+  bool has_changes : 1;
+  bool has_external_tokens : 1;
+  bool has_external_scanner_state_change : 1;
+  bool depends_on_column: 1;
+  bool is_missing : 1;
+  bool is_keyword : 1;
+
+  union {
+    // Non-terminal subtrees (`child_count > 0`)
+    struct {
+      uint32_t visible_child_count;
+      uint32_t named_child_count;
+      uint32_t visible_descendant_count;
+      int32_t dynamic_precedence;
+      uint16_t repeat_depth;
+      uint16_t production_id;
+      struct {
+        TSSymbol symbol;
+        TSStateId parse_state;
+      } first_leaf;
+    };
+
+    // External terminal subtrees (`child_count == 0 && has_external_tokens`)
+    ExternalScannerState external_scanner_state;
+
+    // Error terminal subtrees (`child_count == 0 && symbol == ts_builtin_sym_error`)
+    int32_t lookahead_char;
+  };
+} SubtreeHeapData;
+
+// The fundamental building block of a syntax tree.
+typedef union {
+  SubtreeInlineData data;
+  const SubtreeHeapData *ptr;
+} Subtree;
+
+// Like Subtree, but mutable.
+typedef union {
+  SubtreeInlineData data;
+  SubtreeHeapData *ptr;
+} MutableSubtree;
+
+typedef Array(Subtree) SubtreeArray;
+typedef Array(MutableSubtree) MutableSubtreeArray;
+
+typedef struct {
+  MutableSubtreeArray free_trees;
+  MutableSubtreeArray tree_stack;
+} SubtreePool;
+
+void ts_external_scanner_state_init(ExternalScannerState *self, const char *data, unsigned length);
+const char *ts_external_scanner_state_data(const ExternalScannerState *self);
+bool ts_external_scanner_state_eq(const ExternalScannerState *self, const char *buffer, unsigned length);
+void ts_external_scanner_state_delete(ExternalScannerState *self);
+
+void ts_subtree_array_copy(SubtreeArray self, SubtreeArray *dest);
+void ts_subtree_array_clear(SubtreePool *pool, SubtreeArray *self);
+void ts_subtree_array_delete(SubtreePool *pool, SubtreeArray *self);
+void ts_subtree_array_remove_trailing_extras(SubtreeArray *self, SubtreeArray *destination);
+void ts_subtree_array_reverse(SubtreeArray *self);
+
+SubtreePool ts_subtree_pool_new(uint32_t capacity);
+void ts_subtree_pool_delete(SubtreePool *self);
+
+Subtree ts_subtree_new_leaf(
+  SubtreePool *pool, TSSymbol symbol, Length padding, Length size,
+  uint32_t lookahead_bytes, TSStateId parse_state,
+  bool has_external_tokens, bool depends_on_column,
+  bool is_keyword, const TSLanguage *language
+);
+Subtree ts_subtree_new_error(
+  SubtreePool *pool, int32_t lookahead_char, Length padding, Length size,
+  uint32_t bytes_scanned, TSStateId parse_state, const TSLanguage *language
+);
+MutableSubtree ts_subtree_new_node(
+  TSSymbol symbol,
+  SubtreeArray *chiildren,
+  unsigned production_id,
+  const TSLanguage *language
+);
+Subtree ts_subtree_new_error_node(
+  SubtreeArray *children,
+  bool extra,
+  const TSLanguage * language
+);
+Subtree ts_subtree_new_missing_leaf(
+  SubtreePool *pool,
+  TSSymbol symbol,
+  Length padding,
+  uint32_t lookahead_bytes,
+  const TSLanguage *language
+);
+MutableSubtree ts_subtree_make_mut(SubtreePool *pool, Subtree self);
+void ts_subtree_retain(Subtree self);
+void ts_subtree_release(SubtreePool *pool, Subtree self);
+int ts_subtree_compare(Subtree left, Subtree right, SubtreePool *pool);
+void ts_subtree_set_symbol(MutableSubtree *self, TSSymbol symbol, const TSLanguage *language);
+void ts_subtree_compress(MutableSubtree self, unsigned count, const TSLanguage *language, MutableSubtreeArray *stack);
+void ts_subtree_summarize_children(MutableSubtree self, const TSLanguage *language);
+Subtree ts_subtree_edit(Subtree self, const TSInputEdit *edit, SubtreePool *pool);
+char *ts_subtree_string(Subtree self, TSSymbol alias_symbol, bool alias_is_named, const TSLanguage *language, bool include_all);
+void ts_subtree_print_dot_graph(Subtree self, const TSLanguage *language, FILE *f);
+Subtree ts_subtree_last_external_token(Subtree tree);
+const ExternalScannerState *ts_subtree_external_scanner_state(Subtree self);
+bool ts_subtree_external_scanner_state_eq(Subtree self, Subtree other);
+
+#define SUBTREE_GET(self, name) ((self).data.is_inline ? (self).data.name : (self).ptr->name)
+
+static inline TSSymbol ts_subtree_symbol(Subtree self) { return SUBTREE_GET(self, symbol); }
+static inline bool ts_subtree_visible(Subtree self) { return SUBTREE_GET(self, visible); }
+static inline bool ts_subtree_named(Subtree self) { return SUBTREE_GET(self, named); }
+static inline bool ts_subtree_extra(Subtree self) { return SUBTREE_GET(self, extra); }
+static inline bool ts_subtree_has_changes(Subtree self) { return SUBTREE_GET(self, has_changes); }
+static inline bool ts_subtree_missing(Subtree self) { return SUBTREE_GET(self, is_missing); }
+static inline bool ts_subtree_is_keyword(Subtree self) { return SUBTREE_GET(self, is_keyword); }
+static inline TSStateId ts_subtree_parse_state(Subtree self) { return SUBTREE_GET(self, parse_state); }
+static inline uint32_t ts_subtree_lookahead_bytes(Subtree self) { return SUBTREE_GET(self, lookahead_bytes); }
+
+#undef SUBTREE_GET
+
+// Get the size needed to store a heap-allocated subtree with the given
+// number of children.
+static inline size_t ts_subtree_alloc_size(uint32_t child_count) {
+  return child_count * sizeof(Subtree) + sizeof(SubtreeHeapData);
+}
+
+// Get a subtree's children, which are allocated immediately before the
+// tree's own heap data.
+#define ts_subtree_children(self) \
+  ((self).data.is_inline ? NULL : (Subtree *)((self).ptr) - (self).ptr->child_count)
+
+static inline void ts_subtree_set_extra(MutableSubtree *self, bool is_extra) {
+  if (self->data.is_inline) {
+    self->data.extra = is_extra;
+  } else {
+    self->ptr->extra = is_extra;
+  }
+}
+
+static inline TSSymbol ts_subtree_leaf_symbol(Subtree self) {
+  if (self.data.is_inline) return self.data.symbol;
+  if (self.ptr->child_count == 0) return self.ptr->symbol;
+  return self.ptr->first_leaf.symbol;
+}
+
+static inline TSStateId ts_subtree_leaf_parse_state(Subtree self) {
+  if (self.data.is_inline) return self.data.parse_state;
+  if (self.ptr->child_count == 0) return self.ptr->parse_state;
+  return self.ptr->first_leaf.parse_state;
+}
+
+static inline Length ts_subtree_padding(Subtree self) {
+  if (self.data.is_inline) {
+    Length result = {self.data.padding_bytes, {self.data.padding_rows, self.data.padding_columns}};
+    return result;
+  } else {
+    return self.ptr->padding;
+  }
+}
+
+static inline Length ts_subtree_size(Subtree self) {
+  if (self.data.is_inline) {
+    Length result = {self.data.size_bytes, {0, self.data.size_bytes}};
+    return result;
+  } else {
+    return self.ptr->size;
+  }
+}
+
+static inline Length ts_subtree_total_size(Subtree self) {
+  return length_add(ts_subtree_padding(self), ts_subtree_size(self));
+}
+
+static inline uint32_t ts_subtree_total_bytes(Subtree self) {
+  return ts_subtree_total_size(self).bytes;
+}
+
+static inline uint32_t ts_subtree_child_count(Subtree self) {
+  return self.data.is_inline ? 0 : self.ptr->child_count;
+}
+
+static inline uint32_t ts_subtree_repeat_depth(Subtree self) {
+  return self.data.is_inline ? 0 : self.ptr->repeat_depth;
+}
+
+static inline uint32_t ts_subtree_is_repetition(Subtree self) {
+  return self.data.is_inline
+    ? 0
+    : !self.ptr->named && !self.ptr->visible && self.ptr->child_count != 0;
+}
+
+static inline uint32_t ts_subtree_visible_descendant_count(Subtree self) {
+  return (self.data.is_inline || self.ptr->child_count == 0)
+    ? 0
+    : self.ptr->visible_descendant_count;
+}
+
+static inline uint32_t ts_subtree_visible_child_count(Subtree self) {
+  if (ts_subtree_child_count(self) > 0) {
+    return self.ptr->visible_child_count;
+  } else {
+    return 0;
+  }
+}
+
+static inline uint32_t ts_subtree_error_cost(Subtree self) {
+  if (ts_subtree_missing(self)) {
+    return ERROR_COST_PER_MISSING_TREE + ERROR_COST_PER_RECOVERY;
+  } else {
+    return self.data.is_inline ? 0 : self.ptr->error_cost;
+  }
+}
+
+static inline int32_t ts_subtree_dynamic_precedence(Subtree self) {
+  return (self.data.is_inline || self.ptr->child_count == 0) ? 0 : self.ptr->dynamic_precedence;
+}
+
+static inline uint16_t ts_subtree_production_id(Subtree self) {
+  if (ts_subtree_child_count(self) > 0) {
+    return self.ptr->production_id;
+  } else {
+    return 0;
+  }
+}
+
+static inline bool ts_subtree_fragile_left(Subtree self) {
+  return self.data.is_inline ? false : self.ptr->fragile_left;
+}
+
+static inline bool ts_subtree_fragile_right(Subtree self) {
+  return self.data.is_inline ? false : self.ptr->fragile_right;
+}
+
+static inline bool ts_subtree_has_external_tokens(Subtree self) {
+  return self.data.is_inline ? false : self.ptr->has_external_tokens;
+}
+
+static inline bool ts_subtree_has_external_scanner_state_change(Subtree self) {
+  return self.data.is_inline ? false : self.ptr->has_external_scanner_state_change;
+}
+
+static inline bool ts_subtree_depends_on_column(Subtree self) {
+  return self.data.is_inline ? false : self.ptr->depends_on_column;
+}
+
+static inline bool ts_subtree_is_fragile(Subtree self) {
+  return self.data.is_inline ? false : (self.ptr->fragile_left || self.ptr->fragile_right);
+}
+
+static inline bool ts_subtree_is_error(Subtree self) {
+  return ts_subtree_symbol(self) == ts_builtin_sym_error;
+}
+
+static inline bool ts_subtree_is_eof(Subtree self) {
+  return ts_subtree_symbol(self) == ts_builtin_sym_end;
+}
+
+static inline Subtree ts_subtree_from_mut(MutableSubtree self) {
+  Subtree result;
+  result.data = self.data;
+  return result;
+}
+
+static inline MutableSubtree ts_subtree_to_mut_unsafe(Subtree self) {
+  MutableSubtree result;
+  result.data = self.data;
+  return result;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_SUBTREE_H_

third_party/tree-sitter/lib/src/tree.c

@@ -0,0 +1,170 @@
+#include "tree_sitter/api.h"
+#include "./array.h"
+#include "./get_changed_ranges.h"
+#include "./length.h"
+#include "./subtree.h"
+#include "./tree_cursor.h"
+#include "./tree.h"
+
+TSTree *ts_tree_new(
+  Subtree root, const TSLanguage *language,
+  const TSRange *included_ranges, unsigned included_range_count
+) {
+  TSTree *result = ts_malloc(sizeof(TSTree));
+  result->root = root;
+  result->language = ts_language_copy(language);
+  result->included_ranges = ts_calloc(included_range_count, sizeof(TSRange));
+  memcpy(result->included_ranges, included_ranges, included_range_count * sizeof(TSRange));
+  result->included_range_count = included_range_count;
+  return result;
+}
+
+TSTree *ts_tree_copy(const TSTree *self) {
+  ts_subtree_retain(self->root);
+  return ts_tree_new(self->root, self->language, self->included_ranges, self->included_range_count);
+}
+
+void ts_tree_delete(TSTree *self) {
+  if (!self) return;
+
+  SubtreePool pool = ts_subtree_pool_new(0);
+  ts_subtree_release(&pool, self->root);
+  ts_subtree_pool_delete(&pool);
+  ts_language_delete(self->language);
+  ts_free(self->included_ranges);
+  ts_free(self);
+}
+
+TSNode ts_tree_root_node(const TSTree *self) {
+  return ts_node_new(self, &self->root, ts_subtree_padding(self->root), 0);
+}
+
+TSNode ts_tree_root_node_with_offset(
+  const TSTree *self,
+  uint32_t offset_bytes,
+  TSPoint offset_extent
+) {
+  Length offset = {offset_bytes, offset_extent};
+  return ts_node_new(self, &self->root, length_add(offset, ts_subtree_padding(self->root)), 0);
+}
+
+const TSLanguage *ts_tree_language(const TSTree *self) {
+  return self->language;
+}
+
+void ts_tree_edit(TSTree *self, const TSInputEdit *edit) {
+  for (unsigned i = 0; i < self->included_range_count; i++) {
+    TSRange *range = &self->included_ranges[i];
+    if (range->end_byte >= edit->old_end_byte) {
+      if (range->end_byte != UINT32_MAX) {
+        range->end_byte = edit->new_end_byte + (range->end_byte - edit->old_end_byte);
+        range->end_point = point_add(
+          edit->new_end_point,
+          point_sub(range->end_point, edit->old_end_point)
+        );
+        if (range->end_byte < edit->new_end_byte) {
+          range->end_byte = UINT32_MAX;
+          range->end_point = POINT_MAX;
+        }
+      }
+    } else if (range->end_byte > edit->start_byte) {
+      range->end_byte = edit->start_byte;
+      range->end_point = edit->start_point;
+    }
+    if (range->start_byte >= edit->old_end_byte) {
+      range->start_byte = edit->new_end_byte + (range->start_byte - edit->old_end_byte);
+      range->start_point = point_add(
+        edit->new_end_point,
+        point_sub(range->start_point, edit->old_end_point)
+      );
+      if (range->start_byte < edit->new_end_byte) {
+        range->start_byte = UINT32_MAX;
+        range->start_point = POINT_MAX;
+      }
+    } else if (range->start_byte > edit->start_byte) {
+      range->start_byte = edit->start_byte;
+      range->start_point = edit->start_point;
+    }
+  }
+
+  SubtreePool pool = ts_subtree_pool_new(0);
+  self->root = ts_subtree_edit(self->root, edit, &pool);
+  ts_subtree_pool_delete(&pool);
+}
+
+TSRange *ts_tree_included_ranges(const TSTree *self, uint32_t *length) {
+  *length = self->included_range_count;
+  TSRange *ranges = ts_calloc(self->included_range_count, sizeof(TSRange));
+  memcpy(ranges, self->included_ranges, self->included_range_count * sizeof(TSRange));
+  return ranges;
+}
+
+TSRange *ts_tree_get_changed_ranges(const TSTree *old_tree, const TSTree *new_tree, uint32_t *length) {
+  TreeCursor cursor1 = {NULL, array_new(), 0};
+  TreeCursor cursor2 = {NULL, array_new(), 0};
+  ts_tree_cursor_init(&cursor1, ts_tree_root_node(old_tree));
+  ts_tree_cursor_init(&cursor2, ts_tree_root_node(new_tree));
+
+  TSRangeArray included_range_differences = array_new();
+  ts_range_array_get_changed_ranges(
+    old_tree->included_ranges, old_tree->included_range_count,
+    new_tree->included_ranges, new_tree->included_range_count,
+    &included_range_differences
+  );
+
+  TSRange *result;
+  *length = ts_subtree_get_changed_ranges(
+    &old_tree->root, &new_tree->root, &cursor1, &cursor2,
+    old_tree->language, &included_range_differences, &result
+  );
+
+  array_delete(&included_range_differences);
+  array_delete(&cursor1.stack);
+  array_delete(&cursor2.stack);
+  return result;
+}
+
+#ifdef _WIN32
+
+#include <io.h>
+#include <windows.h>
+
+int _ts_dup(HANDLE handle) {
+  HANDLE dup_handle;
+  if (!DuplicateHandle(
+    GetCurrentProcess(), handle,
+    GetCurrentProcess(), &dup_handle,
+    0, FALSE, DUPLICATE_SAME_ACCESS
+  )) return -1;
+
+  return _open_osfhandle((intptr_t)dup_handle, 0);
+}
+
+void ts_tree_print_dot_graph(const TSTree *self, int fd) {
+  FILE *file = _fdopen(_ts_dup((HANDLE)_get_osfhandle(fd)), "a");
+  ts_subtree_print_dot_graph(self->root, self->language, file);
+  fclose(file);
+}
+
+#elif !defined(__wasi__) // WASI doesn't support dup
+
+#include <unistd.h>
+
+int _ts_dup(int file_descriptor) {
+  return dup(file_descriptor);
+}
+
+void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) {
+  FILE *file = fdopen(_ts_dup(file_descriptor), "a");
+  ts_subtree_print_dot_graph(self->root, self->language, file);
+  fclose(file);
+}
+
+#else
+
+void ts_tree_print_dot_graph(const TSTree *self, int file_descriptor) {
+  (void)self;
+  (void)file_descriptor;
+}
+
+#endif

third_party/tree-sitter/lib/src/tree.h

@@ -0,0 +1,31 @@
+#ifndef TREE_SITTER_TREE_H_
+#define TREE_SITTER_TREE_H_
+
+#include "./subtree.h"
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct {
+  const Subtree *child;
+  const Subtree *parent;
+  Length position;
+  TSSymbol alias_symbol;
+} ParentCacheEntry;
+
+struct TSTree {
+  Subtree root;
+  const TSLanguage *language;
+  TSRange *included_ranges;
+  unsigned included_range_count;
+};
+
+TSTree *ts_tree_new(Subtree root, const TSLanguage *language, const TSRange *included_ranges, unsigned included_range_count);
+TSNode ts_node_new(const TSTree *tree, const Subtree *subtree, Length position, TSSymbol alias);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_TREE_H_

third_party/tree-sitter/lib/src/tree_cursor.c

@@ -0,0 +1,716 @@
+#include "tree_sitter/api.h"
+#include "./tree_cursor.h"
+#include "./language.h"
+#include "./tree.h"
+
+typedef struct {
+  Subtree parent;
+  const TSTree *tree;
+  Length position;
+  uint32_t child_index;
+  uint32_t structural_child_index;
+  uint32_t descendant_index;
+  const TSSymbol *alias_sequence;
+} CursorChildIterator;
+
+// CursorChildIterator
+
+static inline bool ts_tree_cursor_is_entry_visible(const TreeCursor *self, uint32_t index) {
+  TreeCursorEntry *entry = array_get(&self->stack, index);
+  if (index == 0 || ts_subtree_visible(*entry->subtree)) {
+    return true;
+  } else if (!ts_subtree_extra(*entry->subtree)) {
+    TreeCursorEntry *parent_entry = array_get(&self->stack, index - 1);
+    return ts_language_alias_at(
+      self->tree->language,
+      parent_entry->subtree->ptr->production_id,
+      entry->structural_child_index
+    );
+  } else {
+    return false;
+  }
+}
+
+static inline CursorChildIterator ts_tree_cursor_iterate_children(const TreeCursor *self) {
+  TreeCursorEntry *last_entry = array_back(&self->stack);
+  if (ts_subtree_child_count(*last_entry->subtree) == 0) {
+    return (CursorChildIterator) {NULL_SUBTREE, self->tree, length_zero(), 0, 0, 0, NULL};
+  }
+  const TSSymbol *alias_sequence = ts_language_alias_sequence(
+    self->tree->language,
+    last_entry->subtree->ptr->production_id
+  );
+
+  uint32_t descendant_index = last_entry->descendant_index;
+  if (ts_tree_cursor_is_entry_visible(self, self->stack.size - 1)) {
+    descendant_index += 1;
+  }
+
+  return (CursorChildIterator) {
+    .tree = self->tree,
+    .parent = *last_entry->subtree,
+    .position = last_entry->position,
+    .child_index = 0,
+    .structural_child_index = 0,
+    .descendant_index = descendant_index,
+    .alias_sequence = alias_sequence,
+  };
+}
+
+static inline bool ts_tree_cursor_child_iterator_next(
+  CursorChildIterator *self,
+  TreeCursorEntry *result,
+  bool *visible
+) {
+  if (!self->parent.ptr || self->child_index == self->parent.ptr->child_count) return false;
+  const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
+  *result = (TreeCursorEntry) {
+    .subtree = child,
+    .position = self->position,
+    .child_index = self->child_index,
+    .structural_child_index = self->structural_child_index,
+    .descendant_index = self->descendant_index,
+  };
+  *visible = ts_subtree_visible(*child);
+  bool extra = ts_subtree_extra(*child);
+  if (!extra) {
+    if (self->alias_sequence) {
+      *visible |= self->alias_sequence[self->structural_child_index];
+    }
+    self->structural_child_index++;
+  }
+
+  self->descendant_index += ts_subtree_visible_descendant_count(*child);
+  if (*visible) {
+    self->descendant_index += 1;
+  }
+
+  self->position = length_add(self->position, ts_subtree_size(*child));
+  self->child_index++;
+
+  if (self->child_index < self->parent.ptr->child_count) {
+    Subtree next_child = ts_subtree_children(self->parent)[self->child_index];
+    self->position = length_add(self->position, ts_subtree_padding(next_child));
+  }
+
+  return true;
+}
+
+// Return a position that, when `b` is added to it, yields `a`. This
+// can only be computed if `b` has zero rows. Otherwise, this function
+// returns `LENGTH_UNDEFINED`, and the caller needs to recompute
+// the position some other way.
+static inline Length length_backtrack(Length a, Length b) {
+  if (length_is_undefined(a) || b.extent.row != 0) {
+    return LENGTH_UNDEFINED;
+  }
+
+  Length result;
+  result.bytes = a.bytes - b.bytes;
+  result.extent.row = a.extent.row;
+  result.extent.column = a.extent.column - b.extent.column;
+  return result;
+}
+
+static inline bool ts_tree_cursor_child_iterator_previous(
+  CursorChildIterator *self,
+  TreeCursorEntry *result,
+  bool *visible
+) {
+  // this is mostly a reverse `ts_tree_cursor_child_iterator_next` taking into
+  // account unsigned underflow
+  if (!self->parent.ptr || (int8_t)self->child_index == -1) return false;
+  const Subtree *child = &ts_subtree_children(self->parent)[self->child_index];
+  *result = (TreeCursorEntry) {
+    .subtree = child,
+    .position = self->position,
+    .child_index = self->child_index,
+    .structural_child_index = self->structural_child_index,
+  };
+  *visible = ts_subtree_visible(*child);
+  bool extra = ts_subtree_extra(*child);
+
+  self->position = length_backtrack(self->position, ts_subtree_padding(*child));
+  self->child_index--;
+
+  if (!extra && self->alias_sequence) {
+    *visible |= self->alias_sequence[self->structural_child_index];
+    if (self->structural_child_index > 0) {
+      self->structural_child_index--;
+    }
+  }
+
+  // unsigned can underflow so compare it to child_count
+  if (self->child_index < self->parent.ptr->child_count) {
+    Subtree previous_child = ts_subtree_children(self->parent)[self->child_index];
+    Length size = ts_subtree_size(previous_child);
+    self->position = length_backtrack(self->position, size);
+  }
+
+  return true;
+}
+
+// TSTreeCursor - lifecycle
+
+TSTreeCursor ts_tree_cursor_new(TSNode node) {
+  TSTreeCursor self = {NULL, NULL, {0, 0, 0}};
+  ts_tree_cursor_init((TreeCursor *)&self, node);
+  return self;
+}
+
+void ts_tree_cursor_reset(TSTreeCursor *_self, TSNode node) {
+  ts_tree_cursor_init((TreeCursor *)_self, node);
+}
+
+void ts_tree_cursor_init(TreeCursor *self, TSNode node) {
+  self->tree = node.tree;
+  self->root_alias_symbol = node.context[3];
+  array_clear(&self->stack);
+  array_push(&self->stack, ((TreeCursorEntry) {
+    .subtree = (const Subtree *)node.id,
+    .position = {
+      ts_node_start_byte(node),
+      ts_node_start_point(node)
+    },
+    .child_index = 0,
+    .structural_child_index = 0,
+    .descendant_index = 0,
+  }));
+}
+
+void ts_tree_cursor_delete(TSTreeCursor *_self) {
+  TreeCursor *self = (TreeCursor *)_self;
+  array_delete(&self->stack);
+}
+
+// TSTreeCursor - walking the tree
+
+TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self) {
+  TreeCursor *self = (TreeCursor *)_self;
+  bool visible;
+  TreeCursorEntry entry;
+  CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
+  while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
+    if (visible) {
+      array_push(&self->stack, entry);
+      return TreeCursorStepVisible;
+    }
+    if (ts_subtree_visible_child_count(*entry.subtree) > 0) {
+      array_push(&self->stack, entry);
+      return TreeCursorStepHidden;
+    }
+  }
+  return TreeCursorStepNone;
+}
+
+bool ts_tree_cursor_goto_first_child(TSTreeCursor *self) {
+  for (;;) {
+    switch (ts_tree_cursor_goto_first_child_internal(self)) {
+      case TreeCursorStepHidden:
+        continue;
+      case TreeCursorStepVisible:
+        return true;
+      default:
+        return false;
+    }
+  }
+}
+
+TreeCursorStep ts_tree_cursor_goto_last_child_internal(TSTreeCursor *_self) {
+  TreeCursor *self = (TreeCursor *)_self;
+  bool visible;
+  TreeCursorEntry entry;
+  CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
+  if (!iterator.parent.ptr || iterator.parent.ptr->child_count == 0) return TreeCursorStepNone;
+
+  TreeCursorEntry last_entry = {0};
+  TreeCursorStep last_step = TreeCursorStepNone;
+  while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
+    if (visible) {
+      last_entry = entry;
+      last_step = TreeCursorStepVisible;
+    }
+    else if (ts_subtree_visible_child_count(*entry.subtree) > 0) {
+      last_entry = entry;
+      last_step = TreeCursorStepHidden;
+    }
+  }
+  if (last_entry.subtree) {
+    array_push(&self->stack, last_entry);
+    return last_step;
+  }
+
+  return TreeCursorStepNone;
+}
+
+bool ts_tree_cursor_goto_last_child(TSTreeCursor *self) {
+  for (;;) {
+    switch (ts_tree_cursor_goto_last_child_internal(self)) {
+      case TreeCursorStepHidden:
+        continue;
+      case TreeCursorStepVisible:
+        return true;
+      default:
+        return false;
+    }
+  }
+}
+
+static inline int64_t ts_tree_cursor_goto_first_child_for_byte_and_point(
+  TSTreeCursor *_self,
+  uint32_t goal_byte,
+  TSPoint goal_point
+) {
+  TreeCursor *self = (TreeCursor *)_self;
+  uint32_t initial_size = self->stack.size;
+  uint32_t visible_child_index = 0;
+
+  bool did_descend;
+  do {
+    did_descend = false;
+
+    bool visible;
+    TreeCursorEntry entry;
+    CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
+    while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
+      Length entry_end = length_add(entry.position, ts_subtree_size(*entry.subtree));
+      bool at_goal = entry_end.bytes > goal_byte && point_gt(entry_end.extent, goal_point);
+      uint32_t visible_child_count = ts_subtree_visible_child_count(*entry.subtree);
+      if (at_goal) {
+        if (visible) {
+          array_push(&self->stack, entry);
+          return visible_child_index;
+        }
+        if (visible_child_count > 0) {
+          array_push(&self->stack, entry);
+          did_descend = true;
+          break;
+        }
+      } else if (visible) {
+        visible_child_index++;
+      } else {
+        visible_child_index += visible_child_count;
+      }
+    }
+  } while (did_descend);
+
+  self->stack.size = initial_size;
+  return -1;
+}
+
+int64_t ts_tree_cursor_goto_first_child_for_byte(TSTreeCursor *self, uint32_t goal_byte) {
+  return ts_tree_cursor_goto_first_child_for_byte_and_point(self, goal_byte, POINT_ZERO);
+}
+
+int64_t ts_tree_cursor_goto_first_child_for_point(TSTreeCursor *self, TSPoint goal_point) {
+  return ts_tree_cursor_goto_first_child_for_byte_and_point(self, 0, goal_point);
+}
+
+TreeCursorStep ts_tree_cursor_goto_sibling_internal(
+  TSTreeCursor *_self,
+  bool (*advance)(CursorChildIterator *, TreeCursorEntry *, bool *)
+) {
+  TreeCursor *self = (TreeCursor *)_self;
+  uint32_t initial_size = self->stack.size;
+
+  while (self->stack.size > 1) {
+    TreeCursorEntry entry = array_pop(&self->stack);
+    CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
+    iterator.child_index = entry.child_index;
+    iterator.structural_child_index = entry.structural_child_index;
+    iterator.position = entry.position;
+    iterator.descendant_index = entry.descendant_index;
+
+    bool visible = false;
+    advance(&iterator, &entry, &visible);
+    if (visible && self->stack.size + 1 < initial_size) break;
+
+    while (advance(&iterator, &entry, &visible)) {
+      if (visible) {
+        array_push(&self->stack, entry);
+        return TreeCursorStepVisible;
+      }
+
+      if (ts_subtree_visible_child_count(*entry.subtree)) {
+        array_push(&self->stack, entry);
+        return TreeCursorStepHidden;
+      }
+    }
+  }
+
+  self->stack.size = initial_size;
+  return TreeCursorStepNone;
+}
+
+TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self) {
+  return ts_tree_cursor_goto_sibling_internal(_self, ts_tree_cursor_child_iterator_next);
+}
+
+bool ts_tree_cursor_goto_next_sibling(TSTreeCursor *self) {
+  switch (ts_tree_cursor_goto_next_sibling_internal(self)) {
+    case TreeCursorStepHidden:
+      ts_tree_cursor_goto_first_child(self);
+      return true;
+    case TreeCursorStepVisible:
+      return true;
+    default:
+      return false;
+  }
+}
+
+TreeCursorStep ts_tree_cursor_goto_previous_sibling_internal(TSTreeCursor *_self) {
+  // since subtracting across row loses column information, we may have to
+  // restore it
+  TreeCursor *self = (TreeCursor *)_self;
+
+  // for that, save current position before traversing
+  TreeCursorStep step = ts_tree_cursor_goto_sibling_internal(
+      _self, ts_tree_cursor_child_iterator_previous);
+  if (step == TreeCursorStepNone)
+    return step;
+
+  // if length is already valid, there's no need to recompute it
+  if (!length_is_undefined(array_back(&self->stack)->position))
+    return step;
+
+  // restore position from the parent node
+  const TreeCursorEntry *parent = array_get(&self->stack, self->stack.size - 2);
+  Length position = parent->position;
+  uint32_t child_index = array_back(&self->stack)->child_index;
+  const Subtree *children = ts_subtree_children((*(parent->subtree)));
+
+  if (child_index > 0) {
+    // skip first child padding since its position should match the position of the parent
+    position = length_add(position, ts_subtree_size(children[0]));
+    for (uint32_t i = 1; i < child_index; ++i) {
+      position = length_add(position, ts_subtree_total_size(children[i]));
+    }
+    position = length_add(position, ts_subtree_padding(children[child_index]));
+  }
+
+  array_back(&self->stack)->position = position;
+
+  return step;
+}
+
+bool ts_tree_cursor_goto_previous_sibling(TSTreeCursor *self) {
+  switch (ts_tree_cursor_goto_previous_sibling_internal(self)) {
+    case TreeCursorStepHidden:
+      ts_tree_cursor_goto_last_child(self);
+      return true;
+    case TreeCursorStepVisible:
+      return true;
+    default:
+      return false;
+  }
+}
+
+bool ts_tree_cursor_goto_parent(TSTreeCursor *_self) {
+  TreeCursor *self = (TreeCursor *)_self;
+  for (unsigned i = self->stack.size - 2; i + 1 > 0; i--) {
+    if (ts_tree_cursor_is_entry_visible(self, i)) {
+      self->stack.size = i + 1;
+      return true;
+    }
+  }
+  return false;
+}
+
+void ts_tree_cursor_goto_descendant(
+  TSTreeCursor *_self,
+  uint32_t goal_descendant_index
+) {
+  TreeCursor *self = (TreeCursor *)_self;
+
+  // Ascend to the lowest ancestor that contains the goal node.
+  for (;;) {
+    uint32_t i = self->stack.size - 1;
+    TreeCursorEntry *entry = array_get(&self->stack, i);
+    uint32_t next_descendant_index =
+      entry->descendant_index +
+      (ts_tree_cursor_is_entry_visible(self, i) ? 1 : 0) +
+      ts_subtree_visible_descendant_count(*entry->subtree);
+    if (
+      (entry->descendant_index <= goal_descendant_index) &&
+      (next_descendant_index > goal_descendant_index)
+    ) {
+      break;
+    } else if (self->stack.size <= 1) {
+      return;
+    } else {
+      self->stack.size--;
+    }
+  }
+
+  // Descend to the goal node.
+  bool did_descend = true;
+  do {
+    did_descend = false;
+    bool visible;
+    TreeCursorEntry entry;
+    CursorChildIterator iterator = ts_tree_cursor_iterate_children(self);
+    if (iterator.descendant_index > goal_descendant_index) {
+      return;
+    }
+
+    while (ts_tree_cursor_child_iterator_next(&iterator, &entry, &visible)) {
+      if (iterator.descendant_index > goal_descendant_index) {
+        array_push(&self->stack, entry);
+        if (visible && entry.descendant_index == goal_descendant_index) {
+          return;
+        } else {
+          did_descend = true;
+          break;
+        }
+      }
+    }
+  } while (did_descend);
+}
+
+uint32_t ts_tree_cursor_current_descendant_index(const TSTreeCursor *_self) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+  TreeCursorEntry *last_entry = array_back(&self->stack);
+  return last_entry->descendant_index;
+}
+
+TSNode ts_tree_cursor_current_node(const TSTreeCursor *_self) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+  TreeCursorEntry *last_entry = array_back(&self->stack);
+  bool is_extra = ts_subtree_extra(*last_entry->subtree);
+  TSSymbol alias_symbol = is_extra ? 0 : self->root_alias_symbol;
+  if (self->stack.size > 1 && !is_extra) {
+    TreeCursorEntry *parent_entry = array_get(&self->stack, self->stack.size - 2);
+    alias_symbol = ts_language_alias_at(
+      self->tree->language,
+      parent_entry->subtree->ptr->production_id,
+      last_entry->structural_child_index
+    );
+  }
+  return ts_node_new(
+    self->tree,
+    last_entry->subtree,
+    last_entry->position,
+    alias_symbol
+  );
+}
+
+// Private - Get various facts about the current node that are needed
+// when executing tree queries.
+void ts_tree_cursor_current_status(
+  const TSTreeCursor *_self,
+  TSFieldId *field_id,
+  bool *has_later_siblings,
+  bool *has_later_named_siblings,
+  bool *can_have_later_siblings_with_this_field,
+  TSSymbol *supertypes,
+  unsigned *supertype_count
+) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+  unsigned max_supertypes = *supertype_count;
+  *field_id = 0;
+  *supertype_count = 0;
+  *has_later_siblings = false;
+  *has_later_named_siblings = false;
+  *can_have_later_siblings_with_this_field = false;
+
+  // Walk up the tree, visiting the current node and its invisible ancestors,
+  // because fields can refer to nodes through invisible *wrapper* nodes,
+  for (unsigned i = self->stack.size - 1; i > 0; i--) {
+    TreeCursorEntry *entry = array_get(&self->stack, i);
+    TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1);
+
+    const TSSymbol *alias_sequence = ts_language_alias_sequence(
+      self->tree->language,
+      parent_entry->subtree->ptr->production_id
+    );
+
+    #define subtree_symbol(subtree, structural_child_index) \
+      ((                                                    \
+        !ts_subtree_extra(subtree) &&                       \
+        alias_sequence &&                                   \
+        alias_sequence[structural_child_index]              \
+      ) ?                                                   \
+        alias_sequence[structural_child_index] :            \
+        ts_subtree_symbol(subtree))
+
+    // Stop walking up when a visible ancestor is found.
+    TSSymbol entry_symbol = subtree_symbol(
+      *entry->subtree,
+      entry->structural_child_index
+    );
+    TSSymbolMetadata entry_metadata = ts_language_symbol_metadata(
+      self->tree->language,
+      entry_symbol
+    );
+    if (i != self->stack.size - 1 && entry_metadata.visible) break;
+
+    // Record any supertypes
+    if (entry_metadata.supertype && *supertype_count < max_supertypes) {
+      supertypes[*supertype_count] = entry_symbol;
+      (*supertype_count)++;
+    }
+
+    // Determine if the current node has later siblings.
+    if (!*has_later_siblings) {
+      unsigned sibling_count = parent_entry->subtree->ptr->child_count;
+      unsigned structural_child_index = entry->structural_child_index;
+      if (!ts_subtree_extra(*entry->subtree)) structural_child_index++;
+      for (unsigned j = entry->child_index + 1; j < sibling_count; j++) {
+        Subtree sibling = ts_subtree_children(*parent_entry->subtree)[j];
+        TSSymbolMetadata sibling_metadata = ts_language_symbol_metadata(
+          self->tree->language,
+          subtree_symbol(sibling, structural_child_index)
+        );
+        if (sibling_metadata.visible) {
+          *has_later_siblings = true;
+          if (*has_later_named_siblings) break;
+          if (sibling_metadata.named) {
+            *has_later_named_siblings = true;
+            break;
+          }
+        } else if (ts_subtree_visible_child_count(sibling) > 0) {
+          *has_later_siblings = true;
+          if (*has_later_named_siblings) break;
+          if (sibling.ptr->named_child_count > 0) {
+            *has_later_named_siblings = true;
+            break;
+          }
+        }
+        if (!ts_subtree_extra(sibling)) structural_child_index++;
+      }
+    }
+
+    #undef subtree_symbol
+
+    if (!ts_subtree_extra(*entry->subtree)) {
+      const TSFieldMapEntry *field_map, *field_map_end;
+      ts_language_field_map(
+        self->tree->language,
+        parent_entry->subtree->ptr->production_id,
+        &field_map, &field_map_end
+      );
+
+      // Look for a field name associated with the current node.
+      if (!*field_id) {
+        for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
+          if (!map->inherited && map->child_index == entry->structural_child_index) {
+            *field_id = map->field_id;
+            break;
+          }
+        }
+      }
+
+      // Determine if the current node can have later siblings with the same field name.
+      if (*field_id) {
+        for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
+          if (
+            map->field_id == *field_id &&
+            map->child_index > entry->structural_child_index
+          ) {
+            *can_have_later_siblings_with_this_field = true;
+            break;
+          }
+        }
+      }
+    }
+  }
+}
+
+uint32_t ts_tree_cursor_current_depth(const TSTreeCursor *_self) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+  uint32_t depth = 0;
+  for (unsigned i = 1; i < self->stack.size; i++) {
+    if (ts_tree_cursor_is_entry_visible(self, i)) {
+      depth++;
+    }
+  }
+  return depth;
+}
+
+TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+  for (int i = (int)self->stack.size - 2; i >= 0; i--) {
+    TreeCursorEntry *entry = array_get(&self->stack, i);
+    bool is_visible = true;
+    TSSymbol alias_symbol = 0;
+    if (i > 0) {
+      TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1);
+      alias_symbol = ts_language_alias_at(
+        self->tree->language,
+        parent_entry->subtree->ptr->production_id,
+        entry->structural_child_index
+      );
+      is_visible = (alias_symbol != 0) || ts_subtree_visible(*entry->subtree);
+    }
+    if (is_visible) {
+      return ts_node_new(
+        self->tree,
+        entry->subtree,
+        entry->position,
+        alias_symbol
+      );
+    }
+  }
+  return ts_node_new(NULL, NULL, length_zero(), 0);
+}
+
+TSFieldId ts_tree_cursor_current_field_id(const TSTreeCursor *_self) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+
+  // Walk up the tree, visiting the current node and its invisible ancestors.
+  for (unsigned i = self->stack.size - 1; i > 0; i--) {
+    TreeCursorEntry *entry = array_get(&self->stack, i);
+    TreeCursorEntry *parent_entry = array_get(&self->stack, i - 1);
+
+    // Stop walking up when another visible node is found.
+    if (
+      i != self->stack.size - 1 &&
+      ts_tree_cursor_is_entry_visible(self, i)
+    ) break;
+
+    if (ts_subtree_extra(*entry->subtree)) break;
+
+    const TSFieldMapEntry *field_map, *field_map_end;
+    ts_language_field_map(
+      self->tree->language,
+      parent_entry->subtree->ptr->production_id,
+      &field_map, &field_map_end
+    );
+    for (const TSFieldMapEntry *map = field_map; map < field_map_end; map++) {
+      if (!map->inherited && map->child_index == entry->structural_child_index) {
+        return map->field_id;
+      }
+    }
+  }
+  return 0;
+}
+
+const char *ts_tree_cursor_current_field_name(const TSTreeCursor *_self) {
+  TSFieldId id = ts_tree_cursor_current_field_id(_self);
+  if (id) {
+    const TreeCursor *self = (const TreeCursor *)_self;
+    return self->tree->language->field_names[id];
+  } else {
+    return NULL;
+  }
+}
+
+TSTreeCursor ts_tree_cursor_copy(const TSTreeCursor *_cursor) {
+  const TreeCursor *cursor = (const TreeCursor *)_cursor;
+  TSTreeCursor res = {NULL, NULL, {0, 0}};
+  TreeCursor *copy = (TreeCursor *)&res;
+  copy->tree = cursor->tree;
+  copy->root_alias_symbol = cursor->root_alias_symbol;
+  array_init(&copy->stack);
+  array_push_all(&copy->stack, &cursor->stack);
+  return res;
+}
+
+void ts_tree_cursor_reset_to(TSTreeCursor *_dst, const TSTreeCursor *_src) {
+  const TreeCursor *cursor = (const TreeCursor *)_src;
+  TreeCursor *copy = (TreeCursor *)_dst;
+  copy->tree = cursor->tree;
+  copy->root_alias_symbol = cursor->root_alias_symbol;
+  array_clear(&copy->stack);
+  array_push_all(&copy->stack, &cursor->stack);
+}

third_party/tree-sitter/lib/src/tree_cursor.h

@@ -0,0 +1,48 @@
+#ifndef TREE_SITTER_TREE_CURSOR_H_
+#define TREE_SITTER_TREE_CURSOR_H_
+
+#include "./subtree.h"
+
+typedef struct {
+  const Subtree *subtree;
+  Length position;
+  uint32_t child_index;
+  uint32_t structural_child_index;
+  uint32_t descendant_index;
+} TreeCursorEntry;
+
+typedef struct {
+  const TSTree *tree;
+  Array(TreeCursorEntry) stack;
+  TSSymbol root_alias_symbol;
+} TreeCursor;
+
+typedef enum {
+  TreeCursorStepNone,
+  TreeCursorStepHidden,
+  TreeCursorStepVisible,
+} TreeCursorStep;
+
+void ts_tree_cursor_init(TreeCursor *self, TSNode node);
+void ts_tree_cursor_current_status(
+  const TSTreeCursor *_self,
+  TSFieldId *field_id,
+  bool *has_later_siblings,
+  bool *has_later_named_siblings,
+  bool *can_have_later_siblings_with_this_field,
+  TSSymbol *supertypes,
+  unsigned *supertype_count
+);
+
+TreeCursorStep ts_tree_cursor_goto_first_child_internal(TSTreeCursor *_self);
+TreeCursorStep ts_tree_cursor_goto_next_sibling_internal(TSTreeCursor *_self);
+
+static inline Subtree ts_tree_cursor_current_subtree(const TSTreeCursor *_self) {
+  const TreeCursor *self = (const TreeCursor *)_self;
+  TreeCursorEntry *last_entry = array_back(&self->stack);
+  return *last_entry->subtree;
+}
+
+TSNode ts_tree_cursor_parent_node(const TSTreeCursor *_self);
+
+#endif  // TREE_SITTER_TREE_CURSOR_H_

third_party/tree-sitter/lib/src/ts_assert.h

@@ -0,0 +1,11 @@
+#ifndef TREE_SITTER_ASSERT_H_
+#define TREE_SITTER_ASSERT_H_
+
+#ifdef NDEBUG
+#define ts_assert(e) ((void)(e))
+#else
+#include <assert.h>
+#define ts_assert(e) assert(e)
+#endif
+
+#endif // TREE_SITTER_ASSERT_H_

third_party/tree-sitter/lib/src/unicode.h

@@ -0,0 +1,75 @@
+#ifndef TREE_SITTER_UNICODE_H_
+#define TREE_SITTER_UNICODE_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include <limits.h>
+#include <stdint.h>
+
+#define U_EXPORT
+#define U_EXPORT2
+#include "unicode/utf8.h"
+#include "unicode/utf16.h"
+#include "portable/endian.h"
+
+#define U16_NEXT_LE(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=le16toh((s)[(i)++]); \
+    if(U16_IS_LEAD(c)) { \
+        uint16_t __c2; \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+            ++(i); \
+            (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+#define U16_NEXT_BE(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=be16toh((s)[(i)++]); \
+    if(U16_IS_LEAD(c)) { \
+        uint16_t __c2; \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+            ++(i); \
+            (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+static const int32_t TS_DECODE_ERROR = U_SENTINEL;
+
+static inline uint32_t ts_decode_utf8(
+  const uint8_t *string,
+  uint32_t length,
+  int32_t *code_point
+) {
+  uint32_t i = 0;
+  U8_NEXT(string, i, length, *code_point);
+  return i;
+}
+
+static inline uint32_t ts_decode_utf16_le(
+  const uint8_t *string,
+  uint32_t length,
+  int32_t *code_point
+) {
+  uint32_t i = 0;
+  U16_NEXT_LE(((uint16_t *)string), i, length, *code_point);
+  return i * 2;
+}
+
+static inline uint32_t ts_decode_utf16_be(
+  const uint8_t *string,
+  uint32_t length,
+  int32_t *code_point
+) {
+  uint32_t i = 0;
+  U16_NEXT_BE(((uint16_t *)string), i, length, *code_point);
+  return i * 2;
+}
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_UNICODE_H_

third_party/tree-sitter/lib/src/unicode/ICU_SHA

@@ -0,0 +1 @@
+552b01f61127d30d6589aa4bf99468224979b661

third_party/tree-sitter/lib/src/unicode/LICENSE

@@ -0,0 +1,414 @@
+COPYRIGHT AND PERMISSION NOTICE (ICU 58 and later)
+
+Copyright © 1991-2019 Unicode, Inc. All rights reserved.
+Distributed under the Terms of Use in https://www.unicode.org/copyright.html.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of the Unicode data files and any associated documentation
+(the "Data Files") or Unicode software and any associated documentation
+(the "Software") to deal in the Data Files or Software
+without restriction, including without limitation the rights to use,
+copy, modify, merge, publish, distribute, and/or sell copies of
+the Data Files or Software, and to permit persons to whom the Data Files
+or Software are furnished to do so, provided that either
+(a) this copyright and permission notice appear with all copies
+of the Data Files or Software, or
+(b) this copyright and permission notice appear in associated
+Documentation.
+
+THE DATA FILES AND SOFTWARE ARE PROVIDED "AS IS", WITHOUT WARRANTY OF
+ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE
+WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT OF THIRD PARTY RIGHTS.
+IN NO EVENT SHALL THE COPYRIGHT HOLDER OR HOLDERS INCLUDED IN THIS
+NOTICE BE LIABLE FOR ANY CLAIM, OR ANY SPECIAL INDIRECT OR CONSEQUENTIAL
+DAMAGES, OR ANY DAMAGES WHATSOEVER RESULTING FROM LOSS OF USE,
+DATA OR PROFITS, WHETHER IN AN ACTION OF CONTRACT, NEGLIGENCE OR OTHER
+TORTIOUS ACTION, ARISING OUT OF OR IN CONNECTION WITH THE USE OR
+PERFORMANCE OF THE DATA FILES OR SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder
+shall not be used in advertising or otherwise to promote the sale,
+use or other dealings in these Data Files or Software without prior
+written authorization of the copyright holder.
+
+---------------------
+
+Third-Party Software Licenses
+
+This section contains third-party software notices and/or additional
+terms for licensed third-party software components included within ICU
+libraries.
+
+1. ICU License - ICU 1.8.1 to ICU 57.1
+
+COPYRIGHT AND PERMISSION NOTICE
+
+Copyright (c) 1995-2016 International Business Machines Corporation and others
+All rights reserved.
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, and/or sell copies of the Software, and to permit persons
+to whom the Software is furnished to do so, provided that the above
+copyright notice(s) and this permission notice appear in all copies of
+the Software and that both the above copyright notice(s) and this
+permission notice appear in supporting documentation.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT
+OF THIRD PARTY RIGHTS. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR
+HOLDERS INCLUDED IN THIS NOTICE BE LIABLE FOR ANY CLAIM, OR ANY
+SPECIAL INDIRECT OR CONSEQUENTIAL DAMAGES, OR ANY DAMAGES WHATSOEVER
+RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION OF
+CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
+CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
+
+Except as contained in this notice, the name of a copyright holder
+shall not be used in advertising or otherwise to promote the sale, use
+or other dealings in this Software without prior written authorization
+of the copyright holder.
+
+All trademarks and registered trademarks mentioned herein are the
+property of their respective owners.
+
+2. Chinese/Japanese Word Break Dictionary Data (cjdict.txt)
+
+ #     The Google Chrome software developed by Google is licensed under
+ # the BSD license. Other software included in this distribution is
+ # provided under other licenses, as set forth below.
+ #
+ #  The BSD License
+ #  http://opensource.org/licenses/bsd-license.php
+ #  Copyright (C) 2006-2008, Google Inc.
+ #
+ #  All rights reserved.
+ #
+ #  Redistribution and use in source and binary forms, with or without
+ # modification, are permitted provided that the following conditions are met:
+ #
+ #  Redistributions of source code must retain the above copyright notice,
+ # this list of conditions and the following disclaimer.
+ #  Redistributions in binary form must reproduce the above
+ # copyright notice, this list of conditions and the following
+ # disclaimer in the documentation and/or other materials provided with
+ # the distribution.
+ #  Neither the name of  Google Inc. nor the names of its
+ # contributors may be used to endorse or promote products derived from
+ # this software without specific prior written permission.
+ #
+ #
+ #  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ # CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ # INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ # MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ # DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
+ # LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ # CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ # SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR
+ # BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
+ # LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
+ # NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
+ # SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ #
+ #
+ #  The word list in cjdict.txt are generated by combining three word lists
+ # listed below with further processing for compound word breaking. The
+ # frequency is generated with an iterative training against Google web
+ # corpora.
+ #
+ #  * Libtabe (Chinese)
+ #    - https://sourceforge.net/project/?group_id=1519
+ #    - Its license terms and conditions are shown below.
+ #
+ #  * IPADIC (Japanese)
+ #    - http://chasen.aist-nara.ac.jp/chasen/distribution.html
+ #    - Its license terms and conditions are shown below.
+ #
+ #  ---------COPYING.libtabe ---- BEGIN--------------------
+ #
+ #  /*
+ #   * Copyright (c) 1999 TaBE Project.
+ #   * Copyright (c) 1999 Pai-Hsiang Hsiao.
+ #   * All rights reserved.
+ #   *
+ #   * Redistribution and use in source and binary forms, with or without
+ #   * modification, are permitted provided that the following conditions
+ #   * are met:
+ #   *
+ #   * . Redistributions of source code must retain the above copyright
+ #   *   notice, this list of conditions and the following disclaimer.
+ #   * . Redistributions in binary form must reproduce the above copyright
+ #   *   notice, this list of conditions and the following disclaimer in
+ #   *   the documentation and/or other materials provided with the
+ #   *   distribution.
+ #   * . Neither the name of the TaBE Project nor the names of its
+ #   *   contributors may be used to endorse or promote products derived
+ #   *   from this software without specific prior written permission.
+ #   *
+ #   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ #   * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ #   * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ #   * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ #   * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ #   * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ #   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ #   * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ #   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ #   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ #   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ #   * OF THE POSSIBILITY OF SUCH DAMAGE.
+ #   */
+ #
+ #  /*
+ #   * Copyright (c) 1999 Computer Systems and Communication Lab,
+ #   *                    Institute of Information Science, Academia
+ #       *                    Sinica. All rights reserved.
+ #   *
+ #   * Redistribution and use in source and binary forms, with or without
+ #   * modification, are permitted provided that the following conditions
+ #   * are met:
+ #   *
+ #   * . Redistributions of source code must retain the above copyright
+ #   *   notice, this list of conditions and the following disclaimer.
+ #   * . Redistributions in binary form must reproduce the above copyright
+ #   *   notice, this list of conditions and the following disclaimer in
+ #   *   the documentation and/or other materials provided with the
+ #   *   distribution.
+ #   * . Neither the name of the Computer Systems and Communication Lab
+ #   *   nor the names of its contributors may be used to endorse or
+ #   *   promote products derived from this software without specific
+ #   *   prior written permission.
+ #   *
+ #   * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ #   * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ #   * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ #   * FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ #   * REGENTS OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
+ #   * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ #   * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ #   * SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ #   * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ #   * STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ #   * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ #   * OF THE POSSIBILITY OF SUCH DAMAGE.
+ #   */
+ #
+ #  Copyright 1996 Chih-Hao Tsai @ Beckman Institute,
+ #      University of Illinois
+ #  c-tsai4@uiuc.edu  http://casper.beckman.uiuc.edu/~c-tsai4
+ #
+ #  ---------------COPYING.libtabe-----END--------------------------------
+ #
+ #
+ #  ---------------COPYING.ipadic-----BEGIN-------------------------------
+ #
+ #  Copyright 2000, 2001, 2002, 2003 Nara Institute of Science
+ #  and Technology.  All Rights Reserved.
+ #
+ #  Use, reproduction, and distribution of this software is permitted.
+ #  Any copy of this software, whether in its original form or modified,
+ #  must include both the above copyright notice and the following
+ #  paragraphs.
+ #
+ #  Nara Institute of Science and Technology (NAIST),
+ #  the copyright holders, disclaims all warranties with regard to this
+ #  software, including all implied warranties of merchantability and
+ #  fitness, in no event shall NAIST be liable for
+ #  any special, indirect or consequential damages or any damages
+ #  whatsoever resulting from loss of use, data or profits, whether in an
+ #  action of contract, negligence or other tortuous action, arising out
+ #  of or in connection with the use or performance of this software.
+ #
+ #  A large portion of the dictionary entries
+ #  originate from ICOT Free Software.  The following conditions for ICOT
+ #  Free Software applies to the current dictionary as well.
+ #
+ #  Each User may also freely distribute the Program, whether in its
+ #  original form or modified, to any third party or parties, PROVIDED
+ #  that the provisions of Section 3 ("NO WARRANTY") will ALWAYS appear
+ #  on, or be attached to, the Program, which is distributed substantially
+ #  in the same form as set out herein and that such intended
+ #  distribution, if actually made, will neither violate or otherwise
+ #  contravene any of the laws and regulations of the countries having
+ #  jurisdiction over the User or the intended distribution itself.
+ #
+ #  NO WARRANTY
+ #
+ #  The program was produced on an experimental basis in the course of the
+ #  research and development conducted during the project and is provided
+ #  to users as so produced on an experimental basis.  Accordingly, the
+ #  program is provided without any warranty whatsoever, whether express,
+ #  implied, statutory or otherwise.  The term "warranty" used herein
+ #  includes, but is not limited to, any warranty of the quality,
+ #  performance, merchantability and fitness for a particular purpose of
+ #  the program and the nonexistence of any infringement or violation of
+ #  any right of any third party.
+ #
+ #  Each user of the program will agree and understand, and be deemed to
+ #  have agreed and understood, that there is no warranty whatsoever for
+ #  the program and, accordingly, the entire risk arising from or
+ #  otherwise connected with the program is assumed by the user.
+ #
+ #  Therefore, neither ICOT, the copyright holder, or any other
+ #  organization that participated in or was otherwise related to the
+ #  development of the program and their respective officials, directors,
+ #  officers and other employees shall be held liable for any and all
+ #  damages, including, without limitation, general, special, incidental
+ #  and consequential damages, arising out of or otherwise in connection
+ #  with the use or inability to use the program or any product, material
+ #  or result produced or otherwise obtained by using the program,
+ #  regardless of whether they have been advised of, or otherwise had
+ #  knowledge of, the possibility of such damages at any time during the
+ #  project or thereafter.  Each user will be deemed to have agreed to the
+ #  foregoing by his or her commencement of use of the program.  The term
+ #  "use" as used herein includes, but is not limited to, the use,
+ #  modification, copying and distribution of the program and the
+ #  production of secondary products from the program.
+ #
+ #  In the case where the program, whether in its original form or
+ #  modified, was distributed or delivered to or received by a user from
+ #  any person, organization or entity other than ICOT, unless it makes or
+ #  grants independently of ICOT any specific warranty to the user in
+ #  writing, such person, organization or entity, will also be exempted
+ #  from and not be held liable to the user for any such damages as noted
+ #  above as far as the program is concerned.
+ #
+ #  ---------------COPYING.ipadic-----END----------------------------------
+
+3. Lao Word Break Dictionary Data (laodict.txt)
+
+ #  Copyright (c) 2013 International Business Machines Corporation
+ #  and others. All Rights Reserved.
+ #
+ # Project: http://code.google.com/p/lao-dictionary/
+ # Dictionary: http://lao-dictionary.googlecode.com/git/Lao-Dictionary.txt
+ # License: http://lao-dictionary.googlecode.com/git/Lao-Dictionary-LICENSE.txt
+ #              (copied below)
+ #
+ #  This file is derived from the above dictionary, with slight
+ #  modifications.
+ #  ----------------------------------------------------------------------
+ #  Copyright (C) 2013 Brian Eugene Wilson, Robert Martin Campbell.
+ #  All rights reserved.
+ #
+ #  Redistribution and use in source and binary forms, with or without
+ #  modification,
+ #  are permitted provided that the following conditions are met:
+ #
+ #
+ # Redistributions of source code must retain the above copyright notice, this
+ #  list of conditions and the following disclaimer. Redistributions in
+ #  binary form must reproduce the above copyright notice, this list of
+ #  conditions and the following disclaimer in the documentation and/or
+ #  other materials provided with the distribution.
+ #
+ #
+ # THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+ # "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+ # LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
+ # FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
+ # COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT,
+ # INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
+ # (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+ # SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ # HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT,
+ # STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ # ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED
+ # OF THE POSSIBILITY OF SUCH DAMAGE.
+ #  --------------------------------------------------------------------------
+
+4. Burmese Word Break Dictionary Data (burmesedict.txt)
+
+ #  Copyright (c) 2014 International Business Machines Corporation
+ #  and others. All Rights Reserved.
+ #
+ #  This list is part of a project hosted at:
+ #    github.com/kanyawtech/myanmar-karen-word-lists
+ #
+ #  --------------------------------------------------------------------------
+ #  Copyright (c) 2013, LeRoy Benjamin Sharon
+ #  All rights reserved.
+ #
+ #  Redistribution and use in source and binary forms, with or without
+ #  modification, are permitted provided that the following conditions
+ #  are met: Redistributions of source code must retain the above
+ #  copyright notice, this list of conditions and the following
+ #  disclaimer.  Redistributions in binary form must reproduce the
+ #  above copyright notice, this list of conditions and the following
+ #  disclaimer in the documentation and/or other materials provided
+ #  with the distribution.
+ #
+ #    Neither the name Myanmar Karen Word Lists, nor the names of its
+ #    contributors may be used to endorse or promote products derived
+ #    from this software without specific prior written permission.
+ #
+ #  THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND
+ #  CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES,
+ #  INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF
+ #  MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+ #  DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS
+ #  BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
+ #  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
+ #  TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ #  DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
+ #  ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR
+ #  TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF
+ #  THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ #  SUCH DAMAGE.
+ #  --------------------------------------------------------------------------
+
+5. Time Zone Database
+
+  ICU uses the public domain data and code derived from Time Zone
+Database for its time zone support. The ownership of the TZ database
+is explained in BCP 175: Procedure for Maintaining the Time Zone
+Database section 7.
+
+ # 7.  Database Ownership
+ #
+ #    The TZ database itself is not an IETF Contribution or an IETF
+ #    document.  Rather it is a pre-existing and regularly updated work
+ #    that is in the public domain, and is intended to remain in the
+ #    public domain.  Therefore, BCPs 78 [RFC5378] and 79 [RFC3979] do
+ #    not apply to the TZ Database or contributions that individuals make
+ #    to it.  Should any claims be made and substantiated against the TZ
+ #    Database, the organization that is providing the IANA
+ #    Considerations defined in this RFC, under the memorandum of
+ #    understanding with the IETF, currently ICANN, may act in accordance
+ #    with all competent court orders.  No ownership claims will be made
+ #    by ICANN or the IETF Trust on the database or the code.  Any person
+ #    making a contribution to the database or code waives all rights to
+ #    future claims in that contribution or in the TZ Database.
+
+6. Google double-conversion
+
+Copyright 2006-2011, the V8 project authors. All rights reserved.
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are
+met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+    * Neither the name of Google Inc. nor the names of its
+      contributors may be used to endorse or promote products derived
+      from this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

third_party/tree-sitter/lib/src/unicode/README.md

@@ -0,0 +1,29 @@
+# ICU Parts
+
+This directory contains a small subset of files from the Unicode organization's [ICU repository](https://github.com/unicode-org/icu).
+
+### License
+
+The license for these files is contained in the `LICENSE` file within this directory.
+
+### Contents
+
+* Source files taken from the [`icu4c/source/common/unicode`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c/source/common/unicode) directory:
+  * `utf8.h`
+  * `utf16.h`
+  * `umachine.h`
+* Empty source files that are referenced by the above source files, but whose original contents in `libicu` are not needed:
+  * `ptypes.h`
+  * `urename.h`
+  * `utf.h`
+* `ICU_SHA` - File containing the Git SHA of the commit in the `icu` repository from which the files were obtained.
+* `LICENSE` - The license file from the [`icu4c`](https://github.com/unicode-org/icu/tree/552b01f61127d30d6589aa4bf99468224979b661/icu4c) directory of the `icu` repository.
+* `README.md` - This text file.
+
+### Updating ICU
+
+To incorporate changes from the upstream `icu` repository:
+
+* Update `ICU_SHA` with the new Git SHA.
+* Update `LICENSE` with the license text from the directory mentioned above.
+* Update `utf8.h`, `utf16.h`, and `umachine.h` with their new contents in the `icu` repository.

third_party/tree-sitter/lib/src/unicode/ptypes.h

@@ -0,0 +1 @@
+// This file must exist in order for `utf8.h` and `utf16.h` to be used.

third_party/tree-sitter/lib/src/unicode/umachine.h

@@ -0,0 +1,448 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+******************************************************************************
+*
+*   Copyright (C) 1999-2015, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+******************************************************************************
+*   file name:  umachine.h
+*   encoding:   UTF-8
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999sep13
+*   created by: Markus W. Scherer
+*
+*   This file defines basic types and constants for ICU to be
+*   platform-independent. umachine.h and utf.h are included into
+*   utypes.h to provide all the general definitions for ICU.
+*   All of these definitions used to be in utypes.h before
+*   the UTF-handling macros made this unmaintainable.
+*/
+
+#ifndef __UMACHINE_H__
+#define __UMACHINE_H__
+
+
+/**
+ * \file
+ * \brief Basic types and constants for UTF
+ *
+ * <h2> Basic types and constants for UTF </h2>
+ *   This file defines basic types and constants for utf.h to be
+ *   platform-independent. umachine.h and utf.h are included into
+ *   utypes.h to provide all the general definitions for ICU.
+ *   All of these definitions used to be in utypes.h before
+ *   the UTF-handling macros made this unmaintainable.
+ *
+ */
+/*==========================================================================*/
+/* Include platform-dependent definitions                                   */
+/* which are contained in the platform-specific file platform.h             */
+/*==========================================================================*/
+
+#include "unicode/ptypes.h" /* platform.h is included in ptypes.h */
+
+/*
+ * ANSI C headers:
+ * stddef.h defines wchar_t
+ */
+#include <stddef.h>
+
+/*==========================================================================*/
+/* For C wrappers, we use the symbol U_STABLE.                                */
+/* This works properly if the includer is C or C++.                         */
+/* Functions are declared   U_STABLE return-type U_EXPORT2 function-name()... */
+/*==========================================================================*/
+
+/**
+ * \def U_CFUNC
+ * This is used in a declaration of a library private ICU C function.
+ * @stable ICU 2.4
+ */
+
+/**
+ * \def U_CDECL_BEGIN
+ * This is used to begin a declaration of a library private ICU C API.
+ * @stable ICU 2.4
+ */
+
+/**
+ * \def U_CDECL_END
+ * This is used to end a declaration of a library private ICU C API
+ * @stable ICU 2.4
+ */
+
+#ifdef __cplusplus
+#   define U_CFUNC extern "C"
+#   define U_CDECL_BEGIN extern "C" {
+#   define U_CDECL_END   }
+#else
+#   define U_CFUNC extern
+#   define U_CDECL_BEGIN
+#   define U_CDECL_END
+#endif
+
+#ifndef U_ATTRIBUTE_DEPRECATED
+/**
+ * \def U_ATTRIBUTE_DEPRECATED
+ *  This is used for GCC specific attributes
+ * @internal
+ */
+#if U_GCC_MAJOR_MINOR >= 302
+#    define U_ATTRIBUTE_DEPRECATED __attribute__ ((deprecated))
+/**
+ * \def U_ATTRIBUTE_DEPRECATED
+ * This is used for Visual C++ specific attributes
+ * @internal
+ */
+#elif defined(_MSC_VER) && (_MSC_VER >= 1400)
+#    define U_ATTRIBUTE_DEPRECATED __declspec(deprecated)
+#else
+#    define U_ATTRIBUTE_DEPRECATED
+#endif
+#endif
+
+/** This is used to declare a function as a public ICU C API @stable ICU 2.0*/
+#define U_CAPI U_CFUNC U_EXPORT
+/** This is used to declare a function as a stable public ICU C API*/
+#define U_STABLE U_CAPI
+/** This is used to declare a function as a draft public ICU C API  */
+#define U_DRAFT  U_CAPI
+/** This is used to declare a function as a deprecated public ICU C API  */
+#define U_DEPRECATED U_CAPI U_ATTRIBUTE_DEPRECATED
+/** This is used to declare a function as an obsolete public ICU C API  */
+#define U_OBSOLETE U_CAPI
+/** This is used to declare a function as an internal ICU C API  */
+#define U_INTERNAL U_CAPI
+
+/**
+ * \def U_OVERRIDE
+ * Defined to the C++11 "override" keyword if available.
+ * Denotes a class or member which is an override of the base class.
+ * May result in an error if it applied to something not an override.
+ * @internal
+ */
+#ifndef U_OVERRIDE
+#define U_OVERRIDE override
+#endif
+
+/**
+ * \def U_FINAL
+ * Defined to the C++11 "final" keyword if available.
+ * Denotes a class or member which may not be overridden in subclasses.
+ * May result in an error if subclasses attempt to override.
+ * @internal
+ */
+#if !defined(U_FINAL) || defined(U_IN_DOXYGEN)
+#define U_FINAL final
+#endif
+
+// Before ICU 65, function-like, multi-statement ICU macros were just defined as
+// series of statements wrapped in { } blocks and the caller could choose to
+// either treat them as if they were actual functions and end the invocation
+// with a trailing ; creating an empty statement after the block or else omit
+// this trailing ; using the knowledge that the macro would expand to { }.
+//
+// But doing so doesn't work well with macros that look like functions and
+// compiler warnings about empty statements (ICU-20601) and ICU 65 therefore
+// switches to the standard solution of wrapping such macros in do { } while.
+//
+// This will however break existing code that depends on being able to invoke
+// these macros without a trailing ; so to be able to remain compatible with
+// such code the wrapper is itself defined as macros so that it's possible to
+// build ICU 65 and later with the old macro behaviour, like this:
+//
+// CPPFLAGS='-DUPRV_BLOCK_MACRO_BEGIN="" -DUPRV_BLOCK_MACRO_END=""'
+// runConfigureICU ...
+
+/**
+ * \def UPRV_BLOCK_MACRO_BEGIN
+ * Defined as the "do" keyword by default.
+ * @internal
+ */
+#ifndef UPRV_BLOCK_MACRO_BEGIN
+#define UPRV_BLOCK_MACRO_BEGIN do
+#endif
+
+/**
+ * \def UPRV_BLOCK_MACRO_END
+ * Defined as "while (FALSE)" by default.
+ * @internal
+ */
+#ifndef UPRV_BLOCK_MACRO_END
+#define UPRV_BLOCK_MACRO_END while (FALSE)
+#endif
+
+/*==========================================================================*/
+/* limits for int32_t etc., like in POSIX inttypes.h                        */
+/*==========================================================================*/
+
+#ifndef INT8_MIN
+/** The smallest value an 8 bit signed integer can hold @stable ICU 2.0 */
+#   define INT8_MIN        ((int8_t)(-128))
+#endif
+#ifndef INT16_MIN
+/** The smallest value a 16 bit signed integer can hold @stable ICU 2.0 */
+#   define INT16_MIN       ((int16_t)(-32767-1))
+#endif
+#ifndef INT32_MIN
+/** The smallest value a 32 bit signed integer can hold @stable ICU 2.0 */
+#   define INT32_MIN       ((int32_t)(-2147483647-1))
+#endif
+
+#ifndef INT8_MAX
+/** The largest value an 8 bit signed integer can hold @stable ICU 2.0 */
+#   define INT8_MAX        ((int8_t)(127))
+#endif
+#ifndef INT16_MAX
+/** The largest value a 16 bit signed integer can hold @stable ICU 2.0 */
+#   define INT16_MAX       ((int16_t)(32767))
+#endif
+#ifndef INT32_MAX
+/** The largest value a 32 bit signed integer can hold @stable ICU 2.0 */
+#   define INT32_MAX       ((int32_t)(2147483647))
+#endif
+
+#ifndef UINT8_MAX
+/** The largest value an 8 bit unsigned integer can hold @stable ICU 2.0 */
+#   define UINT8_MAX       ((uint8_t)(255U))
+#endif
+#ifndef UINT16_MAX
+/** The largest value a 16 bit unsigned integer can hold @stable ICU 2.0 */
+#   define UINT16_MAX      ((uint16_t)(65535U))
+#endif
+#ifndef UINT32_MAX
+/** The largest value a 32 bit unsigned integer can hold @stable ICU 2.0 */
+#   define UINT32_MAX      ((uint32_t)(4294967295U))
+#endif
+
+#if defined(U_INT64_T_UNAVAILABLE)
+# error int64_t is required for decimal format and rule-based number format.
+#else
+# ifndef INT64_C
+/**
+ * Provides a platform independent way to specify a signed 64-bit integer constant.
+ * note: may be wrong for some 64 bit platforms - ensure your compiler provides INT64_C
+ * @stable ICU 2.8
+ */
+#   define INT64_C(c) c ## LL
+# endif
+# ifndef UINT64_C
+/**
+ * Provides a platform independent way to specify an unsigned 64-bit integer constant.
+ * note: may be wrong for some 64 bit platforms - ensure your compiler provides UINT64_C
+ * @stable ICU 2.8
+ */
+#   define UINT64_C(c) c ## ULL
+# endif
+# ifndef U_INT64_MIN
+/** The smallest value a 64 bit signed integer can hold @stable ICU 2.8 */
+#     define U_INT64_MIN       ((int64_t)(INT64_C(-9223372036854775807)-1))
+# endif
+# ifndef U_INT64_MAX
+/** The largest value a 64 bit signed integer can hold @stable ICU 2.8 */
+#     define U_INT64_MAX       ((int64_t)(INT64_C(9223372036854775807)))
+# endif
+# ifndef U_UINT64_MAX
+/** The largest value a 64 bit unsigned integer can hold @stable ICU 2.8 */
+#     define U_UINT64_MAX      ((uint64_t)(UINT64_C(18446744073709551615)))
+# endif
+#endif
+
+/*==========================================================================*/
+/* Boolean data type                                                        */
+/*==========================================================================*/
+
+/** The ICU boolean type @stable ICU 2.0 */
+typedef int8_t UBool;
+
+#ifndef TRUE
+/** The TRUE value of a UBool @stable ICU 2.0 */
+#   define TRUE  1
+#endif
+#ifndef FALSE
+/** The FALSE value of a UBool @stable ICU 2.0 */
+#   define FALSE 0
+#endif
+
+
+/*==========================================================================*/
+/* Unicode data types                                                       */
+/*==========================================================================*/
+
+/* wchar_t-related definitions -------------------------------------------- */
+
+/*
+ * \def U_WCHAR_IS_UTF16
+ * Defined if wchar_t uses UTF-16.
+ *
+ * @stable ICU 2.0
+ */
+/*
+ * \def U_WCHAR_IS_UTF32
+ * Defined if wchar_t uses UTF-32.
+ *
+ * @stable ICU 2.0
+ */
+#if !defined(U_WCHAR_IS_UTF16) && !defined(U_WCHAR_IS_UTF32)
+#   ifdef __STDC_ISO_10646__
+#       if (U_SIZEOF_WCHAR_T==2)
+#           define U_WCHAR_IS_UTF16
+#       elif (U_SIZEOF_WCHAR_T==4)
+#           define  U_WCHAR_IS_UTF32
+#       endif
+#   elif defined __UCS2__
+#       if (U_PF_OS390 <= U_PLATFORM && U_PLATFORM <= U_PF_OS400) && (U_SIZEOF_WCHAR_T==2)
+#           define U_WCHAR_IS_UTF16
+#       endif
+#   elif defined(__UCS4__) || (U_PLATFORM == U_PF_OS400 && defined(__UTF32__))
+#       if (U_SIZEOF_WCHAR_T==4)
+#           define U_WCHAR_IS_UTF32
+#       endif
+#   elif U_PLATFORM_IS_DARWIN_BASED || (U_SIZEOF_WCHAR_T==4 && U_PLATFORM_IS_LINUX_BASED)
+#       define U_WCHAR_IS_UTF32
+#   elif U_PLATFORM_HAS_WIN32_API
+#       define U_WCHAR_IS_UTF16
+#   endif
+#endif
+
+/* UChar and UChar32 definitions -------------------------------------------- */
+
+/** Number of bytes in a UChar. @stable ICU 2.0 */
+#define U_SIZEOF_UCHAR 2
+
+/**
+ * \def U_CHAR16_IS_TYPEDEF
+ * If 1, then char16_t is a typedef and not a real type (yet)
+ * @internal
+ */
+#if (U_PLATFORM == U_PF_AIX) && defined(__cplusplus) &&(U_CPLUSPLUS_VERSION < 11)
+// for AIX, uchar.h needs to be included
+# include <uchar.h>
+# define U_CHAR16_IS_TYPEDEF 1
+#elif defined(_MSC_VER) && (_MSC_VER < 1900)
+// Versions of Visual Studio/MSVC below 2015 do not support char16_t as a real type,
+// and instead use a typedef.  https://msdn.microsoft.com/library/bb531344.aspx
+# define U_CHAR16_IS_TYPEDEF 1
+#else
+# define U_CHAR16_IS_TYPEDEF 0
+#endif
+
+
+/**
+ * \var UChar
+ *
+ * The base type for UTF-16 code units and pointers.
+ * Unsigned 16-bit integer.
+ * Starting with ICU 59, C++ API uses char16_t directly, while C API continues to use UChar.
+ *
+ * UChar is configurable by defining the macro UCHAR_TYPE
+ * on the preprocessor or compiler command line:
+ * -DUCHAR_TYPE=uint16_t or -DUCHAR_TYPE=wchar_t (if U_SIZEOF_WCHAR_T==2) etc.
+ * (The UCHAR_TYPE can also be \#defined earlier in this file, for outside the ICU library code.)
+ * This is for transitional use from application code that uses uint16_t or wchar_t for UTF-16.
+ *
+ * The default is UChar=char16_t.
+ *
+ * C++11 defines char16_t as bit-compatible with uint16_t, but as a distinct type.
+ *
+ * In C, char16_t is a simple typedef of uint_least16_t.
+ * ICU requires uint_least16_t=uint16_t for data memory mapping.
+ * On macOS, char16_t is not available because the uchar.h standard header is missing.
+ *
+ * @stable ICU 4.4
+ */
+
+#if 1
+    // #if 1 is normal. UChar defaults to char16_t in C++.
+    // For configuration testing of UChar=uint16_t temporarily change this to #if 0.
+    // The intltest Makefile #defines UCHAR_TYPE=char16_t,
+    // so we only #define it to uint16_t if it is undefined so far.
+#elif !defined(UCHAR_TYPE)
+#   define UCHAR_TYPE uint16_t
+#endif
+
+#if defined(U_COMBINED_IMPLEMENTATION) || defined(U_COMMON_IMPLEMENTATION) || \
+        defined(U_I18N_IMPLEMENTATION) || defined(U_IO_IMPLEMENTATION)
+    // Inside the ICU library code, never configurable.
+    typedef char16_t UChar;
+#elif defined(UCHAR_TYPE)
+    typedef UCHAR_TYPE UChar;
+#elif defined(__cplusplus)
+    typedef char16_t UChar;
+#else
+    typedef uint16_t UChar;
+#endif
+
+/**
+ * \var OldUChar
+ * Default ICU 58 definition of UChar.
+ * A base type for UTF-16 code units and pointers.
+ * Unsigned 16-bit integer.
+ *
+ * Define OldUChar to be wchar_t if that is 16 bits wide.
+ * If wchar_t is not 16 bits wide, then define UChar to be uint16_t.
+ *
+ * This makes the definition of OldUChar platform-dependent
+ * but allows direct string type compatibility with platforms with
+ * 16-bit wchar_t types.
+ *
+ * This is how UChar was defined in ICU 58, for transition convenience.
+ * Exception: ICU 58 UChar was defined to UCHAR_TYPE if that macro was defined.
+ * The current UChar responds to UCHAR_TYPE but OldUChar does not.
+ *
+ * @stable ICU 59
+ */
+#if U_SIZEOF_WCHAR_T==2
+    typedef wchar_t OldUChar;
+#elif defined(__CHAR16_TYPE__)
+    typedef __CHAR16_TYPE__ OldUChar;
+#else
+    typedef uint16_t OldUChar;
+#endif
+
+/**
+ * Define UChar32 as a type for single Unicode code points.
+ * UChar32 is a signed 32-bit integer (same as int32_t).
+ *
+ * The Unicode code point range is 0..0x10ffff.
+ * All other values (negative or >=0x110000) are illegal as Unicode code points.
+ * They may be used as sentinel values to indicate "done", "error"
+ * or similar non-code point conditions.
+ *
+ * Before ICU 2.4 (Jitterbug 2146), UChar32 was defined
+ * to be wchar_t if that is 32 bits wide (wchar_t may be signed or unsigned)
+ * or else to be uint32_t.
+ * That is, the definition of UChar32 was platform-dependent.
+ *
+ * @see U_SENTINEL
+ * @stable ICU 2.4
+ */
+typedef int32_t UChar32;
+
+/**
+ * This value is intended for sentinel values for APIs that
+ * (take or) return single code points (UChar32).
+ * It is outside of the Unicode code point range 0..0x10ffff.
+ *
+ * For example, a "done" or "error" value in a new API
+ * could be indicated with U_SENTINEL.
+ *
+ * ICU APIs designed before ICU 2.4 usually define service-specific "done"
+ * values, mostly 0xffff.
+ * Those may need to be distinguished from
+ * actual U+ffff text contents by calling functions like
+ * CharacterIterator::hasNext() or UnicodeString::length().
+ *
+ * @return -1
+ * @see UChar32
+ * @stable ICU 2.4
+ */
+#define U_SENTINEL (-1)
+
+#include "unicode/urename.h"
+
+#endif

third_party/tree-sitter/lib/src/unicode/urename.h

@@ -0,0 +1 @@
+// This file must exist in order for `utf8.h` and `utf16.h` to be used.

third_party/tree-sitter/lib/src/unicode/utf.h

@@ -0,0 +1 @@
+// This file must exist in order for `utf8.h` and `utf16.h` to be used.

third_party/tree-sitter/lib/src/unicode/utf16.h

@@ -0,0 +1,733 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+*
+*   Copyright (C) 1999-2012, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  utf16.h
+*   encoding:   UTF-8
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999sep09
+*   created by: Markus W. Scherer
+*/
+
+/**
+ * \file
+ * \brief C API: 16-bit Unicode handling macros
+ * 
+ * This file defines macros to deal with 16-bit Unicode (UTF-16) code units and strings.
+ *
+ * For more information see utf.h and the ICU User Guide Strings chapter
+ * (http://userguide.icu-project.org/strings).
+ *
+ * <em>Usage:</em>
+ * ICU coding guidelines for if() statements should be followed when using these macros.
+ * Compound statements (curly braces {}) must be used  for if-else-while... 
+ * bodies and all macro statements should be terminated with semicolon.
+ */
+
+#ifndef __UTF16_H__
+#define __UTF16_H__
+
+#include "unicode/umachine.h"
+#ifndef __UTF_H__
+#   include "unicode/utf.h"
+#endif
+
+/* single-code point definitions -------------------------------------------- */
+
+/**
+ * Does this code unit alone encode a code point (BMP, not a surrogate)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_SINGLE(c) !U_IS_SURROGATE(c)
+
+/**
+ * Is this code unit a lead surrogate (U+d800..U+dbff)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_LEAD(c) (((c)&0xfffffc00)==0xd800)
+
+/**
+ * Is this code unit a trail surrogate (U+dc00..U+dfff)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_TRAIL(c) (((c)&0xfffffc00)==0xdc00)
+
+/**
+ * Is this code unit a surrogate (U+d800..U+dfff)?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_SURROGATE(c) U_IS_SURROGATE(c)
+
+/**
+ * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)),
+ * is it a lead surrogate?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U16_IS_SURROGATE_LEAD(c) (((c)&0x400)==0)
+
+/**
+ * Assuming c is a surrogate code point (U16_IS_SURROGATE(c)),
+ * is it a trail surrogate?
+ * @param c 16-bit code unit
+ * @return TRUE or FALSE
+ * @stable ICU 4.2
+ */
+#define U16_IS_SURROGATE_TRAIL(c) (((c)&0x400)!=0)
+
+/**
+ * Helper constant for U16_GET_SUPPLEMENTARY.
+ * @internal
+ */
+#define U16_SURROGATE_OFFSET ((0xd800<<10UL)+0xdc00-0x10000)
+
+/**
+ * Get a supplementary code point value (U+10000..U+10ffff)
+ * from its lead and trail surrogates.
+ * The result is undefined if the input values are not
+ * lead and trail surrogates.
+ *
+ * @param lead lead surrogate (U+d800..U+dbff)
+ * @param trail trail surrogate (U+dc00..U+dfff)
+ * @return supplementary code point (U+10000..U+10ffff)
+ * @stable ICU 2.4
+ */
+#define U16_GET_SUPPLEMENTARY(lead, trail) \
+    (((UChar32)(lead)<<10UL)+(UChar32)(trail)-U16_SURROGATE_OFFSET)
+
+
+/**
+ * Get the lead surrogate (0xd800..0xdbff) for a
+ * supplementary code point (0x10000..0x10ffff).
+ * @param supplementary 32-bit code point (U+10000..U+10ffff)
+ * @return lead surrogate (U+d800..U+dbff) for supplementary
+ * @stable ICU 2.4
+ */
+#define U16_LEAD(supplementary) (UChar)(((supplementary)>>10)+0xd7c0)
+
+/**
+ * Get the trail surrogate (0xdc00..0xdfff) for a
+ * supplementary code point (0x10000..0x10ffff).
+ * @param supplementary 32-bit code point (U+10000..U+10ffff)
+ * @return trail surrogate (U+dc00..U+dfff) for supplementary
+ * @stable ICU 2.4
+ */
+#define U16_TRAIL(supplementary) (UChar)(((supplementary)&0x3ff)|0xdc00)
+
+/**
+ * How many 16-bit code units are used to encode this Unicode code point? (1 or 2)
+ * The result is not defined if c is not a Unicode code point (U+0000..U+10ffff).
+ * @param c 32-bit code point
+ * @return 1 or 2
+ * @stable ICU 2.4
+ */
+#define U16_LENGTH(c) ((uint32_t)(c)<=0xffff ? 1 : 2)
+
+/**
+ * The maximum number of 16-bit code units per Unicode code point (U+0000..U+10ffff).
+ * @return 2
+ * @stable ICU 2.4
+ */
+#define U16_MAX_LENGTH 2
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * The offset may point to either the lead or trail surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the adjacent matching surrogate as well.
+ * The result is undefined if the offset points to a single, unpaired surrogate.
+ * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U16_GET
+ * @stable ICU 2.4
+ */
+#define U16_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[i]; \
+    if(U16_IS_SURROGATE(c)) { \
+        if(U16_IS_SURROGATE_LEAD(c)) { \
+            (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)+1]); \
+        } else { \
+            (c)=U16_GET_SUPPLEMENTARY((s)[(i)-1], (c)); \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The offset may point to either the lead or trail surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the adjacent matching surrogate as well.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * If the offset points to a single, unpaired surrogate, then
+ * c is set to that unpaired surrogate.
+ * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i<length
+ * @param length string length
+ * @param c output UChar32 variable
+ * @see U16_GET_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_GET(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[i]; \
+    if(U16_IS_SURROGATE(c)) { \
+        uint16_t __c2; \
+        if(U16_IS_SURROGATE_LEAD(c)) { \
+            if((i)+1!=(length) && U16_IS_TRAIL(__c2=(s)[(i)+1])) { \
+                (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
+            } \
+        } else { \
+            if((i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \
+                (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \
+            } \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The offset may point to either the lead or trail surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the adjacent matching surrogate as well.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * If the offset points to a single, unpaired surrogate, then
+ * c is set to U+FFFD.
+ * Iteration through a string is more efficient with U16_NEXT_UNSAFE or U16_NEXT_OR_FFFD.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i<length
+ * @param length string length
+ * @param c output UChar32 variable
+ * @see U16_GET_UNSAFE
+ * @stable ICU 60
+ */
+#define U16_GET_OR_FFFD(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[i]; \
+    if(U16_IS_SURROGATE(c)) { \
+        uint16_t __c2; \
+        if(U16_IS_SURROGATE_LEAD(c)) { \
+            if((i)+1!=(length) && U16_IS_TRAIL(__c2=(s)[(i)+1])) { \
+                (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
+            } else { \
+                (c)=0xfffd; \
+            } \
+        } else { \
+            if((i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \
+                (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \
+            } else { \
+                (c)=0xfffd; \
+            } \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/* definitions with forward iteration --------------------------------------- */
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * The offset may point to the lead surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the following trail surrogate as well.
+ * If the offset points to a trail surrogate, then that itself
+ * will be returned as the code point.
+ * The result is undefined if the offset points to a single, unpaired lead surrogate.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U16_NEXT
+ * @stable ICU 2.4
+ */
+#define U16_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[(i)++]; \
+    if(U16_IS_LEAD(c)) { \
+        (c)=U16_GET_SUPPLEMENTARY((c), (s)[(i)++]); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * The offset may point to the lead surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the following trail surrogate as well.
+ * If the offset points to a trail surrogate or
+ * to a single, unpaired lead surrogate, then c is set to that unpaired surrogate.
+ *
+ * @param s const UChar * string
+ * @param i string offset, must be i<length
+ * @param length string length
+ * @param c output UChar32 variable
+ * @see U16_NEXT_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_NEXT(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[(i)++]; \
+    if(U16_IS_LEAD(c)) { \
+        uint16_t __c2; \
+        if((i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+            ++(i); \
+            (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * The offset may point to the lead surrogate unit
+ * for a supplementary code point, in which case the macro will read
+ * the following trail surrogate as well.
+ * If the offset points to a trail surrogate or
+ * to a single, unpaired lead surrogate, then c is set to U+FFFD.
+ *
+ * @param s const UChar * string
+ * @param i string offset, must be i<length
+ * @param length string length
+ * @param c output UChar32 variable
+ * @see U16_NEXT_UNSAFE
+ * @stable ICU 60
+ */
+#define U16_NEXT_OR_FFFD(s, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[(i)++]; \
+    if(U16_IS_SURROGATE(c)) { \
+        uint16_t __c2; \
+        if(U16_IS_SURROGATE_LEAD(c) && (i)!=(length) && U16_IS_TRAIL(__c2=(s)[(i)])) { \
+            ++(i); \
+            (c)=U16_GET_SUPPLEMENTARY((c), __c2); \
+        } else { \
+            (c)=0xfffd; \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Append a code point to a string, overwriting 1 or 2 code units.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Unsafe" macro, assumes a valid code point and sufficient space in the string.
+ * Otherwise, the result is undefined.
+ *
+ * @param s const UChar * string buffer
+ * @param i string offset
+ * @param c code point to append
+ * @see U16_APPEND
+ * @stable ICU 2.4
+ */
+#define U16_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    if((uint32_t)(c)<=0xffff) { \
+        (s)[(i)++]=(uint16_t)(c); \
+    } else { \
+        (s)[(i)++]=(uint16_t)(((c)>>10)+0xd7c0); \
+        (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Append a code point to a string, overwriting 1 or 2 code units.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Safe" macro, checks for a valid code point.
+ * If a surrogate pair is written, checks for sufficient space in the string.
+ * If the code point is not valid or a trail surrogate does not fit,
+ * then isError is set to TRUE.
+ *
+ * @param s const UChar * string buffer
+ * @param i string offset, must be i<capacity
+ * @param capacity size of the string buffer
+ * @param c code point to append
+ * @param isError output UBool set to TRUE if an error occurs, otherwise not modified
+ * @see U16_APPEND_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_APPEND(s, i, capacity, c, isError) UPRV_BLOCK_MACRO_BEGIN { \
+    if((uint32_t)(c)<=0xffff) { \
+        (s)[(i)++]=(uint16_t)(c); \
+    } else if((uint32_t)(c)<=0x10ffff && (i)+1<(capacity)) { \
+        (s)[(i)++]=(uint16_t)(((c)>>10)+0xd7c0); \
+        (s)[(i)++]=(uint16_t)(((c)&0x3ff)|0xdc00); \
+    } else /* c>0x10ffff or not enough space */ { \
+        (isError)=TRUE; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_FWD_1
+ * @stable ICU 2.4
+ */
+#define U16_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_LEAD((s)[(i)++])) { \
+        ++(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * @param s const UChar * string
+ * @param i string offset, must be i<length
+ * @param length string length
+ * @see U16_FWD_1_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_FWD_1(s, i, length) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_LEAD((s)[(i)++]) && (i)!=(length) && U16_IS_TRAIL((s)[i])) { \
+        ++(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param n number of code points to skip
+ * @see U16_FWD_N
+ * @stable ICU 2.4
+ */
+#define U16_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0) { \
+        U16_FWD_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * @param s const UChar * string
+ * @param i int32_t string offset, must be i<length
+ * @param length int32_t string length
+ * @param n number of code points to skip
+ * @see U16_FWD_N_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_FWD_N(s, i, length, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \
+        U16_FWD_1(s, i, length); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to the trail surrogate of a surrogate pair,
+ * then the offset is decremented.
+ * Otherwise, it is not modified.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_SET_CP_START
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_TRAIL((s)[i])) { \
+        --(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to the trail surrogate of a surrogate pair,
+ * then the offset is decremented.
+ * Otherwise, it is not modified.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<=i
+ * @see U16_SET_CP_START_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_TRAIL((s)[i]) && (i)>(start) && U16_IS_LEAD((s)[(i)-1])) { \
+        --(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/* definitions with backward iteration -------------------------------------- */
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a trail surrogate unit
+ * for a supplementary code point, then the macro will read
+ * the preceding lead surrogate as well.
+ * If the offset is behind a lead surrogate, then that itself
+ * will be returned as the code point.
+ * The result is undefined if the offset is behind a single, unpaired trail surrogate.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U16_PREV
+ * @stable ICU 2.4
+ */
+#define U16_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[--(i)]; \
+    if(U16_IS_TRAIL(c)) { \
+        (c)=U16_GET_SUPPLEMENTARY((s)[--(i)], (c)); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a trail surrogate unit
+ * for a supplementary code point, then the macro will read
+ * the preceding lead surrogate as well.
+ * If the offset is behind a lead surrogate or behind a single, unpaired
+ * trail surrogate, then c is set to that unpaired surrogate.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<i
+ * @param c output UChar32 variable
+ * @see U16_PREV_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_PREV(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[--(i)]; \
+    if(U16_IS_TRAIL(c)) { \
+        uint16_t __c2; \
+        if((i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \
+            --(i); \
+            (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a trail surrogate unit
+ * for a supplementary code point, then the macro will read
+ * the preceding lead surrogate as well.
+ * If the offset is behind a lead surrogate or behind a single, unpaired
+ * trail surrogate, then c is set to U+FFFD.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<i
+ * @param c output UChar32 variable
+ * @see U16_PREV_UNSAFE
+ * @stable ICU 60
+ */
+#define U16_PREV_OR_FFFD(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(s)[--(i)]; \
+    if(U16_IS_SURROGATE(c)) { \
+        uint16_t __c2; \
+        if(U16_IS_SURROGATE_TRAIL(c) && (i)>(start) && U16_IS_LEAD(__c2=(s)[(i)-1])) { \
+            --(i); \
+            (c)=U16_GET_SUPPLEMENTARY(__c2, (c)); \
+        } else { \
+            (c)=0xfffd; \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_BACK_1
+ * @stable ICU 2.4
+ */
+#define U16_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_TRAIL((s)[--(i)])) { \
+        --(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start starting string offset (usually 0)
+ * @param i string offset, must be start<i
+ * @see U16_BACK_1_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_BACK_1(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_TRAIL((s)[--(i)]) && (i)>(start) && U16_IS_LEAD((s)[(i)-1])) { \
+        --(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @param n number of code points to skip
+ * @see U16_BACK_N
+ * @stable ICU 2.4
+ */
+#define U16_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0) { \
+        U16_BACK_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * @param s const UChar * string
+ * @param start start of string
+ * @param i string offset, must be start<i
+ * @param n number of code points to skip
+ * @see U16_BACK_N_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_BACK_N(s, start, i, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0 && (i)>(start)) { \
+        U16_BACK_1(s, start, i); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind the lead surrogate of a surrogate pair,
+ * then the offset is incremented.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-16.
+ *
+ * @param s const UChar * string
+ * @param i string offset
+ * @see U16_SET_CP_LIMIT
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U16_IS_LEAD((s)[(i)-1])) { \
+        ++(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind the lead surrogate of a surrogate pair,
+ * then the offset is incremented.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Safe" macro, handles unpaired surrogates and checks for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * @param s const UChar * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param i int32_t string offset, start<=i<=length
+ * @param length int32_t string length
+ * @see U16_SET_CP_LIMIT_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U16_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \
+    if((start)<(i) && ((i)<(length) || (length)<0) && U16_IS_LEAD((s)[(i)-1]) && U16_IS_TRAIL((s)[i])) { \
+        ++(i); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+#endif

third_party/tree-sitter/lib/src/unicode/utf8.h

@@ -0,0 +1,881 @@
+// © 2016 and later: Unicode, Inc. and others.
+// License & terms of use: http://www.unicode.org/copyright.html
+/*
+*******************************************************************************
+*
+*   Copyright (C) 1999-2015, International Business Machines
+*   Corporation and others.  All Rights Reserved.
+*
+*******************************************************************************
+*   file name:  utf8.h
+*   encoding:   UTF-8
+*   tab size:   8 (not used)
+*   indentation:4
+*
+*   created on: 1999sep13
+*   created by: Markus W. Scherer
+*/
+
+/**
+ * \file
+ * \brief C API: 8-bit Unicode handling macros
+ *
+ * This file defines macros to deal with 8-bit Unicode (UTF-8) code units (bytes) and strings.
+ *
+ * For more information see utf.h and the ICU User Guide Strings chapter
+ * (http://userguide.icu-project.org/strings).
+ *
+ * <em>Usage:</em>
+ * ICU coding guidelines for if() statements should be followed when using these macros.
+ * Compound statements (curly braces {}) must be used  for if-else-while...
+ * bodies and all macro statements should be terminated with semicolon.
+ */
+
+#ifndef __UTF8_H__
+#define __UTF8_H__
+
+#include "unicode/umachine.h"
+#ifndef __UTF_H__
+#   include "unicode/utf.h"
+#endif
+
+/* internal definitions ----------------------------------------------------- */
+
+/**
+ * Counts the trail bytes for a UTF-8 lead byte.
+ * Returns 0 for 0..0xc1 as well as for 0xf5..0xff.
+ * leadByte might be evaluated multiple times.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is called by public macros in this file and thus must remain stable.
+ *
+ * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff.
+ * @internal
+ */
+#define U8_COUNT_TRAIL_BYTES(leadByte) \
+    (U8_IS_LEAD(leadByte) ? \
+        ((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0)+1 : 0)
+
+/**
+ * Counts the trail bytes for a UTF-8 lead byte of a valid UTF-8 sequence.
+ * Returns 0 for 0..0xc1. Undefined for 0xf5..0xff.
+ * leadByte might be evaluated multiple times.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is called by public macros in this file and thus must remain stable.
+ *
+ * @param leadByte The first byte of a UTF-8 sequence. Must be 0..0xff.
+ * @internal
+ */
+#define U8_COUNT_TRAIL_BYTES_UNSAFE(leadByte) \
+    (((uint8_t)(leadByte)>=0xc2)+((uint8_t)(leadByte)>=0xe0)+((uint8_t)(leadByte)>=0xf0))
+
+/**
+ * Mask a UTF-8 lead byte, leave only the lower bits that form part of the code point value.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is called by public macros in this file and thus must remain stable.
+ * @internal
+ */
+#define U8_MASK_LEAD_BYTE(leadByte, countTrailBytes) ((leadByte)&=(1<<(6-(countTrailBytes)))-1)
+
+/**
+ * Internal bit vector for 3-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD3_AND_T1.
+ * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence.
+ * Lead byte E0..EF bits 3..0 are used as byte index,
+ * first trail byte bits 7..5 are used as bit index into that byte.
+ * @see U8_IS_VALID_LEAD3_AND_T1
+ * @internal
+ */
+#define U8_LEAD3_T1_BITS "\x20\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x30\x10\x30\x30"
+
+/**
+ * Internal 3-byte UTF-8 validity check.
+ * Non-zero if lead byte E0..EF and first trail byte 00..FF start a valid sequence.
+ * @internal
+ */
+#define U8_IS_VALID_LEAD3_AND_T1(lead, t1) (U8_LEAD3_T1_BITS[(lead)&0xf]&(1<<((uint8_t)(t1)>>5)))
+
+/**
+ * Internal bit vector for 4-byte UTF-8 validity check, for use in U8_IS_VALID_LEAD4_AND_T1.
+ * Each bit indicates whether one lead byte + first trail byte pair starts a valid sequence.
+ * First trail byte bits 7..4 are used as byte index,
+ * lead byte F0..F4 bits 2..0 are used as bit index into that byte.
+ * @see U8_IS_VALID_LEAD4_AND_T1
+ * @internal
+ */
+#define U8_LEAD4_T1_BITS "\x00\x00\x00\x00\x00\x00\x00\x00\x1E\x0F\x0F\x0F\x00\x00\x00\x00"
+
+/**
+ * Internal 4-byte UTF-8 validity check.
+ * Non-zero if lead byte F0..F4 and first trail byte 00..FF start a valid sequence.
+ * @internal
+ */
+#define U8_IS_VALID_LEAD4_AND_T1(lead, t1) (U8_LEAD4_T1_BITS[(uint8_t)(t1)>>4]&(1<<((lead)&7)))
+
+/**
+ * Function for handling "next code point" with error-checking.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
+ * file and thus must remain stable, and should not be hidden when other internal
+ * functions are hidden (otherwise public macros would fail to compile).
+ * @internal
+ */
+U_STABLE UChar32 U_EXPORT2
+utf8_nextCharSafeBody(const uint8_t *s, int32_t *pi, int32_t length, UChar32 c, UBool strict);
+
+/**
+ * Function for handling "append code point" with error-checking.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
+ * file and thus must remain stable, and should not be hidden when other internal
+ * functions are hidden (otherwise public macros would fail to compile).
+ * @internal
+ */
+U_STABLE int32_t U_EXPORT2
+utf8_appendCharSafeBody(uint8_t *s, int32_t i, int32_t length, UChar32 c, UBool *pIsError);
+
+/**
+ * Function for handling "previous code point" with error-checking.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
+ * file and thus must remain stable, and should not be hidden when other internal
+ * functions are hidden (otherwise public macros would fail to compile).
+ * @internal
+ */
+U_STABLE UChar32 U_EXPORT2
+utf8_prevCharSafeBody(const uint8_t *s, int32_t start, int32_t *pi, UChar32 c, UBool strict);
+
+/**
+ * Function for handling "skip backward one code point" with error-checking.
+ *
+ * This is internal since it is not meant to be called directly by external clients;
+ * however it is U_STABLE (not U_INTERNAL) since it is called by public macros in this
+ * file and thus must remain stable, and should not be hidden when other internal
+ * functions are hidden (otherwise public macros would fail to compile).
+ * @internal
+ */
+U_STABLE int32_t U_EXPORT2
+utf8_back1SafeBody(const uint8_t *s, int32_t start, int32_t i);
+
+/* single-code point definitions -------------------------------------------- */
+
+/**
+ * Does this code unit (byte) encode a code point by itself (US-ASCII 0..0x7f)?
+ * @param c 8-bit code unit (byte)
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U8_IS_SINGLE(c) (((c)&0x80)==0)
+
+/**
+ * Is this code unit (byte) a UTF-8 lead byte? (0xC2..0xF4)
+ * @param c 8-bit code unit (byte)
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U8_IS_LEAD(c) ((uint8_t)((c)-0xc2)<=0x32)
+// 0x32=0xf4-0xc2
+
+/**
+ * Is this code unit (byte) a UTF-8 trail byte? (0x80..0xBF)
+ * @param c 8-bit code unit (byte)
+ * @return TRUE or FALSE
+ * @stable ICU 2.4
+ */
+#define U8_IS_TRAIL(c) ((int8_t)(c)<-0x40)
+
+/**
+ * How many code units (bytes) are used for the UTF-8 encoding
+ * of this Unicode code point?
+ * @param c 32-bit code point
+ * @return 1..4, or 0 if c is a surrogate or not a Unicode code point
+ * @stable ICU 2.4
+ */
+#define U8_LENGTH(c) \
+    ((uint32_t)(c)<=0x7f ? 1 : \
+        ((uint32_t)(c)<=0x7ff ? 2 : \
+            ((uint32_t)(c)<=0xd7ff ? 3 : \
+                ((uint32_t)(c)<=0xdfff || (uint32_t)(c)>0x10ffff ? 0 : \
+                    ((uint32_t)(c)<=0xffff ? 3 : 4)\
+                ) \
+            ) \
+        ) \
+    )
+
+/**
+ * The maximum number of UTF-8 code units (bytes) per Unicode code point (U+0000..U+10ffff).
+ * @return 4
+ * @stable ICU 2.4
+ */
+#define U8_MAX_LENGTH 4
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * The offset may point to either the lead byte or one of the trail bytes
+ * for a code point, in which case the macro will read all of the bytes
+ * for the code point.
+ * The result is undefined if the offset points to an illegal UTF-8
+ * byte sequence.
+ * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U8_GET
+ * @stable ICU 2.4
+ */
+#define U8_GET_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t _u8_get_unsafe_index=(int32_t)(i); \
+    U8_SET_CP_START_UNSAFE(s, _u8_get_unsafe_index); \
+    U8_NEXT_UNSAFE(s, _u8_get_unsafe_index, c); \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * The offset may point to either the lead byte or one of the trail bytes
+ * for a code point, in which case the macro will read all of the bytes
+ * for the code point.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * If the offset points to an illegal UTF-8 byte sequence, then
+ * c is set to a negative value.
+ * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset
+ * @param i int32_t string offset, must be start<=i<length
+ * @param length int32_t string length
+ * @param c output UChar32 variable, set to <0 in case of an error
+ * @see U8_GET_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_GET(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t _u8_get_index=(i); \
+    U8_SET_CP_START(s, start, _u8_get_index); \
+    U8_NEXT(s, _u8_get_index, length, c); \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a random-access offset,
+ * without changing the offset.
+ * The offset may point to either the lead byte or one of the trail bytes
+ * for a code point, in which case the macro will read all of the bytes
+ * for the code point.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * If the offset points to an illegal UTF-8 byte sequence, then
+ * c is set to U+FFFD.
+ * Iteration through a string is more efficient with U8_NEXT_UNSAFE or U8_NEXT_OR_FFFD.
+ *
+ * This macro does not distinguish between a real U+FFFD in the text
+ * and U+FFFD returned for an ill-formed sequence.
+ * Use U8_GET() if that distinction is important.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset
+ * @param i int32_t string offset, must be start<=i<length
+ * @param length int32_t string length
+ * @param c output UChar32 variable, set to U+FFFD in case of an error
+ * @see U8_GET
+ * @stable ICU 51
+ */
+#define U8_GET_OR_FFFD(s, start, i, length, c) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t _u8_get_index=(i); \
+    U8_SET_CP_START(s, start, _u8_get_index); \
+    U8_NEXT_OR_FFFD(s, _u8_get_index, length, c); \
+} UPRV_BLOCK_MACRO_END
+
+/* definitions with forward iteration --------------------------------------- */
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * The offset may point to the lead byte of a multi-byte sequence,
+ * in which case the macro will read the whole sequence.
+ * The result is undefined if the offset points to a trail byte
+ * or an illegal UTF-8 sequence.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U8_NEXT
+ * @stable ICU 2.4
+ */
+#define U8_NEXT_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(uint8_t)(s)[(i)++]; \
+    if(!U8_IS_SINGLE(c)) { \
+        if((c)<0xe0) { \
+            (c)=(((c)&0x1f)<<6)|((s)[(i)++]&0x3f); \
+        } else if((c)<0xf0) { \
+            /* no need for (c&0xf) because the upper bits are truncated after <<12 in the cast to (UChar) */ \
+            (c)=(UChar)(((c)<<12)|(((s)[i]&0x3f)<<6)|((s)[(i)+1]&0x3f)); \
+            (i)+=2; \
+        } else { \
+            (c)=(((c)&7)<<18)|(((s)[i]&0x3f)<<12)|(((s)[(i)+1]&0x3f)<<6)|((s)[(i)+2]&0x3f); \
+            (i)+=3; \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * The offset may point to the lead byte of a multi-byte sequence,
+ * in which case the macro will read the whole sequence.
+ * If the offset points to a trail byte or an illegal UTF-8 sequence, then
+ * c is set to a negative value.
+ *
+ * @param s const uint8_t * string
+ * @param i int32_t string offset, must be i<length
+ * @param length int32_t string length
+ * @param c output UChar32 variable, set to <0 in case of an error
+ * @see U8_NEXT_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_NEXT(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, U_SENTINEL)
+
+/**
+ * Get a code point from a string at a code point boundary offset,
+ * and advance the offset to the next code point boundary.
+ * (Post-incrementing forward iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * The offset may point to the lead byte of a multi-byte sequence,
+ * in which case the macro will read the whole sequence.
+ * If the offset points to a trail byte or an illegal UTF-8 sequence, then
+ * c is set to U+FFFD.
+ *
+ * This macro does not distinguish between a real U+FFFD in the text
+ * and U+FFFD returned for an ill-formed sequence.
+ * Use U8_NEXT() if that distinction is important.
+ *
+ * @param s const uint8_t * string
+ * @param i int32_t string offset, must be i<length
+ * @param length int32_t string length
+ * @param c output UChar32 variable, set to U+FFFD in case of an error
+ * @see U8_NEXT
+ * @stable ICU 51
+ */
+#define U8_NEXT_OR_FFFD(s, i, length, c) U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, 0xfffd)
+
+/** @internal */
+#define U8_INTERNAL_NEXT_OR_SUB(s, i, length, c, sub) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(uint8_t)(s)[(i)++]; \
+    if(!U8_IS_SINGLE(c)) { \
+        uint8_t __t = 0; \
+        if((i)!=(length) && \
+            /* fetch/validate/assemble all but last trail byte */ \
+            ((c)>=0xe0 ? \
+                ((c)<0xf0 ?  /* U+0800..U+FFFF except surrogates */ \
+                    U8_LEAD3_T1_BITS[(c)&=0xf]&(1<<((__t=(s)[i])>>5)) && \
+                    (__t&=0x3f, 1) \
+                :  /* U+10000..U+10FFFF */ \
+                    ((c)-=0xf0)<=4 && \
+                    U8_LEAD4_T1_BITS[(__t=(s)[i])>>4]&(1<<(c)) && \
+                    ((c)=((c)<<6)|(__t&0x3f), ++(i)!=(length)) && \
+                    (__t=(s)[i]-0x80)<=0x3f) && \
+                /* valid second-to-last trail byte */ \
+                ((c)=((c)<<6)|__t, ++(i)!=(length)) \
+            :  /* U+0080..U+07FF */ \
+                (c)>=0xc2 && ((c)&=0x1f, 1)) && \
+            /* last trail byte */ \
+            (__t=(s)[i]-0x80)<=0x3f && \
+            ((c)=((c)<<6)|__t, ++(i), 1)) { \
+        } else { \
+            (c)=(sub);  /* ill-formed*/ \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Append a code point to a string, overwriting 1 to 4 bytes.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Unsafe" macro, assumes a valid code point and sufficient space in the string.
+ * Otherwise, the result is undefined.
+ *
+ * @param s const uint8_t * string buffer
+ * @param i string offset
+ * @param c code point to append
+ * @see U8_APPEND
+ * @stable ICU 2.4
+ */
+#define U8_APPEND_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    uint32_t __uc=(c); \
+    if(__uc<=0x7f) { \
+        (s)[(i)++]=(uint8_t)__uc; \
+    } else { \
+        if(__uc<=0x7ff) { \
+            (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \
+        } else { \
+            if(__uc<=0xffff) { \
+                (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \
+            } else { \
+                (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \
+                (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \
+            } \
+            (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \
+        } \
+        (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Append a code point to a string, overwriting 1 to 4 bytes.
+ * The offset points to the current end of the string contents
+ * and is advanced (post-increment).
+ * "Safe" macro, checks for a valid code point.
+ * If a non-ASCII code point is written, checks for sufficient space in the string.
+ * If the code point is not valid or trail bytes do not fit,
+ * then isError is set to TRUE.
+ *
+ * @param s const uint8_t * string buffer
+ * @param i int32_t string offset, must be i<capacity
+ * @param capacity int32_t size of the string buffer
+ * @param c UChar32 code point to append
+ * @param isError output UBool set to TRUE if an error occurs, otherwise not modified
+ * @see U8_APPEND_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_APPEND(s, i, capacity, c, isError) UPRV_BLOCK_MACRO_BEGIN { \
+    uint32_t __uc=(c); \
+    if(__uc<=0x7f) { \
+        (s)[(i)++]=(uint8_t)__uc; \
+    } else if(__uc<=0x7ff && (i)+1<(capacity)) { \
+        (s)[(i)++]=(uint8_t)((__uc>>6)|0xc0); \
+        (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
+    } else if((__uc<=0xd7ff || (0xe000<=__uc && __uc<=0xffff)) && (i)+2<(capacity)) { \
+        (s)[(i)++]=(uint8_t)((__uc>>12)|0xe0); \
+        (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \
+        (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
+    } else if(0xffff<__uc && __uc<=0x10ffff && (i)+3<(capacity)) { \
+        (s)[(i)++]=(uint8_t)((__uc>>18)|0xf0); \
+        (s)[(i)++]=(uint8_t)(((__uc>>12)&0x3f)|0x80); \
+        (s)[(i)++]=(uint8_t)(((__uc>>6)&0x3f)|0x80); \
+        (s)[(i)++]=(uint8_t)((__uc&0x3f)|0x80); \
+    } else { \
+        (isError)=TRUE; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_FWD_1
+ * @stable ICU 2.4
+ */
+#define U8_FWD_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    (i)+=1+U8_COUNT_TRAIL_BYTES_UNSAFE((s)[i]); \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the next.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * @param s const uint8_t * string
+ * @param i int32_t string offset, must be i<length
+ * @param length int32_t string length
+ * @see U8_FWD_1_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_FWD_1(s, i, length) UPRV_BLOCK_MACRO_BEGIN { \
+    uint8_t __b=(s)[(i)++]; \
+    if(U8_IS_LEAD(__b) && (i)!=(length)) { \
+        uint8_t __t1=(s)[i]; \
+        if((0xe0<=__b && __b<0xf0)) { \
+            if(U8_IS_VALID_LEAD3_AND_T1(__b, __t1) && \
+                    ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \
+                ++(i); \
+            } \
+        } else if(__b<0xe0) { \
+            if(U8_IS_TRAIL(__t1)) { \
+                ++(i); \
+            } \
+        } else /* c>=0xf0 */ { \
+            if(U8_IS_VALID_LEAD4_AND_T1(__b, __t1) && \
+                    ++(i)!=(length) && U8_IS_TRAIL((s)[i]) && \
+                    ++(i)!=(length) && U8_IS_TRAIL((s)[i])) { \
+                ++(i); \
+            } \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param n number of code points to skip
+ * @see U8_FWD_N
+ * @stable ICU 2.4
+ */
+#define U8_FWD_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0) { \
+        U8_FWD_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Advance the string offset from one code point boundary to the n-th next one,
+ * i.e., move forward by n code points.
+ * (Post-incrementing iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * @param s const uint8_t * string
+ * @param i int32_t string offset, must be i<length
+ * @param length int32_t string length
+ * @param n number of code points to skip
+ * @see U8_FWD_N_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_FWD_N(s, i, length, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0 && ((i)<(length) || ((length)<0 && (s)[i]!=0))) { \
+        U8_FWD_1(s, i, length); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to a UTF-8 trail byte,
+ * then the offset is moved backward to the corresponding lead byte.
+ * Otherwise, it is not modified.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_SET_CP_START
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_START_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    while(U8_IS_TRAIL((s)[i])) { --(i); } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary
+ * at the start of a code point.
+ * If the offset points to a UTF-8 trail byte,
+ * then the offset is moved backward to the corresponding lead byte.
+ * Otherwise, it is not modified.
+ *
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ * Unlike U8_TRUNCATE_IF_INCOMPLETE(), this macro always reads s[i].
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param i int32_t string offset, must be start<=i
+ * @see U8_SET_CP_START_UNSAFE
+ * @see U8_TRUNCATE_IF_INCOMPLETE
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_START(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U8_IS_TRAIL((s)[(i)])) { \
+        (i)=utf8_back1SafeBody(s, start, (i)); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * If the string ends with a UTF-8 byte sequence that is valid so far
+ * but incomplete, then reduce the length of the string to end before
+ * the lead byte of that incomplete sequence.
+ * For example, if the string ends with E1 80, the length is reduced by 2.
+ *
+ * In all other cases (the string ends with a complete sequence, or it is not
+ * possible for any further trail byte to extend the trailing sequence)
+ * the length remains unchanged.
+ *
+ * Useful for processing text split across multiple buffers
+ * (save the incomplete sequence for later)
+ * and for optimizing iteration
+ * (check for string length only once per character).
+ *
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ * Unlike U8_SET_CP_START(), this macro never reads s[length].
+ *
+ * (In UTF-16, simply check for U16_IS_LEAD(last code unit).)
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param length int32_t string length (usually start<=length)
+ * @see U8_SET_CP_START
+ * @stable ICU 61
+ */
+#define U8_TRUNCATE_IF_INCOMPLETE(s, start, length) UPRV_BLOCK_MACRO_BEGIN { \
+    if((length)>(start)) { \
+        uint8_t __b1=s[(length)-1]; \
+        if(U8_IS_SINGLE(__b1)) { \
+            /* common ASCII character */ \
+        } else if(U8_IS_LEAD(__b1)) { \
+            --(length); \
+        } else if(U8_IS_TRAIL(__b1) && ((length)-2)>=(start)) { \
+            uint8_t __b2=s[(length)-2]; \
+            if(0xe0<=__b2 && __b2<=0xf4) { \
+                if(__b2<0xf0 ? U8_IS_VALID_LEAD3_AND_T1(__b2, __b1) : \
+                        U8_IS_VALID_LEAD4_AND_T1(__b2, __b1)) { \
+                    (length)-=2; \
+                } \
+            } else if(U8_IS_TRAIL(__b2) && ((length)-3)>=(start)) { \
+                uint8_t __b3=s[(length)-3]; \
+                if(0xf0<=__b3 && __b3<=0xf4 && U8_IS_VALID_LEAD4_AND_T1(__b3, __b2)) { \
+                    (length)-=3; \
+                } \
+            } \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/* definitions with backward iteration -------------------------------------- */
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a multi-byte sequence, then the macro will read
+ * the whole sequence.
+ * If the offset is behind a lead byte, then that itself
+ * will be returned as the code point.
+ * The result is undefined if the offset is behind an illegal UTF-8 sequence.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param c output UChar32 variable
+ * @see U8_PREV
+ * @stable ICU 2.4
+ */
+#define U8_PREV_UNSAFE(s, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(uint8_t)(s)[--(i)]; \
+    if(U8_IS_TRAIL(c)) { \
+        uint8_t __b, __count=1, __shift=6; \
+\
+        /* c is a trail byte */ \
+        (c)&=0x3f; \
+        for(;;) { \
+            __b=(s)[--(i)]; \
+            if(__b>=0xc0) { \
+                U8_MASK_LEAD_BYTE(__b, __count); \
+                (c)|=(UChar32)__b<<__shift; \
+                break; \
+            } else { \
+                (c)|=(UChar32)(__b&0x3f)<<__shift; \
+                ++__count; \
+                __shift+=6; \
+            } \
+        } \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a multi-byte sequence, then the macro will read
+ * the whole sequence.
+ * If the offset is behind a lead byte, then that itself
+ * will be returned as the code point.
+ * If the offset is behind an illegal UTF-8 sequence, then c is set to a negative value.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param i int32_t string offset, must be start<i
+ * @param c output UChar32 variable, set to <0 in case of an error
+ * @see U8_PREV_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_PREV(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(uint8_t)(s)[--(i)]; \
+    if(!U8_IS_SINGLE(c)) { \
+        (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -1); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one
+ * and get the code point between them.
+ * (Pre-decrementing backward iteration.)
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The input offset may be the same as the string length.
+ * If the offset is behind a multi-byte sequence, then the macro will read
+ * the whole sequence.
+ * If the offset is behind a lead byte, then that itself
+ * will be returned as the code point.
+ * If the offset is behind an illegal UTF-8 sequence, then c is set to U+FFFD.
+ *
+ * This macro does not distinguish between a real U+FFFD in the text
+ * and U+FFFD returned for an ill-formed sequence.
+ * Use U8_PREV() if that distinction is important.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param i int32_t string offset, must be start<i
+ * @param c output UChar32 variable, set to U+FFFD in case of an error
+ * @see U8_PREV
+ * @stable ICU 51
+ */
+#define U8_PREV_OR_FFFD(s, start, i, c) UPRV_BLOCK_MACRO_BEGIN { \
+    (c)=(uint8_t)(s)[--(i)]; \
+    if(!U8_IS_SINGLE(c)) { \
+        (c)=utf8_prevCharSafeBody((const uint8_t *)s, start, &(i), c, -3); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_BACK_1
+ * @stable ICU 2.4
+ */
+#define U8_BACK_1_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    while(U8_IS_TRAIL((s)[--(i)])) {} \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the previous one.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param i int32_t string offset, must be start<i
+ * @see U8_BACK_1_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_BACK_1(s, start, i) UPRV_BLOCK_MACRO_BEGIN { \
+    if(U8_IS_TRAIL((s)[--(i)])) { \
+        (i)=utf8_back1SafeBody(s, start, (i)); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @param n number of code points to skip
+ * @see U8_BACK_N
+ * @stable ICU 2.4
+ */
+#define U8_BACK_N_UNSAFE(s, i, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0) { \
+        U8_BACK_1_UNSAFE(s, i); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Move the string offset from one code point boundary to the n-th one before it,
+ * i.e., move backward by n code points.
+ * (Pre-decrementing backward iteration.)
+ * The input offset may be the same as the string length.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t index of the start of the string
+ * @param i int32_t string offset, must be start<i
+ * @param n number of code points to skip
+ * @see U8_BACK_N_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_BACK_N(s, start, i, n) UPRV_BLOCK_MACRO_BEGIN { \
+    int32_t __N=(n); \
+    while(__N>0 && (i)>(start)) { \
+        U8_BACK_1(s, start, i); \
+        --__N; \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind a partial multi-byte sequence,
+ * then the offset is incremented to behind the whole sequence.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Unsafe" macro, assumes well-formed UTF-8.
+ *
+ * @param s const uint8_t * string
+ * @param i string offset
+ * @see U8_SET_CP_LIMIT
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_LIMIT_UNSAFE(s, i) UPRV_BLOCK_MACRO_BEGIN { \
+    U8_BACK_1_UNSAFE(s, i); \
+    U8_FWD_1_UNSAFE(s, i); \
+} UPRV_BLOCK_MACRO_END
+
+/**
+ * Adjust a random-access offset to a code point boundary after a code point.
+ * If the offset is behind a partial multi-byte sequence,
+ * then the offset is incremented to behind the whole sequence.
+ * Otherwise, it is not modified.
+ * The input offset may be the same as the string length.
+ * "Safe" macro, checks for illegal sequences and for string boundaries.
+ *
+ * The length can be negative for a NUL-terminated string.
+ *
+ * @param s const uint8_t * string
+ * @param start int32_t starting string offset (usually 0)
+ * @param i int32_t string offset, must be start<=i<=length
+ * @param length int32_t string length
+ * @see U8_SET_CP_LIMIT_UNSAFE
+ * @stable ICU 2.4
+ */
+#define U8_SET_CP_LIMIT(s, start, i, length) UPRV_BLOCK_MACRO_BEGIN { \
+    if((start)<(i) && ((i)<(length) || (length)<0)) { \
+        U8_BACK_1(s, start, i); \
+        U8_FWD_1(s, i, length); \
+    } \
+} UPRV_BLOCK_MACRO_END
+
+#endif

third_party/tree-sitter/lib/src/wasm/stdlib-symbols.txt

@@ -0,0 +1,24 @@
+"calloc",
+"free",
+"iswalnum",
+"iswalpha",
+"iswblank",
+"iswdigit",
+"iswlower",
+"iswspace",
+"iswupper",
+"iswxdigit",
+"malloc",
+"memchr",
+"memcmp",
+"memcpy",
+"memmove",
+"memset",
+"realloc",
+"strcmp",
+"strlen",
+"strncat",
+"strncmp",
+"strncpy",
+"towlower",
+"towupper",

third_party/tree-sitter/lib/src/wasm/stdlib.c

@@ -0,0 +1,113 @@
+// This file implements a very simple allocator for external scanners running
+// in WASM. Allocation is just bumping a static pointer and growing the heap
+// as needed, and freeing is mostly a noop. But in the special case of freeing
+// the last-allocated pointer, we'll reuse that pointer again.
+
+#ifdef TREE_SITTER_FEATURE_WASM
+
+#include <stdio.h>
+#include <unistd.h>
+#include <stdlib.h>
+#include <string.h>
+
+extern void tree_sitter_debug_message(const char *, size_t);
+
+#define PAGESIZE 0x10000
+#define MAX_HEAP_SIZE (4 * 1024 * 1024)
+
+typedef struct {
+  size_t size;
+  char data[0];
+} Region;
+
+static Region *heap_end = NULL;
+static Region *heap_start = NULL;
+static Region *next = NULL;
+
+// Get the region metadata for the given heap pointer.
+static inline Region *region_for_ptr(void *ptr) {
+  return ((Region *)ptr) - 1;
+}
+
+// Get the location of the next region after the given region,
+// if the given region had the given size.
+static inline Region *region_after(Region *self, size_t len) {
+  char *address = self->data + len;
+  char *aligned = (char *)((uintptr_t)(address + 3) & ~0x3);
+  return (Region *)aligned;
+}
+
+static void *get_heap_end() {
+  return (void *)(__builtin_wasm_memory_size(0) * PAGESIZE);
+}
+
+static int grow_heap(size_t size) {
+  size_t new_page_count = ((size - 1) / PAGESIZE) + 1;
+  return __builtin_wasm_memory_grow(0, new_page_count) != SIZE_MAX;
+}
+
+// Clear out the heap, and move it to the given address.
+void reset_heap(void *new_heap_start) {
+  heap_start = new_heap_start;
+  next = new_heap_start;
+  heap_end = get_heap_end();
+}
+
+void *malloc(size_t size) {
+  Region *region_end = region_after(next, size);
+
+  if (region_end > heap_end) {
+    if ((char *)region_end - (char *)heap_start > MAX_HEAP_SIZE) {
+      return NULL;
+    }
+    if (!grow_heap(size)) return NULL;
+    heap_end = get_heap_end();
+  }
+
+  void *result = &next->data;
+  next->size = size;
+  next = region_end;
+
+  return result;
+}
+
+void free(void *ptr) {
+  if (ptr == NULL) return;
+
+  Region *region = region_for_ptr(ptr);
+  Region *region_end = region_after(region, region->size);
+
+  // When freeing the last allocated pointer, re-use that
+  // pointer for the next allocation.
+  if (region_end == next) {
+    next = region;
+  }
+}
+
+void *calloc(size_t count, size_t size) {
+  void *result = malloc(count * size);
+  memset(result, 0, count * size);
+  return result;
+}
+
+void *realloc(void *ptr, size_t new_size) {
+  if (ptr == NULL) {
+    return malloc(new_size);
+  }
+
+  Region *region = region_for_ptr(ptr);
+  Region *region_end = region_after(region, region->size);
+
+  // When reallocating the last allocated region, return
+  // the same pointer, and skip copying the data.
+  if (region_end == next) {
+    next = region;
+    return malloc(new_size);
+  }
+
+  void *result = malloc(new_size);
+  memcpy(result, &region->data, region->size);
+  return result;
+}
+
+#endif

third_party/tree-sitter/lib/src/wasm_store.h

@@ -0,0 +1,31 @@
+#ifndef TREE_SITTER_WASM_H_
+#define TREE_SITTER_WASM_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#include "tree_sitter/api.h"
+#include "./parser.h"
+
+bool ts_wasm_store_start(TSWasmStore *self, TSLexer *lexer, const TSLanguage *language);
+void ts_wasm_store_reset(TSWasmStore *self);
+bool ts_wasm_store_has_error(const TSWasmStore *self);
+
+bool ts_wasm_store_call_lex_main(TSWasmStore *self, TSStateId state);
+bool ts_wasm_store_call_lex_keyword(TSWasmStore *self, TSStateId state);
+
+uint32_t ts_wasm_store_call_scanner_create(TSWasmStore *self);
+void ts_wasm_store_call_scanner_destroy(TSWasmStore *self, uint32_t scanner_address);
+bool ts_wasm_store_call_scanner_scan(TSWasmStore *self, uint32_t scanner_address, uint32_t valid_tokens_ix);
+uint32_t ts_wasm_store_call_scanner_serialize(TSWasmStore *self, uint32_t scanner_address, char *buffer);
+void ts_wasm_store_call_scanner_deserialize(TSWasmStore *self, uint32_t scanner, const char *buffer, unsigned length);
+
+void ts_wasm_language_retain(const TSLanguage *self);
+void ts_wasm_language_release(const TSLanguage *self);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif  // TREE_SITTER_WASM_H_

todo.md

@@ -1,10 +1,14 @@
 # Bugs
-- Fix crash when cursor is over a new-line
 - Fix jumping forward a word jumping past consecutive brackets
 - Odd scrolling behavior on small screen heights
-- Closing the only panel crashes
+- Scrolling past end/beginning of results panics
+
+# Visual QOL
+- Split grep search results into a table to avoid funky unaligned text
 
 # Planned Features
+- [ ] Jump List
+- [x] Use grouped lifetimes exclusively for memory allocation/freeing
 - [ ] Highlight which panel is currently active
 - [ ] Persist end of line cursor position
 - Testing Harness
@@ -26,8 +30,12 @@
     - [ ] Go-to Definition/
     - [ ] Find references
 - Re-implement lost features from Plugins
-    - [ ] Syntax Highlighting
+    - [ ] Integrate tree-sitter
-        - [ ] Integrate tree-sitter
+        - [x] Syntax Highlighting
+        - [ ] Auto Setup Parsers
+            - [ ] Download parser
+            - [ ] Compile/"Install"
+        - [ ] Auto-indent?
     - [ ] Bootleg Telescope
         - [ ] Grepping Files
             - [x] Query across project
@@ -42,10 +50,13 @@
     - [x] New UI
     - [ ] Styling
 - Undo/Redo
+    - [x] Basic/Naive Undo/Redo
+    - [ ] Interface for undo-able actions
     - [ ] Edit History Tree
     - [ ] Undo history saved to disk
 - Finish selections
     - [x] Guarantee that start and end are always ordered
+    - [ ] Whole Line Selections
     - Add in text actions
         - [x] Yank
         - [x] Delete