Add extensible highlighter registration and Tree-sitter support.

- Implemented runtime API for registering custom highlighters.
- Added optional Tree-sitter integration for advanced syntax parsing (disabled by default).
- Updated buffer initialization and copying to support dynamic highlighter configuration.
- Introduced `NullHighlighter` as a fallback for unsupported filetypes.
- Enhanced CMake configuration with `KTE_ENABLE_TREESITTER` option.

Buffer.cc

@@ -6,6 +6,9 @@
 #include "Buffer.h"
 #include "UndoSystem.h"
 #include "UndoTree.h"
+// For reconstructing highlighter state on copies
+#include "HighlighterRegistry.h"
+#include "NullHighlighter.h"
 
 
 Buffer::Buffer()
@@ -40,9 +43,32 @@ Buffer::Buffer(const Buffer &other)
 	mark_set_       = other.mark_set_;
 	mark_curx_      = other.mark_curx_;
 	mark_cury_      = other.mark_cury_;
+	// Copy syntax/highlighting flags
+	version_        = other.version_;
+	syntax_enabled_ = other.syntax_enabled_;
+	filetype_       = other.filetype_;
 	// Fresh undo system for the copy
 	undo_tree_ = std::make_unique<UndoTree>();
 	undo_sys_  = std::make_unique<UndoSystem>(*this, *undo_tree_);
+
+	// Recreate a highlighter engine for this copy based on filetype/syntax state
+	if (syntax_enabled_) {
+		// Allocate engine and install an appropriate highlighter
+		highlighter_ = std::make_unique<kte::HighlighterEngine>();
+		if (!filetype_.empty()) {
+			auto hl = kte::HighlighterRegistry::CreateFor(filetype_);
+			if (hl) {
+				highlighter_->SetHighlighter(std::move(hl));
+			} else {
+				// Unsupported filetype -> NullHighlighter keeps syntax pipeline active
+				highlighter_->SetHighlighter(std::make_unique<kte::NullHighlighter>());
+			}
+		} else {
+			// No filetype -> keep syntax enabled but use NullHighlighter
+			highlighter_->SetHighlighter(std::make_unique<kte::NullHighlighter>());
+		}
+		// Fresh engine has empty caches; nothing to invalidate
+	}
 }
 
 
@@ -65,9 +91,28 @@ Buffer::operator=(const Buffer &other)
 	mark_set_       = other.mark_set_;
 	mark_curx_      = other.mark_curx_;
 	mark_cury_      = other.mark_cury_;
+	version_        = other.version_;
+	syntax_enabled_ = other.syntax_enabled_;
+	filetype_       = other.filetype_;
 	// Recreate undo system for this instance
 	undo_tree_ = std::make_unique<UndoTree>();
 	undo_sys_  = std::make_unique<UndoSystem>(*this, *undo_tree_);
+
+	// Recreate highlighter engine consistent with syntax settings
+	highlighter_.reset();
+	if (syntax_enabled_) {
+		highlighter_ = std::make_unique<kte::HighlighterEngine>();
+		if (!filetype_.empty()) {
+			auto hl = kte::HighlighterRegistry::CreateFor(filetype_);
+			if (hl) {
+				highlighter_->SetHighlighter(std::move(hl));
+			} else {
+				highlighter_->SetHighlighter(std::make_unique<kte::NullHighlighter>());
+			}
+		} else {
+			highlighter_->SetHighlighter(std::make_unique<kte::NullHighlighter>());
+		}
+	}
 	return *this;
 }
 
@@ -91,6 +136,11 @@ Buffer::Buffer(Buffer &&other) noexcept
 	  undo_tree_(std::move(other.undo_tree_)),
 	  undo_sys_(std::move(other.undo_sys_))
 {
+	// Move syntax/highlighting state
+	version_        = other.version_;
+	syntax_enabled_ = other.syntax_enabled_;
+	filetype_       = std::move(other.filetype_);
+	highlighter_    = std::move(other.highlighter_);
 	// Update UndoSystem's buffer reference to point to this object
 	if (undo_sys_) {
 		undo_sys_->UpdateBufferReference(*this);
@@ -122,6 +172,12 @@ Buffer::operator=(Buffer &&other) noexcept
 	undo_tree_      = std::move(other.undo_tree_);
 	undo_sys_       = std::move(other.undo_sys_);
 
+	// Move syntax/highlighting state
+	version_        = other.version_;
+	syntax_enabled_ = other.syntax_enabled_;
+	filetype_       = std::move(other.filetype_);
+	highlighter_    = std::move(other.highlighter_);
+
 	// Update UndoSystem's buffer reference to point to this object
 	if (undo_sys_) {
 		undo_sys_->UpdateBufferReference(*this);

CMakeLists.txt

@@ -4,7 +4,7 @@ project(kte)
 include(GNUInstallDirs)
 
 set(CMAKE_CXX_STANDARD 17)
-set(KTE_VERSION "1.1.2")
+set(KTE_VERSION "1.2.0")
 
 # Default to terminal-only build to avoid SDL/OpenGL dependency by default.
 # Enable with -DBUILD_GUI=ON when SDL2/OpenGL/Freetype are available.
@@ -13,6 +13,7 @@ set(BUILD_TESTS OFF CACHE BOOL "Enable building test programs.")
 option(KTE_USE_PIECE_TABLE "Use PieceTable instead of GapBuffer implementation" ON)
 set(KTE_FONT_SIZE "18.0" CACHE STRING "Default font size for GUI")
 option(KTE_UNDO_DEBUG "Enable undo instrumentation logs" OFF)
+option(KTE_ENABLE_TREESITTER "Enable optional Tree-sitter highlighter adapter" OFF)
 
 if (CMAKE_HOST_UNIX)
     message(STATUS "Build system is POSIX.")
@@ -37,6 +38,9 @@ else ()
 endif ()
 add_compile_definitions(KGE_PLATFORM=${CMAKE_HOST_SYSTEM_NAME})
 add_compile_definitions(KTE_VERSION_STR="v${KTE_VERSION}")
+if (KTE_ENABLE_TREESITTER)
+    add_compile_definitions(KTE_ENABLE_TREESITTER)
+endif ()
 
 message(STATUS "Build system: ${CMAKE_HOST_SYSTEM_NAME}")
 
@@ -80,6 +84,11 @@ set(COMMON_SOURCES
         LispHighlighter.cc
 )
 
+if (KTE_ENABLE_TREESITTER)
+    list(APPEND COMMON_SOURCES
+            TreeSitterHighlighter.cc)
+endif ()
+
 set(COMMON_HEADERS
         GapBuffer.h
         PieceTable.h
@@ -116,6 +125,11 @@ set(COMMON_HEADERS
         LispHighlighter.h
 )
 
+if (KTE_ENABLE_TREESITTER)
+    list(APPEND COMMON_HEADERS
+            TreeSitterHighlighter.h)
+endif ()
+
 # kte (terminal-first) executable
 add_executable(kte
         main.cc
@@ -132,6 +146,18 @@ endif ()
 
 target_link_libraries(kte ${CURSES_LIBRARIES})
 
+if (KTE_ENABLE_TREESITTER)
+    # Users can provide their own tree-sitter include/lib via cache variables
+    set(TREESITTER_INCLUDE_DIR "" CACHE PATH "Path to tree-sitter include directory")
+    set(TREESITTER_LIBRARY "" CACHE FILEPATH "Path to tree-sitter library (.a/.dylib)")
+    if (TREESITTER_INCLUDE_DIR)
+        target_include_directories(kte PRIVATE ${TREESITTER_INCLUDE_DIR})
+    endif ()
+    if (TREESITTER_LIBRARY)
+        target_link_libraries(kte ${TREESITTER_LIBRARY})
+    endif ()
+endif ()
+
 install(TARGETS kte
         RUNTIME DESTINATION ${CMAKE_INSTALL_BINDIR}
 )
@@ -157,6 +183,14 @@ if (BUILD_TESTS)
 
 
     target_link_libraries(test_undo ${CURSES_LIBRARIES})
+    if (KTE_ENABLE_TREESITTER)
+        if (TREESITTER_INCLUDE_DIR)
+            target_include_directories(test_undo PRIVATE ${TREESITTER_INCLUDE_DIR})
+        endif ()
+        if (TREESITTER_LIBRARY)
+            target_link_libraries(test_undo ${TREESITTER_LIBRARY})
+        endif ()
+    endif ()
 endif ()
 
 if (${BUILD_GUI})

Editor.cc

@@ -1,6 +1,8 @@
 #include <algorithm>
 #include <utility>
 #include <filesystem>
+#include "HighlighterRegistry.h"
+#include "NullHighlighter.h"
 
 #include "Editor.h"
 #include "HighlighterRegistry.h"
@@ -222,6 +224,27 @@ Editor::SwitchTo(std::size_t index)
         return false;
     }
     curbuf_ = index;
+    // Robustness: ensure a valid highlighter is installed when switching buffers
+    Buffer &b = buffers_[curbuf_];
+    if (b.SyntaxEnabled()) {
+        b.EnsureHighlighter();
+        if (auto *eng = b.Highlighter()) {
+            if (!eng->HasHighlighter()) {
+                // Try to set based on existing filetype; fall back to NullHighlighter
+                if (!b.Filetype().empty()) {
+                    auto hl = kte::HighlighterRegistry::CreateFor(b.Filetype());
+                    if (hl) {
+                        eng->SetHighlighter(std::move(hl));
+                    } else {
+                        eng->SetHighlighter(std::make_unique<kte::NullHighlighter>());
+                    }
+                } else {
+                    eng->SetHighlighter(std::make_unique<kte::NullHighlighter>());
+                }
+                eng->InvalidateFrom(0);
+            }
+        }
+    }
     return true;
 }
 

GUIInputHandler.cc

@@ -3,6 +3,7 @@
 #include <ncurses.h>
 
 #include <SDL.h>
+#include <imgui.h>
 
 #include "GUIInputHandler.h"
 #include "KKeymap.h"
@@ -284,6 +285,14 @@ GUIInputHandler::ProcessSDLEvent(const SDL_Event &e)
 	bool produced = false;
 	switch (e.type) {
 	case SDL_MOUSEWHEEL: {
+		// If ImGui wants to capture the mouse (e.g., hovering the File Picker list),
+		// don't translate wheel events into editor scrolling.
+		// This prevents background buffer scroll while using GUI widgets.
+		ImGuiIO &io = ImGui::GetIO();
+		if (io.WantCaptureMouse) {
+			return true; // consumed by GUI
+		}
+
 		// Map vertical wheel to line-wise cursor movement (MoveUp/MoveDown)
 		int dy = e.wheel.y;
 #ifdef SDL_MOUSEWHEEL_FLIPPED

GUIRenderer.cc

@@ -155,6 +155,12 @@ GUIRenderer::Draw(Editor &ed)
                     last_row  = first_row + vis_rows - 1;
                 }
             }
+            // Phase 3: prefetch visible viewport highlights and warm around in background
+            if (buf->SyntaxEnabled() && buf->Highlighter() && buf->Highlighter()->HasHighlighter()) {
+                int fr = static_cast<int>(std::max(0L, first_row));
+                int rc = static_cast<int>(std::max(1L, vis_rows));
+                buf->Highlighter()->PrefetchViewport(*buf, fr, rc, buf->Version());
+            }
         }
 		// Handle mouse click before rendering to avoid dependent on drawn items
 		if (ImGui::IsWindowHovered() && ImGui::IsMouseClicked(ImGuiMouseButton_Left)) {

HighlighterEngine.cc

@@ -1,40 +1,69 @@
 #include "HighlighterEngine.h"
 #include "Buffer.h"
 #include "LanguageHighlighter.h"
+#include <thread>
 
 namespace kte {
 
 HighlighterEngine::HighlighterEngine() = default;
-HighlighterEngine::~HighlighterEngine() = default;
+HighlighterEngine::~HighlighterEngine()
+{
+    // stop background worker
+    if (worker_running_.load()) {
+        {
+            std::lock_guard<std::mutex> lock(mtx_);
+            worker_running_.store(false);
+            has_request_ = true; // wake it up to exit
+        }
+        cv_.notify_one();
+        if (worker_.joinable()) worker_.join();
+    }
+}
 
 void
 HighlighterEngine::SetHighlighter(std::unique_ptr<LanguageHighlighter> hl)
 {
+    std::lock_guard<std::mutex> lock(mtx_);
     hl_ = std::move(hl);
     cache_.clear();
     state_cache_.clear();
+    state_last_contig_.clear();
 }
 
 const LineHighlight &
 HighlighterEngine::GetLine(const Buffer &buf, int row, std::uint64_t buf_version) const
 {
+    std::unique_lock<std::mutex> lock(mtx_);
     auto it = cache_.find(row);
-    if (it != cache_.end()) {
+    if (it != cache_.end() && it->second.version == buf_version) {
-        if (it->second.version == buf_version) {
         return it->second;
     }
-    }
+
-    LineHighlight updated;
+    // Prepare destination slot to reuse its capacity and avoid allocations
-    updated.version = buf_version;
+    LineHighlight &slot = cache_[row];
-    updated.spans.clear();
+    slot.version = buf_version;
+    slot.spans.clear();
+
     if (!hl_) {
-        auto &slot = cache_[row];
+        return slot;
-        slot = std::move(updated);
-        return cache_[row];
     }
 
-    if (auto *stateful = dynamic_cast<StatefulHighlighter *>(hl_.get())) {
+    // Copy shared_ptr-like raw pointer for use outside critical sections
-        // Find nearest cached state at or before row-1 with matching version
+    LanguageHighlighter *hl_ptr = hl_.get();
+    bool is_stateful = dynamic_cast<StatefulHighlighter *>(hl_ptr) != nullptr;
+
+    if (!is_stateful) {
+        // Stateless fast path: we can release the lock while computing to reduce contention
+        auto &out = slot.spans;
+        lock.unlock();
+        hl_ptr->HighlightLine(buf, row, out);
+        return cache_.at(row);
+    }
+
+    // Stateful path: we need to walk from a known previous state. Keep lock while consulting caches,
+    // but release during heavy computation.
+    auto *stateful = static_cast<StatefulHighlighter *>(hl_ptr);
+
     StatefulHighlighter::LineState prev_state;
     int start_row = -1;
     if (!state_cache_.empty()) {
@@ -43,9 +72,7 @@ HighlighterEngine::GetLine(const Buffer &buf, int row, std::uint64_t buf_version
         for (const auto &kv : state_cache_) {
             int r = kv.first;
             if (r <= row - 1 && kv.second.version == buf_version) {
-                    if (r > best) {
+                if (r > best) best = r;
-                        best = r;
-                    }
             }
         }
         if (best >= 0) {
@@ -54,31 +81,32 @@ HighlighterEngine::GetLine(const Buffer &buf, int row, std::uint64_t buf_version
         }
     }
 
-        // Walk from start_row+1 up to row computing states; only collect spans at the target row
+    // We'll compute states and the target line's spans without holding the lock for most of the work.
+    // Create a local copy of prev_state and iterate rows; we will update caches under lock.
+    lock.unlock();
+    StatefulHighlighter::LineState cur_state = prev_state;
     for (int r = start_row + 1; r <= row; ++r) {
         std::vector<HighlightSpan> tmp;
-            std::vector<HighlightSpan> &out = (r == row) ? updated.spans : tmp;
+        std::vector<HighlightSpan> &out = (r == row) ? slot.spans : tmp;
-            auto next_state = stateful->HighlightLineStateful(buf, r, prev_state, out);
+        auto next_state = stateful->HighlightLineStateful(buf, r, cur_state, out);
-            // store state for this row (state after finishing r)
+        // Update state cache for r
+        std::lock_guard<std::mutex> gl(mtx_);
         StateEntry se;
         se.version = buf_version;
         se.state = next_state;
         state_cache_[r] = se;
-            prev_state = next_state;
+        cur_state = next_state;
-        }
-    } else {
-        // Stateless path
-        hl_->HighlightLine(buf, row, updated.spans);
     }
 
-    auto &slot = cache_[row];
+    // Return reference under lock to ensure slot's address stability in map
-    slot = std::move(updated);
+    lock.lock();
-    return cache_[row];
+    return cache_.at(row);
 }
 
 void
 HighlighterEngine::InvalidateFrom(int row)
 {
+    std::lock_guard<std::mutex> lock(mtx_);
     if (cache_.empty()) return;
     // Simple implementation: erase all rows >= row
     for (auto it = cache_.begin(); it != cache_.end(); ) {
@@ -91,4 +119,63 @@ HighlighterEngine::InvalidateFrom(int row)
     }
 }
 
+void HighlighterEngine::ensure_worker_started() const
+{
+    if (worker_running_.load()) return;
+    worker_running_.store(true);
+    worker_ = std::thread([this]() { this->worker_loop(); });
+}
+
+void HighlighterEngine::worker_loop() const
+{
+    std::unique_lock<std::mutex> lock(mtx_);
+    while (worker_running_.load()) {
+        cv_.wait(lock, [this]() { return has_request_ || !worker_running_.load(); });
+        if (!worker_running_.load()) break;
+        WarmRequest req = pending_;
+        has_request_ = false;
+        // Copy locals then release lock while computing
+        lock.unlock();
+        if (req.buf) {
+            int start = std::max(0, req.start_row);
+            int end = std::max(start, req.end_row);
+            for (int r = start; r <= end; ++r) {
+                // Re-check version staleness quickly by peeking cache version; not strictly necessary
+                // Compute line; GetLine is thread-safe
+                (void)this->GetLine(*req.buf, r, req.version);
+            }
+        }
+        lock.lock();
+    }
+}
+
+void HighlighterEngine::PrefetchViewport(const Buffer &buf, int first_row, int row_count, std::uint64_t buf_version, int warm_margin) const
+{
+    if (row_count <= 0) return;
+    // Synchronously compute visible rows to ensure cache hits during draw
+    int start = std::max(0, first_row);
+    int end = start + row_count - 1;
+    int max_rows = static_cast<int>(buf.Nrows());
+    if (start >= max_rows) return;
+    if (end >= max_rows) end = max_rows - 1;
+
+    for (int r = start; r <= end; ++r) {
+        (void)GetLine(buf, r, buf_version);
+    }
+
+    // Enqueue background warm-around
+    int warm_start = std::max(0, start - warm_margin);
+    int warm_end = std::min(max_rows - 1, end + warm_margin);
+    {
+        std::lock_guard<std::mutex> lock(mtx_);
+        pending_.buf = &buf;
+        pending_.version = buf_version;
+        pending_.start_row = warm_start;
+        pending_.end_row = warm_end;
+        has_request_ = true;
+    }
+    ensure_worker_started();
+    cv_.notify_one();
+}
+
 } // namespace kte

HighlighterEngine.h

@@ -5,6 +5,10 @@
 #include <memory>
 #include <unordered_map>
 #include <vector>
+#include <mutex>
+#include <condition_variable>
+#include <atomic>
+#include <thread>
 
 #include "Highlight.h"
 #include "LanguageHighlighter.h"
@@ -29,6 +33,11 @@ public:
 
     bool HasHighlighter() const { return static_cast<bool>(hl_); }
 
+    // Phase 3: viewport-first prefetch and background warming
+    // Compute only the visible range now, and enqueue a background warm-around task.
+    // warm_margin: how many extra lines above/below to warm in the background.
+    void PrefetchViewport(const Buffer &buf, int first_row, int row_count, std::uint64_t buf_version, int warm_margin = 200) const;
+
 private:
     std::unique_ptr<LanguageHighlighter> hl_;
     // Simple cache by row index (mutable to allow caching in const GetLine)
@@ -40,6 +49,28 @@ private:
         StatefulHighlighter::LineState state;
     };
     mutable std::unordered_map<int, StateEntry> state_cache_;
+
+    // Track best known contiguous state row for a given version to avoid O(n) scans
+    mutable std::unordered_map<std::uint64_t, int> state_last_contig_;
+
+    // Thread-safety for caches and background worker state
+    mutable std::mutex mtx_;
+
+    // Background warmer
+    struct WarmRequest {
+        const Buffer *buf{nullptr};
+        std::uint64_t version{0};
+        int start_row{0};
+        int end_row{0}; // inclusive
+    };
+    mutable std::condition_variable cv_;
+    mutable std::thread worker_;
+    mutable std::atomic<bool> worker_running_{false};
+    mutable bool has_request_{false};
+    mutable WarmRequest pending_{};
+
+    void ensure_worker_started() const;
+    void worker_loop() const;
 };
 
 } // namespace kte

HighlighterRegistry.cc

@@ -3,9 +3,22 @@
 
 #include <algorithm>
 #include <filesystem>
+#include <vector>
+#include <cctype>
 
 // Forward declare simple highlighters implemented in this project
 namespace kte {
+
+// Registration storage
+struct RegEntry {
+    std::string ft; // normalized
+    HighlighterRegistry::Factory factory;
+};
+
+static std::vector<RegEntry> &registry() {
+    static std::vector<RegEntry> reg;
+    return reg;
+}
 class JSONHighlighter; class MarkdownHighlighter; class ShellHighlighter;
 class GoHighlighter; class PythonHighlighter; class RustHighlighter; class LispHighlighter;
 }
@@ -45,6 +58,10 @@ std::string HighlighterRegistry::Normalize(std::string_view ft)
 std::unique_ptr<LanguageHighlighter> HighlighterRegistry::CreateFor(std::string_view filetype)
 {
     std::string ft = Normalize(filetype);
+    // Prefer externally registered factories
+    for (const auto &e : registry()) {
+        if (e.ft == ft && e.factory) return e.factory();
+    }
     if (ft == "cpp") return std::make_unique<CppHighlighter>();
     if (ft == "json") return std::make_unique<JSONHighlighter>();
     if (ft == "markdown") return std::make_unique<MarkdownHighlighter>();
@@ -91,3 +108,50 @@ std::string HighlighterRegistry::DetectForPath(std::string_view path, std::strin
 }
 
 } // namespace kte
+
+// Extensibility API implementations
+namespace kte {
+
+void HighlighterRegistry::Register(std::string_view filetype, Factory factory, bool override_existing)
+{
+    std::string ft = Normalize(filetype);
+    for (auto &e : registry()) {
+        if (e.ft == ft) {
+            if (override_existing) e.factory = std::move(factory);
+            return;
+        }
+    }
+    registry().push_back(RegEntry{ft, std::move(factory)});
+}
+
+bool HighlighterRegistry::IsRegistered(std::string_view filetype)
+{
+    std::string ft = Normalize(filetype);
+    for (const auto &e : registry()) if (e.ft == ft) return true;
+    return false;
+}
+
+std::vector<std::string> HighlighterRegistry::RegisteredFiletypes()
+{
+    std::vector<std::string> out;
+    out.reserve(registry().size());
+    for (const auto &e : registry()) out.push_back(e.ft);
+    return out;
+}
+
+#ifdef KTE_ENABLE_TREESITTER
+// Forward declare adapter factory
+std::unique_ptr<LanguageHighlighter> CreateTreeSitterHighlighter(const char* filetype,
+                                                                 const void* (*get_lang)());
+
+void HighlighterRegistry::RegisterTreeSitter(std::string_view filetype,
+                                             const TSLanguage* (*get_language)())
+{
+    std::string ft = Normalize(filetype);
+    Register(ft, [ft, get_language]() {
+        return CreateTreeSitterHighlighter(ft.c_str(), reinterpret_cast<const void* (*)()>(get_language));
+    }, /*override_existing=*/true);
+}
+#endif
+
+} // namespace kte

HighlighterRegistry.h

@@ -1,10 +1,11 @@
-// HighlighterRegistry.h - create/detect language highlighters
+// HighlighterRegistry.h - create/detect language highlighters and allow external registration
 #pragma once
 
 #include <functional>
 #include <memory>
 #include <string>
 #include <string_view>
+#include <vector>
 
 #include "LanguageHighlighter.h"
 
@@ -12,6 +13,8 @@ namespace kte {
 
 class HighlighterRegistry {
 public:
+    using Factory = std::function<std::unique_ptr<LanguageHighlighter>()>;
+
     // Create a highlighter for normalized filetype id (e.g., "cpp", "json", "markdown", "shell", "go", "python", "rust", "lisp").
     static std::unique_ptr<LanguageHighlighter> CreateFor(std::string_view filetype);
 
@@ -21,6 +24,26 @@ public:
 
     // Normalize various aliases/extensions to canonical ids.
     static std::string Normalize(std::string_view ft);
+
+    // Extensibility: allow external code to register highlighters at runtime.
+    // The filetype key is normalized via Normalize(). If a factory is already registered for the
+    // normalized key and override=false, the existing factory is kept.
+    static void Register(std::string_view filetype, Factory factory, bool override_existing = true);
+
+    // Returns true if a factory is registered for the (normalized) filetype.
+    static bool IsRegistered(std::string_view filetype);
+
+    // Return a list of currently registered (normalized) filetypes. Primarily for diagnostics/tests.
+    static std::vector<std::string> RegisteredFiletypes();
+
+#ifdef KTE_ENABLE_TREESITTER
+    // Forward declaration to avoid hard dependency when disabled.
+    struct TSLanguage;
+    // Convenience: register a Tree-sitter-backed highlighter for a filetype.
+    // The getter should return a non-null language pointer for the grammar.
+    static void RegisterTreeSitter(std::string_view filetype,
+                                   const TSLanguage* (*get_language)());
+#endif
 };
 
 } // namespace kte

TerminalRenderer.cc

@@ -42,6 +42,13 @@ TerminalRenderer::Draw(Editor &ed)
 		std::size_t coloffs = buf->Coloffs();
 
 		const int tabw = 8;
+  // Phase 3: prefetch visible viewport highlights (current terminal area)
+        if (buf->SyntaxEnabled() && buf->Highlighter() && buf->Highlighter()->HasHighlighter()) {
+            int fr = static_cast<int>(rowoffs);
+            int rc = std::max(0, content_rows);
+            buf->Highlighter()->PrefetchViewport(*buf, fr, rc, buf->Version());
+        }
+
         for (int r = 0; r < content_rows; ++r) {
             move(r, 0);
             std::size_t li         = rowoffs + static_cast<std::size_t>(r);

TreeSitterHighlighter.cc

@@ -0,0 +1,46 @@
+#include "TreeSitterHighlighter.h"
+
+#ifdef KTE_ENABLE_TREESITTER
+
+#include "Buffer.h"
+#include <utility>
+
+namespace kte {
+
+TreeSitterHighlighter::TreeSitterHighlighter(const TSLanguage* lang, std::string filetype)
+    : language_(lang), filetype_(std::move(filetype))
+{
+}
+
+TreeSitterHighlighter::~TreeSitterHighlighter()
+{
+    disposeParser();
+}
+
+void TreeSitterHighlighter::ensureParsed(const Buffer& /*buf*/) const
+{
+    // Intentionally a stub to avoid pulling the Tree-sitter API and library by default.
+    // In future, when linking against tree-sitter, initialize parser_, set language_,
+    // and build tree_ from the buffer contents.
+}
+
+void TreeSitterHighlighter::disposeParser() const
+{
+    // Stub; nothing to dispose when not actually creating parser/tree
+}
+
+void TreeSitterHighlighter::HighlightLine(const Buffer &/*buf*/, int /*row*/, std::vector<HighlightSpan> &/*out*/) const
+{
+    // For now, no-op. When tree-sitter is wired, map nodes to TokenKind spans per line.
+}
+
+std::unique_ptr<LanguageHighlighter> CreateTreeSitterHighlighter(const char* filetype,
+                                                                 const void* (*get_lang)())
+{
+    const auto* lang = reinterpret_cast<const TSLanguage*>(get_lang ? get_lang() : nullptr);
+    return std::make_unique<TreeSitterHighlighter>(lang, filetype ? std::string(filetype) : std::string());
+}
+
+} // namespace kte
+
+#endif // KTE_ENABLE_TREESITTER

TreeSitterHighlighter.h

@@ -0,0 +1,48 @@
+// TreeSitterHighlighter.h - optional adapter for Tree-sitter (behind KTE_ENABLE_TREESITTER)
+#pragma once
+
+#ifdef KTE_ENABLE_TREESITTER
+
+#include <memory>
+#include <string>
+#include <vector>
+
+#include "LanguageHighlighter.h"
+
+// Forward-declare Tree-sitter C API to avoid hard coupling in headers if includes are not present
+extern "C" {
+struct TSLanguage;
+struct TSParser;
+struct TSTree;
+}
+
+namespace kte {
+
+// A minimal adapter that uses Tree-sitter to parse the whole buffer and then, for now,
+// does very limited token classification. This acts as a scaffold for future richer
+// queries. If no queries are provided, it currently produces no spans (safe fallback).
+class TreeSitterHighlighter : public LanguageHighlighter {
+public:
+    explicit TreeSitterHighlighter(const TSLanguage* lang, std::string filetype);
+    ~TreeSitterHighlighter() override;
+
+    void HighlightLine(const Buffer &buf, int row, std::vector<HighlightSpan> &out) const override;
+
+private:
+    const TSLanguage* language_{nullptr};
+    std::string filetype_;
+    // Lazy parser to avoid startup cost; mutable to allow creation in const method
+    mutable TSParser* parser_{nullptr};
+    mutable TSTree* tree_{nullptr};
+
+    void ensureParsed(const Buffer& buf) const;
+    void disposeParser() const;
+};
+
+// Factory used by HighlighterRegistry when registering via RegisterTreeSitter.
+std::unique_ptr<LanguageHighlighter> CreateTreeSitterHighlighter(const char* filetype,
+                                                                 const void* (*get_lang)());
+
+} // namespace kte
+
+#endif // KTE_ENABLE_TREESITTER

docs/lsp plan.md

@@ -0,0 +1,525 @@
+# LSP Support Implementation Plan for kte
+
+## Executive Summary
+
+This plan outlines a comprehensive approach to integrating Language Server Protocol (LSP) support into kte while
+respecting its core architectural principles: **frontend/backend separation**, **testability**, and **dual terminal/GUI
+support**.
+
+---
+
+## 1. Core Architecture
+
+### 1.1 LSP Client Module Structure
+
+```c++
+// LspClient.h - Core LSP client abstraction
+class LspClient {
+public:
+    virtual ~LspClient() = default;
+    
+    // Lifecycle
+    virtual bool initialize(const std::string& rootPath) = 0;
+    virtual void shutdown() = 0;
+    
+    // Document Synchronization
+    virtual void didOpen(const std::string& uri, const std::string& languageId, 
+                         int version, const std::string& text) = 0;
+    virtual void didChange(const std::string& uri, int version, 
+                           const std::vector<TextDocumentContentChangeEvent>& changes) = 0;
+    virtual void didClose(const std::string& uri) = 0;
+    virtual void didSave(const std::string& uri) = 0;
+    
+    // Language Features
+    virtual void completion(const std::string& uri, Position pos, 
+                           CompletionCallback callback) = 0;
+    virtual void hover(const std::string& uri, Position pos, 
+                      HoverCallback callback) = 0;
+    virtual void definition(const std::string& uri, Position pos, 
+                           LocationCallback callback) = 0;
+    virtual void references(const std::string& uri, Position pos, 
+                           LocationsCallback callback) = 0;
+    virtual void diagnostics(DiagnosticsCallback callback) = 0;
+    
+    // Process Management
+    virtual bool isRunning() const = 0;
+    virtual std::string getServerName() const = 0;
+};
+```
+
+### 1.2 Process-based LSP Implementation
+
+```c++
+// LspProcessClient.h - Manages LSP server subprocess
+class LspProcessClient : public LspClient {
+private:
+    std::string serverCommand_;
+    std::vector<std::string> serverArgs_;
+    std::unique_ptr<Process> process_;
+    std::unique_ptr<JsonRpcTransport> transport_;
+    std::unordered_map<int, PendingRequest> pendingRequests_;
+    int nextRequestId_ = 1;
+    
+    // Async I/O handling
+    std::thread readerThread_;
+    std::mutex mutex_;
+    std::condition_variable cv_;
+    
+public:
+    LspProcessClient(const std::string& command, 
+                     const std::vector<std::string>& args);
+    // ... implementation of LspClient interface
+};
+```
+
+### 1.3 JSON-RPC Transport Layer
+
+```c++
+// JsonRpcTransport.h
+class JsonRpcTransport {
+public:
+    // Send a request and get the request ID
+    int sendRequest(const std::string& method, const nlohmann::json& params);
+    
+    // Send a notification (no response expected)
+    void sendNotification(const std::string& method, const nlohmann::json& params);
+    
+    // Read next message (blocking)
+    std::optional<JsonRpcMessage> readMessage();
+    
+private:
+    void writeMessage(const nlohmann::json& message);
+    std::string readContentLength();
+    
+    int fdIn_;   // stdin to server
+    int fdOut_;  // stdout from server
+};
+```
+
+---
+
+## 2. Incremental Document Updates
+
+### 2.1 Change Tracking in Buffer
+
+The key to efficient LSP integration is tracking changes incrementally. This integrates with the existing `Buffer`
+class:
+
+```c++
+// TextDocumentContentChangeEvent.h
+struct TextDocumentContentChangeEvent {
+    std::optional<Range> range;     // If nullopt, entire document changed
+    std::optional<int> rangeLength; // Deprecated but some servers use it
+    std::string text;
+};
+
+// BufferChangeTracker.h - Integrates with Buffer to track changes
+class BufferChangeTracker {
+public:
+    explicit BufferChangeTracker(Buffer* buffer);
+    
+    // Called by Buffer on each edit operation
+    void recordInsertion(Position pos, const std::string& text);
+    void recordDeletion(Range range, const std::string& deletedText);
+    
+    // Get accumulated changes since last sync
+    std::vector<TextDocumentContentChangeEvent> getChanges();
+    
+    // Clear changes after sending to LSP
+    void clearChanges();
+    
+    // Get current document version
+    int getVersion() const { return version_; }
+    
+private:
+    Buffer* buffer_;
+    int version_ = 0;
+    std::vector<TextDocumentContentChangeEvent> pendingChanges_;
+    
+    // Optional: Coalesce adjacent changes
+    void coalesceChanges();
+};
+```
+
+### 2.2 Integration with Buffer Operations
+
+```c++
+// Buffer.h additions
+class Buffer {
+    // ... existing code ...
+    
+    // LSP integration
+    void setChangeTracker(std::unique_ptr<BufferChangeTracker> tracker);
+    BufferChangeTracker* getChangeTracker() { return changeTracker_.get(); }
+    
+    // These methods should call tracker when present
+    void insertText(Position pos, const std::string& text);
+    void deleteRange(Range range);
+    
+private:
+    std::unique_ptr<BufferChangeTracker> changeTracker_;
+};
+```
+
+### 2.3 Sync Strategy Selection
+
+```c++
+// LspSyncMode.h
+enum class LspSyncMode {
+    None,        // No sync
+    Full,        // Send full document on each change
+    Incremental  // Send only changes (preferred)
+};
+
+// Determined during server capability negotiation
+LspSyncMode negotiateSyncMode(const ServerCapabilities& caps);
+```
+
+---
+
+## 3. Diagnostics Display System
+
+### 3.1 Diagnostic Data Model
+
+```c++
+// Diagnostic.h
+enum class DiagnosticSeverity {
+    Error = 1,
+    Warning = 2,
+    Information = 3,
+    Hint = 4
+};
+
+struct Diagnostic {
+    Range range;
+    DiagnosticSeverity severity;
+    std::optional<std::string> code;
+    std::optional<std::string> source;
+    std::string message;
+    std::vector<DiagnosticRelatedInformation> relatedInfo;
+};
+
+// DiagnosticStore.h - Central storage for diagnostics
+class DiagnosticStore {
+public:
+    void setDiagnostics(const std::string& uri, 
+                        std::vector<Diagnostic> diagnostics);
+    const std::vector<Diagnostic>& getDiagnostics(const std::string& uri) const;
+    std::vector<Diagnostic> getDiagnosticsAtLine(const std::string& uri, 
+                                                  int line) const;
+    std::optional<Diagnostic> getDiagnosticAtPosition(const std::string& uri,
+                                                       Position pos) const;
+    void clear(const std::string& uri);
+    void clearAll();
+    
+    // Statistics
+    int getErrorCount(const std::string& uri) const;
+    int getWarningCount(const std::string& uri) const;
+    
+private:
+    std::unordered_map<std::string, std::vector<Diagnostic>> diagnostics_;
+};
+```
+
+### 3.2 Frontend-Agnostic Diagnostic Display Interface
+
+Following kte's existing abstraction pattern with `Frontend`, `Renderer`, and `InputHandler`:
+
+```c++
+// DiagnosticDisplay.h - Abstract interface for showing diagnostics
+class DiagnosticDisplay {
+public:
+    virtual ~DiagnosticDisplay() = default;
+    
+    // Update the diagnostic indicators for a buffer
+    virtual void updateDiagnostics(const std::string& uri, 
+                                   const std::vector<Diagnostic>& diagnostics) = 0;
+    
+    // Show inline diagnostic at cursor position
+    virtual void showInlineDiagnostic(const Diagnostic& diagnostic) = 0;
+    
+    // Show diagnostic list/panel
+    virtual void showDiagnosticList(const std::vector<Diagnostic>& diagnostics) = 0;
+    virtual void hideDiagnosticList() = 0;
+    
+    // Status bar summary
+    virtual void updateStatusBar(int errorCount, int warningCount) = 0;
+};
+```
+
+### 3.3 Terminal Diagnostic Display
+
+```c++
+// TerminalDiagnosticDisplay.h
+class TerminalDiagnosticDisplay : public DiagnosticDisplay {
+public:
+    explicit TerminalDiagnosticDisplay(TerminalRenderer* renderer);
+    
+    void updateDiagnostics(const std::string& uri, 
+                          const std::vector<Diagnostic>& diagnostics) override;
+    void showInlineDiagnostic(const Diagnostic& diagnostic) override;
+    void showDiagnosticList(const std::vector<Diagnostic>& diagnostics) override;
+    void hideDiagnosticList() override;
+    void updateStatusBar(int errorCount, int warningCount) override;
+    
+private:
+    TerminalRenderer* renderer_;
+    
+    // Terminal-specific display strategies
+    void renderGutterMarkers(const std::vector<Diagnostic>& diagnostics);
+    void renderUnderlines(const std::vector<Diagnostic>& diagnostics);
+    void renderVirtualText(const Diagnostic& diagnostic);
+};
+```
+
+**Terminal Display Strategies:**
+
+1. **Gutter markers**: Show `E` (error), `W` (warning), `I` (info), `H` (hint) in left gutter
+2. **Underlines**: Use terminal underline/curly underline capabilities (where supported)
+3. **Virtual text**: Display diagnostic message at end of line (configurable)
+4. **Status line**: `[E:3 W:5]` summary
+5. **Message line**: Full diagnostic on cursor line shown in bottom bar
+
+```
+1 │ fn main() {
+E 2 │     let x: i32 = "hello";
+  3 │ }                        
+──────────────────────────────────────
+error[E0308]: mismatched types       
+expected `i32`, found `&str`         
+[E:1 W:0] main.rs
+```
+
+### 3.4 GUI Diagnostic Display
+
+```c++
+// GUIDiagnosticDisplay.h
+class GUIDiagnosticDisplay : public DiagnosticDisplay {
+public:
+    explicit GUIDiagnosticDisplay(GUIRenderer* renderer, GUITheme* theme);
+    
+    void updateDiagnostics(const std::string& uri, 
+                          const std::vector<Diagnostic>& diagnostics) override;
+    void showInlineDiagnostic(const Diagnostic& diagnostic) override;
+    void showDiagnosticList(const std::vector<Diagnostic>& diagnostics) override;
+    void hideDiagnosticList() override;
+    void updateStatusBar(int errorCount, int warningCount) override;
+    
+private:
+    GUIRenderer* renderer_;
+    GUITheme* theme_;
+    
+    // GUI-specific display
+    void renderWavyUnderlines(const std::vector<Diagnostic>& diagnostics);
+    void renderTooltip(Position pos, const Diagnostic& diagnostic);
+    void renderDiagnosticPanel();
+};
+```
+
+**GUI Display Features:**
+
+1. **Wavy underlines**: Classic IDE-style (red for errors, yellow for warnings, etc.)
+2. **Gutter icons**: Colored icons/dots in the gutter
+3. **Hover tooltips**: Rich tooltips on hover showing full diagnostic
+4. **Diagnostic panel**: Bottom panel with clickable diagnostic list
+5. **Minimap markers**: Colored marks on the minimap (if present)
+
+---
+
+## 4. LspManager - Central Coordination
+
+```c++
+// LspManager.h
+class LspManager {
+public:
+    explicit LspManager(Editor* editor, DiagnosticDisplay* display);
+    
+    // Server management
+    void registerServer(const std::string& languageId, 
+                       const LspServerConfig& config);
+    bool startServerForBuffer(Buffer* buffer);
+    void stopServer(const std::string& languageId);
+    void stopAllServers();
+    
+    // Document sync
+    void onBufferOpened(Buffer* buffer);
+    void onBufferChanged(Buffer* buffer);
+    void onBufferClosed(Buffer* buffer);
+    void onBufferSaved(Buffer* buffer);
+    
+    // Feature requests
+    void requestCompletion(Buffer* buffer, Position pos, 
+                          CompletionCallback callback);
+    void requestHover(Buffer* buffer, Position pos, 
+                     HoverCallback callback);
+    void requestDefinition(Buffer* buffer, Position pos, 
+                          LocationCallback callback);
+    
+    // Configuration
+    void setDebugLogging(bool enabled);
+    
+private:
+    Editor* editor_;
+    DiagnosticDisplay* display_;
+    DiagnosticStore diagnosticStore_;
+    std::unordered_map<std::string, std::unique_ptr<LspClient>> servers_;
+    std::unordered_map<std::string, LspServerConfig> serverConfigs_;
+    
+    void handleDiagnostics(const std::string& uri, 
+                          const std::vector<Diagnostic>& diagnostics);
+    std::string getLanguageId(Buffer* buffer);
+    std::string getUri(Buffer* buffer);
+};
+```
+
+---
+
+## 5. Configuration
+
+```c++
+// LspServerConfig.h
+struct LspServerConfig {
+    std::string command;
+    std::vector<std::string> args;
+    std::vector<std::string> filePatterns;  // e.g., {"*.rs", "*.toml"}
+    std::string rootPatterns;               // e.g., "Cargo.toml"
+    LspSyncMode preferredSyncMode = LspSyncMode::Incremental;
+    bool autostart = true;
+    std::unordered_map<std::string, nlohmann::json> initializationOptions;
+    std::unordered_map<std::string, nlohmann::json> settings;
+};
+
+// Default configurations
+std::vector<LspServerConfig> getDefaultServerConfigs() {
+    return {
+        {
+            .command = "rust-analyzer",
+            .filePatterns = {"*.rs"},
+            .rootPatterns = "Cargo.toml"
+        },
+        {
+            .command = "clangd",
+            .args = {"--background-index"},
+            .filePatterns = {"*.c", "*.cc", "*.cpp", "*.h", "*.hpp"},
+            .rootPatterns = "compile_commands.json"
+        },
+        {
+            .command = "gopls",
+            .filePatterns = {"*.go"},
+            .rootPatterns = "go.mod"
+        },
+        // ... more servers
+    };
+}
+```
+
+---
+
+## 6. Implementation Phases
+
+### Phase 1: Foundation (2-3 weeks)
+
+- [ ] JSON-RPC transport layer
+- [ ] Process management for LSP servers
+- [ ] Basic `LspClient` with initialize/shutdown
+- [ ] `textDocument/didOpen`, `textDocument/didClose` (full sync)
+
+### Phase 2: Incremental Sync (1-2 weeks)
+
+- [ ] `BufferChangeTracker` integration with `Buffer`
+- [ ] `textDocument/didChange` with incremental updates
+- [ ] Change coalescing for rapid edits
+- [ ] Version tracking
+
+### Phase 3: Diagnostics (2-3 weeks)
+
+- [ ] `DiagnosticStore` implementation
+- [ ] `TerminalDiagnosticDisplay` with gutter markers & status line
+- [ ] `GUIDiagnosticDisplay` with wavy underlines & tooltips
+- [ ] `textDocument/publishDiagnostics` handling
+
+### Phase 4: Language Features (3-4 weeks)
+
+- [ ] Completion (`textDocument/completion`)
+- [ ] Hover (`textDocument/hover`)
+- [ ] Go to definition (`textDocument/definition`)
+- [ ] Find references (`textDocument/references`)
+- [ ] Code actions (`textDocument/codeAction`)
+
+### Phase 5: Polish & Advanced Features (2-3 weeks)
+
+- [ ] Multiple server support
+- [ ] Server auto-detection
+- [ ] Configuration file support
+- [ ] Workspace symbol search
+- [ ] Rename refactoring
+
+---
+
+## 7. Alignment with kte Core Principles
+
+### 7.1 Frontend/Backend Separation
+
+- LSP logic is completely separate from display
+- `DiagnosticDisplay` interface allows identical behavior across Terminal/GUI
+- Follows existing pattern: `Renderer`, `InputHandler`, `Frontend`
+
+### 7.2 Testability
+
+- `LspClient` is abstract, enabling `MockLspClient` for testing
+- `DiagnosticDisplay` can be mocked for testing diagnostic flow
+- Change tracking can be unit tested in isolation
+
+### 7.3 Performance
+
+- Incremental sync minimizes data sent to LSP servers
+- Async message handling doesn't block UI
+- Diagnostic rendering is batched
+
+### 7.4 Simplicity
+
+- Minimal dependencies (nlohmann/json for JSON handling)
+- Self-contained process management
+- Clear separation of concerns
+
+---
+
+## 8. File Organization
+
+```
+kte/
+├── lsp/
+│   ├── LspClient.h
+│   ├── LspProcessClient.h
+│   ├── LspProcessClient.cc
+│   ├── LspManager.h
+│   ├── LspManager.cc
+│   ├── LspServerConfig.h
+│   ├── JsonRpcTransport.h
+│   ├── JsonRpcTransport.cc
+│   ├── LspTypes.h           # Position, Range, Location, etc.
+│   ├── Diagnostic.h
+│   ├── DiagnosticStore.h
+│   ├── DiagnosticStore.cc
+│   └── BufferChangeTracker.h
+├── diagnostic/
+│   ├── DiagnosticDisplay.h
+│   ├── TerminalDiagnosticDisplay.h
+│   ├── TerminalDiagnosticDisplay.cc
+│   ├── GUIDiagnosticDisplay.h
+│   └── GUIDiagnosticDisplay.cc
+```
+
+---
+
+## 9. Dependencies
+
+- **nlohmann/json**: JSON parsing/serialization (header-only)
+- **POSIX/Windows process APIs**: For spawning LSP servers
+- Existing kte infrastructure: `Buffer`, `Renderer`, `Frontend`, etc.
+
+---
+
+This plan provides a solid foundation for LSP support while maintaining kte's clean architecture. The key insight is
+that LSP is fundamentally a backend feature that should be displayed through the existing frontend abstraction layer,
+ensuring consistent behavior across terminal and GUI modes.

docs/syntax.md

@@ -50,3 +50,21 @@ Renderer integration
 
 - Terminal and GUI renderers request line spans via `Highlighter()->GetLine(buf, row, buf.Version())`.
 - Search highlight and cursor overlays take precedence over syntax colors.
+
+Extensibility (Phase 4)
+-----------------------
+
+- Public registration API: external code can register custom highlighters by filetype.
+  - Use `HighlighterRegistry::Register("mylang", []{ return std::make_unique<MyHighlighter>(); });`
+  - Registered factories are preferred over built-ins for the same filetype key.
+  - Filetype keys are normalized via `HighlighterRegistry::Normalize()`.
+- Optional Tree-sitter adapter: disabled by default to keep dependencies minimal.
+  - Enable with CMake option `-DKTE_ENABLE_TREESITTER=ON` and provide
+    `-DTREESITTER_INCLUDE_DIR=...` and `-DTREESITTER_LIBRARY=...` if needed.
+  - Register a Tree-sitter-backed highlighter for a language (example assumes you link a grammar):
+    ```c++
+    extern "C" const TSLanguage* tree_sitter_c();
+    kte::HighlighterRegistry::RegisterTreeSitter("c", &tree_sitter_c);
+    ```
+  - Current adapter is a stub scaffold; it compiles and integrates cleanly when enabled, but
+    intentionally emits no spans until Tree-sitter node-to-token mapping is implemented.