diff --git a/CLAUDE.md b/CLAUDE.md index 25b5e2b..7877058 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -36,6 +36,7 @@ The test binary runs all tests, or only those whose names contain one of its arg | `ENABLE_ASAN` | OFF | AddressSanitizer | | `KTE_STATIC_LINK` | OFF | Static linking (Linux only) | | `KTE_ENABLE_TREESITTER` | OFF | Tree-sitter syntax highlighting | +| `KTE_USE_PCRE2` | ON | Regex search/replace via PCRE2 when `libpcre2-8` is found (std::regex otherwise; see `RegexEngine.h`) | ### Nix diff --git a/CMakeLists.txt b/CMakeLists.txt index 36267d2..5762b8e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,6 +15,7 @@ set(KTE_FONT_SIZE "18.0" CACHE STRING "Default font size for GUI") option(KTE_UNDO_DEBUG "Enable undo instrumentation logs" OFF) option(KTE_ENABLE_TREESITTER "Enable optional Tree-sitter highlighter adapter" OFF) option(KTE_STATIC_LINK "Enable static linking on Linux" OFF) +option(KTE_USE_PCRE2 "Use PCRE2 (libpcre2-8) for regex search when found; std::regex otherwise" ON) # Optionally enable AddressSanitizer (ASan) option(ENABLE_ASAN "Enable AddressSanitizer for builds" OFF) @@ -66,6 +67,26 @@ endif () message(STATUS "Build system: ${CMAKE_HOST_SYSTEM_NAME}") +# Regex engine (see RegexEngine.h). Linked by name (-lpcre2-8) so that a +# static build picks up the static library. +if (KTE_USE_PCRE2) + find_package(PkgConfig QUIET) + if (PKG_CONFIG_FOUND) + pkg_check_modules(PCRE2 QUIET libpcre2-8) + endif () + if (PCRE2_FOUND) + message(STATUS "Regex engine: PCRE2 ${PCRE2_VERSION}") + add_compile_definitions(KTE_USE_PCRE2) + include_directories(${PCRE2_INCLUDE_DIRS}) + link_directories(${PCRE2_LIBRARY_DIRS}) + link_libraries(${PCRE2_LIBRARIES}) + else () + message(STATUS "Regex engine: std::regex (libpcre2-8 not found)") + endif () +else () + message(STATUS "Regex engine: std::regex") +endif () + if (BUILD_GUI) include(cmake/imgui.cmake) endif () @@ -175,6 +196,7 @@ set(COMMON_SOURCES SyscallWrappers.cc ErrorRecovery.cc RegexGuard.cc + RegexEngine.cc TerminalInputHandler.cc TerminalRenderer.cc TerminalFrontend.cc @@ -273,6 +295,7 @@ set(COMMON_HEADERS TestRenderer.h TestFrontend.h TextSpan.h + RegexEngine.h UndoNode.h UndoTree.h UndoSystem.h @@ -389,6 +412,7 @@ if (BUILD_TESTS) SyscallWrappers.cc ErrorRecovery.cc RegexGuard.cc + RegexEngine.cc KKeymap.cc SwapRecorder.h OptimizedSearch.cc diff --git a/Command.cc b/Command.cc index 075510a..3742ce2 100644 --- a/Command.cc +++ b/Command.cc @@ -13,6 +13,7 @@ #include "Command.h" #include "ErrorHandler.h" #include "RegexGuard.h" +#include "RegexEngine.h" #include "TermWidth.h" #include "syntax/HighlighterRegistry.h" #include "syntax/NullHighlighter.h" @@ -927,39 +928,20 @@ static constexpr std::size_t kSearchCountBytesPlain = std::size_t{64} << 20; static constexpr std::size_t kSearchCountBytesRegex = std::size_t{4} << 20; -class RegexMatcher { -public: - bool Compile(const std::string &pattern, std::string &err) - { - err.clear(); - try { - rx_ = std::regex(pattern); - return true; - } catch (const std::regex_error &e) { - err = e.what(); - return false; - } - } +using RegexMatcher = kte::Regex; // see RegexEngine.h - // First match in `line` starting at byte >= from. - bool Find(std::string_view line, std::size_t from, std::size_t &pos, std::size_t &len) const - { - if (from > line.size()) - return false; - std::cmatch m; - const auto flags = from > 0 ? std::regex_constants::match_prev_avail - : std::regex_constants::match_default; - if (!std::regex_search(line.data() + from, line.data() + line.size(), m, rx_, flags)) - return false; - pos = from + static_cast(m.position(0)); - len = static_cast(m.length(0)); - return true; - } - -private: - std::regex rx_; -}; +// Run regex work: std::regex recurses per matched character, so it runs on +// a large stack (RegexGuard.h); PCRE2 does not recurse on the C stack. +template +static void +run_regex_work(Fn &&fn) +{ + if (std::string_view(kte::Regex::EngineName()) == "PCRE2") + fn(); + else + kte::RunWithLargeStack(fn); +} static std::string_view line_text_view(const Buffer &buf, std::size_t y); @@ -1034,23 +1016,88 @@ plain_find(const Buffer &buf, const std::string &q, std::size_t y, std::size_t x } +// Walks the lines of the materialized text (a view per line; no per-row +// buffer calls) forward or backward from row y, wrapping. +class LineWalker { +public: + LineWalker(const Buffer &buf, std::size_t y) + : text_(buffer_text_view(buf)), nrows_(std::max(buf.Nrows(), 1)), + y_(std::min(y, nrows_ - 1)), start_(std::min(buf.RowColToOffset(y_, 0), text_.size())) {} + + + [[nodiscard]] std::size_t Row() const + { + return y_; + } + + + [[nodiscard]] std::string_view Line() const + { + const std::size_t e = text_.find('\n', start_); + return text_.substr(start_, (e == std::string_view::npos ? text_.size() : e) - start_); + } + + + void Next() + { + const std::size_t e = text_.find('\n', start_); + if (e == std::string_view::npos || y_ + 1 >= nrows_) { + y_ = 0; + start_ = 0; + } else { + ++y_; + start_ = e + 1; + } + } + + + void Prev() + { + if (y_ == 0 || start_ == 0) { + y_ = nrows_ - 1; + const std::size_t n = text_.rfind('\n'); + start_ = n == std::string_view::npos ? 0 : n + 1; + return; + } + --y_; + // start_ - 1 is the newline ending the previous row. + const std::size_t n = start_ >= 2 ? text_.rfind('\n', start_ - 2) : std::string_view::npos; + start_ = n == std::string_view::npos ? 0 : n + 1; + } + +private: + std::string_view text_; + std::size_t nrows_; + std::size_t y_; + std::size_t start_; +}; + + // Regex: like plain_find, line by line. Lines longer than line_limit are -// skipped (counted in *skipped). +// skipped (counted in *skipped); *limited counts lines where the engine hit +// its match limit (catastrophic backtracking) and gave up. static bool regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t x0, bool forward, - std::size_t line_limit, std::size_t *skipped, SearchHit &hit) + std::size_t line_limit, std::size_t *skipped, std::size_t *limited, SearchHit &hit) { const std::size_t nrows = buf.Nrows(); if (nrows == 0) return false; y0 = std::min(y0, nrows - 1); bool found = false; - kte::RunWithLargeStack([&] { + auto scan = [&] { + LineWalker w(buf, y0); // nrows + 1 visits: the start row is visited again after wrapping, // for the part on the other side of x0. for (std::size_t i = 0; i <= nrows && !found; ++i) { - const std::size_t y = forward ? (y0 + i) % nrows : (y0 + nrows - i % nrows) % nrows; - const std::string_view line = line_text_view(buf, y); + if (i > 0) { + if (forward) + w.Next(); + else + w.Prev(); + } + const std::size_t y = w.Row(); + const std::string_view line = w.Line(); if (line.size() > line_limit) { if (skipped && i < nrows) ++*skipped; @@ -1058,9 +1105,15 @@ regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t } const bool first = i == 0, again = i == nrows; std::size_t pos = 0, len = 0; + auto search = [&](std::size_t from) { + const bool ok = m.Search(line, from, pos, len); + if (!ok && limited && m.LimitHit()) + ++*limited; + return ok; + }; if (forward) { const std::size_t from = first ? x0 : 0; - if (m.Find(line, from, pos, len) && (!again || pos < x0)) { + if (search(from) && (!again || pos < x0)) { hit = SearchHit{y, pos, len}; found = true; } @@ -1070,7 +1123,7 @@ regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t bool have = false; SearchHit best{}; std::size_t from = 0; - while (from <= line.size() && m.Find(line, from, pos, len)) { + while (from <= line.size() && search(from)) { if (first && pos >= x0) break; if (!again || pos >= x0) { @@ -1085,7 +1138,8 @@ regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t } } } - }); + }; + run_regex_work(scan); return found; } @@ -1126,14 +1180,15 @@ search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMat } std::size_t scanned = 0; bool stopped = false; - kte::RunWithLargeStack([&] { - for (std::size_t y = 0; y < nrows && !stopped; ++y) { - const std::string_view line = line_text_view(buf, y); + run_regex_work([&] { + LineWalker w(buf, 0); + for (std::size_t y = 0; y < nrows && !stopped; ++y, w.Next()) { + const std::string_view line = w.Line(); if (line.size() > line_limit) continue; scanned += line.size() + 1; std::size_t from = 0, pos = 0, len = 0; - while (from <= line.size() && m->Find(line, from, pos, len)) { + while (from <= line.size() && m->Search(line, from, pos, len)) { if (y < cur.y || (y == cur.y && pos < cur.x)) ++c.before; else @@ -1167,7 +1222,7 @@ enum class SearchStep { Here, Next, Prev }; // status. Here: from the current match (so it stays while the extended query // still matches there) or the search origin; Next/Prev: after/before the // current match. incremental limits regex to lines up to -// kRegexIncrementalLineLimit (search-as-you-type). +// Regex::IncrementalLineLimit() (search-as-you-type). static void run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incremental) { @@ -1195,15 +1250,21 @@ run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incrementa std::string err; SearchHit hit; bool found = false; - std::size_t skipped = 0; - const std::size_t line_limit = (regex && incremental) ? kte::kRegexIncrementalLineLimit + std::size_t skipped = 0, limited = 0; + const std::size_t line_limit = (regex && incremental) ? kte::Regex::IncrementalLineLimit() : static_cast(-1); if (!q.empty()) { if (!regex) found = plain_find(buf, q, y, x, step != SearchStep::Prev, hit); else if (m.Compile(q, err)) - found = regex_find(buf, m, y, x, step != SearchStep::Prev, line_limit, &skipped, hit); + found = regex_find(buf, m, y, x, step != SearchStep::Prev, line_limit, &skipped, &limited, hit); } + // Notes appended to the status. + std::string notes; + if (skipped > 0) + notes += " [" + std::to_string(skipped) + " long line(s) skipped; Left/Right search all]"; + if (limited > 0) + notes += " [pattern too complex on " + std::to_string(limited) + " line(s); stopped there]"; if (!found) { ed.SetSearchMatch(0, 0, 0); @@ -1215,8 +1276,8 @@ run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incrementa std::string status = prefix + q; if (!err.empty()) status += " [error: " + err + "]"; - else if (skipped > 0) - status += " [" + std::to_string(skipped) + " long line(s) skipped; Left/Right search all]"; + else + status += notes; ed.SetStatus(status); return; } @@ -1233,10 +1294,7 @@ run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incrementa where = std::to_string(c.before + 1) + "/" + std::to_string(c.total) + "+"; else where = std::to_string(c.total) + "+ matches"; - std::string status = prefix + q + " " + where; - if (skipped > 0) - status += " [" + std::to_string(skipped) + " long line(s) skipped; Left/Right search all]"; - ed.SetStatus(status); + ed.SetStatus(prefix + q + " " + where + notes); } @@ -3570,11 +3628,10 @@ cmd_newline(CommandContext &ctx) ctx.editor.SetSearchIndex(-1); return true; } - std::regex rx; - try { - rx = std::regex(patt); - } catch (const std::regex_error &e) { - ctx.editor.SetStatus(std::string("Regex error: ") + e.what()); + kte::Regex rx; + std::string rx_err; + if (!rx.Compile(patt, rx_err)) { + ctx.editor.SetStatus("Regex error: " + rx_err); // Clear search UI state ctx.editor.SetSearchActive(false); ctx.editor.SetSearchQuery(""); @@ -3592,12 +3649,11 @@ cmd_newline(CommandContext &ctx) if (nrows > 1 && buf->GetLineString(nrows - 1).empty()) --nrows; // Build the new text line by line and apply it as one edit (one - // edit per changed line was quadratic on big files). std::regex - // recurses per matched character, so match on a large stack - // (RegexGuard.h). + // edit per changed line was quadratic on big files). const std::size_t all_rows = buf->Nrows(); std::string after; - kte::RunWithLargeStack([&] { + bool limited = false; + run_regex_work([&] { for (std::size_t y = 0; y < all_rows; ++y) { if (y > 0) after.push_back('\n'); @@ -3606,7 +3662,8 @@ cmd_newline(CommandContext &ctx) after += before_line; // the empty row after a final '\n' continue; } - const std::string replaced = std::regex_replace(before_line, rx, repl); + const std::string replaced = rx.ReplaceAll(before_line, repl); + limited = limited || rx.LimitHit(); if (replaced != before_line) ++changed; after += replaced; @@ -3616,7 +3673,8 @@ cmd_newline(CommandContext &ctx) replace_buffer_text(*buf, buffer_text(*buf), after, ru); clamp_cursor_to_buffer(*buf); buf->SetDirty(true); - ctx.editor.SetStatus("Regex replaced in " + std::to_string(changed) + " line(s)"); + ctx.editor.SetStatus("Regex replaced in " + std::to_string(changed) + " line(s)" + + (limited ? " (pattern too complex on some lines; those left as they were)" : "")); // Clear search UI state ctx.editor.SetSearchActive(false); ctx.editor.SetSearchQuery(""); diff --git a/Dockerfile b/Dockerfile index e4636a7..5dbf713 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,6 +9,8 @@ RUN apk add --no-cache \ cmake \ make \ ncurses-dev \ + pcre2-dev \ + pkgconf \ sdl2-dev \ mesa-dev \ freetype-dev \ diff --git a/ImGuiRenderer.cc b/ImGuiRenderer.cc index 3abee55..1fa2bcf 100644 --- a/ImGuiRenderer.cc +++ b/ImGuiRenderer.cc @@ -7,10 +7,9 @@ #include #include -#include #include "ImGuiRenderer.h" -#include "RegexGuard.h" +#include "RegexEngine.h" #include "Highlight.h" #include "GUITheme.h" #include "Buffer.h" @@ -164,15 +163,11 @@ ImGuiRenderer::Draw(Editor &ed) const bool regex_mode = search_mode && ed.PromptActive() && ( ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind); - std::regex search_rx; + kte::Regex search_rx; // RegexEngine.h bool search_rx_valid = false; if (regex_mode) { - try { - search_rx = std::regex(ed.SearchQuery()); - search_rx_valid = true; - } catch (const std::regex_error &) { - search_rx_valid = false; - } + std::string rx_err; + search_rx_valid = search_rx.Compile(ed.SearchQuery(), rx_err); } // Compute the visible row range and skip rendering work for off-screen @@ -360,15 +355,13 @@ ImGuiRenderer::Draw(Editor &ed) if (search_mode) { // In regex mode, reuse the compiled regex hoisted above the loop. if (regex_mode) { - // std::regex recursion is kept off the main stack for long - // lines (RegexGuard.h); very long lines are not highlighted. - if (search_rx_valid && line.size() <= kte::kRegexRenderLineLimit) { - try { - kte::ForEachRegexMatch(line, search_rx, [&](std::size_t pos, std::size_t len) { - hl_src_ranges.emplace_back(pos, pos + len); - }); - } catch (const std::regex_error &) { - // ignore invalid patterns here; status line already shows the error + // Very long lines are not highlighted (see + // Regex::IncrementalLineLimit). + if (search_rx_valid && line.size() <= kte::Regex::IncrementalLineLimit()) { + std::size_t from = 0, pos = 0, len = 0; + while (from <= line.size() && search_rx.Search(line, from, pos, len)) { + hl_src_ranges.emplace_back(pos, pos + len); + from = pos + std::max(len, 1); } } } else { diff --git a/RegexEngine.cc b/RegexEngine.cc new file mode 100644 index 0000000..c1fa32f --- /dev/null +++ b/RegexEngine.cc @@ -0,0 +1,324 @@ +#include "RegexEngine.h" + +#include + +#if defined(KTE_USE_PCRE2) +#define PCRE2_CODE_UNIT_WIDTH 8 +#include +#endif + +namespace kte { +namespace { +// Expand an ECMAScript replacement for one match, as std::regex_replace +// does: $` is the text since the previous match (match_results::prefix()). +// group(i, begin, end) returns whether group i participated and where; +// ngroups counts group 0. +template +void +append_format(std::string &out, std::string_view fmt, std::string_view subject, std::size_t prev_end, + std::size_t ms, std::size_t me, std::size_t ngroups, GroupFn &&group) +{ + for (std::size_t i = 0; i < fmt.size(); ++i) { + const char c = fmt[i]; + if (c != '$' || i + 1 >= fmt.size()) { + out.push_back(c); + continue; + } + const char n = fmt[i + 1]; + if (n == '$') { + out.push_back('$'); + ++i; + } else if (n == '&') { + out.append(subject.substr(ms, me - ms)); + ++i; + } else if (n == '`') { + out.append(subject.substr(prev_end, ms - prev_end)); + ++i; + } else if (n == '\'') { + out.append(subject.substr(me)); + ++i; + } else if (n >= '0' && n <= '9') { + // $nn if that group exists, else $n; $0 is not a reference. + std::size_t idx = static_cast(n - '0'); + std::size_t used = 1; + if (i + 2 < fmt.size() && fmt[i + 2] >= '0' && fmt[i + 2] <= '9') { + const std::size_t two = idx * 10 + static_cast(fmt[i + 2] - '0'); + if (two >= 1 && two < ngroups) { + idx = two; + used = 2; + } + } + if (idx == 0 || idx >= ngroups) { + out.push_back(c); // not a group reference: literal + continue; + } + std::size_t b = 0, e = 0; + if (group(idx, b, e)) + out.append(subject.substr(b, e - b)); + i += used; + } else { + out.push_back(c); + } + } +} +} // namespace + + +#if defined(KTE_USE_PCRE2) + +// PCRE2 gives up on a match attempt after this many internal steps +// (catastrophic backtracking), reporting it instead of running for ages. +static constexpr std::uint32_t kMatchLimit = 10000000; + + +struct Regex::Impl { + pcre2_code *code = nullptr; + pcre2_match_data *md = nullptr; + pcre2_match_context *mctx = nullptr; + std::size_t ngroups = 1; // including group 0 + mutable bool limit_hit = false; + + + ~Impl() + { + if (md) + pcre2_match_data_free(md); + if (mctx) + pcre2_match_context_free(mctx); + if (code) + pcre2_code_free(code); + } + + + // pcre2_match with a subject that is never a null pointer. + int match(std::string_view s, std::size_t from, std::uint32_t opts) const + { + static const char empty = '\0'; + const char *p = s.data() ? s.data() : ∅ + const int rc = pcre2_match(code, reinterpret_cast(p), s.size(), from, opts, md, mctx); + if (rc == PCRE2_ERROR_MATCHLIMIT || rc == PCRE2_ERROR_DEPTHLIMIT || rc == PCRE2_ERROR_HEAPLIMIT) + limit_hit = true; + return rc; + } +}; + + +Regex::Regex() = default; + +Regex::~Regex() = default; + +Regex::Regex(Regex &&other) noexcept = default; + +Regex &Regex::operator=(Regex &&other) noexcept = default; + + +bool +Regex::Compile(std::string_view pattern, std::string &err) +{ + err.clear(); + impl_.reset(); + auto impl = std::make_unique(); + int errcode = 0; + PCRE2_SIZE erroff = 0; + static const char empty = '\0'; + impl->code = pcre2_compile(reinterpret_cast(pattern.data() ? pattern.data() : &empty), + pattern.size(), 0, &errcode, &erroff, nullptr); + if (!impl->code) { + PCRE2_UCHAR msg[256]; + pcre2_get_error_message(errcode, msg, sizeof(msg)); + err = std::string(reinterpret_cast(msg)) + " at offset " + std::to_string(erroff); + return false; + } + (void) pcre2_jit_compile(impl->code, PCRE2_JIT_COMPLETE); // interpreted if unavailable + impl->md = pcre2_match_data_create_from_pattern(impl->code, nullptr); + impl->mctx = pcre2_match_context_create(nullptr); + if (!impl->md || !impl->mctx) { + err = "out of memory compiling the pattern"; + return false; + } + pcre2_set_match_limit(impl->mctx, kMatchLimit); + std::uint32_t captures = 0; + (void) pcre2_pattern_info(impl->code, PCRE2_INFO_CAPTURECOUNT, &captures); + impl->ngroups = static_cast(captures) + 1; + impl_ = std::move(impl); + return true; +} + + +bool +Regex::Search(std::string_view subject, std::size_t from, std::size_t &pos, std::size_t &len) const +{ + if (!impl_ || from > subject.size()) + return false; + impl_->limit_hit = false; + if (impl_->match(subject, from, 0) <= 0) + return false; + const PCRE2_SIZE *ov = pcre2_get_ovector_pointer(impl_->md); + pos = ov[0]; + len = ov[1] - ov[0]; + return true; +} + + +std::string +Regex::ReplaceAll(std::string_view s, std::string_view fmt) const +{ + if (!impl_) + return std::string(s); + impl_->limit_hit = false; + std::string out; + std::size_t copied = 0; // subject bytes up to here are in `out` + std::size_t start = 0; + std::uint32_t opts = 0; + while (start <= s.size()) { + const int rc = impl_->match(s, start, opts); + if (rc == PCRE2_ERROR_NOMATCH && opts != 0) { + // No non-empty match right after an empty one: step past a byte. + if (start >= s.size()) + break; + ++start; + opts = 0; + continue; + } + if (rc <= 0) { + if (impl_->limit_hit) + return std::string(s); // gave up: leave the text as it was + break; + } + const PCRE2_SIZE *ov = pcre2_get_ovector_pointer(impl_->md); + const std::size_t ms = ov[0], me = ov[1]; + out.append(s.substr(copied, ms - copied)); + append_format(out, fmt, s, copied, ms, me, impl_->ngroups, [&](std::size_t i, std::size_t &b, std::size_t &e) { + if (i >= static_cast(rc) || ov[2 * i] == PCRE2_UNSET) + return false; + b = ov[2 * i]; + e = ov[2 * i + 1]; + return true; + }); + copied = me; + start = me; + // After an empty match, the next one must not be empty at the same place. + opts = (me == ms) ? (PCRE2_NOTEMPTY_ATSTART | PCRE2_ANCHORED) : 0; + } + out.append(s.substr(copied)); + return out; +} + + +bool +Regex::Valid() const +{ + return static_cast(impl_); +} + + +bool +Regex::LimitHit() const +{ + return impl_ && impl_->limit_hit; +} + + +const char * +Regex::EngineName() +{ + return "PCRE2"; +} + + +std::size_t +Regex::IncrementalLineLimit() +{ + return std::size_t{1} << 20; +} + +#else // std::regex + +struct Regex::Impl { + std::regex rx; +}; + + +Regex::Regex() = default; + +Regex::~Regex() = default; + +Regex::Regex(Regex &&other) noexcept = default; + +Regex &Regex::operator=(Regex &&other) noexcept = default; + + +bool +Regex::Compile(std::string_view pattern, std::string &err) +{ + err.clear(); + impl_.reset(); + try { + auto impl = std::make_unique(); + impl->rx = std::regex(pattern.begin(), pattern.end()); + impl_ = std::move(impl); + return true; + } catch (const std::regex_error &e) { + err = e.what(); + return false; + } +} + + +bool +Regex::Search(std::string_view subject, std::size_t from, std::size_t &pos, std::size_t &len) const +{ + if (!impl_ || from > subject.size()) + return false; + std::cmatch m; + const auto flags = from > 0 ? std::regex_constants::match_prev_avail : std::regex_constants::match_default; + const char *b = subject.data() + from; + const char *e = subject.data() + subject.size(); + if (!std::regex_search(b, e, m, impl_->rx, flags)) + return false; + pos = from + static_cast(m.position(0)); + len = static_cast(m.length(0)); + return true; +} + + +std::string +Regex::ReplaceAll(std::string_view s, std::string_view fmt) const +{ + if (!impl_) + return std::string(s); + std::string out; + std::regex_replace(std::back_inserter(out), s.begin(), s.end(), impl_->rx, std::string(fmt)); + return out; +} + + +bool +Regex::Valid() const +{ + return static_cast(impl_); +} + + +bool +Regex::LimitHit() const +{ + return false; +} + + +const char * +Regex::EngineName() +{ + return "std::regex"; +} + + +std::size_t +Regex::IncrementalLineLimit() +{ + return 2000; +} + +#endif +} // namespace kte diff --git a/RegexEngine.h b/RegexEngine.h new file mode 100644 index 0000000..0c18b19 --- /dev/null +++ b/RegexEngine.h @@ -0,0 +1,68 @@ +/* + * RegexEngine.h - the regex engine behind search, replace and match + * highlighting. + * + * With KTE_USE_PCRE2 (the default when libpcre2-8 is found) patterns are + * compiled by PCRE2 with its JIT: typically one to two orders of magnitude + * faster than std::regex, no recursion on the C stack, and a match limit + * that stops catastrophic backtracking (e.g. "^(a+)+$") instead of hanging + * the editor. Otherwise std::regex (ECMAScript) is used, run on a large + * stack (see RegexGuard.h). The syntax both accept covers ordinary use; + * PCRE2 additionally has lookbehind, possessive quantifiers and the like. + * + * Replacement strings use ECMAScript syntax with either engine, as + * std::regex_replace reads it: $& (the match), $1..$99 (groups), $` (the + * text since the previous match), $' (the rest of the line), $$ (a dollar). + */ +#pragma once + +#include +#include +#include +#include + +namespace kte { +class Regex { +public: + Regex(); + + ~Regex(); + + Regex(Regex &&other) noexcept; + + Regex &operator=(Regex &&other) noexcept; + + Regex(const Regex &) = delete; + + Regex &operator=(const Regex &) = delete; + + // Compile; on failure err describes the problem and Valid() is false. + bool Compile(std::string_view pattern, std::string &err); + + [[nodiscard]] bool Valid() const; + + // The first match in `subject` starting at byte `from` or later; `from` + // may be inside the subject (anchors and lookbehind see what precedes + // it). False if there is none, or if the engine gave up (LimitHit()). + bool Search(std::string_view subject, std::size_t from, std::size_t &pos, std::size_t &len) const; + + // Every non-overlapping match replaced by `fmt` (ECMAScript syntax, see + // above); an empty match is not repeated at the same position. + [[nodiscard]] std::string ReplaceAll(std::string_view subject, std::string_view fmt) const; + + // Whether the last Search/ReplaceAll stopped at the engine's match limit. + [[nodiscard]] bool LimitHit() const; + + // "PCRE2" or "std::regex". + static const char *EngineName(); + + // The longest line search-as-you-type and match highlighting run the + // regex over (per keystroke/frame); std::regex can be quadratic in line + // length with no way to interrupt it, PCRE2 is bounded by its limit. + static std::size_t IncrementalLineLimit(); + +private: + struct Impl; + std::unique_ptr impl_; +}; +} // namespace kte diff --git a/RegexGuard.h b/RegexGuard.h index b37bb09..46b9f01 100644 --- a/RegexGuard.h +++ b/RegexGuard.h @@ -12,13 +12,14 @@ #include namespace kte { -// Renderers highlight regex matches only on lines up to this many bytes, and -// search-as-you-type skips longer lines (an explicit search does not): +// Used where std::regex is still called directly (the Qt frontend's match +// highlighting; search and the other renderers use RegexEngine.h, whose +// Regex::IncrementalLineLimit() applies the same limits for std::regex). +// Regex matches are highlighted only on lines up to this many bytes: // std::regex can take time quadratic in line length even for ordinary // patterns (".*q" on a 20 KB line: 12 s), with no way to interrupt it, and // these run on every frame or keystroke. -constexpr std::size_t kRegexRenderLineLimit = 2000; -constexpr std::size_t kRegexIncrementalLineLimit = 2000; +constexpr std::size_t kRegexRenderLineLimit = 2000; // Run fn to completion on a thread with a large stack (address space is // reserved, pages are committed only as used), so regex work over long lines diff --git a/Swap.cc b/Swap.cc index 1af3386..08419cf 100644 --- a/Swap.cc +++ b/Swap.cc @@ -1246,12 +1246,12 @@ SwapManager::maybe_request_checkpoint(Buffer &buf, const std::size_t approx_edit if (ctx.last_chkpt_ns == 0) ctx.last_chkpt_ns = now; // A checkpoint writes the whole buffer. For a large one, checkpoint - // after edits amounting to half its size (and on the timer only - // once edits reach a sixteenth), so the bytes written stay in - // proportion to the bytes edited; retry a gap less often too. + // after edits amounting to its size (and on the timer only once + // edits reach a sixteenth), so the bytes written stay in proportion + // to the bytes edited; retry a gap less often too. const std::size_t size = buf.ContentBytes(); const bool large = size > kLargeCheckpointBytes; - const std::size_t bytes_limit = std::max(cfg.checkpoint_bytes, size / 2); + const std::size_t bytes_limit = std::max(cfg.checkpoint_bytes, large ? size : 0); const bool bytes_hit = (cfg.checkpoint_bytes > 0) && ctx.edit_bytes_since_chkpt >= bytes_limit; const bool time_hit = (cfg.checkpoint_interval_ms > 0) && (((now - ctx.last_chkpt_ns) / 1000000ULL) >= cfg.checkpoint_interval_ms) && @@ -1324,6 +1324,13 @@ SwapManager::RecordCheckpoint(Buffer &buf, const bool urgent_flush) auto it = journals_.find(&buf); if (it == journals_.end() || it->second.suspended) return; + // One queued checkpoint at a time: records queued after it apply on + // top of it, so a second one adds nothing but another full copy of + // the buffer in memory (several whole-file edits in a row on a large + // buffer queued one each). + if (it->second.chkpt_queued) + return; + it->second.chkpt_queued = true; } // Any size: the writer frames large checkpoints as several records. @@ -1416,6 +1423,13 @@ SwapManager::process_one(const Pending &p) { if (!p.buf) return; + if (p.type == SwapRecType::CHKPT) { + // Being handled now: a later request queues a new checkpoint. + std::lock_guard lg(mtx_); + auto it = journals_.find(p.buf); + if (it != journals_.end()) + it->second.chkpt_queued = false; + } // Any record that is not written leaves a gap in the journal. auto mark_gap = [&]() { diff --git a/Swap.h b/Swap.h index 1880962..7df7425 100644 --- a/Swap.h +++ b/Swap.h @@ -236,6 +236,7 @@ private: // drop them until a checkpoint re-establishes the full content. bool gap{false}; bool gap_notified{false}; // the user was told about the current gap + bool chkpt_queued{false}; // a checkpoint is queued and not yet being written bool lockout_notified{false}; // told about the lock; kept across ResetJournal std::uint64_t gap_chkpt_request_ns{0}; bool gap_unfixable_reported{false}; // gap on a buffer too large to checkpoint diff --git a/TerminalRenderer.cc b/TerminalRenderer.cc index ef9b94c..e42acea 100644 --- a/TerminalRenderer.cc +++ b/TerminalRenderer.cc @@ -6,11 +6,10 @@ #include #include #include -#include #include #include "TerminalRenderer.h" -#include "RegexGuard.h" +#include "RegexEngine.h" #include "TermWidth.h" #include "Buffer.h" #include "Editor.h" @@ -58,6 +57,16 @@ TerminalRenderer::Draw(Editor &ed) buf->Highlighter()->PrefetchViewport(*buf, fr, rc, buf->Version()); } + // The search pattern, compiled once per frame (it was compiled for + // every row). + kte::Regex draw_rx; + bool draw_rx_ok = false; + if (ed.SearchActive() && !ed.SearchQuery().empty() && ed.PromptActive() && + (ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || + ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) { + std::string rx_err; + draw_rx_ok = draw_rx.Compile(ed.SearchQuery(), rx_err); + } for (int r = 0; r < content_rows; ++r) { move(r, 0); std::size_t li = rowoffs + static_cast(r); @@ -72,15 +81,15 @@ TerminalRenderer::Draw(Editor &ed) if (ed.PromptActive() && ( ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || ed. CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) { - // Long lines are not highlighted: std::regex recursion - // could overflow the stack (RegexGuard.h). - if (sline.size() <= kte::kRegexRenderLineLimit) try { - std::regex rx(ed.SearchQuery()); - kte::ForEachRegexMatch(sline, rx, [&](std::size_t pos, std::size_t len) { + // Very long lines are not highlighted (see + // Regex::IncrementalLineLimit); an invalid pattern + // highlights nothing (the status shows the error). + if (draw_rx_ok && sline.size() <= kte::Regex::IncrementalLineLimit()) { + std::size_t from = 0, pos = 0, len = 0; + while (from <= sline.size() && draw_rx.Search(sline, from, pos, len)) { ranges.emplace_back(pos, pos + len); - }); - } catch (const std::regex_error &) { - // ignore invalid patterns here; status shows error + from = pos + std::max(len, 1); + } } } else { const std::string &q = ed.SearchQuery(); diff --git a/default.nix b/default.nix index 687d04c..49b6fb1 100644 --- a/default.nix +++ b/default.nix @@ -2,7 +2,9 @@ lib, stdenv, cmake, + pkg-config, ncurses, + pcre2, SDL2, libGL, xorg, @@ -30,6 +32,7 @@ stdenv.mkDerivation { nativeBuildInputs = [ cmake + pkg-config installShellFiles ] ++ lib.optionals graphical [ copyDesktopItems @@ -39,6 +42,7 @@ stdenv.mkDerivation { buildInputs = [ ncurses + pcre2 # regex engine (optional; std::regex without it) ] ++ lib.optionals graphical [ SDL2 libGL diff --git a/docs/audits/claude-findings-20260923.md b/docs/audits/claude-findings-20260923.md index 483d066..e98dcb4 100644 --- a/docs/audits/claude-findings-20260923.md +++ b/docs/audits/claude-findings-20260923.md @@ -544,9 +544,24 @@ before the fix (unless noted). thread (it was skipped), so a touched but unchanged large file no longer makes its journal unusable for recovery. +**Follow-up: regex engine** +- kte::Regex (RegexEngine.h) wraps PCRE2 with its JIT when libpcre2-8 + is found (KTE_USE_PCRE2, default ON), std::regex otherwise. Search, + regex replace-all and the terminal and ImGui renderers use it (the Qt + frontend's highlighting still calls std::regex). PCRE2 has a match + limit, so catastrophic backtracking ("^(a+)+$") stops with a status + note instead of hanging, and does not recurse on the C stack, so its + per-keystroke line limit is 1 MiB instead of 2,000 bytes. Replacement + strings keep std::regex_replace semantics with either engine. +- On 26 MB: regex keystroke 1.6 -> 0.14 ms, next 62 -> 6 ms, + replace-all 929 -> 438 ms; a rarely matching regex over 105 MB: + 1.55 s -> 0.10 s per keystroke. +- Only one checkpoint per journal is queued at a time; whole-file edits + in a row on a large buffer each queued a full copy of it. + **Known limitations (not fixed)** -- Catastrophic regex backtracking (e.g. `(a*)*b`) can still take very - long; std::regex has no time limit. +- Without PCRE2 (std::regex builds), catastrophic regex backtracking + can still take very long; std::regex has no limit. - Typing in a multi-megabyte single line costs O(line length) per keystroke (cursor column computed by scanning the line). - GUI-only items B7, B8, B10 and highlighter edge cases B13 are @@ -555,5 +570,5 @@ before the fix (unless noted). - Journals are created at the first edit, so a second kte that opens the file before the first one edits it is not warned at open; the session that is locked out is told at its first edit instead. -- A regex that matches rarely or not at all still scans the whole - buffer with std::regex (1.6 s on 105 MB). +- A regex that matches rarely or not at all scans the whole buffer + (0.1 s on 105 MB with PCRE2, 1.6 s with std::regex). diff --git a/tests/test_audit_regressions.cc b/tests/test_audit_regressions.cc index f6c088c..8f66e35 100644 --- a/tests/test_audit_regressions.cc +++ b/tests/test_audit_regressions.cc @@ -9,6 +9,7 @@ #include "ErrorRecovery.h" #include "PieceTable.h" #include "RegexGuard.h" +#include "RegexEngine.h" #include "Swap.h" #include "UndoTree.h" #include "syntax/HighlighterEngine.h" @@ -1285,14 +1286,16 @@ TEST(Audit_RegexSearch_LongLineFoundByNext) { TestHarness h; Editor &ed = h.EditorRef(); - h.Buf().insert_text(0, 0, "short\n" + std::string(25000, 'a') + "NEEDLE\nend\n"); + // Longer than search-as-you-type scans (depends on the regex engine). + const std::size_t long_len = kte::Regex::IncrementalLineLimit() + 1000; + h.Buf().insert_text(0, 0, "short\n" + std::string(long_len, 'a') + "NEEDLE\nend\n"); h.Buf().SetCursor(0, 0); ASSERT_TRUE(h.Exec(CommandId::RegexFindStart)); ASSERT_TRUE(h.Exec(CommandId::InsertText, "NEE+DLE")); ASSERT_TRUE(ed.Status().find("long line") != std::string::npos); ASSERT_TRUE(h.Exec(CommandId::MoveRight)); ASSERT_EQ(h.Buf().Cury(), (std::size_t) 1); - ASSERT_EQ(h.Buf().Curx(), (std::size_t) 25000); + ASSERT_EQ(h.Buf().Curx(), long_len); ASSERT_TRUE(h.Exec(CommandId::Refresh)); ASSERT_TRUE(!ed.PromptActive()); // The regex replace prompt steps through matches the same way. @@ -1302,7 +1305,7 @@ TEST(Audit_RegexSearch_LongLineFoundByNext) ASSERT_TRUE(ed.Status().find("Left/Right search all") != std::string::npos); ASSERT_TRUE(h.Exec(CommandId::MoveRight)); ASSERT_EQ(h.Buf().Cury(), (std::size_t) 1); - ASSERT_EQ(h.Buf().Curx(), (std::size_t) 25000); + ASSERT_EQ(h.Buf().Curx(), long_len); } @@ -2179,3 +2182,49 @@ TEST(Swap_DeferredBaseCrc) b.SetSwapRecorder(nullptr); sm.Detach(&b, true); } + + +// kte::Regex replacements follow std::regex_replace (ECMAScript), whichever +// engine is built in. (References to groups the pattern does not have, like +// $3 with two groups, are implementation-defined and left out.) +TEST(RegexEngine_ReplaceMatchesStdRegex) +{ + const char *subjects[] = {"", "baaac", "one two three", "a-b-c", "xyz", "aaa", " lead", "tail "}; + const std::pair cases[] = { + {"a*", "X"}, {"a", "[$&]"}, {"(\\w+)", "<$1>"}, {"(o)(n)?", "$2$1"}, {"-", "$`|$'"}, + {"\\s+", "$$"}, {"^", ">"}, {"$", "<"}, {"b|", "_"}, {"\\b", "|"}, + {"([a-z])([a-z])", "$2$1"}, {"(a)(x)?", "[$2]"} + }; + for (const char *subj: subjects) { + for (const auto &[pat, fmt]: cases) { + kte::Regex rx; + std::string err; + ASSERT_TRUE(rx.Compile(pat, err)); + const std::string got = rx.ReplaceAll(subj, fmt); + const std::string want = std::regex_replace(std::string(subj), std::regex(pat), fmt); + if (got != want) + fprintf(stderr, "subj=[%s] pat=[%s] fmt=[%s] got=[%s] want=[%s]\n", subj, pat, fmt, + got.c_str(), want.c_str()); + ASSERT_EQ(got, want); + } + } +} + + +// With PCRE2, catastrophic backtracking stops at the match limit and says +// so, instead of hanging the editor. +TEST(RegexEngine_CatastrophicPatternStops) +{ + if (std::string(kte::Regex::EngineName()) != "PCRE2") + return; // std::regex has no limit (a known limitation) + TestHarness h; + h.Buf().insert_text(0, 0, std::string(40, 'a') + "!\n"); + h.Buf().SetCursor(0, 0); + const auto t0 = std::chrono::steady_clock::now(); + ASSERT_TRUE(h.Exec(CommandId::RegexFindStart)); + ASSERT_TRUE(h.Exec(CommandId::InsertText, "^(a+)+$")); + const double secs = std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + ASSERT_TRUE(secs < 10.0); + ASSERT_TRUE(h.EditorRef().Status().find("too complex") != std::string::npos); + ASSERT_TRUE(h.Exec(CommandId::Refresh)); +}