diff --git a/CMakeLists.txt b/CMakeLists.txt index f1d5f3a..5db8e4b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -197,6 +197,7 @@ set(COMMON_SOURCES ErrorRecovery.cc RegexGuard.cc RegexEngine.cc + SearchHighlight.cc TerminalInputHandler.cc TerminalRenderer.cc TerminalFrontend.cc @@ -296,6 +297,7 @@ set(COMMON_HEADERS TestFrontend.h TextSpan.h RegexEngine.h + SearchHighlight.h UndoNode.h UndoTree.h UndoSystem.h @@ -413,6 +415,7 @@ if (BUILD_TESTS) ErrorRecovery.cc RegexGuard.cc RegexEngine.cc + SearchHighlight.cc KKeymap.cc SwapRecorder.h OptimizedSearch.cc diff --git a/Command.cc b/Command.cc index a6e5cd6..7f185a7 100644 --- a/Command.cc +++ b/Command.cc @@ -12,7 +12,6 @@ #include "Command.h" #include "ErrorHandler.h" -#include "RegexGuard.h" #include "RegexEngine.h" #include "TermWidth.h" #include "syntax/HighlighterRegistry.h" @@ -928,22 +927,6 @@ static constexpr std::size_t kSearchCountBytesPlain = std::size_t{64} << 20; static constexpr std::size_t kSearchCountBytesRegex = std::size_t{4} << 20; -using RegexMatcher = kte::Regex; // see RegexEngine.h - - -// Run regex work: std::regex recurses per matched character, so it runs on -// a large stack (RegexGuard.h); PCRE2 does not recurse on the C stack. -template -static void -run_regex_work(Fn &&fn) -{ - if (std::string_view(kte::Regex::EngineName()) == "PCRE2") - fn(); - else - kte::RunWithLargeStack(fn); -} - - static std::string_view line_text_view(const Buffer &buf, std::size_t y); @@ -1077,7 +1060,7 @@ private: // skipped (counted in *skipped); *limited counts lines where the engine hit // its match limit (catastrophic backtracking) and gave up. static bool -regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t x0, bool forward, +regex_find(const Buffer &buf, const kte::Regex &m, std::size_t y0, std::size_t x0, bool forward, std::size_t line_limit, std::size_t *skipped, std::size_t *limited, SearchHit &hit) { const std::size_t nrows = buf.Nrows(); @@ -1139,7 +1122,7 @@ regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t } } }; - run_regex_work(scan); + kte::Regex::RunGuarded(scan); return found; } @@ -1153,7 +1136,7 @@ struct SearchCount { static SearchCount -search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMatcher *m, const SearchHit &cur, +search_count(const Buffer &buf, bool regex, const std::string &q, const kte::Regex *m, const SearchHit &cur, std::size_t line_limit) { SearchCount c; @@ -1180,7 +1163,7 @@ search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMat } std::size_t scanned = 0; bool stopped = false; - run_regex_work([&] { + kte::Regex::RunGuarded([&] { LineWalker w(buf, 0); for (std::size_t y = 0; y < nrows && !stopped; ++y, w.Next()) { const std::string_view line = w.Line(); @@ -1248,7 +1231,7 @@ run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incrementa } } - RegexMatcher m; + kte::Regex m; std::string err; SearchHit hit; bool found = false; @@ -3655,7 +3638,7 @@ cmd_newline(CommandContext &ctx) const std::size_t all_rows = buf->Nrows(); std::string after; bool limited = false; - run_regex_work([&] { + kte::Regex::RunGuarded([&] { for (std::size_t y = 0; y < all_rows; ++y) { if (y > 0) after.push_back('\n'); diff --git a/ImGuiRenderer.cc b/ImGuiRenderer.cc index 1fa2bcf..e2b0e46 100644 --- a/ImGuiRenderer.cc +++ b/ImGuiRenderer.cc @@ -9,7 +9,7 @@ #include #include "ImGuiRenderer.h" -#include "RegexEngine.h" +#include "SearchHighlight.h" #include "Highlight.h" #include "GUITheme.h" #include "Buffer.h" @@ -157,18 +157,11 @@ ImGuiRenderer::Draw(Editor &ed) max_width_version_ = buf->Version(); } - // Hoist the search-regex compilation out of the per-line loop. Compiling - // std::regex per line per frame was a large source of idle CPU on macOS. - const bool search_mode = ed.SearchActive() && !ed.SearchQuery().empty(); - const bool regex_mode = search_mode && ed.PromptActive() && ( - ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || - ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind); - kte::Regex search_rx; // RegexEngine.h - bool search_rx_valid = false; - if (regex_mode) { - std::string rx_err; - search_rx_valid = search_rx.Compile(ed.SearchQuery(), rx_err); - } + // Search matches: the pattern is compiled once per frame (compiling it + // per line per frame was a large source of idle CPU on macOS). + const kte::SearchHighlight search_hl(ed); + const bool search_mode = search_hl.Active(); + std::vector > hl_src_ranges; // Compute the visible row range and skip rendering work for off-screen // lines. ImGui clips drawing, but string allocation, tab expansion, @@ -350,29 +343,8 @@ ImGuiRenderer::Draw(Editor &ed) expanded.c_str() + end).x; }; - // Compute search highlight ranges for this line in source indices - std::vector > hl_src_ranges; - if (search_mode) { - // In regex mode, reuse the compiled regex hoisted above the loop. - if (regex_mode) { - // Very long lines are not highlighted (see - // Regex::IncrementalLineLimit). - if (search_rx_valid && line.size() <= kte::Regex::IncrementalLineLimit()) { - std::size_t from = 0, pos = 0, len = 0; - while (from <= line.size() && search_rx.Search(line, from, pos, len)) { - hl_src_ranges.emplace_back(pos, pos + len); - from = pos + std::max(len, 1); - } - } - } else { - const std::string &q = ed.SearchQuery(); - std::size_t pos = 0; - while (!q.empty() && (pos = line.find(q, pos)) != std::string::npos) { - hl_src_ranges.emplace_back(pos, pos + q.size()); - pos += q.size(); - } - } - } + // Search highlight ranges for this line in source indices + search_hl.Ranges(line, hl_src_ranges); auto src_to_rx = [&](std::size_t upto_src_exclusive) -> std::size_t { std::size_t rx = 0; std::size_t s = 0; diff --git a/QtFrontend.cc b/QtFrontend.cc index 406cf66..968c852 100644 --- a/QtFrontend.cc +++ b/QtFrontend.cc @@ -1,5 +1,5 @@ #include "QtFrontend.h" -#include "RegexGuard.h" +#include "SearchHighlight.h" #include #include @@ -14,7 +14,6 @@ #include #include #include -#include #include "Editor.h" #include "Command.h" @@ -142,6 +141,10 @@ protected: p.save(); p.setClipRect(viewport); + // Search matches: the pattern is compiled once per frame. + const kte::SearchHighlight search_hl(*ed_); + std::vector > hl_src_ranges; + // Iterate visible lines for (std::size_t i = rowoffs, vis_idx = 0; i < last_row; ++i, ++vis_idx) { // Get line as string for regex/iterator usage and general string ops. @@ -159,34 +162,8 @@ protected: }; // Search-match background highlights first (under text) - if (ed_->SearchActive() && !ed_->SearchQuery().empty()) { - std::vector > hl_src_ranges; - // Compute ranges per line (source indices) - if (ed_->PromptActive() && - (ed_->CurrentPromptKind() == Editor::PromptKind::RegexSearch || - ed_->CurrentPromptKind() == - Editor::PromptKind::RegexReplaceFind)) { - // std::regex recursion is kept off the main stack for long - // lines (RegexGuard.h); very long lines are not highlighted. - if (line.size() <= kte::kRegexRenderLineLimit) try { - std::regex rx(ed_->SearchQuery()); - kte::ForEachRegexMatch(line, rx, [&](std::size_t pos, std::size_t len) { - hl_src_ranges.emplace_back(pos, pos + len); - }); - } catch (const std::regex_error &) { - // Invalid regex: ignore, status line already shows errors - } - } else { - const std::string &q = ed_->SearchQuery(); - if (!q.empty()) { - std::size_t pos = 0; - while ((pos = line.find(q, pos)) != std::string::npos) { - hl_src_ranges.emplace_back(pos, pos + q.size()); - pos += q.size(); - } - } - } - + if (search_hl.Active()) { + search_hl.Ranges(line, hl_src_ranges); if (!hl_src_ranges.empty()) { const bool has_current = ed_->SearchMatchLen() > 0 && ed_->SearchMatchY() == i; diff --git a/RegexEngine.cc b/RegexEngine.cc index 6b911b7..c37233c 100644 --- a/RegexEngine.cc +++ b/RegexEngine.cc @@ -2,6 +2,8 @@ #include +#include "RegexGuard.h" + #if defined(KTE_USE_PCRE2) #define PCRE2_CODE_UNIT_WIDTH 8 #include @@ -223,6 +225,13 @@ Regex::LimitHit() const } +void +Regex::RunGuarded(const std::function &fn) +{ + fn(); +} + + const char * Regex::EngineName() { @@ -311,6 +320,13 @@ Regex::LimitHit() const } +void +Regex::RunGuarded(const std::function &fn) +{ + RunWithLargeStack(fn); +} + + const char * Regex::EngineName() { diff --git a/RegexEngine.h b/RegexEngine.h index a40ab85..27bea07 100644 --- a/RegexEngine.h +++ b/RegexEngine.h @@ -7,7 +7,7 @@ * faster than std::regex, no recursion on the C stack, and a match limit * that stops catastrophic backtracking (e.g. "^(a+)+$") instead of hanging * the editor. Otherwise std::regex (ECMAScript) is used, run on a large - * stack (see RegexGuard.h). The syntax both accept covers ordinary use; + * stack (see RunGuarded). The syntax both accept covers ordinary use; * PCRE2 additionally has lookbehind, possessive quantifiers and the like. * * Replacement strings use ECMAScript syntax with either engine, as @@ -20,6 +20,7 @@ #pragma once #include +#include #include #include #include @@ -56,6 +57,13 @@ public: // Whether the last Search/ReplaceAll stopped at the engine's match limit. [[nodiscard]] bool LimitHit() const; + // Run fn, which does regex work that may span long text, where the + // engine cannot overflow the stack: std::regex recurses once per matched + // character, so fn runs to completion on a thread with a large stack + // (RegexGuard.h); PCRE2 does not recurse on the C stack, so fn runs + // inline. Exceptions thrown by fn are rethrown. + static void RunGuarded(const std::function &fn); + // "PCRE2" or "std::regex". static const char *EngineName(); diff --git a/RegexGuard.cc b/RegexGuard.cc index c9d61e7..ddbbdd6 100644 --- a/RegexGuard.cc +++ b/RegexGuard.cc @@ -49,21 +49,4 @@ RunWithLargeStack(const std::function &fn) std::rethrow_exception(job.error); } - -void -ForEachRegexMatch(const std::string &line, const std::regex &rx, - const std::function &on_match) -{ - auto scan = [&] { - for (auto it = std::sregex_iterator(line.begin(), line.end(), rx); it != std::sregex_iterator(); ++it) - on_match(static_cast(it->position()), static_cast(it->length())); - }; - // Stack use grows with match length; a few hundred bytes stays far below - // any thread's stack for every pattern we measured. - constexpr std::size_t kInlineLimit = 512; - if (line.size() <= kInlineLimit) - scan(); - else - RunWithLargeStack(scan); -} } // namespace kte diff --git a/RegexGuard.h b/RegexGuard.h index 46b9f01..9cae1ee 100644 --- a/RegexGuard.h +++ b/RegexGuard.h @@ -6,31 +6,13 @@ // stack and crashes the editor. #pragma once -#include #include -#include -#include namespace kte { -// Used where std::regex is still called directly (the Qt frontend's match -// highlighting; search and the other renderers use RegexEngine.h, whose -// Regex::IncrementalLineLimit() applies the same limits for std::regex). -// Regex matches are highlighted only on lines up to this many bytes: -// std::regex can take time quadratic in line length even for ordinary -// patterns (".*q" on a 20 KB line: 12 s), with no way to interrupt it, and -// these run on every frame or keystroke. -constexpr std::size_t kRegexRenderLineLimit = 2000; - // Run fn to completion on a thread with a large stack (address space is // reserved, pages are committed only as used), so regex work over long lines // cannot overflow. If such a thread cannot be created, fn runs on the calling // thread. Exceptions thrown by fn are rethrown to the caller. void RunWithLargeStack(const std::function &fn); -// Call on_match(position, length) for each match of rx in line. Lines short -// enough to be safe on any stack are matched inline; longer ones on a large -// stack (a 10,000-byte line already overflowed 8 MiB for some patterns). -// Throws what std::regex throws. -void ForEachRegexMatch(const std::string &line, const std::regex &rx, - const std::function &on_match); } // namespace kte diff --git a/SearchHighlight.cc b/SearchHighlight.cc new file mode 100644 index 0000000..6759e72 --- /dev/null +++ b/SearchHighlight.cc @@ -0,0 +1,45 @@ +#include "SearchHighlight.h" + +#include + +#include "Editor.h" + +namespace kte { +SearchHighlight::SearchHighlight(const Editor &ed) +{ + active_ = ed.SearchActive() && !ed.SearchQuery().empty(); + if (!active_) + return; + query_ = ed.SearchQuery(); + regex_ = ed.PromptActive() && (ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || + ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind); + if (regex_) { + std::string err; + rx_valid_ = rx_.Compile(query_, err); + } +} + + +void +SearchHighlight::Ranges(std::string_view line, std::vector > &out) const +{ + out.clear(); + if (!active_) + return; + if (regex_) { + if (!rx_valid_ || line.size() > Regex::IncrementalLineLimit()) + return; + std::size_t from = 0, pos = 0, len = 0; + while (from <= line.size() && rx_.Search(line, from, pos, len)) { + out.emplace_back(pos, pos + len); + from = pos + std::max(len, 1); + } + return; + } + std::size_t pos = 0; + while ((pos = line.find(query_, pos)) != std::string_view::npos) { + out.emplace_back(pos, pos + query_.size()); + pos += query_.size(); + } +} +} // namespace kte diff --git a/SearchHighlight.h b/SearchHighlight.h new file mode 100644 index 0000000..e50cd14 --- /dev/null +++ b/SearchHighlight.h @@ -0,0 +1,44 @@ +/* + * SearchHighlight.h - the search-match ranges every renderer highlights. + * + * Built once per frame from the editor's search state (the regex, when the + * regex search or regex replace prompt is up, is compiled here once rather + * than per line), then asked for each visible line's matches. + */ +#pragma once + +#include +#include +#include +#include +#include + +#include "RegexEngine.h" + +class Editor; + +namespace kte { +class SearchHighlight { +public: + explicit SearchHighlight(const Editor &ed); + + // Whether a search is up with a non-empty query. + [[nodiscard]] bool Active() const + { + return active_; + } + + // The matches in one line as [start, end) byte ranges, ordered and not + // overlapping. Nothing for an invalid pattern (the status line shows the + // error) or, in regex mode, a line longer than + // Regex::IncrementalLineLimit(). + void Ranges(std::string_view line, std::vector > &out) const; + +private: + bool active_ = false; + bool regex_ = false; + bool rx_valid_ = false; + std::string query_; + Regex rx_; +}; +} // namespace kte diff --git a/TerminalRenderer.cc b/TerminalRenderer.cc index e42acea..971254d 100644 --- a/TerminalRenderer.cc +++ b/TerminalRenderer.cc @@ -9,7 +9,7 @@ #include #include "TerminalRenderer.h" -#include "RegexEngine.h" +#include "SearchHighlight.h" #include "TermWidth.h" #include "Buffer.h" #include "Editor.h" @@ -57,49 +57,18 @@ TerminalRenderer::Draw(Editor &ed) buf->Highlighter()->PrefetchViewport(*buf, fr, rc, buf->Version()); } - // The search pattern, compiled once per frame (it was compiled for - // every row). - kte::Regex draw_rx; - bool draw_rx_ok = false; - if (ed.SearchActive() && !ed.SearchQuery().empty() && ed.PromptActive() && - (ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || - ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) { - std::string rx_err; - draw_rx_ok = draw_rx.Compile(ed.SearchQuery(), rx_err); - } + // Search matches: the pattern is compiled once per frame. + const kte::SearchHighlight search_hl(ed); + std::vector > ranges; // [start, end) for (int r = 0; r < content_rows; ++r) { move(r, 0); std::size_t li = rowoffs + static_cast(r); std::size_t render_col = 0; std::size_t src_i = 0; - // Compute matches for this line if search highlighting is active - bool search_mode = ed.SearchActive() && !ed.SearchQuery().empty(); - std::vector > ranges; // [start, end) - if (search_mode && li < nlines) { - std::string sline = buf->GetLineString(li); - // If regex search prompt is active (RegexSearch or RegexReplaceFind), use regex to compute highlight ranges - if (ed.PromptActive() && ( - ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || ed. - CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) { - // Very long lines are not highlighted (see - // Regex::IncrementalLineLimit); an invalid pattern - // highlights nothing (the status shows the error). - if (draw_rx_ok && sline.size() <= kte::Regex::IncrementalLineLimit()) { - std::size_t from = 0, pos = 0, len = 0; - while (from <= sline.size() && draw_rx.Search(sline, from, pos, len)) { - ranges.emplace_back(pos, pos + len); - from = pos + std::max(len, 1); - } - } - } else { - const std::string &q = ed.SearchQuery(); - std::size_t pos = 0; - while (!q.empty() && (pos = sline.find(q, pos)) != std::string::npos) { - ranges.emplace_back(pos, pos + q.size()); - pos += q.size(); - } - } - } + // Per-row copy: GetLineView() would materialize the whole buffer + // after every edit. + const std::string line = li < nlines ? buf->GetLineString(li) : std::string(); + search_hl.Ranges(line, ranges); auto is_src_in_hl = [&](std::size_t si) -> bool { if (ranges.empty()) return false; @@ -152,7 +121,6 @@ TerminalRenderer::Draw(Editor &ed) }; int written = 0; if (li < nlines) { - std::string line = buf->GetLineString(li); const bool vsel_on_line = vsel_active && li >= vsel_sy && li <= vsel_ey; const std::size_t vsel_spot_src = vsel_on_line ? std::min(buf->Curx(), line.size()) @@ -333,7 +301,7 @@ TerminalRenderer::Draw(Editor &ed) vsel_on_line && !vsel_spot_is_eol && src_i == vsel_spot_src; bool in_sel = in_mark || in_vsel; - bool in_hl = search_mode && is_src_in_hl(src_i); + bool in_hl = search_hl.Active() && is_src_in_hl(src_i); bool in_cur = has_current && li == cur_my && src_i >= cur_mx && @@ -390,7 +358,7 @@ TerminalRenderer::Draw(Editor &ed) } } bool in_sel = in_mark || in_vsel; - bool in_hl = search_mode && from_src && is_src_in_hl(src_i); + bool in_hl = search_hl.Active() && from_src && is_src_in_hl(src_i); bool in_cur = has_current && li == cur_my && from_src && src_i >= cur_mx && src_i < cur_mend; attr_t a = A_NORMAL; diff --git a/tests/test_audit_regressions.cc b/tests/test_audit_regressions.cc index c1a0864..38add2e 100644 --- a/tests/test_audit_regressions.cc +++ b/tests/test_audit_regressions.cc @@ -24,6 +24,7 @@ #include #include #include +#include #include #include #include @@ -888,17 +889,20 @@ TEST(Audit_Commands_FilesystemErrorsDoNotThrow) } -// Renderer regex highlighting on a long line with a deeply recursive pattern -// (overflowed an 8 MiB stack at under 10,000 bytes) runs off the main stack. -TEST(Audit_ForEachRegexMatch_LongLine) +// std::regex on a long line with a deeply recursive pattern (overflowed an +// 8 MiB stack at under 10,000 bytes) completes on the large stack. +TEST(Audit_RunWithLargeStack_LongLine) { std::string line; while (line.size() < 200000) line += "word, more words; and so on. "; std::size_t n = 0, total = 0; - kte::ForEachRegexMatch(line, std::regex("(\\w|\\s|[.,;])+"), [&](std::size_t, std::size_t len) { - ++n; - total += len; + const std::regex rx("(\\w|\\s|[.,;])+"); + kte::RunWithLargeStack([&] { + for (auto it = std::sregex_iterator(line.begin(), line.end(), rx); it != std::sregex_iterator(); ++it) { + ++n; + total += static_cast(it->length()); + } }); ASSERT_EQ(n, (std::size_t) 1); ASSERT_EQ(total, line.size());