kyle/kte
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Share search-match highlighting across renderers; engine owns its stack policy

Browse Source 
SearchHighlight compiles the search pattern once per frame and yields each
line's match ranges; the terminal, ImGui and Qt renderers had three copies
of this, and Qt compiled a std::regex for every visible line on every
frame. Qt now uses kte::Regex like the others.

kte::Regex::RunGuarded replaces Command.cc's run_regex_work, which chose
the large stack by comparing engine names. ForEachRegexMatch had no
callers left and is removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB
This commit is contained in:
Claude
2026-09-24 12:19:59 -07:00
committed by Kyle Isom
parent 2496722b43
commit 6bc3a6e0af
12 changed files with 158 additions and 173 deletions
Download Patch File Download Diff File Expand all files Collapse all files
CMakeLists.txt
+3
View File
@@ -197,6 +197,7 @@ set(COMMON_SOURCES
ErrorRecovery.cc
RegexGuard.cc
RegexEngine.cc
SearchHighlight.cc
TerminalInputHandler.cc
TerminalRenderer.cc
TerminalFrontend.cc
@@ -296,6 +297,7 @@ set(COMMON_HEADERS
TestFrontend.h
TextSpan.h
RegexEngine.h
SearchHighlight.h
UndoNode.h
UndoTree.h
UndoSystem.h
@@ -413,6 +415,7 @@ if (BUILD_TESTS)
ErrorRecovery.cc
RegexGuard.cc
RegexEngine.cc
SearchHighlight.cc
KKeymap.cc
SwapRecorder.h
OptimizedSearch.cc
Command.cc
+6 -23
View File
@@ -12,7 +12,6 @@
#include "Command.h"
#include "ErrorHandler.h"
#include "RegexGuard.h"
#include "RegexEngine.h"
#include "TermWidth.h"
#include "syntax/HighlighterRegistry.h"
@@ -928,22 +927,6 @@ static constexpr std::size_t kSearchCountBytesPlain = std::size_t{64} << 20;
static constexpr std::size_t kSearchCountBytesRegex = std::size_t{4} << 20;
using RegexMatcher = kte::Regex; // see RegexEngine.h
// Run regex work: std::regex recurses per matched character, so it runs on
// a large stack (RegexGuard.h); PCRE2 does not recurse on the C stack.
template<typename Fn>
static void
run_regex_work(Fn &&fn)
{
if (std::string_view(kte::Regex::EngineName()) == "PCRE2")
fn();
else
kte::RunWithLargeStack(fn);
}
static std::string_view line_text_view(const Buffer &buf, std::size_t y);
@@ -1077,7 +1060,7 @@ private:
// skipped (counted in *skipped); *limited counts lines where the engine hit
// its match limit (catastrophic backtracking) and gave up.
static bool
regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t x0, bool forward,
regex_find(const Buffer &buf, const kte::Regex &m, std::size_t y0, std::size_t x0, bool forward,
std::size_t line_limit, std::size_t *skipped, std::size_t *limited, SearchHit &hit)
{
const std::size_t nrows = buf.Nrows();
@@ -1139,7 +1122,7 @@ regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t
}
}
};
run_regex_work(scan);
kte::Regex::RunGuarded(scan);
return found;
}
@@ -1153,7 +1136,7 @@ struct SearchCount {
static SearchCount
search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMatcher *m, const SearchHit &cur,
search_count(const Buffer &buf, bool regex, const std::string &q, const kte::Regex *m, const SearchHit &cur,
std::size_t line_limit)
{
SearchCount c;
@@ -1180,7 +1163,7 @@ search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMat
}
std::size_t scanned = 0;
bool stopped = false;
run_regex_work([&] {
kte::Regex::RunGuarded([&] {
LineWalker w(buf, 0);
for (std::size_t y = 0; y < nrows && !stopped; ++y, w.Next()) {
const std::string_view line = w.Line();
@@ -1248,7 +1231,7 @@ run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incrementa
}
}
RegexMatcher m;
kte::Regex m;
std::string err;
SearchHit hit;
bool found = false;
@@ -3655,7 +3638,7 @@ cmd_newline(CommandContext &ctx)
const std::size_t all_rows = buf->Nrows();
std::string after;
bool limited = false;
run_regex_work([&] {
kte::Regex::RunGuarded([&] {
for (std::size_t y = 0; y < all_rows; ++y) {
if (y > 0)
after.push_back('\n');
ImGuiRenderer.cc
+8 -36
View File
@@ -9,7 +9,7 @@
#include <imgui.h>
#include "ImGuiRenderer.h"
#include "RegexEngine.h"
#include "SearchHighlight.h"
#include "Highlight.h"
#include "GUITheme.h"
#include "Buffer.h"
@@ -157,18 +157,11 @@ ImGuiRenderer::Draw(Editor &ed)
max_width_version_ = buf->Version();
}
// Hoist the search-regex compilation out of the per-line loop. Compiling
// std::regex per line per frame was a large source of idle CPU on macOS.
const bool search_mode = ed.SearchActive() && !ed.SearchQuery().empty();
const bool regex_mode = search_mode && ed.PromptActive() && (
ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind);
kte::Regex search_rx; // RegexEngine.h
bool search_rx_valid = false;
if (regex_mode) {
std::string rx_err;
search_rx_valid = search_rx.Compile(ed.SearchQuery(), rx_err);
}
// Search matches: the pattern is compiled once per frame (compiling it
// per line per frame was a large source of idle CPU on macOS).
const kte::SearchHighlight search_hl(ed);
const bool search_mode = search_hl.Active();
std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
// Compute the visible row range and skip rendering work for off-screen
// lines. ImGui clips drawing, but string allocation, tab expansion,
@@ -350,29 +343,8 @@ ImGuiRenderer::Draw(Editor &ed)
expanded.c_str() + end).x;
};
// Compute search highlight ranges for this line in source indices
std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
if (search_mode) {
// In regex mode, reuse the compiled regex hoisted above the loop.
if (regex_mode) {
// Very long lines are not highlighted (see
// Regex::IncrementalLineLimit).
if (search_rx_valid && line.size() <= kte::Regex::IncrementalLineLimit()) {
std::size_t from = 0, pos = 0, len = 0;
while (from <= line.size() && search_rx.Search(line, from, pos, len)) {
hl_src_ranges.emplace_back(pos, pos + len);
from = pos + std::max<std::size_t>(len, 1);
}
}
} else {
const std::string &q = ed.SearchQuery();
std::size_t pos = 0;
while (!q.empty() && (pos = line.find(q, pos)) != std::string::npos) {
hl_src_ranges.emplace_back(pos, pos + q.size());
pos += q.size();
}
}
}
// Search highlight ranges for this line in source indices
search_hl.Ranges(line, hl_src_ranges);
auto src_to_rx = [&](std::size_t upto_src_exclusive) -> std::size_t {
std::size_t rx = 0;
std::size_t s = 0;
QtFrontend.cc
+7 -30
View File
@@ -1,5 +1,5 @@
#include "QtFrontend.h"
#include "RegexGuard.h"
#include "SearchHighlight.h"
#include <QApplication>
#include <QWidget>
@@ -14,7 +14,6 @@
#include <QPainter>
#include <QPaintEvent>
#include <QWheelEvent>
#include <regex>
#include "Editor.h"
#include "Command.h"
@@ -142,6 +141,10 @@ protected:
p.save();
p.setClipRect(viewport);
// Search matches: the pattern is compiled once per frame.
const kte::SearchHighlight search_hl(*ed_);
std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
// Iterate visible lines
for (std::size_t i = rowoffs, vis_idx = 0; i < last_row; ++i, ++vis_idx) {
// Get line as string for regex/iterator usage and general string ops.
@@ -159,34 +162,8 @@ protected:
};
// Search-match background highlights first (under text)
if (ed_->SearchActive() && !ed_->SearchQuery().empty()) {
std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
// Compute ranges per line (source indices)
if (ed_->PromptActive() &&
(ed_->CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
ed_->CurrentPromptKind() ==
Editor::PromptKind::RegexReplaceFind)) {
// std::regex recursion is kept off the main stack for long
// lines (RegexGuard.h); very long lines are not highlighted.
if (line.size() <= kte::kRegexRenderLineLimit) try {
std::regex rx(ed_->SearchQuery());
kte::ForEachRegexMatch(line, rx, [&](std::size_t pos, std::size_t len) {
hl_src_ranges.emplace_back(pos, pos + len);
});
} catch (const std::regex_error &) {
// Invalid regex: ignore, status line already shows errors
}
} else {
const std::string &q = ed_->SearchQuery();
if (!q.empty()) {
std::size_t pos = 0;
while ((pos = line.find(q, pos)) != std::string::npos) {
hl_src_ranges.emplace_back(pos, pos + q.size());
pos += q.size();
}
}
}
if (search_hl.Active()) {
search_hl.Ranges(line, hl_src_ranges);
if (!hl_src_ranges.empty()) {
const bool has_current =
ed_->SearchMatchLen() > 0 && ed_->SearchMatchY() == i;
RegexEngine.cc
+16
View File
@@ -2,6 +2,8 @@
#include <regex>
#include "RegexGuard.h"
#if defined(KTE_USE_PCRE2)
#define PCRE2_CODE_UNIT_WIDTH 8
#include <pcre2.h>
@@ -223,6 +225,13 @@ Regex::LimitHit() const
}
void
Regex::RunGuarded(const std::function<void()> &fn)
{
fn();
}
const char *
Regex::EngineName()
{
@@ -311,6 +320,13 @@ Regex::LimitHit() const
}
void
Regex::RunGuarded(const std::function<void()> &fn)
{
RunWithLargeStack(fn);
}
const char *
Regex::EngineName()
{
RegexEngine.h
+9 -1
View File
@@ -7,7 +7,7 @@
* faster than std::regex, no recursion on the C stack, and a match limit
* that stops catastrophic backtracking (e.g. "^(a+)+$") instead of hanging
* the editor. Otherwise std::regex (ECMAScript) is used, run on a large
* stack (see RegexGuard.h). The syntax both accept covers ordinary use;
* stack (see RunGuarded). The syntax both accept covers ordinary use;
* PCRE2 additionally has lookbehind, possessive quantifiers and the like.
*
* Replacement strings use ECMAScript syntax with either engine, as
@@ -20,6 +20,7 @@
#pragma once
#include <cstddef>
#include <functional>
#include <memory>
#include <string>
#include <string_view>
@@ -56,6 +57,13 @@ public:
// Whether the last Search/ReplaceAll stopped at the engine's match limit.
[[nodiscard]] bool LimitHit() const;
// Run fn, which does regex work that may span long text, where the
// engine cannot overflow the stack: std::regex recurses once per matched
// character, so fn runs to completion on a thread with a large stack
// (RegexGuard.h); PCRE2 does not recurse on the C stack, so fn runs
// inline. Exceptions thrown by fn are rethrown.
static void RunGuarded(const std::function<void()> &fn);
// "PCRE2" or "std::regex".
static const char *EngineName();
RegexGuard.cc
-17
View File
@@ -49,21 +49,4 @@ RunWithLargeStack(const std::function<void()> &fn)
std::rethrow_exception(job.error);
}
void
ForEachRegexMatch(const std::string &line, const std::regex &rx,
const std::function<void(std::size_t, std::size_t)> &on_match)
{
auto scan = [&] {
for (auto it = std::sregex_iterator(line.begin(), line.end(), rx); it != std::sregex_iterator(); ++it)
on_match(static_cast<std::size_t>(it->position()), static_cast<std::size_t>(it->length()));
};
// Stack use grows with match length; a few hundred bytes stays far below
// any thread's stack for every pattern we measured.
constexpr std::size_t kInlineLimit = 512;
if (line.size() <= kInlineLimit)
scan();
else
RunWithLargeStack(scan);
}
} // namespace kte
RegexGuard.h
-18
View File
@@ -6,31 +6,13 @@
// stack and crashes the editor.
#pragma once
#include <cstddef>
#include <functional>
#include <regex>
#include <string>
namespace kte {
// Used where std::regex is still called directly (the Qt frontend's match
// highlighting; search and the other renderers use RegexEngine.h, whose
// Regex::IncrementalLineLimit() applies the same limits for std::regex).
// Regex matches are highlighted only on lines up to this many bytes:
// std::regex can take time quadratic in line length even for ordinary
// patterns (".*q" on a 20 KB line: 12 s), with no way to interrupt it, and
// these run on every frame or keystroke.
constexpr std::size_t kRegexRenderLineLimit = 2000;
// Run fn to completion on a thread with a large stack (address space is
// reserved, pages are committed only as used), so regex work over long lines
// cannot overflow. If such a thread cannot be created, fn runs on the calling
// thread. Exceptions thrown by fn are rethrown to the caller.
void RunWithLargeStack(const std::function<void()> &fn);
// Call on_match(position, length) for each match of rx in line. Lines short
// enough to be safe on any stack are matched inline; longer ones on a large
// stack (a 10,000-byte line already overflowed 8 MiB for some patterns).
// Throws what std::regex throws.
void ForEachRegexMatch(const std::string &line, const std::regex &rx,
const std::function<void(std::size_t, std::size_t)> &on_match);
} // namespace kte
SearchHighlight.cc
+45
View File
@@ -0,0 +1,45 @@
#include "SearchHighlight.h"
#include <algorithm>
#include "Editor.h"
namespace kte {
SearchHighlight::SearchHighlight(const Editor &ed)
{
active_ = ed.SearchActive() && !ed.SearchQuery().empty();
if (!active_)
return;
query_ = ed.SearchQuery();
regex_ = ed.PromptActive() && (ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind);
if (regex_) {
std::string err;
rx_valid_ = rx_.Compile(query_, err);
}
}
void
SearchHighlight::Ranges(std::string_view line, std::vector<std::pair<std::size_t, std::size_t> > &out) const
{
out.clear();
if (!active_)
return;
if (regex_) {
if (!rx_valid_ || line.size() > Regex::IncrementalLineLimit())
return;
std::size_t from = 0, pos = 0, len = 0;
while (from <= line.size() && rx_.Search(line, from, pos, len)) {
out.emplace_back(pos, pos + len);
from = pos + std::max<std::size_t>(len, 1);
}
return;
}
std::size_t pos = 0;
while ((pos = line.find(query_, pos)) != std::string_view::npos) {
out.emplace_back(pos, pos + query_.size());
pos += query_.size();
}
}
} // namespace kte
SearchHighlight.h
+44
View File
@@ -0,0 +1,44 @@
/*
* SearchHighlight.h - the search-match ranges every renderer highlights.
*
* Built once per frame from the editor's search state (the regex, when the
* regex search or regex replace prompt is up, is compiled here once rather
* than per line), then asked for each visible line's matches.
*/
#pragma once
#include <cstddef>
#include <string>
#include <string_view>
#include <utility>
#include <vector>
#include "RegexEngine.h"
class Editor;
namespace kte {
class SearchHighlight {
public:
explicit SearchHighlight(const Editor &ed);
// Whether a search is up with a non-empty query.
[[nodiscard]] bool Active() const
{
return active_;
}
// The matches in one line as [start, end) byte ranges, ordered and not
// overlapping. Nothing for an invalid pattern (the status line shows the
// error) or, in regex mode, a line longer than
// Regex::IncrementalLineLimit().
void Ranges(std::string_view line, std::vector<std::pair<std::size_t, std::size_t> > &out) const;
private:
bool active_ = false;
bool regex_ = false;
bool rx_valid_ = false;
std::string query_;
Regex rx_;
};
} // namespace kte
TerminalRenderer.cc
+10 -42
View File
@@ -9,7 +9,7 @@
#include <string>
#include "TerminalRenderer.h"
#include "RegexEngine.h"
#include "SearchHighlight.h"
#include "TermWidth.h"
#include "Buffer.h"
#include "Editor.h"
@@ -57,49 +57,18 @@ TerminalRenderer::Draw(Editor &ed)
buf->Highlighter()->PrefetchViewport(*buf, fr, rc, buf->Version());
}
// The search pattern, compiled once per frame (it was compiled for
// every row).
kte::Regex draw_rx;
bool draw_rx_ok = false;
if (ed.SearchActive() && !ed.SearchQuery().empty() && ed.PromptActive() &&
(ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) {
std::string rx_err;
draw_rx_ok = draw_rx.Compile(ed.SearchQuery(), rx_err);
}
// Search matches: the pattern is compiled once per frame.
const kte::SearchHighlight search_hl(ed);
std::vector<std::pair<std::size_t, std::size_t> > ranges; // [start, end)
for (int r = 0; r < content_rows; ++r) {
move(r, 0);
std::size_t li = rowoffs + static_cast<std::size_t>(r);
std::size_t render_col = 0;
std::size_t src_i = 0;
// Compute matches for this line if search highlighting is active
bool search_mode = ed.SearchActive() && !ed.SearchQuery().empty();
std::vector<std::pair<std::size_t, std::size_t> > ranges; // [start, end)
if (search_mode && li < nlines) {
std::string sline = buf->GetLineString(li);
// If regex search prompt is active (RegexSearch or RegexReplaceFind), use regex to compute highlight ranges
if (ed.PromptActive() && (
ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || ed.
CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) {
// Very long lines are not highlighted (see
// Regex::IncrementalLineLimit); an invalid pattern
// highlights nothing (the status shows the error).
if (draw_rx_ok && sline.size() <= kte::Regex::IncrementalLineLimit()) {
std::size_t from = 0, pos = 0, len = 0;
while (from <= sline.size() && draw_rx.Search(sline, from, pos, len)) {
ranges.emplace_back(pos, pos + len);
from = pos + std::max<std::size_t>(len, 1);
}
}
} else {
const std::string &q = ed.SearchQuery();
std::size_t pos = 0;
while (!q.empty() && (pos = sline.find(q, pos)) != std::string::npos) {
ranges.emplace_back(pos, pos + q.size());
pos += q.size();
}
}
}
// Per-row copy: GetLineView() would materialize the whole buffer
// after every edit.
const std::string line = li < nlines ? buf->GetLineString(li) : std::string();
search_hl.Ranges(line, ranges);
auto is_src_in_hl = [&](std::size_t si) -> bool {
if (ranges.empty())
return false;
@@ -152,7 +121,6 @@ TerminalRenderer::Draw(Editor &ed)
};
int written = 0;
if (li < nlines) {
std::string line = buf->GetLineString(li);
const bool vsel_on_line = vsel_active && li >= vsel_sy && li <= vsel_ey;
const std::size_t vsel_spot_src = vsel_on_line
? std::min(buf->Curx(), line.size())
@@ -333,7 +301,7 @@ TerminalRenderer::Draw(Editor &ed)
vsel_on_line && !vsel_spot_is_eol && src_i ==
vsel_spot_src;
bool in_sel = in_mark || in_vsel;
bool in_hl = search_mode && is_src_in_hl(src_i);
bool in_hl = search_hl.Active() && is_src_in_hl(src_i);
bool in_cur =
has_current && li == cur_my && src_i >= cur_mx
&&
@@ -390,7 +358,7 @@ TerminalRenderer::Draw(Editor &ed)
}
}
bool in_sel = in_mark || in_vsel;
bool in_hl = search_mode && from_src && is_src_in_hl(src_i);
bool in_hl = search_hl.Active() && from_src && is_src_in_hl(src_i);
bool in_cur = has_current && li == cur_my && from_src && src_i >= cur_mx &&
src_i < cur_mend;
attr_t a = A_NORMAL;
tests/test_audit_regressions.cc
+9 -5
View File
@@ -24,6 +24,7 @@
#include <sys/stat.h>
#include <unistd.h>
#include <random>
#include <regex>
#include <set>
#include <vector>
#include <clocale>
@@ -888,17 +889,20 @@ TEST(Audit_Commands_FilesystemErrorsDoNotThrow)
}
// Renderer regex highlighting on a long line with a deeply recursive pattern
// (overflowed an 8 MiB stack at under 10,000 bytes) runs off the main stack.
TEST(Audit_ForEachRegexMatch_LongLine)
// std::regex on a long line with a deeply recursive pattern (overflowed an
// 8 MiB stack at under 10,000 bytes) completes on the large stack.
TEST(Audit_RunWithLargeStack_LongLine)
{
std::string line;
while (line.size() < 200000)
line += "word, more words; and so on. ";
std::size_t n = 0, total = 0;
kte::ForEachRegexMatch(line, std::regex("(\\w|\\s|[.,;])+"), [&](std::size_t, std::size_t len) {
const std::regex rx("(\\w|\\s|[.,;])+");
kte::RunWithLargeStack([&] {
for (auto it = std::sregex_iterator(line.begin(), line.end(), rx); it != std::sregex_iterator(); ++it) {
++n;
total += len;
total += static_cast<std::size_t>(it->length());
}
});
ASSERT_EQ(n, (std::size_t) 1);
ASSERT_EQ(total, line.size());