Share search-match highlighting across renderers; engine owns its stack policy

SearchHighlight compiles the search pattern once per frame and yields each
line's match ranges; the terminal, ImGui and Qt renderers had three copies
of this, and Qt compiled a std::regex for every visible line on every
frame. Qt now uses kte::Regex like the others.

kte::Regex::RunGuarded replaces Command.cc's run_regex_work, which chose
the large stack by comparing engine names. ForEachRegexMatch had no
callers left and is removed.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB

CMakeLists.txt

@@ -197,6 +197,7 @@ set(COMMON_SOURCES
         ErrorRecovery.cc
         RegexGuard.cc
         RegexEngine.cc
+        SearchHighlight.cc
         TerminalInputHandler.cc
         TerminalRenderer.cc
         TerminalFrontend.cc
@@ -296,6 +297,7 @@ set(COMMON_HEADERS
         TestFrontend.h
         TextSpan.h
         RegexEngine.h
+        SearchHighlight.h
         UndoNode.h
         UndoTree.h
         UndoSystem.h
@@ -413,6 +415,7 @@ if (BUILD_TESTS)
             ErrorRecovery.cc
             RegexGuard.cc
             RegexEngine.cc
+            SearchHighlight.cc
             KKeymap.cc
             SwapRecorder.h
             OptimizedSearch.cc

Command.cc

@@ -12,7 +12,6 @@
 
 #include "Command.h"
 #include "ErrorHandler.h"
-#include "RegexGuard.h"
 #include "RegexEngine.h"
 #include "TermWidth.h"
 #include "syntax/HighlighterRegistry.h"
@@ -928,22 +927,6 @@ static constexpr std::size_t kSearchCountBytesPlain = std::size_t{64} << 20;
 static constexpr std::size_t kSearchCountBytesRegex = std::size_t{4} << 20;
 
 
-using RegexMatcher = kte::Regex; // see RegexEngine.h
-
-
-// Run regex work: std::regex recurses per matched character, so it runs on
-// a large stack (RegexGuard.h); PCRE2 does not recurse on the C stack.
-template<typename Fn>
-static void
-run_regex_work(Fn &&fn)
-{
-	if (std::string_view(kte::Regex::EngineName()) == "PCRE2")
-		fn();
-	else
-		kte::RunWithLargeStack(fn);
-}
-
-
 static std::string_view line_text_view(const Buffer &buf, std::size_t y);
 
 
@@ -1077,7 +1060,7 @@ private:
 // skipped (counted in *skipped); *limited counts lines where the engine hit
 // its match limit (catastrophic backtracking) and gave up.
 static bool
-regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t x0, bool forward,
+regex_find(const Buffer &buf, const kte::Regex &m, std::size_t y0, std::size_t x0, bool forward,
            std::size_t line_limit, std::size_t *skipped, std::size_t *limited, SearchHit &hit)
 {
 	const std::size_t nrows = buf.Nrows();
@@ -1139,7 +1122,7 @@ regex_find(const Buffer &buf, const RegexMatcher &m, std::size_t y0, std::size_t
 			}
 		}
 	};
-	run_regex_work(scan);
+	kte::Regex::RunGuarded(scan);
 	return found;
 }
 
@@ -1153,7 +1136,7 @@ struct SearchCount {
 
 
 static SearchCount
-search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMatcher *m, const SearchHit &cur,
+search_count(const Buffer &buf, bool regex, const std::string &q, const kte::Regex *m, const SearchHit &cur,
              std::size_t line_limit)
 {
 	SearchCount c;
@@ -1180,7 +1163,7 @@ search_count(const Buffer &buf, bool regex, const std::string &q, const RegexMat
 	}
 	std::size_t scanned = 0;
 	bool stopped        = false;
-	run_regex_work([&] {
+	kte::Regex::RunGuarded([&] {
 		LineWalker w(buf, 0);
 		for (std::size_t y = 0; y < nrows && !stopped; ++y, w.Next()) {
 			const std::string_view line = w.Line();
@@ -1248,7 +1231,7 @@ run_search(Editor &ed, Buffer &buf, bool regex, SearchStep step, bool incrementa
 		}
 	}
 
-	RegexMatcher m;
+	kte::Regex m;
 	std::string err;
 	SearchHit hit;
 	bool found          = false;
@@ -3655,7 +3638,7 @@ cmd_newline(CommandContext &ctx)
 			const std::size_t all_rows = buf->Nrows();
 			std::string after;
 			bool limited = false;
-			run_regex_work([&] {
+			kte::Regex::RunGuarded([&] {
 				for (std::size_t y = 0; y < all_rows; ++y) {
 					if (y > 0)
 						after.push_back('\n');

ImGuiRenderer.cc

@@ -9,7 +9,7 @@
 #include <imgui.h>
 
 #include "ImGuiRenderer.h"
-#include "RegexEngine.h"
+#include "SearchHighlight.h"
 #include "Highlight.h"
 #include "GUITheme.h"
 #include "Buffer.h"
@@ -157,18 +157,11 @@ ImGuiRenderer::Draw(Editor &ed)
 			max_width_version_ = buf->Version();
 		}
 
-		// Hoist the search-regex compilation out of the per-line loop. Compiling
+		// Search matches: the pattern is compiled once per frame (compiling it
-		// std::regex per line per frame was a large source of idle CPU on macOS.
+		// per line per frame was a large source of idle CPU on macOS).
-		const bool search_mode = ed.SearchActive() && !ed.SearchQuery().empty();
+		const kte::SearchHighlight search_hl(ed);
-		const bool regex_mode  = search_mode && ed.PromptActive() && (
+		const bool search_mode = search_hl.Active();
-			ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
+		std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
-			ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind);
-		kte::Regex search_rx; // RegexEngine.h
-		bool search_rx_valid = false;
-		if (regex_mode) {
-			std::string rx_err;
-			search_rx_valid = search_rx.Compile(ed.SearchQuery(), rx_err);
-		}
 
 		// Compute the visible row range and skip rendering work for off-screen
 		// lines. ImGui clips drawing, but string allocation, tab expansion,
@@ -350,29 +343,8 @@ ImGuiRenderer::Draw(Editor &ed)
 				                           expanded.c_str() + end).x;
 			};
 
-			// Compute search highlight ranges for this line in source indices
+			// Search highlight ranges for this line in source indices
-			std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
+			search_hl.Ranges(line, hl_src_ranges);
-			if (search_mode) {
-				// In regex mode, reuse the compiled regex hoisted above the loop.
-				if (regex_mode) {
-					// Very long lines are not highlighted (see
-					// Regex::IncrementalLineLimit).
-					if (search_rx_valid && line.size() <= kte::Regex::IncrementalLineLimit()) {
-						std::size_t from = 0, pos = 0, len = 0;
-						while (from <= line.size() && search_rx.Search(line, from, pos, len)) {
-							hl_src_ranges.emplace_back(pos, pos + len);
-							from = pos + std::max<std::size_t>(len, 1);
-						}
-					}
-				} else {
-					const std::string &q = ed.SearchQuery();
-					std::size_t pos      = 0;
-					while (!q.empty() && (pos = line.find(q, pos)) != std::string::npos) {
-						hl_src_ranges.emplace_back(pos, pos + q.size());
-						pos += q.size();
-					}
-				}
-			}
 			auto src_to_rx = [&](std::size_t upto_src_exclusive) -> std::size_t {
 				std::size_t rx = 0;
 				std::size_t s  = 0;

QtFrontend.cc

@@ -1,5 +1,5 @@
 #include "QtFrontend.h"
-#include "RegexGuard.h"
+#include "SearchHighlight.h"
 
 #include <QApplication>
 #include <QWidget>
@@ -14,7 +14,6 @@
 #include <QPainter>
 #include <QPaintEvent>
 #include <QWheelEvent>
-#include <regex>
 
 #include "Editor.h"
 #include "Command.h"
@@ -142,6 +141,10 @@ protected:
 				p.save();
 				p.setClipRect(viewport);
 
+				// Search matches: the pattern is compiled once per frame.
+				const kte::SearchHighlight search_hl(*ed_);
+				std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
+
 				// Iterate visible lines
 				for (std::size_t i = rowoffs, vis_idx = 0; i < last_row; ++i, ++vis_idx) {
 					// Get line as string for regex/iterator usage and general string ops.
@@ -159,34 +162,8 @@ protected:
 					};
 
 					// Search-match background highlights first (under text)
-					if (ed_->SearchActive() && !ed_->SearchQuery().empty()) {
+					if (search_hl.Active()) {
-						std::vector<std::pair<std::size_t, std::size_t> > hl_src_ranges;
+						search_hl.Ranges(line, hl_src_ranges);
-						// Compute ranges per line (source indices)
-						if (ed_->PromptActive() &&
-						    (ed_->CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
-						     ed_->CurrentPromptKind() ==
-						     Editor::PromptKind::RegexReplaceFind)) {
-							// std::regex recursion is kept off the main stack for long
-							// lines (RegexGuard.h); very long lines are not highlighted.
-							if (line.size() <= kte::kRegexRenderLineLimit) try {
-								std::regex rx(ed_->SearchQuery());
-								kte::ForEachRegexMatch(line, rx, [&](std::size_t pos, std::size_t len) {
-									hl_src_ranges.emplace_back(pos, pos + len);
-								});
-							} catch (const std::regex_error &) {
-								// Invalid regex: ignore, status line already shows errors
-							}
-						} else {
-							const std::string &q = ed_->SearchQuery();
-							if (!q.empty()) {
-								std::size_t pos = 0;
-								while ((pos = line.find(q, pos)) != std::string::npos) {
-									hl_src_ranges.emplace_back(pos, pos + q.size());
-									pos += q.size();
-								}
-							}
-						}
-
 						if (!hl_src_ranges.empty()) {
 							const bool has_current =
 								ed_->SearchMatchLen() > 0 && ed_->SearchMatchY() == i;

RegexEngine.cc

@@ -2,6 +2,8 @@
 
 #include <regex>
 
+#include "RegexGuard.h"
+
 #if defined(KTE_USE_PCRE2)
 #define PCRE2_CODE_UNIT_WIDTH 8
 #include <pcre2.h>
@@ -223,6 +225,13 @@ Regex::LimitHit() const
 }
 
 
+void
+Regex::RunGuarded(const std::function<void()> &fn)
+{
+	fn();
+}
+
+
 const char *
 Regex::EngineName()
 {
@@ -311,6 +320,13 @@ Regex::LimitHit() const
 }
 
 
+void
+Regex::RunGuarded(const std::function<void()> &fn)
+{
+	RunWithLargeStack(fn);
+}
+
+
 const char *
 Regex::EngineName()
 {

RegexEngine.h

@@ -7,7 +7,7 @@
  * faster than std::regex, no recursion on the C stack, and a match limit
  * that stops catastrophic backtracking (e.g. "^(a+)+$") instead of hanging
  * the editor. Otherwise std::regex (ECMAScript) is used, run on a large
- * stack (see RegexGuard.h). The syntax both accept covers ordinary use;
+ * stack (see RunGuarded). The syntax both accept covers ordinary use;
  * PCRE2 additionally has lookbehind, possessive quantifiers and the like.
  *
  * Replacement strings use ECMAScript syntax with either engine, as
@@ -20,6 +20,7 @@
 #pragma once
 
 #include <cstddef>
+#include <functional>
 #include <memory>
 #include <string>
 #include <string_view>
@@ -56,6 +57,13 @@ public:
 	// Whether the last Search/ReplaceAll stopped at the engine's match limit.
 	[[nodiscard]] bool LimitHit() const;
 
+	// Run fn, which does regex work that may span long text, where the
+	// engine cannot overflow the stack: std::regex recurses once per matched
+	// character, so fn runs to completion on a thread with a large stack
+	// (RegexGuard.h); PCRE2 does not recurse on the C stack, so fn runs
+	// inline. Exceptions thrown by fn are rethrown.
+	static void RunGuarded(const std::function<void()> &fn);
+
 	// "PCRE2" or "std::regex".
 	static const char *EngineName();
 

RegexGuard.cc

@@ -49,21 +49,4 @@ RunWithLargeStack(const std::function<void()> &fn)
 		std::rethrow_exception(job.error);
 }
 
-
-void
-ForEachRegexMatch(const std::string &line, const std::regex &rx,
-                  const std::function<void(std::size_t, std::size_t)> &on_match)
-{
-	auto scan = [&] {
-		for (auto it = std::sregex_iterator(line.begin(), line.end(), rx); it != std::sregex_iterator(); ++it)
-			on_match(static_cast<std::size_t>(it->position()), static_cast<std::size_t>(it->length()));
-	};
-	// Stack use grows with match length; a few hundred bytes stays far below
-	// any thread's stack for every pattern we measured.
-	constexpr std::size_t kInlineLimit = 512;
-	if (line.size() <= kInlineLimit)
-		scan();
-	else
-		RunWithLargeStack(scan);
-}
 } // namespace kte

RegexGuard.h

@@ -6,31 +6,13 @@
 // stack and crashes the editor.
 #pragma once
 
-#include <cstddef>
 #include <functional>
-#include <regex>
-#include <string>
 
 namespace kte {
-// Used where std::regex is still called directly (the Qt frontend's match
-// highlighting; search and the other renderers use RegexEngine.h, whose
-// Regex::IncrementalLineLimit() applies the same limits for std::regex).
-// Regex matches are highlighted only on lines up to this many bytes:
-// std::regex can take time quadratic in line length even for ordinary
-// patterns (".*q" on a 20 KB line: 12 s), with no way to interrupt it, and
-// these run on every frame or keystroke.
-constexpr std::size_t kRegexRenderLineLimit = 2000;
-
 // Run fn to completion on a thread with a large stack (address space is
 // reserved, pages are committed only as used), so regex work over long lines
 // cannot overflow. If such a thread cannot be created, fn runs on the calling
 // thread. Exceptions thrown by fn are rethrown to the caller.
 void RunWithLargeStack(const std::function<void()> &fn);
 
-// Call on_match(position, length) for each match of rx in line. Lines short
-// enough to be safe on any stack are matched inline; longer ones on a large
-// stack (a 10,000-byte line already overflowed 8 MiB for some patterns).
-// Throws what std::regex throws.
-void ForEachRegexMatch(const std::string &line, const std::regex &rx,
-                       const std::function<void(std::size_t, std::size_t)> &on_match);
 } // namespace kte

SearchHighlight.cc

@@ -0,0 +1,45 @@
+#include "SearchHighlight.h"
+
+#include <algorithm>
+
+#include "Editor.h"
+
+namespace kte {
+SearchHighlight::SearchHighlight(const Editor &ed)
+{
+	active_ = ed.SearchActive() && !ed.SearchQuery().empty();
+	if (!active_)
+		return;
+	query_ = ed.SearchQuery();
+	regex_ = ed.PromptActive() && (ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
+	                               ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind);
+	if (regex_) {
+		std::string err;
+		rx_valid_ = rx_.Compile(query_, err);
+	}
+}
+
+
+void
+SearchHighlight::Ranges(std::string_view line, std::vector<std::pair<std::size_t, std::size_t> > &out) const
+{
+	out.clear();
+	if (!active_)
+		return;
+	if (regex_) {
+		if (!rx_valid_ || line.size() > Regex::IncrementalLineLimit())
+			return;
+		std::size_t from = 0, pos = 0, len = 0;
+		while (from <= line.size() && rx_.Search(line, from, pos, len)) {
+			out.emplace_back(pos, pos + len);
+			from = pos + std::max<std::size_t>(len, 1);
+		}
+		return;
+	}
+	std::size_t pos = 0;
+	while ((pos = line.find(query_, pos)) != std::string_view::npos) {
+		out.emplace_back(pos, pos + query_.size());
+		pos += query_.size();
+	}
+}
+} // namespace kte

SearchHighlight.h

@@ -0,0 +1,44 @@
+/*
+ * SearchHighlight.h - the search-match ranges every renderer highlights.
+ *
+ * Built once per frame from the editor's search state (the regex, when the
+ * regex search or regex replace prompt is up, is compiled here once rather
+ * than per line), then asked for each visible line's matches.
+ */
+#pragma once
+
+#include <cstddef>
+#include <string>
+#include <string_view>
+#include <utility>
+#include <vector>
+
+#include "RegexEngine.h"
+
+class Editor;
+
+namespace kte {
+class SearchHighlight {
+public:
+	explicit SearchHighlight(const Editor &ed);
+
+	// Whether a search is up with a non-empty query.
+	[[nodiscard]] bool Active() const
+	{
+		return active_;
+	}
+
+	// The matches in one line as [start, end) byte ranges, ordered and not
+	// overlapping. Nothing for an invalid pattern (the status line shows the
+	// error) or, in regex mode, a line longer than
+	// Regex::IncrementalLineLimit().
+	void Ranges(std::string_view line, std::vector<std::pair<std::size_t, std::size_t> > &out) const;
+
+private:
+	bool active_   = false;
+	bool regex_    = false;
+	bool rx_valid_ = false;
+	std::string query_;
+	Regex rx_;
+};
+} // namespace kte

TerminalRenderer.cc

@@ -9,7 +9,7 @@
 #include <string>
 
 #include "TerminalRenderer.h"
-#include "RegexEngine.h"
+#include "SearchHighlight.h"
 #include "TermWidth.h"
 #include "Buffer.h"
 #include "Editor.h"
@@ -57,49 +57,18 @@ TerminalRenderer::Draw(Editor &ed)
 			buf->Highlighter()->PrefetchViewport(*buf, fr, rc, buf->Version());
 		}
 
-		// The search pattern, compiled once per frame (it was compiled for
+		// Search matches: the pattern is compiled once per frame.
-		// every row).
+		const kte::SearchHighlight search_hl(ed);
-		kte::Regex draw_rx;
+		std::vector<std::pair<std::size_t, std::size_t> > ranges; // [start, end)
-		bool draw_rx_ok = false;
-		if (ed.SearchActive() && !ed.SearchQuery().empty() && ed.PromptActive() &&
-		    (ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch ||
-		     ed.CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) {
-			std::string rx_err;
-			draw_rx_ok = draw_rx.Compile(ed.SearchQuery(), rx_err);
-		}
 		for (int r = 0; r < content_rows; ++r) {
 			move(r, 0);
 			std::size_t li         = rowoffs + static_cast<std::size_t>(r);
 			std::size_t render_col = 0;
 			std::size_t src_i      = 0;
-			// Compute matches for this line if search highlighting is active
+			// Per-row copy: GetLineView() would materialize the whole buffer
-			bool search_mode = ed.SearchActive() && !ed.SearchQuery().empty();
+			// after every edit.
-			std::vector<std::pair<std::size_t, std::size_t> > ranges; // [start, end)
+			const std::string line = li < nlines ? buf->GetLineString(li) : std::string();
-			if (search_mode && li < nlines) {
+			search_hl.Ranges(line, ranges);
-				std::string sline = buf->GetLineString(li);
-				// If regex search prompt is active (RegexSearch or RegexReplaceFind), use regex to compute highlight ranges
-				if (ed.PromptActive() && (
-					    ed.CurrentPromptKind() == Editor::PromptKind::RegexSearch || ed.
-					    CurrentPromptKind() == Editor::PromptKind::RegexReplaceFind)) {
-					// Very long lines are not highlighted (see
-					// Regex::IncrementalLineLimit); an invalid pattern
-					// highlights nothing (the status shows the error).
-					if (draw_rx_ok && sline.size() <= kte::Regex::IncrementalLineLimit()) {
-						std::size_t from = 0, pos = 0, len = 0;
-						while (from <= sline.size() && draw_rx.Search(sline, from, pos, len)) {
-							ranges.emplace_back(pos, pos + len);
-							from = pos + std::max<std::size_t>(len, 1);
-						}
-					}
-				} else {
-					const std::string &q = ed.SearchQuery();
-					std::size_t pos      = 0;
-					while (!q.empty() && (pos = sline.find(q, pos)) != std::string::npos) {
-						ranges.emplace_back(pos, pos + q.size());
-						pos += q.size();
-					}
-				}
-			}
 			auto is_src_in_hl = [&](std::size_t si) -> bool {
 				if (ranges.empty())
 					return false;
@@ -152,7 +121,6 @@ TerminalRenderer::Draw(Editor &ed)
 			};
 			int written = 0;
 			if (li < nlines) {
-				std::string line                = buf->GetLineString(li);
 				const bool vsel_on_line         = vsel_active && li >= vsel_sy && li <= vsel_ey;
 				const std::size_t vsel_spot_src = vsel_on_line
 					                                  ? std::min(buf->Curx(), line.size())
@@ -333,7 +301,7 @@ TerminalRenderer::Draw(Editor &ed)
 									vsel_on_line && !vsel_spot_is_eol && src_i ==
 									vsel_spot_src;
 								bool in_sel = in_mark || in_vsel;
-								bool in_hl  = search_mode && is_src_in_hl(src_i);
+								bool in_hl  = search_hl.Active() && is_src_in_hl(src_i);
 								bool in_cur =
 									has_current && li == cur_my && src_i >= cur_mx
 									&&
@@ -390,7 +358,7 @@ TerminalRenderer::Draw(Editor &ed)
 						}
 					}
 					bool in_sel = in_mark || in_vsel;
-					bool in_hl  = search_mode && from_src && is_src_in_hl(src_i);
+					bool in_hl  = search_hl.Active() && from_src && is_src_in_hl(src_i);
 					bool in_cur = has_current && li == cur_my && from_src && src_i >= cur_mx &&
 					              src_i < cur_mend;
 					attr_t a = A_NORMAL;

tests/test_audit_regressions.cc

@@ -24,6 +24,7 @@
 #include <sys/stat.h>
 #include <unistd.h>
 #include <random>
+#include <regex>
 #include <set>
 #include <vector>
 #include <clocale>
@@ -888,17 +889,20 @@ TEST(Audit_Commands_FilesystemErrorsDoNotThrow)
 }
 
 
-// Renderer regex highlighting on a long line with a deeply recursive pattern
+// std::regex on a long line with a deeply recursive pattern (overflowed an
-// (overflowed an 8 MiB stack at under 10,000 bytes) runs off the main stack.
+// 8 MiB stack at under 10,000 bytes) completes on the large stack.
-TEST(Audit_ForEachRegexMatch_LongLine)
+TEST(Audit_RunWithLargeStack_LongLine)
 {
 	std::string line;
 	while (line.size() < 200000)
 		line += "word, more words; and so on. ";
 	std::size_t n = 0, total = 0;
-	kte::ForEachRegexMatch(line, std::regex("(\\w|\\s|[.,;])+"), [&](std::size_t, std::size_t len) {
+	const std::regex rx("(\\w|\\s|[.,;])+");
-		++n;
+	kte::RunWithLargeStack([&] {
-		total += len;
+		for (auto it = std::sregex_iterator(line.begin(), line.end(), rx); it != std::sregex_iterator(); ++it) {
+			++n;
+			total += static_cast<std::size_t>(it->length());
+		}
 	});
 	ASSERT_EQ(n, (std::size_t) 1);
 	ASSERT_EQ(total, line.size());