Draw non-ASCII text whole in the GUIs (B7); one span sanitizer

Highlighters emit one span per byte of a non-ASCII character, and the
ImGui and Qt renderers decoded each span on its own, so every such
character was drawn as replacement glyphs (reproduced under Xvfb:
"héllo" showed as "h??llo"). kte::SanitizeSpans (Highlight.h) widens
spans to whole UTF-8 characters and removes overlaps (the earlier span
wins), and replaces the three renderers' copies of the clamp/sort code.

The terminal renderer's token lookup rescanned the spans from the first
for every byte drawn; it now walks them with a cursor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB

Highlight.h

@@ -1,7 +1,10 @@
 // Highlight.h - core syntax highlighting types for kte
 #pragma once
 
+#include <algorithm>
+#include <cstddef>
 #include <cstdint>
+#include <string_view>
 #include <vector>
 
 namespace kte {
@@ -34,4 +37,43 @@ struct LineHighlight {
 	std::vector<HighlightSpan> spans;
 	std::uint64_t version{0}; // buffer version used for this line
 };
+
+
+// A line's spans ready to draw: clamped to the line, ordered by start, not
+// overlapping (the earlier span wins), and widened to whole UTF-8
+// characters. Highlighters may emit one span per byte of a non-ASCII
+// character; drawn separately, each byte showed as a replacement glyph.
+inline void
+SanitizeSpans(std::string_view line, const std::vector<HighlightSpan> &in, std::vector<HighlightSpan> &out)
+{
+	out.clear();
+	const long len = static_cast<long>(line.size());
+	for (const HighlightSpan &sp: in) {
+		long s = std::clamp<long>(std::min(sp.col_start, sp.col_end), 0, len);
+		long e = std::clamp<long>(std::max(sp.col_start, sp.col_end), 0, len);
+		if (e > s)
+			out.push_back(HighlightSpan{static_cast<int>(s), static_cast<int>(e), sp.kind});
+	}
+	std::stable_sort(out.begin(), out.end(), [](const HighlightSpan &a, const HighlightSpan &b) {
+		return a.col_start < b.col_start;
+	});
+	auto continuation = [&](long i) {
+		return i < len && (static_cast<unsigned char>(line[static_cast<std::size_t>(i)]) & 0xC0) == 0x80;
+	};
+	std::size_t w = 0;
+	long prev_end = 0;
+	for (const HighlightSpan &sp: out) {
+		long s = sp.col_start, e = sp.col_end;
+		while (s > 0 && continuation(s))
+			--s;
+		while (e < len && continuation(e))
+			++e;
+		s = std::max(s, prev_end);
+		if (e <= s)
+			continue;
+		out[w++] = HighlightSpan{static_cast<int>(s), static_cast<int>(e), sp.kind};
+		prev_end = e;
+	}
+	out.resize(w);
+}
 } // namespace kte

ImGuiRenderer.cc

@@ -440,42 +440,19 @@ ImGuiRenderer::Draw(Editor &ed)
 			if (buf->SyntaxEnabled() && buf->Highlighter() && buf->Highlighter()->HasHighlighter()) {
 				const kte::LineHighlight &lh = buf->Highlighter()->GetLine(
 					*buf, static_cast<int>(i), buf->Version());
-				// Sanitize spans defensively: clamp to [0, line.size()], ensure end>=start, drop empties
+				std::vector<kte::HighlightSpan> spans; // clamped, ordered, whole characters
-				struct SSpan {
+				kte::SanitizeSpans(line, lh.spans, spans);
-					std::size_t s;
-					std::size_t e;
-					kte::TokenKind k;
-				};
-				std::vector<SSpan> spans;
-				spans.reserve(lh.spans.size());
-				const std::size_t line_len = line.size();
-				for (const auto &sp: lh.spans) {
-					int s_raw = sp.col_start;
-					int e_raw = sp.col_end;
-					if (e_raw < s_raw)
-						std::swap(e_raw, s_raw);
-					std::size_t s = static_cast<std::size_t>(std::max(
-						0, std::min(s_raw, static_cast<int>(line_len))));
-					std::size_t e = static_cast<std::size_t>(std::max(
-						static_cast<int>(s), std::min(e_raw, static_cast<int>(line_len))));
-					if (e <= s)
-						continue;
-					spans.push_back(SSpan{s, e, sp.kind});
-				}
-				std::sort(spans.begin(), spans.end(), [](const SSpan &a, const SSpan &b) {
-					return a.s < b.s;
-				});
 
 				for (const auto &sp: spans) {
-					std::size_t rx_s = src_to_rx(sp.s);
+					std::size_t rx_s = src_to_rx(static_cast<std::size_t>(sp.col_start));
-					std::size_t rx_e = src_to_rx(sp.e);
+					std::size_t rx_e = src_to_rx(static_cast<std::size_t>(sp.col_end));
 					std::size_t draw_start = rx_s;
 					if (draw_start >= expanded.size())
 						continue;
 					std::size_t draw_end = std::min<std::size_t>(rx_e, expanded.size());
 					if (draw_end <= draw_start)
 						continue;
-					ImU32 col = ImGui::GetColorU32(kte::SyntaxInk(sp.k));
+					ImU32 col = ImGui::GetColorU32(kte::SyntaxInk(sp.kind));
 					ImVec2 p = ImVec2(line_pos.x + rx_to_px(draw_start),
 					                  line_pos.y);
 					ImGui::GetWindowDrawList()->AddText(

QtFrontend.cc

@@ -310,30 +310,8 @@ protected:
 					    HasHighlighter()) {
 						const kte::LineHighlight &lh = buf->Highlighter()->GetLine(
 							*buf, static_cast<int>(i), buf->Version());
-						struct SSpan {
+						std::vector<kte::HighlightSpan> spans; // clamped, ordered, whole characters
-							std::size_t s;
+						kte::SanitizeSpans(line, lh.spans, spans);
-							std::size_t e;
-							kte::TokenKind k;
-						};
-						std::vector<SSpan> spans;
-						spans.reserve(lh.spans.size());
-						const std::size_t line_len = line.size();
-						for (const auto &sp: lh.spans) {
-							int s_raw = sp.col_start;
-							int e_raw = sp.col_end;
-							if (e_raw < s_raw)
-								std::swap(e_raw, s_raw);
-							std::size_t s = static_cast<std::size_t>(std::max(
-								0, std::min(s_raw, (int) line_len)));
-							std::size_t e = static_cast<std::size_t>(std::max(
-								(int) s, std::min(e_raw, (int) line_len)));
-							if (s < e)
-								spans.push_back({s, e, sp.kind});
-						}
-						std::sort(spans.begin(), spans.end(),
-						          [](const SSpan &a, const SSpan &b) {
-							          return a.s < b.s;
-						          });
 
 						auto colorFor = [](kte::TokenKind k) -> QColor {
 							// GUITheme provides colors via ImGui vector; avoid direct dependency types
@@ -344,10 +322,10 @@ protected:
 
 						std::size_t pos = 0;
 						for (const auto &sp: spans) {
-							const std::size_t s = std::max(sp.s, pos);
+							const std::size_t s = std::max(static_cast<std::size_t>(sp.col_start), pos);
 							draw_seg(pos, s, fg);
-							draw_seg(s, sp.e, colorFor(sp.k));
+							draw_seg(s, static_cast<std::size_t>(sp.col_end), colorFor(sp.kind));
-							pos = std::max(pos, sp.e);
+							pos = std::max(pos, static_cast<std::size_t>(sp.col_end));
 						}
 						draw_seg(pos, line.size(), fg);
 					} else {

TerminalRenderer.cc

@@ -165,40 +165,20 @@ TerminalRenderer::Draw(Editor &ed)
 				    HasHighlighter()) {
 					const kte::LineHighlight &lh_val = buf->Highlighter()->GetLine(
 						*buf, static_cast<int>(li), buf->Version());
-					// Sanitize defensively: clamp to [0, line.size()], ensure end>=start, drop empties
+					kte::SanitizeSpans(line, lh_val.spans, sane_spans);
-					const std::size_t line_len = line.size();
-					sane_spans.reserve(lh_val.spans.size());
-					for (const auto &sp: lh_val.spans) {
-						int s_raw = sp.col_start;
-						int e_raw = sp.col_end;
-						if (e_raw < s_raw)
-							std::swap(e_raw, s_raw);
-						std::size_t s = static_cast<std::size_t>(std::max(
-							0, std::min(s_raw, static_cast<int>(line_len))));
-						std::size_t e = static_cast<std::size_t>(std::max(
-							static_cast<int>(s),
-							std::min(e_raw, static_cast<int>(line_len))));
-						if (e <= s)
-							continue;
-						sane_spans.push_back(kte::HighlightSpan{
-							static_cast<int>(s), static_cast<int>(e), sp.kind
-						});
-					}
-					std::sort(sane_spans.begin(), sane_spans.end(),
-					          [](const kte::HighlightSpan &a, const kte::HighlightSpan &b) {
-						          return a.col_start < b.col_start;
-					          });
 				}
+				// Called left to right while drawing: walk the spans with a
+				// cursor (rescanning from the first span for every byte was
+				// quadratic on long highlighted lines).
+				std::size_t span_i = 0;
 				auto token_at = [&](std::size_t src_index) -> kte::TokenKind {
-					if (sane_spans.empty())
+					const int si = static_cast<int>(src_index);
-						return kte::TokenKind::Default;
+					if (span_i > 0 && span_i <= sane_spans.size() && si < sane_spans[span_i - 1].col_start)
-					int si = static_cast<int>(src_index);
+						span_i = 0; // moved backwards: restart
-					for (const auto &sp: sane_spans) {
+					while (span_i < sane_spans.size() && sane_spans[span_i].col_end <= si)
-						if (si < sp.col_start)
+						++span_i;
-							break;
+					if (span_i < sane_spans.size() && sane_spans[span_i].col_start <= si)
-						if (si >= sp.col_start && si < sp.col_end)
+						return sane_spans[span_i].kind;
-							return sp.kind;
-					}
 					return kte::TokenKind::Default;
 				};
 				auto token_attr = [&](kte::TokenKind k) -> attr_t {

tests/test_audit_regressions.cc

@@ -2328,3 +2328,34 @@ TEST(Swap_V1JournalGetsNoChunkedRecords)
 	b.SetSwapRecorder(nullptr);
 	sm.Detach(&b, true);
 }
+
+
+// B7: spans that split a UTF-8 character (highlighters emit one per byte of
+// a non-ASCII character) are widened to whole characters, and overlaps are
+// removed, before renderers decode them.
+TEST(Audit_B7_SanitizeSpans_WholeCharacters)
+{
+	const std::string line = "h\xC3\xA9llo \xE2\x9C\x93!"; // "héllo ✓!"
+	using kte::HighlightSpan;
+	using kte::TokenKind;
+	// Bytes: h 0, é 1-2, l 3, l 4, o 5, space 6, ✓ 7-9, ! 10.
+	const std::vector<HighlightSpan> in = {
+		{10, 11, TokenKind::Operator}, // '!' (listed out of order)
+		{0, 1, TokenKind::Identifier},
+		{1, 2, TokenKind::Identifier}, // first byte of é
+		{2, 3, TokenKind::Identifier}, // second byte of é
+		{3, 6, TokenKind::Identifier},
+		{8, 9, TokenKind::Error},   // middle byte of ✓
+		{9, 50, TokenKind::Error},  // last byte of ✓, runs past the end
+		{5, 4, TokenKind::Keyword}, // reversed, overlaps [3,6)
+	};
+	std::vector<HighlightSpan> out;
+	kte::SanitizeSpans(line, in, out);
+	std::vector<std::pair<int, int> > got;
+	for (const auto &sp: out)
+		got.emplace_back(sp.col_start, sp.col_end);
+	const std::vector<std::pair<int, int> > want = {{0, 1}, {1, 3}, {3, 6}, {7, 10}, {10, 11}};
+	ASSERT_EQ(got, want);
+	ASSERT_TRUE(out[3].kind == TokenKind::Error);
+	ASSERT_TRUE(out[4].kind == TokenKind::Error); // the earlier-starting span wins
+}