From c5df5035faf8c302320d926db4a6bfd0579d20b6 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 17:13:41 +0000 Subject: [PATCH] Draw non-ASCII text whole in the GUIs (B7); one span sanitizer MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Highlighters emit one span per byte of a non-ASCII character, and the ImGui and Qt renderers decoded each span on its own, so every such character was drawn as replacement glyphs (reproduced under Xvfb: "héllo" showed as "h??llo"). kte::SanitizeSpans (Highlight.h) widens spans to whole UTF-8 characters and removes overlaps (the earlier span wins), and replaces the three renderers' copies of the clamp/sort code. The terminal renderer's token lookup rescanned the spans from the first for every byte drawn; it now walks them with a cursor. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB --- Highlight.h | 42 +++++++++++++++++++++++++++++++ ImGuiRenderer.cc | 33 ++++--------------------- QtFrontend.cc | 32 ++++-------------------- TerminalRenderer.cc | 44 +++++++++------------------------ tests/test_audit_regressions.cc | 31 +++++++++++++++++++++++ 5 files changed, 95 insertions(+), 87 deletions(-) diff --git a/Highlight.h b/Highlight.h index a3581e3..b9f16cc 100644 --- a/Highlight.h +++ b/Highlight.h @@ -1,7 +1,10 @@ // Highlight.h - core syntax highlighting types for kte #pragma once +#include +#include #include +#include #include namespace kte { @@ -34,4 +37,43 @@ struct LineHighlight { std::vector spans; std::uint64_t version{0}; // buffer version used for this line }; + + +// A line's spans ready to draw: clamped to the line, ordered by start, not +// overlapping (the earlier span wins), and widened to whole UTF-8 +// characters. Highlighters may emit one span per byte of a non-ASCII +// character; drawn separately, each byte showed as a replacement glyph. +inline void +SanitizeSpans(std::string_view line, const std::vector &in, std::vector &out) +{ + out.clear(); + const long len = static_cast(line.size()); + for (const HighlightSpan &sp: in) { + long s = std::clamp(std::min(sp.col_start, sp.col_end), 0, len); + long e = std::clamp(std::max(sp.col_start, sp.col_end), 0, len); + if (e > s) + out.push_back(HighlightSpan{static_cast(s), static_cast(e), sp.kind}); + } + std::stable_sort(out.begin(), out.end(), [](const HighlightSpan &a, const HighlightSpan &b) { + return a.col_start < b.col_start; + }); + auto continuation = [&](long i) { + return i < len && (static_cast(line[static_cast(i)]) & 0xC0) == 0x80; + }; + std::size_t w = 0; + long prev_end = 0; + for (const HighlightSpan &sp: out) { + long s = sp.col_start, e = sp.col_end; + while (s > 0 && continuation(s)) + --s; + while (e < len && continuation(e)) + ++e; + s = std::max(s, prev_end); + if (e <= s) + continue; + out[w++] = HighlightSpan{static_cast(s), static_cast(e), sp.kind}; + prev_end = e; + } + out.resize(w); +} } // namespace kte diff --git a/ImGuiRenderer.cc b/ImGuiRenderer.cc index 70a6146..4cd9cd4 100644 --- a/ImGuiRenderer.cc +++ b/ImGuiRenderer.cc @@ -440,42 +440,19 @@ ImGuiRenderer::Draw(Editor &ed) if (buf->SyntaxEnabled() && buf->Highlighter() && buf->Highlighter()->HasHighlighter()) { const kte::LineHighlight &lh = buf->Highlighter()->GetLine( *buf, static_cast(i), buf->Version()); - // Sanitize spans defensively: clamp to [0, line.size()], ensure end>=start, drop empties - struct SSpan { - std::size_t s; - std::size_t e; - kte::TokenKind k; - }; - std::vector spans; - spans.reserve(lh.spans.size()); - const std::size_t line_len = line.size(); - for (const auto &sp: lh.spans) { - int s_raw = sp.col_start; - int e_raw = sp.col_end; - if (e_raw < s_raw) - std::swap(e_raw, s_raw); - std::size_t s = static_cast(std::max( - 0, std::min(s_raw, static_cast(line_len)))); - std::size_t e = static_cast(std::max( - static_cast(s), std::min(e_raw, static_cast(line_len)))); - if (e <= s) - continue; - spans.push_back(SSpan{s, e, sp.kind}); - } - std::sort(spans.begin(), spans.end(), [](const SSpan &a, const SSpan &b) { - return a.s < b.s; - }); + std::vector spans; // clamped, ordered, whole characters + kte::SanitizeSpans(line, lh.spans, spans); for (const auto &sp: spans) { - std::size_t rx_s = src_to_rx(sp.s); - std::size_t rx_e = src_to_rx(sp.e); + std::size_t rx_s = src_to_rx(static_cast(sp.col_start)); + std::size_t rx_e = src_to_rx(static_cast(sp.col_end)); std::size_t draw_start = rx_s; if (draw_start >= expanded.size()) continue; std::size_t draw_end = std::min(rx_e, expanded.size()); if (draw_end <= draw_start) continue; - ImU32 col = ImGui::GetColorU32(kte::SyntaxInk(sp.k)); + ImU32 col = ImGui::GetColorU32(kte::SyntaxInk(sp.kind)); ImVec2 p = ImVec2(line_pos.x + rx_to_px(draw_start), line_pos.y); ImGui::GetWindowDrawList()->AddText( diff --git a/QtFrontend.cc b/QtFrontend.cc index 6417655..5915272 100644 --- a/QtFrontend.cc +++ b/QtFrontend.cc @@ -310,30 +310,8 @@ protected: HasHighlighter()) { const kte::LineHighlight &lh = buf->Highlighter()->GetLine( *buf, static_cast(i), buf->Version()); - struct SSpan { - std::size_t s; - std::size_t e; - kte::TokenKind k; - }; - std::vector spans; - spans.reserve(lh.spans.size()); - const std::size_t line_len = line.size(); - for (const auto &sp: lh.spans) { - int s_raw = sp.col_start; - int e_raw = sp.col_end; - if (e_raw < s_raw) - std::swap(e_raw, s_raw); - std::size_t s = static_cast(std::max( - 0, std::min(s_raw, (int) line_len))); - std::size_t e = static_cast(std::max( - (int) s, std::min(e_raw, (int) line_len))); - if (s < e) - spans.push_back({s, e, sp.kind}); - } - std::sort(spans.begin(), spans.end(), - [](const SSpan &a, const SSpan &b) { - return a.s < b.s; - }); + std::vector spans; // clamped, ordered, whole characters + kte::SanitizeSpans(line, lh.spans, spans); auto colorFor = [](kte::TokenKind k) -> QColor { // GUITheme provides colors via ImGui vector; avoid direct dependency types @@ -344,10 +322,10 @@ protected: std::size_t pos = 0; for (const auto &sp: spans) { - const std::size_t s = std::max(sp.s, pos); + const std::size_t s = std::max(static_cast(sp.col_start), pos); draw_seg(pos, s, fg); - draw_seg(s, sp.e, colorFor(sp.k)); - pos = std::max(pos, sp.e); + draw_seg(s, static_cast(sp.col_end), colorFor(sp.kind)); + pos = std::max(pos, static_cast(sp.col_end)); } draw_seg(pos, line.size(), fg); } else { diff --git a/TerminalRenderer.cc b/TerminalRenderer.cc index 142e795..e535833 100644 --- a/TerminalRenderer.cc +++ b/TerminalRenderer.cc @@ -165,40 +165,20 @@ TerminalRenderer::Draw(Editor &ed) HasHighlighter()) { const kte::LineHighlight &lh_val = buf->Highlighter()->GetLine( *buf, static_cast(li), buf->Version()); - // Sanitize defensively: clamp to [0, line.size()], ensure end>=start, drop empties - const std::size_t line_len = line.size(); - sane_spans.reserve(lh_val.spans.size()); - for (const auto &sp: lh_val.spans) { - int s_raw = sp.col_start; - int e_raw = sp.col_end; - if (e_raw < s_raw) - std::swap(e_raw, s_raw); - std::size_t s = static_cast(std::max( - 0, std::min(s_raw, static_cast(line_len)))); - std::size_t e = static_cast(std::max( - static_cast(s), - std::min(e_raw, static_cast(line_len)))); - if (e <= s) - continue; - sane_spans.push_back(kte::HighlightSpan{ - static_cast(s), static_cast(e), sp.kind - }); - } - std::sort(sane_spans.begin(), sane_spans.end(), - [](const kte::HighlightSpan &a, const kte::HighlightSpan &b) { - return a.col_start < b.col_start; - }); + kte::SanitizeSpans(line, lh_val.spans, sane_spans); } + // Called left to right while drawing: walk the spans with a + // cursor (rescanning from the first span for every byte was + // quadratic on long highlighted lines). + std::size_t span_i = 0; auto token_at = [&](std::size_t src_index) -> kte::TokenKind { - if (sane_spans.empty()) - return kte::TokenKind::Default; - int si = static_cast(src_index); - for (const auto &sp: sane_spans) { - if (si < sp.col_start) - break; - if (si >= sp.col_start && si < sp.col_end) - return sp.kind; - } + const int si = static_cast(src_index); + if (span_i > 0 && span_i <= sane_spans.size() && si < sane_spans[span_i - 1].col_start) + span_i = 0; // moved backwards: restart + while (span_i < sane_spans.size() && sane_spans[span_i].col_end <= si) + ++span_i; + if (span_i < sane_spans.size() && sane_spans[span_i].col_start <= si) + return sane_spans[span_i].kind; return kte::TokenKind::Default; }; auto token_attr = [&](kte::TokenKind k) -> attr_t { diff --git a/tests/test_audit_regressions.cc b/tests/test_audit_regressions.cc index 38add2e..198c5b5 100644 --- a/tests/test_audit_regressions.cc +++ b/tests/test_audit_regressions.cc @@ -2328,3 +2328,34 @@ TEST(Swap_V1JournalGetsNoChunkedRecords) b.SetSwapRecorder(nullptr); sm.Detach(&b, true); } + + +// B7: spans that split a UTF-8 character (highlighters emit one per byte of +// a non-ASCII character) are widened to whole characters, and overlaps are +// removed, before renderers decode them. +TEST(Audit_B7_SanitizeSpans_WholeCharacters) +{ + const std::string line = "h\xC3\xA9llo \xE2\x9C\x93!"; // "héllo ✓!" + using kte::HighlightSpan; + using kte::TokenKind; + // Bytes: h 0, é 1-2, l 3, l 4, o 5, space 6, ✓ 7-9, ! 10. + const std::vector in = { + {10, 11, TokenKind::Operator}, // '!' (listed out of order) + {0, 1, TokenKind::Identifier}, + {1, 2, TokenKind::Identifier}, // first byte of é + {2, 3, TokenKind::Identifier}, // second byte of é + {3, 6, TokenKind::Identifier}, + {8, 9, TokenKind::Error}, // middle byte of ✓ + {9, 50, TokenKind::Error}, // last byte of ✓, runs past the end + {5, 4, TokenKind::Keyword}, // reversed, overlaps [3,6) + }; + std::vector out; + kte::SanitizeSpans(line, in, out); + std::vector > got; + for (const auto &sp: out) + got.emplace_back(sp.col_start, sp.col_end); + const std::vector > want = {{0, 1}, {1, 3}, {3, 6}, {7, 10}, {10, 11}}; + ASSERT_EQ(got, want); + ASSERT_TRUE(out[3].kind == TokenKind::Error); + ASSERT_TRUE(out[4].kind == TokenKind::Error); // the earlier-starting span wins +}