kyle/kte
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Draw non-ASCII text whole in the GUIs (B7); one span sanitizer

Browse Source 
Highlighters emit one span per byte of a non-ASCII character, and the
ImGui and Qt renderers decoded each span on its own, so every such
character was drawn as replacement glyphs (reproduced under Xvfb:
"héllo" showed as "h??llo"). kte::SanitizeSpans (Highlight.h) widens
spans to whole UTF-8 characters and removes overlaps (the earlier span
wins), and replaces the three renderers' copies of the clamp/sort code.

The terminal renderer's token lookup rescanned the spans from the first
for every byte drawn; it now walks them with a cursor.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB
This commit is contained in:
Claude
2026-09-24 12:19:59 -07:00
committed by Kyle Isom
parent d490245f30
commit c5df5035fa
5 changed files with 95 additions and 87 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Highlight.h
+42
View File
@@ -1,7 +1,10 @@
// Highlight.h - core syntax highlighting types for kte
#pragma once
#include <algorithm>
#include <cstddef>
#include <cstdint>
#include <string_view>
#include <vector>
namespace kte {
@@ -34,4 +37,43 @@ struct LineHighlight {
std::vector<HighlightSpan> spans;
std::uint64_t version{0}; // buffer version used for this line
};
// A line's spans ready to draw: clamped to the line, ordered by start, not
// overlapping (the earlier span wins), and widened to whole UTF-8
// characters. Highlighters may emit one span per byte of a non-ASCII
// character; drawn separately, each byte showed as a replacement glyph.
inline void
SanitizeSpans(std::string_view line, const std::vector<HighlightSpan> &in, std::vector<HighlightSpan> &out)
{
out.clear();
const long len = static_cast<long>(line.size());
for (const HighlightSpan &sp: in) {
long s = std::clamp<long>(std::min(sp.col_start, sp.col_end), 0, len);
long e = std::clamp<long>(std::max(sp.col_start, sp.col_end), 0, len);
if (e > s)
out.push_back(HighlightSpan{static_cast<int>(s), static_cast<int>(e), sp.kind});
}
std::stable_sort(out.begin(), out.end(), [](const HighlightSpan &a, const HighlightSpan &b) {
return a.col_start < b.col_start;
});
auto continuation = [&](long i) {
return i < len && (static_cast<unsigned char>(line[static_cast<std::size_t>(i)]) & 0xC0) == 0x80;
};
std::size_t w = 0;
long prev_end = 0;
for (const HighlightSpan &sp: out) {
long s = sp.col_start, e = sp.col_end;
while (s > 0 && continuation(s))
--s;
while (e < len && continuation(e))
++e;
s = std::max(s, prev_end);
if (e <= s)
continue;
out[w++] = HighlightSpan{static_cast<int>(s), static_cast<int>(e), sp.kind};
prev_end = e;
}
out.resize(w);
}
} // namespace kte
ImGuiRenderer.cc
+5 -28
View File
@@ -440,42 +440,19 @@ ImGuiRenderer::Draw(Editor &ed)
if (buf->SyntaxEnabled() && buf->Highlighter() && buf->Highlighter()->HasHighlighter()) {
const kte::LineHighlight &lh = buf->Highlighter()->GetLine(
*buf, static_cast<int>(i), buf->Version());
// Sanitize spans defensively: clamp to [0, line.size()], ensure end>=start, drop empties
struct SSpan {
std::size_t s;
std::size_t e;
kte::TokenKind k;
};
std::vector<SSpan> spans;
spans.reserve(lh.spans.size());
const std::size_t line_len = line.size();
for (const auto &sp: lh.spans) {
int s_raw = sp.col_start;
int e_raw = sp.col_end;
if (e_raw < s_raw)
std::swap(e_raw, s_raw);
std::size_t s = static_cast<std::size_t>(std::max(
0, std::min(s_raw, static_cast<int>(line_len))));
std::size_t e = static_cast<std::size_t>(std::max(
static_cast<int>(s), std::min(e_raw, static_cast<int>(line_len))));
if (e <= s)
continue;
spans.push_back(SSpan{s, e, sp.kind});
}
std::sort(spans.begin(), spans.end(), [](const SSpan &a, const SSpan &b) {
return a.s < b.s;
});
std::vector<kte::HighlightSpan> spans; // clamped, ordered, whole characters
kte::SanitizeSpans(line, lh.spans, spans);
for (const auto &sp: spans) {
std::size_t rx_s = src_to_rx(sp.s);
std::size_t rx_e = src_to_rx(sp.e);
std::size_t rx_s = src_to_rx(static_cast<std::size_t>(sp.col_start));
std::size_t rx_e = src_to_rx(static_cast<std::size_t>(sp.col_end));
std::size_t draw_start = rx_s;
if (draw_start >= expanded.size())
continue;
std::size_t draw_end = std::min<std::size_t>(rx_e, expanded.size());
if (draw_end <= draw_start)
continue;
ImU32 col = ImGui::GetColorU32(kte::SyntaxInk(sp.k));
ImU32 col = ImGui::GetColorU32(kte::SyntaxInk(sp.kind));
ImVec2 p = ImVec2(line_pos.x + rx_to_px(draw_start),
line_pos.y);
ImGui::GetWindowDrawList()->AddText(
QtFrontend.cc
+5 -27
View File
@@ -310,30 +310,8 @@ protected:
HasHighlighter()) {
const kte::LineHighlight &lh = buf->Highlighter()->GetLine(
*buf, static_cast<int>(i), buf->Version());
struct SSpan {
std::size_t s;
std::size_t e;
kte::TokenKind k;
};
std::vector<SSpan> spans;
spans.reserve(lh.spans.size());
const std::size_t line_len = line.size();
for (const auto &sp: lh.spans) {
int s_raw = sp.col_start;
int e_raw = sp.col_end;
if (e_raw < s_raw)
std::swap(e_raw, s_raw);
std::size_t s = static_cast<std::size_t>(std::max(
0, std::min(s_raw, (int) line_len)));
std::size_t e = static_cast<std::size_t>(std::max(
(int) s, std::min(e_raw, (int) line_len)));
if (s < e)
spans.push_back({s, e, sp.kind});
}
std::sort(spans.begin(), spans.end(),
[](const SSpan &a, const SSpan &b) {
return a.s < b.s;
});
std::vector<kte::HighlightSpan> spans; // clamped, ordered, whole characters
kte::SanitizeSpans(line, lh.spans, spans);
auto colorFor = [](kte::TokenKind k) -> QColor {
// GUITheme provides colors via ImGui vector; avoid direct dependency types
@@ -344,10 +322,10 @@ protected:
std::size_t pos = 0;
for (const auto &sp: spans) {
const std::size_t s = std::max(sp.s, pos);
const std::size_t s = std::max(static_cast<std::size_t>(sp.col_start), pos);
draw_seg(pos, s, fg);
draw_seg(s, sp.e, colorFor(sp.k));
pos = std::max(pos, sp.e);
draw_seg(s, static_cast<std::size_t>(sp.col_end), colorFor(sp.kind));
pos = std::max(pos, static_cast<std::size_t>(sp.col_end));
}
draw_seg(pos, line.size(), fg);
} else {
TerminalRenderer.cc
+12 -32
View File
@@ -165,40 +165,20 @@ TerminalRenderer::Draw(Editor &ed)
HasHighlighter()) {
const kte::LineHighlight &lh_val = buf->Highlighter()->GetLine(
*buf, static_cast<int>(li), buf->Version());
// Sanitize defensively: clamp to [0, line.size()], ensure end>=start, drop empties
const std::size_t line_len = line.size();
sane_spans.reserve(lh_val.spans.size());
for (const auto &sp: lh_val.spans) {
int s_raw = sp.col_start;
int e_raw = sp.col_end;
if (e_raw < s_raw)
std::swap(e_raw, s_raw);
std::size_t s = static_cast<std::size_t>(std::max(
0, std::min(s_raw, static_cast<int>(line_len))));
std::size_t e = static_cast<std::size_t>(std::max(
static_cast<int>(s),
std::min(e_raw, static_cast<int>(line_len))));
if (e <= s)
continue;
sane_spans.push_back(kte::HighlightSpan{
static_cast<int>(s), static_cast<int>(e), sp.kind
});
}
std::sort(sane_spans.begin(), sane_spans.end(),
[](const kte::HighlightSpan &a, const kte::HighlightSpan &b) {
return a.col_start < b.col_start;
});
kte::SanitizeSpans(line, lh_val.spans, sane_spans);
}
// Called left to right while drawing: walk the spans with a
// cursor (rescanning from the first span for every byte was
// quadratic on long highlighted lines).
std::size_t span_i = 0;
auto token_at = [&](std::size_t src_index) -> kte::TokenKind {
if (sane_spans.empty())
return kte::TokenKind::Default;
int si = static_cast<int>(src_index);
for (const auto &sp: sane_spans) {
if (si < sp.col_start)
break;
if (si >= sp.col_start && si < sp.col_end)
return sp.kind;
}
const int si = static_cast<int>(src_index);
if (span_i > 0 && span_i <= sane_spans.size() && si < sane_spans[span_i - 1].col_start)
span_i = 0; // moved backwards: restart
while (span_i < sane_spans.size() && sane_spans[span_i].col_end <= si)
++span_i;
if (span_i < sane_spans.size() && sane_spans[span_i].col_start <= si)
return sane_spans[span_i].kind;
return kte::TokenKind::Default;
};
auto token_attr = [&](kte::TokenKind k) -> attr_t {
tests/test_audit_regressions.cc
+31
View File
@@ -2328,3 +2328,34 @@ TEST(Swap_V1JournalGetsNoChunkedRecords)
b.SetSwapRecorder(nullptr);
sm.Detach(&b, true);
}
// B7: spans that split a UTF-8 character (highlighters emit one per byte of
// a non-ASCII character) are widened to whole characters, and overlaps are
// removed, before renderers decode them.
TEST(Audit_B7_SanitizeSpans_WholeCharacters)
{
const std::string line = "h\xC3\xA9llo \xE2\x9C\x93!"; // "héllo ✓!"
using kte::HighlightSpan;
using kte::TokenKind;
// Bytes: h 0, é 1-2, l 3, l 4, o 5, space 6, ✓ 7-9, ! 10.
const std::vector<HighlightSpan> in = {
{10, 11, TokenKind::Operator}, // '!' (listed out of order)
{0, 1, TokenKind::Identifier},
{1, 2, TokenKind::Identifier}, // first byte of é
{2, 3, TokenKind::Identifier}, // second byte of é
{3, 6, TokenKind::Identifier},
{8, 9, TokenKind::Error}, // middle byte of ✓
{9, 50, TokenKind::Error}, // last byte of ✓, runs past the end
{5, 4, TokenKind::Keyword}, // reversed, overlaps [3,6)
};
std::vector<HighlightSpan> out;
kte::SanitizeSpans(line, in, out);
std::vector<std::pair<int, int> > got;
for (const auto &sp: out)
got.emplace_back(sp.col_start, sp.col_end);
const std::vector<std::pair<int, int> > want = {{0, 1}, {1, 3}, {3, 6}, {7, 10}, {10, 11}};
ASSERT_EQ(got, want);
ASSERT_TRUE(out[3].kind == TokenKind::Error);
ASSERT_TRUE(out[4].kind == TokenKind::Error); // the earlier-starting span wins
}