From 868ee3fe2f8b381b434efc42a445405d578ebf2a Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 24 Sep 2026 08:12:57 +0000 Subject: [PATCH] Terminal: decode UTF-8 input in kte; visual-line edits keep characters whole - TerminalInputHandler reads bytes with getch() and assembles UTF-8 itself, waiting up to a second for a character's continuation bytes. The frontend reads the rest of an input batch with a zero timeout, and get_wch() then gave up on a character whose bytes had not all arrived; ncurses discarded the rest as an invalid sequence, so text pasted over a slow link silently lost characters and what followed them (typing 'ab\xc3', then '\xa9cd' 50 ms later saved 'ab'). - Visual-line insert, delete, backspace and newline carried the cursor's byte column to the other selected lines, splitting multibyte characters there (the saved file became invalid UTF-8). The column is now carried as a character count. - Visual-line delete-char left the cursor on the last selected line. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB --- Command.cc | 48 +++++++++++++++++----- TerminalInputHandler.cc | 72 ++++++++++++++++++++++++++++++--- tests/test_audit_regressions.cc | 41 +++++++++++++++++++ 3 files changed, 145 insertions(+), 16 deletions(-) diff --git a/Command.cc b/Command.cc index 3479793..965c2df 100644 --- a/Command.cc +++ b/Command.cc @@ -108,6 +108,31 @@ utf8_next_len(std::string_view line, std::size_t x) } +// Visual-line mode applies an edit at "the cursor's column" on every +// selected line. Columns are byte offsets, so the same number on a line with +// different characters can fall inside a multibyte one: carry the column as +// a character count instead. +static std::size_t +utf8_char_count(std::string_view line, std::size_t x) +{ + x = std::min(x, line.size()); + std::size_t n = 0; + for (std::size_t i = 0; i < x; i += utf8_next_len(line, i)) + ++n; + return n; +} + + +static std::size_t +utf8_byte_of_char(std::string_view line, std::size_t nchars) +{ + std::size_t i = 0; + for (; nchars > 0 && i < line.size(); --nchars) + i += utf8_next_len(line, i); + return i; +} + + static inline std::string_view line_view(const std::string &l) { @@ -2549,12 +2574,11 @@ cmd_insert_text(CommandContext &ctx) for (int i = 0; i < repeat; ++i) ins += ctx.arg; } + const std::size_t xchars = y < rows.size() ? utf8_char_count(line_view(rows[y]), x) : x; for (std::size_t yy = sy; yy <= ey; ++yy) { if (yy >= rows.size()) break; - std::size_t xx = x; - if (xx > rows[yy].size()) - xx = rows[yy].size(); + std::size_t xx = utf8_byte_of_char(line_view(rows[yy]), xchars); if (!ins.empty()) { buf->SetCursor(xx, yy); if (u) @@ -3476,11 +3500,12 @@ cmd_newline(CommandContext &ctx) }; // Iterate bottom-up to keep row indices stable while splitting. - const std::size_t nrows = buf->Nrows(); + const std::size_t nrows = buf->Nrows(); + const std::size_t xchars = y < nrows ? utf8_char_count(buf->GetLineString(y), x) : x; for (std::size_t yy = ey + 1; yy-- > sy;) { if (yy >= nrows) continue; - const std::size_t xx = std::min(x, buf->GetLineString(yy).size()); + const std::size_t xx = utf8_byte_of_char(buf->GetLineString(yy), xchars); // First split at the cursor column; subsequent splits create blank lines. split_at(yy, xx); for (int i = 1; i < repeat; ++i) @@ -3667,12 +3692,11 @@ cmd_backspace(CommandContext &ctx) gid = u->BeginGroup(); (void) gid; std::size_t cx = x; + const std::size_t xchars = y < rows.size() ? utf8_char_count(line_view(rows[y]), x) : x; for (std::size_t yy = sy; yy <= ey; ++yy) { if (yy >= rows.size()) break; - std::size_t xx = x; - if (xx > rows[yy].size()) - xx = rows[yy].size(); + std::size_t xx = utf8_byte_of_char(line_view(rows_of(*buf)[yy]), xchars); std::string deleted; for (int i = 0; i < repeat; ++i) { if (xx == 0) @@ -3777,12 +3801,11 @@ cmd_delete_char(CommandContext &ctx) if (u) gid = u->BeginGroup(); (void) gid; + const std::size_t xchars = y < rows.size() ? utf8_char_count(line_view(rows[y]), x) : x; for (std::size_t yy = sy; yy <= ey; ++yy) { if (yy >= rows.size()) break; - std::size_t xx = x; - if (xx > rows[yy].size()) - xx = rows[yy].size(); + std::size_t xx = utf8_byte_of_char(line_view(rows_of(*buf)[yy]), xchars); std::string deleted; for (int i = 0; i < repeat; ++i) { const auto &rows_view = rows_of(*buf); @@ -3802,6 +3825,9 @@ cmd_delete_char(CommandContext &ctx) } if (u) u->EndGroup(); + // Recording each line's undo moved the cursor; deleting forward + // leaves it where it was. + buf->SetCursor(x, y); buf->SetDirty(true); ensure_cursor_visible(ctx.editor, *buf); return true; diff --git a/TerminalInputHandler.cc b/TerminalInputHandler.cc index 305102d..8130a46 100644 --- a/TerminalInputHandler.cc +++ b/TerminalInputHandler.cc @@ -358,16 +358,78 @@ map_key_to_command(const int ch, } +// Read one key: an ncurses KEY_* code, or a character decoded from UTF-8 +// here rather than by get_wch(). With a zero timeout (the frontend reads the +// rest of a batch without waiting), get_wch() gave up on a character whose +// bytes had not all arrived yet, and ncurses then discarded the remaining +// bytes as an invalid sequence: text pasted over a slow link lost +// characters, and whatever followed them in the same read. +// Returns false when there is no input. +static bool +read_key(int &ch, bool &is_keycode) +{ + const int c = getch(); + if (c == ERR) + return false; + is_keycode = c > 0xFF; + ch = c; + if (is_keycode || c < 0x80) + return true; + + // Lead byte of a multibyte character: how many continuation bytes follow. + int need; + char32_t cp; + if (c >= 0xC2 && c <= 0xDF) { + need = 1; + cp = static_cast(c & 0x1F); + } else if (c >= 0xE0 && c <= 0xEF) { + need = 2; + cp = static_cast(c & 0x0F); + } else if (c >= 0xF0 && c <= 0xF4) { + need = 3; + cp = static_cast(c & 0x07); + } else { + ch = -1; // stray continuation or invalid byte: ignore it + return true; + } + // The rest of the character is on its way; wait for it (restoring the + // caller's timeout), but not forever if it never comes. + const int saved_delay = wgetdelay(stdscr); + timeout(1000); + for (int i = 0; i < need; ++i) { + const int b = getch(); + if (b == ERR || b > 0xFF || (b & 0xC0) != 0x80) { + if (b != ERR) + ungetch(b); // not part of this character; read it next + ch = -1; + timeout(saved_delay); + return true; + } + cp = (cp << 6) | static_cast(b & 0x3F); + } + timeout(saved_delay); + // Reject overlong forms and surrogates. + if ((need == 2 && cp < 0x800) || (need == 3 && cp < 0x10000) || (cp >= 0xD800 && cp <= 0xDFFF) || + cp > 0x10FFFF) { + ch = -1; + return true; + } + ch = static_cast(cp); + return true; +} + + bool TerminalInputHandler::decode_(MappedInput &out) { - wint_t wch; - int ret = get_wch(&wch); - if (ret == ERR) { + int ch = 0; + bool is_keycode = false; + if (!read_key(ch, is_keycode)) return false; // no input + if (ch < 0) { + out.hasCommand = false; // invalid byte sequence, dropped + return true; } - const bool is_keycode = (ret == KEY_CODE_YES); - const int ch = static_cast(wch); bool consumed = map_key_to_command( ch, is_keycode, k_prefix_, esc_meta_, diff --git a/tests/test_audit_regressions.cc b/tests/test_audit_regressions.cc index 46c7e85..762861e 100644 --- a/tests/test_audit_regressions.cc +++ b/tests/test_audit_regressions.cc @@ -1400,3 +1400,44 @@ TEST(Audit_VisualYank_CursorStaysInBuffer) (void) h.Exec(CommandId::Undo); ASSERT_EQ(h.Text(), std::string("abc def\n")); } + + +// Visual-line edits apply at the cursor's column on every selected line; a +// byte column taken from an ASCII line fell inside the CJK characters of +// the next line and split them. The column is now carried in characters. +TEST(Audit_VisualLine_EditsKeepUtf8Whole) +{ + const std::string cjk = "\xE4\xB8\x96\xE7\x95\x8C"; // two 3-byte characters + { + TestHarness h; + h.Buf().insert_text(0, 0, "abcdef\n" + cjk + "\n"); + h.Buf().SetCursor(0, 1); + ASSERT_TRUE(h.Exec(CommandId::VisualLineModeToggle)); + ASSERT_TRUE(h.Exec(CommandId::MoveUp)); + ASSERT_TRUE(h.Exec(CommandId::MoveRight)); + ASSERT_TRUE(h.Exec(CommandId::InsertText, "X")); + ASSERT_EQ(h.Text(), std::string("aXbcdef\n\xE4\xB8\x96X\xE7\x95\x8C\n")); + } + { + TestHarness h; + h.Buf().insert_text(0, 0, "abcdef\n" + cjk + "\n"); + h.Buf().SetCursor(0, 1); + ASSERT_TRUE(h.Exec(CommandId::VisualLineModeToggle)); + ASSERT_TRUE(h.Exec(CommandId::MoveUp)); + ASSERT_TRUE(h.Exec(CommandId::DeleteChar)); + ASSERT_EQ(h.Text(), std::string("bcdef\n\xE7\x95\x8C\n")); + ASSERT_TRUE(h.Exec(CommandId::MoveRight)); + ASSERT_TRUE(h.Exec(CommandId::Backspace)); + ASSERT_EQ(h.Text(), std::string("cdef\n\n")); + } + { + TestHarness h; + h.Buf().insert_text(0, 0, "abcdef\n" + cjk + "\n"); + h.Buf().SetCursor(0, 1); + ASSERT_TRUE(h.Exec(CommandId::VisualLineModeToggle)); + ASSERT_TRUE(h.Exec(CommandId::MoveUp)); + ASSERT_TRUE(h.Exec(CommandId::MoveRight)); + ASSERT_TRUE(h.Exec(CommandId::Newline)); + ASSERT_EQ(h.Text(), std::string("a\nbcdef\n\xE4\xB8\x96\n\xE7\x95\x8C\n")); + } +}