Terminal: decode UTF-8 input in kte; visual-line edits keep characters whole

- TerminalInputHandler reads bytes with getch() and assembles UTF-8
  itself, waiting up to a second for a character's continuation bytes.
  The frontend reads the rest of an input batch with a zero timeout, and
  get_wch() then gave up on a character whose bytes had not all arrived;
  ncurses discarded the rest as an invalid sequence, so text pasted over
  a slow link silently lost characters and what followed them (typing
  'ab\xc3', then '\xa9cd' 50 ms later saved 'ab').
- Visual-line insert, delete, backspace and newline carried the cursor's
  byte column to the other selected lines, splitting multibyte
  characters there (the saved file became invalid UTF-8). The column is
  now carried as a character count.
- Visual-line delete-char left the cursor on the last selected line.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_015rcVQiiWyTzoNrsvLSaAfB

Command.cc

@@ -108,6 +108,31 @@ utf8_next_len(std::string_view line, std::size_t x)
 }
 
 
+// Visual-line mode applies an edit at "the cursor's column" on every
+// selected line. Columns are byte offsets, so the same number on a line with
+// different characters can fall inside a multibyte one: carry the column as
+// a character count instead.
+static std::size_t
+utf8_char_count(std::string_view line, std::size_t x)
+{
+	x = std::min(x, line.size());
+	std::size_t n = 0;
+	for (std::size_t i = 0; i < x; i += utf8_next_len(line, i))
+		++n;
+	return n;
+}
+
+
+static std::size_t
+utf8_byte_of_char(std::string_view line, std::size_t nchars)
+{
+	std::size_t i = 0;
+	for (; nchars > 0 && i < line.size(); --nchars)
+		i += utf8_next_len(line, i);
+	return i;
+}
+
+
 static inline std::string_view
 line_view(const std::string &l)
 {
@@ -2549,12 +2574,11 @@ cmd_insert_text(CommandContext &ctx)
 			for (int i = 0; i < repeat; ++i)
 				ins += ctx.arg;
 		}
+		const std::size_t xchars = y < rows.size() ? utf8_char_count(line_view(rows[y]), x) : x;
 		for (std::size_t yy = sy; yy <= ey; ++yy) {
 			if (yy >= rows.size())
 				break;
-			std::size_t xx = x;
-			if (xx > rows[yy].size())
-				xx = rows[yy].size();
+			std::size_t xx = utf8_byte_of_char(line_view(rows[yy]), xchars);
 			if (!ins.empty()) {
 				buf->SetCursor(xx, yy);
 				if (u)
@@ -3476,11 +3500,12 @@ cmd_newline(CommandContext &ctx)
 		};
 
 		// Iterate bottom-up to keep row indices stable while splitting.
-		const std::size_t nrows = buf->Nrows();
+		const std::size_t nrows  = buf->Nrows();
+		const std::size_t xchars = y < nrows ? utf8_char_count(buf->GetLineString(y), x) : x;
 		for (std::size_t yy = ey + 1; yy-- > sy;) {
 			if (yy >= nrows)
 				continue;
-			const std::size_t xx = std::min(x, buf->GetLineString(yy).size());
+			const std::size_t xx = utf8_byte_of_char(buf->GetLineString(yy), xchars);
 			// First split at the cursor column; subsequent splits create blank lines.
 			split_at(yy, xx);
 			for (int i = 1; i < repeat; ++i)
@@ -3667,12 +3692,11 @@ cmd_backspace(CommandContext &ctx)
 			gid = u->BeginGroup();
 		(void) gid;
 		std::size_t cx = x;
+		const std::size_t xchars = y < rows.size() ? utf8_char_count(line_view(rows[y]), x) : x;
 		for (std::size_t yy = sy; yy <= ey; ++yy) {
 			if (yy >= rows.size())
 				break;
-			std::size_t xx = x;
-			if (xx > rows[yy].size())
-				xx = rows[yy].size();
+			std::size_t xx = utf8_byte_of_char(line_view(rows_of(*buf)[yy]), xchars);
 			std::string deleted;
 			for (int i = 0; i < repeat; ++i) {
 				if (xx == 0)
@@ -3777,12 +3801,11 @@ cmd_delete_char(CommandContext &ctx)
 		if (u)
 			gid = u->BeginGroup();
 		(void) gid;
+		const std::size_t xchars = y < rows.size() ? utf8_char_count(line_view(rows[y]), x) : x;
 		for (std::size_t yy = sy; yy <= ey; ++yy) {
 			if (yy >= rows.size())
 				break;
-			std::size_t xx = x;
-			if (xx > rows[yy].size())
-				xx = rows[yy].size();
+			std::size_t xx = utf8_byte_of_char(line_view(rows_of(*buf)[yy]), xchars);
 			std::string deleted;
 			for (int i = 0; i < repeat; ++i) {
 				const auto &rows_view = rows_of(*buf);
@@ -3802,6 +3825,9 @@ cmd_delete_char(CommandContext &ctx)
 		}
 		if (u)
 			u->EndGroup();
+		// Recording each line's undo moved the cursor; deleting forward
+		// leaves it where it was.
+		buf->SetCursor(x, y);
 		buf->SetDirty(true);
 		ensure_cursor_visible(ctx.editor, *buf);
 		return true;

TerminalInputHandler.cc

@@ -358,16 +358,78 @@ map_key_to_command(const int ch,
 }
 
 
+// Read one key: an ncurses KEY_* code, or a character decoded from UTF-8
+// here rather than by get_wch(). With a zero timeout (the frontend reads the
+// rest of a batch without waiting), get_wch() gave up on a character whose
+// bytes had not all arrived yet, and ncurses then discarded the remaining
+// bytes as an invalid sequence: text pasted over a slow link lost
+// characters, and whatever followed them in the same read.
+// Returns false when there is no input.
+static bool
+read_key(int &ch, bool &is_keycode)
+{
+	const int c = getch();
+	if (c == ERR)
+		return false;
+	is_keycode = c > 0xFF;
+	ch         = c;
+	if (is_keycode || c < 0x80)
+		return true;
+
+	// Lead byte of a multibyte character: how many continuation bytes follow.
+	int need;
+	char32_t cp;
+	if (c >= 0xC2 && c <= 0xDF) {
+		need = 1;
+		cp   = static_cast<char32_t>(c & 0x1F);
+	} else if (c >= 0xE0 && c <= 0xEF) {
+		need = 2;
+		cp   = static_cast<char32_t>(c & 0x0F);
+	} else if (c >= 0xF0 && c <= 0xF4) {
+		need = 3;
+		cp   = static_cast<char32_t>(c & 0x07);
+	} else {
+		ch = -1; // stray continuation or invalid byte: ignore it
+		return true;
+	}
+	// The rest of the character is on its way; wait for it (restoring the
+	// caller's timeout), but not forever if it never comes.
+	const int saved_delay = wgetdelay(stdscr);
+	timeout(1000);
+	for (int i = 0; i < need; ++i) {
+		const int b = getch();
+		if (b == ERR || b > 0xFF || (b & 0xC0) != 0x80) {
+			if (b != ERR)
+				ungetch(b); // not part of this character; read it next
+			ch = -1;
+			timeout(saved_delay);
+			return true;
+		}
+		cp = (cp << 6) | static_cast<char32_t>(b & 0x3F);
+	}
+	timeout(saved_delay);
+	// Reject overlong forms and surrogates.
+	if ((need == 2 && cp < 0x800) || (need == 3 && cp < 0x10000) || (cp >= 0xD800 && cp <= 0xDFFF) ||
+	    cp > 0x10FFFF) {
+		ch = -1;
+		return true;
+	}
+	ch = static_cast<int>(cp);
+	return true;
+}
+
+
 bool
 TerminalInputHandler::decode_(MappedInput &out)
 {
-	wint_t wch;
-	int ret = get_wch(&wch);
-	if (ret == ERR) {
+	int ch          = 0;
+	bool is_keycode = false;
+	if (!read_key(ch, is_keycode))
 		return false; // no input
+	if (ch < 0) {
+		out.hasCommand = false; // invalid byte sequence, dropped
+		return true;
 	}
-	const bool is_keycode = (ret == KEY_CODE_YES);
-	const int ch          = static_cast<int>(wch);
 	bool consumed = map_key_to_command(
 		ch, is_keycode,
 		k_prefix_, esc_meta_,

tests/test_audit_regressions.cc

@@ -1400,3 +1400,44 @@ TEST(Audit_VisualYank_CursorStaysInBuffer)
 		(void) h.Exec(CommandId::Undo);
 	ASSERT_EQ(h.Text(), std::string("abc def\n"));
 }
+
+
+// Visual-line edits apply at the cursor's column on every selected line; a
+// byte column taken from an ASCII line fell inside the CJK characters of
+// the next line and split them. The column is now carried in characters.
+TEST(Audit_VisualLine_EditsKeepUtf8Whole)
+{
+	const std::string cjk = "\xE4\xB8\x96\xE7\x95\x8C"; // two 3-byte characters
+	{
+		TestHarness h;
+		h.Buf().insert_text(0, 0, "abcdef\n" + cjk + "\n");
+		h.Buf().SetCursor(0, 1);
+		ASSERT_TRUE(h.Exec(CommandId::VisualLineModeToggle));
+		ASSERT_TRUE(h.Exec(CommandId::MoveUp));
+		ASSERT_TRUE(h.Exec(CommandId::MoveRight));
+		ASSERT_TRUE(h.Exec(CommandId::InsertText, "X"));
+		ASSERT_EQ(h.Text(), std::string("aXbcdef\n\xE4\xB8\x96X\xE7\x95\x8C\n"));
+	}
+	{
+		TestHarness h;
+		h.Buf().insert_text(0, 0, "abcdef\n" + cjk + "\n");
+		h.Buf().SetCursor(0, 1);
+		ASSERT_TRUE(h.Exec(CommandId::VisualLineModeToggle));
+		ASSERT_TRUE(h.Exec(CommandId::MoveUp));
+		ASSERT_TRUE(h.Exec(CommandId::DeleteChar));
+		ASSERT_EQ(h.Text(), std::string("bcdef\n\xE7\x95\x8C\n"));
+		ASSERT_TRUE(h.Exec(CommandId::MoveRight));
+		ASSERT_TRUE(h.Exec(CommandId::Backspace));
+		ASSERT_EQ(h.Text(), std::string("cdef\n\n"));
+	}
+	{
+		TestHarness h;
+		h.Buf().insert_text(0, 0, "abcdef\n" + cjk + "\n");
+		h.Buf().SetCursor(0, 1);
+		ASSERT_TRUE(h.Exec(CommandId::VisualLineModeToggle));
+		ASSERT_TRUE(h.Exec(CommandId::MoveUp));
+		ASSERT_TRUE(h.Exec(CommandId::MoveRight));
+		ASSERT_TRUE(h.Exec(CommandId::Newline));
+		ASSERT_EQ(h.Text(), std::string("a\nbcdef\n\xE4\xB8\x96\n\xE7\x95\x8C\n"));
+	}
+}