bxcore: the model's tags and sanitize::neutralise

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>
2026-09-27 19:08:42 -07:00
parent f594e0d9d4
commit 1817e7f117
7 changed files with 769 additions and 0 deletions
@@ -13,6 +13,7 @@ pub mod hosts;
 pub mod ids;
 pub mod log;
 pub mod memory;
+pub mod sanitize;
 #[cfg(feature = "secrets")]
 pub mod secrets;
 pub mod sha1;
@@ -0,0 +1,6 @@
+//! The sanitizer (M5b spec, section 6): what is done to text from outside before it reaches the
+//! model or memory. Neutralising the model's chat-template tags (6.2) is in `tags`.
+
+mod tags;
+
+pub use tags::{TAGS, Tag, neutralise};
@@ -0,0 +1,133 @@
+//! The model's chat-template and control tags, and `neutralise` (M5b spec, section 6.2).
+//!
+//! Inside any message, a tag becomes the model's own token: text holding one can end its turn and
+//! forge another (`docs/research/2026-09-24-chat-template-tokens.md`). `neutralise` writes each
+//! tag's angle brackets as `&lt;` and `&gt;`, which the model still reads as the tag's name and
+//! the tokenizer reads as ordinary characters.
+//!
+//! The list was read on straylight on 2026-09-26 from the model file the router loads for
+//! `ornith-1.5-35b-a3b` (`Ornith-1.5-35B-Q4_K_M.gguf`, snapshot `12393612fd4f`, sha256
+//! `42739874cc2c…`), llama-server b10964-b29c606: every vocabulary entry of type control (3) or
+//! user-defined (4), 33 in all, each a tag. Each was checked with the server's `/tokenize`: exactly
+//! one token, with the id below. The 243 entries of type unused are not parsed as one token and
+//! are not listed.
+
+use std::borrow::Cow;
+
+// Every tag is built from pieces, so that this file never holds one whole (tip T33): a model
+// reading the file would take a whole tag for its own token.
+macro_rules! bar {
+    ($name:literal) => {
+        concat!("<", "|", $name, "|", ">")
+    };
+}
+
+macro_rules! open {
+    ($name:literal) => {
+        concat!("<", $name, ">")
+    };
+}
+
+macro_rules! close {
+    ($name:literal) => {
+        concat!("<", "/", $name, ">")
+    };
+}
+
+/// One of the model's template or control tags.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Tag {
+    /// The tag as the vocabulary spells it: one `<` first, one `>` last, ASCII.
+    pub text: &'static str,
+    /// Its token id in the model's vocabulary.
+    pub id: u32,
+}
+
+const fn tag(text: &'static str, id: u32) -> Tag {
+    Tag { text, id }
+}
+
+/// The tags, in id order. No tag holds another, so the order does not change what matches.
+pub const TAGS: &[Tag] = &[
+    tag(bar!("endoftext"), 248044),
+    tag(bar!("im_start"), 248045),
+    tag(bar!("im_end"), 248046),
+    tag(bar!("object_ref_start"), 248047),
+    tag(bar!("object_ref_end"), 248048),
+    tag(bar!("box_start"), 248049),
+    tag(bar!("box_end"), 248050),
+    tag(bar!("quad_start"), 248051),
+    tag(bar!("quad_end"), 248052),
+    tag(bar!("vision_start"), 248053),
+    tag(bar!("vision_end"), 248054),
+    tag(bar!("vision_pad"), 248055),
+    tag(bar!("image_pad"), 248056),
+    tag(bar!("video_pad"), 248057),
+    tag(open!("tool_call"), 248058),
+    tag(close!("tool_call"), 248059),
+    tag(bar!("fim_prefix"), 248060),
+    tag(bar!("fim_middle"), 248061),
+    tag(bar!("fim_suffix"), 248062),
+    tag(bar!("fim_pad"), 248063),
+    tag(bar!("repo_name"), 248064),
+    tag(bar!("file_sep"), 248065),
+    tag(open!("tool_response"), 248066),
+    tag(close!("tool_response"), 248067),
+    tag(open!("think"), 248068),
+    tag(close!("think"), 248069),
+    tag(bar!("audio_start"), 248070),
+    tag(bar!("audio_end"), 248071),
+    tag(open!("tts_pad"), 248072),
+    tag(open!("tts_text_bos"), 248073),
+    tag(open!("tts_text_eod"), 248074),
+    tag(open!("tts_text_bos_single"), 248075),
+    tag(bar!("audio_pad"), 248076),
+];
+
+/// `text` with each listed tag's angle brackets written as `&lt;` and `&gt;`, everywhere it
+/// occurs. Matching is exact and case-sensitive, and finds a tag inside a word too: that is what
+/// the tokenizer does (measured, 2026-09-26). Text with no listed tag is returned borrowed.
+pub fn neutralise(text: &str) -> Cow<'_, str> {
+    let Some((before, tag, mut rest)) = next_tag(text) else {
+        return Cow::Borrowed(text);
+    };
+    let mut out = String::with_capacity(text.len() + 16);
+    out.push_str(before);
+    push_escaped(&mut out, tag);
+    while let Some((before, tag, after)) = next_tag(rest) {
+        out.push_str(before);
+        push_escaped(&mut out, tag);
+        rest = after;
+    }
+    out.push_str(rest);
+    Cow::Owned(out)
+}
+
+/// The first listed tag in `text`: the text before it, the tag, and the text after it.
+fn next_tag(text: &str) -> Option<(&str, &'static Tag, &str)> {
+    // For each `(at, _)` of `text.match_indices('<')`: `let (before, from) =
+    // text.split_at_checked(at)?;`, then `TAGS.iter().find(|tag| from.starts_with(tag.text))`.
+    // If it finds one: `let after = from.get(tag.text.len()..)?;` and return
+    // `Some((before, tag, after))`. After the loop, `None`.
+    for (at, _) in text.match_indices('<') {
+        let (before, from) = text.split_at_checked(at)?;
+        if let Some(tag) = TAGS.iter().find(|tag| from.starts_with(tag.text)) {
+            let after = from.get(tag.text.len()..)?;
+            return Some((before, tag, after));
+        }
+    }
+    None
+}
+
+/// Appends `tag.text` to `out` with `<` written as `&lt;` and `>` as `&gt;`.
+fn push_escaped(out: &mut String, tag: &Tag) {
+    // For each `c` of `tag.text.chars()`, a `match c`: `'<'` pushes the string "&lt;", `'>'`
+    // pushes "&gt;", and any other character is pushed as it is (`out.push(c)`).
+    for c in tag.text.chars() {
+        match c {
+            '<' => out.push_str("&lt;"),
+            '>' => out.push_str("&gt;"),
+            other => out.push(other),
+        }
+    }
+}