bxcore: the model's tags and sanitize::neutralise

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>
2026-09-27 19:08:42 -07:00
parent f594e0d9d4
commit 1817e7f117
7 changed files with 769 additions and 0 deletions
@@ -0,0 +1,85 @@
+//! The model's chat-template tags for the sanitizer's tests, built from their names so that no
+//! test file holds one whole (tip T33), and a naive oracle for `neutralise`. No assertion prints
+//! a tag: a failing case is named, never shown. Do not edit.
+
+#![allow(dead_code)] // each test file uses a different part of this module
+
+/// An angle-bar tag: a less-than sign and a vertical bar, the name, a vertical bar and a
+/// greater-than sign.
+pub fn bar(name: &str) -> String {
+    ["<", "|", name, "|", ">"].concat()
+}
+
+/// An opening angle-bracket tag: a less-than sign, the name, a greater-than sign.
+pub fn open(name: &str) -> String {
+    ["<", name, ">"].concat()
+}
+
+/// A closing angle-bracket tag: a less-than sign and a slash, the name, a greater-than sign.
+pub fn close(name: &str) -> String {
+    ["<", "/", name, ">"].concat()
+}
+
+/// `text` with every angle bracket written as its HTML entity: a neutralised tag.
+pub fn escaped(text: &str) -> String {
+    text.replace('<', "&lt;").replace('>', "&gt;")
+}
+
+/// The tags measured on straylight on 2026-09-26, with their token ids, in id order: every
+/// vocabulary entry of type control or user-defined in the model file, each checked with the
+/// server's `/tokenize` to be exactly one token with this id.
+pub fn measured() -> Vec<(String, u32)> {
+    vec![
+        (bar("endoftext"), 248044),
+        (bar("im_start"), 248045),
+        (bar("im_end"), 248046),
+        (bar("object_ref_start"), 248047),
+        (bar("object_ref_end"), 248048),
+        (bar("box_start"), 248049),
+        (bar("box_end"), 248050),
+        (bar("quad_start"), 248051),
+        (bar("quad_end"), 248052),
+        (bar("vision_start"), 248053),
+        (bar("vision_end"), 248054),
+        (bar("vision_pad"), 248055),
+        (bar("image_pad"), 248056),
+        (bar("video_pad"), 248057),
+        (open("tool_call"), 248058),
+        (close("tool_call"), 248059),
+        (bar("fim_prefix"), 248060),
+        (bar("fim_middle"), 248061),
+        (bar("fim_suffix"), 248062),
+        (bar("fim_pad"), 248063),
+        (bar("repo_name"), 248064),
+        (bar("file_sep"), 248065),
+        (open("tool_response"), 248066),
+        (close("tool_response"), 248067),
+        (open("think"), 248068),
+        (close("think"), 248069),
+        (bar("audio_start"), 248070),
+        (bar("audio_end"), 248071),
+        (open("tts_pad"), 248072),
+        (open("tts_text_bos"), 248073),
+        (open("tts_text_eod"), 248074),
+        (open("tts_text_bos_single"), 248075),
+        (bar("audio_pad"), 248076),
+    ]
+}
+
+/// What `neutralise` must return, computed the slow way: each measured tag replaced everywhere by
+/// its escaped form. The order does not matter: no tag holds another, and no escaped form holds
+/// an angle bracket.
+pub fn oracle(text: &str) -> String {
+    let mut out = text.to_string();
+    for (tag, _) in measured() {
+        out = out.replace(&tag, &escaped(&tag));
+    }
+    out
+}
+
+/// Whether `text` still holds any measured tag whole.
+pub fn holds_a_tag(text: &str) -> bool {
+    measured()
+        .iter()
+        .any(|(tag, _)| text.contains(tag.as_str()))
+}