loopd: the tag check asks /tokenize twice, not once per tag

Found in the M5b device check on straylight: the self-test asked about
each of the 33 tags and each neutralised form, 66 requests, one
connection each, and inferproxy's token bucket (burst 10) refused the
ninth, so loopd would not start. Now it asks about all the tags at once,
one per line, and then all their neutralised forms: the answer to the
first must hold each tag's recorded token in order, the second none of
them. Each tag is a special token, split out before anything else, so
the lines between cannot change it; and a neutralised form is only
dangerous as a tag's own token, so that is what the second answer must
not hold. The errors name a tag by its id, as before.

The fake tokenizer reads text with several tags in it; the tests cover
each tag as ordinary tokens, as another listed or unlisted token, a
duplicated token in the middle and at the end, a form holding each tag,
and both requests failing. Spec section 8 and the runbook entry follow.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

crates/loopd/tests/selftest_tags.rs

@@ -1,8 +1,10 @@
-//! The self-test's tag check (M5b spec, section 8): each of the model's tags must be one token,
-//! the one recorded with it, and its neutralised form more than one, or `loopd` does not start.
-//! Against the fake server's tokenizer, with one answer changed at a time. The tags come from
+//! The self-test's tag check (M5b spec, section 8): asked about all the model's tags at once,
+//! the server must give each one's recorded token, in order; asked about all their neutralised
+//! forms, none of those tokens. Two `/tokenize` requests, so that the check fits in `inferproxy`'s
+//! burst of connections (found on straylight in the M5b device check). Against the fake server's
+//! tokenizer, with one tag's answer changed at a time. The tags come from
 //! `bxcore::sanitize::TAGS`, and no assertion prints one (tip T33): a failure names the tag by its
-//! id, and the tests check that it does. Do not edit.
+//! id, and the tests check that it does.
 
 mod support;
 #[path = "support/tags.rs"]
@@ -15,7 +17,7 @@ use std::time::{Duration, Instant};
 use bxcore::sanitize::{TAGS, neutralise};
 use loopd::llama::Client;
 use loopd::selftest::{SelfTestError, run};
-use support::{FakeServer, Home, Recorded, Reply, tokenizer, tokens};
+use support::{FakeServer, Home, Recorded, Reply, tokenize, tokenizer, tokens};
 use tags::{holds_a_tag, same};
 
 const CHAT: &str = "/v1/chat/completions";
@@ -45,15 +47,33 @@ fn content(request: &Recorded) -> String {
     body["content"].as_str().unwrap_or_default().to_string()
 }
 
-/// The fake tokenizer, except that `text` is answered with `ids`.
-fn except(text: &str, ids: Vec<u32>) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static {
-    let text = text.to_string();
+/// The texts the check asks about: every tag, one per line; then every neutralised form.
+fn asked_texts() -> [String; 2] {
+    let tags: Vec<&str> = TAGS.iter().map(|t| t.text).collect();
+    let forms: Vec<String> = TAGS
+        .iter()
+        .map(|t| neutralise(t.text).into_owned())
+        .collect();
+    [tags.join("\n"), forms.join("\n")]
+}
+
+/// The fake tokenizer, except that the tag `text` is answered with `ids` wherever it appears.
+fn except(
+    text: &'static str,
+    ids: Vec<u32>,
+) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static {
+    move |request: &Recorded| tokens(&tokenize(&content(request), &[(text, ids.clone())]))
+}
+
+/// The fake tokenizer, except that the neutralised forms' answer also holds `id`.
+fn forms_holding(id: u32) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static {
     move |request: &Recorded| {
-        if content(request) == text {
-            tokens(&ids)
-        } else {
-            tokenizer(request)
+        let text = content(request);
+        let mut ids = tokenize(&text, &[]);
+        if text == asked_texts()[1] {
+            ids.insert(ids.len() / 2, id);
         }
+        tokens(&ids)
     }
 }
 
@@ -65,20 +85,27 @@ fn run_on(server: &FakeServer) -> (Result<(), SelfTestError>, Vec<String>) {
     (result, steps)
 }
 
-/// A failure of the tag check stops the self-test before any prompt, names the tag by its id,
-/// holds no tag, and points at the tags' own runbook entry.
+/// A failure of the tag check stops the self-test before any prompt, holds no tag, names `id` if
+/// given, and points at the tags' own runbook entry.
 #[track_caller]
-fn stopped_before_any_prompt(server: &FakeServer, e: &SelfTestError, steps: &[String], id: u32) {
+fn stopped_before_any_prompt(
+    server: &FakeServer,
+    e: &SelfTestError,
+    steps: &[String],
+    id: Option<u32>,
+) {
     let text = e.to_string();
-    assert!(!holds_a_tag(&text), "tag {id}: the message holds a tag");
-    assert!(text.contains(&id.to_string()), "tag {id}: {text}");
-    assert_eq!(e.pointer(), POINTER, "tag {id}");
-    assert_eq!(steps.len(), 2, "tag {id}: {steps:?}");
-    assert!(server.requests_to(CHAT).is_empty(), "tag {id}: no prompt");
+    assert!(!holds_a_tag(&text), "{id:?}: the message holds a tag");
+    if let Some(id) = id {
+        assert!(text.contains(&id.to_string()), "tag {id}: {text}");
+    }
+    assert_eq!(e.pointer(), POINTER, "{id:?}");
+    assert_eq!(steps.len(), 2, "{id:?}: {steps:?}");
+    assert!(server.requests_to(CHAT).is_empty(), "{id:?}: no prompt");
 }
 
 #[test]
-fn every_tag_and_its_neutralised_form_tokenize_as_recorded() {
+fn the_tags_then_their_neutralised_forms_are_asked_in_two_requests() {
     let server = server_with(tokenizer);
     let (result, steps) = run_on(&server);
     assert!(result.is_ok(), "{result:?}");
@@ -95,30 +122,27 @@ fn every_tag_and_its_neutralised_form_tokenize_as_recorded() {
     let asked = server.requests_to(TOKENIZE);
     assert_eq!(
         asked.len(),
-        2 * TAGS.len(),
-        "one request per tag and per form"
+        2,
+        "one request for the tags, one for the forms"
+    );
+    let got: Vec<String> = asked.iter().map(content).collect();
+    same(
+        &got,
+        &asked_texts().to_vec(),
+        "the texts asked about, in order",
     );
-    let mut want: Vec<String> = TAGS
-        .iter()
-        .flat_map(|t| [t.text.to_string(), neutralise(t.text).into_owned()])
-        .collect();
-    let mut got: Vec<String> = asked.iter().map(content).collect();
-    want.sort();
-    got.sort();
-    same(&got, &want, "the texts asked about");
     for (i, request) in asked.iter().enumerate() {
         assert_eq!(request.method, "POST", "request {i}");
-        let body = request.json();
         let fields = serde_json::json!({
             "model": "test-model",
             "content": content(request),
             "add_special": false,
             "parse_special": true,
         });
-        same(&body, &fields, &format!("request {i}: the body"));
+        same(&request.json(), &fields, &format!("request {i}: the body"));
     }
 
-    // Every tag is checked before the first prompt is sent.
+    // Both are asked before the first prompt is sent.
     let paths: Vec<String> = server
         .requests()
         .iter()
@@ -130,38 +154,59 @@ fn every_tag_and_its_neutralised_form_tokenize_as_recorded() {
 }
 
 #[test]
-fn a_tag_that_is_not_one_token_fails() {
-    for (i, tag) in TAGS.iter().enumerate() {
-        // Six ordinary tokens, as a tag the tokenizer no longer knows would be; none for the
-        // first tag.
-        let ids = if i == 0 {
-            Vec::new()
-        } else {
-            vec![27, 91, 316, 6018, 91, 29]
-        };
-        let count = ids.len();
-        let server = server_with(except(tag.text, ids));
-        let (result, steps) = run_on(&server);
-        match &result {
-            Err(e @ SelfTestError::TagNotOneToken { id, count: got }) => {
-                assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id);
-                stopped_before_any_prompt(&server, e, &steps, tag.id);
-            }
-            other => panic!("tag {}: {other:?}", tag.id),
-        }
-    }
-}
-
-#[test]
-fn a_tag_that_is_another_token_fails() {
+fn a_tag_that_is_ordinary_tokens_fails_and_is_named() {
     for tag in TAGS {
-        let other = tag.id + 1;
-        let server = server_with(except(tag.text, vec![other]));
+        // Six ordinary tokens, as a tag the tokenizer no longer knows would be.
+        let server = server_with(except(tag.text, vec![27, 91, 316, 6018, 91, 29]));
         let (result, steps) = run_on(&server);
         match &result {
-            Err(e @ SelfTestError::TagOtherId { id, got }) => {
-                assert_eq!((*id, *got), (tag.id, other), "tag {}", tag.id);
-                stopped_before_any_prompt(&server, e, &steps, tag.id);
+            Err(e @ SelfTestError::TagNotRecorded { id }) => {
+                assert_eq!(*id, tag.id, "tag {}", tag.id);
+                stopped_before_any_prompt(&server, e, &steps, Some(tag.id));
+            }
+            other => panic!("tag {}: {other:?}", tag.id),
+        }
+        assert_eq!(
+            server.requests_to(TOKENIZE).len(),
+            1,
+            "tag {}: the forms are not asked",
+            tag.id
+        );
+    }
+}
+
+#[test]
+fn a_tag_that_is_another_token_fails_and_is_named() {
+    for tag in TAGS {
+        // Another tag's token, and a token outside the list.
+        let listed = if tag.id == TAGS[0].id {
+            TAGS[1].id
+        } else {
+            TAGS[0].id
+        };
+        for other in [listed, 7] {
+            let server = server_with(except(tag.text, vec![other]));
+            let (result, steps) = run_on(&server);
+            match &result {
+                Err(e @ SelfTestError::TagNotRecorded { id }) => {
+                    assert_eq!(*id, tag.id, "tag {} as {other}", tag.id);
+                    stopped_before_any_prompt(&server, e, &steps, Some(tag.id));
+                }
+                other_result => panic!("tag {} as {other}: {other_result:?}", tag.id),
+            }
+        }
+    }
+}
+
+#[test]
+fn a_tag_that_is_its_token_twice_fails() {
+    // A middle tag shifts every later place; the last one leaves an extra token at the end.
+    for tag in [&TAGS[4], &TAGS[TAGS.len() - 1]] {
+        let server = server_with(except(tag.text, vec![tag.id, tag.id]));
+        let (result, steps) = run_on(&server);
+        match &result {
+            Err(e @ SelfTestError::TagNotRecorded { .. }) => {
+                stopped_before_any_prompt(&server, e, &steps, None);
             }
             other => panic!("tag {}: {other:?}", tag.id),
         }
@@ -169,18 +214,14 @@ fn a_tag_that_is_another_token_fails() {
 }
 
 #[test]
-fn a_neutralised_form_that_is_one_token_or_none_fails() {
-    for (i, tag) in TAGS.iter().enumerate() {
-        // The tag's own token, as a tokenizer that saw through the escape would give; none for
-        // the first tag.
-        let ids = if i == 0 { Vec::new() } else { vec![tag.id] };
-        let count = ids.len();
-        let server = server_with(except(&neutralise(tag.text), ids));
+fn a_neutralised_form_that_holds_a_tag_fails_and_names_it() {
+    for tag in TAGS {
+        let server = server_with(forms_holding(tag.id));
         let (result, steps) = run_on(&server);
         match &result {
-            Err(e @ SelfTestError::NeutralisedTooFew { id, count: got }) => {
-                assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id);
-                stopped_before_any_prompt(&server, e, &steps, tag.id);
+            Err(e @ SelfTestError::NeutralisedIsTag { id }) => {
+                assert_eq!(*id, tag.id, "tag {}", tag.id);
+                stopped_before_any_prompt(&server, e, &steps, Some(tag.id));
             }
             other => panic!("tag {}: {other:?}", tag.id),
         }
@@ -188,9 +229,16 @@ fn a_neutralised_form_that_is_one_token_or_none_fails() {
 }
 
 #[test]
-fn a_neutralised_form_of_two_tokens_passes() {
-    // "More than one" is the rule; two is enough.
-    let server = server_with(except(&neutralise(TAGS[0].text), vec![5, 6]));
+fn neutralised_forms_of_ordinary_tokens_pass_however_few() {
+    // Only a tag's own token is dangerous; a form the tokenizer happens to know as one ordinary
+    // token is not.
+    let server = server_with(move |request: &Recorded| {
+        if content(request) == asked_texts()[1] {
+            tokens(&[5])
+        } else {
+            tokenizer(request)
+        }
+    });
     let (result, steps) = run_on(&server);
     assert!(result.is_ok(), "{result:?}");
     assert_eq!(steps.len(), 4);
@@ -198,7 +246,6 @@ fn a_neutralised_form_of_two_tokens_passes() {
 
 #[test]
 fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() {
-    let first = TAGS[0].id;
     let cases: [(&str, Option<Reply>); 5] = [
         ("no /tokenize at all", None),
         (
@@ -230,9 +277,12 @@ fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() {
         }
         let (result, steps) = run_on(&server);
         match &result {
-            Err(e @ SelfTestError::Tokenize { id, .. }) => {
-                assert_eq!(*id, first, "{what}");
-                stopped_before_any_prompt(&server, e, &steps, first);
+            Err(e @ SelfTestError::Tokenize { .. }) => {
+                assert!(
+                    e.to_string().starts_with("could not tokenize the tags: "),
+                    "{what}: {e}"
+                );
+                stopped_before_any_prompt(&server, e, &steps, None);
             }
             other => panic!("{what}: {other:?}"),
         }
@@ -240,6 +290,32 @@ fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() {
     }
 }
 
+#[test]
+fn a_tokenizer_that_fails_on_the_forms_fails_closed() {
+    let server = FakeServer::start();
+    server.route("/props", vec![Reply::fixture("props")]);
+    server.route(CHAT, vec![Reply::fixture("tool_call")]);
+    server.respond(TOKENIZE, |request: &Recorded| {
+        if content(request) == asked_texts()[0] {
+            tokenizer(request)
+        } else {
+            Reply::json(500, r#"{"error":{"code":500,"message":"boom"}}"#)
+        }
+    });
+    let (result, steps) = run_on(&server);
+    match &result {
+        Err(e @ SelfTestError::Tokenize { .. }) => {
+            assert!(
+                e.to_string()
+                    .starts_with("could not tokenize the neutralised tags: "),
+                "{e}"
+            );
+            stopped_before_any_prompt(&server, e, &steps, None);
+        }
+        other => panic!("{other:?}"),
+    }
+}
+
 #[test]
 fn the_other_failures_keep_the_self_tests_entry() {
     let e = SelfTestError::Hash;

crates/loopd/tests/serve.rs

@@ -85,8 +85,8 @@ fn serve_runs_the_self_test_then_binds_the_socket_with_mode_0600() {
     );
     assert_eq!(
         server.requests_to("/tokenize").len(),
-        2 * bxcore::sanitize::TAGS.len(),
-        "the tag check ran: each tag and its neutralised form"
+        2,
+        "the tag check ran: the tags, then their neutralised forms"
     );
 }
 

crates/loopd/tests/support/mod.rs

@@ -180,11 +180,36 @@ pub fn tokenizer(request: &Recorded) -> Reply {
     let Some(content) = body.get("content").and_then(|c| c.as_str()) else {
         return Reply::json(400, r#"{"error":"no content"}"#);
     };
-    let ids: Vec<u32> = match bxcore::sanitize::TAGS.iter().find(|t| t.text == content) {
-        Some(tag) => vec![tag.id],
-        None => content.bytes().map(u32::from).collect(),
-    };
-    tokens(&ids)
+    tokens(&tokenize(content, &[]))
+}
+
+/// The fake tokenizer's ids for `text`: the id of each listed tag in it, the longest where two
+/// start at the same place, and one id per byte for everything else. A tag whose text is a key of
+/// `overrides` gets that entry's ids instead.
+pub fn tokenize(text: &str, overrides: &[(&str, Vec<u32>)]) -> Vec<u32> {
+    let mut ids = Vec::new();
+    let mut rest = text;
+    while let Some(c) = rest.chars().next() {
+        let tag = bxcore::sanitize::TAGS
+            .iter()
+            .filter(|t| rest.starts_with(t.text))
+            .max_by_key(|t| t.text.len());
+        let len = match tag {
+            Some(tag) => {
+                match overrides.iter().find(|(text, _)| *text == tag.text) {
+                    Some((_, instead)) => ids.extend(instead),
+                    None => ids.push(tag.id),
+                }
+                tag.text.len()
+            }
+            None => {
+                ids.extend(c.to_string().bytes().map(u32::from));
+                c.len_utf8()
+            }
+        };
+        rest = &rest[len..];
+    }
+    ids
 }
 
 /// A `/tokenize` answer as llama-server gives one without `with_pieces`: `{"tokens": [ids]}`.