loopd: the self-test checks the model's tags against the tokenizer

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/loopd/src/llama/info.rs

@@ -25,6 +25,12 @@ pub struct SlotInfo {
     pub is_processing: bool,
 }
 
+/// `/tokenize`'s answer without `with_pieces`: `{"tokens": [ids]}`. Other fields are ignored.
+#[derive(serde::Deserialize)]
+struct TokenIds {
+    tokens: Vec<u32>,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum CacheOutcome {
     Hit,
@@ -143,6 +149,23 @@ impl Client {
             .ok_or_else(|| InferError::Protocol("tokenize response has no tokens".to_string()))
     }
 
+    /// The token ids the server gives `text`, with special tokens parsed as a chat prompt's are
+    /// and no BOS added. The self-test checks the model's tags with it (M5b spec, section 8).
+    pub fn token_ids(&self, text: &str) -> Result<Vec<u32>, InferError> {
+        let body = serde_json::to_string(&serde_json::json!({
+            "model": self.cfg.infer.model,
+            "content": text,
+            "add_special": false,
+            "parse_special": true,
+        }))
+        .map_err(|_| InferError::Protocol("could not encode tokenize request".to_string()))?;
+        let bytes = self.call("POST", "/tokenize", Some(body.as_bytes()))?;
+        let ids: TokenIds = serde_json::from_slice(&bytes).map_err(|_| {
+            InferError::Protocol("tokenize response has no list of token ids".to_string())
+        })?;
+        Ok(ids.tokens)
+    }
+
     pub(crate) fn end_reasoning(&self, completion_id: &str) -> Result<bool, InferError> {
         let body = serde_json::to_string(&serde_json::json!({
             "id": completion_id,

crates/loopd/src/main.rs

@@ -42,7 +42,7 @@ fn run_selftest_check(client: &Client) -> Result<(), SelfTestError> {
             Ok(())
         }
         Err(e) => {
-            eprintln!("selftest: FAILED: {e}; see docs/runbook.md#loopd-selftest-failed");
+            eprintln!("selftest: FAILED: {e}; {}", e.pointer());
             Err(e)
         }
     }

crates/loopd/src/selftest.rs

@@ -1,6 +1,9 @@
 //! The startup self-test: before `loopd` serves anyone, it checks that the server is the one its
-//! config describes, that a tool call comes back parsed, and that a second turn reuses the first
-//! turn's cache. Any check failing means `loopd` does not start.
+//! config describes, that the model's tags tokenize as recorded, that a tool call comes back
+//! parsed, and that a second turn reuses the first turn's cache. Any check failing means `loopd`
+//! does not start.
+
+use bxcore::sanitize::{TAGS, Tag, neutralise};
 
 use crate::llama::{
     ChatMessage, ChatRequest, Client, FinishReason, InferError, ToolSchema,
@@ -21,6 +24,45 @@ pub enum SelfTestError {
     },
     Infer(InferError),
     Hash,
+    /// The tag with the recorded id `id` is `count` tokens, not one.
+    TagNotOneToken {
+        id: u32,
+        count: usize,
+    },
+    /// The tag with the recorded id `id` is one token, but the token `got`.
+    TagOtherId {
+        id: u32,
+        got: u32,
+    },
+    /// The neutralised form of the tag with the recorded id `id` is `count` tokens, not more
+    /// than one.
+    NeutralisedTooFew {
+        id: u32,
+        count: usize,
+    },
+    /// `/tokenize` failed for the tag with the recorded id `id`, or for its neutralised form.
+    Tokenize {
+        id: u32,
+        error: InferError,
+    },
+}
+
+impl SelfTestError {
+    /// The runbook pointer that ends the line `loopd` prints for this failure. A tag failure has
+    /// its own entry; every other failure shares the self-test's.
+    pub fn pointer(&self) -> &'static str {
+        match self {
+            SelfTestError::TagNotOneToken { .. }
+            | SelfTestError::TagOtherId { .. }
+            | SelfTestError::NeutralisedTooFew { .. }
+            | SelfTestError::Tokenize { .. } => "see docs/runbook.md#loopd-tags-unexpected",
+            SelfTestError::Mismatch { .. }
+            | SelfTestError::ToolCall(_)
+            | SelfTestError::CacheMiss { .. }
+            | SelfTestError::Infer(_)
+            | SelfTestError::Hash => "see docs/runbook.md#loopd-selftest-failed",
+        }
+    }
 }
 
 impl std::fmt::Display for SelfTestError {
@@ -40,6 +82,19 @@ impl std::fmt::Display for SelfTestError {
             ),
             SelfTestError::Infer(e) => write!(f, "{e}"),
             SelfTestError::Hash => write!(f, "the chat template could not be hashed"),
+            SelfTestError::TagNotOneToken { id, count } => {
+                write!(f, "the tag with id {id} is {count} tokens, not one")
+            }
+            SelfTestError::TagOtherId { id, got } => {
+                write!(f, "the tag with id {id} is one token, but the token {got}")
+            }
+            SelfTestError::NeutralisedTooFew { id, count } => write!(
+                f,
+                "the neutralised form of the tag with id {id} does not split into several tokens (the server gave {count})"
+            ),
+            SelfTestError::Tokenize { id, error } => {
+                write!(f, "could not tokenize the tag with id {id}: {error}")
+            }
         }
     }
 }
@@ -54,11 +109,14 @@ impl From<InferError> for SelfTestError {
 
 const SYSTEM: &str = "You are Boxmaker, a careful personal agent.";
 
-/// Runs the three checks in order. `on_step` is told the name of each check as it starts.
+/// Runs the four checks in order. `on_step` is told the name of each check as it starts.
 pub fn run(client: &Client, on_step: &mut dyn FnMut(&str)) -> Result<(), SelfTestError> {
     on_step("server matches config");
     server_matches(client)?;
 
+    on_step("tags tokenize as recorded");
+    tags_tokenize(client)?;
+
     on_step("tool call round trip");
     tool_call_round_trip(client)?;
 
@@ -103,7 +161,45 @@ fn server_matches(client: &Client) -> Result<(), SelfTestError> {
     Ok(())
 }
 
-/// Check 2: a tool call comes back parsed, with a `read_file` tool and a string `path`.
+/// Check 2: each of the model's tags (`bxcore::sanitize::TAGS`) is one token, the one recorded
+/// with it, and its neutralised form is more than one (M5b spec, section 8). Like check 1 it sends
+/// no prompt and uses no slot, so a server whose tokenizer is not the recorded one is sent no
+/// prompt either. A tag missing from the list cannot be found: the server does not list them.
+fn tags_tokenize(client: &Client) -> Result<(), SelfTestError> {
+    for tag in TAGS {
+        check_tag(client, tag)?;
+    }
+    Ok(())
+}
+
+/// Two `/tokenize` requests: `tag.text`, then `neutralise(tag.text)`. An error names the tag by
+/// its id, never by its text, which the operator's terminal and log would show whole.
+fn check_tag(client: &Client, tag: &Tag) -> Result<(), SelfTestError> {
+    let id = tag.id;
+    let tokenize = |text: &str| {
+        client
+            .token_ids(text)
+            .map_err(|error| SelfTestError::Tokenize { id, error })
+    };
+    let ids = tokenize(tag.text)?;
+    match ids.as_slice() {
+        [got] if *got == id => {}
+        [got] => return Err(SelfTestError::TagOtherId { id, got: *got }),
+        _ => {
+            return Err(SelfTestError::TagNotOneToken {
+                id,
+                count: ids.len(),
+            });
+        }
+    }
+    let count = tokenize(&neutralise(tag.text))?.len();
+    if count <= 1 {
+        return Err(SelfTestError::NeutralisedTooFew { id, count });
+    }
+    Ok(())
+}
+
+/// Check 3: a tool call comes back parsed, with a `read_file` tool and a string `path`.
 fn tool_call_round_trip(client: &Client) -> Result<(), SelfTestError> {
     let req = ChatRequest {
         slot: client.config().slots.main,
@@ -159,7 +255,7 @@ fn tool_call_round_trip(client: &Client) -> Result<(), SelfTestError> {
     Ok(())
 }
 
-/// Check 3: a second turn that extends the first reuses the first turn's cache.
+/// Check 4: a second turn that extends the first reuses the first turn's cache.
 fn cache_hit(client: &Client) -> Result<(), SelfTestError> {
     let first = ChatRequest {
         slot: client.config().slots.main,

crates/loopd/tests/pointers.rs

@@ -14,7 +14,7 @@ use loopd::broker_port::{not_configured_line, unavailable_line};
 use loopd::memory::{MemoryPort, MemorySource};
 use loopd::session::{Session, SessionError};
 use loopd::tools::Registry;
-use support::{FakeServer, Home, Reply};
+use support::{FakeServer, Home, Reply, tokenizer};
 
 // `want` is the whole pointer, written out: scripts/check-runbook.sh reads the anchors of every
 // pointer in the source, and cannot read one built with `format!`.
@@ -150,6 +150,7 @@ home = "{}"
 
 fn healthy_routes(server: &FakeServer) {
     server.route("/props", vec![Reply::fixture("props")]);
+    server.respond("/tokenize", tokenizer);
     server.route(
         "/v1/chat/completions",
         vec![

crates/loopd/tests/selftest.rs

@@ -4,17 +4,26 @@ mod support;
 
 use loopd::llama::Client;
 use loopd::selftest::{SelfTestError, run};
-use support::{FakeServer, Reply};
+use support::{FakeServer, Reply, tokenizer};
 
 const CHAT: &str = "/v1/chat/completions";
+const TOKENIZE: &str = "/tokenize";
 
 /// A change to the config before the self-test runs.
 type Edit = fn(&mut loopd::config::Config);
 
-/// A server that passes: the recorded props, then the three recorded completions in order.
-fn healthy() -> FakeServer {
+/// The recorded props and the fake tokenizer, and no completion yet.
+fn props_and_tags() -> FakeServer {
     let server = FakeServer::start();
     server.route("/props", vec![Reply::fixture("props")]);
+    server.respond(TOKENIZE, tokenizer);
+    server
+}
+
+/// A server that passes: the recorded props, the fake tokenizer, then the three recorded
+/// completions in order.
+fn healthy() -> FakeServer {
+    let server = props_and_tags();
     server.route(
         CHAT,
         vec![
@@ -38,7 +47,7 @@ fn run_with(
 }
 
 #[test]
-fn a_healthy_server_passes_all_three_checks_in_order() {
+fn a_healthy_server_passes_all_four_checks_in_order() {
     let server = healthy();
     let (result, steps) = run_with(&server, |_| {});
     assert!(result.is_ok(), "{result:?}");
@@ -46,6 +55,7 @@ fn a_healthy_server_passes_all_three_checks_in_order() {
         steps,
         [
             "server matches config",
+            "tags tokenize as recorded",
             "tool call round trip",
             "turn 2 cache hit"
         ]
@@ -115,27 +125,26 @@ fn each_expected_value_is_checked() {
             "a server that is not the expected one is not sent any prompt"
         );
         assert!(server.requests_to(CHAT).is_empty());
+        assert!(server.requests_to(TOKENIZE).is_empty());
     }
 }
 
 #[test]
-fn a_completion_without_the_tool_call_fails_the_second_check() {
-    let server = FakeServer::start();
-    server.route("/props", vec![Reply::fixture("props")]);
+fn a_completion_without_the_tool_call_fails_the_tool_call_check() {
+    let server = props_and_tags();
     server.route(CHAT, vec![Reply::fixture("plain")]);
     let (result, steps) = run_with(&server, |_| {});
     assert!(
         matches!(result, Err(SelfTestError::ToolCall(_))),
         "{result:?}"
     );
-    assert_eq!(steps.len(), 2);
+    assert_eq!(steps.len(), 3);
 }
 
 #[test]
-fn a_cold_second_turn_fails_the_third_check() {
+fn a_cold_second_turn_fails_the_cache_check() {
     // turn1 twice: the "second turn" reuses 15 tokens where 46 were left.
-    let server = FakeServer::start();
-    server.route("/props", vec![Reply::fixture("props")]);
+    let server = props_and_tags();
     server.route(
         CHAT,
         vec![
@@ -149,13 +158,12 @@ fn a_cold_second_turn_fails_the_third_check() {
         Err(SelfTestError::CacheMiss { expected, got }) => assert_eq!((expected, got), (7052, 15)),
         other => panic!("{other:?}"),
     }
-    assert_eq!(steps.len(), 3);
+    assert_eq!(steps.len(), 4);
 }
 
 #[test]
 fn a_server_that_is_down_is_an_inference_error_after_retries() {
-    let server = FakeServer::start();
-    server.route("/props", vec![Reply::fixture("props")]);
+    let server = props_and_tags();
     server.route(CHAT, vec![Reply::fixture("tool_call").cut_after(500)]);
     let (result, _) = run_with(&server, |c| c.limits.retry_attempts = 2);
     assert!(matches!(result, Err(SelfTestError::Infer(_))), "{result:?}");

crates/loopd/tests/selftest_tags.rs

@@ -0,0 +1,349 @@
+//! The self-test's tag check (M5b spec, section 8): each of the model's tags must be one token,
+//! the one recorded with it, and its neutralised form more than one, or `loopd` does not start.
+//! Against the fake server's tokenizer, with one answer changed at a time. The tags come from
+//! `bxcore::sanitize::TAGS`, and no assertion prints one (tip T33): a failure names the tag by its
+//! id, and the tests check that it does. Do not edit.
+
+mod support;
+#[path = "support/tags.rs"]
+mod tags;
+
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command, Stdio};
+use std::time::{Duration, Instant};
+
+use bxcore::sanitize::{TAGS, neutralise};
+use loopd::llama::Client;
+use loopd::selftest::{SelfTestError, run};
+use support::{FakeServer, Home, Recorded, Reply, tokenizer, tokens};
+use tags::{holds_a_tag, same};
+
+const CHAT: &str = "/v1/chat/completions";
+const TOKENIZE: &str = "/tokenize";
+const POINTER: &str = "see docs/runbook.md#loopd-tags-unexpected";
+
+/// The recorded props and completions, and `tokenize` as the server's `/tokenize`.
+fn server_with(tokenize: impl Fn(&Recorded) -> Reply + Send + Sync + 'static) -> FakeServer {
+    let server = FakeServer::start();
+    server.route("/props", vec![Reply::fixture("props")]);
+    server.route(
+        CHAT,
+        vec![
+            Reply::fixture("tool_call"),
+            Reply::fixture("turn1"),
+            Reply::fixture("turn2"),
+            Reply::fixture("plain"),
+        ],
+    );
+    server.respond(TOKENIZE, tokenize);
+    server
+}
+
+/// The text a `/tokenize` request asked about.
+fn content(request: &Recorded) -> String {
+    let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap_or_default();
+    body["content"].as_str().unwrap_or_default().to_string()
+}
+
+/// The fake tokenizer, except that `text` is answered with `ids`.
+fn except(text: &str, ids: Vec<u32>) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static {
+    let text = text.to_string();
+    move |request: &Recorded| {
+        if content(request) == text {
+            tokens(&ids)
+        } else {
+            tokenizer(request)
+        }
+    }
+}
+
+/// Runs the self-test against `server` with the test config; the result and the steps started.
+fn run_on(server: &FakeServer) -> (Result<(), SelfTestError>, Vec<String>) {
+    let cfg = support::test_config(&server.socket);
+    let mut steps = Vec::new();
+    let result = run(&Client::new(cfg), &mut |s| steps.push(s.to_string()));
+    (result, steps)
+}
+
+/// A failure of the tag check stops the self-test before any prompt, names the tag by its id,
+/// holds no tag, and points at the tags' own runbook entry.
+#[track_caller]
+fn stopped_before_any_prompt(server: &FakeServer, e: &SelfTestError, steps: &[String], id: u32) {
+    let text = e.to_string();
+    assert!(!holds_a_tag(&text), "tag {id}: the message holds a tag");
+    assert!(text.contains(&id.to_string()), "tag {id}: {text}");
+    assert_eq!(e.pointer(), POINTER, "tag {id}");
+    assert_eq!(steps.len(), 2, "tag {id}: {steps:?}");
+    assert!(server.requests_to(CHAT).is_empty(), "tag {id}: no prompt");
+}
+
+#[test]
+fn every_tag_and_its_neutralised_form_tokenize_as_recorded() {
+    let server = server_with(tokenizer);
+    let (result, steps) = run_on(&server);
+    assert!(result.is_ok(), "{result:?}");
+    assert_eq!(
+        steps,
+        [
+            "server matches config",
+            "tags tokenize as recorded",
+            "tool call round trip",
+            "turn 2 cache hit"
+        ]
+    );
+
+    let asked = server.requests_to(TOKENIZE);
+    assert_eq!(
+        asked.len(),
+        2 * TAGS.len(),
+        "one request per tag and per form"
+    );
+    let mut want: Vec<String> = TAGS
+        .iter()
+        .flat_map(|t| [t.text.to_string(), neutralise(t.text).into_owned()])
+        .collect();
+    let mut got: Vec<String> = asked.iter().map(content).collect();
+    want.sort();
+    got.sort();
+    same(&got, &want, "the texts asked about");
+    for (i, request) in asked.iter().enumerate() {
+        assert_eq!(request.method, "POST", "request {i}");
+        let body = request.json();
+        let fields = serde_json::json!({
+            "model": "test-model",
+            "content": content(request),
+            "add_special": false,
+            "parse_special": true,
+        });
+        same(&body, &fields, &format!("request {i}: the body"));
+    }
+
+    // Every tag is checked before the first prompt is sent.
+    let paths: Vec<String> = server
+        .requests()
+        .iter()
+        .map(|r| r.path().to_string())
+        .collect();
+    let last_tokenize = paths.iter().rposition(|p| p == TOKENIZE);
+    let first_chat = paths.iter().position(|p| p == CHAT);
+    assert!(last_tokenize < first_chat, "{paths:?}");
+}
+
+#[test]
+fn a_tag_that_is_not_one_token_fails() {
+    for (i, tag) in TAGS.iter().enumerate() {
+        // Six ordinary tokens, as a tag the tokenizer no longer knows would be; none for the
+        // first tag.
+        let ids = if i == 0 {
+            Vec::new()
+        } else {
+            vec![27, 91, 316, 6018, 91, 29]
+        };
+        let count = ids.len();
+        let server = server_with(except(tag.text, ids));
+        let (result, steps) = run_on(&server);
+        match &result {
+            Err(e @ SelfTestError::TagNotOneToken { id, count: got }) => {
+                assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id);
+                stopped_before_any_prompt(&server, e, &steps, tag.id);
+            }
+            other => panic!("tag {}: {other:?}", tag.id),
+        }
+    }
+}
+
+#[test]
+fn a_tag_that_is_another_token_fails() {
+    for tag in TAGS {
+        let other = tag.id + 1;
+        let server = server_with(except(tag.text, vec![other]));
+        let (result, steps) = run_on(&server);
+        match &result {
+            Err(e @ SelfTestError::TagOtherId { id, got }) => {
+                assert_eq!((*id, *got), (tag.id, other), "tag {}", tag.id);
+                stopped_before_any_prompt(&server, e, &steps, tag.id);
+            }
+            other => panic!("tag {}: {other:?}", tag.id),
+        }
+    }
+}
+
+#[test]
+fn a_neutralised_form_that_is_one_token_or_none_fails() {
+    for (i, tag) in TAGS.iter().enumerate() {
+        // The tag's own token, as a tokenizer that saw through the escape would give; none for
+        // the first tag.
+        let ids = if i == 0 { Vec::new() } else { vec![tag.id] };
+        let count = ids.len();
+        let server = server_with(except(&neutralise(tag.text), ids));
+        let (result, steps) = run_on(&server);
+        match &result {
+            Err(e @ SelfTestError::NeutralisedTooFew { id, count: got }) => {
+                assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id);
+                stopped_before_any_prompt(&server, e, &steps, tag.id);
+            }
+            other => panic!("tag {}: {other:?}", tag.id),
+        }
+    }
+}
+
+#[test]
+fn a_neutralised_form_of_two_tokens_passes() {
+    // "More than one" is the rule; two is enough.
+    let server = server_with(except(&neutralise(TAGS[0].text), vec![5, 6]));
+    let (result, steps) = run_on(&server);
+    assert!(result.is_ok(), "{result:?}");
+    assert_eq!(steps.len(), 4);
+}
+
+#[test]
+fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() {
+    let first = TAGS[0].id;
+    let cases: [(&str, Option<Reply>); 5] = [
+        ("no /tokenize at all", None),
+        (
+            "an HTTP error",
+            Some(Reply::json(
+                500,
+                r#"{"error":{"code":500,"message":"boom"}}"#,
+            )),
+        ),
+        (
+            "a count, not ids",
+            Some(Reply::json(200, r#"{"tokens": 1}"#)),
+        ),
+        (
+            "pieces, not ids",
+            Some(Reply::json(200, r#"{"tokens":[{"id":1,"piece":"x"}]}"#)),
+        ),
+        (
+            "a negative id",
+            Some(Reply::json(200, r#"{"tokens":[-1]}"#)),
+        ),
+    ];
+    for (what, reply) in cases {
+        let server = FakeServer::start();
+        server.route("/props", vec![Reply::fixture("props")]);
+        server.route(CHAT, vec![Reply::fixture("tool_call")]);
+        if let Some(reply) = reply {
+            server.route(TOKENIZE, vec![reply]);
+        }
+        let (result, steps) = run_on(&server);
+        match &result {
+            Err(e @ SelfTestError::Tokenize { id, .. }) => {
+                assert_eq!(*id, first, "{what}");
+                stopped_before_any_prompt(&server, e, &steps, first);
+            }
+            other => panic!("{what}: {other:?}"),
+        }
+        assert_eq!(server.requests_to(TOKENIZE).len(), 1, "{what}: no retry");
+    }
+}
+
+#[test]
+fn the_other_failures_keep_the_self_tests_entry() {
+    let e = SelfTestError::Hash;
+    assert_eq!(e.pointer(), "see docs/runbook.md#loopd-selftest-failed");
+    let e = SelfTestError::CacheMiss {
+        expected: 2,
+        got: 1,
+    };
+    assert_eq!(e.pointer(), "see docs/runbook.md#loopd-selftest-failed");
+}
+
+/// A `loopd` process that is killed when the test ends, however it ends.
+struct Running(Child);
+
+impl Drop for Running {
+    fn drop(&mut self) {
+        let _ = self.0.kill();
+        let _ = self.0.wait();
+    }
+}
+
+/// Runs `loopd <command> --config <config>` for at most ten seconds: its exit code (None if it
+/// was still running and was killed) and what it printed on standard error.
+fn loopd(command: &str, config: &Path) -> (Option<i32>, String) {
+    let child = Command::new(env!("CARGO_BIN_EXE_loopd"))
+        .args([command, "--config"])
+        .arg(config)
+        .stdout(Stdio::null())
+        .stderr(Stdio::piped())
+        .spawn()
+        .unwrap();
+    let mut running = Running(child);
+    let deadline = Instant::now() + Duration::from_secs(10);
+    let mut code = None;
+    while Instant::now() < deadline {
+        if let Some(status) = running.0.try_wait().unwrap() {
+            code = status.code();
+            break;
+        }
+        std::thread::sleep(Duration::from_millis(20));
+    }
+    let _ = running.0.kill();
+    let _ = running.0.wait();
+    let mut stderr = String::new();
+    if let Some(mut pipe) = running.0.stderr.take() {
+        std::io::Read::read_to_string(&mut pipe, &mut stderr).unwrap();
+    }
+    (code, stderr)
+}
+
+fn config_file(home: &Home, server: &FakeServer) -> PathBuf {
+    let text = format!(
+        r#"
+[infer]
+socket = "{}"
+model = "test-model"
+[slots]
+main = 0
+background = 1
+[expect]
+template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
+n_ctx = 131072
+slots = 2
+[limits]
+poll_ms = 40
+liveness_ms = 500
+retry_backoff_ms = [10]
+[paths]
+home = "{}"
+"#,
+        server.socket.display(),
+        home.dir.display()
+    );
+    let path = home.dir.join("config.toml");
+    std::fs::write(&path, text).unwrap();
+    path
+}
+
+#[test]
+fn selftest_and_serve_stop_on_a_tag_and_name_its_entry() {
+    let home = Home::new();
+    // The end-of-turn tag, as six ordinary tokens.
+    let tag = &TAGS[2];
+    let server = server_with(except(tag.text, vec![27, 91, 316, 6018, 91, 29]));
+    let config = config_file(&home, &server);
+    for command in ["selftest", "serve"] {
+        let (code, stderr) = loopd(command, &config);
+        assert!(
+            !holds_a_tag(&stderr),
+            "{command}: what it printed holds a tag"
+        );
+        assert_eq!(code, Some(1), "{command}: {stderr}");
+        let line = stderr
+            .lines()
+            .find(|l| l.starts_with("selftest: FAILED: "))
+            .unwrap_or_else(|| panic!("{command}: no FAILED line: {stderr}"));
+        assert!(line.ends_with(POINTER), "{command}: {line}");
+        assert_eq!(line.matches("runbook.md#").count(), 1, "{command}: {line}");
+        assert!(line.contains(&tag.id.to_string()), "{command}: {line}");
+        assert!(!stderr.contains("serving on"), "{command}: {stderr}");
+    }
+    assert!(
+        !home.dir.join("run/loop/loop.sock").exists(),
+        "serve bound no socket"
+    );
+    assert!(server.requests_to(CHAT).is_empty(), "no prompt was sent");
+}

crates/loopd/tests/serve.rs

@@ -5,7 +5,7 @@ mod support;
 use std::os::unix::fs::PermissionsExt;
 use std::process::{Command, Stdio};
 use std::time::Duration;
-use support::{FakeServer, Home, Reply};
+use support::{FakeServer, Home, Reply, tokenizer};
 
 fn config_file(home: &Home, server: &FakeServer, expect_slots: u32) -> std::path::PathBuf {
     let text = format!(
@@ -37,6 +37,7 @@ home = "{}"
 
 fn healthy_routes(server: &FakeServer) {
     server.route("/props", vec![Reply::fixture("props")]);
+    server.respond("/tokenize", tokenizer);
     server.route(
         "/v1/chat/completions",
         vec![
@@ -82,6 +83,11 @@ fn serve_runs_the_self_test_then_binds_the_socket_with_mode_0600() {
         3,
         "the three self-test completions ran"
     );
+    assert_eq!(
+        server.requests_to("/tokenize").len(),
+        2 * bxcore::sanitize::TAGS.len(),
+        "the tag check ran: each tag and its neutralised form"
+    );
 }
 
 #[test]

crates/loopd/tests/support/mod.rs

@@ -3,7 +3,8 @@
 //! It listens on a Unix socket in a temporary directory. Each path has a list of replies that are
 //! served in order; the last one repeats. A reply is raw bytes, normally a response recorded from
 //! the real server (`tests/fixtures/http/*.http`), and can be delayed, sent in small pieces, cut
-//! short, or left hanging. Every request is recorded.
+//! short, or left hanging. A path can instead compute each reply from its request, as the fake
+//! tokenizer does. Every request is recorded.
 
 #![allow(dead_code)] // each test file uses a different part of this module
 
@@ -171,6 +172,26 @@ impl loopd::tools::ToolPort for ReturnsPendingPort {
     }
 }
 
+/// The fake `/tokenize`, for `FakeServer::respond`: the text of the request's `content` is one
+/// token when it is one of the model's tags (`bxcore::sanitize::TAGS`), the tag's id, and
+/// otherwise one token per byte, the byte's value. A body without a string `content` gets a 400.
+pub fn tokenizer(request: &Recorded) -> Reply {
+    let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap_or_default();
+    let Some(content) = body.get("content").and_then(|c| c.as_str()) else {
+        return Reply::json(400, r#"{"error":"no content"}"#);
+    };
+    let ids: Vec<u32> = match bxcore::sanitize::TAGS.iter().find(|t| t.text == content) {
+        Some(tag) => vec![tag.id],
+        None => content.bytes().map(u32::from).collect(),
+    };
+    tokens(&ids)
+}
+
+/// A `/tokenize` answer as llama-server gives one without `with_pieces`: `{"tokens": [ids]}`.
+pub fn tokens(ids: &[u32]) -> Reply {
+    Reply::json(200, &serde_json::json!({ "tokens": ids }).to_string())
+}
+
 /// What an independent script derived from a recorded stream: `tests/fixtures/expected/`.
 pub fn expected(name: &str) -> serde_json::Value {
     serde_json::from_slice(&fixture_bytes("expected", &format!("{name}.json"))).unwrap()
@@ -273,8 +294,12 @@ impl Recorded {
     }
 }
 
+/// A reply computed from its request.
+type Responder = Arc<dyn Fn(&Recorded) -> Reply + Send + Sync>;
+
 struct State {
     routes: Mutex<Vec<(String, VecDeque<Reply>)>>,
+    responders: Mutex<Vec<(String, Responder)>>,
     requests: Mutex<Vec<Recorded>>,
 }
 
@@ -295,6 +320,7 @@ impl FakeServer {
         let listener = UnixListener::bind(&socket).unwrap();
         let state = Arc::new(State {
             routes: Mutex::new(Vec::new()),
+            responders: Mutex::new(Vec::new()),
             requests: Mutex::new(Vec::new()),
         });
         let accept_state = Arc::clone(&state);
@@ -308,14 +334,29 @@ impl FakeServer {
         FakeServer { socket, state, dir }
     }
 
-    /// Replies for `path` (the query is ignored), served in order. The last one repeats.
+    /// Replies for `path` (the query is ignored), served in order. The last one repeats. They
+    /// replace a responder for `path`.
     pub fn route(&self, path: &str, replies: Vec<Reply>) {
         assert!(!replies.is_empty());
+        self.state
+            .responders
+            .lock()
+            .unwrap()
+            .retain(|(p, _)| p != path);
         let mut routes = self.state.routes.lock().unwrap();
         routes.retain(|(p, _)| p != path);
         routes.push((path.to_string(), replies.into()));
     }
 
+    /// For each request to `path` (the query is ignored), the reply `reply` computes from it. It
+    /// replaces the replies `route` gave `path`.
+    pub fn respond(&self, path: &str, reply: impl Fn(&Recorded) -> Reply + Send + Sync + 'static) {
+        self.state.routes.lock().unwrap().retain(|(p, _)| p != path);
+        let mut responders = self.state.responders.lock().unwrap();
+        responders.retain(|(p, _)| p != path);
+        responders.push((path.to_string(), Arc::new(reply)));
+    }
+
     pub fn requests(&self) -> Vec<Recorded> {
         self.state.requests.lock().unwrap().clone()
     }
@@ -376,15 +417,23 @@ fn serve(mut stream: UnixStream, state: &State) {
         return;
     };
     let path = request.path().to_string();
+    let responder = state
+        .responders
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|(p, _)| *p == path)
+        .map(|(_, reply)| Arc::clone(reply));
+    let computed = responder.map(|reply| reply(&request));
     state.requests.lock().unwrap().push(request);
-    let reply = {
+    let reply = computed.or_else(|| {
         let mut routes = state.routes.lock().unwrap();
         match routes.iter_mut().find(|(p, _)| *p == path) {
             Some((_, replies)) if replies.len() > 1 => replies.pop_front(),
             Some((_, replies)) => replies.front().cloned(),
             None => None,
         }
-    };
+    });
     let reply =
         reply.unwrap_or_else(|| Reply::json(404, r#"{"error":"no route in the fake server"}"#));
     thread::sleep(Duration::from_millis(reply.head_delay_ms));

docs/implementer-log.md

@@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone.
 
 | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model |
 |---|---|---|---|---|---|---|---|
+| M5b/08-loopd-selftest-tags | 2026-09-26 | done | 2 | pass | none | Copied the five given test files from `docs/plans/M5b/files/`: `selftest_tags.rs` (new) and new versions of `selftest.rs`, `serve.rs`, `pointers.rs` and `support/mod.rs` (did not open them). Step 2 showed exactly the four named failures (`selftest`: three steps without `"tags tokenize as recorded"`; `a_completion_without_the_tool_call_fails_the_tool_call_check` left 2 right 3; `a_cold_second_turn_fails_the_cache_check` left 3 right 4; `serve_runs_the_self_test_then_binds_the_socket_with_mode_0600` left 0 right 66) and `selftest_tags.rs` failing to compile with 7 E0599s (no `TagNotOneToken`/`TagOtherId`/`NeutralisedTooFew`/`Tokenize` variant, no `pointer`). Appended `runbook-loopd-tags-unexpected.md`; `check-runbook.sh` printed nothing and exited 0, `grep -c '^## loopd-tags-unexpected$'` printed 1. Made the edits in order: I1 the private `TokenIds` struct before `CacheOutcome` and I2 the `pub token_ids` method after `tokenize` (left `tokenize` itself unchanged); S1 the `use bxcore::sanitize::{TAGS, Tag, neutralise}` import, S2 the four tag variants plus `SelfTestError::pointer` with each pointer written whole, S3 the four Display arms, S4 the second check `tags tokenize as recorded`, S5 the `tags_tokenize` and `check_tag` functions, S6 check 3 to check 4; M1 `e.pointer()` in `run_selftest_check`. `cargo fmt --all`; `cargo check -p loopd` clean; `selftest_tags` 8, `selftest` 6, `serve` 3, `pointers` 7 passed, every `loopd` suite passes. `grep -rn 'runbook.md#loopd-' crates/loopd/src` prints exactly two lines, both in `selftest.rs`. `make gate` printed `gate: ok` with 1050 tests (1042 before this task, plus the 8 in `selftest_tags.rs`). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
 | M5b/07-loopd-neutralise | 2026-09-26 | done | 2 | fail | none | Copied the three given tests from `docs/plans/M5b/files/`: `neutralise.rs`, `neutralise_replay.rs` and `support/tags.rs` (did not open them). Step 2 showed exactly the seven named failures (all six in `neutralise`, `the_memory_block_is_neutralised_when_the_baseline_is_assembled` in `neutralise_replay`); the two that should pass passed. Made the edits in order: T1 the module comment, T2 the `use bxcore::sanitize::neutralise` import, T3 `content: neutralise(content).into_owned()` on the `LogRecord::User`, T4 the capped-then-neutralised `let text = neutralise(&text).into_owned()` on the `LogRecord::ToolResult`; B1 the import, B2 the `assemble_with` comment, B3 `neutralise(memory.trim())` with `&memory` pushed. `cargo fmt --all`; `cargo check -p loopd` clean; `neutralise` 6 passed, `neutralise_replay` 3 passed, every `loopd` suite passes. `grep -rn 'neutralise(' crates/loopd/src` prints exactly three lines (two in `turn.rs`, one in `baseline.rs`). First `make gate` run failed on a flaky brokerd concurrency test, `a_torn_tail_is_recovered_and_it_serves` (one fail out of the serve suite under full-workspace load); it passes 3/3 in isolation and the serve suite 9/9 three times on the same tree, and the loopd edits touch no brokerd path, so it is a timing flake, not this task. Re-ran `make gate`: `gate: ok` with 1042 tests. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
 | M5b/06-memoryd-sanitize | 2026-09-26 | done | 1 | pass | none | Copied the five given tests from `docs/plans/M5b/files/`: `sanitize.rs`, `serve_sanitize.rs`, and new versions of `edit.rs`, `files.rs` and `serve.rs` (did not open them). Step 2 showed exactly the seven named failures (`refusals_change_nothing` in `edit`, `an_entry_that_fails_the_check_is_left_out_and_logged` in `serve`, all five in `serve_sanitize`); `files` did not compile (one E0609, no field `rule` on `&memoryd::check::Refused`) and `sanitize` did not compile (13: E0432 for `memoryd::edit::flagged`/`refused`, E0609 for `flag` on `Outcome` and `&Record` and `rule` on `Refused`, E0615 for `sentence`). Appended the three runbook fragments to `docs/runbook.md`; `check-runbook.sh` printed nothing and exited 0. `git rm`'d `crates/memoryd/src/check.rs` and its test. Made the edits in order: E1 removed `pub mod check;` from `lib.rs`; E2-E10 in `edit.rs` (imports, `Outcome` gains `flag`, five functions replacing `checked`, the refuse check in `apply`, the flag check after `edited`, `flag: None`, `text.trim()` in `edited`, `noted` plus `as_notes`, `text.trim()` in `consolidate`); E11-E13 in `files.rs` (imports, `Block`, the two `block` comments, the `refuse` match); E14-E19 in `journal.rs` (`Record` gains `flag` after `untrusted`, `record` sets `flag: outcome.flag.map(str::to_string)`, `log` appends `  flagged {rule}`, `undo` sets `flag: None`); E20 in `review.rs` (`reject` sets `flag: None`); E21-E23 in `serve.rs` (`block` logs by `refusal.rule` with the runbook pointer, the `memory` comment, and `memory` calls `edit::refused` first then `edit::flagged` to keep flagged writes off review and logs the rule). `cargo fmt --all`; `cargo check -p memoryd` clean; `cargo test -p memoryd` every suite passes (sanitize 11, serve_sanitize 5, edit 8, files 8, serve 8); `make gate` printed `gate: ok` with 1033 tests; `grep -rn 'memoryd::check\|crate::check' crates/` prints nothing. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
 | M5b/05-sanitize-flag | 2026-09-26 | done | 1 | pass | none | Copied the three given tests from `docs/plans/M5b/files/`: `bxcore/tests/sanitize_flag.rs`, `sanitize_flag_corpus.rs`, `sanitize_flag_edges.rs`. Step 2 showed exactly the named E0432s (`Flag`/`flag`/`normalise` for `sanitize_flag`, `flag` for the other two). Copied the two skeletons, `bxcore/src/sanitize/normalise.rs` (five functions) and `flag.rs` (seven), then made the two edits: E1 replaced `bxcore/src/sanitize/mod.rs` whole (added `mod flag; mod normalise;` and `pub use flag::{Flag, flag};` / `pub use normalise::normalise;`), E2 made `invisible` `pub(super)` in `refuse.rs` (nothing else changed). Filled `normalise.rs` in the given order, each from its comment and `cargo check` after: `leet` (`LEET.iter().find` the digit, `map_or(c, |&(_, letter)| letter)`); `homoglyph` (`HOMOGLYPHS.iter().find` the `from`, `map` the `to`); `full_width` (`u32::from(c)` inside `FULL_WIDTH`, `checked_sub(FULL_WIDTH_OFFSET).and_then(char::from_u32).unwrap_or(c)`); `push_folded` (`full_width` then `homoglyph`, pushing the letter when `Some`, else `c.to_lowercase().map(leet)`); `normalise` (walk chars, `space` bool: whitespace sets it, `invisible` dropped leaving it, any other char pushes one `' '` when `space` and the output is not empty, clears `space`, then `push_folded`). Filled `flag.rs` in the given order: `in_sentence` (`!word.ends_with(['.','!','?',';']) && !["&&","||","|"].contains(&word)`); `filler` (`!word.is_empty() && word.chars().all(|c| c.is_alphanumeric() || c == '_')`); `is` (trim non-word ends with `trim_matches`, compare the word and its stripped form to each keyword); `ends_clause` (`word.ends_with(CLAUSE_ENDS)`, or `after.is_empty()`, or `after` starts with a `CLAUSE_FOLLOWERS` matched word by word as `is`); `one_word` (`split_first`, `test` the first word with the words after it, then `seq_at` the remaining parts on those words); `skip` (counts `0..=max`, `split_at_checked`: a skipped word failing `test` or a count past the end returns `false`, otherwise when all pass check `seq_at`); `seq_at` (`split_first`; `Is`/`Ends`/`Holds` go to `one_word` with `is`, `is && ends_clause`, `contains`, `Filler`/`Gap` to `skip` with `filler`/`in_sentence`). My first `one_word` edit dropped a `]` and `)` from the `impl Fn` signature; caught by diffing against the skeleton and fixed. After the fills `cargo check -p bxcore` is clean. `sanitize_flag` 22 passed, `sanitize_flag_corpus` 2, `sanitize_flag_edges` 2; `sanitize_refuse` 14, `sanitize_refuse_corpus` 2, `sanitize_tags` 15, `sanitize_corpus` 3. `cargo fmt --all`; `make gate` printed `gate: ok`; `cargo test --workspace` reports 1022 passed, plus the gate's separate loopd `end_to_end` ignored test, for 1023 total. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |

docs/runbook.md

@@ -728,3 +728,57 @@ Entries are separated by a line holding only `§`. A secret does not belong in m
 if it was real.
 
 **Check.** Start a new conversation: `memoryd` does not log the line again.
+
+## loopd-tags-unexpected
+
+**What you see.** `loopd selftest` or `loopd serve` prints `selftest: tags tokenize as recorded`,
+then `selftest: FAILED: <reason>; see docs/runbook.md#loopd-tags-unexpected`, and exits 1. No
+prompt was sent. The reason names a tag by its token id, never by its text:
+
+- `the tag with id <id> is <n> tokens, not one`
+- `the tag with id <id> is one token, but the token <other>`
+- `the neutralised form of the tag with id <id> does not split into several tokens (the server
+  gave <n>)`
+- `could not tokenize the tag with id <id>: <error>`
+
+**Why.** `loopd` neutralises the model's chat-template tags in every text from outside, writing
+their angle brackets as `&lt;` and `&gt;` (M5b spec, sections 6.2 and 8). The list of tags, each
+with its token id, is compiled into `bxcore` (`crates/bxcore/src/sanitize/tags.rs`), as measured
+on straylight. At startup `loopd` asks the server's `/tokenize` about each tag and its neutralised
+form. If a tag is no longer the one token recorded with it, the model or its tokenizer has changed
+and the list may no longer match it; if a neutralised form is a single token, neutralising no
+longer keeps that tag from the model. Either way the model could receive text from outside as its
+own control tokens, so `loopd` does not start. The check cannot find a new tag missing from the
+list: the server does not list its special tokens.
+
+**Confirm.** Find the tag: `grep -n '<id>' crates/bxcore/src/sanitize/tags.rs` shows its entry,
+built from its name with `bar!`, `open!` or `close!`. Ask the server yourself, building the tag
+from its name as the file does. For a `bar!` tag named `im_end`:
+
+```sh
+printf '{"model":"ornith-1.5-35b-a3b","content":"<|%s|>","parse_special":true}' im_end |
+  curl -s -d @- http://straylight:11434/tokenize
+printf '{"model":"ornith-1.5-35b-a3b","content":"&lt;|%s|&gt;","parse_special":true}' im_end |
+  curl -s -d @- http://straylight:11434/tokenize
+```
+
+For an `open!` tag the content is `<%s>`, for a `close!` tag `</%s>`. The first answer should be
+`{"tokens":[<id>]}`, the second a list of several ids. Check which model file the server loaded:
+`curl -s 'http://straylight:11434/props?model=ornith-1.5-35b-a3b' | jq .model_path`.
+
+**Fix, by reason.**
+
+- `could not tokenize …`: the server did not answer `/tokenize`. A router in front of
+  llama-server may not forward it (crossbar does not, `docs/decisions.md`, P22): point
+  `inferproxy` at llama-server itself. Otherwise check that the server is up:
+  `curl -s http://straylight:11434/health`, and run `loopd selftest --config <path>` again.
+- A tag with another id, or several tokens: the model file changed. Do not run `loopd` against
+  it. Measure the new model's tags as M5b task 03 did (every vocabulary entry of type control or
+  user-defined, each checked with `/tokenize`), and change `TAGS` and the tests' list in
+  `crates/bxcore/tests/support/tags.rs` together; the change reaches the agent when it is rebuilt.
+  A new model also invalidates the measurements in `docs/inference-contract.md`.
+- A neutralised form that is one token: the tokenizer has an entry for the escaped spelling, and
+  the replacement of the M5b spec, section 6.2, no longer works for that tag. That is a design
+  change: record it in `docs/decisions.md` and do not run `loopd` until it is settled.
+
+**Check.** `loopd selftest --config <path>` prints `selftest: ok`.