diff --git a/crates/loopd/src/llama/info.rs b/crates/loopd/src/llama/info.rs index 0f95b0c..b1b1c4b 100644 --- a/crates/loopd/src/llama/info.rs +++ b/crates/loopd/src/llama/info.rs @@ -25,6 +25,12 @@ pub struct SlotInfo { pub is_processing: bool, } +/// `/tokenize`'s answer without `with_pieces`: `{"tokens": [ids]}`. Other fields are ignored. +#[derive(serde::Deserialize)] +struct TokenIds { + tokens: Vec, +} + #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum CacheOutcome { Hit, @@ -143,6 +149,23 @@ impl Client { .ok_or_else(|| InferError::Protocol("tokenize response has no tokens".to_string())) } + /// The token ids the server gives `text`, with special tokens parsed as a chat prompt's are + /// and no BOS added. The self-test checks the model's tags with it (M5b spec, section 8). + pub fn token_ids(&self, text: &str) -> Result, InferError> { + let body = serde_json::to_string(&serde_json::json!({ + "model": self.cfg.infer.model, + "content": text, + "add_special": false, + "parse_special": true, + })) + .map_err(|_| InferError::Protocol("could not encode tokenize request".to_string()))?; + let bytes = self.call("POST", "/tokenize", Some(body.as_bytes()))?; + let ids: TokenIds = serde_json::from_slice(&bytes).map_err(|_| { + InferError::Protocol("tokenize response has no list of token ids".to_string()) + })?; + Ok(ids.tokens) + } + pub(crate) fn end_reasoning(&self, completion_id: &str) -> Result { let body = serde_json::to_string(&serde_json::json!({ "id": completion_id, diff --git a/crates/loopd/src/main.rs b/crates/loopd/src/main.rs index 0b94203..2b32b51 100644 --- a/crates/loopd/src/main.rs +++ b/crates/loopd/src/main.rs @@ -42,7 +42,7 @@ fn run_selftest_check(client: &Client) -> Result<(), SelfTestError> { Ok(()) } Err(e) => { - eprintln!("selftest: FAILED: {e}; see docs/runbook.md#loopd-selftest-failed"); + eprintln!("selftest: FAILED: {e}; {}", e.pointer()); Err(e) } } diff --git a/crates/loopd/src/selftest.rs b/crates/loopd/src/selftest.rs index 57eb3d8..1e72be0 100644 --- a/crates/loopd/src/selftest.rs +++ b/crates/loopd/src/selftest.rs @@ -1,6 +1,9 @@ //! The startup self-test: before `loopd` serves anyone, it checks that the server is the one its -//! config describes, that a tool call comes back parsed, and that a second turn reuses the first -//! turn's cache. Any check failing means `loopd` does not start. +//! config describes, that the model's tags tokenize as recorded, that a tool call comes back +//! parsed, and that a second turn reuses the first turn's cache. Any check failing means `loopd` +//! does not start. + +use bxcore::sanitize::{TAGS, Tag, neutralise}; use crate::llama::{ ChatMessage, ChatRequest, Client, FinishReason, InferError, ToolSchema, @@ -21,6 +24,45 @@ pub enum SelfTestError { }, Infer(InferError), Hash, + /// The tag with the recorded id `id` is `count` tokens, not one. + TagNotOneToken { + id: u32, + count: usize, + }, + /// The tag with the recorded id `id` is one token, but the token `got`. + TagOtherId { + id: u32, + got: u32, + }, + /// The neutralised form of the tag with the recorded id `id` is `count` tokens, not more + /// than one. + NeutralisedTooFew { + id: u32, + count: usize, + }, + /// `/tokenize` failed for the tag with the recorded id `id`, or for its neutralised form. + Tokenize { + id: u32, + error: InferError, + }, +} + +impl SelfTestError { + /// The runbook pointer that ends the line `loopd` prints for this failure. A tag failure has + /// its own entry; every other failure shares the self-test's. + pub fn pointer(&self) -> &'static str { + match self { + SelfTestError::TagNotOneToken { .. } + | SelfTestError::TagOtherId { .. } + | SelfTestError::NeutralisedTooFew { .. } + | SelfTestError::Tokenize { .. } => "see docs/runbook.md#loopd-tags-unexpected", + SelfTestError::Mismatch { .. } + | SelfTestError::ToolCall(_) + | SelfTestError::CacheMiss { .. } + | SelfTestError::Infer(_) + | SelfTestError::Hash => "see docs/runbook.md#loopd-selftest-failed", + } + } } impl std::fmt::Display for SelfTestError { @@ -40,6 +82,19 @@ impl std::fmt::Display for SelfTestError { ), SelfTestError::Infer(e) => write!(f, "{e}"), SelfTestError::Hash => write!(f, "the chat template could not be hashed"), + SelfTestError::TagNotOneToken { id, count } => { + write!(f, "the tag with id {id} is {count} tokens, not one") + } + SelfTestError::TagOtherId { id, got } => { + write!(f, "the tag with id {id} is one token, but the token {got}") + } + SelfTestError::NeutralisedTooFew { id, count } => write!( + f, + "the neutralised form of the tag with id {id} does not split into several tokens (the server gave {count})" + ), + SelfTestError::Tokenize { id, error } => { + write!(f, "could not tokenize the tag with id {id}: {error}") + } } } } @@ -54,11 +109,14 @@ impl From for SelfTestError { const SYSTEM: &str = "You are Boxmaker, a careful personal agent."; -/// Runs the three checks in order. `on_step` is told the name of each check as it starts. +/// Runs the four checks in order. `on_step` is told the name of each check as it starts. pub fn run(client: &Client, on_step: &mut dyn FnMut(&str)) -> Result<(), SelfTestError> { on_step("server matches config"); server_matches(client)?; + on_step("tags tokenize as recorded"); + tags_tokenize(client)?; + on_step("tool call round trip"); tool_call_round_trip(client)?; @@ -103,7 +161,45 @@ fn server_matches(client: &Client) -> Result<(), SelfTestError> { Ok(()) } -/// Check 2: a tool call comes back parsed, with a `read_file` tool and a string `path`. +/// Check 2: each of the model's tags (`bxcore::sanitize::TAGS`) is one token, the one recorded +/// with it, and its neutralised form is more than one (M5b spec, section 8). Like check 1 it sends +/// no prompt and uses no slot, so a server whose tokenizer is not the recorded one is sent no +/// prompt either. A tag missing from the list cannot be found: the server does not list them. +fn tags_tokenize(client: &Client) -> Result<(), SelfTestError> { + for tag in TAGS { + check_tag(client, tag)?; + } + Ok(()) +} + +/// Two `/tokenize` requests: `tag.text`, then `neutralise(tag.text)`. An error names the tag by +/// its id, never by its text, which the operator's terminal and log would show whole. +fn check_tag(client: &Client, tag: &Tag) -> Result<(), SelfTestError> { + let id = tag.id; + let tokenize = |text: &str| { + client + .token_ids(text) + .map_err(|error| SelfTestError::Tokenize { id, error }) + }; + let ids = tokenize(tag.text)?; + match ids.as_slice() { + [got] if *got == id => {} + [got] => return Err(SelfTestError::TagOtherId { id, got: *got }), + _ => { + return Err(SelfTestError::TagNotOneToken { + id, + count: ids.len(), + }); + } + } + let count = tokenize(&neutralise(tag.text))?.len(); + if count <= 1 { + return Err(SelfTestError::NeutralisedTooFew { id, count }); + } + Ok(()) +} + +/// Check 3: a tool call comes back parsed, with a `read_file` tool and a string `path`. fn tool_call_round_trip(client: &Client) -> Result<(), SelfTestError> { let req = ChatRequest { slot: client.config().slots.main, @@ -159,7 +255,7 @@ fn tool_call_round_trip(client: &Client) -> Result<(), SelfTestError> { Ok(()) } -/// Check 3: a second turn that extends the first reuses the first turn's cache. +/// Check 4: a second turn that extends the first reuses the first turn's cache. fn cache_hit(client: &Client) -> Result<(), SelfTestError> { let first = ChatRequest { slot: client.config().slots.main, diff --git a/crates/loopd/tests/pointers.rs b/crates/loopd/tests/pointers.rs index c64d44d..f4f93ff 100644 --- a/crates/loopd/tests/pointers.rs +++ b/crates/loopd/tests/pointers.rs @@ -14,7 +14,7 @@ use loopd::broker_port::{not_configured_line, unavailable_line}; use loopd::memory::{MemoryPort, MemorySource}; use loopd::session::{Session, SessionError}; use loopd::tools::Registry; -use support::{FakeServer, Home, Reply}; +use support::{FakeServer, Home, Reply, tokenizer}; // `want` is the whole pointer, written out: scripts/check-runbook.sh reads the anchors of every // pointer in the source, and cannot read one built with `format!`. @@ -150,6 +150,7 @@ home = "{}" fn healthy_routes(server: &FakeServer) { server.route("/props", vec![Reply::fixture("props")]); + server.respond("/tokenize", tokenizer); server.route( "/v1/chat/completions", vec![ diff --git a/crates/loopd/tests/selftest.rs b/crates/loopd/tests/selftest.rs index ad7abec..db99333 100644 --- a/crates/loopd/tests/selftest.rs +++ b/crates/loopd/tests/selftest.rs @@ -4,17 +4,26 @@ mod support; use loopd::llama::Client; use loopd::selftest::{SelfTestError, run}; -use support::{FakeServer, Reply}; +use support::{FakeServer, Reply, tokenizer}; const CHAT: &str = "/v1/chat/completions"; +const TOKENIZE: &str = "/tokenize"; /// A change to the config before the self-test runs. type Edit = fn(&mut loopd::config::Config); -/// A server that passes: the recorded props, then the three recorded completions in order. -fn healthy() -> FakeServer { +/// The recorded props and the fake tokenizer, and no completion yet. +fn props_and_tags() -> FakeServer { let server = FakeServer::start(); server.route("/props", vec![Reply::fixture("props")]); + server.respond(TOKENIZE, tokenizer); + server +} + +/// A server that passes: the recorded props, the fake tokenizer, then the three recorded +/// completions in order. +fn healthy() -> FakeServer { + let server = props_and_tags(); server.route( CHAT, vec![ @@ -38,7 +47,7 @@ fn run_with( } #[test] -fn a_healthy_server_passes_all_three_checks_in_order() { +fn a_healthy_server_passes_all_four_checks_in_order() { let server = healthy(); let (result, steps) = run_with(&server, |_| {}); assert!(result.is_ok(), "{result:?}"); @@ -46,6 +55,7 @@ fn a_healthy_server_passes_all_three_checks_in_order() { steps, [ "server matches config", + "tags tokenize as recorded", "tool call round trip", "turn 2 cache hit" ] @@ -115,27 +125,26 @@ fn each_expected_value_is_checked() { "a server that is not the expected one is not sent any prompt" ); assert!(server.requests_to(CHAT).is_empty()); + assert!(server.requests_to(TOKENIZE).is_empty()); } } #[test] -fn a_completion_without_the_tool_call_fails_the_second_check() { - let server = FakeServer::start(); - server.route("/props", vec![Reply::fixture("props")]); +fn a_completion_without_the_tool_call_fails_the_tool_call_check() { + let server = props_and_tags(); server.route(CHAT, vec![Reply::fixture("plain")]); let (result, steps) = run_with(&server, |_| {}); assert!( matches!(result, Err(SelfTestError::ToolCall(_))), "{result:?}" ); - assert_eq!(steps.len(), 2); + assert_eq!(steps.len(), 3); } #[test] -fn a_cold_second_turn_fails_the_third_check() { +fn a_cold_second_turn_fails_the_cache_check() { // turn1 twice: the "second turn" reuses 15 tokens where 46 were left. - let server = FakeServer::start(); - server.route("/props", vec![Reply::fixture("props")]); + let server = props_and_tags(); server.route( CHAT, vec![ @@ -149,13 +158,12 @@ fn a_cold_second_turn_fails_the_third_check() { Err(SelfTestError::CacheMiss { expected, got }) => assert_eq!((expected, got), (7052, 15)), other => panic!("{other:?}"), } - assert_eq!(steps.len(), 3); + assert_eq!(steps.len(), 4); } #[test] fn a_server_that_is_down_is_an_inference_error_after_retries() { - let server = FakeServer::start(); - server.route("/props", vec![Reply::fixture("props")]); + let server = props_and_tags(); server.route(CHAT, vec![Reply::fixture("tool_call").cut_after(500)]); let (result, _) = run_with(&server, |c| c.limits.retry_attempts = 2); assert!(matches!(result, Err(SelfTestError::Infer(_))), "{result:?}"); diff --git a/crates/loopd/tests/selftest_tags.rs b/crates/loopd/tests/selftest_tags.rs new file mode 100644 index 0000000..6ef1fe7 --- /dev/null +++ b/crates/loopd/tests/selftest_tags.rs @@ -0,0 +1,349 @@ +//! The self-test's tag check (M5b spec, section 8): each of the model's tags must be one token, +//! the one recorded with it, and its neutralised form more than one, or `loopd` does not start. +//! Against the fake server's tokenizer, with one answer changed at a time. The tags come from +//! `bxcore::sanitize::TAGS`, and no assertion prints one (tip T33): a failure names the tag by its +//! id, and the tests check that it does. Do not edit. + +mod support; +#[path = "support/tags.rs"] +mod tags; + +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use bxcore::sanitize::{TAGS, neutralise}; +use loopd::llama::Client; +use loopd::selftest::{SelfTestError, run}; +use support::{FakeServer, Home, Recorded, Reply, tokenizer, tokens}; +use tags::{holds_a_tag, same}; + +const CHAT: &str = "/v1/chat/completions"; +const TOKENIZE: &str = "/tokenize"; +const POINTER: &str = "see docs/runbook.md#loopd-tags-unexpected"; + +/// The recorded props and completions, and `tokenize` as the server's `/tokenize`. +fn server_with(tokenize: impl Fn(&Recorded) -> Reply + Send + Sync + 'static) -> FakeServer { + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route( + CHAT, + vec![ + Reply::fixture("tool_call"), + Reply::fixture("turn1"), + Reply::fixture("turn2"), + Reply::fixture("plain"), + ], + ); + server.respond(TOKENIZE, tokenize); + server +} + +/// The text a `/tokenize` request asked about. +fn content(request: &Recorded) -> String { + let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap_or_default(); + body["content"].as_str().unwrap_or_default().to_string() +} + +/// The fake tokenizer, except that `text` is answered with `ids`. +fn except(text: &str, ids: Vec) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static { + let text = text.to_string(); + move |request: &Recorded| { + if content(request) == text { + tokens(&ids) + } else { + tokenizer(request) + } + } +} + +/// Runs the self-test against `server` with the test config; the result and the steps started. +fn run_on(server: &FakeServer) -> (Result<(), SelfTestError>, Vec) { + let cfg = support::test_config(&server.socket); + let mut steps = Vec::new(); + let result = run(&Client::new(cfg), &mut |s| steps.push(s.to_string())); + (result, steps) +} + +/// A failure of the tag check stops the self-test before any prompt, names the tag by its id, +/// holds no tag, and points at the tags' own runbook entry. +#[track_caller] +fn stopped_before_any_prompt(server: &FakeServer, e: &SelfTestError, steps: &[String], id: u32) { + let text = e.to_string(); + assert!(!holds_a_tag(&text), "tag {id}: the message holds a tag"); + assert!(text.contains(&id.to_string()), "tag {id}: {text}"); + assert_eq!(e.pointer(), POINTER, "tag {id}"); + assert_eq!(steps.len(), 2, "tag {id}: {steps:?}"); + assert!(server.requests_to(CHAT).is_empty(), "tag {id}: no prompt"); +} + +#[test] +fn every_tag_and_its_neutralised_form_tokenize_as_recorded() { + let server = server_with(tokenizer); + let (result, steps) = run_on(&server); + assert!(result.is_ok(), "{result:?}"); + assert_eq!( + steps, + [ + "server matches config", + "tags tokenize as recorded", + "tool call round trip", + "turn 2 cache hit" + ] + ); + + let asked = server.requests_to(TOKENIZE); + assert_eq!( + asked.len(), + 2 * TAGS.len(), + "one request per tag and per form" + ); + let mut want: Vec = TAGS + .iter() + .flat_map(|t| [t.text.to_string(), neutralise(t.text).into_owned()]) + .collect(); + let mut got: Vec = asked.iter().map(content).collect(); + want.sort(); + got.sort(); + same(&got, &want, "the texts asked about"); + for (i, request) in asked.iter().enumerate() { + assert_eq!(request.method, "POST", "request {i}"); + let body = request.json(); + let fields = serde_json::json!({ + "model": "test-model", + "content": content(request), + "add_special": false, + "parse_special": true, + }); + same(&body, &fields, &format!("request {i}: the body")); + } + + // Every tag is checked before the first prompt is sent. + let paths: Vec = server + .requests() + .iter() + .map(|r| r.path().to_string()) + .collect(); + let last_tokenize = paths.iter().rposition(|p| p == TOKENIZE); + let first_chat = paths.iter().position(|p| p == CHAT); + assert!(last_tokenize < first_chat, "{paths:?}"); +} + +#[test] +fn a_tag_that_is_not_one_token_fails() { + for (i, tag) in TAGS.iter().enumerate() { + // Six ordinary tokens, as a tag the tokenizer no longer knows would be; none for the + // first tag. + let ids = if i == 0 { + Vec::new() + } else { + vec![27, 91, 316, 6018, 91, 29] + }; + let count = ids.len(); + let server = server_with(except(tag.text, ids)); + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::TagNotOneToken { id, count: got }) => { + assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id); + stopped_before_any_prompt(&server, e, &steps, tag.id); + } + other => panic!("tag {}: {other:?}", tag.id), + } + } +} + +#[test] +fn a_tag_that_is_another_token_fails() { + for tag in TAGS { + let other = tag.id + 1; + let server = server_with(except(tag.text, vec![other])); + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::TagOtherId { id, got }) => { + assert_eq!((*id, *got), (tag.id, other), "tag {}", tag.id); + stopped_before_any_prompt(&server, e, &steps, tag.id); + } + other => panic!("tag {}: {other:?}", tag.id), + } + } +} + +#[test] +fn a_neutralised_form_that_is_one_token_or_none_fails() { + for (i, tag) in TAGS.iter().enumerate() { + // The tag's own token, as a tokenizer that saw through the escape would give; none for + // the first tag. + let ids = if i == 0 { Vec::new() } else { vec![tag.id] }; + let count = ids.len(); + let server = server_with(except(&neutralise(tag.text), ids)); + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::NeutralisedTooFew { id, count: got }) => { + assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id); + stopped_before_any_prompt(&server, e, &steps, tag.id); + } + other => panic!("tag {}: {other:?}", tag.id), + } + } +} + +#[test] +fn a_neutralised_form_of_two_tokens_passes() { + // "More than one" is the rule; two is enough. + let server = server_with(except(&neutralise(TAGS[0].text), vec![5, 6])); + let (result, steps) = run_on(&server); + assert!(result.is_ok(), "{result:?}"); + assert_eq!(steps.len(), 4); +} + +#[test] +fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() { + let first = TAGS[0].id; + let cases: [(&str, Option); 5] = [ + ("no /tokenize at all", None), + ( + "an HTTP error", + Some(Reply::json( + 500, + r#"{"error":{"code":500,"message":"boom"}}"#, + )), + ), + ( + "a count, not ids", + Some(Reply::json(200, r#"{"tokens": 1}"#)), + ), + ( + "pieces, not ids", + Some(Reply::json(200, r#"{"tokens":[{"id":1,"piece":"x"}]}"#)), + ), + ( + "a negative id", + Some(Reply::json(200, r#"{"tokens":[-1]}"#)), + ), + ]; + for (what, reply) in cases { + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route(CHAT, vec![Reply::fixture("tool_call")]); + if let Some(reply) = reply { + server.route(TOKENIZE, vec![reply]); + } + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::Tokenize { id, .. }) => { + assert_eq!(*id, first, "{what}"); + stopped_before_any_prompt(&server, e, &steps, first); + } + other => panic!("{what}: {other:?}"), + } + assert_eq!(server.requests_to(TOKENIZE).len(), 1, "{what}: no retry"); + } +} + +#[test] +fn the_other_failures_keep_the_self_tests_entry() { + let e = SelfTestError::Hash; + assert_eq!(e.pointer(), "see docs/runbook.md#loopd-selftest-failed"); + let e = SelfTestError::CacheMiss { + expected: 2, + got: 1, + }; + assert_eq!(e.pointer(), "see docs/runbook.md#loopd-selftest-failed"); +} + +/// A `loopd` process that is killed when the test ends, however it ends. +struct Running(Child); + +impl Drop for Running { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +/// Runs `loopd --config ` for at most ten seconds: its exit code (None if it +/// was still running and was killed) and what it printed on standard error. +fn loopd(command: &str, config: &Path) -> (Option, String) { + let child = Command::new(env!("CARGO_BIN_EXE_loopd")) + .args([command, "--config"]) + .arg(config) + .stdout(Stdio::null()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let mut running = Running(child); + let deadline = Instant::now() + Duration::from_secs(10); + let mut code = None; + while Instant::now() < deadline { + if let Some(status) = running.0.try_wait().unwrap() { + code = status.code(); + break; + } + std::thread::sleep(Duration::from_millis(20)); + } + let _ = running.0.kill(); + let _ = running.0.wait(); + let mut stderr = String::new(); + if let Some(mut pipe) = running.0.stderr.take() { + std::io::Read::read_to_string(&mut pipe, &mut stderr).unwrap(); + } + (code, stderr) +} + +fn config_file(home: &Home, server: &FakeServer) -> PathBuf { + let text = format!( + r#" +[infer] +socket = "{}" +model = "test-model" +[slots] +main = 0 +background = 1 +[expect] +template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" +n_ctx = 131072 +slots = 2 +[limits] +poll_ms = 40 +liveness_ms = 500 +retry_backoff_ms = [10] +[paths] +home = "{}" +"#, + server.socket.display(), + home.dir.display() + ); + let path = home.dir.join("config.toml"); + std::fs::write(&path, text).unwrap(); + path +} + +#[test] +fn selftest_and_serve_stop_on_a_tag_and_name_its_entry() { + let home = Home::new(); + // The end-of-turn tag, as six ordinary tokens. + let tag = &TAGS[2]; + let server = server_with(except(tag.text, vec![27, 91, 316, 6018, 91, 29])); + let config = config_file(&home, &server); + for command in ["selftest", "serve"] { + let (code, stderr) = loopd(command, &config); + assert!( + !holds_a_tag(&stderr), + "{command}: what it printed holds a tag" + ); + assert_eq!(code, Some(1), "{command}: {stderr}"); + let line = stderr + .lines() + .find(|l| l.starts_with("selftest: FAILED: ")) + .unwrap_or_else(|| panic!("{command}: no FAILED line: {stderr}")); + assert!(line.ends_with(POINTER), "{command}: {line}"); + assert_eq!(line.matches("runbook.md#").count(), 1, "{command}: {line}"); + assert!(line.contains(&tag.id.to_string()), "{command}: {line}"); + assert!(!stderr.contains("serving on"), "{command}: {stderr}"); + } + assert!( + !home.dir.join("run/loop/loop.sock").exists(), + "serve bound no socket" + ); + assert!(server.requests_to(CHAT).is_empty(), "no prompt was sent"); +} diff --git a/crates/loopd/tests/serve.rs b/crates/loopd/tests/serve.rs index b295257..6b0c873 100644 --- a/crates/loopd/tests/serve.rs +++ b/crates/loopd/tests/serve.rs @@ -5,7 +5,7 @@ mod support; use std::os::unix::fs::PermissionsExt; use std::process::{Command, Stdio}; use std::time::Duration; -use support::{FakeServer, Home, Reply}; +use support::{FakeServer, Home, Reply, tokenizer}; fn config_file(home: &Home, server: &FakeServer, expect_slots: u32) -> std::path::PathBuf { let text = format!( @@ -37,6 +37,7 @@ home = "{}" fn healthy_routes(server: &FakeServer) { server.route("/props", vec![Reply::fixture("props")]); + server.respond("/tokenize", tokenizer); server.route( "/v1/chat/completions", vec![ @@ -82,6 +83,11 @@ fn serve_runs_the_self_test_then_binds_the_socket_with_mode_0600() { 3, "the three self-test completions ran" ); + assert_eq!( + server.requests_to("/tokenize").len(), + 2 * bxcore::sanitize::TAGS.len(), + "the tag check ran: each tag and its neutralised form" + ); } #[test] diff --git a/crates/loopd/tests/support/mod.rs b/crates/loopd/tests/support/mod.rs index 1642767..2092dab 100644 --- a/crates/loopd/tests/support/mod.rs +++ b/crates/loopd/tests/support/mod.rs @@ -3,7 +3,8 @@ //! It listens on a Unix socket in a temporary directory. Each path has a list of replies that are //! served in order; the last one repeats. A reply is raw bytes, normally a response recorded from //! the real server (`tests/fixtures/http/*.http`), and can be delayed, sent in small pieces, cut -//! short, or left hanging. Every request is recorded. +//! short, or left hanging. A path can instead compute each reply from its request, as the fake +//! tokenizer does. Every request is recorded. #![allow(dead_code)] // each test file uses a different part of this module @@ -171,6 +172,26 @@ impl loopd::tools::ToolPort for ReturnsPendingPort { } } +/// The fake `/tokenize`, for `FakeServer::respond`: the text of the request's `content` is one +/// token when it is one of the model's tags (`bxcore::sanitize::TAGS`), the tag's id, and +/// otherwise one token per byte, the byte's value. A body without a string `content` gets a 400. +pub fn tokenizer(request: &Recorded) -> Reply { + let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap_or_default(); + let Some(content) = body.get("content").and_then(|c| c.as_str()) else { + return Reply::json(400, r#"{"error":"no content"}"#); + }; + let ids: Vec = match bxcore::sanitize::TAGS.iter().find(|t| t.text == content) { + Some(tag) => vec![tag.id], + None => content.bytes().map(u32::from).collect(), + }; + tokens(&ids) +} + +/// A `/tokenize` answer as llama-server gives one without `with_pieces`: `{"tokens": [ids]}`. +pub fn tokens(ids: &[u32]) -> Reply { + Reply::json(200, &serde_json::json!({ "tokens": ids }).to_string()) +} + /// What an independent script derived from a recorded stream: `tests/fixtures/expected/`. pub fn expected(name: &str) -> serde_json::Value { serde_json::from_slice(&fixture_bytes("expected", &format!("{name}.json"))).unwrap() @@ -273,8 +294,12 @@ impl Recorded { } } +/// A reply computed from its request. +type Responder = Arc Reply + Send + Sync>; + struct State { routes: Mutex)>>, + responders: Mutex>, requests: Mutex>, } @@ -295,6 +320,7 @@ impl FakeServer { let listener = UnixListener::bind(&socket).unwrap(); let state = Arc::new(State { routes: Mutex::new(Vec::new()), + responders: Mutex::new(Vec::new()), requests: Mutex::new(Vec::new()), }); let accept_state = Arc::clone(&state); @@ -308,14 +334,29 @@ impl FakeServer { FakeServer { socket, state, dir } } - /// Replies for `path` (the query is ignored), served in order. The last one repeats. + /// Replies for `path` (the query is ignored), served in order. The last one repeats. They + /// replace a responder for `path`. pub fn route(&self, path: &str, replies: Vec) { assert!(!replies.is_empty()); + self.state + .responders + .lock() + .unwrap() + .retain(|(p, _)| p != path); let mut routes = self.state.routes.lock().unwrap(); routes.retain(|(p, _)| p != path); routes.push((path.to_string(), replies.into())); } + /// For each request to `path` (the query is ignored), the reply `reply` computes from it. It + /// replaces the replies `route` gave `path`. + pub fn respond(&self, path: &str, reply: impl Fn(&Recorded) -> Reply + Send + Sync + 'static) { + self.state.routes.lock().unwrap().retain(|(p, _)| p != path); + let mut responders = self.state.responders.lock().unwrap(); + responders.retain(|(p, _)| p != path); + responders.push((path.to_string(), Arc::new(reply))); + } + pub fn requests(&self) -> Vec { self.state.requests.lock().unwrap().clone() } @@ -376,15 +417,23 @@ fn serve(mut stream: UnixStream, state: &State) { return; }; let path = request.path().to_string(); + let responder = state + .responders + .lock() + .unwrap() + .iter() + .find(|(p, _)| *p == path) + .map(|(_, reply)| Arc::clone(reply)); + let computed = responder.map(|reply| reply(&request)); state.requests.lock().unwrap().push(request); - let reply = { + let reply = computed.or_else(|| { let mut routes = state.routes.lock().unwrap(); match routes.iter_mut().find(|(p, _)| *p == path) { Some((_, replies)) if replies.len() > 1 => replies.pop_front(), Some((_, replies)) => replies.front().cloned(), None => None, } - }; + }); let reply = reply.unwrap_or_else(|| Reply::json(404, r#"{"error":"no route in the fake server"}"#)); thread::sleep(Duration::from_millis(reply.head_delay_ms)); diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 23b50bd..77e5e65 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone. | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| M5b/08-loopd-selftest-tags | 2026-09-26 | done | 2 | pass | none | Copied the five given test files from `docs/plans/M5b/files/`: `selftest_tags.rs` (new) and new versions of `selftest.rs`, `serve.rs`, `pointers.rs` and `support/mod.rs` (did not open them). Step 2 showed exactly the four named failures (`selftest`: three steps without `"tags tokenize as recorded"`; `a_completion_without_the_tool_call_fails_the_tool_call_check` left 2 right 3; `a_cold_second_turn_fails_the_cache_check` left 3 right 4; `serve_runs_the_self_test_then_binds_the_socket_with_mode_0600` left 0 right 66) and `selftest_tags.rs` failing to compile with 7 E0599s (no `TagNotOneToken`/`TagOtherId`/`NeutralisedTooFew`/`Tokenize` variant, no `pointer`). Appended `runbook-loopd-tags-unexpected.md`; `check-runbook.sh` printed nothing and exited 0, `grep -c '^## loopd-tags-unexpected$'` printed 1. Made the edits in order: I1 the private `TokenIds` struct before `CacheOutcome` and I2 the `pub token_ids` method after `tokenize` (left `tokenize` itself unchanged); S1 the `use bxcore::sanitize::{TAGS, Tag, neutralise}` import, S2 the four tag variants plus `SelfTestError::pointer` with each pointer written whole, S3 the four Display arms, S4 the second check `tags tokenize as recorded`, S5 the `tags_tokenize` and `check_tag` functions, S6 check 3 to check 4; M1 `e.pointer()` in `run_selftest_check`. `cargo fmt --all`; `cargo check -p loopd` clean; `selftest_tags` 8, `selftest` 6, `serve` 3, `pointers` 7 passed, every `loopd` suite passes. `grep -rn 'runbook.md#loopd-' crates/loopd/src` prints exactly two lines, both in `selftest.rs`. `make gate` printed `gate: ok` with 1050 tests (1042 before this task, plus the 8 in `selftest_tags.rs`). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | | M5b/07-loopd-neutralise | 2026-09-26 | done | 2 | fail | none | Copied the three given tests from `docs/plans/M5b/files/`: `neutralise.rs`, `neutralise_replay.rs` and `support/tags.rs` (did not open them). Step 2 showed exactly the seven named failures (all six in `neutralise`, `the_memory_block_is_neutralised_when_the_baseline_is_assembled` in `neutralise_replay`); the two that should pass passed. Made the edits in order: T1 the module comment, T2 the `use bxcore::sanitize::neutralise` import, T3 `content: neutralise(content).into_owned()` on the `LogRecord::User`, T4 the capped-then-neutralised `let text = neutralise(&text).into_owned()` on the `LogRecord::ToolResult`; B1 the import, B2 the `assemble_with` comment, B3 `neutralise(memory.trim())` with `&memory` pushed. `cargo fmt --all`; `cargo check -p loopd` clean; `neutralise` 6 passed, `neutralise_replay` 3 passed, every `loopd` suite passes. `grep -rn 'neutralise(' crates/loopd/src` prints exactly three lines (two in `turn.rs`, one in `baseline.rs`). First `make gate` run failed on a flaky brokerd concurrency test, `a_torn_tail_is_recovered_and_it_serves` (one fail out of the serve suite under full-workspace load); it passes 3/3 in isolation and the serve suite 9/9 three times on the same tree, and the loopd edits touch no brokerd path, so it is a timing flake, not this task. Re-ran `make gate`: `gate: ok` with 1042 tests. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | | M5b/06-memoryd-sanitize | 2026-09-26 | done | 1 | pass | none | Copied the five given tests from `docs/plans/M5b/files/`: `sanitize.rs`, `serve_sanitize.rs`, and new versions of `edit.rs`, `files.rs` and `serve.rs` (did not open them). Step 2 showed exactly the seven named failures (`refusals_change_nothing` in `edit`, `an_entry_that_fails_the_check_is_left_out_and_logged` in `serve`, all five in `serve_sanitize`); `files` did not compile (one E0609, no field `rule` on `&memoryd::check::Refused`) and `sanitize` did not compile (13: E0432 for `memoryd::edit::flagged`/`refused`, E0609 for `flag` on `Outcome` and `&Record` and `rule` on `Refused`, E0615 for `sentence`). Appended the three runbook fragments to `docs/runbook.md`; `check-runbook.sh` printed nothing and exited 0. `git rm`'d `crates/memoryd/src/check.rs` and its test. Made the edits in order: E1 removed `pub mod check;` from `lib.rs`; E2-E10 in `edit.rs` (imports, `Outcome` gains `flag`, five functions replacing `checked`, the refuse check in `apply`, the flag check after `edited`, `flag: None`, `text.trim()` in `edited`, `noted` plus `as_notes`, `text.trim()` in `consolidate`); E11-E13 in `files.rs` (imports, `Block`, the two `block` comments, the `refuse` match); E14-E19 in `journal.rs` (`Record` gains `flag` after `untrusted`, `record` sets `flag: outcome.flag.map(str::to_string)`, `log` appends ` flagged {rule}`, `undo` sets `flag: None`); E20 in `review.rs` (`reject` sets `flag: None`); E21-E23 in `serve.rs` (`block` logs by `refusal.rule` with the runbook pointer, the `memory` comment, and `memory` calls `edit::refused` first then `edit::flagged` to keep flagged writes off review and logs the rule). `cargo fmt --all`; `cargo check -p memoryd` clean; `cargo test -p memoryd` every suite passes (sanitize 11, serve_sanitize 5, edit 8, files 8, serve 8); `make gate` printed `gate: ok` with 1033 tests; `grep -rn 'memoryd::check\|crate::check' crates/` prints nothing. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | | M5b/05-sanitize-flag | 2026-09-26 | done | 1 | pass | none | Copied the three given tests from `docs/plans/M5b/files/`: `bxcore/tests/sanitize_flag.rs`, `sanitize_flag_corpus.rs`, `sanitize_flag_edges.rs`. Step 2 showed exactly the named E0432s (`Flag`/`flag`/`normalise` for `sanitize_flag`, `flag` for the other two). Copied the two skeletons, `bxcore/src/sanitize/normalise.rs` (five functions) and `flag.rs` (seven), then made the two edits: E1 replaced `bxcore/src/sanitize/mod.rs` whole (added `mod flag; mod normalise;` and `pub use flag::{Flag, flag};` / `pub use normalise::normalise;`), E2 made `invisible` `pub(super)` in `refuse.rs` (nothing else changed). Filled `normalise.rs` in the given order, each from its comment and `cargo check` after: `leet` (`LEET.iter().find` the digit, `map_or(c, |&(_, letter)| letter)`); `homoglyph` (`HOMOGLYPHS.iter().find` the `from`, `map` the `to`); `full_width` (`u32::from(c)` inside `FULL_WIDTH`, `checked_sub(FULL_WIDTH_OFFSET).and_then(char::from_u32).unwrap_or(c)`); `push_folded` (`full_width` then `homoglyph`, pushing the letter when `Some`, else `c.to_lowercase().map(leet)`); `normalise` (walk chars, `space` bool: whitespace sets it, `invisible` dropped leaving it, any other char pushes one `' '` when `space` and the output is not empty, clears `space`, then `push_folded`). Filled `flag.rs` in the given order: `in_sentence` (`!word.ends_with(['.','!','?',';']) && !["&&","||","|"].contains(&word)`); `filler` (`!word.is_empty() && word.chars().all(|c| c.is_alphanumeric() || c == '_')`); `is` (trim non-word ends with `trim_matches`, compare the word and its stripped form to each keyword); `ends_clause` (`word.ends_with(CLAUSE_ENDS)`, or `after.is_empty()`, or `after` starts with a `CLAUSE_FOLLOWERS` matched word by word as `is`); `one_word` (`split_first`, `test` the first word with the words after it, then `seq_at` the remaining parts on those words); `skip` (counts `0..=max`, `split_at_checked`: a skipped word failing `test` or a count past the end returns `false`, otherwise when all pass check `seq_at`); `seq_at` (`split_first`; `Is`/`Ends`/`Holds` go to `one_word` with `is`, `is && ends_clause`, `contains`, `Filler`/`Gap` to `skip` with `filler`/`in_sentence`). My first `one_word` edit dropped a `]` and `)` from the `impl Fn` signature; caught by diffing against the skeleton and fixed. After the fills `cargo check -p bxcore` is clean. `sanitize_flag` 22 passed, `sanitize_flag_corpus` 2, `sanitize_flag_edges` 2; `sanitize_refuse` 14, `sanitize_refuse_corpus` 2, `sanitize_tags` 15, `sanitize_corpus` 3. `cargo fmt --all`; `make gate` printed `gate: ok`; `cargo test --workspace` reports 1022 passed, plus the gate's separate loopd `end_to_end` ignored test, for 1023 total. No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b | diff --git a/docs/runbook.md b/docs/runbook.md index 97baee8..04498f4 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -728,3 +728,57 @@ Entries are separated by a line holding only `§`. A secret does not belong in m if it was real. **Check.** Start a new conversation: `memoryd` does not log the line again. + +## loopd-tags-unexpected + +**What you see.** `loopd selftest` or `loopd serve` prints `selftest: tags tokenize as recorded`, +then `selftest: FAILED: ; see docs/runbook.md#loopd-tags-unexpected`, and exits 1. No +prompt was sent. The reason names a tag by its token id, never by its text: + +- `the tag with id is tokens, not one` +- `the tag with id is one token, but the token ` +- `the neutralised form of the tag with id does not split into several tokens (the server + gave )` +- `could not tokenize the tag with id : ` + +**Why.** `loopd` neutralises the model's chat-template tags in every text from outside, writing +their angle brackets as `<` and `>` (M5b spec, sections 6.2 and 8). The list of tags, each +with its token id, is compiled into `bxcore` (`crates/bxcore/src/sanitize/tags.rs`), as measured +on straylight. At startup `loopd` asks the server's `/tokenize` about each tag and its neutralised +form. If a tag is no longer the one token recorded with it, the model or its tokenizer has changed +and the list may no longer match it; if a neutralised form is a single token, neutralising no +longer keeps that tag from the model. Either way the model could receive text from outside as its +own control tokens, so `loopd` does not start. The check cannot find a new tag missing from the +list: the server does not list its special tokens. + +**Confirm.** Find the tag: `grep -n '' crates/bxcore/src/sanitize/tags.rs` shows its entry, +built from its name with `bar!`, `open!` or `close!`. Ask the server yourself, building the tag +from its name as the file does. For a `bar!` tag named `im_end`: + +```sh +printf '{"model":"ornith-1.5-35b-a3b","content":"<|%s|>","parse_special":true}' im_end | + curl -s -d @- http://straylight:11434/tokenize +printf '{"model":"ornith-1.5-35b-a3b","content":"<|%s|>","parse_special":true}' im_end | + curl -s -d @- http://straylight:11434/tokenize +``` + +For an `open!` tag the content is `<%s>`, for a `close!` tag ``. The first answer should be +`{"tokens":[]}`, the second a list of several ids. Check which model file the server loaded: +`curl -s 'http://straylight:11434/props?model=ornith-1.5-35b-a3b' | jq .model_path`. + +**Fix, by reason.** + +- `could not tokenize …`: the server did not answer `/tokenize`. A router in front of + llama-server may not forward it (crossbar does not, `docs/decisions.md`, P22): point + `inferproxy` at llama-server itself. Otherwise check that the server is up: + `curl -s http://straylight:11434/health`, and run `loopd selftest --config ` again. +- A tag with another id, or several tokens: the model file changed. Do not run `loopd` against + it. Measure the new model's tags as M5b task 03 did (every vocabulary entry of type control or + user-defined, each checked with `/tokenize`), and change `TAGS` and the tests' list in + `crates/bxcore/tests/support/tags.rs` together; the change reaches the agent when it is rebuilt. + A new model also invalidates the measurements in `docs/inference-contract.md`. +- A neutralised form that is one token: the tokenizer has an entry for the escaped spelling, and + the replacement of the M5b spec, section 6.2, no longer works for that tag. That is a design + change: record it in `docs/decisions.md` and do not run `loopd` until it is settled. + +**Check.** `loopd selftest --config ` prints `selftest: ok`.