diff --git a/crates/loopd/src/selftest.rs b/crates/loopd/src/selftest.rs index 1e72be0..b308771 100644 --- a/crates/loopd/src/selftest.rs +++ b/crates/loopd/src/selftest.rs @@ -3,7 +3,7 @@ //! parsed, and that a second turn reuses the first turn's cache. Any check failing means `loopd` //! does not start. -use bxcore::sanitize::{TAGS, Tag, neutralise}; +use bxcore::sanitize::{TAGS, neutralise}; use crate::llama::{ ChatMessage, ChatRequest, Client, FinishReason, InferError, ToolSchema, @@ -24,25 +24,18 @@ pub enum SelfTestError { }, Infer(InferError), Hash, - /// The tag with the recorded id `id` is `count` tokens, not one. - TagNotOneToken { + /// The first tag, by its recorded id, whose token is not in its place in the server's answer + /// about the tags (or, past the last tag, an extra tag token the server gave). + TagNotRecorded { id: u32, - count: usize, }, - /// The tag with the recorded id `id` is one token, but the token `got`. - TagOtherId { + /// The server's answer about the neutralised forms holds the tag token `id`. + NeutralisedIsTag { id: u32, - got: u32, }, - /// The neutralised form of the tag with the recorded id `id` is `count` tokens, not more - /// than one. - NeutralisedTooFew { - id: u32, - count: usize, - }, - /// `/tokenize` failed for the tag with the recorded id `id`, or for its neutralised form. + /// `/tokenize` failed; `what` is "the tags" or "the neutralised tags". Tokenize { - id: u32, + what: &'static str, error: InferError, }, } @@ -52,9 +45,8 @@ impl SelfTestError { /// its own entry; every other failure shares the self-test's. pub fn pointer(&self) -> &'static str { match self { - SelfTestError::TagNotOneToken { .. } - | SelfTestError::TagOtherId { .. } - | SelfTestError::NeutralisedTooFew { .. } + SelfTestError::TagNotRecorded { .. } + | SelfTestError::NeutralisedIsTag { .. } | SelfTestError::Tokenize { .. } => "see docs/runbook.md#loopd-tags-unexpected", SelfTestError::Mismatch { .. } | SelfTestError::ToolCall(_) @@ -82,18 +74,17 @@ impl std::fmt::Display for SelfTestError { ), SelfTestError::Infer(e) => write!(f, "{e}"), SelfTestError::Hash => write!(f, "the chat template could not be hashed"), - SelfTestError::TagNotOneToken { id, count } => { - write!(f, "the tag with id {id} is {count} tokens, not one") + SelfTestError::TagNotRecorded { id } => { + write!( + f, + "the tag with id {id} does not tokenize as its recorded token" + ) } - SelfTestError::TagOtherId { id, got } => { - write!(f, "the tag with id {id} is one token, but the token {got}") + SelfTestError::NeutralisedIsTag { id } => { + write!(f, "a neutralised tag tokenizes to the tag with id {id}") } - SelfTestError::NeutralisedTooFew { id, count } => write!( - f, - "the neutralised form of the tag with id {id} does not split into several tokens (the server gave {count})" - ), - SelfTestError::Tokenize { id, error } => { - write!(f, "could not tokenize the tag with id {id}: {error}") + SelfTestError::Tokenize { what, error } => { + write!(f, "could not tokenize {what}: {error}") } } } @@ -161,42 +152,49 @@ fn server_matches(client: &Client) -> Result<(), SelfTestError> { Ok(()) } -/// Check 2: each of the model's tags (`bxcore::sanitize::TAGS`) is one token, the one recorded -/// with it, and its neutralised form is more than one (M5b spec, section 8). Like check 1 it sends -/// no prompt and uses no slot, so a server whose tokenizer is not the recorded one is sent no -/// prompt either. A tag missing from the list cannot be found: the server does not list them. +/// Check 2 (M5b spec, section 8): asked about every tag of `bxcore::sanitize::TAGS` at once, one +/// per line, the server gives each tag's recorded token, in the list's order; asked about every +/// neutralised form, it gives none of those tokens. Each tag is a special token, which the +/// tokenizer splits out before anything else, so the lines between cannot change it. Two requests, +/// so that the check fits in `inferproxy`'s burst of connections. Like check 1 it sends no prompt +/// and uses no slot. A tag missing from the list cannot be found: the server does not list them. +/// An error names a tag by its id, never by its text, which the operator's terminal and log would +/// show whole. fn tags_tokenize(client: &Client) -> Result<(), SelfTestError> { - for tag in TAGS { - check_tag(client, tag)?; + let recorded: Vec = TAGS.iter().map(|tag| tag.id).collect(); + let tags: Vec<&str> = TAGS.iter().map(|tag| tag.text).collect(); + let got: Vec = client + .token_ids(&tags.join("\n")) + .map_err(|error| SelfTestError::Tokenize { + what: "the tags", + error, + })? + .into_iter() + .filter(|id| recorded.contains(id)) + .collect(); + let places = recorded.len().max(got.len()); + if let Some(at) = (0..places).find(|&at| got.get(at) != recorded.get(at)) { + let id = recorded + .get(at) + .or(got.get(at)) + .copied() + .unwrap_or_default(); + return Err(SelfTestError::TagNotRecorded { id }); } - Ok(()) -} - -/// Two `/tokenize` requests: `tag.text`, then `neutralise(tag.text)`. An error names the tag by -/// its id, never by its text, which the operator's terminal and log would show whole. -fn check_tag(client: &Client, tag: &Tag) -> Result<(), SelfTestError> { - let id = tag.id; - let tokenize = |text: &str| { - client - .token_ids(text) - .map_err(|error| SelfTestError::Tokenize { id, error }) - }; - let ids = tokenize(tag.text)?; - match ids.as_slice() { - [got] if *got == id => {} - [got] => return Err(SelfTestError::TagOtherId { id, got: *got }), - _ => { - return Err(SelfTestError::TagNotOneToken { - id, - count: ids.len(), - }); - } + let forms: Vec = TAGS + .iter() + .map(|tag| neutralise(tag.text).into_owned()) + .collect(); + let got = client + .token_ids(&forms.join("\n")) + .map_err(|error| SelfTestError::Tokenize { + what: "the neutralised tags", + error, + })?; + match got.into_iter().find(|id| recorded.contains(id)) { + Some(id) => Err(SelfTestError::NeutralisedIsTag { id }), + None => Ok(()), } - let count = tokenize(&neutralise(tag.text))?.len(); - if count <= 1 { - return Err(SelfTestError::NeutralisedTooFew { id, count }); - } - Ok(()) } /// Check 3: a tool call comes back parsed, with a `read_file` tool and a string `path`. diff --git a/crates/loopd/tests/selftest_tags.rs b/crates/loopd/tests/selftest_tags.rs index 6ef1fe7..6af15b6 100644 --- a/crates/loopd/tests/selftest_tags.rs +++ b/crates/loopd/tests/selftest_tags.rs @@ -1,8 +1,10 @@ -//! The self-test's tag check (M5b spec, section 8): each of the model's tags must be one token, -//! the one recorded with it, and its neutralised form more than one, or `loopd` does not start. -//! Against the fake server's tokenizer, with one answer changed at a time. The tags come from +//! The self-test's tag check (M5b spec, section 8): asked about all the model's tags at once, +//! the server must give each one's recorded token, in order; asked about all their neutralised +//! forms, none of those tokens. Two `/tokenize` requests, so that the check fits in `inferproxy`'s +//! burst of connections (found on straylight in the M5b device check). Against the fake server's +//! tokenizer, with one tag's answer changed at a time. The tags come from //! `bxcore::sanitize::TAGS`, and no assertion prints one (tip T33): a failure names the tag by its -//! id, and the tests check that it does. Do not edit. +//! id, and the tests check that it does. mod support; #[path = "support/tags.rs"] @@ -15,7 +17,7 @@ use std::time::{Duration, Instant}; use bxcore::sanitize::{TAGS, neutralise}; use loopd::llama::Client; use loopd::selftest::{SelfTestError, run}; -use support::{FakeServer, Home, Recorded, Reply, tokenizer, tokens}; +use support::{FakeServer, Home, Recorded, Reply, tokenize, tokenizer, tokens}; use tags::{holds_a_tag, same}; const CHAT: &str = "/v1/chat/completions"; @@ -45,15 +47,33 @@ fn content(request: &Recorded) -> String { body["content"].as_str().unwrap_or_default().to_string() } -/// The fake tokenizer, except that `text` is answered with `ids`. -fn except(text: &str, ids: Vec) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static { - let text = text.to_string(); +/// The texts the check asks about: every tag, one per line; then every neutralised form. +fn asked_texts() -> [String; 2] { + let tags: Vec<&str> = TAGS.iter().map(|t| t.text).collect(); + let forms: Vec = TAGS + .iter() + .map(|t| neutralise(t.text).into_owned()) + .collect(); + [tags.join("\n"), forms.join("\n")] +} + +/// The fake tokenizer, except that the tag `text` is answered with `ids` wherever it appears. +fn except( + text: &'static str, + ids: Vec, +) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static { + move |request: &Recorded| tokens(&tokenize(&content(request), &[(text, ids.clone())])) +} + +/// The fake tokenizer, except that the neutralised forms' answer also holds `id`. +fn forms_holding(id: u32) -> impl Fn(&Recorded) -> Reply + Send + Sync + 'static { move |request: &Recorded| { - if content(request) == text { - tokens(&ids) - } else { - tokenizer(request) + let text = content(request); + let mut ids = tokenize(&text, &[]); + if text == asked_texts()[1] { + ids.insert(ids.len() / 2, id); } + tokens(&ids) } } @@ -65,20 +85,27 @@ fn run_on(server: &FakeServer) -> (Result<(), SelfTestError>, Vec) { (result, steps) } -/// A failure of the tag check stops the self-test before any prompt, names the tag by its id, -/// holds no tag, and points at the tags' own runbook entry. +/// A failure of the tag check stops the self-test before any prompt, holds no tag, names `id` if +/// given, and points at the tags' own runbook entry. #[track_caller] -fn stopped_before_any_prompt(server: &FakeServer, e: &SelfTestError, steps: &[String], id: u32) { +fn stopped_before_any_prompt( + server: &FakeServer, + e: &SelfTestError, + steps: &[String], + id: Option, +) { let text = e.to_string(); - assert!(!holds_a_tag(&text), "tag {id}: the message holds a tag"); - assert!(text.contains(&id.to_string()), "tag {id}: {text}"); - assert_eq!(e.pointer(), POINTER, "tag {id}"); - assert_eq!(steps.len(), 2, "tag {id}: {steps:?}"); - assert!(server.requests_to(CHAT).is_empty(), "tag {id}: no prompt"); + assert!(!holds_a_tag(&text), "{id:?}: the message holds a tag"); + if let Some(id) = id { + assert!(text.contains(&id.to_string()), "tag {id}: {text}"); + } + assert_eq!(e.pointer(), POINTER, "{id:?}"); + assert_eq!(steps.len(), 2, "{id:?}: {steps:?}"); + assert!(server.requests_to(CHAT).is_empty(), "{id:?}: no prompt"); } #[test] -fn every_tag_and_its_neutralised_form_tokenize_as_recorded() { +fn the_tags_then_their_neutralised_forms_are_asked_in_two_requests() { let server = server_with(tokenizer); let (result, steps) = run_on(&server); assert!(result.is_ok(), "{result:?}"); @@ -95,30 +122,27 @@ fn every_tag_and_its_neutralised_form_tokenize_as_recorded() { let asked = server.requests_to(TOKENIZE); assert_eq!( asked.len(), - 2 * TAGS.len(), - "one request per tag and per form" + 2, + "one request for the tags, one for the forms" + ); + let got: Vec = asked.iter().map(content).collect(); + same( + &got, + &asked_texts().to_vec(), + "the texts asked about, in order", ); - let mut want: Vec = TAGS - .iter() - .flat_map(|t| [t.text.to_string(), neutralise(t.text).into_owned()]) - .collect(); - let mut got: Vec = asked.iter().map(content).collect(); - want.sort(); - got.sort(); - same(&got, &want, "the texts asked about"); for (i, request) in asked.iter().enumerate() { assert_eq!(request.method, "POST", "request {i}"); - let body = request.json(); let fields = serde_json::json!({ "model": "test-model", "content": content(request), "add_special": false, "parse_special": true, }); - same(&body, &fields, &format!("request {i}: the body")); + same(&request.json(), &fields, &format!("request {i}: the body")); } - // Every tag is checked before the first prompt is sent. + // Both are asked before the first prompt is sent. let paths: Vec = server .requests() .iter() @@ -130,38 +154,59 @@ fn every_tag_and_its_neutralised_form_tokenize_as_recorded() { } #[test] -fn a_tag_that_is_not_one_token_fails() { - for (i, tag) in TAGS.iter().enumerate() { - // Six ordinary tokens, as a tag the tokenizer no longer knows would be; none for the - // first tag. - let ids = if i == 0 { - Vec::new() - } else { - vec![27, 91, 316, 6018, 91, 29] - }; - let count = ids.len(); - let server = server_with(except(tag.text, ids)); - let (result, steps) = run_on(&server); - match &result { - Err(e @ SelfTestError::TagNotOneToken { id, count: got }) => { - assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id); - stopped_before_any_prompt(&server, e, &steps, tag.id); - } - other => panic!("tag {}: {other:?}", tag.id), - } - } -} - -#[test] -fn a_tag_that_is_another_token_fails() { +fn a_tag_that_is_ordinary_tokens_fails_and_is_named() { for tag in TAGS { - let other = tag.id + 1; - let server = server_with(except(tag.text, vec![other])); + // Six ordinary tokens, as a tag the tokenizer no longer knows would be. + let server = server_with(except(tag.text, vec![27, 91, 316, 6018, 91, 29])); let (result, steps) = run_on(&server); match &result { - Err(e @ SelfTestError::TagOtherId { id, got }) => { - assert_eq!((*id, *got), (tag.id, other), "tag {}", tag.id); - stopped_before_any_prompt(&server, e, &steps, tag.id); + Err(e @ SelfTestError::TagNotRecorded { id }) => { + assert_eq!(*id, tag.id, "tag {}", tag.id); + stopped_before_any_prompt(&server, e, &steps, Some(tag.id)); + } + other => panic!("tag {}: {other:?}", tag.id), + } + assert_eq!( + server.requests_to(TOKENIZE).len(), + 1, + "tag {}: the forms are not asked", + tag.id + ); + } +} + +#[test] +fn a_tag_that_is_another_token_fails_and_is_named() { + for tag in TAGS { + // Another tag's token, and a token outside the list. + let listed = if tag.id == TAGS[0].id { + TAGS[1].id + } else { + TAGS[0].id + }; + for other in [listed, 7] { + let server = server_with(except(tag.text, vec![other])); + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::TagNotRecorded { id }) => { + assert_eq!(*id, tag.id, "tag {} as {other}", tag.id); + stopped_before_any_prompt(&server, e, &steps, Some(tag.id)); + } + other_result => panic!("tag {} as {other}: {other_result:?}", tag.id), + } + } + } +} + +#[test] +fn a_tag_that_is_its_token_twice_fails() { + // A middle tag shifts every later place; the last one leaves an extra token at the end. + for tag in [&TAGS[4], &TAGS[TAGS.len() - 1]] { + let server = server_with(except(tag.text, vec![tag.id, tag.id])); + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::TagNotRecorded { .. }) => { + stopped_before_any_prompt(&server, e, &steps, None); } other => panic!("tag {}: {other:?}", tag.id), } @@ -169,18 +214,14 @@ fn a_tag_that_is_another_token_fails() { } #[test] -fn a_neutralised_form_that_is_one_token_or_none_fails() { - for (i, tag) in TAGS.iter().enumerate() { - // The tag's own token, as a tokenizer that saw through the escape would give; none for - // the first tag. - let ids = if i == 0 { Vec::new() } else { vec![tag.id] }; - let count = ids.len(); - let server = server_with(except(&neutralise(tag.text), ids)); +fn a_neutralised_form_that_holds_a_tag_fails_and_names_it() { + for tag in TAGS { + let server = server_with(forms_holding(tag.id)); let (result, steps) = run_on(&server); match &result { - Err(e @ SelfTestError::NeutralisedTooFew { id, count: got }) => { - assert_eq!((*id, *got), (tag.id, count), "tag {}", tag.id); - stopped_before_any_prompt(&server, e, &steps, tag.id); + Err(e @ SelfTestError::NeutralisedIsTag { id }) => { + assert_eq!(*id, tag.id, "tag {}", tag.id); + stopped_before_any_prompt(&server, e, &steps, Some(tag.id)); } other => panic!("tag {}: {other:?}", tag.id), } @@ -188,9 +229,16 @@ fn a_neutralised_form_that_is_one_token_or_none_fails() { } #[test] -fn a_neutralised_form_of_two_tokens_passes() { - // "More than one" is the rule; two is enough. - let server = server_with(except(&neutralise(TAGS[0].text), vec![5, 6])); +fn neutralised_forms_of_ordinary_tokens_pass_however_few() { + // Only a tag's own token is dangerous; a form the tokenizer happens to know as one ordinary + // token is not. + let server = server_with(move |request: &Recorded| { + if content(request) == asked_texts()[1] { + tokens(&[5]) + } else { + tokenizer(request) + } + }); let (result, steps) = run_on(&server); assert!(result.is_ok(), "{result:?}"); assert_eq!(steps.len(), 4); @@ -198,7 +246,6 @@ fn a_neutralised_form_of_two_tokens_passes() { #[test] fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() { - let first = TAGS[0].id; let cases: [(&str, Option); 5] = [ ("no /tokenize at all", None), ( @@ -230,9 +277,12 @@ fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() { } let (result, steps) = run_on(&server); match &result { - Err(e @ SelfTestError::Tokenize { id, .. }) => { - assert_eq!(*id, first, "{what}"); - stopped_before_any_prompt(&server, e, &steps, first); + Err(e @ SelfTestError::Tokenize { .. }) => { + assert!( + e.to_string().starts_with("could not tokenize the tags: "), + "{what}: {e}" + ); + stopped_before_any_prompt(&server, e, &steps, None); } other => panic!("{what}: {other:?}"), } @@ -240,6 +290,32 @@ fn a_tokenizer_that_fails_or_answers_wrong_fails_closed() { } } +#[test] +fn a_tokenizer_that_fails_on_the_forms_fails_closed() { + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route(CHAT, vec![Reply::fixture("tool_call")]); + server.respond(TOKENIZE, |request: &Recorded| { + if content(request) == asked_texts()[0] { + tokenizer(request) + } else { + Reply::json(500, r#"{"error":{"code":500,"message":"boom"}}"#) + } + }); + let (result, steps) = run_on(&server); + match &result { + Err(e @ SelfTestError::Tokenize { .. }) => { + assert!( + e.to_string() + .starts_with("could not tokenize the neutralised tags: "), + "{e}" + ); + stopped_before_any_prompt(&server, e, &steps, None); + } + other => panic!("{other:?}"), + } +} + #[test] fn the_other_failures_keep_the_self_tests_entry() { let e = SelfTestError::Hash; diff --git a/crates/loopd/tests/serve.rs b/crates/loopd/tests/serve.rs index 6b0c873..b0313eb 100644 --- a/crates/loopd/tests/serve.rs +++ b/crates/loopd/tests/serve.rs @@ -85,8 +85,8 @@ fn serve_runs_the_self_test_then_binds_the_socket_with_mode_0600() { ); assert_eq!( server.requests_to("/tokenize").len(), - 2 * bxcore::sanitize::TAGS.len(), - "the tag check ran: each tag and its neutralised form" + 2, + "the tag check ran: the tags, then their neutralised forms" ); } diff --git a/crates/loopd/tests/support/mod.rs b/crates/loopd/tests/support/mod.rs index 2092dab..1e6f90b 100644 --- a/crates/loopd/tests/support/mod.rs +++ b/crates/loopd/tests/support/mod.rs @@ -180,11 +180,36 @@ pub fn tokenizer(request: &Recorded) -> Reply { let Some(content) = body.get("content").and_then(|c| c.as_str()) else { return Reply::json(400, r#"{"error":"no content"}"#); }; - let ids: Vec = match bxcore::sanitize::TAGS.iter().find(|t| t.text == content) { - Some(tag) => vec![tag.id], - None => content.bytes().map(u32::from).collect(), - }; - tokens(&ids) + tokens(&tokenize(content, &[])) +} + +/// The fake tokenizer's ids for `text`: the id of each listed tag in it, the longest where two +/// start at the same place, and one id per byte for everything else. A tag whose text is a key of +/// `overrides` gets that entry's ids instead. +pub fn tokenize(text: &str, overrides: &[(&str, Vec)]) -> Vec { + let mut ids = Vec::new(); + let mut rest = text; + while let Some(c) = rest.chars().next() { + let tag = bxcore::sanitize::TAGS + .iter() + .filter(|t| rest.starts_with(t.text)) + .max_by_key(|t| t.text.len()); + let len = match tag { + Some(tag) => { + match overrides.iter().find(|(text, _)| *text == tag.text) { + Some((_, instead)) => ids.extend(instead), + None => ids.push(tag.id), + } + tag.text.len() + } + None => { + ids.extend(c.to_string().bytes().map(u32::from)); + c.len_utf8() + } + }; + rest = &rest[len..]; + } + ids } /// A `/tokenize` answer as llama-server gives one without `with_pieces`: `{"tokens": [ids]}`. diff --git a/docs/runbook.md b/docs/runbook.md index ddf8372..bdf3d0b 100644 --- a/docs/runbook.md +++ b/docs/runbook.md @@ -735,21 +735,20 @@ if it was real. then `selftest: FAILED: ; see docs/runbook.md#loopd-tags-unexpected`, and exits 1. No prompt was sent. The reason names a tag by its token id, never by its text: -- `the tag with id is tokens, not one` -- `the tag with id is one token, but the token ` -- `the neutralised form of the tag with id does not split into several tokens (the server - gave )` -- `could not tokenize the tag with id : ` +- `the tag with id does not tokenize as its recorded token` +- `a neutralised tag tokenizes to the tag with id ` +- `could not tokenize the tags: `, or `… the neutralised tags: ` **Why.** `loopd` neutralises the model's chat-template tags in every text from outside, writing their angle brackets as `<` and `>` (M5b spec, sections 6.2 and 8). The list of tags, each -with its token id, is compiled into `bxcore` (`crates/bxcore/src/sanitize/tags.rs`), as measured -on straylight. At startup `loopd` asks the server's `/tokenize` about each tag and its neutralised -form. If a tag is no longer the one token recorded with it, the model or its tokenizer has changed -and the list may no longer match it; if a neutralised form is a single token, neutralising no -longer keeps that tag from the model. Either way the model could receive text from outside as its -own control tokens, so `loopd` does not start. The check cannot find a new tag missing from the -list: the server does not list its special tokens. +with its token id, is compiled into `bxcore` (`crates/bxcore/src/sanitize/tags.rs`), as measured on +straylight. At startup `loopd` asks the server's `/tokenize` about all the tags at once, one per +line, then about all their neutralised forms: two requests, so that the check fits in `inferproxy`'s +burst of connections. If a tag is no longer the token recorded with it, the model or its tokenizer +has changed and the list may no longer match it; if a neutralised form comes back as a tag's token, +neutralising no longer keeps that tag from the model. Either way the model could receive text from +outside as its own control tokens, so `loopd` does not start. The check cannot find a new tag +missing from the list: the server does not list its special tokens. **Confirm.** Find the tag: `grep -n '' crates/bxcore/src/sanitize/tags.rs` shows its entry, built from its name with `bar!`, `open!` or `close!`. Ask the server yourself, building the tag @@ -763,23 +762,23 @@ printf '{"model":"ornith-1.5-35b-a3b","content":"<|%s|>","parse_special":t ``` For an `open!` tag the content is `<%s>`, for a `close!` tag ``. The first answer should be -`{"tokens":[]}`, the second a list of several ids. Check which model file the server loaded: -`curl -s 'http://straylight:11434/props?model=ornith-1.5-35b-a3b' | jq .model_path`. +`{"tokens":[]}`, the second a list of ids without `` in it. Check which model file the +server loaded: `curl -s 'http://straylight:11434/props?model=ornith-1.5-35b-a3b' | jq .model_path`. **Fix, by reason.** -- `could not tokenize …`: the server did not answer `/tokenize`. A router in front of - llama-server may not forward it (crossbar does not, `docs/decisions.md`, P22): point - `inferproxy` at llama-server itself. Otherwise check that the server is up: +- `could not tokenize …`: the server did not answer `/tokenize`. A router in front of llama-server + may not forward it (crossbar does not, `docs/decisions.md`, P22): point `inferproxy` at + llama-server itself. Otherwise check that the server is up: `curl -s http://straylight:11434/health`, and run `loopd selftest --config ` again. -- A tag with another id, or several tokens: the model file changed. Do not run `loopd` against - it. Measure the new model's tags as M5b task 03 did (every vocabulary entry of type control or - user-defined, each checked with `/tokenize`), and change `TAGS` and the tests' list in - `crates/bxcore/tests/support/tags.rs` together; the change reaches the agent when it is rebuilt. - A new model also invalidates the measurements in `docs/inference-contract.md`. -- A neutralised form that is one token: the tokenizer has an entry for the escaped spelling, and - the replacement of the M5b spec, section 6.2, no longer works for that tag. That is a design - change: record it in `docs/decisions.md` and do not run `loopd` until it is settled. +- A tag that does not tokenize as its recorded token: the model file changed. Do not run `loopd` + against it. Measure the new model's tags as M5b task 03 did (every vocabulary entry of type + control or user-defined, each checked with `/tokenize`), and change `TAGS` and the tests' list in + `crates/bxcore/tests/support/tags.rs` together; the change reaches the agent when it is rebuilt. A + new model also invalidates the measurements in `docs/inference-contract.md`. +- A neutralised tag that tokenizes to a tag: the tokenizer reads the escaped spelling as the tag, + and the replacement of the M5b spec, section 6.2, no longer works for it. That is a design change: + record it in `docs/decisions.md` and do not run `loopd` until it is settled. **Check.** `loopd selftest --config ` prints `selftest: ok`. diff --git a/docs/specs/2026-09-26-m5b-hardening.md b/docs/specs/2026-09-26-m5b-hardening.md index 871cb32..5e7ba49 100644 --- a/docs/specs/2026-09-26-m5b-hardening.md +++ b/docs/specs/2026-09-26-m5b-hardening.md @@ -239,9 +239,12 @@ are what every later request replays and the inference contract holds: The model's own output (reasoning, content, tool calls) is not touched: it is logged and replayed exactly as the server returned it. -**The self-test** gains a fourth check, run at startup before `loopd` serves anyone: for each tag in -the list, the server's `/tokenize` must give one token for the tag, and more than one for its -neutralised form. If either fails, `loopd` does not start, with a new runbook entry. This catches a +**The self-test** gains a fourth check, run at startup before `loopd` serves anyone: asked about +every tag in the list at once, one per line, the server's `/tokenize` must give each tag's recorded +token, in order; asked about every neutralised form, it must give none of those tokens. If either +fails, `loopd` does not start, with a new runbook entry. Two requests, not one per tag: the device +check (2026-09-27) found that 66 requests, one connection each, overrun `inferproxy`'s burst of +ten connections. This catches a list entry that no longer works after a model or template change; it cannot find a tag missing from the list, because the server does not list its special tokens.