From f1d880568fbdf765d3f007a139543c835344a7d4 Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Fri, 18 Sep 2026 15:28:37 -0700 Subject: [PATCH] Add the startup self-test and the loopd selftest command Implemented-By: OpenCode session (model recorded in docs/implementer-log.md) --- crates/loopd/src/lib.rs | 1 + crates/loopd/src/main.rs | 49 +++++++- crates/loopd/src/selftest.rs | 211 +++++++++++++++++++++++++++++++++ crates/loopd/tests/selftest.rs | 169 ++++++++++++++++++++++++++ docs/implementer-log.md | 1 + 5 files changed, 428 insertions(+), 3 deletions(-) create mode 100644 crates/loopd/src/selftest.rs create mode 100644 crates/loopd/tests/selftest.rs diff --git a/crates/loopd/src/lib.rs b/crates/loopd/src/lib.rs index edb30d0..4891d26 100644 --- a/crates/loopd/src/lib.rs +++ b/crates/loopd/src/lib.rs @@ -3,4 +3,5 @@ pub mod config; pub mod http; pub mod llama; +pub mod selftest; pub mod sse; diff --git a/crates/loopd/src/main.rs b/crates/loopd/src/main.rs index 081fcd6..a2f852b 100644 --- a/crates/loopd/src/main.rs +++ b/crates/loopd/src/main.rs @@ -1,4 +1,47 @@ -fn main() { - eprintln!("loopd: not implemented until M2"); - std::process::exit(2); +//! `loopd`: the agent loop daemon. Its one command today is `selftest`, which runs the startup +//! checks before `loopd` serves anyone. + +use std::path::Path; +use std::process::ExitCode; + +use loopd::config::Config; +use loopd::llama::Client; +use loopd::selftest::run; + +fn main() -> ExitCode { + let args: Vec = std::env::args().skip(1).collect(); + let args: Vec<&str> = args.iter().map(String::as_str).collect(); + match args.as_slice() { + ["selftest", "--config", path] => run_selftest(path), + _ => { + eprintln!("usage: loopd selftest --config "); + ExitCode::from(2) + } + } +} + +fn run_selftest(path: &str) -> ExitCode { + let cfg = match Config::load(Path::new(path)) { + Ok(cfg) => cfg, + Err(e) => { + eprintln!("loopd: {e}"); + return ExitCode::from(1); + } + }; + + let mut on_step = |step: &str| { + eprintln!("selftest: {step}"); + }; + let result = run(&Client::new(cfg), &mut on_step); + + match result { + Ok(()) => { + eprintln!("selftest: ok"); + ExitCode::SUCCESS + } + Err(e) => { + eprintln!("selftest: FAILED: {e}"); + ExitCode::from(1) + } + } } diff --git a/crates/loopd/src/selftest.rs b/crates/loopd/src/selftest.rs new file mode 100644 index 0000000..45dc586 --- /dev/null +++ b/crates/loopd/src/selftest.rs @@ -0,0 +1,211 @@ +//! The startup self-test: before `loopd` serves anyone, it checks that the server is the one its +//! config describes, that a tool call comes back parsed, and that a second turn reuses the first +//! turn's cache. Any check failing means `loopd` does not start. + +use crate::llama::{ + ChatMessage, ChatRequest, Client, FinishReason, InferError, ToolSchema, + info::{CacheOutcome, cache_outcome}, +}; + +#[derive(Debug)] +pub enum SelfTestError { + Mismatch { + what: &'static str, + expected: String, + got: String, + }, + ToolCall(String), + CacheMiss { + expected: u64, + got: u64, + }, + Infer(InferError), + Hash, +} + +impl std::fmt::Display for SelfTestError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + SelfTestError::Mismatch { + what, + expected, + got, + } => write!(f, "{what}: expected {expected}, got {got}"), + SelfTestError::ToolCall(reason) => { + write!(f, "the tool call did not come back parsed: {reason}") + } + SelfTestError::CacheMiss { expected, got } => write!( + f, + "turn 2 did not reuse turn 1's cache: expected {expected}, saw {got}" + ), + SelfTestError::Infer(e) => write!(f, "{e}"), + SelfTestError::Hash => write!(f, "the chat template could not be hashed"), + } + } +} + +impl std::error::Error for SelfTestError {} + +impl From for SelfTestError { + fn from(e: InferError) -> Self { + SelfTestError::Infer(e) + } +} + +const SYSTEM: &str = "You are Boxmaker, a careful personal agent."; + +/// Runs the three checks in order. `on_step` is told the name of each check as it starts. +pub fn run(client: &Client, on_step: &mut dyn FnMut(&str)) -> Result<(), SelfTestError> { + on_step("server matches config"); + server_matches(client)?; + + on_step("tool call round trip"); + tool_call_round_trip(client)?; + + on_step("turn 2 cache hit"); + cache_hit(client)?; + + Ok(()) +} + +/// Check 1: the server's chat template hash and slot settings are the ones the config expects. +/// A server that is not the expected one is sent no prompt at all. +fn server_matches(client: &Client) -> Result<(), SelfTestError> { + let props = client.props().map_err(SelfTestError::Infer)?; + let expect = &client.config().expect; + + let expected = expect.template_sha256; + let got = proto::sha256(props.chat_template.as_bytes()).map_err(|_| SelfTestError::Hash)?; + if expected != got { + return Err(SelfTestError::Mismatch { + what: "chat template sha256", + expected: expected.to_hex(), + got: got.to_hex(), + }); + } + + if expect.n_ctx != props.n_ctx { + return Err(SelfTestError::Mismatch { + what: "context per slot", + expected: expect.n_ctx.to_string(), + got: props.n_ctx.to_string(), + }); + } + + if expect.slots != props.total_slots { + return Err(SelfTestError::Mismatch { + what: "slot count", + expected: expect.slots.to_string(), + got: props.total_slots.to_string(), + }); + } + + Ok(()) +} + +/// Check 2: a tool call comes back parsed, with a `read_file` tool and a string `path`. +fn tool_call_round_trip(client: &Client) -> Result<(), SelfTestError> { + let req = ChatRequest { + slot: client.config().slots.main, + messages: vec![ + ChatMessage::System { + content: SYSTEM.to_string(), + }, + ChatMessage::User { + content: "Read /etc/hostname and tell me what it says.".to_string(), + }, + ], + tools: vec![ToolSchema { + name: "read_file".to_string(), + description: "Read a text file and return its contents.".to_string(), + parameters: serde_json::json!({ + "type": "object", + "properties": { "path": { "type": "string", "description": "Absolute path" } }, + "required": ["path"], + }), + }], + thinking: false, + }; + + let completion = client + .chat_with_retry(&req, &mut |_| {}) + .map_err(SelfTestError::Infer)?; + + if completion.finish_reason != FinishReason::ToolCalls { + return Err(SelfTestError::ToolCall( + "the completion did not end with a tool call".to_string(), + )); + } + let Some(call) = completion.tool_calls.first() else { + return Err(SelfTestError::ToolCall( + "the completion made no tool call".to_string(), + )); + }; + if call.name != "read_file" { + return Err(SelfTestError::ToolCall(format!( + "the first tool call was {}, not read_file", + call.name + ))); + } + let arguments: serde_json::Value = serde_json::from_str(&call.arguments).map_err(|_| { + SelfTestError::ToolCall("the tool call arguments were not valid JSON".to_string()) + })?; + if arguments.get("path").and_then(|v| v.as_str()).is_none() { + return Err(SelfTestError::ToolCall( + "the tool call arguments had no string path".to_string(), + )); + } + + Ok(()) +} + +/// Check 3: a second turn that extends the first reuses the first turn's cache. +fn cache_hit(client: &Client) -> Result<(), SelfTestError> { + let first = ChatRequest { + slot: client.config().slots.main, + messages: vec![ + ChatMessage::System { + content: SYSTEM.to_string(), + }, + ChatMessage::User { + content: "Name one colour. One word.".to_string(), + }, + ], + tools: Vec::new(), + thinking: false, + }; + let turn1 = client + .chat_with_retry(&first, &mut |_| {}) + .map_err(SelfTestError::Infer)?; + + let mut messages = vec![ + ChatMessage::System { + content: SYSTEM.to_string(), + }, + ChatMessage::User { + content: "Name one colour. One word.".to_string(), + }, + ]; + messages.push(ChatMessage::Assistant { + content: turn1.content, + reasoning_content: turn1.reasoning_content, + tool_calls: turn1.tool_calls, + }); + messages.push(ChatMessage::User { + content: "Name another. One word.".to_string(), + }); + let second = ChatRequest { + slot: client.config().slots.main, + messages, + tools: Vec::new(), + thinking: false, + }; + let turn2 = client + .chat_with_retry(&second, &mut |_| {}) + .map_err(SelfTestError::Infer)?; + + match cache_outcome(&turn1.timings, &turn2.timings) { + CacheOutcome::Hit => Ok(()), + CacheOutcome::Loss { expected, got } => Err(SelfTestError::CacheMiss { expected, got }), + } +} diff --git a/crates/loopd/tests/selftest.rs b/crates/loopd/tests/selftest.rs new file mode 100644 index 0000000..24cf2ba --- /dev/null +++ b/crates/loopd/tests/selftest.rs @@ -0,0 +1,169 @@ +//! Tests for the startup self-test, against the fake server. Do not edit. + +mod support; + +use loopd::llama::Client; +use loopd::selftest::{SelfTestError, run}; +use support::{FakeServer, Reply}; + +const CHAT: &str = "/v1/chat/completions"; + +/// A change to the config before the self-test runs. +type Edit = fn(&mut loopd::config::Config); + +/// A server that passes: the recorded props, then the three recorded completions in order. +fn healthy() -> FakeServer { + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route( + CHAT, + vec![ + Reply::fixture("tool_call"), + Reply::fixture("turn1"), + Reply::fixture("turn2"), + ], + ); + server +} + +fn run_with( + server: &FakeServer, + edit: impl FnOnce(&mut loopd::config::Config), +) -> (Result<(), SelfTestError>, Vec) { + let mut cfg = support::test_config(&server.socket); + edit(&mut cfg); + let mut steps = Vec::new(); + let result = run(&Client::new(cfg), &mut |s| steps.push(s.to_string())); + (result, steps) +} + +#[test] +fn a_healthy_server_passes_all_three_checks_in_order() { + let server = healthy(); + let (result, steps) = run_with(&server, |_| {}); + assert!(result.is_ok(), "{result:?}"); + assert_eq!( + steps, + [ + "server matches config", + "tool call round trip", + "turn 2 cache hit" + ] + ); + + let chats = server.requests_to(CHAT); + assert_eq!(chats.len(), 3); + // The tool-call check offers exactly one tool; the cache check offers none. + assert_eq!(chats[0].json()["tools"].as_array().map(Vec::len), Some(1)); + assert!(chats[1].json().get("tools").is_none()); + // All three run on the main slot. + assert!(chats.iter().all(|c| c.json()["id_slot"] == 0)); + // Turn 2 is turn 1 plus the assistant's answer plus a new question: a strict extension. + let turn1 = chats[1].json()["messages"].as_array().unwrap().clone(); + let turn2 = chats[2].json()["messages"].as_array().unwrap().clone(); + assert_eq!(turn2.len(), turn1.len() + 2); + assert_eq!(turn2[..turn1.len()], turn1[..]); + assert_eq!( + turn2[turn1.len()], + serde_json::json!({"role": "assistant", "content": "Blue"}) + ); +} + +#[test] +fn the_main_slot_comes_from_the_config() { + let server = healthy(); + let (result, _) = run_with(&server, |c| c.slots.main = 1); + assert!(result.is_ok(), "{result:?}"); + assert!( + server + .requests_to(CHAT) + .iter() + .all(|c| c.json()["id_slot"] == 1) + ); +} + +#[test] +fn each_expected_value_is_checked() { + let zero = proto::Hash32::ZERO; + let cases: [(&str, Edit); 3] = [ + ("chat template sha256", |c| { + c.expect.template_sha256 = proto::Hash32::ZERO + }), + ("context per slot", |c| c.expect.n_ctx = 4096), + ("slot count", |c| c.expect.slots = 3), + ]; + for (want_what, edit) in cases { + let server = healthy(); + let (result, steps) = run_with(&server, edit); + match result { + Err(SelfTestError::Mismatch { + what, + expected, + got, + }) => { + assert_eq!(what, want_what); + assert_ne!(expected, got); + if what == "chat template sha256" { + assert_eq!(expected, zero.to_hex()); + } + } + other => panic!("{want_what}: {other:?}"), + } + assert_eq!( + steps.len(), + 1, + "a server that is not the expected one is not sent any prompt" + ); + assert!(server.requests_to(CHAT).is_empty()); + } +} + +#[test] +fn a_completion_without_the_tool_call_fails_the_second_check() { + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route(CHAT, vec![Reply::fixture("plain")]); + let (result, steps) = run_with(&server, |_| {}); + assert!( + matches!(result, Err(SelfTestError::ToolCall(_))), + "{result:?}" + ); + assert_eq!(steps.len(), 2); +} + +#[test] +fn a_cold_second_turn_fails_the_third_check() { + // turn1 twice: the "second turn" reuses 15 tokens where 46 were left. + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route( + CHAT, + vec![ + Reply::fixture("tool_call"), + Reply::fixture("progress"), + Reply::fixture("turn1"), + ], + ); + let (result, steps) = run_with(&server, |_| {}); + match result { + Err(SelfTestError::CacheMiss { expected, got }) => assert_eq!((expected, got), (7052, 15)), + other => panic!("{other:?}"), + } + assert_eq!(steps.len(), 3); +} + +#[test] +fn a_server_that_is_down_is_an_inference_error_after_retries() { + let server = FakeServer::start(); + server.route("/props", vec![Reply::fixture("props")]); + server.route(CHAT, vec![Reply::fixture("tool_call").cut_after(500)]); + let (result, _) = run_with(&server, |c| c.limits.retry_attempts = 2); + assert!(matches!(result, Err(SelfTestError::Infer(_))), "{result:?}"); + assert_eq!( + server.requests_to(CHAT).len(), + 2, + "the self-test retries like everything else" + ); + let e: Box = Box::new(result.unwrap_err()); + assert!(!e.to_string().is_empty()); +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 4095731..3e3b452 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -26,6 +26,7 @@ reviewer adds findings under "Reviews" once per milestone. | M2a/11-llama-gate-retry | 2026-09-18 | done | 3 | fail | none | Prerequisite `chat` (M2a/09) now exists, so the task was possible. Implemented `SlotGate` in gate.rs: per-slot holder + a `VecDeque` of arrival tickets, `notify_all`, a woken waiter takes the slot only if free and its ticket is at the front (and claims it by setting holder), `Drop` frees and wakes; mutex/condvar poison recovered via `unwrap_or_else(...into_inner)`, no `unwrap`. Implemented `chat_with_retry` + `is_retryable` (all nine variants, a new one is a compile error) + `backoff_ms` in retry.rs: base is `schedule[retry-1]` or last or 0, jitter clamped and computed in `i128` so `u64::MAX` never overflows, jitter from sub-second nanos. `chat` acquires the gate for `req.slot` and maps `GateFull`->`InferError::Busy`; `Client` gained a `gate` field. First gate failed on three clippy lints (derivable `Default`, `or_insert_with`->`or_default`), fixed. One logic bug caught by `waiters_are_served_in_order`: `take_if_front` claimed the ticket but not `holder`, letting two permits overlap — set holder on claim. All 79 loopd tests pass; retry 13/13 over ten runs; `make gate` prints `gate: ok`. | ? | | M2a/09-llama-chat | 2026-09-18 | done | 1 | pass | none | Wrote crates/loopd/src/llama/chat.rs (`chat`, with the head wait in a separate `wait_for_head`) and registered `pub mod chat;`. `chat` builds the body (build error -> Protocol), opens and POSTs, then `wait_for_head` loops `read_head` at `poll_ms`: a Timeout is classified Idle/Busy/Unavailable by `received_any` then a `slots()` poll, emits `Waiting { slot_busy }` on every poll, keeps a per-state `since` that resets on state change, and returns `WaitTimeout`/`LoadTimeout`/`Stalled` at the right limits; 200 streams via `Events`+`Assembler` mapping `Timeout->Stalled`, `Truncated->StreamClosedEarly`, else `Protocol`, then `finish(false)`; non-200 returns `Http { status, error_text }`. All 13 chat tests pass five runs in a row. Table-to-test map: state (table 1) Busy -> a_busy_slot_is_waited_out / a_slot_that_stays_busy_is_a_wait_timeout, Idle-not-processing -> only_the_requests_own_slot_counts_as_busy, Unavailable -> an_unavailable_server_is_a_load_timeout, Idle-received_any -> a_slow_but_steady_stream_is_not_a_stall (turn1 head is 738 B, pieces are 3140/8=392 B, so the head-wait times out with a partial head); thresholds (table 2) -> a_slot_that_stays_busy_is_a_wait_timeout / an_unavailable_server_is_a_load_timeout / only_the_requests_own_slot_counts_as_busy; stream (table 3) Data/Done/None -> the recorded completion and trickle tests, Timeout -> silence_in_mid_stream_is_a_stall, Truncated -> a_stream_that_is_cut_is_closed_early_wherever_the_cut_falls, other -> garbage_in_the_stream_is_a_protocol_error; clock-restart -> the_wait_clocks_restart_when_the_state_changes. One path fix: info and request live under crate::llama, not crate::, so the imports use crate::llama::{info::..., request::...}. | ? | | M2a/10-llama-cap | 2026-09-18 | done | 1 | pass | none | Added `Client::end_reasoning` to info.rs: POSTs `{"id","action":"reasoning_end","model"}` to `/v1/chat/completions/control` via `call`, reads `success` as a bool from the server's JSON (ignoring `message`), non-200 stays an Err through `call`, a missing/non-bool `success` is Protocol. Threaded the cap into `chat` step 4: after passing a chunk's events on, when `assembler.in_reasoning()`, a local `cap_at: Option` holds where the cap fired (None while it has not fired); on `tokens >= thinking_cap` it calls `end_reasoning(assembler.id())` once, remembers `tokens` and emits `ThinkingCapped` on `Ok(true)`, returns `ThinkingOverrun` on `Ok(false)`/`Err`, and after firing returns `ThinkingOverrun` once `tokens >= at + thinking_overrun`; `finish(cap_at.is_some())`. The `the_allowance_is_exact` test passes with `>=` in both rows (63 is not `20+44`, and is `>= 20+43`). One guard: a reasoning chunk with no id at cap time is Protocol rather than a panic. 6 cap tests + 13 chat tests pass; `make gate` prints `gate: ok`. | ? | +| M2a/12-selftest | 2026-09-18 | done | 1 | pass | none | Wrote crates/loopd/src/selftest.rs (SelfTestError with Display/std::error::Error/From, and `run` doing the three checks in order) and registered `pub mod selftest;` in lib.rs; rewrote main.rs into `loopd selftest --config `. Check 1 calls props() and compares chat-template sha256, then n_ctx, then total_slots, a sha256 error being Hash so a wrong server is sent no prompt; check 2 posts one read_file tool with chat_with_retry and requires finish_reason ToolCalls, a first call named read_file whose arguments parse as JSON carrying a string path, wrapping an InferError as Infer; check 3 runs turn 1 then an extension of it and maps a cache Loss to CacheMiss. The copied test's cache_outcome/CacheOutcome live at crate::llama::info, so the import follows that. main.rs parses args as &[&str] via a two-step String->&str collect; unknown/missing args are exit 2 and a config load failure is `loopd: ` exit 1. Real server via inferproxy against straylight: minimal.toml gave three step lines and `selftest: ok` exit 0; setting slots=3 gave `selftest: FAILED: slot count: expected 3, got 2` exit 1. | ? | ## Reviews