Add the startup self-test and the loopd selftest command

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/loopd/src/lib.rs

@@ -3,4 +3,5 @@
 pub mod config;
 pub mod http;
 pub mod llama;
+pub mod selftest;
 pub mod sse;

crates/loopd/src/main.rs

@@ -1,4 +1,47 @@
-fn main() {
-    eprintln!("loopd: not implemented until M2");
-    std::process::exit(2);
+//! `loopd`: the agent loop daemon. Its one command today is `selftest`, which runs the startup
+//! checks before `loopd` serves anyone.
+
+use std::path::Path;
+use std::process::ExitCode;
+
+use loopd::config::Config;
+use loopd::llama::Client;
+use loopd::selftest::run;
+
+fn main() -> ExitCode {
+    let args: Vec<String> = std::env::args().skip(1).collect();
+    let args: Vec<&str> = args.iter().map(String::as_str).collect();
+    match args.as_slice() {
+        ["selftest", "--config", path] => run_selftest(path),
+        _ => {
+            eprintln!("usage: loopd selftest --config <path>");
+            ExitCode::from(2)
+        }
+    }
+}
+
+fn run_selftest(path: &str) -> ExitCode {
+    let cfg = match Config::load(Path::new(path)) {
+        Ok(cfg) => cfg,
+        Err(e) => {
+            eprintln!("loopd: {e}");
+            return ExitCode::from(1);
+        }
+    };
+
+    let mut on_step = |step: &str| {
+        eprintln!("selftest: {step}");
+    };
+    let result = run(&Client::new(cfg), &mut on_step);
+
+    match result {
+        Ok(()) => {
+            eprintln!("selftest: ok");
+            ExitCode::SUCCESS
+        }
+        Err(e) => {
+            eprintln!("selftest: FAILED: {e}");
+            ExitCode::from(1)
+        }
+    }
 }

crates/loopd/src/selftest.rs

@@ -0,0 +1,211 @@
+//! The startup self-test: before `loopd` serves anyone, it checks that the server is the one its
+//! config describes, that a tool call comes back parsed, and that a second turn reuses the first
+//! turn's cache. Any check failing means `loopd` does not start.
+
+use crate::llama::{
+    ChatMessage, ChatRequest, Client, FinishReason, InferError, ToolSchema,
+    info::{CacheOutcome, cache_outcome},
+};
+
+#[derive(Debug)]
+pub enum SelfTestError {
+    Mismatch {
+        what: &'static str,
+        expected: String,
+        got: String,
+    },
+    ToolCall(String),
+    CacheMiss {
+        expected: u64,
+        got: u64,
+    },
+    Infer(InferError),
+    Hash,
+}
+
+impl std::fmt::Display for SelfTestError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            SelfTestError::Mismatch {
+                what,
+                expected,
+                got,
+            } => write!(f, "{what}: expected {expected}, got {got}"),
+            SelfTestError::ToolCall(reason) => {
+                write!(f, "the tool call did not come back parsed: {reason}")
+            }
+            SelfTestError::CacheMiss { expected, got } => write!(
+                f,
+                "turn 2 did not reuse turn 1's cache: expected {expected}, saw {got}"
+            ),
+            SelfTestError::Infer(e) => write!(f, "{e}"),
+            SelfTestError::Hash => write!(f, "the chat template could not be hashed"),
+        }
+    }
+}
+
+impl std::error::Error for SelfTestError {}
+
+impl From<InferError> for SelfTestError {
+    fn from(e: InferError) -> Self {
+        SelfTestError::Infer(e)
+    }
+}
+
+const SYSTEM: &str = "You are Boxmaker, a careful personal agent.";
+
+/// Runs the three checks in order. `on_step` is told the name of each check as it starts.
+pub fn run(client: &Client, on_step: &mut dyn FnMut(&str)) -> Result<(), SelfTestError> {
+    on_step("server matches config");
+    server_matches(client)?;
+
+    on_step("tool call round trip");
+    tool_call_round_trip(client)?;
+
+    on_step("turn 2 cache hit");
+    cache_hit(client)?;
+
+    Ok(())
+}
+
+/// Check 1: the server's chat template hash and slot settings are the ones the config expects.
+/// A server that is not the expected one is sent no prompt at all.
+fn server_matches(client: &Client) -> Result<(), SelfTestError> {
+    let props = client.props().map_err(SelfTestError::Infer)?;
+    let expect = &client.config().expect;
+
+    let expected = expect.template_sha256;
+    let got = proto::sha256(props.chat_template.as_bytes()).map_err(|_| SelfTestError::Hash)?;
+    if expected != got {
+        return Err(SelfTestError::Mismatch {
+            what: "chat template sha256",
+            expected: expected.to_hex(),
+            got: got.to_hex(),
+        });
+    }
+
+    if expect.n_ctx != props.n_ctx {
+        return Err(SelfTestError::Mismatch {
+            what: "context per slot",
+            expected: expect.n_ctx.to_string(),
+            got: props.n_ctx.to_string(),
+        });
+    }
+
+    if expect.slots != props.total_slots {
+        return Err(SelfTestError::Mismatch {
+            what: "slot count",
+            expected: expect.slots.to_string(),
+            got: props.total_slots.to_string(),
+        });
+    }
+
+    Ok(())
+}
+
+/// Check 2: a tool call comes back parsed, with a `read_file` tool and a string `path`.
+fn tool_call_round_trip(client: &Client) -> Result<(), SelfTestError> {
+    let req = ChatRequest {
+        slot: client.config().slots.main,
+        messages: vec![
+            ChatMessage::System {
+                content: SYSTEM.to_string(),
+            },
+            ChatMessage::User {
+                content: "Read /etc/hostname and tell me what it says.".to_string(),
+            },
+        ],
+        tools: vec![ToolSchema {
+            name: "read_file".to_string(),
+            description: "Read a text file and return its contents.".to_string(),
+            parameters: serde_json::json!({
+                "type": "object",
+                "properties": { "path": { "type": "string", "description": "Absolute path" } },
+                "required": ["path"],
+            }),
+        }],
+        thinking: false,
+    };
+
+    let completion = client
+        .chat_with_retry(&req, &mut |_| {})
+        .map_err(SelfTestError::Infer)?;
+
+    if completion.finish_reason != FinishReason::ToolCalls {
+        return Err(SelfTestError::ToolCall(
+            "the completion did not end with a tool call".to_string(),
+        ));
+    }
+    let Some(call) = completion.tool_calls.first() else {
+        return Err(SelfTestError::ToolCall(
+            "the completion made no tool call".to_string(),
+        ));
+    };
+    if call.name != "read_file" {
+        return Err(SelfTestError::ToolCall(format!(
+            "the first tool call was {}, not read_file",
+            call.name
+        )));
+    }
+    let arguments: serde_json::Value = serde_json::from_str(&call.arguments).map_err(|_| {
+        SelfTestError::ToolCall("the tool call arguments were not valid JSON".to_string())
+    })?;
+    if arguments.get("path").and_then(|v| v.as_str()).is_none() {
+        return Err(SelfTestError::ToolCall(
+            "the tool call arguments had no string path".to_string(),
+        ));
+    }
+
+    Ok(())
+}
+
+/// Check 3: a second turn that extends the first reuses the first turn's cache.
+fn cache_hit(client: &Client) -> Result<(), SelfTestError> {
+    let first = ChatRequest {
+        slot: client.config().slots.main,
+        messages: vec![
+            ChatMessage::System {
+                content: SYSTEM.to_string(),
+            },
+            ChatMessage::User {
+                content: "Name one colour. One word.".to_string(),
+            },
+        ],
+        tools: Vec::new(),
+        thinking: false,
+    };
+    let turn1 = client
+        .chat_with_retry(&first, &mut |_| {})
+        .map_err(SelfTestError::Infer)?;
+
+    let mut messages = vec![
+        ChatMessage::System {
+            content: SYSTEM.to_string(),
+        },
+        ChatMessage::User {
+            content: "Name one colour. One word.".to_string(),
+        },
+    ];
+    messages.push(ChatMessage::Assistant {
+        content: turn1.content,
+        reasoning_content: turn1.reasoning_content,
+        tool_calls: turn1.tool_calls,
+    });
+    messages.push(ChatMessage::User {
+        content: "Name another. One word.".to_string(),
+    });
+    let second = ChatRequest {
+        slot: client.config().slots.main,
+        messages,
+        tools: Vec::new(),
+        thinking: false,
+    };
+    let turn2 = client
+        .chat_with_retry(&second, &mut |_| {})
+        .map_err(SelfTestError::Infer)?;
+
+    match cache_outcome(&turn1.timings, &turn2.timings) {
+        CacheOutcome::Hit => Ok(()),
+        CacheOutcome::Loss { expected, got } => Err(SelfTestError::CacheMiss { expected, got }),
+    }
+}

crates/loopd/tests/selftest.rs

@@ -0,0 +1,169 @@
+//! Tests for the startup self-test, against the fake server. Do not edit.
+
+mod support;
+
+use loopd::llama::Client;
+use loopd::selftest::{SelfTestError, run};
+use support::{FakeServer, Reply};
+
+const CHAT: &str = "/v1/chat/completions";
+
+/// A change to the config before the self-test runs.
+type Edit = fn(&mut loopd::config::Config);
+
+/// A server that passes: the recorded props, then the three recorded completions in order.
+fn healthy() -> FakeServer {
+    let server = FakeServer::start();
+    server.route("/props", vec![Reply::fixture("props")]);
+    server.route(
+        CHAT,
+        vec![
+            Reply::fixture("tool_call"),
+            Reply::fixture("turn1"),
+            Reply::fixture("turn2"),
+        ],
+    );
+    server
+}
+
+fn run_with(
+    server: &FakeServer,
+    edit: impl FnOnce(&mut loopd::config::Config),
+) -> (Result<(), SelfTestError>, Vec<String>) {
+    let mut cfg = support::test_config(&server.socket);
+    edit(&mut cfg);
+    let mut steps = Vec::new();
+    let result = run(&Client::new(cfg), &mut |s| steps.push(s.to_string()));
+    (result, steps)
+}
+
+#[test]
+fn a_healthy_server_passes_all_three_checks_in_order() {
+    let server = healthy();
+    let (result, steps) = run_with(&server, |_| {});
+    assert!(result.is_ok(), "{result:?}");
+    assert_eq!(
+        steps,
+        [
+            "server matches config",
+            "tool call round trip",
+            "turn 2 cache hit"
+        ]
+    );
+
+    let chats = server.requests_to(CHAT);
+    assert_eq!(chats.len(), 3);
+    // The tool-call check offers exactly one tool; the cache check offers none.
+    assert_eq!(chats[0].json()["tools"].as_array().map(Vec::len), Some(1));
+    assert!(chats[1].json().get("tools").is_none());
+    // All three run on the main slot.
+    assert!(chats.iter().all(|c| c.json()["id_slot"] == 0));
+    // Turn 2 is turn 1 plus the assistant's answer plus a new question: a strict extension.
+    let turn1 = chats[1].json()["messages"].as_array().unwrap().clone();
+    let turn2 = chats[2].json()["messages"].as_array().unwrap().clone();
+    assert_eq!(turn2.len(), turn1.len() + 2);
+    assert_eq!(turn2[..turn1.len()], turn1[..]);
+    assert_eq!(
+        turn2[turn1.len()],
+        serde_json::json!({"role": "assistant", "content": "Blue"})
+    );
+}
+
+#[test]
+fn the_main_slot_comes_from_the_config() {
+    let server = healthy();
+    let (result, _) = run_with(&server, |c| c.slots.main = 1);
+    assert!(result.is_ok(), "{result:?}");
+    assert!(
+        server
+            .requests_to(CHAT)
+            .iter()
+            .all(|c| c.json()["id_slot"] == 1)
+    );
+}
+
+#[test]
+fn each_expected_value_is_checked() {
+    let zero = proto::Hash32::ZERO;
+    let cases: [(&str, Edit); 3] = [
+        ("chat template sha256", |c| {
+            c.expect.template_sha256 = proto::Hash32::ZERO
+        }),
+        ("context per slot", |c| c.expect.n_ctx = 4096),
+        ("slot count", |c| c.expect.slots = 3),
+    ];
+    for (want_what, edit) in cases {
+        let server = healthy();
+        let (result, steps) = run_with(&server, edit);
+        match result {
+            Err(SelfTestError::Mismatch {
+                what,
+                expected,
+                got,
+            }) => {
+                assert_eq!(what, want_what);
+                assert_ne!(expected, got);
+                if what == "chat template sha256" {
+                    assert_eq!(expected, zero.to_hex());
+                }
+            }
+            other => panic!("{want_what}: {other:?}"),
+        }
+        assert_eq!(
+            steps.len(),
+            1,
+            "a server that is not the expected one is not sent any prompt"
+        );
+        assert!(server.requests_to(CHAT).is_empty());
+    }
+}
+
+#[test]
+fn a_completion_without_the_tool_call_fails_the_second_check() {
+    let server = FakeServer::start();
+    server.route("/props", vec![Reply::fixture("props")]);
+    server.route(CHAT, vec![Reply::fixture("plain")]);
+    let (result, steps) = run_with(&server, |_| {});
+    assert!(
+        matches!(result, Err(SelfTestError::ToolCall(_))),
+        "{result:?}"
+    );
+    assert_eq!(steps.len(), 2);
+}
+
+#[test]
+fn a_cold_second_turn_fails_the_third_check() {
+    // turn1 twice: the "second turn" reuses 15 tokens where 46 were left.
+    let server = FakeServer::start();
+    server.route("/props", vec![Reply::fixture("props")]);
+    server.route(
+        CHAT,
+        vec![
+            Reply::fixture("tool_call"),
+            Reply::fixture("progress"),
+            Reply::fixture("turn1"),
+        ],
+    );
+    let (result, steps) = run_with(&server, |_| {});
+    match result {
+        Err(SelfTestError::CacheMiss { expected, got }) => assert_eq!((expected, got), (7052, 15)),
+        other => panic!("{other:?}"),
+    }
+    assert_eq!(steps.len(), 3);
+}
+
+#[test]
+fn a_server_that_is_down_is_an_inference_error_after_retries() {
+    let server = FakeServer::start();
+    server.route("/props", vec![Reply::fixture("props")]);
+    server.route(CHAT, vec![Reply::fixture("tool_call").cut_after(500)]);
+    let (result, _) = run_with(&server, |c| c.limits.retry_attempts = 2);
+    assert!(matches!(result, Err(SelfTestError::Infer(_))), "{result:?}");
+    assert_eq!(
+        server.requests_to(CHAT).len(),
+        2,
+        "the self-test retries like everything else"
+    );
+    let e: Box<dyn std::error::Error> = Box::new(result.unwrap_err());
+    assert!(!e.to_string().is_empty());
+}