Extend on-device verification to the agent loop

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
2026-09-18 20:38:56 -07:00
parent 06cda69509
commit 34951084cc
4 changed files with 213 additions and 1 deletions
@@ -231,3 +231,212 @@ fn a_second_turn_reuses_the_first_turns_cache() {
        turn2.timings
    );
 }
+
+// ---- M2b: the agent loop on the real server ----
+
+/// A `loopd serve` on a private home, killed on drop.
+struct Served {
+    child: Child,
+    home: PathBuf,
+    socket: PathBuf,
+}
+
+fn config_text(infer: &Path, home: &Path) -> String {
+    let model =
+        std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
+    format!(
+        r#"
+[infer]
+socket = "{}"
+model = "{model}"
+[slots]
+main = 0
+background = 1
+[expect]
+template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
+n_ctx = 131072
+slots = 2
+[paths]
+home = "{}"
+"#,
+        infer.display(),
+        home.display()
+    )
+}
+
+impl Served {
+    /// Writes the config and the repository's `system.md` into `home`, and starts `loopd serve`.
+    fn start(infer: &Path, home: &Path) -> Served {
+        std::fs::create_dir_all(home).unwrap();
+        let config = home.join("config.toml");
+        std::fs::write(&config, config_text(infer, home)).unwrap();
+        let prompt = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
+        std::fs::copy(&prompt, home.join("system.md"))
+            .expect("config/system.md exists in the repository");
+        let socket = home.join("run").join("loop").join("loop.sock");
+        let _ = std::fs::remove_file(&socket);
+        let child = Command::new(env!("CARGO_BIN_EXE_loopd"))
+            .arg("serve")
+            .arg("--config")
+            .arg(&config)
+            .spawn()
+            .expect("cannot start loopd");
+        let mut served = Served {
+            child,
+            home: home.to_path_buf(),
+            socket,
+        };
+        for _ in 0..600 {
+            if served.socket.exists() {
+                return served;
+            }
+            thread::sleep(Duration::from_millis(100));
+        }
+        served.kill();
+        panic!("loopd did not come up within 60 s");
+    }
+
+    fn kill(&mut self) {
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+    }
+
+    /// One `bxctl chat --say` turn. Returns the answer.
+    fn say(&self, session: &str, text: &str) -> String {
+        let bxctl = std::env::var("BOXMAKER_BXCTL").expect("BOXMAKER_BXCTL is not set");
+        let output = Command::new(bxctl)
+            .arg("chat")
+            .arg("--socket")
+            .arg(&self.socket)
+            .args(["--session", session, "--no-thinking", "--say", text])
+            .output()
+            .expect("cannot run bxctl");
+        assert!(
+            output.status.success(),
+            "bxctl failed: {}",
+            String::from_utf8_lossy(&output.stderr)
+        );
+        String::from_utf8_lossy(&output.stdout)
+            .trim_end()
+            .to_string()
+    }
+
+    fn records(&self, session: &str) -> Vec<proto::LogRecord> {
+        let text =
+            std::fs::read_to_string(self.home.join("sessions").join(session).join("0.jsonl"))
+                .unwrap();
+        text.lines()
+            .map(|l| serde_json::from_str(l).unwrap())
+            .collect()
+    }
+}
+
+impl Drop for Served {
+    fn drop(&mut self) {
+        self.kill();
+    }
+}
+
+#[test]
+#[ignore = "needs the real server; run with make verify-device"]
+fn the_baseline_fits_the_token_budget() {
+    let socket = socket_path("budget");
+    let _proxy = Proxy::start(&socket);
+    let home = socket.parent().unwrap().join("home");
+    std::fs::create_dir_all(&home).unwrap();
+    let mut cfg = config(&socket);
+    cfg.paths.home = home.clone();
+    cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
+    let baseline =
+        loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m2b()).unwrap();
+    let client = Client::new(cfg);
+    // The system text plus every tool schema as the request carries it.
+    let mut text = baseline.system.clone();
+    for tool in &baseline.tools {
+        text.push('\n');
+        text.push_str(&serde_json::to_string(&tool).unwrap());
+    }
+    let tokens = client.tokenize(&text).unwrap();
+    eprintln!("baseline: {tokens} tokens");
+    assert!(
+        tokens <= 3000,
+        "the baseline is {tokens} tokens; the brief allows 3000"
+    );
+}
+
+#[test]
+#[ignore = "needs the real server; run with make verify-device"]
+fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() {
+    let socket = socket_path("loop");
+    let _proxy = Proxy::start(&socket);
+    let home = socket.parent().unwrap().join("home");
+    let session = format!("device-{}", std::process::id());
+    let mut served = Served::start(&socket, &home);
+
+    let a1 = served.say(&session, "Reply with exactly: box made.");
+    assert!(a1.to_lowercase().contains("box made"), "{a1}");
+    let a2 = served.say(
+        &session,
+        "What is the current time? Use your clock tool, then tell me the year.",
+    );
+    assert!(a2.contains("2026") || a2.contains("202"), "{a2}");
+    let a3 = served.say(
+        &session,
+        "Use the echo tool to echo the word cork back to me, and reply with just that word.",
+    );
+    assert!(a3.to_lowercase().contains("cork"), "{a3}");
+
+    served.kill();
+    served = Served::start(&socket, &home);
+    let a4 = served.say(&session, "What word did you echo a moment ago? One word.");
+    assert!(
+        a4.to_lowercase().contains("cork"),
+        "after a restart the session must still know: {a4}"
+    );
+
+    let records = served.records(&session);
+    let tool_names: Vec<String> = records
+        .iter()
+        .filter_map(|r| match r {
+            proto::LogRecord::Assistant { tool_calls, .. } => Some(
+                tool_calls
+                    .iter()
+                    .map(|c| c.name.clone())
+                    .collect::<Vec<_>>(),
+            ),
+            _ => None,
+        })
+        .flatten()
+        .collect();
+    assert!(tool_names.contains(&"clock".to_string()), "{tool_names:?}");
+    assert!(
+        tool_names.contains(&"find_tool".to_string())
+            && tool_names.contains(&"call_tool".to_string()),
+        "{tool_names:?}"
+    );
+    let usages = records
+        .iter()
+        .filter(|r| matches!(r, proto::LogRecord::Usage { .. }))
+        .count();
+    let assistants = records
+        .iter()
+        .filter(|r| matches!(r, proto::LogRecord::Assistant { .. }))
+        .count();
+    assert_eq!(usages, assistants, "one usage record per completion");
+    let losses: Vec<&proto::LogRecord> = records
+        .iter()
+        .filter(|r| matches!(r, proto::LogRecord::CacheLoss { .. }))
+        .collect();
+    assert!(
+        losses.is_empty(),
+        "every request hit the cache, including the one after the restart: {losses:?}"
+    );
+    let results = records
+        .iter()
+        .filter(|r| matches!(r, proto::LogRecord::ToolResult { .. }))
+        .count();
+    assert!(
+        results >= 3,
+        "clock, find_tool and call_tool each left a result: {results}"
+    );
+}