kyleandClaude Opus 5.5
f6841f1155
loopd's self-test caught the change (context per slot 131072 -> 262144, slots 2 -> 4). The device tests keep the expectation in one constant, and the M3a script matches it; verify-device passes 6 of 6 and the M3a device check passes. The inference contract notes which M0 findings rest on the old layout and need re-measuring. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
452 lines
14 KiB
Rust
452 lines
14 KiB
Rust
//! Checks against the real server. Ignored unless run by `make verify-device`. Do not edit.
|
|
//!
|
|
//! They need two environment variables:
|
|
//! BOXMAKER_INFERPROXY path to the built `inferproxy` binary
|
|
//! BOXMAKER_UPSTREAM host:port of the llama-server router, for example straylight:11434
|
|
//! and, optionally, BOXMAKER_MODEL (default ornith-1.5-35b-a3b). They use slot 0.
|
|
//!
|
|
//! They never touch the real `llama-server` process. The "server dies" test kills and restarts
|
|
//! its own `inferproxy`.
|
|
|
|
use loopd::config::Config;
|
|
use loopd::llama::info::{CacheOutcome, cache_outcome};
|
|
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client};
|
|
use std::path::{Path, PathBuf};
|
|
use std::process::{Child, Command};
|
|
use std::sync::mpsc;
|
|
use std::thread;
|
|
use std::time::Duration;
|
|
|
|
struct Proxy {
|
|
child: Child,
|
|
socket: PathBuf,
|
|
}
|
|
|
|
impl Proxy {
|
|
fn start(socket: &Path) -> Proxy {
|
|
let binary = std::env::var("BOXMAKER_INFERPROXY").expect("BOXMAKER_INFERPROXY is not set");
|
|
let upstream = std::env::var("BOXMAKER_UPSTREAM").expect("BOXMAKER_UPSTREAM is not set");
|
|
let child = Command::new(binary)
|
|
.arg("--listen")
|
|
.arg(socket)
|
|
.arg("--upstream")
|
|
.arg(upstream)
|
|
.spawn()
|
|
.expect("cannot start inferproxy");
|
|
let mut proxy = Proxy {
|
|
child,
|
|
socket: socket.to_path_buf(),
|
|
};
|
|
for _ in 0..100 {
|
|
if socket.exists() {
|
|
return proxy;
|
|
}
|
|
thread::sleep(Duration::from_millis(20));
|
|
}
|
|
proxy.kill();
|
|
panic!("inferproxy did not create {}", socket.display());
|
|
}
|
|
|
|
fn kill(&mut self) {
|
|
let _ = self.child.kill();
|
|
let _ = self.child.wait();
|
|
}
|
|
}
|
|
|
|
impl Drop for Proxy {
|
|
fn drop(&mut self) {
|
|
self.kill();
|
|
}
|
|
}
|
|
|
|
/// The server as deployed on straylight, recorded in one place. When the deployment changes on
|
|
/// purpose, change this and `tools/check-m3a-device.sh` together, and note it in
|
|
/// `docs/inference-contract.md`. Since 2026-09-22: four slots over one 262,144-token pool.
|
|
const EXPECT: &str = r#"template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
|
n_ctx = 262144
|
|
slots = 4"#;
|
|
|
|
fn socket_path(name: &str) -> PathBuf {
|
|
let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id()));
|
|
std::fs::create_dir_all(&dir).unwrap();
|
|
dir.join("infer.sock")
|
|
}
|
|
|
|
fn config(socket: &Path) -> Config {
|
|
let model =
|
|
std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
|
|
let text = format!(
|
|
r#"
|
|
[infer]
|
|
socket = "{}"
|
|
model = "{model}"
|
|
[slots]
|
|
main = 0
|
|
background = 1
|
|
[expect]
|
|
{EXPECT}
|
|
"#,
|
|
socket.display()
|
|
);
|
|
Config::parse(&text).unwrap()
|
|
}
|
|
|
|
fn user(text: &str) -> ChatMessage {
|
|
// A different prompt each run, so that an earlier run's cache cannot make a check pass.
|
|
let nonce = std::time::SystemTime::now()
|
|
.duration_since(std::time::UNIX_EPOCH)
|
|
.unwrap()
|
|
.as_nanos();
|
|
ChatMessage::User {
|
|
content: format!("{text} (run {nonce})"),
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "needs the real server; run with make verify-device"]
|
|
fn the_startup_self_test_passes() {
|
|
let socket = socket_path("selftest");
|
|
let _proxy = Proxy::start(&socket);
|
|
let client = Client::new(config(&socket));
|
|
loopd::selftest::run(&client, &mut |step| eprintln!("selftest: {step}")).unwrap();
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "needs the real server; run with make verify-device"]
|
|
fn a_capped_thinking_block_ends_and_the_answer_arrives() {
|
|
let socket = socket_path("cap");
|
|
let _proxy = Proxy::start(&socket);
|
|
let mut cfg = config(&socket);
|
|
cfg.limits.thinking_cap = 60;
|
|
let client = Client::new(cfg);
|
|
let ask = "Think at great length about how many ways 8 rooks fit on a chessboard without attacking \
|
|
each other and with none on the main diagonal. Then answer in one sentence.";
|
|
let req = ChatRequest {
|
|
slot: 0,
|
|
messages: vec![user(ask)],
|
|
tools: vec![],
|
|
thinking: true,
|
|
};
|
|
let mut capped = Vec::new();
|
|
let done = client
|
|
.chat_with_retry(&req, &mut |e| {
|
|
if let ChatEvent::ThinkingCapped { tokens } = e {
|
|
capped.push(*tokens);
|
|
}
|
|
})
|
|
.unwrap();
|
|
assert!(done.thinking_capped);
|
|
assert_eq!(capped.len(), 1);
|
|
assert!(capped[0] >= 60 && capped[0] < 70, "capped at {capped:?}");
|
|
assert!(
|
|
done.reasoning_tokens < 60 + 256,
|
|
"thinking went on to {}",
|
|
done.reasoning_tokens
|
|
);
|
|
assert!(
|
|
done.content.is_some_and(|c| !c.trim().is_empty()),
|
|
"an answer followed the forced end"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "needs the real server; run with make verify-device"]
|
|
fn a_request_survives_its_proxy_being_killed_and_restarted() {
|
|
let socket = socket_path("restart");
|
|
let mut proxy = Proxy::start(&socket);
|
|
let mut cfg = config(&socket);
|
|
cfg.limits.retry_backoff_ms = vec![1_500];
|
|
let client = Client::new(cfg);
|
|
let ask = "Write about 300 words on the history of cork.";
|
|
let req = ChatRequest {
|
|
slot: 0,
|
|
messages: vec![user(ask)],
|
|
tools: vec![],
|
|
thinking: false,
|
|
};
|
|
|
|
let (tx, rx) = mpsc::channel();
|
|
let worker = thread::spawn(move || {
|
|
let mut retries = 0;
|
|
let mut told = false;
|
|
let result = client.chat_with_retry(&req, &mut |e| match e {
|
|
ChatEvent::Content(_) if !told => {
|
|
told = true;
|
|
let _ = tx.send(());
|
|
}
|
|
ChatEvent::Retrying { .. } => retries += 1,
|
|
_ => {}
|
|
});
|
|
(result, retries)
|
|
});
|
|
|
|
rx.recv_timeout(Duration::from_secs(120))
|
|
.expect("no content arrived");
|
|
proxy.kill(); // the stream dies in the middle of the answer
|
|
thread::sleep(Duration::from_millis(300));
|
|
let _proxy = Proxy::start(&proxy.socket);
|
|
|
|
let (result, retries) = worker.join().unwrap();
|
|
let done = result.expect("the retry should have succeeded");
|
|
assert!(retries >= 1, "the request was retried");
|
|
assert!(
|
|
done.content
|
|
.is_some_and(|c| c.split_whitespace().count() > 100)
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "needs the real server; run with make verify-device"]
|
|
fn a_second_turn_reuses_the_first_turns_cache() {
|
|
let socket = socket_path("cache");
|
|
let _proxy = Proxy::start(&socket);
|
|
let client = Client::new(config(&socket));
|
|
let mut messages = vec![user(
|
|
"What is 17 * 23? Think briefly, then answer in one short sentence.",
|
|
)];
|
|
let req = ChatRequest {
|
|
slot: 0,
|
|
messages: messages.clone(),
|
|
tools: vec![],
|
|
thinking: true,
|
|
};
|
|
let turn1 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
|
|
assert!(
|
|
turn1.reasoning_content.is_some(),
|
|
"this check is about replaying a thinking block"
|
|
);
|
|
messages.push(ChatMessage::Assistant {
|
|
content: turn1.content.clone(),
|
|
reasoning_content: turn1.reasoning_content.clone(),
|
|
tool_calls: turn1.tool_calls.clone(),
|
|
});
|
|
messages.push(user("And 17 * 24?"));
|
|
let req = ChatRequest {
|
|
slot: 0,
|
|
messages,
|
|
tools: vec![],
|
|
thinking: true,
|
|
};
|
|
let turn2 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
|
|
assert_eq!(
|
|
cache_outcome(&turn1.timings, &turn2.timings),
|
|
CacheOutcome::Hit,
|
|
"{:?} then {:?}",
|
|
turn1.timings,
|
|
turn2.timings
|
|
);
|
|
}
|
|
|
|
// ---- M2b: the agent loop on the real server ----
|
|
|
|
/// A `loopd serve` on a private home, killed on drop.
|
|
struct Served {
|
|
child: Child,
|
|
home: PathBuf,
|
|
socket: PathBuf,
|
|
}
|
|
|
|
fn config_text(infer: &Path, home: &Path) -> String {
|
|
let model =
|
|
std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
|
|
format!(
|
|
r#"
|
|
[infer]
|
|
socket = "{}"
|
|
model = "{model}"
|
|
[slots]
|
|
main = 0
|
|
background = 1
|
|
[expect]
|
|
{EXPECT}
|
|
[paths]
|
|
home = "{}"
|
|
"#,
|
|
infer.display(),
|
|
home.display()
|
|
)
|
|
}
|
|
|
|
impl Served {
|
|
/// Writes the config and the repository's `system.md` into `home`, and starts `loopd serve`.
|
|
fn start(infer: &Path, home: &Path) -> Served {
|
|
std::fs::create_dir_all(home).unwrap();
|
|
let config = home.join("config.toml");
|
|
std::fs::write(&config, config_text(infer, home)).unwrap();
|
|
let prompt = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
|
|
std::fs::copy(&prompt, home.join("system.md"))
|
|
.expect("config/system.md exists in the repository");
|
|
let socket = home.join("run").join("loop").join("loop.sock");
|
|
let _ = std::fs::remove_file(&socket);
|
|
let child = Command::new(env!("CARGO_BIN_EXE_loopd"))
|
|
.arg("serve")
|
|
.arg("--config")
|
|
.arg(&config)
|
|
.spawn()
|
|
.expect("cannot start loopd");
|
|
let mut served = Served {
|
|
child,
|
|
home: home.to_path_buf(),
|
|
socket,
|
|
};
|
|
for _ in 0..600 {
|
|
if served.socket.exists() {
|
|
return served;
|
|
}
|
|
thread::sleep(Duration::from_millis(100));
|
|
}
|
|
served.kill();
|
|
panic!("loopd did not come up within 60 s");
|
|
}
|
|
|
|
fn kill(&mut self) {
|
|
let _ = self.child.kill();
|
|
let _ = self.child.wait();
|
|
}
|
|
|
|
/// One `bxctl chat --say` turn. Returns the answer.
|
|
fn say(&self, session: &str, text: &str) -> String {
|
|
let bxctl = std::env::var("BOXMAKER_BXCTL").expect("BOXMAKER_BXCTL is not set");
|
|
let output = Command::new(bxctl)
|
|
.arg("chat")
|
|
.arg("--socket")
|
|
.arg(&self.socket)
|
|
.args(["--session", session, "--no-thinking", "--say", text])
|
|
.output()
|
|
.expect("cannot run bxctl");
|
|
assert!(
|
|
output.status.success(),
|
|
"bxctl failed: {}",
|
|
String::from_utf8_lossy(&output.stderr)
|
|
);
|
|
String::from_utf8_lossy(&output.stdout)
|
|
.trim_end()
|
|
.to_string()
|
|
}
|
|
|
|
fn records(&self, session: &str) -> Vec<proto::LogRecord> {
|
|
let text =
|
|
std::fs::read_to_string(self.home.join("sessions").join(session).join("0.jsonl"))
|
|
.unwrap();
|
|
text.lines()
|
|
.map(|l| serde_json::from_str(l).unwrap())
|
|
.collect()
|
|
}
|
|
}
|
|
|
|
impl Drop for Served {
|
|
fn drop(&mut self) {
|
|
self.kill();
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "needs the real server; run with make verify-device"]
|
|
fn the_baseline_fits_the_token_budget() {
|
|
let socket = socket_path("budget");
|
|
let _proxy = Proxy::start(&socket);
|
|
let home = socket.parent().unwrap().join("home");
|
|
std::fs::create_dir_all(&home).unwrap();
|
|
let mut cfg = config(&socket);
|
|
cfg.paths.home = home.clone();
|
|
cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
|
|
let baseline =
|
|
loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m3a()).unwrap();
|
|
let client = Client::new(cfg);
|
|
// The system text plus every tool schema as the request carries it.
|
|
let mut text = baseline.system.clone();
|
|
for tool in &baseline.tools {
|
|
text.push('\n');
|
|
text.push_str(&serde_json::to_string(&tool).unwrap());
|
|
}
|
|
let tokens = client.tokenize(&text).unwrap();
|
|
eprintln!("baseline: {tokens} tokens");
|
|
assert!(
|
|
tokens <= 3000,
|
|
"the baseline is {tokens} tokens; the brief allows 3000"
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
#[ignore = "needs the real server; run with make verify-device"]
|
|
fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() {
|
|
let socket = socket_path("loop");
|
|
let _proxy = Proxy::start(&socket);
|
|
let home = socket.parent().unwrap().join("home");
|
|
let session = format!("device-{}", std::process::id());
|
|
let mut served = Served::start(&socket, &home);
|
|
|
|
let a1 = served.say(&session, "Reply with exactly: box made.");
|
|
assert!(a1.to_lowercase().contains("box made"), "{a1}");
|
|
let a2 = served.say(
|
|
&session,
|
|
"What is the current time? Use your clock tool, then tell me the year.",
|
|
);
|
|
assert!(a2.contains("2026") || a2.contains("202"), "{a2}");
|
|
// No broker is configured here, so the call fails in plain words and the turn goes on. What
|
|
// is checked is the path: find_tool, then call_tool, then an answer.
|
|
let a3 = served.say(
|
|
&session,
|
|
"Find a tool that reads files, use it to read /etc/hostname, and tell me in one \
|
|
sentence what happened.",
|
|
);
|
|
assert!(!a3.trim().is_empty(), "the turn must end in an answer");
|
|
|
|
served.kill();
|
|
served = Served::start(&socket, &home);
|
|
let a4 = served.say(
|
|
&session,
|
|
"What were the exact words I first asked you to reply with?",
|
|
);
|
|
assert!(
|
|
a4.to_lowercase().contains("box made"),
|
|
"after a restart the session must still know: {a4}"
|
|
);
|
|
|
|
let records = served.records(&session);
|
|
let tool_names: Vec<String> = records
|
|
.iter()
|
|
.filter_map(|r| match r {
|
|
proto::LogRecord::Assistant { tool_calls, .. } => Some(
|
|
tool_calls
|
|
.iter()
|
|
.map(|c| c.name.clone())
|
|
.collect::<Vec<_>>(),
|
|
),
|
|
_ => None,
|
|
})
|
|
.flatten()
|
|
.collect();
|
|
assert!(tool_names.contains(&"clock".to_string()), "{tool_names:?}");
|
|
assert!(
|
|
tool_names.contains(&"find_tool".to_string())
|
|
&& tool_names.contains(&"call_tool".to_string()),
|
|
"{tool_names:?}"
|
|
);
|
|
let usages = records
|
|
.iter()
|
|
.filter(|r| matches!(r, proto::LogRecord::Usage { .. }))
|
|
.count();
|
|
let assistants = records
|
|
.iter()
|
|
.filter(|r| matches!(r, proto::LogRecord::Assistant { .. }))
|
|
.count();
|
|
assert_eq!(usages, assistants, "one usage record per completion");
|
|
let losses: Vec<&proto::LogRecord> = records
|
|
.iter()
|
|
.filter(|r| matches!(r, proto::LogRecord::CacheLoss { .. }))
|
|
.collect();
|
|
assert!(
|
|
losses.is_empty(),
|
|
"every request hit the cache, including the one after the restart: {losses:?}"
|
|
);
|
|
let results = records
|
|
.iter()
|
|
.filter(|r| matches!(r, proto::LogRecord::ToolResult { .. }))
|
|
.count();
|
|
assert!(
|
|
results >= 3,
|
|
"clock, find_tool and call_tool each left a result: {results}"
|
|
);
|
|
}
|