//! Checks against the real server. Ignored unless run by `make verify-device`. Do not edit. //! //! They need two environment variables: //! BOXMAKER_INFERPROXY path to the built `inferproxy` binary //! BOXMAKER_UPSTREAM host:port of the llama-server router, for example straylight:11434 //! and, optionally, BOXMAKER_MODEL (default ornith-1.5-35b-a3b). They use slot 0. //! //! They never touch the real `llama-server` process. The "server dies" test kills and restarts //! its own `inferproxy`. use loopd::config::Config; use loopd::llama::info::{CacheOutcome, cache_outcome}; use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client}; use std::path::{Path, PathBuf}; use std::process::{Child, Command}; use std::sync::mpsc; use std::thread; use std::time::Duration; struct Proxy { child: Child, socket: PathBuf, } impl Proxy { fn start(socket: &Path) -> Proxy { let binary = std::env::var("BOXMAKER_INFERPROXY").expect("BOXMAKER_INFERPROXY is not set"); let upstream = std::env::var("BOXMAKER_UPSTREAM").expect("BOXMAKER_UPSTREAM is not set"); let child = Command::new(binary) .arg("--listen") .arg(socket) .arg("--upstream") .arg(upstream) .spawn() .expect("cannot start inferproxy"); let mut proxy = Proxy { child, socket: socket.to_path_buf(), }; for _ in 0..100 { if socket.exists() { return proxy; } thread::sleep(Duration::from_millis(20)); } proxy.kill(); panic!("inferproxy did not create {}", socket.display()); } fn kill(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); } } impl Drop for Proxy { fn drop(&mut self) { self.kill(); } } fn socket_path(name: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); dir.join("infer.sock") } fn config(socket: &Path) -> Config { let model = std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string()); let text = format!( r#" [infer] socket = "{}" model = "{model}" [slots] main = 0 background = 1 [expect] template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" n_ctx = 131072 slots = 2 "#, socket.display() ); Config::parse(&text).unwrap() } fn user(text: &str) -> ChatMessage { // A different prompt each run, so that an earlier run's cache cannot make a check pass. let nonce = std::time::SystemTime::now() .duration_since(std::time::UNIX_EPOCH) .unwrap() .as_nanos(); ChatMessage::User { content: format!("{text} (run {nonce})"), } } #[test] #[ignore = "needs the real server; run with make verify-device"] fn the_startup_self_test_passes() { let socket = socket_path("selftest"); let _proxy = Proxy::start(&socket); let client = Client::new(config(&socket)); loopd::selftest::run(&client, &mut |step| eprintln!("selftest: {step}")).unwrap(); } #[test] #[ignore = "needs the real server; run with make verify-device"] fn a_capped_thinking_block_ends_and_the_answer_arrives() { let socket = socket_path("cap"); let _proxy = Proxy::start(&socket); let mut cfg = config(&socket); cfg.limits.thinking_cap = 60; let client = Client::new(cfg); let ask = "Think at great length about how many ways 8 rooks fit on a chessboard without attacking \ each other and with none on the main diagonal. Then answer in one sentence."; let req = ChatRequest { slot: 0, messages: vec![user(ask)], tools: vec![], thinking: true, }; let mut capped = Vec::new(); let done = client .chat_with_retry(&req, &mut |e| { if let ChatEvent::ThinkingCapped { tokens } = e { capped.push(*tokens); } }) .unwrap(); assert!(done.thinking_capped); assert_eq!(capped.len(), 1); assert!(capped[0] >= 60 && capped[0] < 70, "capped at {capped:?}"); assert!( done.reasoning_tokens < 60 + 256, "thinking went on to {}", done.reasoning_tokens ); assert!( done.content.is_some_and(|c| !c.trim().is_empty()), "an answer followed the forced end" ); } #[test] #[ignore = "needs the real server; run with make verify-device"] fn a_request_survives_its_proxy_being_killed_and_restarted() { let socket = socket_path("restart"); let mut proxy = Proxy::start(&socket); let mut cfg = config(&socket); cfg.limits.retry_backoff_ms = vec![1_500]; let client = Client::new(cfg); let ask = "Write about 300 words on the history of cork."; let req = ChatRequest { slot: 0, messages: vec![user(ask)], tools: vec![], thinking: false, }; let (tx, rx) = mpsc::channel(); let worker = thread::spawn(move || { let mut retries = 0; let mut told = false; let result = client.chat_with_retry(&req, &mut |e| match e { ChatEvent::Content(_) if !told => { told = true; let _ = tx.send(()); } ChatEvent::Retrying { .. } => retries += 1, _ => {} }); (result, retries) }); rx.recv_timeout(Duration::from_secs(120)) .expect("no content arrived"); proxy.kill(); // the stream dies in the middle of the answer thread::sleep(Duration::from_millis(300)); let _proxy = Proxy::start(&proxy.socket); let (result, retries) = worker.join().unwrap(); let done = result.expect("the retry should have succeeded"); assert!(retries >= 1, "the request was retried"); assert!( done.content .is_some_and(|c| c.split_whitespace().count() > 100) ); } #[test] #[ignore = "needs the real server; run with make verify-device"] fn a_second_turn_reuses_the_first_turns_cache() { let socket = socket_path("cache"); let _proxy = Proxy::start(&socket); let client = Client::new(config(&socket)); let mut messages = vec![user( "What is 17 * 23? Think briefly, then answer in one short sentence.", )]; let req = ChatRequest { slot: 0, messages: messages.clone(), tools: vec![], thinking: true, }; let turn1 = client.chat_with_retry(&req, &mut |_| {}).unwrap(); assert!( turn1.reasoning_content.is_some(), "this check is about replaying a thinking block" ); messages.push(ChatMessage::Assistant { content: turn1.content.clone(), reasoning_content: turn1.reasoning_content.clone(), tool_calls: turn1.tool_calls.clone(), }); messages.push(user("And 17 * 24?")); let req = ChatRequest { slot: 0, messages, tools: vec![], thinking: true, }; let turn2 = client.chat_with_retry(&req, &mut |_| {}).unwrap(); assert_eq!( cache_outcome(&turn1.timings, &turn2.timings), CacheOutcome::Hit, "{:?} then {:?}", turn1.timings, turn2.timings ); } // ---- M2b: the agent loop on the real server ---- /// A `loopd serve` on a private home, killed on drop. struct Served { child: Child, home: PathBuf, socket: PathBuf, } fn config_text(infer: &Path, home: &Path) -> String { let model = std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string()); format!( r#" [infer] socket = "{}" model = "{model}" [slots] main = 0 background = 1 [expect] template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" n_ctx = 131072 slots = 2 [paths] home = "{}" "#, infer.display(), home.display() ) } impl Served { /// Writes the config and the repository's `system.md` into `home`, and starts `loopd serve`. fn start(infer: &Path, home: &Path) -> Served { std::fs::create_dir_all(home).unwrap(); let config = home.join("config.toml"); std::fs::write(&config, config_text(infer, home)).unwrap(); let prompt = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md"); std::fs::copy(&prompt, home.join("system.md")) .expect("config/system.md exists in the repository"); let socket = home.join("run").join("loop").join("loop.sock"); let _ = std::fs::remove_file(&socket); let child = Command::new(env!("CARGO_BIN_EXE_loopd")) .arg("serve") .arg("--config") .arg(&config) .spawn() .expect("cannot start loopd"); let mut served = Served { child, home: home.to_path_buf(), socket, }; for _ in 0..600 { if served.socket.exists() { return served; } thread::sleep(Duration::from_millis(100)); } served.kill(); panic!("loopd did not come up within 60 s"); } fn kill(&mut self) { let _ = self.child.kill(); let _ = self.child.wait(); } /// One `bxctl chat --say` turn. Returns the answer. fn say(&self, session: &str, text: &str) -> String { let bxctl = std::env::var("BOXMAKER_BXCTL").expect("BOXMAKER_BXCTL is not set"); let output = Command::new(bxctl) .arg("chat") .arg("--socket") .arg(&self.socket) .args(["--session", session, "--no-thinking", "--say", text]) .output() .expect("cannot run bxctl"); assert!( output.status.success(), "bxctl failed: {}", String::from_utf8_lossy(&output.stderr) ); String::from_utf8_lossy(&output.stdout) .trim_end() .to_string() } fn records(&self, session: &str) -> Vec { let text = std::fs::read_to_string(self.home.join("sessions").join(session).join("0.jsonl")) .unwrap(); text.lines() .map(|l| serde_json::from_str(l).unwrap()) .collect() } } impl Drop for Served { fn drop(&mut self) { self.kill(); } } #[test] #[ignore = "needs the real server; run with make verify-device"] fn the_baseline_fits_the_token_budget() { let socket = socket_path("budget"); let _proxy = Proxy::start(&socket); let home = socket.parent().unwrap().join("home"); std::fs::create_dir_all(&home).unwrap(); let mut cfg = config(&socket); cfg.paths.home = home.clone(); cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md"); let baseline = loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m3a()).unwrap(); let client = Client::new(cfg); // The system text plus every tool schema as the request carries it. let mut text = baseline.system.clone(); for tool in &baseline.tools { text.push('\n'); text.push_str(&serde_json::to_string(&tool).unwrap()); } let tokens = client.tokenize(&text).unwrap(); eprintln!("baseline: {tokens} tokens"); assert!( tokens <= 3000, "the baseline is {tokens} tokens; the brief allows 3000" ); } #[test] #[ignore = "needs the real server; run with make verify-device"] fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() { let socket = socket_path("loop"); let _proxy = Proxy::start(&socket); let home = socket.parent().unwrap().join("home"); let session = format!("device-{}", std::process::id()); let mut served = Served::start(&socket, &home); let a1 = served.say(&session, "Reply with exactly: box made."); assert!(a1.to_lowercase().contains("box made"), "{a1}"); let a2 = served.say( &session, "What is the current time? Use your clock tool, then tell me the year.", ); assert!(a2.contains("2026") || a2.contains("202"), "{a2}"); // No broker is configured here, so the call fails in plain words and the turn goes on. What // is checked is the path: find_tool, then call_tool, then an answer. let a3 = served.say( &session, "Find a tool that reads files, use it to read /etc/hostname, and tell me in one \ sentence what happened.", ); assert!(!a3.trim().is_empty(), "the turn must end in an answer"); served.kill(); served = Served::start(&socket, &home); let a4 = served.say( &session, "What were the exact words I first asked you to reply with?", ); assert!( a4.to_lowercase().contains("box made"), "after a restart the session must still know: {a4}" ); let records = served.records(&session); let tool_names: Vec = records .iter() .filter_map(|r| match r { proto::LogRecord::Assistant { tool_calls, .. } => Some( tool_calls .iter() .map(|c| c.name.clone()) .collect::>(), ), _ => None, }) .flatten() .collect(); assert!(tool_names.contains(&"clock".to_string()), "{tool_names:?}"); assert!( tool_names.contains(&"find_tool".to_string()) && tool_names.contains(&"call_tool".to_string()), "{tool_names:?}" ); let usages = records .iter() .filter(|r| matches!(r, proto::LogRecord::Usage { .. })) .count(); let assistants = records .iter() .filter(|r| matches!(r, proto::LogRecord::Assistant { .. })) .count(); assert_eq!(usages, assistants, "one usage record per completion"); let losses: Vec<&proto::LogRecord> = records .iter() .filter(|r| matches!(r, proto::LogRecord::CacheLoss { .. })) .collect(); assert!( losses.is_empty(), "every request hit the cache, including the one after the restart: {losses:?}" ); let results = records .iter() .filter(|r| matches!(r, proto::LogRecord::ToolResult { .. })) .count(); assert!( results >= 3, "clock, find_tool and call_tool each left a result: {results}" ); }