boxmaker/crates/loopd/tests/device.rs

//! Checks against the real server. Ignored unless run by `make verify-device`. Do not edit.
//!
//! They need two environment variables:
//!   BOXMAKER_INFERPROXY  path to the built `inferproxy` binary
//!   BOXMAKER_UPSTREAM    host:port of the llama-server router, for example straylight:11434
//! and, optionally, BOXMAKER_MODEL (default ornith-1.5-35b-a3b). They use slot 0.
//!
//! They never touch the real `llama-server` process. The "server dies" test kills and restarts
//! its own `inferproxy`.

#[path = "support/memoryd.rs"]
mod memoryd_run;

use loopd::config::Config;
use loopd::llama::info::{CacheOutcome, cache_outcome};
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client};
use std::path::{Path, PathBuf};
use std::process::{Child, Command};
use std::sync::mpsc;
use std::thread;
use std::time::Duration;

struct Proxy {
    child: Child,
    socket: PathBuf,
}

impl Proxy {
    fn start(socket: &Path) -> Proxy {
        let binary = std::env::var("BOXMAKER_INFERPROXY").expect("BOXMAKER_INFERPROXY is not set");
        let upstream = std::env::var("BOXMAKER_UPSTREAM").expect("BOXMAKER_UPSTREAM is not set");
        let child = Command::new(binary)
            .arg("--listen")
            .arg(socket)
            .arg("--upstream")
            .arg(upstream)
            .spawn()
            .expect("cannot start inferproxy");
        let mut proxy = Proxy {
            child,
            socket: socket.to_path_buf(),
        };
        for _ in 0..100 {
            if socket.exists() {
                return proxy;
            }
            thread::sleep(Duration::from_millis(20));
        }
        proxy.kill();
        panic!("inferproxy did not create {}", socket.display());
    }

    fn kill(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
    }
}

impl Drop for Proxy {
    fn drop(&mut self) {
        self.kill();
        // `socket_path` made a directory of its own for the socket; remove it with the proxy
        // (tip T44).
        if let Some(dir) = self.socket.parent() {
            let _ = std::fs::remove_dir_all(dir);
        }
    }
}

/// The server as deployed on straylight, recorded in one place. When the deployment changes on
/// purpose, change this and `tools/check-m3a-device.sh` together, and note it in
/// `docs/inference-contract.md`. Since 2026-09-22: four slots over one 262,144-token pool.
const EXPECT: &str = r#"template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
n_ctx = 262144
slots = 4"#;

fn socket_path(name: &str) -> PathBuf {
    let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id()));
    std::fs::create_dir_all(&dir).unwrap();
    dir.join("infer.sock")
}

fn config(socket: &Path) -> Config {
    let model =
        std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
    let text = format!(
        r#"
[infer]
socket = "{}"
model = "{model}"
[slots]
main = 0
background = 1
[expect]
{EXPECT}
"#,
        socket.display()
    );
    Config::parse(&text).unwrap()
}

fn user(text: &str) -> ChatMessage {
    // A different prompt each run, so that an earlier run's cache cannot make a check pass.
    let nonce = std::time::SystemTime::now()
        .duration_since(std::time::UNIX_EPOCH)
        .unwrap()
        .as_nanos();
    ChatMessage::User {
        content: format!("{text} (run {nonce})"),
    }
}

#[test]
#[ignore = "needs the real server; run with make verify-device"]
fn the_startup_self_test_passes() {
    let socket = socket_path("selftest");
    let _proxy = Proxy::start(&socket);
    let client = Client::new(config(&socket));
    loopd::selftest::run(&client, &mut |step| eprintln!("selftest: {step}")).unwrap();
}

#[test]
#[ignore = "needs the real server; run with make verify-device"]
fn a_capped_thinking_block_ends_and_the_answer_arrives() {
    let socket = socket_path("cap");
    let _proxy = Proxy::start(&socket);
    let mut cfg = config(&socket);
    cfg.limits.thinking_cap = 60;
    let client = Client::new(cfg);
    let ask = "Think at great length about how many ways 8 rooks fit on a chessboard without attacking \
               each other and with none on the main diagonal. Then answer in one sentence.";
    let req = ChatRequest {
        slot: 0,
        messages: vec![user(ask)],
        tools: vec![],
        thinking: true,
    };
    let mut capped = Vec::new();
    let done = client
        .chat_with_retry(&req, &mut |e| {
            if let ChatEvent::ThinkingCapped { tokens } = e {
                capped.push(*tokens);
            }
        })
        .unwrap();
    assert!(done.thinking_capped);
    assert_eq!(capped.len(), 1);
    assert!(capped[0] >= 60 && capped[0] < 70, "capped at {capped:?}");
    assert!(
        done.reasoning_tokens < 60 + 256,
        "thinking went on to {}",
        done.reasoning_tokens
    );
    assert!(
        done.content.is_some_and(|c| !c.trim().is_empty()),
        "an answer followed the forced end"
    );
}

#[test]
#[ignore = "needs the real server; run with make verify-device"]
fn a_request_survives_its_proxy_being_killed_and_restarted() {
    let socket = socket_path("restart");
    let mut proxy = Proxy::start(&socket);
    let mut cfg = config(&socket);
    cfg.limits.retry_backoff_ms = vec![1_500];
    let client = Client::new(cfg);
    let ask = "Write about 300 words on the history of cork.";
    let req = ChatRequest {
        slot: 0,
        messages: vec![user(ask)],
        tools: vec![],
        thinking: false,
    };

    let (tx, rx) = mpsc::channel();
    let worker = thread::spawn(move || {
        let mut retries = 0;
        let mut told = false;
        let result = client.chat_with_retry(&req, &mut |e| match e {
            ChatEvent::Content(_) if !told => {
                told = true;
                let _ = tx.send(());
            }
            ChatEvent::Retrying { .. } => retries += 1,
            _ => {}
        });
        (result, retries)
    });

    rx.recv_timeout(Duration::from_secs(120))
        .expect("no content arrived");
    proxy.kill(); // the stream dies in the middle of the answer
    thread::sleep(Duration::from_millis(300));
    let _proxy = Proxy::start(&proxy.socket);

    let (result, retries) = worker.join().unwrap();
    let done = result.expect("the retry should have succeeded");
    assert!(retries >= 1, "the request was retried");
    assert!(
        done.content
            .is_some_and(|c| c.split_whitespace().count() > 100)
    );
}

#[test]
#[ignore = "needs the real server; run with make verify-device"]
fn a_second_turn_reuses_the_first_turns_cache() {
    let socket = socket_path("cache");
    let _proxy = Proxy::start(&socket);
    let client = Client::new(config(&socket));
    let mut messages = vec![user(
        "What is 17 * 23? Think briefly, then answer in one short sentence.",
    )];
    let req = ChatRequest {
        slot: 0,
        messages: messages.clone(),
        tools: vec![],
        thinking: true,
    };
    let turn1 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
    assert!(
        turn1.reasoning_content.is_some(),
        "this check is about replaying a thinking block"
    );
    messages.push(ChatMessage::Assistant {
        content: turn1.content.clone(),
        reasoning_content: turn1.reasoning_content.clone(),
        tool_calls: turn1.tool_calls.clone(),
    });
    messages.push(user("And 17 * 24?"));
    let req = ChatRequest {
        slot: 0,
        messages,
        tools: vec![],
        thinking: true,
    };
    let turn2 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
    assert_eq!(
        cache_outcome(&turn1.timings, &turn2.timings),
        CacheOutcome::Hit,
        "{:?} then {:?}",
        turn1.timings,
        turn2.timings
    );
}

// ---- M2b: the agent loop on the real server ----

/// A `loopd serve` on a private home, killed on drop.
struct Served {
    child: Child,
    home: PathBuf,
    socket: PathBuf,
    memoryd: Child,
}

fn config_text(infer: &Path, home: &Path) -> String {
    let model =
        std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
    format!(
        r#"
[infer]
socket = "{}"
model = "{model}"
[slots]
main = 0
background = 1
[expect]
{EXPECT}
[paths]
home = "{}"
"#,
        infer.display(),
        home.display()
    )
}

impl Served {
    /// Writes the config and the repository's `system.md` into `home`, and starts `memoryd serve`
    /// (with the repository's starter `soul.md`), then `loopd serve`: `loopd` starts no session
    /// without the memory block.
    fn start(infer: &Path, home: &Path) -> Served {
        std::fs::create_dir_all(home).unwrap();
        let memoryd = memoryd_run::start(home);
        let config = home.join("config.toml");
        std::fs::write(&config, config_text(infer, home)).unwrap();
        let prompt = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
        std::fs::copy(&prompt, home.join("system.md"))
            .expect("config/system.md exists in the repository");
        let socket = home.join("run").join("loop").join("loop.sock");
        let _ = std::fs::remove_file(&socket);
        let child = Command::new(env!("CARGO_BIN_EXE_loopd"))
            .arg("serve")
            .arg("--config")
            .arg(&config)
            .spawn()
            .expect("cannot start loopd");
        let mut served = Served {
            child,
            home: home.to_path_buf(),
            socket,
            memoryd,
        };
        for _ in 0..600 {
            if served.socket.exists() {
                return served;
            }
            thread::sleep(Duration::from_millis(100));
        }
        served.kill();
        panic!("loopd did not come up within 60 s");
    }

    fn kill(&mut self) {
        let _ = self.child.kill();
        let _ = self.child.wait();
        let _ = self.memoryd.kill();
        let _ = self.memoryd.wait();
    }

    /// One `bxctl chat --say` turn. Returns the answer.
    fn say(&self, session: &str, text: &str) -> String {
        let bxctl = std::env::var("BOXMAKER_BXCTL").expect("BOXMAKER_BXCTL is not set");
        let output = Command::new(bxctl)
            .arg("chat")
            .arg("--socket")
            .arg(&self.socket)
            .args(["--session", session, "--no-thinking", "--say", text])
            .output()
            .expect("cannot run bxctl");
        assert!(
            output.status.success(),
            "bxctl failed: {}",
            String::from_utf8_lossy(&output.stderr)
        );
        String::from_utf8_lossy(&output.stdout)
            .trim_end()
            .to_string()
    }

    fn records(&self, session: &str) -> Vec<proto::LogRecord> {
        let text =
            std::fs::read_to_string(self.home.join("sessions").join(session).join("0.jsonl"))
                .unwrap();
        text.lines()
            .map(|l| serde_json::from_str(l).unwrap())
            .collect()
    }
}

impl Drop for Served {
    fn drop(&mut self) {
        self.kill();
    }
}

#[test]
#[ignore = "needs the real server; run with make verify-device"]
fn the_baseline_fits_the_token_budget() {
    let socket = socket_path("budget");
    let _proxy = Proxy::start(&socket);
    let home = socket.parent().unwrap().join("home");
    std::fs::create_dir_all(&home).unwrap();
    let mut cfg = config(&socket);
    cfg.paths.home = home.clone();
    cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
    let root = Path::new(env!("CARGO_MANIFEST_DIR")).join("../..");
    let soul = std::fs::read_to_string(root.join("config/soul.md")).unwrap();
    let prose = std::fs::read_to_string(root.join("docs/design.md")).unwrap();
    let prose: String = prose.split_whitespace().collect::<Vec<_>>().join(" ");
    let client = Client::new(cfg.clone());
    // The memory block as memoryd renders it (M5a spec, section 4): the starter soul with empty
    // files, as a new install sends it; then every file at its limit, the most it can be.
    let empty = format!(
        "# soul.md\n{}\n\n# memory.md (0 of 2,200 characters)\n(empty)\n\n# user.md (0 of 1,375 characters)\n(empty)",
        soul.trim()
    );
    let full = format!(
        "# soul.md\n{}\n\n# memory.md (2,200 of 2,200 characters)\n{}\n\n# user.md (1,375 of 1,375 characters)\n{}",
        prose.chars().take(4000).collect::<String>(),
        prose.chars().skip(4000).take(2200).collect::<String>(),
        prose.chars().skip(6200).take(1375).collect::<String>()
    );
    for (what, block) in [("starter", empty), ("full", full)] {
        let baseline =
            loopd::baseline::Baseline::assemble_with(&cfg, &loopd::tools::Registry::m5a(), &block)
                .unwrap();
        // The system text plus every tool schema as the request carries it.
        let mut text = baseline.system.clone();
        for tool in &baseline.tools {
            text.push('\n');
            text.push_str(&serde_json::to_string(&tool).unwrap());
        }
        let tokens = client.tokenize(&text).unwrap();
        eprintln!("baseline, {what} memory: {tokens} tokens");
        assert!(
            tokens <= 3000,
            "the baseline with {what} memory is {tokens} tokens; the brief allows 3000"
        );
    }
}

#[test]
#[ignore = "needs the real server; run with make verify-device"]
fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() {
    let socket = socket_path("loop");
    let _proxy = Proxy::start(&socket);
    let home = socket.parent().unwrap().join("home");
    let session = format!("device-{}", std::process::id());
    let mut served = Served::start(&socket, &home);

    let a1 = served.say(&session, "Reply with exactly: box made.");
    assert!(a1.to_lowercase().contains("box made"), "{a1}");
    let a2 = served.say(
        &session,
        "What is the current time? Use your clock tool, then tell me the year.",
    );
    assert!(a2.contains("2026") || a2.contains("202"), "{a2}");
    // No broker is configured here, so the call fails in plain words and the turn goes on. What
    // is checked is the path: find_tool, then call_tool, then an answer.
    let a3 = served.say(
        &session,
        "Find a tool that reads files, use it to read /etc/hostname, and tell me in one \
         sentence what happened.",
    );
    assert!(!a3.trim().is_empty(), "the turn must end in an answer");

    served.kill();
    served = Served::start(&socket, &home);
    let a4 = served.say(
        &session,
        "What were the exact words I first asked you to reply with?",
    );
    assert!(
        a4.to_lowercase().contains("box made"),
        "after a restart the session must still know: {a4}"
    );

    let records = served.records(&session);
    let tool_names: Vec<String> = records
        .iter()
        .filter_map(|r| match r {
            proto::LogRecord::Assistant { tool_calls, .. } => Some(
                tool_calls
                    .iter()
                    .map(|c| c.name.clone())
                    .collect::<Vec<_>>(),
            ),
            _ => None,
        })
        .flatten()
        .collect();
    assert!(tool_names.contains(&"clock".to_string()), "{tool_names:?}");
    assert!(
        tool_names.contains(&"find_tool".to_string())
            && tool_names.contains(&"call_tool".to_string()),
        "{tool_names:?}"
    );
    let usages = records
        .iter()
        .filter(|r| matches!(r, proto::LogRecord::Usage { .. }))
        .count();
    let assistants = records
        .iter()
        .filter(|r| matches!(r, proto::LogRecord::Assistant { .. }))
        .count();
    assert_eq!(usages, assistants, "one usage record per completion");
    let losses: Vec<&proto::LogRecord> = records
        .iter()
        .filter(|r| matches!(r, proto::LogRecord::CacheLoss { .. }))
        .collect();
    assert!(
        losses.is_empty(),
        "every request hit the cache, including the one after the restart: {losses:?}"
    );
    let results = records
        .iter()
        .filter(|r| matches!(r, proto::LogRecord::ToolResult { .. }))
        .count();
    assert!(
        results >= 3,
        "clock, find_tool and call_tool each left a result: {results}"
    );
}