From 2249f7fd91949c2577538d96ee03b8ddb0f42ed4 Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Tue, 22 Sep 2026 20:12:05 -0700 Subject: [PATCH] Add the end-to-end test: loopd against the real brokerd binary Implemented-By: Grok 4.6 --- Makefile | 3 + crates/loopd/tests/end_to_end.rs | 168 +++++++++++++++++++++++++++++++ docs/implementer-log.md | 1 + tools/check-m3a-device.sh | 151 +++++++++++++++++++++++++++ 4 files changed, 323 insertions(+) create mode 100644 crates/loopd/tests/end_to_end.rs create mode 100755 tools/check-m3a-device.sh diff --git a/Makefile b/Makefile index ff348f5..619d497 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,9 @@ gate: cargo fmt --all --check cargo clippy --workspace --all-targets --locked --offline -- -D warnings cargo test --workspace --locked --offline + cargo build --workspace --locked --offline + BOXMAKER_BROKERD=$(CURDIR)/target/debug/brokerd \ + cargo test -p loopd --test end_to_end --locked --offline -- --ignored cargo deny --offline check bans licenses sources sh scripts/check-lines.sh sh scripts/check-crate-deps.sh diff --git a/crates/loopd/tests/end_to_end.rs b/crates/loopd/tests/end_to_end.rs new file mode 100644 index 0000000..5647d34 --- /dev/null +++ b/crates/loopd/tests/end_to_end.rs @@ -0,0 +1,168 @@ +//! End to end in two processes: `loopd`'s turn loop, with a `BrokerPort`, against the real +//! `brokerd` binary on a temporary home, and the fake llama server. Do not edit. +//! +//! It cannot be one process: `loopd` may not depend on `brokerd`, even for tests. The path of +//! the `brokerd` binary comes from `BOXMAKER_BROKERD`; without it the test is ignored, and +//! `make gate` builds the workspace and runs it with the variable set. + +mod support; + +use std::os::unix::net::UnixStream; +use std::path::{Path, PathBuf}; +use std::process::{Child, Command, Stdio}; +use std::time::{Duration, Instant}; + +use loopd::baseline::Baseline; +use loopd::broker_port::BrokerPort; +use loopd::llama::Client; +use loopd::session::Session; +use loopd::tools::Registry; +use loopd::turn::{Runtime, run_turn}; +use proto::{ + AuditEvent, AuditRecord, ChainVerifier, DecisionRecord, DenyReason, SessionId, TurnEvent, +}; +use support::{FakeServer, Home, Reply}; + +const CHAT: &str = "/v1/chat/completions"; + +/// `brokerd serve`, killed when dropped. +struct Brokerd(Child); + +impl Drop for Brokerd { + fn drop(&mut self) { + let _ = self.0.kill(); + let _ = self.0.wait(); + } +} + +fn start_brokerd(home: &Path) -> (Brokerd, PathBuf) { + let binary = std::env::var_os("BOXMAKER_BROKERD") + .expect("set BOXMAKER_BROKERD to the brokerd binary; `make gate` does"); + std::fs::create_dir_all(home.join("grants")).unwrap(); + let config = home.join("brokerd.toml"); + let text = format!( + "[paths]\nhome = \"{h}\"\ngrants = \"{h}/grants\"\n", + h = home.display() + ); + std::fs::write(&config, text).unwrap(); + let child = Command::new(binary) + .args(["serve", "--config"]) + .arg(&config) + .stdout(Stdio::null()) + .stderr(Stdio::null()) + .spawn() + .unwrap(); + let brokerd = Brokerd(child); + let socket = home.join("run/loop-broker/broker.sock"); + let until = Instant::now() + Duration::from_secs(10); + while UnixStream::connect(&socket).is_err() { + assert!( + Instant::now() < until, + "brokerd never listened on {}", + socket.display() + ); + std::thread::sleep(Duration::from_millis(20)); + } + (brokerd, socket) +} + +/// Every record under `audit/`, after checking that the chain verifies. +fn audit(dir: &Path) -> Vec { + let mut names: Vec = std::fs::read_dir(dir) + .unwrap() + .map(|e| e.unwrap().file_name().into_string().unwrap()) + .filter(|n| n.ends_with(".jsonl")) + .collect(); + names.sort(); + let mut verifier = ChainVerifier::new(); + let mut records = Vec::new(); + for name in &names { + let bytes = std::fs::read(dir.join(name)).unwrap(); + verifier.feed(name, &bytes); + for line in String::from_utf8(bytes).unwrap().lines() { + records.push(serde_json::from_str(line).unwrap()); + } + } + let report = verifier.finish(); + assert!(report.failure.is_none(), "{:?}", report.failure); + assert!(report.torn_tail.is_none()); + assert_eq!(report.records, records.len() as u64); + records +} + +#[test] +#[ignore = "needs the brokerd binary in BOXMAKER_BROKERD; `make gate` runs it"] +fn a_call_no_grant_allows_reaches_the_model_as_a_denial_and_is_audited() { + let home = Home::new(); + let broker_home = home.dir.join("broker-home"); + let (_brokerd, socket) = start_brokerd(&broker_home); + + let server = FakeServer::start(); + // The recorded model calls `read_file` on /etc/hostname, then answers in plain text. + server.route( + CHAT, + vec![Reply::fixture("tool_call"), Reply::fixture("plain")], + ); + let cfg = home.config(&server.socket); + let client = Client::new(cfg.clone()); + let port = BrokerPort::new(socket, Duration::from_secs(10)); + let registry = Registry::m2b(); + let baseline = Baseline::assemble(&cfg, ®istry).unwrap(); + let id = SessionId::new("e2e").unwrap(); + let mut session = Session::create(&home.dir, id, baseline, cfg.slots.main).unwrap(); + let runtime = Runtime { + cfg: &cfg, + client: &client, + port: &port, + registry: ®istry, + }; + let mut events = Vec::new(); + let outcome = run_turn( + &mut session, + &runtime, + "what is this host called?", + &mut |e| events.push(e.clone()), + ); + assert!( + outcome.is_ok(), + "the turn goes on after a denial: {outcome:?}" + ); + assert!( + events.contains(&TurnEvent::ToolDenied { + name: "read_file".to_string(), + reason: DenyReason::NoGrant, + }), + "{events:?}" + ); + // The model reads the denial in its next request. + let second = server.requests_to(CHAT)[1].json(); + assert_eq!( + second["messages"][3]["content"], + "Denied: no grant allows this call." + ); + + let records = audit(&broker_home.join("audit")); + assert_eq!(records.len(), 1, "{records:?}"); + match &records[0].event { + AuditEvent::Decision { + session, + tool, + arguments, + outcome, + grant, + .. + } => { + assert_eq!(session.as_str(), "e2e"); + assert_eq!(tool, "read_file"); + assert_eq!(arguments, r#"{"path":"/etc/hostname"}"#); + assert_eq!( + *outcome, + DecisionRecord::Denied { + reason: DenyReason::NoGrant + } + ); + assert_eq!(*grant, None); + } + other => panic!("{other:?}"), + } +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 3dd656b..717b385 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone. | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| M3a/22-end-to-end | 2026-09-22 | done | 1 | pass | none | Copied `end_to_end.rs`, `Makefile` (workspace build then ignored e2e with `BOXMAKER_BROKERD`) and `tools/check-m3a-device.sh`. Without the variable: 0 passed, 1 ignored. With it: 1 passed five runs. `--ignored` without the variable fails with "set BOXMAKER_BROKERD". `make gate` ran the e2e line (`1 passed`) and prints `gate: ok`. Did not run the straylight script. | ? | | M3a/21-runbook-check | 2026-09-22 | done | 1 | pass | none | Wrote `scripts/check-runbook.sh`: find `*.rs` under crates (prune `target/`), awk out every `docs/runbook.md#` pointer, empty anchors fail, each remaining anchor must match a whole `## ` line, every missing one is reported with its files, then one exit. Step 5: dropping `-x` from grep failed with "the entry is the whole line, at level two"; `exit 1` at a missing-anchor report failed with "both missing entries and their files are reported". Real tree exits 0. `make gate` prints `gate: ok`. | ? | | M3a/20-bxctl-chat-approvals | 2026-09-22 | done | 1 | pass | `AdminError` has no `Io` variant (task 18 maps write failures to `Protocol` via `From`), so `handle_pending` exit 8 reports every `cmd_approve`/`cmd_refuse` `Err` as `approval {id}: {e}` rather than returning `Err(AdminError::Io(e))`. Writes inside `handle_pending` itself still use `?`. | Moved `stream_turn` into `chat.rs` with `OnPending`/`Approvals`/`TurnIo`/`handle_pending`. The owner is shown `brokerd`'s list item, never the event's tool/args. Only the id typed in full (after stripping one trailing `\n` then one `\r`) approves; anything else refuses. `run` holds one `BufReader` on stdin for both modes. `Printer::event` escapes model text and tool names, prints the three fail-closed runbook lines as whole literals, and prints nothing for `ApprovalPending`. 21/12/20/9/12/8 tests five runs; `make gate` prints `gate: ok`. | ? | | M3a/19-bxctl-audit-verify | 2026-09-21 | stopped | 0 | n/a | none | The task's "The command" section says to grep `crates/bxctl/src/main.rs` for `cannot read the audit log` and stop if it is absent. `grep` returned no match: `main.rs`'s `Command::AuditVerify { home: _ }` arm (line 39) is still the placeholder that prints `bxctl: audit verify is not implemented yet`, ignores `home`, never calls `bxctl::verify::run`, and never prints the error message. The task forbids editing `main.rs` or `lib.rs`, so `bxctl audit verify` cannot be wired to `run` without that edit, and step 5 (which relies on the wiring) cannot pass. Stopped per the task's explicit instruction rather than editing a forbidden file. | ? | diff --git a/tools/check-m3a-device.sh b/tools/check-m3a-device.sh new file mode 100755 index 0000000..e5cd69c --- /dev/null +++ b/tools/check-m3a-device.sh @@ -0,0 +1,151 @@ +#!/bin/sh +# The M3a check on straylight, run by the owner (not part of `make gate` or `verify-device`). +# +# A private home with one `ask` grant for `read_file` on a directory; `brokerd serve` and +# `loopd serve` on it, `loopd` talking to the real server through a private `inferproxy`. A +# `bxctl chat --say` asks Ornith to read a file in that directory; the approval appears in +# `bxctl approvals`; approving it gives the M3a runner's failure, which the model reports. The +# audit log must verify and hold a Decision, an Approval and a Result. +# +# It uses slot 0 only, and first checks that slot 0 is idle: the server is shared. +# +# sh tools/check-m3a-device.sh [host:port] (default straylight:11434) +# +# Needs curl and jq. On success the home is removed; on failure its path is printed. If the +# model is not loaded, `/slots` may fail: load it first (the check does not load models). +set -u + +UPSTREAM="${1:-straylight:11434}" +MODEL="${BOXMAKER_MODEL:-ornith-1.5-35b-a3b}" +ROOT=$(cd "$(dirname "$0")/.." && pwd) || exit 1 +BIN="$ROOT/target/debug" + +fail() { + echo "check-m3a-device: FAILED: $*" >&2 + [ -n "${HOME_DIR:-}" ] && echo "check-m3a-device: the home is kept at $HOME_DIR" >&2 + exit 1 +} + +for tool in curl jq cargo; do + command -v "$tool" > /dev/null || fail "$tool is not installed" +done + +# 1. Slot 0 must be idle. Anything but a clear "not processing" stops the check. +slots=$(curl -sf "http://$UPSTREAM/slots?model=$MODEL") || fail "cannot read /slots from $UPSTREAM" +# Not `jq -e`: it exits 1 when the value is `false`, which is the answer we want. +busy=$(printf '%s' "$slots" | jq '.[] | select(.id == 0) | .is_processing') \ + || fail "the /slots answer is not a list of slots" +[ -n "$busy" ] || fail "slot 0 is not in the /slots answer" +[ "$busy" = "false" ] || fail "slot 0 is busy ($busy); try again later" + +# 2. Build. +cargo build --workspace --locked --manifest-path "$ROOT/Cargo.toml" || fail "cargo build" + +HOME_DIR=$(mktemp -d) || fail "mktemp" +PIDS="" +cleanup() { + for pid in $PIDS; do kill "$pid" 2> /dev/null; done +} +trap cleanup EXIT + +wait_for() { # path, seconds + n=0 + while [ ! -S "$1" ]; do + n=$((n + 1)) + [ "$n" -gt $(($2 * 10)) ] && fail "$1 did not appear within $2 s" + sleep 0.1 + done +} + +# 3. The home: a file to read, one ask grant, the configs, the system prompt. +mkdir -p "$HOME_DIR/files" "$HOME_DIR/grants" "$HOME_DIR/run/infer" || fail "mkdir" +echo "The launch code is BANANA-42." > "$HOME_DIR/files/note.txt" +cat > "$HOME_DIR/grants/files-read.toml" < "$HOME_DIR/brokerd.toml" < "$HOME_DIR/config.toml" < "$HOME_DIR/inferproxy.err" & +PIDS="$PIDS $!" +wait_for "$HOME_DIR/run/infer/infer.sock" 5 +"$BIN/brokerd" serve --config "$HOME_DIR/brokerd.toml" 2> "$HOME_DIR/brokerd.err" & +PIDS="$PIDS $!" +wait_for "$HOME_DIR/run/owner-broker/admin.sock" 10 +"$BIN/loopd" serve --config "$HOME_DIR/config.toml" 2> "$HOME_DIR/loopd.err" & +PIDS="$PIDS $!" +wait_for "$HOME_DIR/run/loop/loop.sock" 60 + +ADMIN="$HOME_DIR/run/owner-broker/admin.sock" + +# 5. The turn, in the background: it waits for the approval. +"$BIN/bxctl" chat --socket "$HOME_DIR/run/loop/loop.sock" --admin-socket "$ADMIN" \ + --session m3a-device --no-thinking \ + --say "Read the file $HOME_DIR/files/note.txt with the read_file tool and tell me exactly what happened." \ + > "$HOME_DIR/chat.out" 2> "$HOME_DIR/chat.err" & +CHAT=$! + +# 6. Wait for the approval, check what it shows, approve it. +n=0 +while :; do + list=$("$BIN/bxctl" approvals --admin-socket "$ADMIN") || fail "bxctl approvals" + [ "$list" != "no pending approvals" ] && break + kill -0 "$CHAT" 2> /dev/null || fail "the turn ended without asking; see $HOME_DIR/chat.out" + n=$((n + 1)) + [ "$n" -gt 300 ] && fail "no approval within 300 s" + sleep 1 +done +echo "$list" +printf '%s\n' "$list" | grep -q "grant files-read" || fail "the block does not name the grant" +printf '%s\n' "$list" | grep -q "read_file {\"path\":\"$HOME_DIR/files/note.txt\"}" \ + || fail "the block does not show the call" +id=$(printf '%s\n' "$list" | head -n 1 | cut -d ' ' -f 1) +"$BIN/bxctl" approve "$id" --admin-socket "$ADMIN" | tee "$HOME_DIR/approve.out" +grep -qx "approved $id: runs" "$HOME_DIR/approve.out" || fail "approve did not say it runs" + +wait "$CHAT" || fail "bxctl chat failed; see $HOME_DIR/chat.err" +echo "--- the model's answer:" +cat "$HOME_DIR/chat.out" +echo "---" + +# 7. The audit log verifies and holds the three records. +"$BIN/bxctl" audit verify --home "$HOME_DIR" || fail "the audit log does not verify" +for type in decision approval result; do + cat "$HOME_DIR"/audit/*.jsonl | grep -q "\"type\":\"$type\"" || fail "no $type record" +done +grep -q "M3b" "$HOME_DIR/chat.out" \ + || echo "check-m3a-device: note: the model's answer does not quote the runner's sentence; read it above" + +cleanup +trap - EXIT +rm -rf "$HOME_DIR" +echo "check-m3a-device: ok"