Add the end-to-end test: loopd against the real brokerd binary
Implemented-By: Grok 4.6
This commit is contained in:
@@ -6,6 +6,9 @@ gate:
|
||||
cargo fmt --all --check
|
||||
cargo clippy --workspace --all-targets --locked --offline -- -D warnings
|
||||
cargo test --workspace --locked --offline
|
||||
cargo build --workspace --locked --offline
|
||||
BOXMAKER_BROKERD=$(CURDIR)/target/debug/brokerd \
|
||||
cargo test -p loopd --test end_to_end --locked --offline -- --ignored
|
||||
cargo deny --offline check bans licenses sources
|
||||
sh scripts/check-lines.sh
|
||||
sh scripts/check-crate-deps.sh
|
||||
|
||||
@@ -0,0 +1,168 @@
|
||||
//! End to end in two processes: `loopd`'s turn loop, with a `BrokerPort`, against the real
|
||||
//! `brokerd` binary on a temporary home, and the fake llama server. Do not edit.
|
||||
//!
|
||||
//! It cannot be one process: `loopd` may not depend on `brokerd`, even for tests. The path of
|
||||
//! the `brokerd` binary comes from `BOXMAKER_BROKERD`; without it the test is ignored, and
|
||||
//! `make gate` builds the workspace and runs it with the variable set.
|
||||
|
||||
mod support;
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use loopd::baseline::Baseline;
|
||||
use loopd::broker_port::BrokerPort;
|
||||
use loopd::llama::Client;
|
||||
use loopd::session::Session;
|
||||
use loopd::tools::Registry;
|
||||
use loopd::turn::{Runtime, run_turn};
|
||||
use proto::{
|
||||
AuditEvent, AuditRecord, ChainVerifier, DecisionRecord, DenyReason, SessionId, TurnEvent,
|
||||
};
|
||||
use support::{FakeServer, Home, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
/// `brokerd serve`, killed when dropped.
|
||||
struct Brokerd(Child);
|
||||
|
||||
impl Drop for Brokerd {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn start_brokerd(home: &Path) -> (Brokerd, PathBuf) {
|
||||
let binary = std::env::var_os("BOXMAKER_BROKERD")
|
||||
.expect("set BOXMAKER_BROKERD to the brokerd binary; `make gate` does");
|
||||
std::fs::create_dir_all(home.join("grants")).unwrap();
|
||||
let config = home.join("brokerd.toml");
|
||||
let text = format!(
|
||||
"[paths]\nhome = \"{h}\"\ngrants = \"{h}/grants\"\n",
|
||||
h = home.display()
|
||||
);
|
||||
std::fs::write(&config, text).unwrap();
|
||||
let child = Command::new(binary)
|
||||
.args(["serve", "--config"])
|
||||
.arg(&config)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let brokerd = Brokerd(child);
|
||||
let socket = home.join("run/loop-broker/broker.sock");
|
||||
let until = Instant::now() + Duration::from_secs(10);
|
||||
while UnixStream::connect(&socket).is_err() {
|
||||
assert!(
|
||||
Instant::now() < until,
|
||||
"brokerd never listened on {}",
|
||||
socket.display()
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
(brokerd, socket)
|
||||
}
|
||||
|
||||
/// Every record under `audit/`, after checking that the chain verifies.
|
||||
fn audit(dir: &Path) -> Vec<AuditRecord> {
|
||||
let mut names: Vec<String> = std::fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||||
.filter(|n| n.ends_with(".jsonl"))
|
||||
.collect();
|
||||
names.sort();
|
||||
let mut verifier = ChainVerifier::new();
|
||||
let mut records = Vec::new();
|
||||
for name in &names {
|
||||
let bytes = std::fs::read(dir.join(name)).unwrap();
|
||||
verifier.feed(name, &bytes);
|
||||
for line in String::from_utf8(bytes).unwrap().lines() {
|
||||
records.push(serde_json::from_str(line).unwrap());
|
||||
}
|
||||
}
|
||||
let report = verifier.finish();
|
||||
assert!(report.failure.is_none(), "{:?}", report.failure);
|
||||
assert!(report.torn_tail.is_none());
|
||||
assert_eq!(report.records, records.len() as u64);
|
||||
records
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the brokerd binary in BOXMAKER_BROKERD; `make gate` runs it"]
|
||||
fn a_call_no_grant_allows_reaches_the_model_as_a_denial_and_is_audited() {
|
||||
let home = Home::new();
|
||||
let broker_home = home.dir.join("broker-home");
|
||||
let (_brokerd, socket) = start_brokerd(&broker_home);
|
||||
|
||||
let server = FakeServer::start();
|
||||
// The recorded model calls `read_file` on /etc/hostname, then answers in plain text.
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let cfg = home.config(&server.socket);
|
||||
let client = Client::new(cfg.clone());
|
||||
let port = BrokerPort::new(socket, Duration::from_secs(10));
|
||||
let registry = Registry::m2b();
|
||||
let baseline = Baseline::assemble(&cfg, ®istry).unwrap();
|
||||
let id = SessionId::new("e2e").unwrap();
|
||||
let mut session = Session::create(&home.dir, id, baseline, cfg.slots.main).unwrap();
|
||||
let runtime = Runtime {
|
||||
cfg: &cfg,
|
||||
client: &client,
|
||||
port: &port,
|
||||
registry: ®istry,
|
||||
};
|
||||
let mut events = Vec::new();
|
||||
let outcome = run_turn(
|
||||
&mut session,
|
||||
&runtime,
|
||||
"what is this host called?",
|
||||
&mut |e| events.push(e.clone()),
|
||||
);
|
||||
assert!(
|
||||
outcome.is_ok(),
|
||||
"the turn goes on after a denial: {outcome:?}"
|
||||
);
|
||||
assert!(
|
||||
events.contains(&TurnEvent::ToolDenied {
|
||||
name: "read_file".to_string(),
|
||||
reason: DenyReason::NoGrant,
|
||||
}),
|
||||
"{events:?}"
|
||||
);
|
||||
// The model reads the denial in its next request.
|
||||
let second = server.requests_to(CHAT)[1].json();
|
||||
assert_eq!(
|
||||
second["messages"][3]["content"],
|
||||
"Denied: no grant allows this call."
|
||||
);
|
||||
|
||||
let records = audit(&broker_home.join("audit"));
|
||||
assert_eq!(records.len(), 1, "{records:?}");
|
||||
match &records[0].event {
|
||||
AuditEvent::Decision {
|
||||
session,
|
||||
tool,
|
||||
arguments,
|
||||
outcome,
|
||||
grant,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(session.as_str(), "e2e");
|
||||
assert_eq!(tool, "read_file");
|
||||
assert_eq!(arguments, r#"{"path":"/etc/hostname"}"#);
|
||||
assert_eq!(
|
||||
*outcome,
|
||||
DecisionRecord::Denied {
|
||||
reason: DenyReason::NoGrant
|
||||
}
|
||||
);
|
||||
assert_eq!(*grant, None);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
@@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone.
|
||||
|
||||
| Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| M3a/22-end-to-end | 2026-09-22 | done | 1 | pass | none | Copied `end_to_end.rs`, `Makefile` (workspace build then ignored e2e with `BOXMAKER_BROKERD`) and `tools/check-m3a-device.sh`. Without the variable: 0 passed, 1 ignored. With it: 1 passed five runs. `--ignored` without the variable fails with "set BOXMAKER_BROKERD". `make gate` ran the e2e line (`1 passed`) and prints `gate: ok`. Did not run the straylight script. | ? |
|
||||
| M3a/21-runbook-check | 2026-09-22 | done | 1 | pass | none | Wrote `scripts/check-runbook.sh`: find `*.rs` under crates (prune `target/`), awk out every `docs/runbook.md#` pointer, empty anchors fail, each remaining anchor must match a whole `## <anchor>` line, every missing one is reported with its files, then one exit. Step 5: dropping `-x` from grep failed with "the entry is the whole line, at level two"; `exit 1` at a missing-anchor report failed with "both missing entries and their files are reported". Real tree exits 0. `make gate` prints `gate: ok`. | ? |
|
||||
| M3a/20-bxctl-chat-approvals | 2026-09-22 | done | 1 | pass | `AdminError` has no `Io` variant (task 18 maps write failures to `Protocol` via `From<io::Error>`), so `handle_pending` exit 8 reports every `cmd_approve`/`cmd_refuse` `Err` as `approval {id}: {e}` rather than returning `Err(AdminError::Io(e))`. Writes inside `handle_pending` itself still use `?`. | Moved `stream_turn` into `chat.rs` with `OnPending`/`Approvals`/`TurnIo`/`handle_pending`. The owner is shown `brokerd`'s list item, never the event's tool/args. Only the id typed in full (after stripping one trailing `\n` then one `\r`) approves; anything else refuses. `run` holds one `BufReader` on stdin for both modes. `Printer::event` escapes model text and tool names, prints the three fail-closed runbook lines as whole literals, and prints nothing for `ApprovalPending`. 21/12/20/9/12/8 tests five runs; `make gate` prints `gate: ok`. | ? |
|
||||
| M3a/19-bxctl-audit-verify | 2026-09-21 | stopped | 0 | n/a | none | The task's "The command" section says to grep `crates/bxctl/src/main.rs` for `cannot read the audit log` and stop if it is absent. `grep` returned no match: `main.rs`'s `Command::AuditVerify { home: _ }` arm (line 39) is still the placeholder that prints `bxctl: audit verify is not implemented yet`, ignores `home`, never calls `bxctl::verify::run`, and never prints the error message. The task forbids editing `main.rs` or `lib.rs`, so `bxctl audit verify` cannot be wired to `run` without that edit, and step 5 (which relies on the wiring) cannot pass. Stopped per the task's explicit instruction rather than editing a forbidden file. | ? |
|
||||
|
||||
Executable
+151
@@ -0,0 +1,151 @@
|
||||
#!/bin/sh
|
||||
# The M3a check on straylight, run by the owner (not part of `make gate` or `verify-device`).
|
||||
#
|
||||
# A private home with one `ask` grant for `read_file` on a directory; `brokerd serve` and
|
||||
# `loopd serve` on it, `loopd` talking to the real server through a private `inferproxy`. A
|
||||
# `bxctl chat --say` asks Ornith to read a file in that directory; the approval appears in
|
||||
# `bxctl approvals`; approving it gives the M3a runner's failure, which the model reports. The
|
||||
# audit log must verify and hold a Decision, an Approval and a Result.
|
||||
#
|
||||
# It uses slot 0 only, and first checks that slot 0 is idle: the server is shared.
|
||||
#
|
||||
# sh tools/check-m3a-device.sh [host:port] (default straylight:11434)
|
||||
#
|
||||
# Needs curl and jq. On success the home is removed; on failure its path is printed. If the
|
||||
# model is not loaded, `/slots` may fail: load it first (the check does not load models).
|
||||
set -u
|
||||
|
||||
UPSTREAM="${1:-straylight:11434}"
|
||||
MODEL="${BOXMAKER_MODEL:-ornith-1.5-35b-a3b}"
|
||||
ROOT=$(cd "$(dirname "$0")/.." && pwd) || exit 1
|
||||
BIN="$ROOT/target/debug"
|
||||
|
||||
fail() {
|
||||
echo "check-m3a-device: FAILED: $*" >&2
|
||||
[ -n "${HOME_DIR:-}" ] && echo "check-m3a-device: the home is kept at $HOME_DIR" >&2
|
||||
exit 1
|
||||
}
|
||||
|
||||
for tool in curl jq cargo; do
|
||||
command -v "$tool" > /dev/null || fail "$tool is not installed"
|
||||
done
|
||||
|
||||
# 1. Slot 0 must be idle. Anything but a clear "not processing" stops the check.
|
||||
slots=$(curl -sf "http://$UPSTREAM/slots?model=$MODEL") || fail "cannot read /slots from $UPSTREAM"
|
||||
# Not `jq -e`: it exits 1 when the value is `false`, which is the answer we want.
|
||||
busy=$(printf '%s' "$slots" | jq '.[] | select(.id == 0) | .is_processing') \
|
||||
|| fail "the /slots answer is not a list of slots"
|
||||
[ -n "$busy" ] || fail "slot 0 is not in the /slots answer"
|
||||
[ "$busy" = "false" ] || fail "slot 0 is busy ($busy); try again later"
|
||||
|
||||
# 2. Build.
|
||||
cargo build --workspace --locked --manifest-path "$ROOT/Cargo.toml" || fail "cargo build"
|
||||
|
||||
HOME_DIR=$(mktemp -d) || fail "mktemp"
|
||||
PIDS=""
|
||||
cleanup() {
|
||||
for pid in $PIDS; do kill "$pid" 2> /dev/null; done
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
wait_for() { # path, seconds
|
||||
n=0
|
||||
while [ ! -S "$1" ]; do
|
||||
n=$((n + 1))
|
||||
[ "$n" -gt $(($2 * 10)) ] && fail "$1 did not appear within $2 s"
|
||||
sleep 0.1
|
||||
done
|
||||
}
|
||||
|
||||
# 3. The home: a file to read, one ask grant, the configs, the system prompt.
|
||||
mkdir -p "$HOME_DIR/files" "$HOME_DIR/grants" "$HOME_DIR/run/infer" || fail "mkdir"
|
||||
echo "The launch code is BANANA-42." > "$HOME_DIR/files/note.txt"
|
||||
cat > "$HOME_DIR/grants/files-read.toml" <<EOF || fail "grant"
|
||||
tool = "read_file"
|
||||
mode = "ask"
|
||||
max_taint = "secret"
|
||||
result_class = "private"
|
||||
untrusted = false
|
||||
|
||||
[constraints]
|
||||
paths = ["$HOME_DIR/files"]
|
||||
EOF
|
||||
cat > "$HOME_DIR/brokerd.toml" <<EOF || fail "brokerd.toml"
|
||||
[paths]
|
||||
home = "$HOME_DIR"
|
||||
grants = "$HOME_DIR/grants"
|
||||
EOF
|
||||
cat > "$HOME_DIR/config.toml" <<EOF || fail "config.toml"
|
||||
[infer]
|
||||
socket = "$HOME_DIR/run/infer/infer.sock"
|
||||
model = "$MODEL"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
[paths]
|
||||
home = "$HOME_DIR"
|
||||
[broker]
|
||||
socket = "$HOME_DIR/run/loop-broker/broker.sock"
|
||||
EOF
|
||||
cp "$ROOT/config/system.md" "$HOME_DIR/system.md" || fail "config/system.md"
|
||||
|
||||
# 4. The three daemons.
|
||||
"$BIN/inferproxy" --listen "$HOME_DIR/run/infer/infer.sock" --upstream "$UPSTREAM" \
|
||||
2> "$HOME_DIR/inferproxy.err" &
|
||||
PIDS="$PIDS $!"
|
||||
wait_for "$HOME_DIR/run/infer/infer.sock" 5
|
||||
"$BIN/brokerd" serve --config "$HOME_DIR/brokerd.toml" 2> "$HOME_DIR/brokerd.err" &
|
||||
PIDS="$PIDS $!"
|
||||
wait_for "$HOME_DIR/run/owner-broker/admin.sock" 10
|
||||
"$BIN/loopd" serve --config "$HOME_DIR/config.toml" 2> "$HOME_DIR/loopd.err" &
|
||||
PIDS="$PIDS $!"
|
||||
wait_for "$HOME_DIR/run/loop/loop.sock" 60
|
||||
|
||||
ADMIN="$HOME_DIR/run/owner-broker/admin.sock"
|
||||
|
||||
# 5. The turn, in the background: it waits for the approval.
|
||||
"$BIN/bxctl" chat --socket "$HOME_DIR/run/loop/loop.sock" --admin-socket "$ADMIN" \
|
||||
--session m3a-device --no-thinking \
|
||||
--say "Read the file $HOME_DIR/files/note.txt with the read_file tool and tell me exactly what happened." \
|
||||
> "$HOME_DIR/chat.out" 2> "$HOME_DIR/chat.err" &
|
||||
CHAT=$!
|
||||
|
||||
# 6. Wait for the approval, check what it shows, approve it.
|
||||
n=0
|
||||
while :; do
|
||||
list=$("$BIN/bxctl" approvals --admin-socket "$ADMIN") || fail "bxctl approvals"
|
||||
[ "$list" != "no pending approvals" ] && break
|
||||
kill -0 "$CHAT" 2> /dev/null || fail "the turn ended without asking; see $HOME_DIR/chat.out"
|
||||
n=$((n + 1))
|
||||
[ "$n" -gt 300 ] && fail "no approval within 300 s"
|
||||
sleep 1
|
||||
done
|
||||
echo "$list"
|
||||
printf '%s\n' "$list" | grep -q "grant files-read" || fail "the block does not name the grant"
|
||||
printf '%s\n' "$list" | grep -q "read_file {\"path\":\"$HOME_DIR/files/note.txt\"}" \
|
||||
|| fail "the block does not show the call"
|
||||
id=$(printf '%s\n' "$list" | head -n 1 | cut -d ' ' -f 1)
|
||||
"$BIN/bxctl" approve "$id" --admin-socket "$ADMIN" | tee "$HOME_DIR/approve.out"
|
||||
grep -qx "approved $id: runs" "$HOME_DIR/approve.out" || fail "approve did not say it runs"
|
||||
|
||||
wait "$CHAT" || fail "bxctl chat failed; see $HOME_DIR/chat.err"
|
||||
echo "--- the model's answer:"
|
||||
cat "$HOME_DIR/chat.out"
|
||||
echo "---"
|
||||
|
||||
# 7. The audit log verifies and holds the three records.
|
||||
"$BIN/bxctl" audit verify --home "$HOME_DIR" || fail "the audit log does not verify"
|
||||
for type in decision approval result; do
|
||||
cat "$HOME_DIR"/audit/*.jsonl | grep -q "\"type\":\"$type\"" || fail "no $type record"
|
||||
done
|
||||
grep -q "M3b" "$HOME_DIR/chat.out" \
|
||||
|| echo "check-m3a-device: note: the model's answer does not quote the runner's sentence; read it above"
|
||||
|
||||
cleanup
|
||||
trap - EXIT
|
||||
rm -rf "$HOME_DIR"
|
||||
echo "check-m3a-device: ok"
|
||||
Reference in New Issue
Block a user