Hand over the M3a plan: 22 tasks, their files, and the check record
Task files, the files they copy in (byte-identical to the reference on m3a-ref), each area's check record, and a README with the per-task table of what each check exposed. The handoff note is done with. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
@@ -0,0 +1,314 @@
|
||||
//! Tests for `BrokerPort` against a fake `brokerd` on a temporary socket: answers, approvals and
|
||||
//! timeouts. Frames that break the protocol are in `broker_port_bad.rs`. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send, socket_path};
|
||||
use loopd::broker_port::{BrokerPort, NoBroker, POINTER, TOO_LARGE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
DenyReason, Message, PROTOCOL_VERSION, Timestamp, ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn a_result_comes_back_as_it_was_sent() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("hello\n"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("hello\n"));
|
||||
assert!(got.pending.is_empty());
|
||||
assert!(got.lines.is_empty(), "nothing to report: {:?}", got.lines);
|
||||
|
||||
// What the broker received: one final frame holding exactly the request.
|
||||
let sent = broker.join().unwrap();
|
||||
assert_eq!(sent.v, PROTOCOL_VERSION);
|
||||
assert!(sent.r#final, "a request is a single final frame");
|
||||
assert_eq!(sent.msg, Message::ToolRequest(request()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_denial_and_a_failure_come_back_as_they_were_sent() {
|
||||
let mut answers: Vec<ToolResponse> = [
|
||||
DenyReason::NoGrant,
|
||||
DenyReason::GrantExpired,
|
||||
DenyReason::TaintTooHigh,
|
||||
DenyReason::DeniedByGrant,
|
||||
DenyReason::ApprovalRefused,
|
||||
DenyReason::ApprovalExpired,
|
||||
DenyReason::GrantsInvalid,
|
||||
DenyReason::AuditUnavailable,
|
||||
DenyReason::InvalidArguments,
|
||||
DenyReason::StateUnreadable,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|reason| ToolResponse::Denied { reason })
|
||||
.collect();
|
||||
answers.push(ToolResponse::Failed {
|
||||
message: "the runner arrives in M3b".to_string(),
|
||||
});
|
||||
for answer in answers {
|
||||
let reply = answer.clone();
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
send(stream, request.id, true, reply);
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, answer);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"a denial is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_frame_reaches_the_callback_and_the_final_frame_is_the_answer() {
|
||||
let expires = in_ms(60_000);
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 41,
|
||||
expires,
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(150));
|
||||
send(stream, request.id, true, result("approved"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("approved"));
|
||||
assert_eq!(
|
||||
got.pending,
|
||||
[Pending {
|
||||
approval: 41,
|
||||
expires
|
||||
}],
|
||||
"called once, with the frame's values"
|
||||
);
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_after_expires_but_within_the_timeout_of_it_is_delivered() {
|
||||
// expires in 100 ms, timeout 1,500 ms: the port waits until 1,600 ms. The answer comes at
|
||||
// about 400 ms, well after `expires`: an approval given at the last moment still gets the
|
||||
// whole timeout to run.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: in_ms(100),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(400));
|
||||
send(stream, request.id, true, result("late but good"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("late but good"));
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_expiry_that_has_already_passed_still_leaves_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: Timestamp::parse("2020-01-01T00:00:00.000Z").unwrap(),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
send(stream, request.id, true, result("fine"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("fine"));
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_socket_is_unavailable() {
|
||||
let socket = socket_path();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let got = call(socket.clone(), 2_000, &request());
|
||||
assert_unavailable(&got, "no socket file");
|
||||
assert!(
|
||||
got.lines[0].contains(&socket.display().to_string()),
|
||||
"the line names the socket: {}",
|
||||
got.lines[0]
|
||||
);
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_without_answering_is_unavailable() {
|
||||
let (socket, broker) = broker(|_, _| {});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed before any frame");
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_500),
|
||||
"a close is seen at once: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_while_pending_is_unavailable() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
// brokerd was restarted: the connection just ends.
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed while pending");
|
||||
assert_eq!(got.pending.len(), 1, "the pending frame was reported first");
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_never_answers_is_unavailable_after_the_timeout() {
|
||||
let (socket, broker) = broker(|_, _| thread::sleep(Duration::from_millis(1_200)));
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(250),
|
||||
"gave up early: {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_100),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_call_nobody_answers_ends_at_expires_plus_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(300),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(1_800));
|
||||
});
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence while pending");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(550),
|
||||
"it must wait for expires (300) plus the timeout (300): {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_700),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_timeout_is_a_deadline_for_the_frame_not_for_each_read() {
|
||||
// The four header bytes arrive 250 ms apart, the last at 1,000 ms. A port that only sets a
|
||||
// 600 ms read timeout sees every single read succeed and returns the result; a port with a
|
||||
// deadline gives up at 600 ms. The answer tells them apart, so no timing is asserted.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&frame(request.id, true, Message::ToolResponse(result("slow"))),
|
||||
)
|
||||
.unwrap();
|
||||
for byte in bytes.iter().take(4) {
|
||||
thread::sleep(Duration::from_millis(250));
|
||||
if stream.write_all(&[*byte]).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = stream.write_all(&bytes[4..]);
|
||||
});
|
||||
let got = call(socket, 600, &request());
|
||||
assert_unavailable(&got, "a trickled frame");
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_timeout_fails_closed_and_does_not_panic() {
|
||||
let (socket, _broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("too late"));
|
||||
});
|
||||
let got = call(socket, 0, &request());
|
||||
assert_unavailable(&got, "timeout_ms = 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_too_large_for_a_frame_is_its_own_failure() {
|
||||
// Nothing is sent, so the fake broker sees a connection that closes or none at all.
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _listener = UnixListener::bind(&path).unwrap();
|
||||
let mut big = request();
|
||||
big.arguments = format!(r#"{{"path":"/{}"}}"#, "a".repeat(proto::MAX_FRAME));
|
||||
let got = call(path, 1_000, &big);
|
||||
assert_eq!(
|
||||
got.response,
|
||||
ToolResponse::Failed {
|
||||
message: TOO_LARGE.to_string()
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"the broker is fine; this is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_call_is_its_own_connection() {
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
for n in 0..3u64 {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
send(
|
||||
&mut stream,
|
||||
request.id,
|
||||
true,
|
||||
result(&format!("answer {n}")),
|
||||
);
|
||||
}
|
||||
});
|
||||
let port = BrokerPort::new(path, Duration::from_millis(2_000));
|
||||
for n in 0..3 {
|
||||
let got = port.call(&request(), &mut |_| {});
|
||||
assert_eq!(got, result(&format!("answer {n}")));
|
||||
}
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_broker_every_call_fails_in_words_the_model_can_repeat() {
|
||||
let mut seen = 0;
|
||||
let got = NoBroker.call(&request(), &mut |_| seen += 1);
|
||||
assert_eq!(
|
||||
got,
|
||||
ToolResponse::Failed {
|
||||
message: "no tool broker is configured".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(seen, 0);
|
||||
assert_eq!(POINTER, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//! Tests for `BrokerPort` when the broker's frames break the protocol. Every one ends in the same
|
||||
//! plain failure and one printed line. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send};
|
||||
use proto::{Envelope, ErrorCode, Message, PROTOCOL_VERSION, ToolResponse, WireError, write_frame};
|
||||
|
||||
#[test]
|
||||
fn frames_that_break_the_protocol_are_unavailable() {
|
||||
type Script = Box<dyn FnOnce(&mut UnixStream, &Envelope) + Send>;
|
||||
let pending = |approval| ToolResponse::PendingApproval {
|
||||
approval,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
let cases: Vec<(&str, Script)> = vec![
|
||||
(
|
||||
"an answer for another request id",
|
||||
Box::new(|s, r| send(s, r.id + 1, true, result("x"))),
|
||||
),
|
||||
(
|
||||
"a final frame that is pending",
|
||||
Box::new(move |s, r| send(s, r.id, true, pending(1))),
|
||||
),
|
||||
(
|
||||
"an answer that is not final",
|
||||
Box::new(|s, r| send(s, r.id, false, result("x"))),
|
||||
),
|
||||
(
|
||||
"a second pending frame",
|
||||
Box::new(move |s, r| {
|
||||
send(s, r.id, false, pending(1));
|
||||
send(s, r.id, false, pending(2));
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"pending, then another id",
|
||||
Box::new(move |s, r| {
|
||||
send(s, r.id, false, pending(1));
|
||||
send(s, r.id + 1, true, result("x"));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"an error message",
|
||||
Box::new(|s, r| {
|
||||
let error = Message::Error(WireError {
|
||||
code: ErrorCode::Forbidden,
|
||||
detail: "tool requests only".to_string(),
|
||||
});
|
||||
let _ = write_frame(s, &frame(r.id, true, error));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a message of another kind",
|
||||
Box::new(|s, r| {
|
||||
let echo = Message::ToolRequest(request());
|
||||
let _ = write_frame(s, &frame(r.id, true, echo));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"another protocol version",
|
||||
Box::new(|s, r| {
|
||||
let mut env = frame(r.id, true, Message::ToolResponse(result("x")));
|
||||
env.v = PROTOCOL_VERSION + 1;
|
||||
let _ = write_frame(s, &env);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a zero length",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 0]);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a length over the maximum",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0xff, 0xff, 0xff, 0xff]);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a body that is not JSON",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 5]);
|
||||
let _ = s.write_all(b"hello");
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a body cut short",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 50]);
|
||||
let _ = s.write_all(b"{\"v\":1");
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (why, script) in cases {
|
||||
let (socket, broker) = broker(script);
|
||||
let got = call(socket, 1_000, &request());
|
||||
assert_unavailable(&got, why);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_error_message_the_broker_sent_is_in_the_line() {
|
||||
let (socket, broker) = broker(|s, r| {
|
||||
let error = Message::Error(WireError {
|
||||
code: ErrorCode::Internal,
|
||||
detail: "the ledger is gone".to_string(),
|
||||
});
|
||||
let _ = write_frame(s, &frame(r.id, true, error));
|
||||
});
|
||||
let got = call(socket, 1_000, &request());
|
||||
assert_unavailable(&got, "an error message");
|
||||
assert!(
|
||||
got.lines[0].contains("the ledger is gone"),
|
||||
"{}",
|
||||
got.lines[0]
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
@@ -0,0 +1,272 @@
|
||||
//! Tests for `loopd`'s configuration. Do not edit: these define the required behaviour.
|
||||
|
||||
use loopd::config::{Config, ConfigError, Limits, Sampling};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn fixture(name: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/config")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimal_file_gets_the_documented_defaults() {
|
||||
let c = Config::load(&fixture("minimal.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.infer.socket,
|
||||
PathBuf::from("/run/boxmaker/infer/infer.sock")
|
||||
);
|
||||
assert_eq!(c.infer.model, "ornith-1.5-35b-a3b");
|
||||
assert_eq!((c.slots.main, c.slots.background), (0, 1));
|
||||
assert_eq!(c.expect.n_ctx, 131_072);
|
||||
assert_eq!(c.expect.slots, 2);
|
||||
assert_eq!(
|
||||
c.expect.template_sha256.to_hex(),
|
||||
"f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
);
|
||||
assert_eq!(
|
||||
c.sampling,
|
||||
Sampling {
|
||||
temperature: 0.6,
|
||||
top_p: 0.95,
|
||||
top_k: 20
|
||||
}
|
||||
);
|
||||
let want = Limits {
|
||||
poll_ms: 5_000,
|
||||
busy_wait_ms: 600_000,
|
||||
load_wait_ms: 180_000,
|
||||
idle_grace_ms: 30_000,
|
||||
liveness_ms: 30_000,
|
||||
thinking_cap: 4_096,
|
||||
thinking_overrun: 256,
|
||||
max_tokens: 8_192,
|
||||
queue_len: 8,
|
||||
retry_attempts: 4,
|
||||
retry_backoff_ms: vec![2_000, 8_000, 30_000],
|
||||
retry_window_ms: 300_000,
|
||||
};
|
||||
assert_eq!(c.limits, want);
|
||||
assert_eq!(Limits::default(), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_file_overrides_every_default() {
|
||||
let c = Config::load(&fixture("full.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.sampling,
|
||||
Sampling {
|
||||
temperature: 0.2,
|
||||
top_p: 0.9,
|
||||
top_k: 40
|
||||
}
|
||||
);
|
||||
let want = Limits {
|
||||
poll_ms: 50,
|
||||
busy_wait_ms: 200,
|
||||
load_wait_ms: 300,
|
||||
idle_grace_ms: 150,
|
||||
liveness_ms: 100,
|
||||
thinking_cap: 20,
|
||||
thinking_overrun: 10,
|
||||
max_tokens: 512,
|
||||
queue_len: 1,
|
||||
retry_attempts: 2,
|
||||
retry_backoff_ms: vec![10],
|
||||
retry_window_ms: 1_000,
|
||||
};
|
||||
assert_eq!(c.limits, want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_partial_limits_table_keeps_the_other_defaults() {
|
||||
let text = std::fs::read_to_string(fixture("minimal.toml")).unwrap();
|
||||
let c = Config::parse(&format!("{text}\n[limits]\nliveness_ms = 1234\n")).unwrap();
|
||||
assert_eq!(c.limits.liveness_ms, 1234);
|
||||
assert_eq!(c.limits.poll_ms, 5_000);
|
||||
}
|
||||
|
||||
/// Every table in the file must reject a key it does not know: a misspelt limit that silently
|
||||
/// fell back to its default would be a limit the owner believes is set and is not.
|
||||
#[test]
|
||||
fn unknown_keys_are_errors_in_every_table() {
|
||||
let text = std::fs::read_to_string(fixture("full.toml")).unwrap();
|
||||
assert!(Config::parse(&text).is_ok());
|
||||
for table in ["infer", "slots", "expect", "sampling", "limits"] {
|
||||
let header = format!("[{table}]\n");
|
||||
assert!(text.contains(&header), "fixture has no [{table}] table");
|
||||
let bad = text.replacen(&header, &format!("{header}zz_unknown = 1\n"), 1);
|
||||
assert!(
|
||||
Config::parse(&bad).is_err(),
|
||||
"[{table}] accepted an unknown key"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
Config::parse(&format!("zz_unknown = 1\n{text}")).is_err(),
|
||||
"top level"
|
||||
);
|
||||
assert!(
|
||||
Config::parse(&format!("{text}\n[zz_unknown]\nx = 1\n")).is_err(),
|
||||
"unknown table"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_tables_and_values_are_checked() {
|
||||
let text = std::fs::read_to_string(fixture("minimal.toml")).unwrap();
|
||||
for table in ["infer", "slots", "expect"] {
|
||||
let without: String = text
|
||||
.split("\n\n")
|
||||
.filter(|block| !block.trim_start().starts_with(&format!("[{table}]")))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
assert!(Config::parse(&without).is_err(), "[{table}] is required");
|
||||
}
|
||||
let bad_hash = text.replace("f55f5293", "F55F5293");
|
||||
assert!(
|
||||
Config::parse(&bad_hash).is_err(),
|
||||
"the hash must be lowercase hex"
|
||||
);
|
||||
let negative = text.replace("main = 0", "main = -1");
|
||||
assert!(Config::parse(&negative).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_reports_which_file_failed() {
|
||||
let missing = fixture("does-not-exist.toml");
|
||||
match Config::load(&missing) {
|
||||
Err(ConfigError::Read(path, _)) => assert_eq!(path, missing),
|
||||
other => panic!("expected a read error, got {other:?}"),
|
||||
}
|
||||
let e: Box<dyn std::error::Error> = Box::new(Config::load(&missing).unwrap_err());
|
||||
assert!(e.to_string().contains("does-not-exist.toml"));
|
||||
}
|
||||
|
||||
// ---- M2b: paths, channel, loop and baseline ----
|
||||
|
||||
#[test]
|
||||
fn the_m2b_tables_have_defaults() {
|
||||
let c = Config::load(&fixture("minimal.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.channel.socket,
|
||||
PathBuf::new(),
|
||||
"empty means: under the home"
|
||||
);
|
||||
assert_eq!(c.channel_socket(), c.paths.home.join("run/loop/loop.sock"));
|
||||
assert_eq!(
|
||||
(
|
||||
c.r#loop.tool_iterations,
|
||||
c.r#loop.repeat_detection,
|
||||
c.r#loop.tool_result_cap
|
||||
),
|
||||
(8, true, 16 * 1024)
|
||||
);
|
||||
// A relative system prompt is taken from the config file's directory.
|
||||
assert_eq!(c.baseline.system, fixture("system.md"));
|
||||
// The home comes from BOXMAKER_HOME when set, else /var/lib/boxmaker. Either way it is absolute.
|
||||
assert!(c.paths.home.is_absolute(), "{:?}", c.paths.home);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_m2b_tables_can_be_set() {
|
||||
let c = Config::load(&fixture("m2b.toml")).unwrap();
|
||||
assert_eq!(c.paths.home, PathBuf::from("/srv/boxmaker"));
|
||||
assert_eq!(
|
||||
c.channel.socket,
|
||||
PathBuf::from("/run/boxmaker/loop/loop.sock")
|
||||
);
|
||||
assert_eq!(
|
||||
c.channel_socket(),
|
||||
PathBuf::from("/run/boxmaker/loop/loop.sock")
|
||||
);
|
||||
assert_eq!(
|
||||
(
|
||||
c.r#loop.tool_iterations,
|
||||
c.r#loop.repeat_detection,
|
||||
c.r#loop.tool_result_cap
|
||||
),
|
||||
(3, false, 1024)
|
||||
);
|
||||
assert_eq!(
|
||||
c.baseline.system,
|
||||
fixture("prompts/agent.md"),
|
||||
"relative to the config file"
|
||||
);
|
||||
let text = std::fs::read_to_string(fixture("m2b.toml")).unwrap();
|
||||
let absolute = text.replace("prompts/agent.md", "/etc/boxmaker/system.md");
|
||||
assert_eq!(
|
||||
Config::parse(&absolute).unwrap().baseline.system,
|
||||
PathBuf::from("/etc/boxmaker/system.md")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unknown_keys_are_errors_in_the_m2b_tables_too() {
|
||||
let text = std::fs::read_to_string(fixture("m2b.toml")).unwrap();
|
||||
assert!(Config::parse(&text).is_ok());
|
||||
for table in ["paths", "channel", "loop", "baseline"] {
|
||||
let header = format!("[{table}]\n");
|
||||
let bad = text.replacen(&header, &format!("{header}zz_unknown = 1\n"), 1);
|
||||
assert!(
|
||||
Config::parse(&bad).is_err(),
|
||||
"[{table}] accepted an unknown key"
|
||||
);
|
||||
}
|
||||
let partial = text.replace("tool_iterations = 3\nrepeat_detection = false\n", "");
|
||||
let c = Config::parse(&partial).unwrap();
|
||||
assert_eq!(
|
||||
(
|
||||
c.r#loop.tool_iterations,
|
||||
c.r#loop.repeat_detection,
|
||||
c.r#loop.tool_result_cap
|
||||
),
|
||||
(8, true, 1024),
|
||||
"a partial table keeps the other defaults"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_table_is_optional_and_its_socket_has_no_default() {
|
||||
let c = Config::load(&fixture("minimal.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.socket, None,
|
||||
"no socket means no broker, never a guessed path"
|
||||
);
|
||||
assert_eq!(c.broker.timeout_ms, 120_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_table_can_be_set_and_rejects_unknown_keys() {
|
||||
let base = std::fs::read_to_string(fixture("m2b.toml")).unwrap();
|
||||
let text = format!(
|
||||
"{base}\n[broker]\nsocket = \"/run/boxmaker/loop-broker/broker.sock\"\ntimeout_ms = 5000\n"
|
||||
);
|
||||
let c = Config::parse(&text).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.socket,
|
||||
Some(PathBuf::from("/run/boxmaker/loop-broker/broker.sock"))
|
||||
);
|
||||
assert_eq!(c.broker.timeout_ms, 5000);
|
||||
|
||||
let only_timeout = format!("{base}\n[broker]\ntimeout_ms = 5000\n");
|
||||
let c = Config::parse(&only_timeout).unwrap();
|
||||
assert_eq!((c.broker.socket, c.broker.timeout_ms), (None, 5000));
|
||||
|
||||
let only_socket = format!("{base}\n[broker]\nsocket = \"/b.sock\"\n");
|
||||
let c = Config::parse(&only_socket).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.timeout_ms, 120_000,
|
||||
"a partial table keeps the default"
|
||||
);
|
||||
|
||||
for bad in [
|
||||
"zz_unknown = 1",
|
||||
"timeout = 5000",
|
||||
"timeout_ms = -1",
|
||||
"timeout_ms = \"5s\"",
|
||||
"socket = 7",
|
||||
] {
|
||||
let text = format!("{base}\n[broker]\n{bad}\n");
|
||||
assert!(Config::parse(&text).is_err(), "[broker] accepted `{bad}`");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,448 @@
|
||||
//! Checks against the real server. Ignored unless run by `make verify-device`. Do not edit.
|
||||
//!
|
||||
//! They need two environment variables:
|
||||
//! BOXMAKER_INFERPROXY path to the built `inferproxy` binary
|
||||
//! BOXMAKER_UPSTREAM host:port of the llama-server router, for example straylight:11434
|
||||
//! and, optionally, BOXMAKER_MODEL (default ornith-1.5-35b-a3b). They use slot 0.
|
||||
//!
|
||||
//! They never touch the real `llama-server` process. The "server dies" test kills and restarts
|
||||
//! its own `inferproxy`.
|
||||
|
||||
use loopd::config::Config;
|
||||
use loopd::llama::info::{CacheOutcome, cache_outcome};
|
||||
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command};
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
struct Proxy {
|
||||
child: Child,
|
||||
socket: PathBuf,
|
||||
}
|
||||
|
||||
impl Proxy {
|
||||
fn start(socket: &Path) -> Proxy {
|
||||
let binary = std::env::var("BOXMAKER_INFERPROXY").expect("BOXMAKER_INFERPROXY is not set");
|
||||
let upstream = std::env::var("BOXMAKER_UPSTREAM").expect("BOXMAKER_UPSTREAM is not set");
|
||||
let child = Command::new(binary)
|
||||
.arg("--listen")
|
||||
.arg(socket)
|
||||
.arg("--upstream")
|
||||
.arg(upstream)
|
||||
.spawn()
|
||||
.expect("cannot start inferproxy");
|
||||
let mut proxy = Proxy {
|
||||
child,
|
||||
socket: socket.to_path_buf(),
|
||||
};
|
||||
for _ in 0..100 {
|
||||
if socket.exists() {
|
||||
return proxy;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
proxy.kill();
|
||||
panic!("inferproxy did not create {}", socket.display());
|
||||
}
|
||||
|
||||
fn kill(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Proxy {
|
||||
fn drop(&mut self) {
|
||||
self.kill();
|
||||
}
|
||||
}
|
||||
|
||||
fn socket_path(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir.join("infer.sock")
|
||||
}
|
||||
|
||||
fn config(socket: &Path) -> Config {
|
||||
let model =
|
||||
std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "{model}"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
"#,
|
||||
socket.display()
|
||||
);
|
||||
Config::parse(&text).unwrap()
|
||||
}
|
||||
|
||||
fn user(text: &str) -> ChatMessage {
|
||||
// A different prompt each run, so that an earlier run's cache cannot make a check pass.
|
||||
let nonce = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
ChatMessage::User {
|
||||
content: format!("{text} (run {nonce})"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn the_startup_self_test_passes() {
|
||||
let socket = socket_path("selftest");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let client = Client::new(config(&socket));
|
||||
loopd::selftest::run(&client, &mut |step| eprintln!("selftest: {step}")).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_capped_thinking_block_ends_and_the_answer_arrives() {
|
||||
let socket = socket_path("cap");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let mut cfg = config(&socket);
|
||||
cfg.limits.thinking_cap = 60;
|
||||
let client = Client::new(cfg);
|
||||
let ask = "Think at great length about how many ways 8 rooks fit on a chessboard without attacking \
|
||||
each other and with none on the main diagonal. Then answer in one sentence.";
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![user(ask)],
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let mut capped = Vec::new();
|
||||
let done = client
|
||||
.chat_with_retry(&req, &mut |e| {
|
||||
if let ChatEvent::ThinkingCapped { tokens } = e {
|
||||
capped.push(*tokens);
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
assert!(done.thinking_capped);
|
||||
assert_eq!(capped.len(), 1);
|
||||
assert!(capped[0] >= 60 && capped[0] < 70, "capped at {capped:?}");
|
||||
assert!(
|
||||
done.reasoning_tokens < 60 + 256,
|
||||
"thinking went on to {}",
|
||||
done.reasoning_tokens
|
||||
);
|
||||
assert!(
|
||||
done.content.is_some_and(|c| !c.trim().is_empty()),
|
||||
"an answer followed the forced end"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_request_survives_its_proxy_being_killed_and_restarted() {
|
||||
let socket = socket_path("restart");
|
||||
let mut proxy = Proxy::start(&socket);
|
||||
let mut cfg = config(&socket);
|
||||
cfg.limits.retry_backoff_ms = vec![1_500];
|
||||
let client = Client::new(cfg);
|
||||
let ask = "Write about 300 words on the history of cork.";
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![user(ask)],
|
||||
tools: vec![],
|
||||
thinking: false,
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let worker = thread::spawn(move || {
|
||||
let mut retries = 0;
|
||||
let mut told = false;
|
||||
let result = client.chat_with_retry(&req, &mut |e| match e {
|
||||
ChatEvent::Content(_) if !told => {
|
||||
told = true;
|
||||
let _ = tx.send(());
|
||||
}
|
||||
ChatEvent::Retrying { .. } => retries += 1,
|
||||
_ => {}
|
||||
});
|
||||
(result, retries)
|
||||
});
|
||||
|
||||
rx.recv_timeout(Duration::from_secs(120))
|
||||
.expect("no content arrived");
|
||||
proxy.kill(); // the stream dies in the middle of the answer
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
let _proxy = Proxy::start(&proxy.socket);
|
||||
|
||||
let (result, retries) = worker.join().unwrap();
|
||||
let done = result.expect("the retry should have succeeded");
|
||||
assert!(retries >= 1, "the request was retried");
|
||||
assert!(
|
||||
done.content
|
||||
.is_some_and(|c| c.split_whitespace().count() > 100)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_second_turn_reuses_the_first_turns_cache() {
|
||||
let socket = socket_path("cache");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let client = Client::new(config(&socket));
|
||||
let mut messages = vec![user(
|
||||
"What is 17 * 23? Think briefly, then answer in one short sentence.",
|
||||
)];
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: messages.clone(),
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let turn1 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
|
||||
assert!(
|
||||
turn1.reasoning_content.is_some(),
|
||||
"this check is about replaying a thinking block"
|
||||
);
|
||||
messages.push(ChatMessage::Assistant {
|
||||
content: turn1.content.clone(),
|
||||
reasoning_content: turn1.reasoning_content.clone(),
|
||||
tool_calls: turn1.tool_calls.clone(),
|
||||
});
|
||||
messages.push(user("And 17 * 24?"));
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages,
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let turn2 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
|
||||
assert_eq!(
|
||||
cache_outcome(&turn1.timings, &turn2.timings),
|
||||
CacheOutcome::Hit,
|
||||
"{:?} then {:?}",
|
||||
turn1.timings,
|
||||
turn2.timings
|
||||
);
|
||||
}
|
||||
|
||||
// ---- M2b: the agent loop on the real server ----
|
||||
|
||||
/// A `loopd serve` on a private home, killed on drop.
|
||||
struct Served {
|
||||
child: Child,
|
||||
home: PathBuf,
|
||||
socket: PathBuf,
|
||||
}
|
||||
|
||||
fn config_text(infer: &Path, home: &Path) -> String {
|
||||
let model =
|
||||
std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
|
||||
format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "{model}"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
[paths]
|
||||
home = "{}"
|
||||
"#,
|
||||
infer.display(),
|
||||
home.display()
|
||||
)
|
||||
}
|
||||
|
||||
impl Served {
|
||||
/// Writes the config and the repository's `system.md` into `home`, and starts `loopd serve`.
|
||||
fn start(infer: &Path, home: &Path) -> Served {
|
||||
std::fs::create_dir_all(home).unwrap();
|
||||
let config = home.join("config.toml");
|
||||
std::fs::write(&config, config_text(infer, home)).unwrap();
|
||||
let prompt = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
|
||||
std::fs::copy(&prompt, home.join("system.md"))
|
||||
.expect("config/system.md exists in the repository");
|
||||
let socket = home.join("run").join("loop").join("loop.sock");
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let child = Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.arg("serve")
|
||||
.arg("--config")
|
||||
.arg(&config)
|
||||
.spawn()
|
||||
.expect("cannot start loopd");
|
||||
let mut served = Served {
|
||||
child,
|
||||
home: home.to_path_buf(),
|
||||
socket,
|
||||
};
|
||||
for _ in 0..600 {
|
||||
if served.socket.exists() {
|
||||
return served;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}
|
||||
served.kill();
|
||||
panic!("loopd did not come up within 60 s");
|
||||
}
|
||||
|
||||
fn kill(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
|
||||
/// One `bxctl chat --say` turn. Returns the answer.
|
||||
fn say(&self, session: &str, text: &str) -> String {
|
||||
let bxctl = std::env::var("BOXMAKER_BXCTL").expect("BOXMAKER_BXCTL is not set");
|
||||
let output = Command::new(bxctl)
|
||||
.arg("chat")
|
||||
.arg("--socket")
|
||||
.arg(&self.socket)
|
||||
.args(["--session", session, "--no-thinking", "--say", text])
|
||||
.output()
|
||||
.expect("cannot run bxctl");
|
||||
assert!(
|
||||
output.status.success(),
|
||||
"bxctl failed: {}",
|
||||
String::from_utf8_lossy(&output.stderr)
|
||||
);
|
||||
String::from_utf8_lossy(&output.stdout)
|
||||
.trim_end()
|
||||
.to_string()
|
||||
}
|
||||
|
||||
fn records(&self, session: &str) -> Vec<proto::LogRecord> {
|
||||
let text =
|
||||
std::fs::read_to_string(self.home.join("sessions").join(session).join("0.jsonl"))
|
||||
.unwrap();
|
||||
text.lines()
|
||||
.map(|l| serde_json::from_str(l).unwrap())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Served {
|
||||
fn drop(&mut self) {
|
||||
self.kill();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn the_baseline_fits_the_token_budget() {
|
||||
let socket = socket_path("budget");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let home = socket.parent().unwrap().join("home");
|
||||
std::fs::create_dir_all(&home).unwrap();
|
||||
let mut cfg = config(&socket);
|
||||
cfg.paths.home = home.clone();
|
||||
cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
|
||||
let baseline =
|
||||
loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m3a()).unwrap();
|
||||
let client = Client::new(cfg);
|
||||
// The system text plus every tool schema as the request carries it.
|
||||
let mut text = baseline.system.clone();
|
||||
for tool in &baseline.tools {
|
||||
text.push('\n');
|
||||
text.push_str(&serde_json::to_string(&tool).unwrap());
|
||||
}
|
||||
let tokens = client.tokenize(&text).unwrap();
|
||||
eprintln!("baseline: {tokens} tokens");
|
||||
assert!(
|
||||
tokens <= 3000,
|
||||
"the baseline is {tokens} tokens; the brief allows 3000"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() {
|
||||
let socket = socket_path("loop");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let home = socket.parent().unwrap().join("home");
|
||||
let session = format!("device-{}", std::process::id());
|
||||
let mut served = Served::start(&socket, &home);
|
||||
|
||||
let a1 = served.say(&session, "Reply with exactly: box made.");
|
||||
assert!(a1.to_lowercase().contains("box made"), "{a1}");
|
||||
let a2 = served.say(
|
||||
&session,
|
||||
"What is the current time? Use your clock tool, then tell me the year.",
|
||||
);
|
||||
assert!(a2.contains("2026") || a2.contains("202"), "{a2}");
|
||||
// No broker is configured here, so the call fails in plain words and the turn goes on. What
|
||||
// is checked is the path: find_tool, then call_tool, then an answer.
|
||||
let a3 = served.say(
|
||||
&session,
|
||||
"Find a tool that reads files, use it to read /etc/hostname, and tell me in one \
|
||||
sentence what happened.",
|
||||
);
|
||||
assert!(!a3.trim().is_empty(), "the turn must end in an answer");
|
||||
|
||||
served.kill();
|
||||
served = Served::start(&socket, &home);
|
||||
let a4 = served.say(
|
||||
&session,
|
||||
"What were the exact words I first asked you to reply with?",
|
||||
);
|
||||
assert!(
|
||||
a4.to_lowercase().contains("box made"),
|
||||
"after a restart the session must still know: {a4}"
|
||||
);
|
||||
|
||||
let records = served.records(&session);
|
||||
let tool_names: Vec<String> = records
|
||||
.iter()
|
||||
.filter_map(|r| match r {
|
||||
proto::LogRecord::Assistant { tool_calls, .. } => Some(
|
||||
tool_calls
|
||||
.iter()
|
||||
.map(|c| c.name.clone())
|
||||
.collect::<Vec<_>>(),
|
||||
),
|
||||
_ => None,
|
||||
})
|
||||
.flatten()
|
||||
.collect();
|
||||
assert!(tool_names.contains(&"clock".to_string()), "{tool_names:?}");
|
||||
assert!(
|
||||
tool_names.contains(&"find_tool".to_string())
|
||||
&& tool_names.contains(&"call_tool".to_string()),
|
||||
"{tool_names:?}"
|
||||
);
|
||||
let usages = records
|
||||
.iter()
|
||||
.filter(|r| matches!(r, proto::LogRecord::Usage { .. }))
|
||||
.count();
|
||||
let assistants = records
|
||||
.iter()
|
||||
.filter(|r| matches!(r, proto::LogRecord::Assistant { .. }))
|
||||
.count();
|
||||
assert_eq!(usages, assistants, "one usage record per completion");
|
||||
let losses: Vec<&proto::LogRecord> = records
|
||||
.iter()
|
||||
.filter(|r| matches!(r, proto::LogRecord::CacheLoss { .. }))
|
||||
.collect();
|
||||
assert!(
|
||||
losses.is_empty(),
|
||||
"every request hit the cache, including the one after the restart: {losses:?}"
|
||||
);
|
||||
let results = records
|
||||
.iter()
|
||||
.filter(|r| matches!(r, proto::LogRecord::ToolResult { .. }))
|
||||
.count();
|
||||
assert!(
|
||||
results >= 3,
|
||||
"clock, find_tool and call_tool each left a result: {results}"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
//! End to end in two processes: `loopd`'s turn loop, with a `BrokerPort`, against the real
|
||||
//! `brokerd` binary on a temporary home, and the fake llama server. Do not edit.
|
||||
//!
|
||||
//! It cannot be one process: `loopd` may not depend on `brokerd`, even for tests. The path of
|
||||
//! the `brokerd` binary comes from `BOXMAKER_BROKERD`; without it the test is ignored, and
|
||||
//! `make gate` builds the workspace and runs it with the variable set.
|
||||
|
||||
mod support;
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use loopd::baseline::Baseline;
|
||||
use loopd::broker_port::BrokerPort;
|
||||
use loopd::llama::Client;
|
||||
use loopd::session::Session;
|
||||
use loopd::tools::Registry;
|
||||
use loopd::turn::{Runtime, run_turn};
|
||||
use proto::{
|
||||
AuditEvent, AuditRecord, ChainVerifier, DecisionRecord, DenyReason, SessionId, TurnEvent,
|
||||
};
|
||||
use support::{FakeServer, Home, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
/// `brokerd serve`, killed when dropped.
|
||||
struct Brokerd(Child);
|
||||
|
||||
impl Drop for Brokerd {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
fn start_brokerd(home: &Path) -> (Brokerd, PathBuf) {
|
||||
let binary = std::env::var_os("BOXMAKER_BROKERD")
|
||||
.expect("set BOXMAKER_BROKERD to the brokerd binary; `make gate` does");
|
||||
std::fs::create_dir_all(home.join("grants")).unwrap();
|
||||
let config = home.join("brokerd.toml");
|
||||
let text = format!(
|
||||
"[paths]\nhome = \"{h}\"\ngrants = \"{h}/grants\"\n",
|
||||
h = home.display()
|
||||
);
|
||||
std::fs::write(&config, text).unwrap();
|
||||
let child = Command::new(binary)
|
||||
.args(["serve", "--config"])
|
||||
.arg(&config)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::null())
|
||||
.spawn()
|
||||
.unwrap();
|
||||
let brokerd = Brokerd(child);
|
||||
let socket = home.join("run/loop-broker/broker.sock");
|
||||
let until = Instant::now() + Duration::from_secs(10);
|
||||
while UnixStream::connect(&socket).is_err() {
|
||||
assert!(
|
||||
Instant::now() < until,
|
||||
"brokerd never listened on {}",
|
||||
socket.display()
|
||||
);
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
(brokerd, socket)
|
||||
}
|
||||
|
||||
/// Every record under `audit/`, after checking that the chain verifies.
|
||||
fn audit(dir: &Path) -> Vec<AuditRecord> {
|
||||
let mut names: Vec<String> = std::fs::read_dir(dir)
|
||||
.unwrap()
|
||||
.map(|e| e.unwrap().file_name().into_string().unwrap())
|
||||
.filter(|n| n.ends_with(".jsonl"))
|
||||
.collect();
|
||||
names.sort();
|
||||
let mut verifier = ChainVerifier::new();
|
||||
let mut records = Vec::new();
|
||||
for name in &names {
|
||||
let bytes = std::fs::read(dir.join(name)).unwrap();
|
||||
verifier.feed(name, &bytes);
|
||||
for line in String::from_utf8(bytes).unwrap().lines() {
|
||||
records.push(serde_json::from_str(line).unwrap());
|
||||
}
|
||||
}
|
||||
let report = verifier.finish();
|
||||
assert!(report.failure.is_none(), "{:?}", report.failure);
|
||||
assert!(report.torn_tail.is_none());
|
||||
assert_eq!(report.records, records.len() as u64);
|
||||
records
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the brokerd binary in BOXMAKER_BROKERD; `make gate` runs it"]
|
||||
fn a_call_no_grant_allows_reaches_the_model_as_a_denial_and_is_audited() {
|
||||
let home = Home::new();
|
||||
let broker_home = home.dir.join("broker-home");
|
||||
let (_brokerd, socket) = start_brokerd(&broker_home);
|
||||
|
||||
let server = FakeServer::start();
|
||||
// The recorded model calls `read_file` on /etc/hostname, then answers in plain text.
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let cfg = home.config(&server.socket);
|
||||
let client = Client::new(cfg.clone());
|
||||
let port = BrokerPort::new(socket, Duration::from_secs(10));
|
||||
let registry = Registry::m2b();
|
||||
let baseline = Baseline::assemble(&cfg, ®istry).unwrap();
|
||||
let id = SessionId::new("e2e").unwrap();
|
||||
let mut session = Session::create(&home.dir, id, baseline, cfg.slots.main).unwrap();
|
||||
let runtime = Runtime {
|
||||
cfg: &cfg,
|
||||
client: &client,
|
||||
port: &port,
|
||||
registry: ®istry,
|
||||
};
|
||||
let mut events = Vec::new();
|
||||
let outcome = run_turn(
|
||||
&mut session,
|
||||
&runtime,
|
||||
"what is this host called?",
|
||||
&mut |e| events.push(e.clone()),
|
||||
);
|
||||
assert!(
|
||||
outcome.is_ok(),
|
||||
"the turn goes on after a denial: {outcome:?}"
|
||||
);
|
||||
assert!(
|
||||
events.contains(&TurnEvent::ToolDenied {
|
||||
name: "read_file".to_string(),
|
||||
reason: DenyReason::NoGrant,
|
||||
}),
|
||||
"{events:?}"
|
||||
);
|
||||
// The model reads the denial in its next request.
|
||||
let second = server.requests_to(CHAT)[1].json();
|
||||
assert_eq!(
|
||||
second["messages"][3]["content"],
|
||||
"Denied: no grant allows this call."
|
||||
);
|
||||
|
||||
let records = audit(&broker_home.join("audit"));
|
||||
assert_eq!(records.len(), 1, "{records:?}");
|
||||
match &records[0].event {
|
||||
AuditEvent::Decision {
|
||||
session,
|
||||
tool,
|
||||
arguments,
|
||||
outcome,
|
||||
grant,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(session.as_str(), "e2e");
|
||||
assert_eq!(tool, "read_file");
|
||||
assert_eq!(arguments, r#"{"path":"/etc/hostname"}"#);
|
||||
assert_eq!(
|
||||
*outcome,
|
||||
DecisionRecord::Denied {
|
||||
reason: DenyReason::NoGrant
|
||||
}
|
||||
);
|
||||
assert_eq!(*grant, None);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,240 @@
|
||||
//! Every fail-closed message `loopd` produces ends with the runbook entry that explains it.
|
||||
//! `scripts/check-runbook.sh` checks that the entries exist; these tests check that the messages
|
||||
//! name them. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::Duration;
|
||||
|
||||
use loopd::baseline::Baseline;
|
||||
use loopd::broker_port::{not_configured_line, unavailable_line};
|
||||
use loopd::session::{Session, SessionError};
|
||||
use loopd::tools::Registry;
|
||||
use proto::SessionId;
|
||||
use support::{FakeServer, Home, Reply};
|
||||
|
||||
// `want` is the whole pointer, written out: scripts/check-runbook.sh reads the anchors of every
|
||||
// pointer in the source, and cannot read one built with `format!`.
|
||||
fn ends_with_pointer(message: &str, want: &str) {
|
||||
assert!(
|
||||
message.ends_with(want),
|
||||
"{message:?} must end with {want:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
message.matches("runbook.md#").count(),
|
||||
1,
|
||||
"one pointer, not two: {message:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_session_log_names_its_entry() {
|
||||
let home = Home::new();
|
||||
let cfg = home.config(Path::new("/tmp/unused.sock"));
|
||||
let id = SessionId::new("a").unwrap();
|
||||
let baseline = Baseline::assemble(&cfg, &Registry::m3a()).unwrap();
|
||||
drop(Session::create(&home.dir, id.clone(), baseline, 0).unwrap());
|
||||
let log = home.dir.join("sessions/a/0.jsonl");
|
||||
let mut text = std::fs::read_to_string(&log).unwrap();
|
||||
text.push_str("{\"type\":\"user\",\"time\":");
|
||||
std::fs::write(&log, text).unwrap();
|
||||
match Session::open(&home.dir, id) {
|
||||
Err(e @ SessionError::Torn { .. }) => {
|
||||
let message = e.to_string();
|
||||
assert!(message.contains("0.jsonl:2: "), "{message}");
|
||||
ends_with_pointer(&message, "see docs/runbook.md#session-log-damaged");
|
||||
}
|
||||
Err(other) => panic!("{other:?}"),
|
||||
Ok(_) => panic!("a torn log was opened"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_torn_error_points_at_the_damaged_log_entry() {
|
||||
let home = Home::new();
|
||||
let missing = Session::open(&home.dir, SessionId::new("nobody").unwrap());
|
||||
let message = match missing {
|
||||
Err(e) => e.to_string(),
|
||||
Ok(_) => panic!("a session nobody created was opened"),
|
||||
};
|
||||
assert!(!message.contains("runbook"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_core_memory_names_its_entry_and_a_missing_system_prompt_does_not() {
|
||||
let home = Home::new();
|
||||
let cfg = home.config(Path::new("/tmp/unused.sock"));
|
||||
home.write("memory/core.md", "memory\n");
|
||||
let core = home.dir.join("memory/core.md");
|
||||
if !running_as_root() {
|
||||
std::fs::set_permissions(&core, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
let result = Baseline::assemble(&cfg, &Registry::m3a());
|
||||
std::fs::set_permissions(&core, std::fs::Permissions::from_mode(0o644)).unwrap();
|
||||
let message = result.expect_err("unreadable core.md").to_string();
|
||||
assert!(message.contains("core.md"), "{message}");
|
||||
ends_with_pointer(&message, "see docs/runbook.md#core-memory-unreadable");
|
||||
}
|
||||
// The system prompt is a different file with a different remedy: no pointer to this entry.
|
||||
std::fs::remove_file(home.dir.join("system.md")).unwrap();
|
||||
let message = Baseline::assemble(&cfg, &Registry::m3a())
|
||||
.expect_err("no system.md")
|
||||
.to_string();
|
||||
assert!(!message.contains("core-memory-unreadable"), "{message}");
|
||||
}
|
||||
|
||||
fn running_as_root() -> bool {
|
||||
std::fs::read_to_string("/proc/self/status")
|
||||
.map(|s| s.lines().any(|l| l.starts_with("Uid:\t0\t")))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_lines_name_their_entry() {
|
||||
let line = unavailable_line("cannot connect to /run/x.sock: No such file");
|
||||
assert_eq!(
|
||||
line,
|
||||
"loopd: the tool broker is unavailable: cannot connect to /run/x.sock: No such file; \
|
||||
see docs/runbook.md#broker-unavailable"
|
||||
);
|
||||
ends_with_pointer(&line, "see docs/runbook.md#broker-unavailable");
|
||||
let line = not_configured_line();
|
||||
assert!(
|
||||
line.starts_with("loopd: no tool broker is configured"),
|
||||
"{line}"
|
||||
);
|
||||
ends_with_pointer(&line, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
|
||||
fn config_file(
|
||||
home: &Home,
|
||||
server: &FakeServer,
|
||||
expect_slots: u32,
|
||||
extra: &str,
|
||||
) -> std::path::PathBuf {
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "test-model"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = {expect_slots}
|
||||
[limits]
|
||||
poll_ms = 40
|
||||
liveness_ms = 500
|
||||
retry_backoff_ms = [10]
|
||||
[paths]
|
||||
home = "{}"
|
||||
{extra}
|
||||
"#,
|
||||
server.socket.display(),
|
||||
home.dir.display()
|
||||
);
|
||||
let path = home.dir.join("config.toml");
|
||||
std::fs::write(&path, text).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn healthy_routes(server: &FakeServer) {
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(
|
||||
"/v1/chat/completions",
|
||||
vec![
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("turn1"),
|
||||
Reply::fixture("turn2"),
|
||||
Reply::fixture("plain"),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/// Waits for `loopd serve` to bind its socket, stops it, and returns what it printed.
|
||||
fn stderr_once_serving(mut child: Child, socket: &Path) -> String {
|
||||
let mut up = false;
|
||||
for _ in 0..200 {
|
||||
if socket.exists() {
|
||||
up = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
child.kill().unwrap();
|
||||
let output = child.wait_with_output().unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
assert!(up, "the socket never appeared; stderr: {stderr}");
|
||||
stderr
|
||||
}
|
||||
|
||||
fn serve(config: &Path) -> Child {
|
||||
Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.args(["serve", "--config"])
|
||||
.arg(config)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_self_test_names_its_entry() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
// The server has two slots; the config expects three.
|
||||
let config = config_file(&home, &server, 3, "");
|
||||
for command in ["selftest", "serve"] {
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.args([command, "--config"])
|
||||
.arg(&config)
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert_eq!(output.status.code(), Some(1), "{command}: {stderr}");
|
||||
let line = stderr
|
||||
.lines()
|
||||
.find(|l| l.starts_with("selftest: FAILED: "))
|
||||
.unwrap_or_else(|| panic!("{command}: no FAILED line: {stderr}"));
|
||||
ends_with_pointer(line, "see docs/runbook.md#loopd-selftest-failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_without_a_broker_says_so_once_with_the_entry() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
let config = config_file(&home, &server, 2, "");
|
||||
let socket = home.dir.join("run/loop/loop.sock");
|
||||
let stderr = stderr_once_serving(serve(&config), &socket);
|
||||
let lines: Vec<&str> = stderr
|
||||
.lines()
|
||||
.filter(|l| l.contains("no tool broker is configured"))
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 1, "once, at startup: {stderr}");
|
||||
ends_with_pointer(lines[0], "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_with_a_broker_socket_does_not_say_it() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
// The socket need not exist: loopd connects per call, and a missing broker is not a reason
|
||||
// to refuse to start.
|
||||
let extra = format!(
|
||||
"[broker]\nsocket = \"{}\"\n",
|
||||
home.dir.join("run/loop-broker/broker.sock").display()
|
||||
);
|
||||
let config = config_file(&home, &server, 2, &extra);
|
||||
let socket = home.dir.join("run/loop/loop.sock");
|
||||
let stderr = stderr_once_serving(serve(&config), &socket);
|
||||
assert!(!stderr.contains("no tool broker"), "{stderr}");
|
||||
assert!(stderr.contains("serving on"), "{stderr}");
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! A fake `brokerd` for the `BrokerPort` tests. Included with `#[path]`. Do not edit.
|
||||
//!
|
||||
//! The fake behaves as the real one will: it reads one request frame, answers on the same
|
||||
//! connection, never half-closes, and closes after the final frame.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use loopd::broker_port::{BrokerPort, UNAVAILABLE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
CallId, DataClass, Envelope, Message, PROTOCOL_VERSION, SessionId, Timestamp, ToolRequest,
|
||||
ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
|
||||
static NEXT: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
pub fn socket_path() -> PathBuf {
|
||||
let n = NEXT.fetch_add(1, Ordering::SeqCst);
|
||||
std::env::temp_dir().join(format!("loopd-broker-{}-{n}.sock", std::process::id()))
|
||||
}
|
||||
|
||||
/// Accepts one connection, reads the request frame and hands both to `script`. The connection
|
||||
/// closes when `script` returns.
|
||||
pub fn broker<F>(script: F) -> (PathBuf, JoinHandle<Envelope>)
|
||||
where
|
||||
F: FnOnce(&mut UnixStream, &Envelope) + Send + 'static,
|
||||
{
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let handle = thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
script(&mut stream, &request);
|
||||
request
|
||||
});
|
||||
(path, handle)
|
||||
}
|
||||
|
||||
pub fn frame(id: u64, r#final: bool, msg: Message) -> Envelope {
|
||||
Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
id,
|
||||
r#final,
|
||||
msg,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send(stream: &mut UnixStream, id: u64, r#final: bool, response: ToolResponse) {
|
||||
// The port may already have given up and gone; the fake does not care.
|
||||
let _ = write_frame(stream, &frame(id, r#final, Message::ToolResponse(response)));
|
||||
}
|
||||
|
||||
pub fn result(content: &str) -> ToolResponse {
|
||||
ToolResponse::Result {
|
||||
content: content.to_string(),
|
||||
class: DataClass::Secret,
|
||||
untrusted: true,
|
||||
truncated: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request() -> ToolRequest {
|
||||
ToolRequest {
|
||||
session: SessionId::new("chat-1").unwrap(),
|
||||
call: CallId(7),
|
||||
tool: "read_file".to_string(),
|
||||
arguments: r#"{"path":"/home/kyle/notes/a.md"}"#.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn in_ms(ms: u64) -> Timestamp {
|
||||
Timestamp::from_unix_millis(Timestamp::now().unix_millis() + ms).unwrap()
|
||||
}
|
||||
|
||||
pub struct Call {
|
||||
pub response: ToolResponse,
|
||||
pub pending: Vec<Pending>,
|
||||
pub lines: Vec<String>,
|
||||
pub took: Duration,
|
||||
}
|
||||
|
||||
pub fn call(socket: PathBuf, timeout_ms: u64, request: &ToolRequest) -> Call {
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = lines.clone();
|
||||
let port = BrokerPort::with_log(
|
||||
socket,
|
||||
Duration::from_millis(timeout_ms),
|
||||
Box::new(move |line| sink.lock().unwrap().push(line.to_string())),
|
||||
);
|
||||
let mut pending = Vec::new();
|
||||
let started = Instant::now();
|
||||
let response = port.call(request, &mut |p| pending.push(*p));
|
||||
let took = started.elapsed();
|
||||
let lines = lines.lock().unwrap().clone();
|
||||
Call {
|
||||
response,
|
||||
pending,
|
||||
lines,
|
||||
took,
|
||||
}
|
||||
}
|
||||
|
||||
/// The answer is the fixed failure, and `loopd` printed exactly one line, with the pointer.
|
||||
pub fn assert_unavailable(call: &Call, why: &str) {
|
||||
assert_eq!(
|
||||
call.response,
|
||||
ToolResponse::Failed {
|
||||
message: UNAVAILABLE.to_string()
|
||||
},
|
||||
"{why}"
|
||||
);
|
||||
assert_eq!(
|
||||
call.lines.len(),
|
||||
1,
|
||||
"{why}: one line per failed call: {:?}",
|
||||
call.lines
|
||||
);
|
||||
let line = &call.lines[0];
|
||||
assert!(
|
||||
line.starts_with("loopd: the tool broker is unavailable: "),
|
||||
"{why}: {line}"
|
||||
);
|
||||
assert!(
|
||||
line.ends_with("; see docs/runbook.md#broker-unavailable"),
|
||||
"{why}: {line}"
|
||||
);
|
||||
assert!(call.pending.is_empty() || why.contains("pending"), "{why}");
|
||||
}
|
||||
@@ -0,0 +1,403 @@
|
||||
//! A scripted stand-in for `llama-server`, for tests. Do not edit.
|
||||
//!
|
||||
//! It listens on a Unix socket in a temporary directory. Each path has a list of replies that are
|
||||
//! served in order; the last one repeats. A reply is raw bytes, normally a response recorded from
|
||||
//! the real server (`tests/fixtures/http/*.http`), and can be delayed, sent in small pieces, cut
|
||||
//! short, or left hanging. Every request is recorded.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
static NEXT: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
pub fn fixture_path(kind: &str, name: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(kind)
|
||||
.join(name)
|
||||
}
|
||||
|
||||
pub fn fixture_bytes(kind: &str, name: &str) -> Vec<u8> {
|
||||
let path = fixture_path(kind, name);
|
||||
std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()))
|
||||
}
|
||||
|
||||
/// A config that points at `socket`, with every limit short enough for a test.
|
||||
pub fn test_config(socket: &Path) -> loopd::config::Config {
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "test-model"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
[limits]
|
||||
poll_ms = 40
|
||||
busy_wait_ms = 400
|
||||
load_wait_ms = 300
|
||||
idle_grace_ms = 200
|
||||
liveness_ms = 150
|
||||
retry_backoff_ms = [10, 20, 30]
|
||||
retry_window_ms = 5000
|
||||
"#,
|
||||
socket.display()
|
||||
);
|
||||
loopd::config::Config::parse(&text).unwrap()
|
||||
}
|
||||
|
||||
/// A temporary `BOXMAKER_HOME` with a `system.md` beside a `config.toml`, for session tests.
|
||||
pub struct Home {
|
||||
pub dir: PathBuf,
|
||||
}
|
||||
|
||||
impl Home {
|
||||
pub fn new() -> Home {
|
||||
let n = NEXT.fetch_add(1, Ordering::SeqCst);
|
||||
let dir = std::env::temp_dir().join(format!("loopd-home-{}-{n}", std::process::id()));
|
||||
let _ = std::fs::remove_dir_all(&dir);
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
std::fs::write(dir.join("system.md"), "You are Boxmaker, a test agent.\n").unwrap();
|
||||
Home { dir }
|
||||
}
|
||||
|
||||
/// A config for `socket` whose home, system prompt and channel socket are all under this
|
||||
/// directory, with the fast test limits.
|
||||
pub fn config(&self, socket: &Path) -> loopd::config::Config {
|
||||
let mut cfg = test_config(socket);
|
||||
cfg.paths.home = self.dir.clone();
|
||||
cfg.baseline.system = self.dir.join("system.md");
|
||||
cfg.channel.socket = self.dir.join("loop.sock");
|
||||
cfg
|
||||
}
|
||||
|
||||
pub fn write(&self, relative: &str, text: &str) {
|
||||
let path = self.dir.join(relative);
|
||||
std::fs::create_dir_all(path.parent().unwrap()).unwrap();
|
||||
std::fs::write(path, text).unwrap();
|
||||
}
|
||||
|
||||
pub fn read(&self, relative: &str) -> String {
|
||||
std::fs::read_to_string(self.dir.join(relative)).unwrap()
|
||||
}
|
||||
|
||||
/// The records of a session's log, epoch 0.
|
||||
pub fn records(&self, session: &str) -> Vec<proto::LogRecord> {
|
||||
let text = self.read(&format!("sessions/{session}/0.jsonl"));
|
||||
text.lines()
|
||||
.map(|l| serde_json::from_str(l).unwrap())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
/// A tool port that answers from a script and records what it was asked.
|
||||
pub struct ScriptedPort {
|
||||
replies: Mutex<VecDeque<proto::ToolResponse>>,
|
||||
calls: Mutex<Vec<proto::ToolRequest>>,
|
||||
}
|
||||
|
||||
impl ScriptedPort {
|
||||
/// Replies are given in order; when they run out, every call gets `fallback`.
|
||||
pub fn new(replies: Vec<proto::ToolResponse>) -> ScriptedPort {
|
||||
ScriptedPort {
|
||||
replies: Mutex::new(replies.into()),
|
||||
calls: Mutex::new(Vec::new()),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn calls(&self) -> Vec<proto::ToolRequest> {
|
||||
self.calls.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ok_result(content: &str) -> proto::ToolResponse {
|
||||
proto::ToolResponse::Result {
|
||||
content: content.to_string(),
|
||||
class: proto::DataClass::Private,
|
||||
untrusted: true,
|
||||
truncated: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn pending(approval: u64, expires: &str) -> proto::ToolResponse {
|
||||
proto::ToolResponse::PendingApproval {
|
||||
approval,
|
||||
expires: proto::Timestamp::parse(expires).unwrap(),
|
||||
}
|
||||
}
|
||||
|
||||
impl loopd::tools::ToolPort for ScriptedPort {
|
||||
/// A scripted `PendingApproval` is reported through `on_pending`, as the real port does with
|
||||
/// the broker's pending frame, and the reply after it is the answer. Use
|
||||
/// `ReturnsPendingPort` for a port that breaks the rule and returns one.
|
||||
fn call(
|
||||
&self,
|
||||
request: &proto::ToolRequest,
|
||||
on_pending: &mut dyn FnMut(&loopd::tools::Pending),
|
||||
) -> proto::ToolResponse {
|
||||
self.calls.lock().unwrap().push(request.clone());
|
||||
let mut replies = self.replies.lock().unwrap();
|
||||
let mut reply = replies.pop_front();
|
||||
if let Some(proto::ToolResponse::PendingApproval { approval, expires }) = reply {
|
||||
on_pending(&loopd::tools::Pending { approval, expires });
|
||||
reply = replies.pop_front();
|
||||
}
|
||||
reply.unwrap_or_else(|| ok_result("scripted"))
|
||||
}
|
||||
}
|
||||
|
||||
/// A port that breaks the rule: it returns a pending frame as its final answer.
|
||||
pub struct ReturnsPendingPort;
|
||||
|
||||
impl loopd::tools::ToolPort for ReturnsPendingPort {
|
||||
fn call(
|
||||
&self,
|
||||
_request: &proto::ToolRequest,
|
||||
_on_pending: &mut dyn FnMut(&loopd::tools::Pending),
|
||||
) -> proto::ToolResponse {
|
||||
pending(1, "2026-09-18T12:00:00.000Z")
|
||||
}
|
||||
}
|
||||
|
||||
/// What an independent script derived from a recorded stream: `tests/fixtures/expected/`.
|
||||
pub fn expected(name: &str) -> serde_json::Value {
|
||||
serde_json::from_slice(&fixture_bytes("expected", &format!("{name}.json"))).unwrap()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Reply {
|
||||
bytes: Vec<u8>,
|
||||
head_delay_ms: u64,
|
||||
piece: usize,
|
||||
piece_delay_ms: u64,
|
||||
stop_after: Option<usize>,
|
||||
hang_ms: u64,
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
/// Exactly these bytes, then close.
|
||||
pub fn raw(bytes: impl Into<Vec<u8>>) -> Reply {
|
||||
Reply {
|
||||
bytes: bytes.into(),
|
||||
head_delay_ms: 0,
|
||||
piece: usize::MAX,
|
||||
piece_delay_ms: 0,
|
||||
stop_after: None,
|
||||
hang_ms: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// A response recorded from the real server: `tests/fixtures/http/<name>.http`.
|
||||
pub fn fixture(name: &str) -> Reply {
|
||||
Reply::raw(fixture_bytes("http", &format!("{name}.http")))
|
||||
}
|
||||
|
||||
/// A small JSON response with a content length.
|
||||
pub fn json(status: u16, body: &str) -> Reply {
|
||||
Reply::raw(format!(
|
||||
"HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}",
|
||||
body.len()
|
||||
))
|
||||
}
|
||||
|
||||
/// Wait this long before sending the first byte, as a queued request does.
|
||||
pub fn head_delay(mut self, ms: u64) -> Reply {
|
||||
self.head_delay_ms = ms;
|
||||
self
|
||||
}
|
||||
|
||||
/// Send `piece` bytes at a time, waiting `delay_ms` before each piece after the first.
|
||||
pub fn trickle(mut self, piece: usize, delay_ms: u64) -> Reply {
|
||||
self.piece = piece.max(1);
|
||||
self.piece_delay_ms = delay_ms;
|
||||
self
|
||||
}
|
||||
|
||||
/// Send only the first `bytes` bytes, then close, as a server that dies does.
|
||||
pub fn cut_after(mut self, bytes: usize) -> Reply {
|
||||
self.stop_after = Some(bytes);
|
||||
self
|
||||
}
|
||||
|
||||
/// Send only the first `bytes` bytes, then stay silent for `ms` before closing.
|
||||
pub fn hang_after(mut self, bytes: usize, ms: u64) -> Reply {
|
||||
self.stop_after = Some(bytes);
|
||||
self.hang_ms = ms;
|
||||
self
|
||||
}
|
||||
|
||||
/// The offset just after the `n`th `data:` line of the body, for use with `cut_after`.
|
||||
pub fn offset_after_events(&self, n: usize) -> usize {
|
||||
let mut seen = 0;
|
||||
let mut at = 0;
|
||||
while let Some(found) = find(&self.bytes[at..], b"\n\n") {
|
||||
at += found + 2;
|
||||
seen += 1;
|
||||
if seen == n {
|
||||
return at;
|
||||
}
|
||||
}
|
||||
panic!("the reply has only {seen} events");
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Recorded {
|
||||
pub method: String,
|
||||
/// Path and query as sent.
|
||||
pub target: String,
|
||||
/// Header lines as sent, without the request line.
|
||||
pub headers: Vec<String>,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Recorded {
|
||||
pub fn path(&self) -> &str {
|
||||
self.target.split('?').next().unwrap_or("")
|
||||
}
|
||||
|
||||
pub fn json(&self) -> serde_json::Value {
|
||||
serde_json::from_slice(&self.body).expect("request body is JSON")
|
||||
}
|
||||
}
|
||||
|
||||
struct State {
|
||||
routes: Mutex<Vec<(String, VecDeque<Reply>)>>,
|
||||
requests: Mutex<Vec<Recorded>>,
|
||||
}
|
||||
|
||||
pub struct FakeServer {
|
||||
pub socket: PathBuf,
|
||||
state: Arc<State>,
|
||||
}
|
||||
|
||||
impl FakeServer {
|
||||
pub fn start() -> FakeServer {
|
||||
let n = NEXT.fetch_add(1, Ordering::SeqCst);
|
||||
let dir = std::env::temp_dir().join(format!("loopd-fake-{}-{n}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let socket = dir.join("infer.sock");
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
let state = Arc::new(State {
|
||||
routes: Mutex::new(Vec::new()),
|
||||
requests: Mutex::new(Vec::new()),
|
||||
});
|
||||
let accept_state = Arc::clone(&state);
|
||||
thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let Ok(stream) = stream else { break };
|
||||
let state = Arc::clone(&accept_state);
|
||||
thread::spawn(move || serve(stream, &state));
|
||||
}
|
||||
});
|
||||
FakeServer { socket, state }
|
||||
}
|
||||
|
||||
/// Replies for `path` (the query is ignored), served in order. The last one repeats.
|
||||
pub fn route(&self, path: &str, replies: Vec<Reply>) {
|
||||
assert!(!replies.is_empty());
|
||||
let mut routes = self.state.routes.lock().unwrap();
|
||||
routes.retain(|(p, _)| p != path);
|
||||
routes.push((path.to_string(), replies.into()));
|
||||
}
|
||||
|
||||
pub fn requests(&self) -> Vec<Recorded> {
|
||||
self.state.requests.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
pub fn requests_to(&self, path: &str) -> Vec<Recorded> {
|
||||
self.requests()
|
||||
.into_iter()
|
||||
.filter(|r| r.path() == path)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack.windows(needle.len()).position(|w| w == needle)
|
||||
}
|
||||
|
||||
fn read_request(stream: &mut UnixStream) -> Option<Recorded> {
|
||||
let mut buf = Vec::new();
|
||||
let mut chunk = [0u8; 4096];
|
||||
let head_end = loop {
|
||||
if let Some(end) = find(&buf, b"\r\n\r\n") {
|
||||
break end;
|
||||
}
|
||||
match stream.read(&mut chunk) {
|
||||
Ok(0) | Err(_) => return None,
|
||||
Ok(n) => buf.extend_from_slice(&chunk[..n]),
|
||||
}
|
||||
};
|
||||
let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
|
||||
let mut lines = head.split("\r\n");
|
||||
let mut request_line = lines.next()?.split(' ');
|
||||
let method = request_line.next()?.to_string();
|
||||
let target = request_line.next()?.to_string();
|
||||
let headers: Vec<String> = lines.map(str::to_string).collect();
|
||||
let length = headers
|
||||
.iter()
|
||||
.filter_map(|h| h.split_once(':'))
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.and_then(|(_, v)| v.trim().parse::<usize>().ok())
|
||||
.unwrap_or(0);
|
||||
let mut body = buf[head_end + 4..].to_vec();
|
||||
while body.len() < length {
|
||||
match stream.read(&mut chunk) {
|
||||
Ok(0) | Err(_) => return None,
|
||||
Ok(n) => body.extend_from_slice(&chunk[..n]),
|
||||
}
|
||||
}
|
||||
Some(Recorded {
|
||||
method,
|
||||
target,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
fn serve(mut stream: UnixStream, state: &State) {
|
||||
let Some(request) = read_request(&mut stream) else {
|
||||
return;
|
||||
};
|
||||
let path = request.path().to_string();
|
||||
state.requests.lock().unwrap().push(request);
|
||||
let reply = {
|
||||
let mut routes = state.routes.lock().unwrap();
|
||||
match routes.iter_mut().find(|(p, _)| *p == path) {
|
||||
Some((_, replies)) if replies.len() > 1 => replies.pop_front(),
|
||||
Some((_, replies)) => replies.front().cloned(),
|
||||
None => None,
|
||||
}
|
||||
};
|
||||
let reply =
|
||||
reply.unwrap_or_else(|| Reply::json(404, r#"{"error":"no route in the fake server"}"#));
|
||||
thread::sleep(Duration::from_millis(reply.head_delay_ms));
|
||||
let end = reply
|
||||
.stop_after
|
||||
.unwrap_or(reply.bytes.len())
|
||||
.min(reply.bytes.len());
|
||||
for (i, piece) in reply.bytes[..end].chunks(reply.piece).enumerate() {
|
||||
if i > 0 {
|
||||
thread::sleep(Duration::from_millis(reply.piece_delay_ms));
|
||||
}
|
||||
if stream.write_all(piece).is_err() {
|
||||
return; // the client went away, which some tests do on purpose
|
||||
}
|
||||
let _ = stream.flush();
|
||||
}
|
||||
thread::sleep(Duration::from_millis(reply.hang_ms));
|
||||
}
|
||||
@@ -0,0 +1,307 @@
|
||||
//! Tests for the registry, dispatch, the denial sentences, the result cap and the fake tools.
|
||||
//! Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::tools::{
|
||||
Dispatch, FakeTools, Pending, Registry, ToolPort, cap_result, denial_text, dispatch,
|
||||
};
|
||||
use proto::{CallId, DenyReason, SessionId, ToolRequest, ToolResponse};
|
||||
|
||||
fn req(tool: &str, arguments: &str) -> ToolRequest {
|
||||
ToolRequest {
|
||||
session: SessionId::new("s").unwrap(),
|
||||
call: CallId(1),
|
||||
tool: tool.to_string(),
|
||||
arguments: arguments.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_core_schemas_are_the_fixed_tools_array() {
|
||||
let names: Vec<String> = Registry::m2b()
|
||||
.core_schemas()
|
||||
.into_iter()
|
||||
.map(|s| s.name)
|
||||
.collect();
|
||||
assert_eq!(names, ["clock", "find_tool", "call_tool"]);
|
||||
let find = &Registry::m2b().core_schemas()[1];
|
||||
assert_eq!(find.parameters["required"], serde_json::json!(["query"]));
|
||||
let call = &Registry::m2b().core_schemas()[2];
|
||||
assert_eq!(
|
||||
call.parameters["required"],
|
||||
serde_json::json!(["name", "arguments"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_matches_name_or_description_case_insensitively() {
|
||||
let r = Registry::m2b();
|
||||
let names = |q: &str| r.find(q).iter().map(|s| s.name.clone()).collect::<Vec<_>>();
|
||||
assert_eq!(names("echo"), ["echo"]);
|
||||
assert_eq!(names("ECHO"), ["echo"]);
|
||||
assert_eq!(names("unchanged"), ["echo"], "matches the description too");
|
||||
assert_eq!(names("time"), ["clock"]);
|
||||
assert!(names("weather").is_empty());
|
||||
assert!(
|
||||
names("").is_empty(),
|
||||
"an empty query matches nothing, not everything"
|
||||
);
|
||||
assert!(names(" ").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_tool_is_answered_locally() {
|
||||
let r = Registry::m2b();
|
||||
match dispatch(&r, "find_tool", r#"{"query":"echo"}"#) {
|
||||
Dispatch::Local(text) => {
|
||||
assert!(text.starts_with("1 tool(s) match:\n"), "{text}");
|
||||
assert!(text.contains("\"name\":\"echo\""), "{text}");
|
||||
assert!(
|
||||
text.contains("\"parameters\""),
|
||||
"the schema is included: {text}"
|
||||
);
|
||||
assert!(text.ends_with("Call it with call_tool."), "{text}");
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
dispatch(&r, "find_tool", r#"{"query":"weather"}"#),
|
||||
Dispatch::Local("No tool matches.".to_string())
|
||||
);
|
||||
assert!(
|
||||
matches!(dispatch(&r, "find_tool", "not json"), Dispatch::Local(t) if t == "No tool matches.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn call_tool_unwraps_a_known_non_core_tool_and_nothing_else() {
|
||||
let r = Registry::m2b();
|
||||
let args = r#"{"name":"echo","arguments":{"text": "box"}}"#;
|
||||
match dispatch(&r, "call_tool", args) {
|
||||
Dispatch::Port { tool, arguments } => {
|
||||
assert_eq!(tool, "echo");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&arguments).unwrap(),
|
||||
serde_json::json!({"text": "box"})
|
||||
);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
let cases = [
|
||||
(
|
||||
r#"{"name":"weather","arguments":{}}"#,
|
||||
"No tool named \"weather\"",
|
||||
),
|
||||
(r#"{"name":"clock","arguments":{}}"#, "core tool"),
|
||||
(r#"{"arguments":{}}"#, "No tool named \"\""),
|
||||
("not json", "needs a JSON object"),
|
||||
];
|
||||
for (args, want) in cases {
|
||||
match dispatch(&r, "call_tool", args) {
|
||||
Dispatch::Local(text) => assert!(text.contains(want), "{args}: {text}"),
|
||||
other => panic!("{args}: {other:?}, must never reach the port"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_other_tool_goes_to_the_port_as_it_is() {
|
||||
let r = Registry::m2b();
|
||||
// Even one the registry does not know: the port (brokerd) decides, not loopd.
|
||||
let want = Dispatch::Port {
|
||||
tool: "read_file".to_string(),
|
||||
arguments: r#"{"path":"/x"}"#.to_string(),
|
||||
};
|
||||
assert_eq!(dispatch(&r, "read_file", r#"{"path":"/x"}"#), want);
|
||||
let want = Dispatch::Port {
|
||||
tool: "weather".to_string(),
|
||||
arguments: "not json".to_string(),
|
||||
};
|
||||
assert_eq!(dispatch(&r, "weather", "not json"), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_clock_is_answered_locally_whatever_its_arguments() {
|
||||
for registry in [Registry::m2b(), Registry::m3a(), Registry::new(vec![])] {
|
||||
for arguments in ["{}", r#"{"zone":"UTC"}"#, "not json", ""] {
|
||||
let before = proto::Timestamp::now();
|
||||
match dispatch(®istry, "clock", arguments) {
|
||||
Dispatch::Local(text) => {
|
||||
let time = proto::Timestamp::parse(&text)
|
||||
.unwrap_or_else(|e| panic!("an RFC 3339 time, got {text:?}: {e:?}"));
|
||||
assert!(time >= before, "{text}");
|
||||
assert!(text.ends_with('Z'), "UTC: {text}");
|
||||
}
|
||||
other => panic!("{arguments:?}: {other:?}, the clock must not reach the port"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_m3a_registry_has_the_same_core_and_the_four_broker_tools() {
|
||||
let r = Registry::m3a();
|
||||
assert_eq!(
|
||||
r.core_schemas(),
|
||||
Registry::m2b().core_schemas(),
|
||||
"the tools array is part of the baseline: it must not change"
|
||||
);
|
||||
let names = |q: &str| r.find(q).iter().map(|s| s.name.clone()).collect::<Vec<_>>();
|
||||
assert_eq!(names("file"), ["read_file", "write_file"]);
|
||||
assert_eq!(names("shell"), ["shell"]);
|
||||
assert_eq!(names("fetch"), ["http_fetch"]);
|
||||
assert_eq!(names("https"), ["http_fetch"]);
|
||||
assert!(names("echo").is_empty(), "echo is a test tool only");
|
||||
|
||||
// The argument schemas are section 3's table: exactly these properties, all strings.
|
||||
let table: [(&str, &[&str], &[&str]); 4] = [
|
||||
("read_file", &["path"], &["path"]),
|
||||
("write_file", &["content", "path"], &["path", "content"]),
|
||||
("shell", &["command", "cwd"], &["command"]),
|
||||
("http_fetch", &["url"], &["url"]),
|
||||
];
|
||||
for (name, properties, required) in table {
|
||||
let entry = r.get(name).unwrap_or_else(|| panic!("{name} is missing"));
|
||||
assert!(!entry.core, "{name} is found with find_tool, not declared");
|
||||
let p = &entry.schema.parameters;
|
||||
assert_eq!(p["type"], "object", "{name}");
|
||||
let mut got: Vec<&str> = p["properties"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
got.sort_unstable();
|
||||
assert_eq!(got, properties, "{name}: properties");
|
||||
for property in properties {
|
||||
assert_eq!(
|
||||
p["properties"][property]["type"], "string",
|
||||
"{name}.{property}"
|
||||
);
|
||||
assert!(
|
||||
p["properties"][property]["description"].is_string(),
|
||||
"{name}.{property} needs a description"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
p["required"],
|
||||
serde_json::json!(required),
|
||||
"{name}: required"
|
||||
);
|
||||
}
|
||||
|
||||
// call_tool lets the four through to the port and nothing else.
|
||||
match dispatch(
|
||||
&r,
|
||||
"call_tool",
|
||||
r#"{"name":"shell","arguments":{"command":"ls"}}"#,
|
||||
) {
|
||||
Dispatch::Port { tool, arguments } => {
|
||||
assert_eq!(tool, "shell");
|
||||
assert_eq!(arguments, r#"{"command":"ls"}"#);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
dispatch(&r, "call_tool", r#"{"name":"echo","arguments":{}}"#),
|
||||
Dispatch::Local(t) if t.contains("No tool named \"echo\"")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_deny_reason_has_its_sentence() {
|
||||
let table = [
|
||||
(DenyReason::NoGrant, "Denied: no grant allows this call."),
|
||||
(
|
||||
DenyReason::GrantExpired,
|
||||
"Denied: the grant for this call has expired.",
|
||||
),
|
||||
(
|
||||
DenyReason::TaintTooHigh,
|
||||
"Denied: this session has seen data too sensitive for this call.",
|
||||
),
|
||||
(
|
||||
DenyReason::DeniedByGrant,
|
||||
"Denied: a grant forbids this call.",
|
||||
),
|
||||
(
|
||||
DenyReason::ApprovalRefused,
|
||||
"Denied: the owner refused this call.",
|
||||
),
|
||||
(
|
||||
DenyReason::ApprovalExpired,
|
||||
"Denied: the approval request expired without an answer.",
|
||||
),
|
||||
(
|
||||
DenyReason::InvalidArguments,
|
||||
"Denied: the arguments are not valid for this tool.",
|
||||
),
|
||||
(
|
||||
DenyReason::GrantsInvalid,
|
||||
"Denied: the grant files have an error; the owner has been told.",
|
||||
),
|
||||
(
|
||||
DenyReason::AuditUnavailable,
|
||||
"Denied: the audit log cannot be written; the owner has been told.",
|
||||
),
|
||||
(
|
||||
DenyReason::StateUnreadable,
|
||||
"Denied: this session's broker state is damaged; the owner has been told.",
|
||||
),
|
||||
];
|
||||
for (reason, want) in table {
|
||||
assert_eq!(denial_text(reason), want, "{reason:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn results_are_cut_on_a_character_boundary_and_marked() {
|
||||
assert_eq!(cap_result("short", 100), ("short".to_string(), false));
|
||||
assert_eq!(cap_result("exactly", 7), ("exactly".to_string(), false));
|
||||
let (text, truncated) = cap_result("abcdefghij", 4);
|
||||
assert_eq!(text, "abcd\n[truncated]");
|
||||
assert!(truncated);
|
||||
// 3 ASCII bytes, then 2-byte characters: byte 4 is inside a character.
|
||||
let (text, truncated) = cap_result("abc\u{e9}\u{e9}\u{e9}", 4);
|
||||
assert_eq!(text, "abc\n[truncated]");
|
||||
assert!(truncated);
|
||||
assert_eq!(cap_result("", 0), (String::new(), false));
|
||||
assert_eq!(cap_result("x", 0), ("\n[truncated]".to_string(), true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fake_tools_answer_echo_deny_the_rest_and_record_calls() {
|
||||
let fake = FakeTools::new();
|
||||
let mut seen = 0;
|
||||
let mut on_pending = |_: &Pending| seen += 1;
|
||||
match fake.call(&req("echo", r#"{"text":"box"}"#), &mut on_pending) {
|
||||
ToolResponse::Result {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
truncated,
|
||||
} => {
|
||||
assert_eq!(content, "box");
|
||||
assert_eq!(class, proto::DataClass::Public);
|
||||
assert!(!untrusted && !truncated);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
fake.call(&req("echo", r#"{"tex":"box"}"#), &mut on_pending),
|
||||
ToolResponse::Failed { .. }
|
||||
));
|
||||
for tool in ["weather", "read_file", "clock"] {
|
||||
assert_eq!(
|
||||
fake.call(&req(tool, "{}"), &mut on_pending),
|
||||
ToolResponse::Denied {
|
||||
reason: DenyReason::NoGrant
|
||||
},
|
||||
"{tool}: the clock is loopd's own now, not the port's"
|
||||
);
|
||||
}
|
||||
assert_eq!(seen, 0, "the fake never asks for approval");
|
||||
assert_eq!(fake.calls().len(), 5);
|
||||
assert_eq!(fake.calls()[0].tool, "echo");
|
||||
}
|
||||
@@ -0,0 +1,297 @@
|
||||
//! Tests for one turn: record sequences and tool dispatch. Do not edit.
|
||||
//! The limits and the append-only property are in `limits.rs`; denials and approvals are in
|
||||
//! `turn_broker.rs`.
|
||||
|
||||
mod support;
|
||||
#[path = "support/turn.rs"]
|
||||
mod turn_support;
|
||||
|
||||
use loopd::tools::Registry;
|
||||
use proto::{DataClass, LogRecord, SessionId, ToolResponse, TurnEvent};
|
||||
use support::{Reply, ok_result};
|
||||
use turn_support::{setup, types};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
#[test]
|
||||
fn a_plain_turn() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(CHAT, vec![Reply::fixture("plain")]);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "hello");
|
||||
let outcome = result.unwrap();
|
||||
assert_eq!(
|
||||
outcome.content,
|
||||
support::expected("plain")["content"].as_str().unwrap()
|
||||
);
|
||||
assert_eq!(
|
||||
(outcome.usage.prompt_n, outcome.usage.predicted_n),
|
||||
(46, 16)
|
||||
);
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
["start", "user", "assistant", "usage"]
|
||||
);
|
||||
assert_eq!(
|
||||
types(&s.home.records("a")),
|
||||
["start", "user", "assistant", "usage"],
|
||||
"on disk too"
|
||||
);
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|e| matches!(e, TurnEvent::Content { .. }))
|
||||
);
|
||||
assert!(
|
||||
!events
|
||||
.iter()
|
||||
.any(|e| matches!(e, TurnEvent::ToolCallStarted { .. }))
|
||||
);
|
||||
assert!(s.port.calls().is_empty());
|
||||
|
||||
let sent = s.server.requests_to(CHAT);
|
||||
assert_eq!(sent.len(), 1);
|
||||
let body = sent[0].json();
|
||||
assert_eq!(body["messages"][0]["role"], "system");
|
||||
assert_eq!(
|
||||
body["messages"][0]["content"],
|
||||
"You are Boxmaker, a test agent."
|
||||
);
|
||||
assert_eq!(
|
||||
body["messages"][1],
|
||||
serde_json::json!({"role": "user", "content": "hello"})
|
||||
);
|
||||
let names: Vec<&str> = body["tools"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|t| t["function"]["name"].as_str().unwrap())
|
||||
.collect();
|
||||
assert_eq!(names, ["clock", "find_tool", "call_tool"]);
|
||||
assert_eq!(body["id_slot"], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tool_turn_goes_through_the_port_and_records_everything() {
|
||||
let s = setup(vec![ok_result("straylight\n")]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "read the hostname");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
[
|
||||
"start",
|
||||
"user",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage"
|
||||
]
|
||||
);
|
||||
|
||||
let calls = s.port.calls();
|
||||
assert_eq!(calls.len(), 1);
|
||||
assert_eq!(calls[0].tool, "read_file");
|
||||
assert_eq!(
|
||||
calls[0].arguments, r#"{"path":"/etc/hostname"}"#,
|
||||
"arguments are passed on unparsed"
|
||||
);
|
||||
assert_eq!(calls[0].session, SessionId::new("a").unwrap());
|
||||
assert_eq!(calls[0].call, proto::CallId(1));
|
||||
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
call,
|
||||
tool_call_id,
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
truncated,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(*call, proto::CallId(1));
|
||||
assert_eq!(
|
||||
tool_call_id, "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F",
|
||||
"the server's id, so the template can pair it"
|
||||
);
|
||||
assert_eq!(content, "straylight\n");
|
||||
assert_eq!(
|
||||
(*class, *untrusted, *truncated),
|
||||
(DataClass::Private, true, false)
|
||||
);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
let kinds: Vec<&str> = events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
TurnEvent::ToolCallStarted { name } => Some(name.as_str()),
|
||||
TurnEvent::ToolResult { name, .. } => Some(name.as_str()),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(kinds, ["read_file", "read_file"]);
|
||||
|
||||
// The second request extends the first: the tool result sits after the assistant turn.
|
||||
let sent = s.server.requests_to(CHAT);
|
||||
let m2 = sent[1].json()["messages"].as_array().unwrap().clone();
|
||||
assert_eq!(m2[2]["role"], "assistant");
|
||||
assert_eq!(
|
||||
m2[2]["tool_calls"][0]["id"],
|
||||
"wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F"
|
||||
);
|
||||
assert_eq!(
|
||||
m2[3],
|
||||
serde_json::json!({"role": "tool", "tool_call_id": "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F", "content": "straylight\n"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_tool_and_call_tool_reach_the_port_only_for_the_target() {
|
||||
let s = setup(vec![ok_result("box")]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::fixture("find_tool"),
|
||||
Reply::fixture("call_tool"),
|
||||
Reply::fixture("plain"),
|
||||
],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, _) = s.turn(&mut session, "echo box");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
[
|
||||
"start",
|
||||
"user",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage"
|
||||
]
|
||||
);
|
||||
let calls = s.port.calls();
|
||||
assert_eq!(
|
||||
calls.len(),
|
||||
1,
|
||||
"find_tool is answered by loopd; only echo reaches the port"
|
||||
);
|
||||
assert_eq!(calls[0].tool, "echo");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&calls[0].arguments).unwrap(),
|
||||
serde_json::json!({"text": "box"})
|
||||
);
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
..
|
||||
} => {
|
||||
assert!(
|
||||
content.contains("\"name\":\"echo\"")
|
||||
&& content.ends_with("Call it with call_tool."),
|
||||
"{content}"
|
||||
);
|
||||
assert_eq!((*class, *untrusted), (DataClass::Public, false));
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_call_tool_for_an_unknown_tool_never_reaches_the_port() {
|
||||
// The recorded call_tool asks for "echo"; with echo removed from the registry it is unknown.
|
||||
let mut s = setup(vec![]);
|
||||
s.registry = Registry::new(vec![loopd::tools::Entry {
|
||||
schema: loopd::tools::clock_schema(),
|
||||
core: true,
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("call_tool"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
assert!(s.turn(&mut session, "x").0.is_ok());
|
||||
assert!(s.port.calls().is_empty());
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content.contains("No tool named \"echo\""))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_result_cap_applies_when_appended() {
|
||||
let mut s = setup(vec![ok_result(&"x".repeat(100))]);
|
||||
s.cfg.r#loop.tool_result_cap = 20;
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok());
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content, truncated, ..
|
||||
} => {
|
||||
assert_eq!(content, &format!("{}\n[truncated]", "x".repeat(20)));
|
||||
assert!(truncated);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(events.iter().any(|e| matches!(
|
||||
e,
|
||||
TurnEvent::ToolResult {
|
||||
truncated: true,
|
||||
..
|
||||
}
|
||||
)));
|
||||
let m2 = s.server.requests_to(CHAT)[1].json();
|
||||
assert_eq!(
|
||||
m2["messages"][3]["content"].as_str().unwrap().len(),
|
||||
20 + "\n[truncated]".len(),
|
||||
"the model sees the capped text"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_tool_failure_becomes_a_result_the_model_can_read() {
|
||||
let s = setup(vec![ToolResponse::Failed {
|
||||
message: "disk on fire".to_string(),
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(content, "The tool failed: disk on fire");
|
||||
assert_eq!((*class, *untrusted), (DataClass::Public, false));
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(
|
||||
!events
|
||||
.iter()
|
||||
.any(|e| matches!(e, TurnEvent::ToolDenied { .. })),
|
||||
"a failure is not a denial"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
//! Tests for what the turn loop does with the broker's answers: denials, pending approvals, and
|
||||
//! a port that misbehaves. Do not edit.
|
||||
|
||||
mod support;
|
||||
#[path = "support/turn.rs"]
|
||||
mod turn_support;
|
||||
|
||||
use proto::{DataClass, DenyReason, LogRecord, Timestamp, ToolResponse, TurnEvent};
|
||||
use support::{Reply, ReturnsPendingPort, ok_result, pending};
|
||||
use turn_support::{setup, types};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
/// The tool events of a turn, in order, as short strings.
|
||||
fn tool_events(events: &[TurnEvent]) -> Vec<String> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
TurnEvent::ToolCallStarted { name } => Some(format!("started {name}")),
|
||||
TurnEvent::ApprovalPending { approval, tool, .. } => {
|
||||
Some(format!("pending {approval} {tool}"))
|
||||
}
|
||||
TurnEvent::ToolDenied { name, reason } => Some(format!("denied {name} {reason:?}")),
|
||||
TurnEvent::ToolResult { name, .. } => Some(format!("result {name}")),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_denial_is_a_fixed_sentence_for_the_model_and_an_event_for_the_owner() {
|
||||
let s = setup(vec![ToolResponse::Denied {
|
||||
reason: DenyReason::TaintTooHigh,
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"the turn goes on after a denial: {result:?}"
|
||||
);
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
truncated,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
content,
|
||||
"Denied: this session has seen data too sensitive for this call."
|
||||
);
|
||||
assert_eq!(
|
||||
(*class, *untrusted, *truncated),
|
||||
(DataClass::Public, false, false)
|
||||
);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"denied read_file TaintTooHigh",
|
||||
"result read_file"
|
||||
],
|
||||
"the denial comes before the result"
|
||||
);
|
||||
// The model reads the sentence in the next request.
|
||||
let m2 = s.server.requests_to(CHAT)[1].json();
|
||||
assert_eq!(
|
||||
m2["messages"][3]["content"],
|
||||
"Denied: this session has seen data too sensitive for this call."
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_denied_call_tool_names_the_target_tool_in_the_denial() {
|
||||
let s = setup(vec![ToolResponse::Denied {
|
||||
reason: DenyReason::NoGrant,
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("call_tool"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "echo box");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started call_tool",
|
||||
"denied echo NoGrant",
|
||||
"result call_tool"
|
||||
],
|
||||
"the owner writes grants for `echo`, not for `call_tool`"
|
||||
);
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "Denied: no grant allows this call.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_approval_is_an_event_and_the_answer_after_it_is_the_result() {
|
||||
let s = setup(vec![
|
||||
pending(41, "2026-09-18T12:15:00.000Z"),
|
||||
ok_result("straylight\n"),
|
||||
]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"pending 41 read_file",
|
||||
"result read_file"
|
||||
]
|
||||
);
|
||||
let expires = events.iter().find_map(|e| match e {
|
||||
TurnEvent::ApprovalPending { expires, .. } => Some(*expires),
|
||||
_ => None,
|
||||
});
|
||||
assert_eq!(
|
||||
expires,
|
||||
Some(Timestamp::parse("2026-09-18T12:15:00.000Z").unwrap())
|
||||
);
|
||||
assert_eq!(s.port.calls().len(), 1, "one call, however long it waited");
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
[
|
||||
"start",
|
||||
"user",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage"
|
||||
],
|
||||
"waiting writes nothing to the log"
|
||||
);
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "straylight\n")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_approval_that_ends_in_a_refusal() {
|
||||
let s = setup(vec![
|
||||
pending(7, "2026-09-18T12:15:00.000Z"),
|
||||
ToolResponse::Denied {
|
||||
reason: DenyReason::ApprovalRefused,
|
||||
},
|
||||
]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"pending 7 read_file",
|
||||
"denied read_file ApprovalRefused",
|
||||
"result read_file"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "Denied: the owner refused this call.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_port_that_returns_a_pending_frame_as_its_answer_is_a_failure_not_a_decision() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let runtime = loopd::turn::Runtime {
|
||||
cfg: &s.cfg,
|
||||
client: &s.client,
|
||||
port: &ReturnsPendingPort,
|
||||
registry: &s.registry,
|
||||
};
|
||||
let mut events = Vec::new();
|
||||
let result =
|
||||
loopd::turn::run_turn(&mut session, &runtime, "x", &mut |e| events.push(e.clone()));
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
content,
|
||||
"The tool failed: the tool broker gave no final answer"
|
||||
);
|
||||
assert_eq!((*class, *untrusted), (DataClass::Public, false));
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
["started read_file", "result read_file"],
|
||||
"neither pending nor denied: the port said neither"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user