M5b plan: task 01, the invalid-arguments detail
Task file and 13 given files (2 new: the wire fixture with a detail and brokerd/tests/invalid_detail.rs; 11 new versions of existing tests, most only adding `detail: None`). Checked against a reference in a scratch worktree with its own target/, on titan (rustc 1.98.1, clippy 0.1.98): make gate prints gate: ok with 956 tests (944 + 12 new); /tmp has no new entries after the gate. All 21 quoted before-blocks occur once in master, and applying the edits to master gives the reference byte for byte. The given files are the reference's, formatted, none over 500 lines; no chat-template tag in the plan. Before the edits, strict's envelopes_reject_unknown_keys_at_every_depth fails and ten test targets do not compile, as step 2 lists. Mutations: 21 one-line breaks of the reference; 20 caught by the given tests. The one not caught (dropping `default` from the detail attribute) cannot change behaviour: serde reads a missing Option field as None. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -0,0 +1,321 @@
|
||||
//! Tests for `BrokerPort` against a fake `brokerd` on a temporary socket: answers, approvals and
|
||||
//! timeouts. Frames that break the protocol are in `broker_port_bad.rs`. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use bxcore::{
|
||||
DenyReason, Message, PROTOCOL_VERSION, Timestamp, ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send, socket_path};
|
||||
use loopd::broker_port::{BrokerPort, NoBroker, POINTER, TOO_LARGE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
|
||||
#[test]
|
||||
fn a_result_comes_back_as_it_was_sent() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("hello\n"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("hello\n"));
|
||||
assert!(got.pending.is_empty());
|
||||
assert!(got.lines.is_empty(), "nothing to report: {:?}", got.lines);
|
||||
|
||||
// What the broker received: one final frame holding exactly the request.
|
||||
let sent = broker.join().unwrap();
|
||||
assert_eq!(sent.v, PROTOCOL_VERSION);
|
||||
assert!(sent.r#final, "a request is a single final frame");
|
||||
assert_eq!(sent.msg, Message::ToolRequest(request()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_denial_and_a_failure_come_back_as_they_were_sent() {
|
||||
let mut answers: Vec<ToolResponse> = [
|
||||
DenyReason::NoGrant,
|
||||
DenyReason::GrantExpired,
|
||||
DenyReason::TaintTooHigh,
|
||||
DenyReason::DeniedByGrant,
|
||||
DenyReason::ApprovalRefused,
|
||||
DenyReason::ApprovalExpired,
|
||||
DenyReason::GrantsInvalid,
|
||||
DenyReason::AuditUnavailable,
|
||||
DenyReason::InvalidArguments,
|
||||
DenyReason::StateUnreadable,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|reason| ToolResponse::Denied {
|
||||
reason,
|
||||
detail: None,
|
||||
})
|
||||
.collect();
|
||||
answers.push(ToolResponse::Denied {
|
||||
reason: DenyReason::InvalidArguments,
|
||||
detail: Some("invalid path: n/a".to_string()),
|
||||
});
|
||||
answers.push(ToolResponse::Failed {
|
||||
message: "the runner arrives in M3b".to_string(),
|
||||
});
|
||||
for answer in answers {
|
||||
let reply = answer.clone();
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
send(stream, request.id, true, reply);
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, answer);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"a denial is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_frame_reaches_the_callback_and_the_final_frame_is_the_answer() {
|
||||
let expires = in_ms(60_000);
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 41,
|
||||
expires,
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(150));
|
||||
send(stream, request.id, true, result("approved"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("approved"));
|
||||
assert_eq!(
|
||||
got.pending,
|
||||
[Pending {
|
||||
approval: 41,
|
||||
expires
|
||||
}],
|
||||
"called once, with the frame's values"
|
||||
);
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_after_expires_but_within_the_timeout_of_it_is_delivered() {
|
||||
// expires in 100 ms, timeout 1,500 ms: the port waits until 1,600 ms. The answer comes at
|
||||
// about 400 ms, well after `expires`: an approval given at the last moment still gets the
|
||||
// whole timeout to run.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: in_ms(100),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(400));
|
||||
send(stream, request.id, true, result("late but good"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("late but good"));
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_expiry_that_has_already_passed_still_leaves_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: Timestamp::parse("2020-01-01T00:00:00.000Z").unwrap(),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
send(stream, request.id, true, result("fine"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("fine"));
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_socket_is_unavailable() {
|
||||
let socket = socket_path();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let got = call(socket.to_path_buf(), 2_000, &request());
|
||||
assert_unavailable(&got, "no socket file");
|
||||
assert!(
|
||||
got.lines[0].contains(&socket.display().to_string()),
|
||||
"the line names the socket: {}",
|
||||
got.lines[0]
|
||||
);
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_without_answering_is_unavailable() {
|
||||
let (socket, broker) = broker(|_, _| {});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed before any frame");
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_500),
|
||||
"a close is seen at once: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_while_pending_is_unavailable() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
// brokerd was restarted: the connection just ends.
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed while pending");
|
||||
assert_eq!(got.pending.len(), 1, "the pending frame was reported first");
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_never_answers_is_unavailable_after_the_timeout() {
|
||||
let (socket, broker) = broker(|_, _| thread::sleep(Duration::from_millis(1_200)));
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(250),
|
||||
"gave up early: {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_100),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_call_nobody_answers_ends_at_expires_plus_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(300),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(1_800));
|
||||
});
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence while pending");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(550),
|
||||
"it must wait for expires (300) plus the timeout (300): {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_700),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_timeout_is_a_deadline_for_the_frame_not_for_each_read() {
|
||||
// The four header bytes arrive 250 ms apart, the last at 1,000 ms. A port that only sets a
|
||||
// 600 ms read timeout sees every single read succeed and returns the result; a port with a
|
||||
// deadline gives up at 600 ms. The answer tells them apart, so no timing is asserted.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&frame(request.id, true, Message::ToolResponse(result("slow"))),
|
||||
)
|
||||
.unwrap();
|
||||
for byte in bytes.iter().take(4) {
|
||||
thread::sleep(Duration::from_millis(250));
|
||||
if stream.write_all(&[*byte]).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = stream.write_all(&bytes[4..]);
|
||||
});
|
||||
let got = call(socket, 600, &request());
|
||||
assert_unavailable(&got, "a trickled frame");
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_timeout_fails_closed_and_does_not_panic() {
|
||||
let (socket, _broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("too late"));
|
||||
});
|
||||
let got = call(socket, 0, &request());
|
||||
assert_unavailable(&got, "timeout_ms = 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_too_large_for_a_frame_is_its_own_failure() {
|
||||
// Nothing is sent, so the fake broker sees a connection that closes or none at all.
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _listener = UnixListener::bind(&path).unwrap();
|
||||
let mut big = request();
|
||||
big.arguments = format!(r#"{{"path":"/{}"}}"#, "a".repeat(bxcore::MAX_FRAME));
|
||||
let got = call(path.to_path_buf(), 1_000, &big);
|
||||
assert_eq!(
|
||||
got.response,
|
||||
ToolResponse::Failed {
|
||||
message: TOO_LARGE.to_string()
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"the broker is fine; this is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_call_is_its_own_connection() {
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
for n in 0..3u64 {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
send(
|
||||
&mut stream,
|
||||
request.id,
|
||||
true,
|
||||
result(&format!("answer {n}")),
|
||||
);
|
||||
}
|
||||
});
|
||||
let port = BrokerPort::new(path.to_path_buf(), Duration::from_millis(2_000));
|
||||
for n in 0..3 {
|
||||
let got = port.call(&request(), &mut |_| {});
|
||||
assert_eq!(got, result(&format!("answer {n}")));
|
||||
}
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_broker_every_call_fails_in_words_the_model_can_repeat() {
|
||||
let mut seen = 0;
|
||||
let got = NoBroker.call(&request(), &mut |_| seen += 1);
|
||||
assert_eq!(
|
||||
got,
|
||||
ToolResponse::Failed {
|
||||
message: "no tool broker is configured".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(seen, 0);
|
||||
assert_eq!(POINTER, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
@@ -0,0 +1,335 @@
|
||||
//! Tests for the registry, dispatch, the denial sentences, the result cap and the fake tools.
|
||||
//! Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use bxcore::{CallId, DenyReason, SessionId, ToolRequest, ToolResponse};
|
||||
use loopd::tools::{
|
||||
Dispatch, FakeTools, Pending, Registry, ToolPort, cap_result, denial_message, denial_text,
|
||||
dispatch,
|
||||
};
|
||||
|
||||
fn req(tool: &str, arguments: &str) -> ToolRequest {
|
||||
ToolRequest {
|
||||
session: SessionId::new("s").unwrap(),
|
||||
call: CallId(1),
|
||||
tool: tool.to_string(),
|
||||
arguments: arguments.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_core_schemas_are_the_fixed_tools_array() {
|
||||
let names: Vec<String> = Registry::m2b()
|
||||
.core_schemas()
|
||||
.into_iter()
|
||||
.map(|s| s.name)
|
||||
.collect();
|
||||
assert_eq!(names, ["clock", "find_tool", "call_tool"]);
|
||||
let find = &Registry::m2b().core_schemas()[1];
|
||||
assert_eq!(find.parameters["required"], serde_json::json!(["query"]));
|
||||
let call = &Registry::m2b().core_schemas()[2];
|
||||
assert_eq!(
|
||||
call.parameters["required"],
|
||||
serde_json::json!(["name", "arguments"])
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_matches_name_or_description_case_insensitively() {
|
||||
let r = Registry::m2b();
|
||||
let names = |q: &str| r.find(q).iter().map(|s| s.name.clone()).collect::<Vec<_>>();
|
||||
assert_eq!(names("echo"), ["echo"]);
|
||||
assert_eq!(names("ECHO"), ["echo"]);
|
||||
assert_eq!(names("unchanged"), ["echo"], "matches the description too");
|
||||
assert_eq!(names("time"), ["clock"]);
|
||||
assert!(names("weather").is_empty());
|
||||
assert!(
|
||||
names("").is_empty(),
|
||||
"an empty query matches nothing, not everything"
|
||||
);
|
||||
assert!(names(" ").is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn find_tool_is_answered_locally() {
|
||||
let r = Registry::m2b();
|
||||
match dispatch(&r, "find_tool", r#"{"query":"echo"}"#) {
|
||||
Dispatch::Local(text) => {
|
||||
assert!(text.starts_with("1 tool(s) match:\n"), "{text}");
|
||||
assert!(text.contains("\"name\":\"echo\""), "{text}");
|
||||
assert!(
|
||||
text.contains("\"parameters\""),
|
||||
"the schema is included: {text}"
|
||||
);
|
||||
assert!(text.ends_with("Call it with call_tool."), "{text}");
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
dispatch(&r, "find_tool", r#"{"query":"weather"}"#),
|
||||
Dispatch::Local("No tool matches.".to_string())
|
||||
);
|
||||
assert!(
|
||||
matches!(dispatch(&r, "find_tool", "not json"), Dispatch::Local(t) if t == "No tool matches.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn call_tool_unwraps_a_known_non_core_tool_and_nothing_else() {
|
||||
let r = Registry::m2b();
|
||||
let args = r#"{"name":"echo","arguments":{"text": "box"}}"#;
|
||||
match dispatch(&r, "call_tool", args) {
|
||||
Dispatch::Port { tool, arguments } => {
|
||||
assert_eq!(tool, "echo");
|
||||
assert_eq!(
|
||||
serde_json::from_str::<serde_json::Value>(&arguments).unwrap(),
|
||||
serde_json::json!({"text": "box"})
|
||||
);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
let cases = [
|
||||
(
|
||||
r#"{"name":"weather","arguments":{}}"#,
|
||||
"No tool named \"weather\"",
|
||||
),
|
||||
(r#"{"name":"clock","arguments":{}}"#, "core tool"),
|
||||
(r#"{"arguments":{}}"#, "No tool named \"\""),
|
||||
("not json", "needs a JSON object"),
|
||||
];
|
||||
for (args, want) in cases {
|
||||
match dispatch(&r, "call_tool", args) {
|
||||
Dispatch::Local(text) => assert!(text.contains(want), "{args}: {text}"),
|
||||
other => panic!("{args}: {other:?}, must never reach the port"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn any_other_tool_goes_to_the_port_as_it_is() {
|
||||
let r = Registry::m2b();
|
||||
// Even one the registry does not know: the port (brokerd) decides, not loopd.
|
||||
let want = Dispatch::Port {
|
||||
tool: "read_file".to_string(),
|
||||
arguments: r#"{"path":"/x"}"#.to_string(),
|
||||
};
|
||||
assert_eq!(dispatch(&r, "read_file", r#"{"path":"/x"}"#), want);
|
||||
let want = Dispatch::Port {
|
||||
tool: "weather".to_string(),
|
||||
arguments: "not json".to_string(),
|
||||
};
|
||||
assert_eq!(dispatch(&r, "weather", "not json"), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_clock_is_answered_locally_whatever_its_arguments() {
|
||||
for registry in [Registry::m2b(), Registry::m3a(), Registry::new(vec![])] {
|
||||
for arguments in ["{}", r#"{"zone":"UTC"}"#, "not json", ""] {
|
||||
let before = bxcore::Timestamp::now();
|
||||
match dispatch(®istry, "clock", arguments) {
|
||||
Dispatch::Local(text) => {
|
||||
let time = bxcore::Timestamp::parse(&text)
|
||||
.unwrap_or_else(|e| panic!("an RFC 3339 time, got {text:?}: {e:?}"));
|
||||
assert!(time >= before, "{text}");
|
||||
assert!(text.ends_with('Z'), "UTC: {text}");
|
||||
}
|
||||
other => panic!("{arguments:?}: {other:?}, the clock must not reach the port"),
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_m3a_registry_has_the_same_core_and_the_four_broker_tools() {
|
||||
let r = Registry::m3a();
|
||||
assert_eq!(
|
||||
r.core_schemas(),
|
||||
Registry::m2b().core_schemas(),
|
||||
"the tools array is part of the baseline: it must not change"
|
||||
);
|
||||
let names = |q: &str| r.find(q).iter().map(|s| s.name.clone()).collect::<Vec<_>>();
|
||||
assert_eq!(names("file"), ["read_file", "write_file"]);
|
||||
assert_eq!(names("shell"), ["shell"]);
|
||||
assert_eq!(names("fetch"), ["http_fetch"]);
|
||||
assert_eq!(names("https"), ["http_fetch"]);
|
||||
assert!(names("echo").is_empty(), "echo is a test tool only");
|
||||
|
||||
// The argument schemas are section 3's table: exactly these properties, all strings.
|
||||
let table: [(&str, &[&str], &[&str]); 4] = [
|
||||
("read_file", &["path"], &["path"]),
|
||||
("write_file", &["content", "path"], &["path", "content"]),
|
||||
("shell", &["command", "cwd"], &["command"]),
|
||||
("http_fetch", &["url"], &["url"]),
|
||||
];
|
||||
for (name, properties, required) in table {
|
||||
let entry = r.get(name).unwrap_or_else(|| panic!("{name} is missing"));
|
||||
assert!(!entry.core, "{name} is found with find_tool, not declared");
|
||||
let p = &entry.schema.parameters;
|
||||
assert_eq!(p["type"], "object", "{name}");
|
||||
let mut got: Vec<&str> = p["properties"]
|
||||
.as_object()
|
||||
.unwrap()
|
||||
.keys()
|
||||
.map(String::as_str)
|
||||
.collect();
|
||||
got.sort_unstable();
|
||||
assert_eq!(got, properties, "{name}: properties");
|
||||
for property in properties {
|
||||
assert_eq!(
|
||||
p["properties"][property]["type"], "string",
|
||||
"{name}.{property}"
|
||||
);
|
||||
assert!(
|
||||
p["properties"][property]["description"].is_string(),
|
||||
"{name}.{property} needs a description"
|
||||
);
|
||||
}
|
||||
assert_eq!(
|
||||
p["required"],
|
||||
serde_json::json!(required),
|
||||
"{name}: required"
|
||||
);
|
||||
}
|
||||
|
||||
// call_tool lets the four through to the port and nothing else.
|
||||
match dispatch(
|
||||
&r,
|
||||
"call_tool",
|
||||
r#"{"name":"shell","arguments":{"command":"ls"}}"#,
|
||||
) {
|
||||
Dispatch::Port { tool, arguments } => {
|
||||
assert_eq!(tool, "shell");
|
||||
assert_eq!(arguments, r#"{"command":"ls"}"#);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
dispatch(&r, "call_tool", r#"{"name":"echo","arguments":{}}"#),
|
||||
Dispatch::Local(t) if t.contains("No tool named \"echo\"")
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_deny_reason_has_its_sentence() {
|
||||
let table = [
|
||||
(DenyReason::NoGrant, "Denied: no grant allows this call."),
|
||||
(
|
||||
DenyReason::GrantExpired,
|
||||
"Denied: the grant for this call has expired.",
|
||||
),
|
||||
(
|
||||
DenyReason::TaintTooHigh,
|
||||
"Denied: this session has seen data too sensitive for this call.",
|
||||
),
|
||||
(
|
||||
DenyReason::DeniedByGrant,
|
||||
"Denied: a grant forbids this call.",
|
||||
),
|
||||
(
|
||||
DenyReason::ApprovalRefused,
|
||||
"Denied: the owner refused this call.",
|
||||
),
|
||||
(
|
||||
DenyReason::ApprovalExpired,
|
||||
"Denied: the approval request expired without an answer.",
|
||||
),
|
||||
(
|
||||
DenyReason::InvalidArguments,
|
||||
"Denied: the arguments are not valid for this tool.",
|
||||
),
|
||||
(
|
||||
DenyReason::GrantsInvalid,
|
||||
"Denied: the grant files have an error; the owner has been told.",
|
||||
),
|
||||
(
|
||||
DenyReason::AuditUnavailable,
|
||||
"Denied: the audit log cannot be written; the owner has been told.",
|
||||
),
|
||||
(
|
||||
DenyReason::StateUnreadable,
|
||||
"Denied: this session's broker state is damaged; the owner has been told.",
|
||||
),
|
||||
];
|
||||
for (reason, want) in table {
|
||||
assert_eq!(denial_text(reason), want, "{reason:?}");
|
||||
assert_eq!(
|
||||
denial_message(reason, None),
|
||||
want,
|
||||
"{reason:?}: without a detail, the sentence alone"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// The broker's detail goes after the sentence, before its full stop.
|
||||
#[test]
|
||||
fn a_denial_with_a_detail_appends_it_to_the_sentence() {
|
||||
assert_eq!(
|
||||
denial_message(
|
||||
DenyReason::InvalidArguments,
|
||||
Some("invalid argument shape: unknown field `file`, expected `text`")
|
||||
),
|
||||
"Denied: the arguments are not valid for this tool: invalid argument shape: unknown field `file`, expected `text`."
|
||||
);
|
||||
assert_eq!(
|
||||
denial_message(DenyReason::InvalidArguments, Some("invalid path: n/a")),
|
||||
"Denied: the arguments are not valid for this tool: invalid path: n/a."
|
||||
);
|
||||
assert_eq!(
|
||||
denial_message(DenyReason::NoGrant, Some("x")),
|
||||
"Denied: no grant allows this call: x.",
|
||||
"any reason's detail is shown the same way"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn results_are_cut_on_a_character_boundary_and_marked() {
|
||||
assert_eq!(cap_result("short", 100), ("short".to_string(), false));
|
||||
assert_eq!(cap_result("exactly", 7), ("exactly".to_string(), false));
|
||||
let (text, truncated) = cap_result("abcdefghij", 4);
|
||||
assert_eq!(text, "abcd\n[truncated]");
|
||||
assert!(truncated);
|
||||
// 3 ASCII bytes, then 2-byte characters: byte 4 is inside a character.
|
||||
let (text, truncated) = cap_result("abc\u{e9}\u{e9}\u{e9}", 4);
|
||||
assert_eq!(text, "abc\n[truncated]");
|
||||
assert!(truncated);
|
||||
assert_eq!(cap_result("", 0), (String::new(), false));
|
||||
assert_eq!(cap_result("x", 0), ("\n[truncated]".to_string(), true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn fake_tools_answer_echo_deny_the_rest_and_record_calls() {
|
||||
let fake = FakeTools::new();
|
||||
let mut seen = 0;
|
||||
let mut on_pending = |_: &Pending| seen += 1;
|
||||
match fake.call(&req("echo", r#"{"text":"box"}"#), &mut on_pending) {
|
||||
ToolResponse::Result {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
truncated,
|
||||
} => {
|
||||
assert_eq!(content, "box");
|
||||
assert_eq!(class, bxcore::DataClass::Public);
|
||||
assert!(!untrusted && !truncated);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert!(matches!(
|
||||
fake.call(&req("echo", r#"{"tex":"box"}"#), &mut on_pending),
|
||||
ToolResponse::Failed { .. }
|
||||
));
|
||||
for tool in ["weather", "read_file", "clock"] {
|
||||
assert_eq!(
|
||||
fake.call(&req(tool, "{}"), &mut on_pending),
|
||||
ToolResponse::Denied {
|
||||
reason: DenyReason::NoGrant,
|
||||
detail: None,
|
||||
},
|
||||
"{tool}: the clock is loopd's own now, not the port's"
|
||||
);
|
||||
}
|
||||
assert_eq!(seen, 0, "the fake never asks for approval");
|
||||
assert_eq!(fake.calls().len(), 5);
|
||||
assert_eq!(fake.calls()[0].tool, "echo");
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
//! Tests for what the turn loop does with the broker's answers: denials, pending approvals, and
|
||||
//! a port that misbehaves. Do not edit.
|
||||
|
||||
mod support;
|
||||
#[path = "support/turn.rs"]
|
||||
mod turn_support;
|
||||
|
||||
use bxcore::{DataClass, DenyReason, LogRecord, Timestamp, ToolResponse, TurnEvent};
|
||||
use support::{Reply, ReturnsPendingPort, ok_result, pending};
|
||||
use turn_support::{setup, types};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
/// The tool events of a turn, in order, as short strings.
|
||||
fn tool_events(events: &[TurnEvent]) -> Vec<String> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
TurnEvent::ToolCallStarted { name } => Some(format!("started {name}")),
|
||||
TurnEvent::ApprovalPending { approval, tool, .. } => {
|
||||
Some(format!("pending {approval} {tool}"))
|
||||
}
|
||||
TurnEvent::ToolDenied { name, reason } => Some(format!("denied {name} {reason:?}")),
|
||||
TurnEvent::ToolResult { name, .. } => Some(format!("result {name}")),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_denial_is_a_fixed_sentence_for_the_model_and_an_event_for_the_owner() {
|
||||
let s = setup(vec![ToolResponse::Denied {
|
||||
reason: DenyReason::TaintTooHigh,
|
||||
detail: None,
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(
|
||||
result.is_ok(),
|
||||
"the turn goes on after a denial: {result:?}"
|
||||
);
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
truncated,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
content,
|
||||
"Denied: this session has seen data too sensitive for this call."
|
||||
);
|
||||
assert_eq!(
|
||||
(*class, *untrusted, *truncated),
|
||||
(DataClass::Public, false, false)
|
||||
);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"denied read_file TaintTooHigh",
|
||||
"result read_file"
|
||||
],
|
||||
"the denial comes before the result"
|
||||
);
|
||||
// The model reads the sentence in the next request.
|
||||
let m2 = s.server.requests_to(CHAT)[1].json();
|
||||
assert_eq!(
|
||||
m2["messages"][3]["content"],
|
||||
"Denied: this session has seen data too sensitive for this call."
|
||||
);
|
||||
}
|
||||
|
||||
/// The broker's detail reaches the model, in the log and in the next request; the owner's event
|
||||
/// is the same as for any denial.
|
||||
#[test]
|
||||
fn an_invalid_call_tells_the_model_why() {
|
||||
let s = setup(vec![ToolResponse::Denied {
|
||||
reason: DenyReason::InvalidArguments,
|
||||
detail: Some("invalid path: n/a".to_string()),
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
let want = "Denied: the arguments are not valid for this tool: invalid path: n/a.";
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
truncated,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(content, want);
|
||||
assert_eq!(
|
||||
(*class, *untrusted, *truncated),
|
||||
(DataClass::Public, false, false)
|
||||
);
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"denied read_file InvalidArguments",
|
||||
"result read_file"
|
||||
]
|
||||
);
|
||||
let m2 = s.server.requests_to(CHAT)[1].json();
|
||||
assert_eq!(m2["messages"][3]["content"], want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_denied_call_tool_names_the_target_tool_in_the_denial() {
|
||||
let s = setup(vec![ToolResponse::Denied {
|
||||
reason: DenyReason::NoGrant,
|
||||
detail: None,
|
||||
}]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("call_tool"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "echo box");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started call_tool",
|
||||
"denied echo NoGrant",
|
||||
"result call_tool"
|
||||
],
|
||||
"the owner writes grants for `echo`, not for `call_tool`"
|
||||
);
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "Denied: no grant allows this call.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_approval_is_an_event_and_the_answer_after_it_is_the_result() {
|
||||
let s = setup(vec![
|
||||
pending(41, "2026-09-18T12:15:00.000Z"),
|
||||
ok_result("straylight\n"),
|
||||
]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"pending 41 read_file",
|
||||
"result read_file"
|
||||
]
|
||||
);
|
||||
let expires = events.iter().find_map(|e| match e {
|
||||
TurnEvent::ApprovalPending { expires, .. } => Some(*expires),
|
||||
_ => None,
|
||||
});
|
||||
assert_eq!(
|
||||
expires,
|
||||
Some(Timestamp::parse("2026-09-18T12:15:00.000Z").unwrap())
|
||||
);
|
||||
assert_eq!(s.port.calls().len(), 1, "one call, however long it waited");
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
[
|
||||
"start",
|
||||
"user",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage"
|
||||
],
|
||||
"waiting writes nothing to the log"
|
||||
);
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "straylight\n")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_approval_that_ends_in_a_refusal() {
|
||||
let s = setup(vec![
|
||||
pending(7, "2026-09-18T12:15:00.000Z"),
|
||||
ToolResponse::Denied {
|
||||
reason: DenyReason::ApprovalRefused,
|
||||
detail: None,
|
||||
},
|
||||
]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
[
|
||||
"started read_file",
|
||||
"pending 7 read_file",
|
||||
"denied read_file ApprovalRefused",
|
||||
"result read_file"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "Denied: the owner refused this call.")
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_port_that_returns_a_pending_frame_as_its_answer_is_a_failure_not_a_decision() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let runtime = loopd::turn::Runtime {
|
||||
cfg: &s.cfg,
|
||||
client: &s.client,
|
||||
port: &ReturnsPendingPort,
|
||||
registry: &s.registry,
|
||||
};
|
||||
let mut events = Vec::new();
|
||||
let result =
|
||||
loopd::turn::run_turn(&mut session, &runtime, "x", &mut |e| events.push(e.clone()));
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
match &session.records()[4] {
|
||||
LogRecord::ToolResult {
|
||||
content,
|
||||
class,
|
||||
untrusted,
|
||||
..
|
||||
} => {
|
||||
assert_eq!(
|
||||
content,
|
||||
"The tool failed: the tool broker gave no final answer"
|
||||
);
|
||||
assert_eq!((*class, *untrusted), (DataClass::Public, false));
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
tool_events(&events),
|
||||
["started read_file", "result read_file"],
|
||||
"neither pending nor denied: the port said neither"
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user