Task files, the files they copy in (byte-identical to the reference on m3a-ref), each area's check record, and a README with the per-task table of what each check exposed. The handoff note is done with. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
223 lines
6.6 KiB
Rust
223 lines
6.6 KiB
Rust
//! Tests for what the turn loop does with the broker's answers: denials, pending approvals, and
|
|
//! a port that misbehaves. Do not edit.
|
|
|
|
mod support;
|
|
#[path = "support/turn.rs"]
|
|
mod turn_support;
|
|
|
|
use proto::{DataClass, DenyReason, LogRecord, Timestamp, ToolResponse, TurnEvent};
|
|
use support::{Reply, ReturnsPendingPort, ok_result, pending};
|
|
use turn_support::{setup, types};
|
|
|
|
const CHAT: &str = "/v1/chat/completions";
|
|
|
|
/// The tool events of a turn, in order, as short strings.
|
|
fn tool_events(events: &[TurnEvent]) -> Vec<String> {
|
|
events
|
|
.iter()
|
|
.filter_map(|e| match e {
|
|
TurnEvent::ToolCallStarted { name } => Some(format!("started {name}")),
|
|
TurnEvent::ApprovalPending { approval, tool, .. } => {
|
|
Some(format!("pending {approval} {tool}"))
|
|
}
|
|
TurnEvent::ToolDenied { name, reason } => Some(format!("denied {name} {reason:?}")),
|
|
TurnEvent::ToolResult { name, .. } => Some(format!("result {name}")),
|
|
_ => None,
|
|
})
|
|
.collect()
|
|
}
|
|
|
|
#[test]
|
|
fn a_denial_is_a_fixed_sentence_for_the_model_and_an_event_for_the_owner() {
|
|
let s = setup(vec![ToolResponse::Denied {
|
|
reason: DenyReason::TaintTooHigh,
|
|
}]);
|
|
s.server.route(
|
|
CHAT,
|
|
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
|
);
|
|
let mut session = s.session("a");
|
|
let (result, events) = s.turn(&mut session, "x");
|
|
assert!(
|
|
result.is_ok(),
|
|
"the turn goes on after a denial: {result:?}"
|
|
);
|
|
match &session.records()[4] {
|
|
LogRecord::ToolResult {
|
|
content,
|
|
class,
|
|
untrusted,
|
|
truncated,
|
|
..
|
|
} => {
|
|
assert_eq!(
|
|
content,
|
|
"Denied: this session has seen data too sensitive for this call."
|
|
);
|
|
assert_eq!(
|
|
(*class, *untrusted, *truncated),
|
|
(DataClass::Public, false, false)
|
|
);
|
|
}
|
|
other => panic!("{other:?}"),
|
|
}
|
|
assert_eq!(
|
|
tool_events(&events),
|
|
[
|
|
"started read_file",
|
|
"denied read_file TaintTooHigh",
|
|
"result read_file"
|
|
],
|
|
"the denial comes before the result"
|
|
);
|
|
// The model reads the sentence in the next request.
|
|
let m2 = s.server.requests_to(CHAT)[1].json();
|
|
assert_eq!(
|
|
m2["messages"][3]["content"],
|
|
"Denied: this session has seen data too sensitive for this call."
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_denied_call_tool_names_the_target_tool_in_the_denial() {
|
|
let s = setup(vec![ToolResponse::Denied {
|
|
reason: DenyReason::NoGrant,
|
|
}]);
|
|
s.server.route(
|
|
CHAT,
|
|
vec![Reply::fixture("call_tool"), Reply::fixture("plain")],
|
|
);
|
|
let mut session = s.session("a");
|
|
let (result, events) = s.turn(&mut session, "echo box");
|
|
assert!(result.is_ok(), "{result:?}");
|
|
assert_eq!(
|
|
tool_events(&events),
|
|
[
|
|
"started call_tool",
|
|
"denied echo NoGrant",
|
|
"result call_tool"
|
|
],
|
|
"the owner writes grants for `echo`, not for `call_tool`"
|
|
);
|
|
assert!(
|
|
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "Denied: no grant allows this call.")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_pending_approval_is_an_event_and_the_answer_after_it_is_the_result() {
|
|
let s = setup(vec![
|
|
pending(41, "2026-09-18T12:15:00.000Z"),
|
|
ok_result("straylight\n"),
|
|
]);
|
|
s.server.route(
|
|
CHAT,
|
|
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
|
);
|
|
let mut session = s.session("a");
|
|
let (result, events) = s.turn(&mut session, "x");
|
|
assert!(result.is_ok(), "{result:?}");
|
|
assert_eq!(
|
|
tool_events(&events),
|
|
[
|
|
"started read_file",
|
|
"pending 41 read_file",
|
|
"result read_file"
|
|
]
|
|
);
|
|
let expires = events.iter().find_map(|e| match e {
|
|
TurnEvent::ApprovalPending { expires, .. } => Some(*expires),
|
|
_ => None,
|
|
});
|
|
assert_eq!(
|
|
expires,
|
|
Some(Timestamp::parse("2026-09-18T12:15:00.000Z").unwrap())
|
|
);
|
|
assert_eq!(s.port.calls().len(), 1, "one call, however long it waited");
|
|
assert_eq!(
|
|
types(session.records()),
|
|
[
|
|
"start",
|
|
"user",
|
|
"assistant",
|
|
"usage",
|
|
"tool_result",
|
|
"assistant",
|
|
"usage"
|
|
],
|
|
"waiting writes nothing to the log"
|
|
);
|
|
assert!(
|
|
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "straylight\n")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_pending_approval_that_ends_in_a_refusal() {
|
|
let s = setup(vec![
|
|
pending(7, "2026-09-18T12:15:00.000Z"),
|
|
ToolResponse::Denied {
|
|
reason: DenyReason::ApprovalRefused,
|
|
},
|
|
]);
|
|
s.server.route(
|
|
CHAT,
|
|
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
|
);
|
|
let mut session = s.session("a");
|
|
let (result, events) = s.turn(&mut session, "x");
|
|
assert!(result.is_ok(), "{result:?}");
|
|
assert_eq!(
|
|
tool_events(&events),
|
|
[
|
|
"started read_file",
|
|
"pending 7 read_file",
|
|
"denied read_file ApprovalRefused",
|
|
"result read_file"
|
|
]
|
|
);
|
|
assert!(
|
|
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content == "Denied: the owner refused this call.")
|
|
);
|
|
}
|
|
|
|
#[test]
|
|
fn a_port_that_returns_a_pending_frame_as_its_answer_is_a_failure_not_a_decision() {
|
|
let s = setup(vec![]);
|
|
s.server.route(
|
|
CHAT,
|
|
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
|
|
);
|
|
let mut session = s.session("a");
|
|
let runtime = loopd::turn::Runtime {
|
|
cfg: &s.cfg,
|
|
client: &s.client,
|
|
port: &ReturnsPendingPort,
|
|
registry: &s.registry,
|
|
};
|
|
let mut events = Vec::new();
|
|
let result =
|
|
loopd::turn::run_turn(&mut session, &runtime, "x", &mut |e| events.push(e.clone()));
|
|
assert!(result.is_ok(), "{result:?}");
|
|
match &session.records()[4] {
|
|
LogRecord::ToolResult {
|
|
content,
|
|
class,
|
|
untrusted,
|
|
..
|
|
} => {
|
|
assert_eq!(
|
|
content,
|
|
"The tool failed: the tool broker gave no final answer"
|
|
);
|
|
assert_eq!((*class, *untrusted), (DataClass::Public, false));
|
|
}
|
|
other => panic!("{other:?}"),
|
|
}
|
|
assert_eq!(
|
|
tool_events(&events),
|
|
["started read_file", "result read_file"],
|
|
"neither pending nor denied: the port said neither"
|
|
);
|
|
}
|