Add BrokerPort: loopd asks brokerd for every tool call
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -25,6 +25,7 @@ pub enum BaselineError {
|
||||
Read(PathBuf, std::io::Error),
|
||||
Parse(PathBuf, serde_json::Error),
|
||||
Hash,
|
||||
Core(PathBuf, std::io::Error),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for BaselineError {
|
||||
@@ -33,6 +34,11 @@ impl std::fmt::Display for BaselineError {
|
||||
BaselineError::Read(path, err) => write!(f, "{}: {err}", path.display()),
|
||||
BaselineError::Parse(path, err) => write!(f, "{}: {err}", path.display()),
|
||||
BaselineError::Hash => write!(f, "hashing the baseline failed"),
|
||||
BaselineError::Core(path, err) => write!(
|
||||
f,
|
||||
"{}: {err}; see docs/runbook.md#core-memory-unreadable",
|
||||
path.display()
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -58,7 +64,7 @@ impl Baseline {
|
||||
// A missing file is fine; an unreadable one is reported so the owner is not given a
|
||||
// session without the memory they curated and no sign of it.
|
||||
Err(e) if e.kind() != std::io::ErrorKind::NotFound => {
|
||||
return Err(BaselineError::Read(core.clone(), e));
|
||||
return Err(BaselineError::Core(core.clone(), e));
|
||||
}
|
||||
Err(_) => {}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
//! The tool port to `brokerd`: one connection per call, a deadline per frame, and plain failures.
|
||||
//!
|
||||
//! Whatever goes wrong reaching `brokerd` or reading its frames, the model gets a plain failure and
|
||||
//! the turn goes on. Every failure prints one line ending at `#broker-unavailable`.
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::PathBuf;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use crate::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
Envelope, FrameError, Message, PROTOCOL_VERSION, Timestamp, ToolRequest, ToolResponse,
|
||||
read_frame, write_frame,
|
||||
};
|
||||
|
||||
/// The failure every outage reaches the model as.
|
||||
pub const UNAVAILABLE: &str = "the tool broker is unavailable";
|
||||
/// What the model is told when no broker is configured.
|
||||
pub const NOT_CONFIGURED: &str = "no tool broker is configured";
|
||||
/// The failure a request too big for one frame reaches.
|
||||
pub const TOO_LARGE: &str = "the request is too large for the tool broker";
|
||||
/// The runbook entry every outage line ends with.
|
||||
pub const POINTER: &str = "see docs/runbook.md#broker-unavailable";
|
||||
|
||||
/// "loopd: {UNAVAILABLE}: {why}; {POINTER}"
|
||||
pub fn unavailable_line(why: &str) -> String {
|
||||
format!("loopd: {UNAVAILABLE}: {why}; {POINTER}")
|
||||
}
|
||||
|
||||
/// "loopd: {NOT_CONFIGURED}: every tool call except the core ones will fail; {POINTER}"
|
||||
pub fn not_configured_line() -> String {
|
||||
format!("loopd: {NOT_CONFIGURED}: every tool call except the core ones will fail; {POINTER}")
|
||||
}
|
||||
|
||||
/// Reads through `stream`, but only what is left before `until`: a read past the deadline fails
|
||||
/// with a timeout, so a peer that sends one byte at a time cannot hold the turn.
|
||||
struct Deadline<'a> {
|
||||
stream: &'a UnixStream,
|
||||
until: Instant,
|
||||
}
|
||||
|
||||
impl std::io::Read for Deadline<'_> {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let left = self.until.saturating_duration_since(Instant::now());
|
||||
if left.is_zero() {
|
||||
return Err(std::io::Error::from(std::io::ErrorKind::TimedOut));
|
||||
}
|
||||
self.stream.set_read_timeout(Some(left))?;
|
||||
let mut stream = self.stream; // `Read` is implemented for `&UnixStream`
|
||||
stream.read(buf)
|
||||
}
|
||||
}
|
||||
|
||||
/// The tool port to `brokerd`. One connection per call, with a deadline per frame.
|
||||
pub struct BrokerPort {
|
||||
socket: PathBuf,
|
||||
timeout: Duration,
|
||||
log: Box<dyn Fn(&str) + Send + Sync>,
|
||||
}
|
||||
|
||||
impl BrokerPort {
|
||||
pub fn new(socket: PathBuf, timeout: Duration) -> BrokerPort {
|
||||
BrokerPort {
|
||||
socket,
|
||||
timeout,
|
||||
log: Box::new(|l| eprintln!("{l}")),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn with_log(
|
||||
socket: PathBuf,
|
||||
timeout: Duration,
|
||||
log: Box<dyn Fn(&str) + Send + Sync>,
|
||||
) -> BrokerPort {
|
||||
BrokerPort {
|
||||
socket,
|
||||
timeout,
|
||||
log,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ToolPort for BrokerPort {
|
||||
fn call(&self, request: &ToolRequest, on_pending: &mut dyn FnMut(&Pending)) -> ToolResponse {
|
||||
let want = request.call.0;
|
||||
|
||||
// The first deadline, taken before connecting.
|
||||
let mut until = match Instant::now().checked_add(self.timeout) {
|
||||
Some(until) => until,
|
||||
None => return self.unavailable("the timeout is too large"),
|
||||
};
|
||||
|
||||
// Exit 2.
|
||||
let mut stream = match UnixStream::connect(&self.socket) {
|
||||
Ok(stream) => stream,
|
||||
Err(e) => {
|
||||
return self
|
||||
.unavailable(&format!("cannot connect to {}: {e}", self.socket.display()));
|
||||
}
|
||||
};
|
||||
|
||||
// Exit 3. `set_write_timeout` fails for a zero timeout.
|
||||
if let Err(e) = stream.set_write_timeout(Some(self.timeout)) {
|
||||
return self.unavailable(&format!("cannot set a timeout: {e}"));
|
||||
}
|
||||
|
||||
// Exit 4. One request frame; the broker is fine if it is too large to send.
|
||||
let request_frame = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
id: want,
|
||||
r#final: true,
|
||||
msg: Message::ToolRequest(request.clone()),
|
||||
};
|
||||
if let Err(e) = write_frame(&mut stream, &request_frame) {
|
||||
return match e {
|
||||
FrameError::TooLarge(_) => ToolResponse::Failed {
|
||||
message: TOO_LARGE.to_string(),
|
||||
},
|
||||
_ => self.unavailable(&format!("cannot send the request: {e}")),
|
||||
};
|
||||
}
|
||||
|
||||
// Exit 5-11, once per frame. A pending frame is reported, then we read again.
|
||||
let mut seen_pending = false;
|
||||
loop {
|
||||
let mut deadline = Deadline {
|
||||
stream: &stream,
|
||||
until,
|
||||
};
|
||||
let env = match read_frame(&mut deadline) {
|
||||
Ok(env) => env,
|
||||
// Exit 5.
|
||||
Err(FrameError::Closed) => {
|
||||
return self.unavailable("the connection closed before the final answer");
|
||||
}
|
||||
Err(FrameError::Io(e))
|
||||
if e.kind() == std::io::ErrorKind::WouldBlock
|
||||
|| e.kind() == std::io::ErrorKind::TimedOut =>
|
||||
{
|
||||
return self.unavailable("no answer in time");
|
||||
}
|
||||
Err(e) => return self.unavailable(&format!("bad frame: {e}")),
|
||||
};
|
||||
|
||||
// Exit 6.
|
||||
if env.id != want {
|
||||
return self
|
||||
.unavailable(&format!("an answer for request {}, not {}", env.id, want));
|
||||
}
|
||||
|
||||
let r#final = env.r#final;
|
||||
// Exit 7.
|
||||
let resp = match env.msg {
|
||||
Message::ToolResponse(resp) => resp,
|
||||
Message::Error(e) => {
|
||||
return self
|
||||
.unavailable(&format!("the broker reported an error: {}", e.detail));
|
||||
}
|
||||
_ => return self.unavailable("an unexpected message"),
|
||||
};
|
||||
|
||||
match resp {
|
||||
// Exit 8.
|
||||
ToolResponse::PendingApproval { approval, expires } => {
|
||||
if r#final {
|
||||
return self.unavailable("a pending frame marked final");
|
||||
}
|
||||
// Exit 10.
|
||||
if seen_pending {
|
||||
return self.unavailable("a second pending frame");
|
||||
}
|
||||
let wait = expires
|
||||
.unix_millis()
|
||||
.saturating_sub(Timestamp::now().unix_millis());
|
||||
until = match Instant::now().checked_add(Duration::from_millis(wait)) {
|
||||
Some(until) => until,
|
||||
None => return self.unavailable("an expiry too far away"),
|
||||
};
|
||||
until = match until.checked_add(self.timeout) {
|
||||
Some(until) => until,
|
||||
None => return self.unavailable("an expiry too far away"),
|
||||
};
|
||||
seen_pending = true;
|
||||
on_pending(&Pending { approval, expires });
|
||||
}
|
||||
// Exit 9 and 11.
|
||||
ToolResponse::Result { .. }
|
||||
| ToolResponse::Failed { .. }
|
||||
| ToolResponse::Denied { .. } => {
|
||||
if r#final {
|
||||
return resp;
|
||||
}
|
||||
return self.unavailable("an answer not marked final");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The one failure no outage reaches the model as, and the one line printed with it.
|
||||
impl BrokerPort {
|
||||
fn unavailable(&self, why: &str) -> ToolResponse {
|
||||
(self.log)(&unavailable_line(why));
|
||||
ToolResponse::Failed {
|
||||
message: UNAVAILABLE.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// The port used when no broker is configured: every call fails with `NOT_CONFIGURED` and prints
|
||||
/// nothing.
|
||||
pub struct NoBroker;
|
||||
|
||||
impl ToolPort for NoBroker {
|
||||
fn call(&self, _request: &ToolRequest, _on_pending: &mut dyn FnMut(&Pending)) -> ToolResponse {
|
||||
ToolResponse::Failed {
|
||||
message: NOT_CONFIGURED.to_string(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -136,6 +136,23 @@ impl Default for Baseline {
|
||||
}
|
||||
}
|
||||
|
||||
/// The tool broker (`brokerd`). No socket means no broker: every non-core tool call fails.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, Deserialize)]
|
||||
#[serde(deny_unknown_fields, default)]
|
||||
pub struct Broker {
|
||||
pub socket: Option<std::path::PathBuf>,
|
||||
pub timeout_ms: u64,
|
||||
}
|
||||
|
||||
impl Default for Broker {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
socket: None,
|
||||
timeout_ms: 120_000,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Deserialize)]
|
||||
#[serde(deny_unknown_fields)]
|
||||
pub struct Config {
|
||||
@@ -154,6 +171,8 @@ pub struct Config {
|
||||
pub r#loop: Loop,
|
||||
#[serde(default)]
|
||||
pub baseline: Baseline,
|
||||
#[serde(default)]
|
||||
pub broker: Broker,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
//! The agent loop: sessions, prompt assembly and memory. It holds no authority.
|
||||
|
||||
pub mod baseline;
|
||||
pub mod broker_port;
|
||||
pub mod channel;
|
||||
pub mod config;
|
||||
pub mod http;
|
||||
|
||||
@@ -6,12 +6,14 @@ use std::os::unix::net::UnixListener;
|
||||
use std::path::Path;
|
||||
use std::process::ExitCode;
|
||||
use std::sync::Arc;
|
||||
use std::time::Duration;
|
||||
|
||||
use loopd::broker_port::{BrokerPort, NoBroker, not_configured_line};
|
||||
use loopd::channel::{self, Context};
|
||||
use loopd::config::Config;
|
||||
use loopd::llama::Client;
|
||||
use loopd::selftest::{SelfTestError, run};
|
||||
use loopd::tools::{FakeTools, Registry};
|
||||
use loopd::tools::{Registry, ToolPort};
|
||||
|
||||
fn main() -> ExitCode {
|
||||
let args: Vec<String> = std::env::args().skip(1).collect();
|
||||
@@ -38,7 +40,7 @@ fn run_selftest_check(client: &Client) -> Result<(), SelfTestError> {
|
||||
Ok(())
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("selftest: FAILED: {e}");
|
||||
eprintln!("selftest: FAILED: {e}; see docs/runbook.md#loopd-selftest-failed");
|
||||
Err(e)
|
||||
}
|
||||
}
|
||||
@@ -104,12 +106,17 @@ fn run_serve(path: &str) -> ExitCode {
|
||||
}
|
||||
eprintln!("loopd: serving on {}", socket.display());
|
||||
|
||||
let ctx = Arc::new(Context::new(
|
||||
cfg,
|
||||
client,
|
||||
Box::new(FakeTools::new()),
|
||||
Registry::m3a(),
|
||||
));
|
||||
let port: Box<dyn ToolPort> = match &cfg.broker.socket {
|
||||
Some(socket) => Box::new(BrokerPort::new(
|
||||
socket.clone(),
|
||||
Duration::from_millis(cfg.broker.timeout_ms),
|
||||
)),
|
||||
None => {
|
||||
eprintln!("{}", not_configured_line());
|
||||
Box::new(NoBroker)
|
||||
}
|
||||
};
|
||||
let ctx = Arc::new(Context::new(cfg, client, port, Registry::m3a()));
|
||||
if let Err(e) = channel::serve(listener, ctx) {
|
||||
eprintln!("loopd: the channel server stopped: {e}");
|
||||
return ExitCode::from(1);
|
||||
|
||||
@@ -29,7 +29,11 @@ impl std::fmt::Display for SessionError {
|
||||
SessionError::Exists(id) => write!(f, "session already exists: {}", id.as_str()),
|
||||
SessionError::NotFound(id) => write!(f, "session not found: {}", id.as_str()),
|
||||
SessionError::Io(path, err) => write!(f, "{}: {err}", path.display()),
|
||||
SessionError::Torn { path, line, why } => write!(f, "{}:{line}: {why}", path.display()),
|
||||
SessionError::Torn { path, line, why } => write!(
|
||||
f,
|
||||
"{}:{line}: {why}; see docs/runbook.md#session-log-damaged",
|
||||
path.display()
|
||||
),
|
||||
SessionError::Baseline(err) => write!(f, "{err}"),
|
||||
SessionError::Encode(err) => write!(f, "{err}"),
|
||||
}
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
//! Tests for `BrokerPort` against a fake `brokerd` on a temporary socket: answers, approvals and
|
||||
//! timeouts. Frames that break the protocol are in `broker_port_bad.rs`. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send, socket_path};
|
||||
use loopd::broker_port::{BrokerPort, NoBroker, POINTER, TOO_LARGE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
DenyReason, Message, PROTOCOL_VERSION, Timestamp, ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn a_result_comes_back_as_it_was_sent() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("hello\n"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("hello\n"));
|
||||
assert!(got.pending.is_empty());
|
||||
assert!(got.lines.is_empty(), "nothing to report: {:?}", got.lines);
|
||||
|
||||
// What the broker received: one final frame holding exactly the request.
|
||||
let sent = broker.join().unwrap();
|
||||
assert_eq!(sent.v, PROTOCOL_VERSION);
|
||||
assert!(sent.r#final, "a request is a single final frame");
|
||||
assert_eq!(sent.msg, Message::ToolRequest(request()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_denial_and_a_failure_come_back_as_they_were_sent() {
|
||||
let mut answers: Vec<ToolResponse> = [
|
||||
DenyReason::NoGrant,
|
||||
DenyReason::GrantExpired,
|
||||
DenyReason::TaintTooHigh,
|
||||
DenyReason::DeniedByGrant,
|
||||
DenyReason::ApprovalRefused,
|
||||
DenyReason::ApprovalExpired,
|
||||
DenyReason::GrantsInvalid,
|
||||
DenyReason::AuditUnavailable,
|
||||
DenyReason::InvalidArguments,
|
||||
DenyReason::StateUnreadable,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|reason| ToolResponse::Denied { reason })
|
||||
.collect();
|
||||
answers.push(ToolResponse::Failed {
|
||||
message: "the runner arrives in M3b".to_string(),
|
||||
});
|
||||
for answer in answers {
|
||||
let reply = answer.clone();
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
send(stream, request.id, true, reply);
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, answer);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"a denial is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_frame_reaches_the_callback_and_the_final_frame_is_the_answer() {
|
||||
let expires = in_ms(60_000);
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 41,
|
||||
expires,
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(150));
|
||||
send(stream, request.id, true, result("approved"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("approved"));
|
||||
assert_eq!(
|
||||
got.pending,
|
||||
[Pending {
|
||||
approval: 41,
|
||||
expires
|
||||
}],
|
||||
"called once, with the frame's values"
|
||||
);
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_after_expires_but_within_the_timeout_of_it_is_delivered() {
|
||||
// expires in 100 ms, timeout 1,500 ms: the port waits until 1,600 ms. The answer comes at
|
||||
// about 400 ms, well after `expires`: an approval given at the last moment still gets the
|
||||
// whole timeout to run.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: in_ms(100),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(400));
|
||||
send(stream, request.id, true, result("late but good"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("late but good"));
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_expiry_that_has_already_passed_still_leaves_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: Timestamp::parse("2020-01-01T00:00:00.000Z").unwrap(),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
send(stream, request.id, true, result("fine"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("fine"));
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_socket_is_unavailable() {
|
||||
let socket = socket_path();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let got = call(socket.clone(), 2_000, &request());
|
||||
assert_unavailable(&got, "no socket file");
|
||||
assert!(
|
||||
got.lines[0].contains(&socket.display().to_string()),
|
||||
"the line names the socket: {}",
|
||||
got.lines[0]
|
||||
);
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_without_answering_is_unavailable() {
|
||||
let (socket, broker) = broker(|_, _| {});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed before any frame");
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_500),
|
||||
"a close is seen at once: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_while_pending_is_unavailable() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
// brokerd was restarted: the connection just ends.
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed while pending");
|
||||
assert_eq!(got.pending.len(), 1, "the pending frame was reported first");
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_never_answers_is_unavailable_after_the_timeout() {
|
||||
let (socket, broker) = broker(|_, _| thread::sleep(Duration::from_millis(1_200)));
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(250),
|
||||
"gave up early: {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_100),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_call_nobody_answers_ends_at_expires_plus_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(300),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(1_800));
|
||||
});
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence while pending");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(550),
|
||||
"it must wait for expires (300) plus the timeout (300): {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_700),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_timeout_is_a_deadline_for_the_frame_not_for_each_read() {
|
||||
// The four header bytes arrive 250 ms apart, the last at 1,000 ms. A port that only sets a
|
||||
// 600 ms read timeout sees every single read succeed and returns the result; a port with a
|
||||
// deadline gives up at 600 ms. The answer tells them apart, so no timing is asserted.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&frame(request.id, true, Message::ToolResponse(result("slow"))),
|
||||
)
|
||||
.unwrap();
|
||||
for byte in bytes.iter().take(4) {
|
||||
thread::sleep(Duration::from_millis(250));
|
||||
if stream.write_all(&[*byte]).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = stream.write_all(&bytes[4..]);
|
||||
});
|
||||
let got = call(socket, 600, &request());
|
||||
assert_unavailable(&got, "a trickled frame");
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_timeout_fails_closed_and_does_not_panic() {
|
||||
let (socket, _broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("too late"));
|
||||
});
|
||||
let got = call(socket, 0, &request());
|
||||
assert_unavailable(&got, "timeout_ms = 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_too_large_for_a_frame_is_its_own_failure() {
|
||||
// Nothing is sent, so the fake broker sees a connection that closes or none at all.
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _listener = UnixListener::bind(&path).unwrap();
|
||||
let mut big = request();
|
||||
big.arguments = format!(r#"{{"path":"/{}"}}"#, "a".repeat(proto::MAX_FRAME));
|
||||
let got = call(path, 1_000, &big);
|
||||
assert_eq!(
|
||||
got.response,
|
||||
ToolResponse::Failed {
|
||||
message: TOO_LARGE.to_string()
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"the broker is fine; this is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_call_is_its_own_connection() {
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
for n in 0..3u64 {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
send(
|
||||
&mut stream,
|
||||
request.id,
|
||||
true,
|
||||
result(&format!("answer {n}")),
|
||||
);
|
||||
}
|
||||
});
|
||||
let port = BrokerPort::new(path, Duration::from_millis(2_000));
|
||||
for n in 0..3 {
|
||||
let got = port.call(&request(), &mut |_| {});
|
||||
assert_eq!(got, result(&format!("answer {n}")));
|
||||
}
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_broker_every_call_fails_in_words_the_model_can_repeat() {
|
||||
let mut seen = 0;
|
||||
let got = NoBroker.call(&request(), &mut |_| seen += 1);
|
||||
assert_eq!(
|
||||
got,
|
||||
ToolResponse::Failed {
|
||||
message: "no tool broker is configured".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(seen, 0);
|
||||
assert_eq!(POINTER, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//! Tests for `BrokerPort` when the broker's frames break the protocol. Every one ends in the same
|
||||
//! plain failure and one printed line. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send};
|
||||
use proto::{Envelope, ErrorCode, Message, PROTOCOL_VERSION, ToolResponse, WireError, write_frame};
|
||||
|
||||
#[test]
|
||||
fn frames_that_break_the_protocol_are_unavailable() {
|
||||
type Script = Box<dyn FnOnce(&mut UnixStream, &Envelope) + Send>;
|
||||
let pending = |approval| ToolResponse::PendingApproval {
|
||||
approval,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
let cases: Vec<(&str, Script)> = vec![
|
||||
(
|
||||
"an answer for another request id",
|
||||
Box::new(|s, r| send(s, r.id + 1, true, result("x"))),
|
||||
),
|
||||
(
|
||||
"a final frame that is pending",
|
||||
Box::new(move |s, r| send(s, r.id, true, pending(1))),
|
||||
),
|
||||
(
|
||||
"an answer that is not final",
|
||||
Box::new(|s, r| send(s, r.id, false, result("x"))),
|
||||
),
|
||||
(
|
||||
"a second pending frame",
|
||||
Box::new(move |s, r| {
|
||||
send(s, r.id, false, pending(1));
|
||||
send(s, r.id, false, pending(2));
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"pending, then another id",
|
||||
Box::new(move |s, r| {
|
||||
send(s, r.id, false, pending(1));
|
||||
send(s, r.id + 1, true, result("x"));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"an error message",
|
||||
Box::new(|s, r| {
|
||||
let error = Message::Error(WireError {
|
||||
code: ErrorCode::Forbidden,
|
||||
detail: "tool requests only".to_string(),
|
||||
});
|
||||
let _ = write_frame(s, &frame(r.id, true, error));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a message of another kind",
|
||||
Box::new(|s, r| {
|
||||
let echo = Message::ToolRequest(request());
|
||||
let _ = write_frame(s, &frame(r.id, true, echo));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"another protocol version",
|
||||
Box::new(|s, r| {
|
||||
let mut env = frame(r.id, true, Message::ToolResponse(result("x")));
|
||||
env.v = PROTOCOL_VERSION + 1;
|
||||
let _ = write_frame(s, &env);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a zero length",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 0]);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a length over the maximum",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0xff, 0xff, 0xff, 0xff]);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a body that is not JSON",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 5]);
|
||||
let _ = s.write_all(b"hello");
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a body cut short",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 50]);
|
||||
let _ = s.write_all(b"{\"v\":1");
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (why, script) in cases {
|
||||
let (socket, broker) = broker(script);
|
||||
let got = call(socket, 1_000, &request());
|
||||
assert_unavailable(&got, why);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_error_message_the_broker_sent_is_in_the_line() {
|
||||
let (socket, broker) = broker(|s, r| {
|
||||
let error = Message::Error(WireError {
|
||||
code: ErrorCode::Internal,
|
||||
detail: "the ledger is gone".to_string(),
|
||||
});
|
||||
let _ = write_frame(s, &frame(r.id, true, error));
|
||||
});
|
||||
let got = call(socket, 1_000, &request());
|
||||
assert_unavailable(&got, "an error message");
|
||||
assert!(
|
||||
got.lines[0].contains("the ledger is gone"),
|
||||
"{}",
|
||||
got.lines[0]
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
@@ -224,3 +224,49 @@ fn unknown_keys_are_errors_in_the_m2b_tables_too() {
|
||||
"a partial table keeps the other defaults"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_table_is_optional_and_its_socket_has_no_default() {
|
||||
let c = Config::load(&fixture("minimal.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.socket, None,
|
||||
"no socket means no broker, never a guessed path"
|
||||
);
|
||||
assert_eq!(c.broker.timeout_ms, 120_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_table_can_be_set_and_rejects_unknown_keys() {
|
||||
let base = std::fs::read_to_string(fixture("m2b.toml")).unwrap();
|
||||
let text = format!(
|
||||
"{base}\n[broker]\nsocket = \"/run/boxmaker/loop-broker/broker.sock\"\ntimeout_ms = 5000\n"
|
||||
);
|
||||
let c = Config::parse(&text).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.socket,
|
||||
Some(PathBuf::from("/run/boxmaker/loop-broker/broker.sock"))
|
||||
);
|
||||
assert_eq!(c.broker.timeout_ms, 5000);
|
||||
|
||||
let only_timeout = format!("{base}\n[broker]\ntimeout_ms = 5000\n");
|
||||
let c = Config::parse(&only_timeout).unwrap();
|
||||
assert_eq!((c.broker.socket, c.broker.timeout_ms), (None, 5000));
|
||||
|
||||
let only_socket = format!("{base}\n[broker]\nsocket = \"/b.sock\"\n");
|
||||
let c = Config::parse(&only_socket).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.timeout_ms, 120_000,
|
||||
"a partial table keeps the default"
|
||||
);
|
||||
|
||||
for bad in [
|
||||
"zz_unknown = 1",
|
||||
"timeout = 5000",
|
||||
"timeout_ms = -1",
|
||||
"timeout_ms = \"5s\"",
|
||||
"socket = 7",
|
||||
] {
|
||||
let text = format!("{base}\n[broker]\n{bad}\n");
|
||||
assert!(Config::parse(&text).is_err(), "[broker] accepted `{bad}`");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,7 +348,7 @@ fn the_baseline_fits_the_token_budget() {
|
||||
cfg.paths.home = home.clone();
|
||||
cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
|
||||
let baseline =
|
||||
loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m2b()).unwrap();
|
||||
loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m3a()).unwrap();
|
||||
let client = Client::new(cfg);
|
||||
// The system text plus every tool schema as the request carries it.
|
||||
let mut text = baseline.system.clone();
|
||||
@@ -380,17 +380,23 @@ fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() {
|
||||
"What is the current time? Use your clock tool, then tell me the year.",
|
||||
);
|
||||
assert!(a2.contains("2026") || a2.contains("202"), "{a2}");
|
||||
// No broker is configured here, so the call fails in plain words and the turn goes on. What
|
||||
// is checked is the path: find_tool, then call_tool, then an answer.
|
||||
let a3 = served.say(
|
||||
&session,
|
||||
"Use the echo tool to echo the word cork back to me, and reply with just that word.",
|
||||
"Find a tool that reads files, use it to read /etc/hostname, and tell me in one \
|
||||
sentence what happened.",
|
||||
);
|
||||
assert!(a3.to_lowercase().contains("cork"), "{a3}");
|
||||
assert!(!a3.trim().is_empty(), "the turn must end in an answer");
|
||||
|
||||
served.kill();
|
||||
served = Served::start(&socket, &home);
|
||||
let a4 = served.say(&session, "What word did you echo a moment ago? One word.");
|
||||
let a4 = served.say(
|
||||
&session,
|
||||
"What were the exact words I first asked you to reply with?",
|
||||
);
|
||||
assert!(
|
||||
a4.to_lowercase().contains("cork"),
|
||||
a4.to_lowercase().contains("box made"),
|
||||
"after a restart the session must still know: {a4}"
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
//! Every fail-closed message `loopd` produces ends with the runbook entry that explains it.
|
||||
//! `scripts/check-runbook.sh` checks that the entries exist; these tests check that the messages
|
||||
//! name them. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::Duration;
|
||||
|
||||
use loopd::baseline::Baseline;
|
||||
use loopd::broker_port::{not_configured_line, unavailable_line};
|
||||
use loopd::session::{Session, SessionError};
|
||||
use loopd::tools::Registry;
|
||||
use proto::SessionId;
|
||||
use support::{FakeServer, Home, Reply};
|
||||
|
||||
// `want` is the whole pointer, written out: scripts/check-runbook.sh reads the anchors of every
|
||||
// pointer in the source, and cannot read one built with `format!`.
|
||||
fn ends_with_pointer(message: &str, want: &str) {
|
||||
assert!(
|
||||
message.ends_with(want),
|
||||
"{message:?} must end with {want:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
message.matches("runbook.md#").count(),
|
||||
1,
|
||||
"one pointer, not two: {message:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_session_log_names_its_entry() {
|
||||
let home = Home::new();
|
||||
let cfg = home.config(Path::new("/tmp/unused.sock"));
|
||||
let id = SessionId::new("a").unwrap();
|
||||
let baseline = Baseline::assemble(&cfg, &Registry::m3a()).unwrap();
|
||||
drop(Session::create(&home.dir, id.clone(), baseline, 0).unwrap());
|
||||
let log = home.dir.join("sessions/a/0.jsonl");
|
||||
let mut text = std::fs::read_to_string(&log).unwrap();
|
||||
text.push_str("{\"type\":\"user\",\"time\":");
|
||||
std::fs::write(&log, text).unwrap();
|
||||
match Session::open(&home.dir, id) {
|
||||
Err(e @ SessionError::Torn { .. }) => {
|
||||
let message = e.to_string();
|
||||
assert!(message.contains("0.jsonl:2: "), "{message}");
|
||||
ends_with_pointer(&message, "see docs/runbook.md#session-log-damaged");
|
||||
}
|
||||
Err(other) => panic!("{other:?}"),
|
||||
Ok(_) => panic!("a torn log was opened"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_torn_error_points_at_the_damaged_log_entry() {
|
||||
let home = Home::new();
|
||||
let missing = Session::open(&home.dir, SessionId::new("nobody").unwrap());
|
||||
let message = match missing {
|
||||
Err(e) => e.to_string(),
|
||||
Ok(_) => panic!("a session nobody created was opened"),
|
||||
};
|
||||
assert!(!message.contains("runbook"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_core_memory_names_its_entry_and_a_missing_system_prompt_does_not() {
|
||||
let home = Home::new();
|
||||
let cfg = home.config(Path::new("/tmp/unused.sock"));
|
||||
home.write("memory/core.md", "memory\n");
|
||||
let core = home.dir.join("memory/core.md");
|
||||
if !running_as_root() {
|
||||
std::fs::set_permissions(&core, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
let result = Baseline::assemble(&cfg, &Registry::m3a());
|
||||
std::fs::set_permissions(&core, std::fs::Permissions::from_mode(0o644)).unwrap();
|
||||
let message = result.expect_err("unreadable core.md").to_string();
|
||||
assert!(message.contains("core.md"), "{message}");
|
||||
ends_with_pointer(&message, "see docs/runbook.md#core-memory-unreadable");
|
||||
}
|
||||
// The system prompt is a different file with a different remedy: no pointer to this entry.
|
||||
std::fs::remove_file(home.dir.join("system.md")).unwrap();
|
||||
let message = Baseline::assemble(&cfg, &Registry::m3a())
|
||||
.expect_err("no system.md")
|
||||
.to_string();
|
||||
assert!(!message.contains("core-memory-unreadable"), "{message}");
|
||||
}
|
||||
|
||||
fn running_as_root() -> bool {
|
||||
std::fs::read_to_string("/proc/self/status")
|
||||
.map(|s| s.lines().any(|l| l.starts_with("Uid:\t0\t")))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_lines_name_their_entry() {
|
||||
let line = unavailable_line("cannot connect to /run/x.sock: No such file");
|
||||
assert_eq!(
|
||||
line,
|
||||
"loopd: the tool broker is unavailable: cannot connect to /run/x.sock: No such file; \
|
||||
see docs/runbook.md#broker-unavailable"
|
||||
);
|
||||
ends_with_pointer(&line, "see docs/runbook.md#broker-unavailable");
|
||||
let line = not_configured_line();
|
||||
assert!(
|
||||
line.starts_with("loopd: no tool broker is configured"),
|
||||
"{line}"
|
||||
);
|
||||
ends_with_pointer(&line, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
|
||||
fn config_file(
|
||||
home: &Home,
|
||||
server: &FakeServer,
|
||||
expect_slots: u32,
|
||||
extra: &str,
|
||||
) -> std::path::PathBuf {
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "test-model"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = {expect_slots}
|
||||
[limits]
|
||||
poll_ms = 40
|
||||
liveness_ms = 500
|
||||
retry_backoff_ms = [10]
|
||||
[paths]
|
||||
home = "{}"
|
||||
{extra}
|
||||
"#,
|
||||
server.socket.display(),
|
||||
home.dir.display()
|
||||
);
|
||||
let path = home.dir.join("config.toml");
|
||||
std::fs::write(&path, text).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn healthy_routes(server: &FakeServer) {
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(
|
||||
"/v1/chat/completions",
|
||||
vec![
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("turn1"),
|
||||
Reply::fixture("turn2"),
|
||||
Reply::fixture("plain"),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/// Waits for `loopd serve` to bind its socket, stops it, and returns what it printed.
|
||||
fn stderr_once_serving(mut child: Child, socket: &Path) -> String {
|
||||
let mut up = false;
|
||||
for _ in 0..200 {
|
||||
if socket.exists() {
|
||||
up = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
child.kill().unwrap();
|
||||
let output = child.wait_with_output().unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
assert!(up, "the socket never appeared; stderr: {stderr}");
|
||||
stderr
|
||||
}
|
||||
|
||||
fn serve(config: &Path) -> Child {
|
||||
Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.args(["serve", "--config"])
|
||||
.arg(config)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_self_test_names_its_entry() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
// The server has two slots; the config expects three.
|
||||
let config = config_file(&home, &server, 3, "");
|
||||
for command in ["selftest", "serve"] {
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.args([command, "--config"])
|
||||
.arg(&config)
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert_eq!(output.status.code(), Some(1), "{command}: {stderr}");
|
||||
let line = stderr
|
||||
.lines()
|
||||
.find(|l| l.starts_with("selftest: FAILED: "))
|
||||
.unwrap_or_else(|| panic!("{command}: no FAILED line: {stderr}"));
|
||||
ends_with_pointer(line, "see docs/runbook.md#loopd-selftest-failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_without_a_broker_says_so_once_with_the_entry() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
let config = config_file(&home, &server, 2, "");
|
||||
let socket = home.dir.join("run/loop/loop.sock");
|
||||
let stderr = stderr_once_serving(serve(&config), &socket);
|
||||
let lines: Vec<&str> = stderr
|
||||
.lines()
|
||||
.filter(|l| l.contains("no tool broker is configured"))
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 1, "once, at startup: {stderr}");
|
||||
ends_with_pointer(lines[0], "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_with_a_broker_socket_does_not_say_it() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
// The socket need not exist: loopd connects per call, and a missing broker is not a reason
|
||||
// to refuse to start.
|
||||
let extra = format!(
|
||||
"[broker]\nsocket = \"{}\"\n",
|
||||
home.dir.join("run/loop-broker/broker.sock").display()
|
||||
);
|
||||
let config = config_file(&home, &server, 2, &extra);
|
||||
let socket = home.dir.join("run/loop/loop.sock");
|
||||
let stderr = stderr_once_serving(serve(&config), &socket);
|
||||
assert!(!stderr.contains("no tool broker"), "{stderr}");
|
||||
assert!(stderr.contains("serving on"), "{stderr}");
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! A fake `brokerd` for the `BrokerPort` tests. Included with `#[path]`. Do not edit.
|
||||
//!
|
||||
//! The fake behaves as the real one will: it reads one request frame, answers on the same
|
||||
//! connection, never half-closes, and closes after the final frame.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use loopd::broker_port::{BrokerPort, UNAVAILABLE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
CallId, DataClass, Envelope, Message, PROTOCOL_VERSION, SessionId, Timestamp, ToolRequest,
|
||||
ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
|
||||
static NEXT: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
pub fn socket_path() -> PathBuf {
|
||||
let n = NEXT.fetch_add(1, Ordering::SeqCst);
|
||||
std::env::temp_dir().join(format!("loopd-broker-{}-{n}.sock", std::process::id()))
|
||||
}
|
||||
|
||||
/// Accepts one connection, reads the request frame and hands both to `script`. The connection
|
||||
/// closes when `script` returns.
|
||||
pub fn broker<F>(script: F) -> (PathBuf, JoinHandle<Envelope>)
|
||||
where
|
||||
F: FnOnce(&mut UnixStream, &Envelope) + Send + 'static,
|
||||
{
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let handle = thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
script(&mut stream, &request);
|
||||
request
|
||||
});
|
||||
(path, handle)
|
||||
}
|
||||
|
||||
pub fn frame(id: u64, r#final: bool, msg: Message) -> Envelope {
|
||||
Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
id,
|
||||
r#final,
|
||||
msg,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send(stream: &mut UnixStream, id: u64, r#final: bool, response: ToolResponse) {
|
||||
// The port may already have given up and gone; the fake does not care.
|
||||
let _ = write_frame(stream, &frame(id, r#final, Message::ToolResponse(response)));
|
||||
}
|
||||
|
||||
pub fn result(content: &str) -> ToolResponse {
|
||||
ToolResponse::Result {
|
||||
content: content.to_string(),
|
||||
class: DataClass::Secret,
|
||||
untrusted: true,
|
||||
truncated: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request() -> ToolRequest {
|
||||
ToolRequest {
|
||||
session: SessionId::new("chat-1").unwrap(),
|
||||
call: CallId(7),
|
||||
tool: "read_file".to_string(),
|
||||
arguments: r#"{"path":"/home/kyle/notes/a.md"}"#.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn in_ms(ms: u64) -> Timestamp {
|
||||
Timestamp::from_unix_millis(Timestamp::now().unix_millis() + ms).unwrap()
|
||||
}
|
||||
|
||||
pub struct Call {
|
||||
pub response: ToolResponse,
|
||||
pub pending: Vec<Pending>,
|
||||
pub lines: Vec<String>,
|
||||
pub took: Duration,
|
||||
}
|
||||
|
||||
pub fn call(socket: PathBuf, timeout_ms: u64, request: &ToolRequest) -> Call {
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = lines.clone();
|
||||
let port = BrokerPort::with_log(
|
||||
socket,
|
||||
Duration::from_millis(timeout_ms),
|
||||
Box::new(move |line| sink.lock().unwrap().push(line.to_string())),
|
||||
);
|
||||
let mut pending = Vec::new();
|
||||
let started = Instant::now();
|
||||
let response = port.call(request, &mut |p| pending.push(*p));
|
||||
let took = started.elapsed();
|
||||
let lines = lines.lock().unwrap().clone();
|
||||
Call {
|
||||
response,
|
||||
pending,
|
||||
lines,
|
||||
took,
|
||||
}
|
||||
}
|
||||
|
||||
/// The answer is the fixed failure, and `loopd` printed exactly one line, with the pointer.
|
||||
pub fn assert_unavailable(call: &Call, why: &str) {
|
||||
assert_eq!(
|
||||
call.response,
|
||||
ToolResponse::Failed {
|
||||
message: UNAVAILABLE.to_string()
|
||||
},
|
||||
"{why}"
|
||||
);
|
||||
assert_eq!(
|
||||
call.lines.len(),
|
||||
1,
|
||||
"{why}: one line per failed call: {:?}",
|
||||
call.lines
|
||||
);
|
||||
let line = &call.lines[0];
|
||||
assert!(
|
||||
line.starts_with("loopd: the tool broker is unavailable: "),
|
||||
"{why}: {line}"
|
||||
);
|
||||
assert!(
|
||||
line.ends_with("; see docs/runbook.md#broker-unavailable"),
|
||||
"{why}: {line}"
|
||||
);
|
||||
assert!(call.pending.is_empty() || why.contains("pending"), "{why}");
|
||||
}
|
||||
Reference in New Issue
Block a user