Add BrokerPort: loopd asks brokerd for every tool call
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -0,0 +1,314 @@
|
||||
//! Tests for `BrokerPort` against a fake `brokerd` on a temporary socket: answers, approvals and
|
||||
//! timeouts. Frames that break the protocol are in `broker_port_bad.rs`. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixListener;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send, socket_path};
|
||||
use loopd::broker_port::{BrokerPort, NoBroker, POINTER, TOO_LARGE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
DenyReason, Message, PROTOCOL_VERSION, Timestamp, ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
|
||||
#[test]
|
||||
fn a_result_comes_back_as_it_was_sent() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("hello\n"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("hello\n"));
|
||||
assert!(got.pending.is_empty());
|
||||
assert!(got.lines.is_empty(), "nothing to report: {:?}", got.lines);
|
||||
|
||||
// What the broker received: one final frame holding exactly the request.
|
||||
let sent = broker.join().unwrap();
|
||||
assert_eq!(sent.v, PROTOCOL_VERSION);
|
||||
assert!(sent.r#final, "a request is a single final frame");
|
||||
assert_eq!(sent.msg, Message::ToolRequest(request()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_denial_and_a_failure_come_back_as_they_were_sent() {
|
||||
let mut answers: Vec<ToolResponse> = [
|
||||
DenyReason::NoGrant,
|
||||
DenyReason::GrantExpired,
|
||||
DenyReason::TaintTooHigh,
|
||||
DenyReason::DeniedByGrant,
|
||||
DenyReason::ApprovalRefused,
|
||||
DenyReason::ApprovalExpired,
|
||||
DenyReason::GrantsInvalid,
|
||||
DenyReason::AuditUnavailable,
|
||||
DenyReason::InvalidArguments,
|
||||
DenyReason::StateUnreadable,
|
||||
]
|
||||
.into_iter()
|
||||
.map(|reason| ToolResponse::Denied { reason })
|
||||
.collect();
|
||||
answers.push(ToolResponse::Failed {
|
||||
message: "the runner arrives in M3b".to_string(),
|
||||
});
|
||||
for answer in answers {
|
||||
let reply = answer.clone();
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
send(stream, request.id, true, reply);
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, answer);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"a denial is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_frame_reaches_the_callback_and_the_final_frame_is_the_answer() {
|
||||
let expires = in_ms(60_000);
|
||||
let (socket, broker) = broker(move |stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 41,
|
||||
expires,
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(150));
|
||||
send(stream, request.id, true, result("approved"));
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_eq!(got.response, result("approved"));
|
||||
assert_eq!(
|
||||
got.pending,
|
||||
[Pending {
|
||||
approval: 41,
|
||||
expires
|
||||
}],
|
||||
"called once, with the frame's values"
|
||||
);
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_answer_after_expires_but_within_the_timeout_of_it_is_delivered() {
|
||||
// expires in 100 ms, timeout 1,500 ms: the port waits until 1,600 ms. The answer comes at
|
||||
// about 400 ms, well after `expires`: an approval given at the last moment still gets the
|
||||
// whole timeout to run.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: in_ms(100),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(400));
|
||||
send(stream, request.id, true, result("late but good"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("late but good"));
|
||||
assert!(got.lines.is_empty(), "{:?}", got.lines);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_expiry_that_has_already_passed_still_leaves_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 1,
|
||||
expires: Timestamp::parse("2020-01-01T00:00:00.000Z").unwrap(),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(200));
|
||||
send(stream, request.id, true, result("fine"));
|
||||
});
|
||||
let got = call(socket, 1_500, &request());
|
||||
assert_eq!(got.response, result("fine"));
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_socket_is_unavailable() {
|
||||
let socket = socket_path();
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let got = call(socket.clone(), 2_000, &request());
|
||||
assert_unavailable(&got, "no socket file");
|
||||
assert!(
|
||||
got.lines[0].contains(&socket.display().to_string()),
|
||||
"the line names the socket: {}",
|
||||
got.lines[0]
|
||||
);
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_without_answering_is_unavailable() {
|
||||
let (socket, broker) = broker(|_, _| {});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed before any frame");
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_500),
|
||||
"a close is seen at once: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_closes_while_pending_is_unavailable() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
// brokerd was restarted: the connection just ends.
|
||||
});
|
||||
let got = call(socket, 2_000, &request());
|
||||
assert_unavailable(&got, "closed while pending");
|
||||
assert_eq!(got.pending.len(), 1, "the pending frame was reported first");
|
||||
assert!(got.took < Duration::from_millis(1_500), "{:?}", got.took);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_broker_that_never_answers_is_unavailable_after_the_timeout() {
|
||||
let (socket, broker) = broker(|_, _| thread::sleep(Duration::from_millis(1_200)));
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(250),
|
||||
"gave up early: {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_100),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_pending_call_nobody_answers_ends_at_expires_plus_the_timeout() {
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let pending = ToolResponse::PendingApproval {
|
||||
approval: 3,
|
||||
expires: in_ms(300),
|
||||
};
|
||||
send(stream, request.id, false, pending);
|
||||
thread::sleep(Duration::from_millis(1_800));
|
||||
});
|
||||
let got = call(socket, 300, &request());
|
||||
assert_unavailable(&got, "silence while pending");
|
||||
assert!(
|
||||
got.took >= Duration::from_millis(550),
|
||||
"it must wait for expires (300) plus the timeout (300): {:?}",
|
||||
got.took
|
||||
);
|
||||
assert!(
|
||||
got.took < Duration::from_millis(1_700),
|
||||
"gave up late: {:?}",
|
||||
got.took
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_timeout_is_a_deadline_for_the_frame_not_for_each_read() {
|
||||
// The four header bytes arrive 250 ms apart, the last at 1,000 ms. A port that only sets a
|
||||
// 600 ms read timeout sees every single read succeed and returns the result; a port with a
|
||||
// deadline gives up at 600 ms. The answer tells them apart, so no timing is asserted.
|
||||
let (socket, broker) = broker(|stream, request| {
|
||||
let mut bytes = Vec::new();
|
||||
write_frame(
|
||||
&mut bytes,
|
||||
&frame(request.id, true, Message::ToolResponse(result("slow"))),
|
||||
)
|
||||
.unwrap();
|
||||
for byte in bytes.iter().take(4) {
|
||||
thread::sleep(Duration::from_millis(250));
|
||||
if stream.write_all(&[*byte]).is_err() {
|
||||
return;
|
||||
}
|
||||
}
|
||||
let _ = stream.write_all(&bytes[4..]);
|
||||
});
|
||||
let got = call(socket, 600, &request());
|
||||
assert_unavailable(&got, "a trickled frame");
|
||||
broker.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_zero_timeout_fails_closed_and_does_not_panic() {
|
||||
let (socket, _broker) = broker(|stream, request| {
|
||||
send(stream, request.id, true, result("too late"));
|
||||
});
|
||||
let got = call(socket, 0, &request());
|
||||
assert_unavailable(&got, "timeout_ms = 0");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_request_too_large_for_a_frame_is_its_own_failure() {
|
||||
// Nothing is sent, so the fake broker sees a connection that closes or none at all.
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let _listener = UnixListener::bind(&path).unwrap();
|
||||
let mut big = request();
|
||||
big.arguments = format!(r#"{{"path":"/{}"}}"#, "a".repeat(proto::MAX_FRAME));
|
||||
let got = call(path, 1_000, &big);
|
||||
assert_eq!(
|
||||
got.response,
|
||||
ToolResponse::Failed {
|
||||
message: TOO_LARGE.to_string()
|
||||
}
|
||||
);
|
||||
assert!(
|
||||
got.lines.is_empty(),
|
||||
"the broker is fine; this is not an outage: {:?}",
|
||||
got.lines
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_call_is_its_own_connection() {
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let server = thread::spawn(move || {
|
||||
for n in 0..3u64 {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
send(
|
||||
&mut stream,
|
||||
request.id,
|
||||
true,
|
||||
result(&format!("answer {n}")),
|
||||
);
|
||||
}
|
||||
});
|
||||
let port = BrokerPort::new(path, Duration::from_millis(2_000));
|
||||
for n in 0..3 {
|
||||
let got = port.call(&request(), &mut |_| {});
|
||||
assert_eq!(got, result(&format!("answer {n}")));
|
||||
}
|
||||
server.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_a_broker_every_call_fails_in_words_the_model_can_repeat() {
|
||||
let mut seen = 0;
|
||||
let got = NoBroker.call(&request(), &mut |_| seen += 1);
|
||||
assert_eq!(
|
||||
got,
|
||||
ToolResponse::Failed {
|
||||
message: "no tool broker is configured".to_string()
|
||||
}
|
||||
);
|
||||
assert_eq!(seen, 0);
|
||||
assert_eq!(POINTER, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//! Tests for `BrokerPort` when the broker's frames break the protocol. Every one ends in the same
|
||||
//! plain failure and one printed line. Do not edit.
|
||||
|
||||
#[path = "support/broker.rs"]
|
||||
mod fake;
|
||||
|
||||
use std::io::Write;
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
use fake::{assert_unavailable, broker, call, frame, in_ms, request, result, send};
|
||||
use proto::{Envelope, ErrorCode, Message, PROTOCOL_VERSION, ToolResponse, WireError, write_frame};
|
||||
|
||||
#[test]
|
||||
fn frames_that_break_the_protocol_are_unavailable() {
|
||||
type Script = Box<dyn FnOnce(&mut UnixStream, &Envelope) + Send>;
|
||||
let pending = |approval| ToolResponse::PendingApproval {
|
||||
approval,
|
||||
expires: in_ms(60_000),
|
||||
};
|
||||
let cases: Vec<(&str, Script)> = vec![
|
||||
(
|
||||
"an answer for another request id",
|
||||
Box::new(|s, r| send(s, r.id + 1, true, result("x"))),
|
||||
),
|
||||
(
|
||||
"a final frame that is pending",
|
||||
Box::new(move |s, r| send(s, r.id, true, pending(1))),
|
||||
),
|
||||
(
|
||||
"an answer that is not final",
|
||||
Box::new(|s, r| send(s, r.id, false, result("x"))),
|
||||
),
|
||||
(
|
||||
"a second pending frame",
|
||||
Box::new(move |s, r| {
|
||||
send(s, r.id, false, pending(1));
|
||||
send(s, r.id, false, pending(2));
|
||||
thread::sleep(Duration::from_millis(100));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"pending, then another id",
|
||||
Box::new(move |s, r| {
|
||||
send(s, r.id, false, pending(1));
|
||||
send(s, r.id + 1, true, result("x"));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"an error message",
|
||||
Box::new(|s, r| {
|
||||
let error = Message::Error(WireError {
|
||||
code: ErrorCode::Forbidden,
|
||||
detail: "tool requests only".to_string(),
|
||||
});
|
||||
let _ = write_frame(s, &frame(r.id, true, error));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a message of another kind",
|
||||
Box::new(|s, r| {
|
||||
let echo = Message::ToolRequest(request());
|
||||
let _ = write_frame(s, &frame(r.id, true, echo));
|
||||
}),
|
||||
),
|
||||
(
|
||||
"another protocol version",
|
||||
Box::new(|s, r| {
|
||||
let mut env = frame(r.id, true, Message::ToolResponse(result("x")));
|
||||
env.v = PROTOCOL_VERSION + 1;
|
||||
let _ = write_frame(s, &env);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a zero length",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 0]);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a length over the maximum",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0xff, 0xff, 0xff, 0xff]);
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a body that is not JSON",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 5]);
|
||||
let _ = s.write_all(b"hello");
|
||||
}),
|
||||
),
|
||||
(
|
||||
"a body cut short",
|
||||
Box::new(|s, _| {
|
||||
let _ = s.write_all(&[0, 0, 0, 50]);
|
||||
let _ = s.write_all(b"{\"v\":1");
|
||||
}),
|
||||
),
|
||||
];
|
||||
for (why, script) in cases {
|
||||
let (socket, broker) = broker(script);
|
||||
let got = call(socket, 1_000, &request());
|
||||
assert_unavailable(&got, why);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_error_message_the_broker_sent_is_in_the_line() {
|
||||
let (socket, broker) = broker(|s, r| {
|
||||
let error = Message::Error(WireError {
|
||||
code: ErrorCode::Internal,
|
||||
detail: "the ledger is gone".to_string(),
|
||||
});
|
||||
let _ = write_frame(s, &frame(r.id, true, error));
|
||||
});
|
||||
let got = call(socket, 1_000, &request());
|
||||
assert_unavailable(&got, "an error message");
|
||||
assert!(
|
||||
got.lines[0].contains("the ledger is gone"),
|
||||
"{}",
|
||||
got.lines[0]
|
||||
);
|
||||
broker.join().unwrap();
|
||||
}
|
||||
@@ -224,3 +224,49 @@ fn unknown_keys_are_errors_in_the_m2b_tables_too() {
|
||||
"a partial table keeps the other defaults"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_table_is_optional_and_its_socket_has_no_default() {
|
||||
let c = Config::load(&fixture("minimal.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.socket, None,
|
||||
"no socket means no broker, never a guessed path"
|
||||
);
|
||||
assert_eq!(c.broker.timeout_ms, 120_000);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_table_can_be_set_and_rejects_unknown_keys() {
|
||||
let base = std::fs::read_to_string(fixture("m2b.toml")).unwrap();
|
||||
let text = format!(
|
||||
"{base}\n[broker]\nsocket = \"/run/boxmaker/loop-broker/broker.sock\"\ntimeout_ms = 5000\n"
|
||||
);
|
||||
let c = Config::parse(&text).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.socket,
|
||||
Some(PathBuf::from("/run/boxmaker/loop-broker/broker.sock"))
|
||||
);
|
||||
assert_eq!(c.broker.timeout_ms, 5000);
|
||||
|
||||
let only_timeout = format!("{base}\n[broker]\ntimeout_ms = 5000\n");
|
||||
let c = Config::parse(&only_timeout).unwrap();
|
||||
assert_eq!((c.broker.socket, c.broker.timeout_ms), (None, 5000));
|
||||
|
||||
let only_socket = format!("{base}\n[broker]\nsocket = \"/b.sock\"\n");
|
||||
let c = Config::parse(&only_socket).unwrap();
|
||||
assert_eq!(
|
||||
c.broker.timeout_ms, 120_000,
|
||||
"a partial table keeps the default"
|
||||
);
|
||||
|
||||
for bad in [
|
||||
"zz_unknown = 1",
|
||||
"timeout = 5000",
|
||||
"timeout_ms = -1",
|
||||
"timeout_ms = \"5s\"",
|
||||
"socket = 7",
|
||||
] {
|
||||
let text = format!("{base}\n[broker]\n{bad}\n");
|
||||
assert!(Config::parse(&text).is_err(), "[broker] accepted `{bad}`");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -348,7 +348,7 @@ fn the_baseline_fits_the_token_budget() {
|
||||
cfg.paths.home = home.clone();
|
||||
cfg.baseline.system = Path::new(env!("CARGO_MANIFEST_DIR")).join("../../config/system.md");
|
||||
let baseline =
|
||||
loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m2b()).unwrap();
|
||||
loopd::baseline::Baseline::assemble(&cfg, &loopd::tools::Registry::m3a()).unwrap();
|
||||
let client = Client::new(cfg);
|
||||
// The system text plus every tool schema as the request carries it.
|
||||
let mut text = baseline.system.clone();
|
||||
@@ -380,17 +380,23 @@ fn a_conversation_uses_tools_survives_a_restart_and_keeps_its_cache() {
|
||||
"What is the current time? Use your clock tool, then tell me the year.",
|
||||
);
|
||||
assert!(a2.contains("2026") || a2.contains("202"), "{a2}");
|
||||
// No broker is configured here, so the call fails in plain words and the turn goes on. What
|
||||
// is checked is the path: find_tool, then call_tool, then an answer.
|
||||
let a3 = served.say(
|
||||
&session,
|
||||
"Use the echo tool to echo the word cork back to me, and reply with just that word.",
|
||||
"Find a tool that reads files, use it to read /etc/hostname, and tell me in one \
|
||||
sentence what happened.",
|
||||
);
|
||||
assert!(a3.to_lowercase().contains("cork"), "{a3}");
|
||||
assert!(!a3.trim().is_empty(), "the turn must end in an answer");
|
||||
|
||||
served.kill();
|
||||
served = Served::start(&socket, &home);
|
||||
let a4 = served.say(&session, "What word did you echo a moment ago? One word.");
|
||||
let a4 = served.say(
|
||||
&session,
|
||||
"What were the exact words I first asked you to reply with?",
|
||||
);
|
||||
assert!(
|
||||
a4.to_lowercase().contains("cork"),
|
||||
a4.to_lowercase().contains("box made"),
|
||||
"after a restart the session must still know: {a4}"
|
||||
);
|
||||
|
||||
|
||||
@@ -0,0 +1,240 @@
|
||||
//! Every fail-closed message `loopd` produces ends with the runbook entry that explains it.
|
||||
//! `scripts/check-runbook.sh` checks that the entries exist; these tests check that the messages
|
||||
//! name them. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
use std::path::Path;
|
||||
use std::process::{Child, Command, Stdio};
|
||||
use std::time::Duration;
|
||||
|
||||
use loopd::baseline::Baseline;
|
||||
use loopd::broker_port::{not_configured_line, unavailable_line};
|
||||
use loopd::session::{Session, SessionError};
|
||||
use loopd::tools::Registry;
|
||||
use proto::SessionId;
|
||||
use support::{FakeServer, Home, Reply};
|
||||
|
||||
// `want` is the whole pointer, written out: scripts/check-runbook.sh reads the anchors of every
|
||||
// pointer in the source, and cannot read one built with `format!`.
|
||||
fn ends_with_pointer(message: &str, want: &str) {
|
||||
assert!(
|
||||
message.ends_with(want),
|
||||
"{message:?} must end with {want:?}"
|
||||
);
|
||||
assert_eq!(
|
||||
message.matches("runbook.md#").count(),
|
||||
1,
|
||||
"one pointer, not two: {message:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_session_log_names_its_entry() {
|
||||
let home = Home::new();
|
||||
let cfg = home.config(Path::new("/tmp/unused.sock"));
|
||||
let id = SessionId::new("a").unwrap();
|
||||
let baseline = Baseline::assemble(&cfg, &Registry::m3a()).unwrap();
|
||||
drop(Session::create(&home.dir, id.clone(), baseline, 0).unwrap());
|
||||
let log = home.dir.join("sessions/a/0.jsonl");
|
||||
let mut text = std::fs::read_to_string(&log).unwrap();
|
||||
text.push_str("{\"type\":\"user\",\"time\":");
|
||||
std::fs::write(&log, text).unwrap();
|
||||
match Session::open(&home.dir, id) {
|
||||
Err(e @ SessionError::Torn { .. }) => {
|
||||
let message = e.to_string();
|
||||
assert!(message.contains("0.jsonl:2: "), "{message}");
|
||||
ends_with_pointer(&message, "see docs/runbook.md#session-log-damaged");
|
||||
}
|
||||
Err(other) => panic!("{other:?}"),
|
||||
Ok(_) => panic!("a torn log was opened"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_torn_error_points_at_the_damaged_log_entry() {
|
||||
let home = Home::new();
|
||||
let missing = Session::open(&home.dir, SessionId::new("nobody").unwrap());
|
||||
let message = match missing {
|
||||
Err(e) => e.to_string(),
|
||||
Ok(_) => panic!("a session nobody created was opened"),
|
||||
};
|
||||
assert!(!message.contains("runbook"), "{message}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_core_memory_names_its_entry_and_a_missing_system_prompt_does_not() {
|
||||
let home = Home::new();
|
||||
let cfg = home.config(Path::new("/tmp/unused.sock"));
|
||||
home.write("memory/core.md", "memory\n");
|
||||
let core = home.dir.join("memory/core.md");
|
||||
if !running_as_root() {
|
||||
std::fs::set_permissions(&core, std::fs::Permissions::from_mode(0o000)).unwrap();
|
||||
let result = Baseline::assemble(&cfg, &Registry::m3a());
|
||||
std::fs::set_permissions(&core, std::fs::Permissions::from_mode(0o644)).unwrap();
|
||||
let message = result.expect_err("unreadable core.md").to_string();
|
||||
assert!(message.contains("core.md"), "{message}");
|
||||
ends_with_pointer(&message, "see docs/runbook.md#core-memory-unreadable");
|
||||
}
|
||||
// The system prompt is a different file with a different remedy: no pointer to this entry.
|
||||
std::fs::remove_file(home.dir.join("system.md")).unwrap();
|
||||
let message = Baseline::assemble(&cfg, &Registry::m3a())
|
||||
.expect_err("no system.md")
|
||||
.to_string();
|
||||
assert!(!message.contains("core-memory-unreadable"), "{message}");
|
||||
}
|
||||
|
||||
fn running_as_root() -> bool {
|
||||
std::fs::read_to_string("/proc/self/status")
|
||||
.map(|s| s.lines().any(|l| l.starts_with("Uid:\t0\t")))
|
||||
.unwrap_or(false)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_broker_lines_name_their_entry() {
|
||||
let line = unavailable_line("cannot connect to /run/x.sock: No such file");
|
||||
assert_eq!(
|
||||
line,
|
||||
"loopd: the tool broker is unavailable: cannot connect to /run/x.sock: No such file; \
|
||||
see docs/runbook.md#broker-unavailable"
|
||||
);
|
||||
ends_with_pointer(&line, "see docs/runbook.md#broker-unavailable");
|
||||
let line = not_configured_line();
|
||||
assert!(
|
||||
line.starts_with("loopd: no tool broker is configured"),
|
||||
"{line}"
|
||||
);
|
||||
ends_with_pointer(&line, "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
|
||||
fn config_file(
|
||||
home: &Home,
|
||||
server: &FakeServer,
|
||||
expect_slots: u32,
|
||||
extra: &str,
|
||||
) -> std::path::PathBuf {
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "test-model"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = {expect_slots}
|
||||
[limits]
|
||||
poll_ms = 40
|
||||
liveness_ms = 500
|
||||
retry_backoff_ms = [10]
|
||||
[paths]
|
||||
home = "{}"
|
||||
{extra}
|
||||
"#,
|
||||
server.socket.display(),
|
||||
home.dir.display()
|
||||
);
|
||||
let path = home.dir.join("config.toml");
|
||||
std::fs::write(&path, text).unwrap();
|
||||
path
|
||||
}
|
||||
|
||||
fn healthy_routes(server: &FakeServer) {
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(
|
||||
"/v1/chat/completions",
|
||||
vec![
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("turn1"),
|
||||
Reply::fixture("turn2"),
|
||||
Reply::fixture("plain"),
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/// Waits for `loopd serve` to bind its socket, stops it, and returns what it printed.
|
||||
fn stderr_once_serving(mut child: Child, socket: &Path) -> String {
|
||||
let mut up = false;
|
||||
for _ in 0..200 {
|
||||
if socket.exists() {
|
||||
up = true;
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(50));
|
||||
}
|
||||
child.kill().unwrap();
|
||||
let output = child.wait_with_output().unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr).to_string();
|
||||
assert!(up, "the socket never appeared; stderr: {stderr}");
|
||||
stderr
|
||||
}
|
||||
|
||||
fn serve(config: &Path) -> Child {
|
||||
Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.args(["serve", "--config"])
|
||||
.arg(config)
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_failed_self_test_names_its_entry() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
// The server has two slots; the config expects three.
|
||||
let config = config_file(&home, &server, 3, "");
|
||||
for command in ["selftest", "serve"] {
|
||||
let output = Command::new(env!("CARGO_BIN_EXE_loopd"))
|
||||
.args([command, "--config"])
|
||||
.arg(&config)
|
||||
.output()
|
||||
.unwrap();
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
assert_eq!(output.status.code(), Some(1), "{command}: {stderr}");
|
||||
let line = stderr
|
||||
.lines()
|
||||
.find(|l| l.starts_with("selftest: FAILED: "))
|
||||
.unwrap_or_else(|| panic!("{command}: no FAILED line: {stderr}"));
|
||||
ends_with_pointer(line, "see docs/runbook.md#loopd-selftest-failed");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_without_a_broker_says_so_once_with_the_entry() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
let config = config_file(&home, &server, 2, "");
|
||||
let socket = home.dir.join("run/loop/loop.sock");
|
||||
let stderr = stderr_once_serving(serve(&config), &socket);
|
||||
let lines: Vec<&str> = stderr
|
||||
.lines()
|
||||
.filter(|l| l.contains("no tool broker is configured"))
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 1, "once, at startup: {stderr}");
|
||||
ends_with_pointer(lines[0], "see docs/runbook.md#broker-unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn serve_with_a_broker_socket_does_not_say_it() {
|
||||
let home = Home::new();
|
||||
let server = FakeServer::start();
|
||||
healthy_routes(&server);
|
||||
// The socket need not exist: loopd connects per call, and a missing broker is not a reason
|
||||
// to refuse to start.
|
||||
let extra = format!(
|
||||
"[broker]\nsocket = \"{}\"\n",
|
||||
home.dir.join("run/loop-broker/broker.sock").display()
|
||||
);
|
||||
let config = config_file(&home, &server, 2, &extra);
|
||||
let socket = home.dir.join("run/loop/loop.sock");
|
||||
let stderr = stderr_once_serving(serve(&config), &socket);
|
||||
assert!(!stderr.contains("no tool broker"), "{stderr}");
|
||||
assert!(stderr.contains("serving on"), "{stderr}");
|
||||
}
|
||||
@@ -0,0 +1,136 @@
|
||||
//! A fake `brokerd` for the `BrokerPort` tests. Included with `#[path]`. Do not edit.
|
||||
//!
|
||||
//! The fake behaves as the real one will: it reads one request frame, answers on the same
|
||||
//! connection, never half-closes, and closes after the final frame.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::{self, JoinHandle};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use loopd::broker_port::{BrokerPort, UNAVAILABLE};
|
||||
use loopd::tools::{Pending, ToolPort};
|
||||
use proto::{
|
||||
CallId, DataClass, Envelope, Message, PROTOCOL_VERSION, SessionId, Timestamp, ToolRequest,
|
||||
ToolResponse, read_frame, write_frame,
|
||||
};
|
||||
|
||||
static NEXT: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
pub fn socket_path() -> PathBuf {
|
||||
let n = NEXT.fetch_add(1, Ordering::SeqCst);
|
||||
std::env::temp_dir().join(format!("loopd-broker-{}-{n}.sock", std::process::id()))
|
||||
}
|
||||
|
||||
/// Accepts one connection, reads the request frame and hands both to `script`. The connection
|
||||
/// closes when `script` returns.
|
||||
pub fn broker<F>(script: F) -> (PathBuf, JoinHandle<Envelope>)
|
||||
where
|
||||
F: FnOnce(&mut UnixStream, &Envelope) + Send + 'static,
|
||||
{
|
||||
let path = socket_path();
|
||||
let _ = std::fs::remove_file(&path);
|
||||
let listener = UnixListener::bind(&path).unwrap();
|
||||
let handle = thread::spawn(move || {
|
||||
let (mut stream, _) = listener.accept().unwrap();
|
||||
let request = read_frame(&mut stream).unwrap();
|
||||
script(&mut stream, &request);
|
||||
request
|
||||
});
|
||||
(path, handle)
|
||||
}
|
||||
|
||||
pub fn frame(id: u64, r#final: bool, msg: Message) -> Envelope {
|
||||
Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
id,
|
||||
r#final,
|
||||
msg,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn send(stream: &mut UnixStream, id: u64, r#final: bool, response: ToolResponse) {
|
||||
// The port may already have given up and gone; the fake does not care.
|
||||
let _ = write_frame(stream, &frame(id, r#final, Message::ToolResponse(response)));
|
||||
}
|
||||
|
||||
pub fn result(content: &str) -> ToolResponse {
|
||||
ToolResponse::Result {
|
||||
content: content.to_string(),
|
||||
class: DataClass::Secret,
|
||||
untrusted: true,
|
||||
truncated: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn request() -> ToolRequest {
|
||||
ToolRequest {
|
||||
session: SessionId::new("chat-1").unwrap(),
|
||||
call: CallId(7),
|
||||
tool: "read_file".to_string(),
|
||||
arguments: r#"{"path":"/home/kyle/notes/a.md"}"#.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn in_ms(ms: u64) -> Timestamp {
|
||||
Timestamp::from_unix_millis(Timestamp::now().unix_millis() + ms).unwrap()
|
||||
}
|
||||
|
||||
pub struct Call {
|
||||
pub response: ToolResponse,
|
||||
pub pending: Vec<Pending>,
|
||||
pub lines: Vec<String>,
|
||||
pub took: Duration,
|
||||
}
|
||||
|
||||
pub fn call(socket: PathBuf, timeout_ms: u64, request: &ToolRequest) -> Call {
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = lines.clone();
|
||||
let port = BrokerPort::with_log(
|
||||
socket,
|
||||
Duration::from_millis(timeout_ms),
|
||||
Box::new(move |line| sink.lock().unwrap().push(line.to_string())),
|
||||
);
|
||||
let mut pending = Vec::new();
|
||||
let started = Instant::now();
|
||||
let response = port.call(request, &mut |p| pending.push(*p));
|
||||
let took = started.elapsed();
|
||||
let lines = lines.lock().unwrap().clone();
|
||||
Call {
|
||||
response,
|
||||
pending,
|
||||
lines,
|
||||
took,
|
||||
}
|
||||
}
|
||||
|
||||
/// The answer is the fixed failure, and `loopd` printed exactly one line, with the pointer.
|
||||
pub fn assert_unavailable(call: &Call, why: &str) {
|
||||
assert_eq!(
|
||||
call.response,
|
||||
ToolResponse::Failed {
|
||||
message: UNAVAILABLE.to_string()
|
||||
},
|
||||
"{why}"
|
||||
);
|
||||
assert_eq!(
|
||||
call.lines.len(),
|
||||
1,
|
||||
"{why}: one line per failed call: {:?}",
|
||||
call.lines
|
||||
);
|
||||
let line = &call.lines[0];
|
||||
assert!(
|
||||
line.starts_with("loopd: the tool broker is unavailable: "),
|
||||
"{why}: {line}"
|
||||
);
|
||||
assert!(
|
||||
line.ends_with("; see docs/runbook.md#broker-unavailable"),
|
||||
"{why}: {line}"
|
||||
);
|
||||
assert!(call.pending.is_empty() || why.contains("pending"), "{why}");
|
||||
}
|
||||
Reference in New Issue
Block a user