Plan M4b: approvals over Mattermost in 15 tasks

Skeletons for new files only; existing files change by exact edits, so no task overwrites an
earlier one's work. Each task's tests were run against a reference built in order from master,
with the gate at each step and mutations checked; each skeleton compiles and fails its tests, and
every todo!() comment was read beside its signature. The reference is kept off this machine.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-09-24 03:34:21 -07:00
co-authored by Claude Opus 5.5
parent b1f732a42a
commit 2d415fc1be
64 changed files with 7701 additions and 2 deletions
@@ -0,0 +1,232 @@
//! A call an `ask` grant matched: the pending frame, the table entry, the wait, and a requester
//! that goes away. The tests answer entries by hand, as `admin` will. Do not edit.
#[path = "support/client.rs"]
mod client;
#[path = "support/rig.rs"]
mod rig;
#[path = "support/runtime.rs"]
mod runtime;
#[path = "support/sink.rs"]
mod sink;
#[path = "support/tmp.rs"]
mod tmp;
use std::os::unix::net::UnixStream;
use std::sync::Arc;
use std::time::{Duration, Instant};
use brokerd::broker::{self, Broker, GONE};
use brokerd::ledger::Answer;
use client::{Serve, next, open};
use proto::{
AuditEvent, DataClass, DecisionRecord, DenyReason, Message, ResultStatus, Timestamp,
ToolResponse,
};
use rig::{Rig, grant_text, request};
use runtime::Recording;
fn ask_notes(rig: &Rig, extra: &str) {
rig.grant(
"notes",
&grant_text("read_file", "ask", extra, "paths = [\"/n\"]"),
);
}
/// Sends a call and reads its pending frame. Returns the connection and the frame's values.
fn start(broker: &Arc<Broker>, arguments: &str) -> (UnixStream, u64, Timestamp) {
let req = request("s1", 9, "read_file", arguments);
let mut stream = open(broker, broker::handle, 9, Message::ToolRequest(req));
let env = next(&mut stream);
assert_eq!((env.id, env.r#final), (9, false), "{env:?}");
match env.msg {
Message::ToolResponse(ToolResponse::PendingApproval { approval, expires }) => {
(stream, approval, expires)
}
other => panic!("{other:?}"),
}
}
/// What `admin` does for an entry it took: record the answer, send the verdict.
fn answer_by_hand(broker: &Broker, id: u64, answer: Answer) -> DecisionRecord {
let entry = broker.table().take(id).expect("the entry is pending");
answer_entry(broker, entry, answer)
}
fn answer_entry(
broker: &Broker,
entry: brokerd::approvals::Entry,
answer: Answer,
) -> DecisionRecord {
let grants = broker.grants();
let done = broker.ledger().answer(
entry.ask,
entry.info.approval,
answer,
&grants,
Timestamp::now(),
);
entry.reply.send(done.verdict).unwrap();
done.outcome
}
fn approved() -> Answer {
Answer::Approved {
by: Some("bxctl".to_string()),
post: None,
}
}
fn final_answer(stream: &mut UnixStream) -> ToolResponse {
let env = next(stream);
assert!(env.r#final, "{env:?}");
match env.msg {
Message::ToolResponse(r) => r,
other => panic!("{other:?}"),
}
}
fn eventually(what: &str, mut done: impl FnMut() -> bool) {
let until = Instant::now() + Duration::from_secs(10);
while !done() {
assert!(Instant::now() < until, "never happened: {what}");
std::thread::sleep(Duration::from_millis(20));
}
}
#[test]
fn an_ask_call_waits_is_listed_and_runs_when_approved() {
let rig = Rig::new("pending-approve");
ask_notes(&rig, "");
let rt = Recording::answering("the notes");
let broker = rig.broker(&rt);
let before = Timestamp::now();
// Spaced JSON: the table shows the arguments as policy parsed them, not as sent.
let (mut stream, approval, expires) = start(&broker, r#"{ "path" : "/n/a" }"#);
assert_eq!(approval, 0, "the approval id is the decision's seq");
let items = broker.table().list();
assert_eq!(items.len(), 1);
let item = &items[0];
assert_eq!(item.approval, 0);
assert_eq!((item.session.as_str(), item.call.0), ("s1", 9));
assert_eq!(item.tool, "read_file");
assert_eq!(item.arguments, r#"{"path":"/n/a"}"#);
assert_eq!(
(item.grant.as_str(), item.taint),
("notes", DataClass::Private)
);
assert!(item.created >= before);
assert_eq!(item.expires, expires);
assert_eq!(
expires.unix_millis() - item.created.unix_millis(),
900_000,
"now plus ttl_ms"
);
assert_eq!(rt.count(), 0, "nothing runs while it waits");
assert_eq!(
answer_by_hand(&broker, 0, approved()),
DecisionRecord::Ask {}
);
assert!(matches!(
final_answer(&mut stream),
ToolResponse::Result { .. }
));
assert_eq!(rt.count(), 1);
let events = rig.events();
assert!(
matches!(
events.as_slice(),
[
AuditEvent::Decision { .. },
AuditEvent::Approval { decision: 0, .. },
AuditEvent::Result { decision: 0, .. }
]
),
"{events:?}"
);
}
#[test]
fn the_expiry_is_the_grants_when_that_is_earlier() {
let rig = Rig::new("pending-grant-expiry");
let soon = Timestamp::from_unix_millis(Timestamp::now().unix_millis() + 60_000).unwrap();
ask_notes(&rig, &format!("expires = \"{}\"", soon.to_rfc3339()));
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (_stream, _, expires) = start(&broker, r#"{"path":"/n/a"}"#);
assert_eq!(expires, soon);
}
#[test]
fn a_denied_verdict_reaches_the_requester_and_nothing_runs() {
let rig = Rig::new("pending-refuse");
ask_notes(&rig, "");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, approval, _) = start(&broker, r#"{"path":"/n/a"}"#);
let refused = Answer::Refused {
by: Some("bxctl".to_string()),
post: None,
reason: None,
};
answer_by_hand(&broker, approval, refused);
assert_eq!(
final_answer(&mut stream),
ToolResponse::Denied {
reason: DenyReason::ApprovalRefused
}
);
assert_eq!(rt.count(), 0);
}
#[test]
fn a_requester_that_leaves_takes_its_own_entry_and_nothing_is_written() {
let rig = Rig::new("pending-leave");
ask_notes(&rig, "");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (stream, _, _) = start(&broker, r#"{"path":"/n/a"}"#);
drop(stream);
// The waiting thread looks at its connection every second.
eventually("the entry is removed", || broker.table().list().is_empty());
std::thread::sleep(Duration::from_millis(200));
assert_eq!(rig.events().len(), 1, "only the decision");
let report = brokerd::audit::verify_dir(&rig.cfg.audit_dir()).unwrap();
assert_eq!(report.abandoned, [0]);
assert!(report.failure.is_none());
assert_eq!(rt.count(), 0);
}
#[test]
fn a_requester_gone_at_the_last_look_runs_nothing_and_closes_the_call() {
let rig = Rig::new("pending-lastlook");
ask_notes(&rig, "");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (stream, approval, _) = start(&broker, r#"{"path":"/n/a"}"#);
// Take the entry first, so the waiting thread cannot take it back when it sees the
// connection gone; then approve.
let entry = broker.table().take(approval).unwrap();
drop(stream);
answer_entry(&broker, entry, approved());
eventually("the call is closed", || rig.events().len() == 3);
match &rig.events()[2] {
AuditEvent::Result {
status,
decision,
sha256,
bytes,
..
} => {
assert_eq!((*status, *decision), (ResultStatus::Failed, 0));
assert_eq!(*sha256, proto::sha256(GONE.as_bytes()).unwrap());
assert_eq!(*bytes, GONE.len() as u64);
}
other => panic!("{other:?}"),
}
assert_eq!(GONE, "the requester went away");
assert_eq!(rt.count(), 0);
let report = brokerd::audit::verify_dir(&rig.cfg.audit_dir()).unwrap();
assert!(report.unfinished.is_empty() && report.abandoned.is_empty());
}
@@ -0,0 +1,242 @@
//! Properties over sequences of calls, through `broker::handle` with many threads at once:
//! the log verifies and no `seq` repeats; taint never goes down; every `Result` follows the
//! record that let its call run; the runtime sees a call only after `allowed` or an approval.
//! Do not edit.
#[path = "support/client.rs"]
mod client;
#[path = "support/rig.rs"]
mod rig;
#[path = "support/runtime.rs"]
mod runtime;
#[path = "support/sink.rs"]
mod sink;
#[path = "support/tmp.rs"]
mod tmp;
use std::collections::BTreeMap;
use std::sync::Arc;
use std::sync::atomic::{AtomicBool, Ordering};
use std::time::Duration;
use brokerd::ledger::Answer;
use client::{Serve, call};
use proto::{
AuditEvent, AuditRecord, CallId, DataClass, DecisionRecord, Message, SessionId, Timestamp,
ToolResponse,
};
use rig::{Rig, grant_text, request};
use runtime::Recording;
fn grants(rig: &Rig) {
let with = |class: &str, max: &str| {
grant_text("read_file", "auto", "", "")
.replace(
"result_class = \"private\"",
&format!("result_class = \"{class}\""),
)
.replace("max_taint = \"secret\"", &format!("max_taint = \"{max}\""))
};
rig.grant("notes", &(with("private", "secret") + "paths = [\"/n\"]\n"));
rig.grant("keys", &(with("secret", "secret") + "paths = [\"/k\"]\n"));
// Stops applying once a session has read a secret.
rig.grant(
"public",
&(with("public", "private") + "paths = [\"/p\"]\n"),
);
rig.grant(
"never",
&grant_text("read_file", "deny", "", "paths = [\"/d\"]"),
);
rig.grant(
"asked",
&grant_text("read_file", "ask", "", "paths = [\"/a\"]"),
);
}
/// Which call ran on what authority: (session, call) of every Decision that allowed and every
/// Approval whose re-decision let the call run, keyed by the decision's seq.
fn check_sequence(records: &[AuditRecord]) -> usize {
let mut may_run: BTreeMap<u64, (SessionId, CallId)> = BTreeMap::new();
let mut taint: BTreeMap<String, DataClass> = BTreeMap::new();
let mut results = 0;
for (i, r) in records.iter().enumerate() {
assert_eq!(r.seq, i as u64, "no seq repeats or skips");
match &r.event {
AuditEvent::Decision {
session,
call,
outcome: DecisionRecord::Allowed {},
..
} => {
may_run.insert(r.seq, (session.clone(), *call));
}
AuditEvent::Approval {
session,
call,
decision,
outcome: DecisionRecord::Allowed {} | DecisionRecord::Ask {},
..
} => {
may_run.insert(*decision, (session.clone(), *call));
}
AuditEvent::Result {
session,
call,
decision,
taint_after,
..
} => {
results += 1;
let allowed = may_run.remove(decision);
assert_eq!(
allowed,
Some((session.clone(), *call)),
"seq {}: a Result with no record letting its call run",
r.seq
);
let before = taint.insert(session.as_str().to_string(), *taint_after);
assert!(
before.is_none_or(|b| b <= *taint_after),
"seq {}: taint went down",
r.seq
);
}
_ => {}
}
}
assert!(
may_run.is_empty(),
"calls allowed but never finished: {may_run:?}"
);
results
}
#[test]
fn eight_threads_two_sessions_fifty_calls_each() {
let rig = Rig::new("sequence-many");
grants(&rig);
let rt = Recording::answering("content");
let broker = rig.broker(&rt);
let paths = ["/n/x", "/k/x", "/p/x", "/d/x", "/none/x"];
let threads: Vec<_> = (0..8u64)
.map(|t| {
let broker = Arc::clone(&broker);
std::thread::spawn(move || {
let session = if t % 2 == 0 { "s-even" } else { "s-odd" };
for i in 0..50u64 {
let path = paths[((t + i) % 5) as usize];
let args = format!(r#"{{"path":"{path}"}}"#);
let req = request(session, t * 1000 + i, "read_file", &args);
let frames = call(&broker, req);
assert_eq!(frames.len(), 1);
}
})
})
.collect();
for t in threads {
t.join().unwrap();
}
let report = brokerd::audit::verify_dir(&rig.cfg.audit_dir()).unwrap();
assert!(report.failure.is_none(), "{:?}", report.failure);
assert!(report.unfinished.is_empty());
let records = rig.records();
let decisions = records
.iter()
.filter(|r| matches!(r.event, AuditEvent::Decision { .. }))
.count();
assert_eq!(decisions, 400);
let results = check_sequence(&records);
assert_eq!(
rt.count(),
results,
"the runtime saw exactly the allowed calls"
);
assert!(results > 0 && results < 400);
// Both sessions read a secret.
for s in ["s-even", "s-odd"] {
let state = rig.state().read(&SessionId::new(s).unwrap()).unwrap();
assert_eq!(state.taint, DataClass::Secret);
}
}
#[test]
fn approved_and_refused_calls_run_only_after_an_approval() {
let rig = Rig::new("sequence-ask");
grants(&rig);
let rt = Recording::answering("content");
let broker = rig.broker(&rt);
// An owner who approves even ids and refuses odd ones, as fast as they appear.
let stop = Arc::new(AtomicBool::new(false));
let owner = {
let (broker, stop) = (Arc::clone(&broker), Arc::clone(&stop));
std::thread::spawn(move || {
while !stop.load(Ordering::SeqCst) {
for item in broker.table().list() {
let Some(entry) = broker.table().take(item.approval) else {
continue;
};
let answer = if item.approval % 2 == 0 {
Answer::Approved {
by: None,
post: None,
}
} else {
Answer::Refused {
by: None,
post: None,
reason: None,
}
};
let grants = broker.grants();
let done = broker.ledger().answer(
entry.ask,
item.approval,
answer,
&grants,
Timestamp::now(),
);
entry.reply.send(done.verdict).unwrap();
}
std::thread::sleep(Duration::from_millis(2));
}
})
};
let threads: Vec<_> = (0..4u64)
.map(|t| {
let broker = Arc::clone(&broker);
std::thread::spawn(move || {
for i in 0..10u64 {
let path = if i % 2 == 0 { "/a/x" } else { "/n/x" };
let args = format!(r#"{{"path":"{path}"}}"#);
let frames = call(&broker, request("s1", t * 100 + i, "read_file", &args));
let last = frames.last().unwrap();
assert!(
matches!(
&last.msg,
Message::ToolResponse(
ToolResponse::Result { .. } | ToolResponse::Denied { .. }
)
),
"{last:?}"
);
}
})
})
.collect();
for t in threads {
t.join().unwrap();
}
stop.store(true, Ordering::SeqCst);
owner.join().unwrap();
let records = rig.records();
let approvals = records
.iter()
.filter(|r| matches!(r.event, AuditEvent::Approval { .. }))
.count();
assert_eq!(approvals, 20, "one Approval for each of the twenty asks");
let results = check_sequence(&records);
assert_eq!(rt.count(), results);
let report = brokerd::audit::verify_dir(&rig.cfg.audit_dir()).unwrap();
assert!(report.failure.is_none() && report.abandoned.is_empty());
}
@@ -0,0 +1,43 @@
//! `[gateway]` in `brokerd.toml`: absent means Mattermost approvals are off; an empty socket is
//! the default under `home` (M4b spec, section 3). Do not edit.
use std::path::PathBuf;
use brokerd::config::Config;
fn parse(text: &str) -> Config {
Config::parse(&format!("[paths]\nhome = \"/srv/bx\"\n{text}")).unwrap()
}
#[test]
fn without_the_section_approvals_stay_in_bxctl() {
let config = parse("");
assert_eq!(config.gateway, None);
assert_eq!(config.gateway_socket(), None);
}
#[test]
fn an_empty_section_or_socket_is_the_default() {
let default = Some(PathBuf::from("/srv/bx/run/gateway/gateway.sock"));
assert_eq!(parse("[gateway]\n").gateway_socket(), default);
assert_eq!(
parse("[gateway]\nsocket = \"\"\n").gateway_socket(),
default
);
}
#[test]
fn a_socket_path_is_used_as_given() {
let config = parse("[gateway]\nsocket = \"/run/bx/gw.sock\"\n");
assert_eq!(
config.gateway_socket(),
Some(PathBuf::from("/run/bx/gw.sock"))
);
}
#[test]
fn unknown_keys_are_errors() {
assert!(Config::parse("[gateway]\nsockets = \"/x\"\n").is_err());
assert!(Config::parse("[gateway]\nsocket = \"/x\"\nenabled = true\n").is_err());
assert!(Config::parse("[gatewy]\nsocket = \"/x\"\n").is_err());
}
@@ -0,0 +1,447 @@
//! The courier carries a pending approval to `gatewayd` and its answer back, against a fake
//! `gatewayd` on a real socket (M4b spec, section 5). Do not edit.
#[path = "support/client.rs"]
mod client;
#[path = "support/fake_gateway.rs"]
mod fake_gateway;
#[path = "support/rig.rs"]
mod rig;
#[path = "support/runtime.rs"]
mod runtime;
#[path = "support/sink.rs"]
mod sink;
#[path = "support/tmp.rs"]
mod tmp;
use std::os::unix::net::UnixStream;
use std::sync::Arc;
use std::sync::atomic::AtomicBool;
use std::time::{Duration, Instant};
use brokerd::broker::{self, Broker};
use brokerd::config::Gateway;
use brokerd::courier::{self, Timing};
use brokerd::ledger::Answer;
use client::{Serve, next, open};
use fake_gateway::{FakeGateway, KYLE, POST};
use proto::{
ApprovalAnswer, ApprovalReply, AuditEvent, DenyReason, GatewayOutcome, Message, Timestamp,
ToolResponse,
};
use rig::{Rig, grant_text, request};
use runtime::Recording;
const WAIT: Duration = Duration::from_secs(5);
/// A rig whose `[gateway]` is a fake gatewayd in its home, and an `ask` grant for /n.
fn rig_with_gateway(tag: &str) -> (Rig, FakeGateway) {
let mut rig = Rig::new(tag);
let socket = rig.dir.path().join("gw.sock");
let fake = FakeGateway::start(&socket);
rig.cfg.gateway = Some(Gateway { socket });
rig.grant(
"notes",
&grant_text("read_file", "ask", "", "paths = [\"/n\"]"),
);
(rig, fake)
}
/// A call that waits for approval; its connection and the approval id.
fn ask(broker: &Arc<Broker>) -> (UnixStream, u64) {
let mut stream = open(
broker,
broker::handle,
9,
Message::ToolRequest(request("s1", 9, "read_file", r#"{"path":"/n/a"}"#)),
);
match next(&mut stream).msg {
Message::ToolResponse(ToolResponse::PendingApproval { approval, .. }) => (stream, approval),
other => panic!("{other:?}"),
}
}
fn final_answer(stream: &mut UnixStream) -> ToolResponse {
let env = next(stream);
assert!(env.r#final, "{env:?}");
match env.msg {
Message::ToolResponse(r) => r,
other => panic!("{other:?}"),
}
}
/// The `Approval` record's (answer, by, post, reason).
fn approval_record(
rig: &Rig,
) -> (
ApprovalAnswer,
Option<String>,
Option<String>,
Option<String>,
) {
let events = rig.events();
let found = events.iter().find_map(|e| match e {
AuditEvent::Approval {
answer,
by,
post,
reason,
..
} => Some((*answer, by.clone(), post.clone(), reason.clone())),
_ => None,
});
found.expect("an Approval record")
}
fn some(s: &str) -> Option<String> {
Some(s.to_string())
}
/// Sets the courier's stop flag when dropped, so a failing assertion ends the courier too.
struct StopOnDrop<'a>(&'a AtomicBool);
impl Drop for StopOnDrop<'_> {
fn drop(&mut self) {
self.0.store(true, std::sync::atomic::Ordering::SeqCst);
}
}
/// The courier must end by itself within five seconds.
fn ends_by_itself<T>(courier: &std::thread::ScopedJoinHandle<'_, T>) {
let until = Instant::now() + WAIT;
while !courier.is_finished() {
assert!(Instant::now() < until, "the courier did not stop");
std::thread::sleep(Duration::from_millis(20));
}
}
#[test]
fn approved_in_mattermost_it_runs_and_the_record_names_who_and_where() {
let (rig, fake) = rig_with_gateway("courier-approve");
let rt = Recording::answering("the notes");
let broker = rig.broker(&rt);
let (mut stream, id) = ask(&broker);
let mut conn = fake.next(WAIT).expect("the approval reaches gatewayd");
assert_eq!(
(conn.request.approval, conn.request.tool.as_str()),
(id, "read_file")
);
assert_eq!(conn.request.arguments, r#"{"path":"/n/a"}"#);
conn.posted();
conn.approve();
assert_eq!(conn.outcome(), Some(GatewayOutcome::Runs {}));
assert!(matches!(
final_answer(&mut stream),
ToolResponse::Result { .. }
));
assert_eq!(rt.count(), 1);
assert_eq!(
approval_record(&rig),
(ApprovalAnswer::Approved, some(KYLE), some(POST), None)
);
}
#[test]
fn refused_in_mattermost_it_is_denied_with_the_reason() {
let (rig, fake) = rig_with_gateway("courier-refuse");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, _) = ask(&broker);
let mut conn = fake.next(WAIT).unwrap();
conn.posted();
conn.refuse("not on a Friday");
assert_eq!(conn.outcome(), Some(GatewayOutcome::Refused {}));
assert_eq!(
final_answer(&mut stream),
ToolResponse::Denied {
reason: DenyReason::ApprovalRefused
}
);
assert_eq!(rt.count(), 0);
assert_eq!(
approval_record(&rig),
(
ApprovalAnswer::Refused,
some(KYLE),
some(POST),
some("not on a Friday")
)
);
}
#[test]
fn approved_after_the_grant_is_gone_it_is_denied_on_the_recheck() {
let (rig, fake) = rig_with_gateway("courier-recheck");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, _) = ask(&broker);
let mut conn = fake.next(WAIT).unwrap();
conn.posted();
rig.remove_grant("notes");
conn.approve();
assert_eq!(
conn.outcome(),
Some(GatewayOutcome::Denied {
reason: DenyReason::NoGrant
})
);
assert_eq!(
final_answer(&mut stream),
ToolResponse::Denied {
reason: DenyReason::NoGrant
}
);
assert_eq!(rt.count(), 0);
}
#[test]
fn answered_in_bxctl_the_courier_closes_without_an_outcome() {
let (rig, fake) = rig_with_gateway("courier-bxctl");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, id) = ask(&broker);
let mut conn = fake.next(WAIT).unwrap();
conn.posted();
let entry = broker.table().take(id).unwrap();
brokerd::admin::answer(
&broker,
entry,
Answer::Approved {
by: some("bxctl"),
post: None,
},
Timestamp::now(),
);
assert!(
conn.closed_within(WAIT),
"the courier closes, and sends nothing"
);
assert!(matches!(
final_answer(&mut stream),
ToolResponse::Result { .. }
));
assert_eq!(approval_record(&rig).1, some("bxctl"));
}
#[test]
fn an_answer_after_it_ended_is_gone() {
let rig = Rig::new("courier-gone");
rig.grant(
"notes",
&grant_text("read_file", "ask", "", "paths = [\"/n\"]"),
);
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (_stream, id) = ask(&broker);
// The courier is run by hand here, so that the answer can arrive after the entry is taken.
let socket = rig.dir.path().join("gw.sock");
let fake = FakeGateway::start(&socket);
let info = broker.table().list().remove(0);
let stop = AtomicBool::new(false);
std::thread::scope(|scope| {
let courier = scope.spawn(|| {
courier::carry(
&broker,
&socket,
&info,
&stop,
Timing {
retry: Duration::from_millis(100),
tick: Duration::from_secs(5),
},
)
});
let _guard = StopOnDrop(&stop);
let mut conn = fake.next(WAIT).unwrap();
conn.posted();
let entry = broker.table().take(id).unwrap();
brokerd::admin::answer(&broker, entry, Answer::Expired, Timestamp::now());
conn.approve();
assert_eq!(conn.outcome(), Some(GatewayOutcome::Gone {}));
ends_by_itself(&courier);
});
}
#[test]
fn if_gatewayd_goes_away_the_approval_is_posted_again() {
let (rig, fake) = rig_with_gateway("courier-again");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, id) = ask(&broker);
let mut first = fake.next(WAIT).unwrap();
first.posted();
first.close();
let mut second = fake.next(Duration::from_secs(10)).expect("posted again");
assert_eq!(second.request.approval, id);
second.posted();
second.approve();
assert_eq!(second.outcome(), Some(GatewayOutcome::Runs {}));
assert!(matches!(
final_answer(&mut stream),
ToolResponse::Result { .. }
));
assert!(
rig.lines.all().iter().any(|l| l.contains(&format!(
"approval {id}: gatewayd went away; posting it again"
))),
"{:?}",
rig.lines.all()
);
}
#[test]
fn a_bad_answer_answers_nothing() {
let (rig, fake) = rig_with_gateway("courier-bad");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (_stream, id) = ask(&broker);
for bad in [
ApprovalReply {
approval: id + 1,
approved: true,
by: KYLE.to_string(),
post: POST.to_string(),
reason: None,
},
ApprovalReply {
approval: id,
approved: true,
by: "kyle".to_string(),
post: POST.to_string(),
reason: None,
},
ApprovalReply {
approval: id,
approved: false,
by: KYLE.to_string(),
post: POST.to_string(),
reason: Some("x".repeat(501)),
},
] {
let mut conn = fake.next(Duration::from_secs(10)).unwrap();
conn.posted();
conn.reply(bad);
assert_eq!(conn.outcome(), None, "no outcome for a bad answer");
assert!(broker.table().contains(id), "still pending");
}
assert_eq!(rt.count(), 0);
}
#[test]
fn unreachable_at_first_then_posted_and_said_once() {
let rig = Rig::new("courier-late");
rig.grant(
"notes",
&grant_text("read_file", "ask", "", "paths = [\"/n\"]"),
);
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (_stream, id) = ask(&broker);
let socket = rig.dir.path().join("gw.sock");
let info = broker.table().list().remove(0);
let stop = AtomicBool::new(false);
let timing = Timing {
retry: Duration::from_millis(100),
tick: Duration::from_millis(20),
};
std::thread::scope(|scope| {
let courier = scope.spawn(|| courier::carry(&broker, &socket, &info, &stop, timing));
let _guard = StopOnDrop(&stop);
std::thread::sleep(Duration::from_millis(500));
let fake = FakeGateway::start(&socket);
let mut conn = fake.next(WAIT).unwrap();
conn.posted();
let until = Instant::now() + WAIT;
while !rig
.lines
.all()
.iter()
.any(|l| l.contains("posted in Mattermost"))
{
assert!(Instant::now() < until, "{:?}", rig.lines.all());
std::thread::sleep(Duration::from_millis(20));
}
broker.table().take(id);
ends_by_itself(&courier);
});
let lines = rig.lines.all();
let unreachable: Vec<&String> = lines
.iter()
.filter(|l| l.contains("gatewayd is not reachable"))
.collect();
assert_eq!(
unreachable.len(),
1,
"said once, not every retry: {lines:?}"
);
assert!(
unreachable[0].starts_with(&format!(
"brokerd: approval {id}: gatewayd is not reachable at "
)),
"{lines:?}"
);
assert!(
unreachable[0]
.ends_with("; bxctl can still answer it\nsee docs/runbook.md#gateway-unreachable"),
"{lines:?}"
);
assert!(
lines.contains(&format!("brokerd: approval {id} posted in Mattermost")),
"{lines:?}"
);
}
#[test]
fn if_gatewayd_cannot_post_it_is_tried_again() {
let (rig, fake) = rig_with_gateway("courier-cannot");
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, _) = ask(&broker);
fake.next(WAIT).unwrap().cannot_post();
let mut conn = fake.next(Duration::from_secs(10)).expect("tried again");
conn.posted();
conn.approve();
assert_eq!(conn.outcome(), Some(GatewayOutcome::Runs {}));
assert!(matches!(
final_answer(&mut stream),
ToolResponse::Result { .. }
));
assert!(
rig.lines
.all()
.iter()
.any(|l| l.contains("gatewayd could not post it: Mattermost is down"))
);
}
#[test]
fn without_a_gateway_nothing_is_carried() {
let rig = Rig::new("courier-none");
rig.grant(
"notes",
&grant_text("read_file", "ask", "", "paths = [\"/n\"]"),
);
let socket = rig.dir.path().join("gw.sock");
let fake = FakeGateway::start(&socket);
let rt = Recording::answering("x");
let broker = rig.broker(&rt);
let (mut stream, id) = ask(&broker);
assert!(
fake.next(Duration::from_millis(500)).is_none(),
"no [gateway]: no connection"
);
let entry = broker.table().take(id).unwrap();
brokerd::admin::answer(
&broker,
entry,
Answer::Approved {
by: some("bxctl"),
post: None,
},
Timestamp::now(),
);
assert!(matches!(
final_answer(&mut stream),
ToolResponse::Result { .. }
));
}
@@ -0,0 +1,264 @@
//! The ledger's second step: an approval decides again and is recorded; a refusal and an expiry
//! are recorded as denials. Do not edit.
#[path = "support/build.rs"]
mod build;
#[path = "support/rig.rs"]
mod rig;
#[path = "support/sink.rs"]
mod sink;
#[path = "support/tmp.rs"]
mod tmp;
use brokerd::approvals::Verdict;
use brokerd::ledger::{Answer, Answered, Decided, Grants, Ledger};
use brokerd::policy::{Ask, Label};
use build::{grant, now, read, set};
use proto::{
ApprovalAnswer, AuditEvent, CallId, DataClass, DecisionRecord, DenyReason, Mode, SessionId,
};
use rig::Rig;
fn asking() -> Grants {
Ok(set(vec![grant("n", "read_file", Mode::Ask).paths(&["/n"])]))
}
fn pending(ledger: &Ledger) -> Ask {
match ledger.decide(read("/n/deep/a"), &asking(), now()) {
Decided::Ask { ask, seq: 0, .. } => ask,
other => panic!("{other:?}"),
}
}
fn bxctl() -> Answer {
Answer::Approved {
by: Some("bxctl".to_string()),
post: None,
}
}
fn denied(reason: DenyReason) -> DecisionRecord {
DecisionRecord::Denied { reason }
}
fn verdict_reason(answered: &Answered) -> Option<DenyReason> {
match &answered.verdict {
Verdict::Run(_) => None,
Verdict::Denied(reason) => Some(*reason),
}
}
/// The one `Approval` record, as (answer, by, reason, outcome, grant).
type Row = (
ApprovalAnswer,
Option<String>,
Option<String>,
DecisionRecord,
Option<String>,
);
fn approval(rig: &Rig) -> Row {
let events = rig.events();
assert_eq!(events.len(), 2, "a decision and one approval: {events:?}");
match &events[1] {
AuditEvent::Approval {
session,
call,
decision,
answer,
by,
post,
reason,
outcome,
grant,
..
} => {
assert_eq!(session, &SessionId::new("s1").unwrap());
assert_eq!((*call, *decision, post), (CallId(1), 0, &None));
(
*answer,
by.clone(),
reason.clone(),
outcome.clone(),
grant.clone(),
)
}
other => panic!("{other:?}"),
}
}
#[test]
fn an_approval_that_still_asks_lets_the_call_run() {
let rig = Rig::new("answer-ask");
let ledger = rig.ledger();
let ask = pending(&ledger);
let answered = ledger.answer(ask, 0, bxctl(), &asking(), now());
match &answered.verdict {
Verdict::Run(decision) => assert_eq!(decision.grant(), "n"),
other => panic!("{other:?}"),
}
assert_eq!(answered.outcome, DecisionRecord::Ask {});
let row = approval(&rig);
assert_eq!(
row,
(
ApprovalAnswer::Approved,
Some("bxctl".to_string()),
None,
DecisionRecord::Ask {},
Some("n".to_string())
)
);
}
#[test]
fn an_approval_under_a_grant_that_is_now_auto_records_allowed() {
let rig = Rig::new("answer-auto");
let ledger = rig.ledger();
let ask = pending(&ledger);
let now_auto = Ok(set(vec![
grant("n", "read_file", Mode::Auto).paths(&["/n"]),
]));
let answered = ledger.answer(ask, 0, bxctl(), &now_auto, now());
assert!(matches!(answered.verdict, Verdict::Run(_)));
assert_eq!(answered.outcome, DecisionRecord::Allowed {});
}
#[test]
fn the_approval_names_the_grant_matched_now() {
let rig = Rig::new("answer-grant");
let ledger = rig.ledger();
let ask = pending(&ledger);
let wider = Ok(set(vec![
grant("n", "read_file", Mode::Ask).paths(&["/n"]),
grant("z-deep", "read_file", Mode::Ask).paths(&["/n/deep"]),
]));
let answered = ledger.answer(ask, 0, bxctl(), &wider, now());
match &answered.verdict {
Verdict::Run(decision) => assert_eq!(decision.grant(), "z-deep"),
other => panic!("{other:?}"),
}
assert_eq!(approval(&rig).4.as_deref(), Some("z-deep"));
}
#[test]
fn an_approval_after_the_grant_is_gone_is_denied() {
let rig = Rig::new("answer-gone");
let ledger = rig.ledger();
let ask = pending(&ledger);
let answered = ledger.answer(ask, 0, bxctl(), &Ok(set(Vec::new())), now());
assert_eq!(verdict_reason(&answered), Some(DenyReason::NoGrant));
assert_eq!(answered.outcome, denied(DenyReason::NoGrant));
let row = approval(&rig);
assert_eq!(
(row.0, row.3, row.4),
(ApprovalAnswer::Approved, denied(DenyReason::NoGrant), None)
);
}
#[test]
fn an_approval_after_the_taint_rose_is_denied() {
let rig = Rig::new("answer-taint");
let ledger = rig.ledger();
let low = || {
Ok(set(vec![
grant("n", "read_file", Mode::Ask)
.paths(&["/n"])
.max_taint(DataClass::Private),
]))
};
let ask = match ledger.decide(read("/n/a"), &low(), now()) {
Decided::Ask { ask, .. } => ask,
other => panic!("{other:?}"),
};
// Another call of the session read a secret while this one waited.
let s1 = SessionId::new("s1").unwrap();
let label = Label {
class: DataClass::Secret,
untrusted: false,
};
rig.state()
.raise(&s1, rig.state().read(&s1).unwrap(), label)
.unwrap();
let answered = ledger.answer(ask, 0, bxctl(), &low(), now());
assert_eq!(verdict_reason(&answered), Some(DenyReason::TaintTooHigh));
match &rig.events()[1] {
AuditEvent::Approval { taint, outcome, .. } => {
assert_eq!(*taint, DataClass::Secret, "the state at the re-decision");
assert_eq!(*outcome, denied(DenyReason::TaintTooHigh));
}
other => panic!("{other:?}"),
}
}
#[test]
fn an_approval_with_invalid_grants_is_denied() {
let rig = Rig::new("answer-invalid");
let ledger = rig.ledger();
let ask = pending(&ledger);
let answered = ledger.answer(ask, 0, bxctl(), &Err(Vec::new()), now());
assert_eq!(verdict_reason(&answered), Some(DenyReason::GrantsInvalid));
}
#[test]
fn a_refusal_is_recorded_with_the_owners_reason() {
let rig = Rig::new("answer-refuse");
let ledger = rig.ledger();
let ask = pending(&ledger);
let refused = Answer::Refused {
by: Some("bxctl".to_string()),
post: None,
reason: Some("not now".to_string()),
};
let answered = ledger.answer(ask, 0, refused, &asking(), now());
assert_eq!(verdict_reason(&answered), Some(DenyReason::ApprovalRefused));
assert_eq!(
approval(&rig),
(
ApprovalAnswer::Refused,
Some("bxctl".to_string()),
Some("not now".to_string()),
denied(DenyReason::ApprovalRefused),
None
)
);
}
#[test]
fn an_expiry_is_recorded_by_nobody() {
let rig = Rig::new("answer-expire");
let ledger = rig.ledger();
let ask = pending(&ledger);
let answered = ledger.answer(ask, 0, Answer::Expired, &asking(), now());
assert_eq!(verdict_reason(&answered), Some(DenyReason::ApprovalExpired));
assert_eq!(
approval(&rig),
(
ApprovalAnswer::Expired,
None,
None,
denied(DenyReason::ApprovalExpired),
None
)
);
}
#[test]
fn an_approval_that_cannot_be_recorded_does_not_run() {
let rig = Rig::new("answer-norecord");
let ledger = rig.ledger();
let ask = pending(&ledger);
rig.switch.fail(true);
let answered = ledger.answer(ask, 0, bxctl(), &asking(), now());
assert_eq!(
verdict_reason(&answered),
Some(DenyReason::AuditUnavailable)
);
assert_eq!(answered.outcome, denied(DenyReason::AuditUnavailable));
assert_eq!(rig.events().len(), 1, "only the decision");
assert!(
!rig.lines
.with("see docs/runbook.md#audit-unavailable")
.is_empty()
);
}
@@ -0,0 +1,118 @@
//! An answer from Mattermost records who answered and in which post, approved or refused; the
//! table says whether an approval is still pending (M4b spec, section 5). Do not edit.
#[path = "support/build.rs"]
mod build;
#[path = "support/rig.rs"]
mod rig;
#[path = "support/sink.rs"]
mod sink;
#[path = "support/tmp.rs"]
mod tmp;
use brokerd::approvals::Table;
use brokerd::ledger::{Answer, Decided, Grants, Ledger};
use brokerd::policy::Ask;
use build::{grant, now, read, set};
use proto::{ApprovalAnswer, AuditEvent, CallId, DataClass, Mode, PendingApproval, SessionId};
use rig::Rig;
const KYLE: &str = "k0000000000000000000000000";
const POST: &str = "c0000000000000000000000001";
fn asking() -> Grants {
Ok(set(vec![grant("n", "read_file", Mode::Ask).paths(&["/n"])]))
}
fn pending(ledger: &Ledger) -> Ask {
match ledger.decide(read("/n/deep/a"), &asking(), now()) {
Decided::Ask { ask, seq: 0, .. } => ask,
other => panic!("{other:?}"),
}
}
/// The one `Approval` record's (answer, by, post, reason).
fn recorded(
rig: &Rig,
) -> (
ApprovalAnswer,
Option<String>,
Option<String>,
Option<String>,
) {
let events = rig.events();
match &events[1] {
AuditEvent::Approval {
answer,
by,
post,
reason,
..
} => (*answer, by.clone(), post.clone(), reason.clone()),
other => panic!("{other:?}"),
}
}
#[test]
fn an_approval_from_mattermost_names_the_user_and_the_post() {
let rig = Rig::new("post-approve");
let ledger = rig.ledger();
let answer = Answer::Approved {
by: Some(KYLE.to_string()),
post: Some(POST.to_string()),
};
ledger.answer(pending(&ledger), 0, answer, &asking(), now());
assert_eq!(
recorded(&rig),
(
ApprovalAnswer::Approved,
Some(KYLE.to_string()),
Some(POST.to_string()),
None
)
);
}
#[test]
fn a_refusal_from_mattermost_names_them_and_keeps_the_reason() {
let rig = Rig::new("post-refuse");
let ledger = rig.ledger();
let answer = Answer::Refused {
by: Some(KYLE.to_string()),
post: Some(POST.to_string()),
reason: Some("not on a Friday".to_string()),
};
ledger.answer(pending(&ledger), 0, answer, &asking(), now());
assert_eq!(
recorded(&rig),
(
ApprovalAnswer::Refused,
Some(KYLE.to_string()),
Some(POST.to_string()),
Some("not on a Friday".to_string())
)
);
}
#[test]
fn the_table_says_what_is_pending() {
let rig = Rig::new("post-table");
let ledger = rig.ledger();
let table = Table::new();
assert!(!table.contains(0));
let info = PendingApproval {
approval: 0,
session: SessionId::new("s1").unwrap(),
call: CallId(1),
tool: "read_file".to_string(),
arguments: "{}".to_string(),
grant: "n".to_string(),
taint: DataClass::Private,
created: now(),
expires: now(),
};
let _verdict = table.insert(info, pending(&ledger));
assert!(table.contains(0) && !table.contains(1));
assert!(table.take(0).is_some());
assert!(!table.contains(0));
}
@@ -0,0 +1,150 @@
//! A fake `gatewayd` on a Unix socket: it accepts each courier's connection, reads the approval,
//! and lets the test post it, answer it, close, or read the outcome. Do not edit.
#![allow(dead_code)] // each test file uses a different part of this module
use std::os::unix::net::{UnixListener, UnixStream};
use std::path::{Path, PathBuf};
use std::sync::mpsc::{self, Receiver};
use std::time::Duration;
use proto::{
ApprovalPosted, ApprovalReply, Envelope, ErrorCode, GatewayOutcome, Message, PROTOCOL_VERSION,
PendingApproval, WireError, read_frame, write_frame,
};
pub const KYLE: &str = "k0000000000000000000000000";
pub const POST: &str = "c0000000000000000000000001";
pub struct FakeGateway {
pub socket: PathBuf,
conns: Receiver<Conn>,
}
/// One courier's connection, its request already read.
pub struct Conn {
stream: UnixStream,
pub request: PendingApproval,
}
fn frame(id: u64, last: bool, msg: Message) -> Envelope {
Envelope {
v: PROTOCOL_VERSION,
id,
r#final: last,
msg,
}
}
impl FakeGateway {
/// Listen on `socket` (created now).
pub fn start(socket: &Path) -> FakeGateway {
let listener = UnixListener::bind(socket).unwrap();
let (tx, conns) = mpsc::channel();
std::thread::spawn(move || {
for stream in listener.incoming() {
let Ok(mut stream) = stream else { continue };
let Ok(env) = read_frame(&mut stream) else {
continue;
};
let Message::ApprovalRequest(req) = env.msg else {
panic!("not a request: {env:?}")
};
assert_eq!((env.id, env.r#final), (1, true));
if tx
.send(Conn {
stream,
request: req.approval,
})
.is_err()
{
return;
}
}
});
FakeGateway {
socket: socket.to_path_buf(),
conns,
}
}
/// The next courier connection within `wait`, or `None`.
pub fn next(&self, wait: Duration) -> Option<Conn> {
self.conns.recv_timeout(wait).ok()
}
}
impl Conn {
pub fn posted(&mut self) {
let posts = vec![POST.to_string()];
write_frame(
&mut self.stream,
&frame(1, false, Message::ApprovalPosted(ApprovalPosted { posts })),
)
.unwrap();
}
pub fn cannot_post(&mut self) {
let e = WireError {
code: ErrorCode::Internal,
detail: "Mattermost is down".to_string(),
};
write_frame(&mut self.stream, &frame(1, true, Message::Error(e))).unwrap();
}
pub fn reply(&mut self, reply: ApprovalReply) {
let _ = write_frame(
&mut self.stream,
&frame(1, true, Message::ApprovalReply(reply)),
);
}
pub fn approve(&mut self) {
let approval = self.request.approval;
self.reply(ApprovalReply {
approval,
approved: true,
by: KYLE.to_string(),
post: POST.to_string(),
reason: None,
});
}
pub fn refuse(&mut self, reason: &str) {
let approval = self.request.approval;
self.reply(ApprovalReply {
approval,
approved: false,
by: KYLE.to_string(),
post: POST.to_string(),
reason: Some(reason.to_string()),
});
}
/// Frame 4 within five seconds, or `None` if the courier closed without one.
pub fn outcome(&mut self) -> Option<GatewayOutcome> {
self.stream
.set_read_timeout(Some(Duration::from_secs(5)))
.unwrap();
match read_frame(&mut self.stream) {
Ok(env) => {
assert_eq!((env.id, env.r#final), (2, true));
match env.msg {
Message::ApprovalOutcome(o) => Some(o.outcome),
other => panic!("not an outcome: {other:?}"),
}
}
Err(_) => None,
}
}
/// Did the courier close the connection within `wait`, without sending anything?
pub fn closed_within(&mut self, wait: Duration) -> bool {
self.stream.set_read_timeout(Some(wait)).unwrap();
matches!(read_frame(&mut self.stream), Err(proto::FrameError::Closed))
}
pub fn close(self) {
let _ = self.stream.shutdown(std::net::Shutdown::Both);
}
}
@@ -0,0 +1,120 @@
//! A home for ledger and broker tests: grants, audit log, session state, a flaky audit sink and
//! a log to read. Do not edit.
//!
//! Included with `#[path = "support/rig.rs"] mod rig;`, beside `tmp` and `sink`. The broker
//! tests add `client`.
#![allow(dead_code)] // each test file uses a different part of this module
use std::path::PathBuf;
use brokerd::audit::Writer;
use brokerd::config::{Approvals, Config, Paths, Sockets};
use brokerd::ledger::Ledger;
use brokerd::state::StateStore;
use proto::{AuditEvent, AuditRecord, CallId, SessionId, ToolRequest};
use crate::sink::{Flaky, Lines, Switch};
use crate::tmp::TempDir;
pub struct Rig {
pub dir: TempDir,
pub cfg: Config,
pub switch: Switch,
pub lines: Lines,
}
impl Rig {
pub fn new(tag: &str) -> Rig {
Rig::with_ttl(tag, 900_000)
}
pub fn with_ttl(tag: &str, ttl_ms: u64) -> Rig {
let dir = TempDir::new(tag);
let grants = dir.path().join("grants");
std::fs::create_dir_all(&grants).unwrap();
let cfg = Config {
paths: Paths {
home: dir.path().to_path_buf(),
grants,
},
sockets: Sockets::default(),
approvals: Approvals { ttl_ms },
runner: None,
gateway: None,
};
Rig {
dir,
cfg,
switch: Switch::default(),
lines: Lines::default(),
}
}
pub fn state(&self) -> StateStore {
StateStore::new(&self.cfg.state_dir())
}
/// Opens the audit log (once: the writer holds its lock) behind the flaky sink.
pub fn ledger(&self) -> Ledger {
let opened = Writer::open(&self.cfg.audit_dir(), false).unwrap();
let sink = Flaky {
writer: opened.writer,
switch: self.switch.clone(),
};
Ledger::new(Box::new(sink), self.state(), self.lines.sink())
}
/// Writes `grants/<id>.toml`.
pub fn grant(&self, id: &str, text: &str) {
std::fs::write(self.cfg.paths.grants.join(format!("{id}.toml")), text).unwrap();
}
pub fn remove_grant(&self, id: &str) {
std::fs::remove_file(self.cfg.paths.grants.join(format!("{id}.toml"))).unwrap();
}
pub fn state_file(&self, session: &str) -> PathBuf {
self.cfg.state_dir().join(format!("{session}.json"))
}
/// Every record in the audit log, in order.
pub fn records(&self) -> Vec<AuditRecord> {
let dir = self.cfg.audit_dir();
let mut names: Vec<String> = std::fs::read_dir(&dir)
.unwrap()
.map(|e| e.unwrap().file_name().into_string().unwrap())
.filter(|n| n.ends_with(".jsonl"))
.collect();
names.sort();
let mut out = Vec::new();
for name in names {
let text = std::fs::read_to_string(dir.join(name)).unwrap();
for line in text.lines() {
out.push(serde_json::from_str(line).unwrap());
}
}
out
}
pub fn events(&self) -> Vec<AuditEvent> {
self.records().into_iter().map(|r| r.event).collect()
}
}
/// A grant file's text. `extra` goes before `[constraints]`, `constraints` after it.
pub fn grant_text(tool: &str, mode: &str, extra: &str, constraints: &str) -> String {
format!(
"tool = \"{tool}\"\nmode = \"{mode}\"\nmax_taint = \"secret\"\nresult_class = \"private\"\n\
untrusted = false\n{extra}\n[constraints]\n{constraints}\n"
)
}
pub fn request(session: &str, call: u64, tool: &str, arguments: &str) -> ToolRequest {
ToolRequest {
session: SessionId::new(session).unwrap(),
call: CallId(call),
tool: tool.to_string(),
arguments: arguments.to_string(),
}
}