//! The courier carries a pending approval to `gatewayd` and its answer back, against a fake //! `gatewayd` on a real socket (M4b spec, section 5). Do not edit. #[path = "support/client.rs"] mod client; #[path = "support/fake_gateway.rs"] mod fake_gateway; #[path = "support/rig.rs"] mod rig; #[path = "support/runtime.rs"] mod runtime; #[path = "support/sink.rs"] mod sink; #[path = "support/tmp.rs"] mod tmp; use std::os::unix::net::UnixStream; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::time::{Duration, Instant}; use brokerd::broker::{self, Broker}; use brokerd::config::Gateway; use brokerd::courier::{self, Timing}; use brokerd::ledger::Answer; use bxcore::{ ApprovalAnswer, ApprovalReply, AuditEvent, DenyReason, GatewayOutcome, Message, Timestamp, ToolResponse, }; use client::{Serve, next, open}; use fake_gateway::{FakeGateway, KYLE, POST}; use rig::{Rig, grant_text, request}; use runtime::Recording; const WAIT: Duration = Duration::from_secs(5); /// A rig whose `[gateway]` is a fake gatewayd in its home, and an `ask` grant for /n. fn rig_with_gateway(tag: &str) -> (Rig, FakeGateway) { let mut rig = Rig::new(tag); let socket = rig.dir.path().join("gw.sock"); let fake = FakeGateway::start(&socket); rig.cfg.gateway = Some(Gateway { socket }); rig.grant( "notes", &grant_text("read_file", "ask", "", "paths = [\"/n\"]"), ); (rig, fake) } /// A call that waits for approval; its connection and the approval id. fn ask(broker: &Arc) -> (UnixStream, u64) { let mut stream = open( broker, broker::handle, 9, Message::ToolRequest(request("s1", 9, "read_file", r#"{"path":"/n/a"}"#)), ); match next(&mut stream).msg { Message::ToolResponse(ToolResponse::PendingApproval { approval, .. }) => (stream, approval), other => panic!("{other:?}"), } } fn final_answer(stream: &mut UnixStream) -> ToolResponse { let env = next(stream); assert!(env.r#final, "{env:?}"); match env.msg { Message::ToolResponse(r) => r, other => panic!("{other:?}"), } } /// The `Approval` record's (answer, by, post, reason). fn approval_record( rig: &Rig, ) -> ( ApprovalAnswer, Option, Option, Option, ) { let events = rig.events(); let found = events.iter().find_map(|e| match e { AuditEvent::Approval { answer, by, post, reason, .. } => Some((*answer, by.clone(), post.clone(), reason.clone())), _ => None, }); found.expect("an Approval record") } fn some(s: &str) -> Option { Some(s.to_string()) } /// Sets the courier's stop flag when dropped, so a failing assertion ends the courier too. struct StopOnDrop<'a>(&'a AtomicBool); impl Drop for StopOnDrop<'_> { fn drop(&mut self) { self.0.store(true, std::sync::atomic::Ordering::SeqCst); } } /// The courier must end by itself within five seconds. fn ends_by_itself(courier: &std::thread::ScopedJoinHandle<'_, T>) { let until = Instant::now() + WAIT; while !courier.is_finished() { assert!(Instant::now() < until, "the courier did not stop"); std::thread::sleep(Duration::from_millis(20)); } } #[test] fn approved_in_mattermost_it_runs_and_the_record_names_who_and_where() { let (rig, fake) = rig_with_gateway("courier-approve"); let rt = Recording::answering("the notes"); let broker = rig.broker(&rt); let (mut stream, id) = ask(&broker); let mut conn = fake.next(WAIT).expect("the approval reaches gatewayd"); assert_eq!( (conn.request.approval, conn.request.tool.as_str()), (id, "read_file") ); assert_eq!(conn.request.arguments, r#"{"path":"/n/a"}"#); conn.posted(); conn.approve(); assert_eq!(conn.outcome(), Some(GatewayOutcome::Runs {})); assert!(matches!( final_answer(&mut stream), ToolResponse::Result { .. } )); assert_eq!(rt.count(), 1); assert_eq!( approval_record(&rig), (ApprovalAnswer::Approved, some(KYLE), some(POST), None) ); } #[test] fn refused_in_mattermost_it_is_denied_with_the_reason() { let (rig, fake) = rig_with_gateway("courier-refuse"); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (mut stream, _) = ask(&broker); let mut conn = fake.next(WAIT).unwrap(); conn.posted(); conn.refuse("not on a Friday"); assert_eq!(conn.outcome(), Some(GatewayOutcome::Refused {})); assert_eq!( final_answer(&mut stream), ToolResponse::Denied { reason: DenyReason::ApprovalRefused, detail: None, } ); assert_eq!(rt.count(), 0); assert_eq!( approval_record(&rig), ( ApprovalAnswer::Refused, some(KYLE), some(POST), some("not on a Friday") ) ); } #[test] fn approved_after_the_grant_is_gone_it_is_denied_on_the_recheck() { let (rig, fake) = rig_with_gateway("courier-recheck"); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (mut stream, _) = ask(&broker); let mut conn = fake.next(WAIT).unwrap(); conn.posted(); rig.remove_grant("notes"); conn.approve(); assert_eq!( conn.outcome(), Some(GatewayOutcome::Denied { reason: DenyReason::NoGrant }) ); assert_eq!( final_answer(&mut stream), ToolResponse::Denied { reason: DenyReason::NoGrant, detail: None, } ); assert_eq!(rt.count(), 0); } #[test] fn answered_in_bxctl_the_courier_closes_without_an_outcome() { let (rig, fake) = rig_with_gateway("courier-bxctl"); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (mut stream, id) = ask(&broker); let mut conn = fake.next(WAIT).unwrap(); conn.posted(); let entry = broker.table().take(id).unwrap(); brokerd::admin::answer( &broker, entry, Answer::Approved { by: some("bxctl"), post: None, }, Timestamp::now(), ); assert!( conn.closed_within(WAIT), "the courier closes, and sends nothing" ); assert!(matches!( final_answer(&mut stream), ToolResponse::Result { .. } )); assert_eq!(approval_record(&rig).1, some("bxctl")); } #[test] fn an_answer_after_it_ended_is_gone() { let rig = Rig::new("courier-gone"); rig.grant( "notes", &grant_text("read_file", "ask", "", "paths = [\"/n\"]"), ); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (_stream, id) = ask(&broker); // The courier is run by hand here, so that the answer can arrive after the entry is taken. let socket = rig.dir.path().join("gw.sock"); let fake = FakeGateway::start(&socket); let info = broker.table().list().remove(0); let stop = AtomicBool::new(false); std::thread::scope(|scope| { let courier = scope.spawn(|| { courier::carry( &broker, &socket, &info, &stop, Timing { retry: Duration::from_millis(100), tick: Duration::from_secs(5), }, ) }); let _guard = StopOnDrop(&stop); let mut conn = fake.next(WAIT).unwrap(); conn.posted(); let entry = broker.table().take(id).unwrap(); brokerd::admin::answer(&broker, entry, Answer::Expired, Timestamp::now()); conn.approve(); assert_eq!(conn.outcome(), Some(GatewayOutcome::Gone {})); ends_by_itself(&courier); }); } #[test] fn if_gatewayd_goes_away_the_approval_is_posted_again() { let (rig, fake) = rig_with_gateway("courier-again"); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (mut stream, id) = ask(&broker); let mut first = fake.next(WAIT).unwrap(); first.posted(); first.close(); let mut second = fake.next(Duration::from_secs(10)).expect("posted again"); assert_eq!(second.request.approval, id); second.posted(); second.approve(); assert_eq!(second.outcome(), Some(GatewayOutcome::Runs {})); assert!(matches!( final_answer(&mut stream), ToolResponse::Result { .. } )); assert!( rig.lines.all().iter().any(|l| l.contains(&format!( "approval {id}: gatewayd went away; posting it again" ))), "{:?}", rig.lines.all() ); } #[test] fn a_bad_answer_answers_nothing() { let (rig, fake) = rig_with_gateway("courier-bad"); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (_stream, id) = ask(&broker); for bad in [ ApprovalReply { approval: id + 1, approved: true, by: KYLE.to_string(), post: POST.to_string(), reason: None, }, ApprovalReply { approval: id, approved: true, by: "kyle".to_string(), post: POST.to_string(), reason: None, }, ApprovalReply { approval: id, approved: false, by: KYLE.to_string(), post: POST.to_string(), reason: Some("x".repeat(501)), }, ] { let mut conn = fake.next(Duration::from_secs(10)).unwrap(); conn.posted(); conn.reply(bad); assert_eq!(conn.outcome(), None, "no outcome for a bad answer"); assert!(broker.table().contains(id), "still pending"); } assert_eq!(rt.count(), 0); } #[test] fn unreachable_at_first_then_posted_and_said_once() { let rig = Rig::new("courier-late"); rig.grant( "notes", &grant_text("read_file", "ask", "", "paths = [\"/n\"]"), ); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (_stream, id) = ask(&broker); let socket = rig.dir.path().join("gw.sock"); let info = broker.table().list().remove(0); let stop = AtomicBool::new(false); let timing = Timing { retry: Duration::from_millis(100), tick: Duration::from_millis(20), }; std::thread::scope(|scope| { let courier = scope.spawn(|| courier::carry(&broker, &socket, &info, &stop, timing)); let _guard = StopOnDrop(&stop); std::thread::sleep(Duration::from_millis(500)); let fake = FakeGateway::start(&socket); let mut conn = fake.next(WAIT).unwrap(); conn.posted(); let until = Instant::now() + WAIT; while !rig .lines .all() .iter() .any(|l| l.contains("posted in Mattermost")) { assert!(Instant::now() < until, "{:?}", rig.lines.all()); std::thread::sleep(Duration::from_millis(20)); } broker.table().take(id); ends_by_itself(&courier); }); let lines = rig.lines.all(); let unreachable: Vec<&String> = lines .iter() .filter(|l| l.contains("gatewayd is not reachable")) .collect(); assert_eq!( unreachable.len(), 1, "said once, not every retry: {lines:?}" ); assert!( unreachable[0].starts_with(&format!( "brokerd: approval {id}: gatewayd is not reachable at " )), "{lines:?}" ); assert!( unreachable[0] .ends_with("; bxctl can still answer it\nsee docs/runbook.md#gateway-unreachable"), "{lines:?}" ); assert!( lines.contains(&format!("brokerd: approval {id} posted in Mattermost")), "{lines:?}" ); } #[test] fn if_gatewayd_cannot_post_it_is_tried_again() { let (rig, fake) = rig_with_gateway("courier-cannot"); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (mut stream, _) = ask(&broker); fake.next(WAIT).unwrap().cannot_post(); let mut conn = fake.next(Duration::from_secs(10)).expect("tried again"); conn.posted(); conn.approve(); assert_eq!(conn.outcome(), Some(GatewayOutcome::Runs {})); assert!(matches!( final_answer(&mut stream), ToolResponse::Result { .. } )); assert!( rig.lines .all() .iter() .any(|l| l.contains("gatewayd could not post it: Mattermost is down")) ); } #[test] fn without_a_gateway_nothing_is_carried() { let rig = Rig::new("courier-none"); rig.grant( "notes", &grant_text("read_file", "ask", "", "paths = [\"/n\"]"), ); let socket = rig.dir.path().join("gw.sock"); let fake = FakeGateway::start(&socket); let rt = Recording::answering("x"); let broker = rig.broker(&rt); let (mut stream, id) = ask(&broker); assert!( fake.next(Duration::from_millis(500)).is_none(), "no [gateway]: no connection" ); let entry = broker.table().take(id).unwrap(); brokerd::admin::answer( &broker, entry, Answer::Approved { by: some("bxctl"), post: None, }, Timestamp::now(), ); assert!(matches!( final_answer(&mut stream), ToolResponse::Result { .. } )); }