Add brokerd serve: startup, both sockets, and the expiry thread

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
2026-09-20 17:24:57 -07:00
parent d250678355
commit 3ecaef3c8b
5 changed files with 725 additions and 4 deletions
+1
View File
@@ -10,4 +10,5 @@ pub mod grants;
pub mod ledger;
pub mod policy;
pub mod runner;
pub mod serve;
pub mod state;
+119 -3
View File
@@ -1,4 +1,120 @@
fn main() {
eprintln!("brokerd: not implemented until M3");
std::process::exit(2);
//! `brokerd`: the broker role. `serve` takes the audit lock, binds both sockets, and serves.
use std::path::PathBuf;
use std::process::ExitCode;
use std::sync::Arc;
use brokerd::audit::{AuditError, RECOVERED_NOTICE};
use brokerd::config::Config;
use brokerd::runner::Refusing;
use brokerd::serve::{self, ServeError};
const USAGE: &str = "usage: brokerd serve --config <path> [--accept-break]";
fn main() -> ExitCode {
// The first argument must be `serve`.
let mut args = std::env::args().skip(1);
match args.next() {
Some(command) if command == "serve" => {}
_ => return usage(),
}
// `--config <path>` exactly once and `--accept-break` at most once, in any order.
let (config_path, accept_break) = match parse_serve(args) {
Ok(pair) => pair,
Err(()) => return usage(),
};
// Read the config before touching the filesystem: a bad config must make nothing.
let cfg = match Config::load(&config_path) {
Ok(cfg) => cfg,
Err(e) => {
eprintln!("brokerd: {e}");
return ExitCode::from(1);
}
};
// Keep the socket paths for the serving notice, before `cfg` is moved into `start`.
let broker_path = cfg.broker_socket();
let admin_path = cfg.admin_socket();
let started = match serve::start(
cfg,
accept_break,
Box::new(Refusing),
Arc::new(|line: &str| eprintln!("{line}")),
) {
Ok(started) => started,
Err(ServeError::Audit(e @ AuditError::NothingToAccept)) => {
eprintln!("brokerd: {e}");
return ExitCode::from(2);
}
Err(e) => {
eprintln!("brokerd: {e}");
return ExitCode::from(1);
}
};
if started.recovered {
eprintln!("{RECOVERED_NOTICE}");
}
if let Some(failure) = &started.accepted {
eprintln!(
"audit: accepted the break at {}:{}: {}",
failure.file, failure.line, failure.what
);
}
eprintln!(
"brokerd: serving tools on {} and approvals on {}",
broker_path.display(),
admin_path.display()
);
match started.run() {
Ok(()) => ExitCode::SUCCESS,
Err(e) => {
eprintln!("brokerd: {e}");
ExitCode::from(1)
}
}
}
/// The argument list was not `serve --config <path> [--accept-break]`.
fn usage() -> ExitCode {
eprintln!("{USAGE}");
ExitCode::from(2)
}
/// The `--config` and `--accept-break` flags after `serve`, or `Err(())` for a bad list.
fn parse_serve<I: Iterator<Item = String>>(mut args: I) -> Result<(PathBuf, bool), ()> {
let mut config: Option<String> = None;
let mut accept_break = false;
loop {
match args.next() {
None => {
return match config {
Some(path) => Ok((PathBuf::from(path), accept_break)),
None => Err(()),
};
}
Some(arg) => match arg.as_str() {
"--config" => match args.next() {
Some(path) => {
if config.is_some() {
return Err(());
}
config = Some(path);
}
None => return Err(()),
},
"--accept-break" => {
if accept_break {
return Err(());
}
accept_break = true;
}
_ => return Err(()),
},
}
}
}
+199
View File
@@ -0,0 +1,199 @@
//! `brokerd serve`: bring the broker up, listen on both sockets, and keep it running.
use std::fs::{DirBuilder, Permissions};
use std::io;
use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
use std::os::unix::net::UnixListener;
use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::sync::mpsc;
use std::time::Duration;
use crate::admin;
use crate::audit::{AuditError, Writer};
use crate::broker::{self, Broker};
use crate::config::Config;
use crate::ledger::Ledger;
use crate::runner::Runtime;
use crate::state::StateStore;
use proto::{ChainFailure, Timestamp};
/// A failure while `brokerd serve` is starting up.
#[derive(Debug)]
pub enum ServeError {
/// The audit chain would not open. Displayed as the audit error's own text.
Audit(AuditError),
/// A directory could not be prepared, or made private.
Dir(PathBuf, io::Error),
/// A socket could not be bound, or made private.
Socket(PathBuf, io::Error),
}
impl std::fmt::Display for ServeError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
ServeError::Audit(e) => write!(f, "{e}"),
ServeError::Dir(path, e) => write!(f, "cannot prepare {}: {e}", path.display()),
ServeError::Socket(path, e) => write!(f, "cannot listen on {}: {e}", path.display()),
}
}
}
impl std::error::Error for ServeError {
fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
match self {
ServeError::Audit(e) => Some(e),
ServeError::Dir(_, e) | ServeError::Socket(_, e) => Some(e),
}
}
}
/// The result of `start`: a running broker with both sockets open.
pub struct Started {
pub broker: Arc<Broker>,
pub recovered: bool,
pub accepted: Option<ChainFailure>,
tools: UnixListener,
admin: UnixListener,
}
/// Start the broker: open the audit chain, bind both sockets, and build the ledger and broker.
pub fn start(
cfg: Config,
accept_break: bool,
runtime: Box<dyn Runtime>,
log: Arc<dyn Fn(&str) + Send + Sync>,
) -> Result<Started, ServeError> {
// The audit lock comes first: if the chain will not open, no directory or socket is made.
let opened = Writer::open(&cfg.audit_dir(), accept_break).map_err(ServeError::Audit)?;
// Both sockets, in the order the task gives.
let tools = listen(&cfg.broker_socket())?;
let admin = listen(&cfg.admin_socket())?;
// The ledger and broker share the one log, each through its own box.
let log1: Box<dyn Fn(&str) + Send + Sync> = {
let log = Arc::clone(&log);
Box::new(move |line: &str| log(line))
};
let ledger = Ledger::new(
Box::new(opened.writer),
StateStore::new(&cfg.state_dir()),
log1,
);
let log2: Box<dyn Fn(&str) + Send + Sync> = {
let log = Arc::clone(&log);
Box::new(move |line: &str| log(line))
};
let broker = Broker::new(cfg, ledger, runtime, log2);
Ok(Started {
broker: Arc::new(broker),
recovered: opened.recovered,
accepted: opened.accepted.map(|b| *b),
tools,
admin,
})
}
/// Bind one socket: prepare its directory at 0700, make it private, remove any stale
/// socket, bind, then make the socket itself private at 0600.
fn listen(socket: &Path) -> Result<UnixListener, ServeError> {
let dir = match socket.parent() {
Some(parent) => parent.to_path_buf(),
None => PathBuf::from("/"),
};
// 1. The directory, at 0700.
DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(&dir)
.map_err(|e| ServeError::Dir(dir.clone(), e))?;
// 2. Make it private, always, even when it was already there at 0755.
std::fs::set_permissions(&dir, Permissions::from_mode(0o700))
.map_err(|e| ServeError::Dir(dir.clone(), e))?;
// 3. A stale socket is removed; a missing one is fine.
match std::fs::remove_file(socket) {
Ok(()) => {}
Err(e) if e.kind() == io::ErrorKind::NotFound => {}
Err(e) => return Err(ServeError::Socket(socket.to_path_buf(), e)),
}
// 4. Bind, then 5. make the socket private.
let listener =
UnixListener::bind(socket).map_err(|e| ServeError::Socket(PathBuf::from(socket), e))?;
std::fs::set_permissions(socket, Permissions::from_mode(0o600))
.map_err(|e| ServeError::Socket(PathBuf::from(socket), e))?;
Ok(listener)
}
/// Which handler the accept thread runs for each accepted stream.
#[derive(Copy, Clone)]
enum Which {
Tools,
Admin,
}
/// Run the broker: start the expiry thread, one accept thread per socket, then wait for the
/// first listener error.
impl Started {
pub fn run(self) -> io::Result<()> {
let Started {
broker,
tools,
admin,
..
} = self;
// The expiry thread: once a second, expire the approvals whose time has run out.
{
let broker = Arc::clone(&broker);
std::thread::spawn(move || {
loop {
std::thread::sleep(Duration::from_secs(1));
admin::expire_due(&broker, Timestamp::now());
}
});
}
// One accept thread per socket, each with its own clone of the broker.
let (tx, rx) = mpsc::channel::<io::Error>();
accept_loop(tools, Arc::clone(&broker), tx.clone(), Which::Tools);
accept_loop(admin, Arc::clone(&broker), tx, Which::Admin);
// The first listener error stops the daemon.
match rx.recv() {
Ok(e) => Err(e),
Err(_) => Err(io::Error::other("a listener thread ended")),
}
}
}
/// Accept streams from one listener forever, spawning a handler thread per stream.
fn accept_loop(
listener: UnixListener,
broker: Arc<Broker>,
tx: mpsc::Sender<io::Error>,
which: Which,
) {
std::thread::spawn(move || {
for stream in listener.incoming() {
match stream {
Ok(stream) => {
let broker = Arc::clone(&broker);
std::thread::spawn(move || match which {
Which::Tools => broker::handle(stream, &broker),
Which::Admin => admin::handle(stream, &broker),
});
}
Err(e) => {
// Once `run` has returned, nobody receives; dropping this is correct.
let _ = tx.send(e);
}
}
}
});
}
+405
View File
@@ -0,0 +1,405 @@
//! `brokerd serve` as a process: its startup, both sockets, and the expiry thread. Do not edit.
#[path = "support/tmp.rs"]
mod tmp;
use std::os::unix::fs::PermissionsExt;
use std::os::unix::net::{UnixListener, UnixStream};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, Output, Stdio};
use std::time::{Duration, Instant};
use brokerd::audit::RECOVERED_NOTICE;
use brokerd::runner::REFUSING;
use proto::{
AuditEvent, CallId, DenyReason, Empty, Envelope, ErrorCode, Message, PROTOCOL_VERSION,
ResultStatus, SessionId, ToolRequest, ToolResponse,
};
use tmp::TempDir;
struct Home {
dir: TempDir,
config: PathBuf,
}
impl Home {
fn new(tag: &str, ttl_ms: u64) -> Home {
let dir = TempDir::new(tag);
std::fs::create_dir_all(dir.path().join("grants")).unwrap();
let text = format!(
"[paths]\nhome = \"{home}\"\ngrants = \"{home}/grants\"\n[approvals]\nttl_ms = {ttl_ms}\n",
home = dir.path().display()
);
let config = dir.write("brokerd.toml", &text);
Home { dir, config }
}
fn path(&self, relative: &str) -> PathBuf {
self.dir.path().join(relative)
}
fn tools(&self) -> PathBuf {
self.path("run/loop-broker/broker.sock")
}
fn admin(&self) -> PathBuf {
self.path("run/owner-broker/admin.sock")
}
fn grant(&self, id: &str, mode: &str) {
let text = format!(
"tool = \"read_file\"\nmode = \"{mode}\"\nmax_taint = \"secret\"\n\
result_class = \"private\"\nuntrusted = false\n[constraints]\npaths = [\"/n\"]\n"
);
std::fs::write(self.path(&format!("grants/{id}.toml")), text).unwrap();
}
fn command(&self, extra: &[&str]) -> Command {
let mut command = Command::new(env!("CARGO_BIN_EXE_brokerd"));
command
.args(["serve", "--config"])
.arg(&self.config)
.args(extra);
command
}
/// Starts `brokerd serve` and waits until both sockets answer.
fn serve(&self) -> Running {
let child = self
.command(&[])
.stdout(Stdio::null())
.stderr(Stdio::piped())
.spawn()
.unwrap();
let running = Running(Some(child));
let until = Instant::now() + Duration::from_secs(10);
while UnixStream::connect(self.tools()).is_err()
|| UnixStream::connect(self.admin()).is_err()
{
assert!(Instant::now() < until, "brokerd never listened");
std::thread::sleep(Duration::from_millis(20));
}
running
}
/// Runs `brokerd serve` expecting it to exit by itself.
fn run(&self, extra: &[&str]) -> Output {
self.command(extra).output().unwrap()
}
fn events(&self) -> Vec<AuditEvent> {
let dir = self.path("audit");
let mut names: Vec<String> = std::fs::read_dir(&dir)
.unwrap()
.map(|e| e.unwrap().file_name().into_string().unwrap())
.filter(|n| n.ends_with(".jsonl"))
.collect();
names.sort();
names
.iter()
.flat_map(|n| {
let text = std::fs::read_to_string(dir.join(n)).unwrap();
text.lines()
.map(|l| serde_json::from_str::<proto::AuditRecord>(l).unwrap().event)
.collect::<Vec<_>>()
})
.collect()
}
}
/// Kills the daemon when dropped; `stop` returns what it printed.
struct Running(Option<Child>);
impl Running {
fn stop(mut self) -> String {
let mut child = self.0.take().unwrap();
child.kill().unwrap();
let output = child.wait_with_output().unwrap();
String::from_utf8_lossy(&output.stderr).to_string()
}
}
impl Drop for Running {
fn drop(&mut self) {
if let Some(child) = &mut self.0 {
let _ = child.kill();
let _ = child.wait();
}
}
}
fn mode(path: &Path) -> u32 {
std::fs::metadata(path).unwrap().permissions().mode() & 0o777
}
fn exchange(socket: &Path, id: u64, msg: Message) -> Vec<Envelope> {
let mut stream = UnixStream::connect(socket).unwrap();
stream
.set_read_timeout(Some(Duration::from_secs(10)))
.unwrap();
let env = Envelope {
v: PROTOCOL_VERSION,
id,
r#final: true,
msg,
};
proto::write_frame(&mut stream, &env).unwrap();
let mut frames = Vec::new();
loop {
let env = proto::read_frame(&mut stream).unwrap();
let last = env.r#final;
frames.push(env);
if last {
return frames;
}
}
}
fn read_notes(call: u64) -> Message {
Message::ToolRequest(ToolRequest {
session: SessionId::new("s1").unwrap(),
call: CallId(call),
tool: "read_file".to_string(),
arguments: r#"{"path":"/n/a"}"#.to_string(),
})
}
fn last_response(frames: &[Envelope]) -> &ToolResponse {
match &frames.last().unwrap().msg {
Message::ToolResponse(r) => r,
other => panic!("{other:?}"),
}
}
fn stderr(output: &Output) -> String {
String::from_utf8_lossy(&output.stderr).to_string()
}
#[test]
fn it_makes_its_directories_0700_and_its_sockets_0600() {
let home = Home::new("serve-modes", 900_000);
// One directory found too open, one made.
std::fs::create_dir_all(home.path("run/owner-broker")).unwrap();
std::fs::set_permissions(
home.path("run/owner-broker"),
std::fs::Permissions::from_mode(0o755),
)
.unwrap();
let running = home.serve();
assert_eq!(mode(&home.path("run/loop-broker")), 0o700);
assert_eq!(mode(&home.path("run/owner-broker")), 0o700);
assert_eq!(mode(&home.tools()), 0o600);
assert_eq!(mode(&home.admin()), 0o600);
assert_eq!(mode(&home.path("audit")), 0o700);
let printed = running.stop();
assert!(printed.contains("serving"), "{printed}");
}
#[test]
fn a_stale_socket_is_replaced() {
let home = Home::new("serve-stale", 900_000);
std::fs::create_dir_all(home.path("run/loop-broker")).unwrap();
drop(UnixListener::bind(home.tools()).unwrap());
assert!(home.tools().exists(), "the stale socket file is there");
let _running = home.serve();
let frames = exchange(&home.tools(), 3, read_notes(3));
assert_eq!(
last_response(&frames),
&ToolResponse::Denied {
reason: DenyReason::NoGrant
}
);
}
#[test]
fn a_second_brokerd_on_the_same_home_refuses_to_start() {
let home = Home::new("serve-twice", 900_000);
let _running = home.serve();
let second = home.run(&[]);
assert_eq!(second.status.code(), Some(1));
let text = stderr(&second);
assert!(text.contains("brokerd is already running"), "{text}");
assert!(
text.trim_end()
.ends_with("see docs/runbook.md#brokerd-already-running"),
"{text}"
);
// The first still has its sockets.
let frames = exchange(&home.admin(), 1, Message::Approvals(Empty {}));
assert!(matches!(frames[0].msg, Message::ApprovalList(_)));
}
#[test]
fn it_answers_each_socket_and_refuses_the_other_kinds() {
let home = Home::new("serve-kinds", 900_000);
home.grant("notes", "auto");
let running = home.serve();
let frames = exchange(&home.tools(), 7, read_notes(7));
assert_eq!(frames[0].id, 7);
// The production runtime runs nothing.
assert_eq!(
last_response(&frames),
&ToolResponse::Failed {
message: REFUSING.to_string()
}
);
let wrong = exchange(&home.tools(), 8, Message::Approvals(Empty {}));
assert!(matches!(&wrong[0].msg, Message::Error(e) if e.code == ErrorCode::Forbidden));
let wrong = exchange(&home.admin(), 9, read_notes(9));
assert!(matches!(&wrong[0].msg, Message::Error(e) if e.code == ErrorCode::Forbidden));
let printed = running.stop();
assert!(
printed.contains("approvals on broker.sock\nsee docs/runbook.md#socket-forbidden"),
"{printed}"
);
assert!(
printed.contains("tool_request on admin.sock\nsee docs/runbook.md#socket-forbidden"),
"{printed}"
);
assert!(matches!(
home.events().as_slice(),
[
AuditEvent::Decision { .. },
AuditEvent::Result {
status: ResultStatus::Failed,
..
}
]
));
}
#[test]
fn an_approval_nobody_answers_expires() {
let home = Home::new("serve-expire", 100);
home.grant("notes", "ask");
let _running = home.serve();
let started = Instant::now();
let frames = exchange(&home.tools(), 2, read_notes(2));
assert_eq!(frames.len(), 2, "{frames:?}");
assert!(matches!(
&frames[0].msg,
Message::ToolResponse(ToolResponse::PendingApproval { approval: 0, .. })
));
assert_eq!(
last_response(&frames),
&ToolResponse::Denied {
reason: DenyReason::ApprovalExpired
}
);
// The expiry thread looks every second.
assert!(
started.elapsed() < Duration::from_secs(5),
"{:?}",
started.elapsed()
);
}
fn copy_case(home: &Home, case: &str, only: &[&str]) {
let from = format!(
"{}/../proto/tests/fixtures/audit/{case}",
env!("CARGO_MANIFEST_DIR")
);
std::fs::create_dir_all(home.path("audit")).unwrap();
for name in only {
std::fs::copy(
format!("{from}/{name}"),
home.path(&format!("audit/{name}")),
)
.unwrap();
}
}
fn snapshot(dir: &Path) -> Vec<(String, Vec<u8>)> {
let mut all: Vec<(String, Vec<u8>)> = std::fs::read_dir(dir)
.unwrap()
.map(|e| e.unwrap())
.filter(|e| e.file_name().to_string_lossy().ends_with(".jsonl"))
.map(|e| {
(
e.file_name().into_string().unwrap(),
std::fs::read(e.path()).unwrap(),
)
})
.collect();
all.sort();
all
}
#[test]
fn a_broken_chain_stops_it_before_any_socket_and_nothing_is_written() {
let home = Home::new("serve-broken", 900_000);
copy_case(&home, "changed-byte", &["2026-09-17.jsonl"]);
let before = snapshot(&home.path("audit"));
let output = home.run(&[]);
assert_eq!(output.status.code(), Some(1));
let text = stderr(&output);
assert!(text.contains("2026-09-17.jsonl:4: "), "{text}");
assert!(
text.trim_end()
.ends_with("see docs/runbook.md#audit-chain-broken"),
"{text}"
);
assert_eq!(snapshot(&home.path("audit")), before);
assert!(!home.tools().exists() && !home.admin().exists());
}
#[test]
fn a_torn_tail_is_recovered_and_it_serves() {
let home = Home::new("serve-torn", 900_000);
copy_case(
&home,
"torn-tail",
&["2026-09-17.jsonl", "2026-09-18.jsonl"],
);
let running = home.serve();
let printed = running.stop();
assert!(printed.contains(RECOVERED_NOTICE), "{printed}");
assert!(RECOVERED_NOTICE.ends_with("see docs/runbook.md#audit-recovered"));
}
#[test]
fn accept_break_with_nothing_to_accept_exits_2() {
let home = Home::new("serve-nothing", 900_000);
let output = home.run(&["--accept-break"]);
assert_eq!(output.status.code(), Some(2));
assert!(
stderr(&output).contains("nothing to accept"),
"{}",
stderr(&output)
);
}
#[test]
fn bad_arguments_and_bad_configs_do_not_start() {
let brokerd = env!("CARGO_BIN_EXE_brokerd");
for args in [
&[][..],
&["serve"][..],
&["serve", "--config"][..],
&["serve", "--config", "a", "--config", "b"][..],
&["serve", "--config", "a", "--loud"][..],
&["run", "--config", "a"][..],
] {
let output = Command::new(brokerd).args(args).output().unwrap();
assert_eq!(output.status.code(), Some(2), "{args:?}");
assert!(
stderr(&output).starts_with("usage: brokerd serve"),
"{args:?}"
);
}
let home = Home::new("serve-config", 900_000);
std::fs::write(&home.config, "[paths]\nhoem = \"/x\"\n").unwrap();
let output = home.run(&[]);
assert_eq!(output.status.code(), Some(1));
assert!(
stderr(&output).contains("brokerd.toml"),
"{}",
stderr(&output)
);
std::fs::remove_file(&home.config).unwrap();
assert_eq!(home.run(&[]).status.code(), Some(1));
assert!(
!home.path("audit").exists(),
"nothing made before the config is read"
);
}