Add the audit writer with its startup check

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/brokerd/src/audit.rs

@@ -0,0 +1,423 @@
+//! The audit writer: a hash-chained log of decisions, approvals and results, one JSON record per
+//! line. The chain is the log's integrity: every record carries the hash of the line before it, so
+//! any change is seen. Opening the log verifies it (a short check for an ordinary start, the whole
+//! log when a break is to be accepted), recovers a torn tail, or accepts a break in an older file
+//! when told to, and only then hands back a writer that appends the next record.
+
+use std::fs::{self, TryLockError};
+use std::io;
+use std::io::Write;
+use std::os::unix::fs::{DirBuilderExt, OpenOptionsExt};
+use std::path::{Path, PathBuf};
+
+use proto::{
+    AuditEvent, AuditRecord, ChainFailure, ChainReport, ChainVerifier, Hash32, Timestamp, sha256,
+};
+
+/// The notice appended to the log when a torn final line is recovered, as the RUNBOOK entry names.
+pub const RECOVERED_NOTICE: &str =
+    "audit: recovered a torn final line\nsee docs/runbook.md#audit-recovered";
+
+/// Where the owner looks when the audit log cannot be read, written or locked.
+const RUNBOOK_UNAVAILABLE: &str = "see docs/runbook.md#audit-unavailable";
+/// The RUNBOOK entry a second writer points to.
+const RUNBOOK_ALREADY_RUNNING: &str = "see docs/runbook.md#brokerd-already-running";
+/// The RUNBOOK entry a broken chain points to.
+const RUNBOOK_BROKEN: &str = "see docs/runbook.md#audit-chain-broken";
+
+/// Why opening or writing the audit log can fail.
+#[derive(Debug)]
+pub enum AuditError {
+    /// The log is already locked by another running brokerd.
+    Locked,
+    /// A record in the log does not chain: the log is damaged.
+    Broken(Box<ChainFailure>),
+    /// Asked to accept a break, but the log verifies: there is none.
+    NothingToAccept,
+    /// The log could not be read, written or locked.
+    Io { what: String, source: io::Error },
+    /// A write failed; the writer has stopped and must be restarted.
+    Stopped,
+}
+
+impl std::fmt::Display for AuditError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            AuditError::Locked => {
+                write!(f, "brokerd is already running\n{RUNBOOK_ALREADY_RUNNING}")
+            }
+            AuditError::Broken(failure) => write!(
+                f,
+                "{}:{}: {}\n{RUNBOOK_BROKEN}",
+                failure.file, failure.line, failure.what
+            ),
+            AuditError::NothingToAccept => {
+                write!(f, "nothing to accept: the audit log verifies")
+            }
+            AuditError::Io { what, source } => {
+                write!(f, "audit: {what}: {source}\n{RUNBOOK_UNAVAILABLE}")
+            }
+            AuditError::Stopped => write!(
+                f,
+                "audit: an earlier write failed; restart brokerd\n{RUNBOOK_UNAVAILABLE}"
+            ),
+        }
+    }
+}
+
+impl std::error::Error for AuditError {
+    fn source(&self) -> Option<&(dyn std::error::Error + 'static)> {
+        match self {
+            AuditError::Io { source, .. } => Some(source),
+            _ => None,
+        }
+    }
+}
+
+impl From<io::Error> for AuditError {
+    fn from(err: io::Error) -> Self {
+        AuditError::Io {
+            what: "audit log".to_string(),
+            source: err,
+        }
+    }
+}
+
+/// A name is a log file when it is `<YYYY-MM-DD>.jsonl`: 16 chars, `-` at 4 and 7, `.jsonl` at 10
+/// to 15, digits elsewhere.
+fn is_log_name(name: &str) -> bool {
+    let b = name.as_bytes();
+    if b.len() != 16 {
+        return false;
+    }
+    b[4] == b'-'
+        && b[7] == b'-'
+        && &b[10..16] == b".jsonl"
+        && b[0..4].iter().all(|c| c.is_ascii_digit())
+        && b[5..6].iter().all(|c| c.is_ascii_digit())
+        && b[8..10].iter().all(|c| c.is_ascii_digit())
+}
+
+/// The log file name for a record's time: the day of the timestamp, `.jsonl`.
+fn day_name(time: Timestamp) -> String {
+    let when = time.to_rfc3339();
+    format!("{}.jsonl", &when[..10])
+}
+
+/// An I/O error as an audit error, the path in what.
+fn io(what: &str, path: impl AsRef<Path>, source: io::Error) -> AuditError {
+    let path = path.as_ref();
+    AuditError::Io {
+        what: format!("{what} {}", path.display()),
+        source,
+    }
+}
+
+/// The bytes after the last newline, once one trailing newline is removed. None if nothing is left.
+fn last_line(bytes: &[u8]) -> Option<&[u8]> {
+    let bytes = match bytes.last() {
+        Some(b'\n') => &bytes[..bytes.len() - 1],
+        _ => bytes,
+    };
+    let line = match bytes.iter().rposition(|b| *b == b'\n') {
+        Some(pos) => &bytes[pos + 1..],
+        None => bytes,
+    };
+    if line.is_empty() { None } else { Some(line) }
+}
+
+/// Every log file in `dir`, by name. An unreadable directory or entry is an I/O error.
+fn log_files(dir: &Path) -> Result<Vec<String>, AuditError> {
+    let mut names: Vec<String> = Vec::new();
+    let mut entries = fs::read_dir(dir).map_err(|e| io("read audit log", dir, e))?;
+    loop {
+        let entry = match entries.next() {
+            Some(entry) => entry,
+            None => break,
+        };
+        let entry = entry.map_err(|e| io("read audit log", dir, e))?;
+        let name = entry.file_name();
+        let name = name.to_string_lossy();
+        if is_log_name(&name) {
+            names.push(name.into_owned());
+        }
+    }
+    names.sort();
+    Ok(names)
+}
+
+/// Verify the whole log, from the first record.
+pub fn verify_dir(dir: &Path) -> Result<ChainReport, AuditError> {
+    let files = log_files(dir)?;
+    let mut verifier = ChainVerifier::new();
+    for file in files {
+        let path = dir.join(&file);
+        let bytes = fs::read(&path).map_err(|e| io("read audit log", &path, e))?;
+        verifier.feed(&file, &bytes);
+    }
+    Ok(verifier.finish())
+}
+
+/// Verify the latest file only, resuming from the last line of the file before it, when that line
+/// is a record. Otherwise the whole log is verified.
+fn short_check(dir: &Path) -> Result<ChainReport, AuditError> {
+    let files = log_files(dir)?;
+    if files.len() < 2 {
+        return verify_dir(dir);
+    }
+    let latest = files.last().unwrap();
+    let before = &files[files.len() - 2];
+    let path_before = dir.join(before);
+    let bytes = fs::read(&path_before).map_err(|e| io("read audit log", &path_before, e))?;
+    let line = match last_line(&bytes) {
+        Some(line) => line,
+        None => return verify_dir(dir),
+    };
+    let record: AuditRecord = match serde_json::from_slice(line) {
+        Ok(record) => record,
+        Err(_) => return verify_dir(dir),
+    };
+    let next_seq = match record.seq.checked_add(1) {
+        Some(seq) => seq,
+        None => return verify_dir(dir),
+    };
+    let prev = match sha256(line) {
+        Ok(hash) => hash,
+        Err(_) => return verify_dir(dir),
+    };
+    let mut verifier = ChainVerifier::resume(next_seq, prev);
+    let path_latest = dir.join(latest);
+    let latest_bytes = fs::read(&path_latest).map_err(|e| io("read audit log", &path_latest, e))?;
+    verifier.feed(latest, &latest_bytes);
+    Ok(verifier.finish())
+}
+
+/// Append one record: serialise it, open the target file, write the line, and return the seq and
+/// the hash of the line it wrote. A fresh file is created mode 0600 and the directory synced.
+fn write_record(
+    dir: &Path,
+    target: &str,
+    newline_first: bool,
+    seq: u64,
+    prev: Hash32,
+    event: AuditEvent,
+    time: Timestamp,
+) -> Result<(u64, Hash32), AuditError> {
+    let record = AuditRecord {
+        seq,
+        time,
+        prev,
+        event,
+    };
+    let json = serde_json::to_string(&record).map_err(io::Error::other)?;
+    let path = dir.join(target);
+    let mut file = fs::OpenOptions::new()
+        .append(true)
+        .create(true)
+        .mode(0o600)
+        .open(&path)
+        .map_err(|e| io("open audit log", &path, e))?;
+    if newline_first {
+        file.write_all(b"\n").map_err(io::Error::other)?;
+    }
+    file.write_all(json.as_bytes()).map_err(io::Error::other)?;
+    file.write_all(b"\n").map_err(io::Error::other)?;
+    file.sync_all().map_err(io::Error::other)?;
+    fs::File::open(dir).map_err(io::Error::other)?.sync_all()?;
+    let hash = sha256(json.as_bytes()).map_err(io::Error::other)?;
+    Ok((seq, hash))
+}
+
+/// The last record's file name: today's, or the latest if that is already current.
+fn target_for(latest: &Option<String>, time: Timestamp) -> String {
+    let candidate = day_name(time);
+    match latest {
+        Some(current) if candidate <= *current => current.clone(),
+        _ => candidate,
+    }
+}
+
+/// A held directory lock: the open `.lock` file, released when it drops. Kept in the `Writer` so
+/// the lock lasts as long as the writer; `fs::File` is not `Debug`, so this wraps it.
+struct Lock(fs::File);
+
+impl std::fmt::Debug for Lock {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_tuple("Lock").field(&"open").finish()
+    }
+}
+
+/// The audit log's writer. Holds the directory lock for its whole life, so a second writer is
+/// refused.
+#[derive(Debug)]
+pub struct Writer {
+    lock_file: Lock,
+    path: PathBuf,
+    latest: Option<String>,
+    prev: Hash32,
+    next_seq: u64,
+    stopped: bool,
+}
+
+impl Writer {
+    /// Open the log at `dir`: create it, lock it, verify it, and recover or accept as needed.
+    pub fn open(dir: &Path, accept_break: bool) -> Result<Opened, AuditError> {
+        // 1. The directory, mode 0700. Create it if missing; if it already exists, that is fine.
+        if let Err(e) = fs::DirBuilder::new().mode(0o700).create(dir)
+            && e.kind() != io::ErrorKind::AlreadyExists
+        {
+            return Err(io("create audit log", dir, e));
+        }
+
+        // 2. The lock, mode 0600: try_lock refuses a second writer while this one lives.
+        let lock = Lock(
+            fs::OpenOptions::new()
+                .write(true)
+                .create(true)
+                .truncate(false)
+                .mode(0o600)
+                .open(dir.join(".lock"))
+                .map_err(|e| io("open audit lock", dir.join(".lock"), e))?,
+        );
+        match lock.0.try_lock() {
+            Ok(()) => (),
+            Err(TryLockError::WouldBlock) => return Err(AuditError::Locked),
+            Err(TryLockError::Error(e)) => return Err(io("open audit lock", dir.join(".lock"), e)),
+        }
+
+        // 3. Verify the log.
+        let report = if accept_break {
+            verify_dir(dir)?
+        } else {
+            short_check(dir)?
+        };
+
+        // The latest log file before any recovery or break, for the target of either.
+        let latest = log_files(dir)
+            .ok()
+            .and_then(|files| files.into_iter().last());
+        let now = Timestamp::now();
+        let recovered = report.torn_tail.is_some();
+
+        // 4-9. A broken chain, nothing to accept, or a torn tail: write the recovery record, if any,
+        // and set the chain state for the next append.
+        let (prev, next_seq) = if let Some(failure) = report.failure.as_ref() {
+            if !accept_break {
+                return Err(AuditError::Broken(Box::new(failure.clone())));
+            }
+            let (seq, hash) = write_record(
+                dir,
+                latest.as_deref().unwrap_or(&day_name(now)),
+                failure.tail_torn,
+                failure.break_seq,
+                failure.break_prev,
+                AuditEvent::AcceptedBreak {
+                    file: failure.file.clone(),
+                    line: failure.line,
+                    last_good: failure.last_good,
+                },
+                now,
+            )?;
+            (hash, seq.checked_add(1).unwrap_or(report.next_seq))
+        } else if accept_break {
+            return Err(AuditError::NothingToAccept);
+        } else if let Some(torn) = report.torn_tail.as_ref() {
+            let (seq, hash) = write_record(
+                dir,
+                latest.as_deref().unwrap_or(&day_name(now)),
+                !torn.has_newline,
+                torn.recovery_seq,
+                torn.recovery_prev,
+                AuditEvent::Recovery {
+                    torn_bytes: torn.bytes,
+                    torn_sha256: torn.sha256,
+                },
+                now,
+            )?;
+            (hash, seq.checked_add(1).unwrap_or(report.next_seq))
+        } else {
+            let prev = match report.head {
+                Some(head) => head,
+                None => {
+                    let files = log_files(dir)?;
+                    if files.is_empty() {
+                        Hash32::ZERO
+                    } else {
+                        let before = &files[files.len() - 2];
+                        let path = dir.join(before);
+                        let bytes = fs::read(&path).map_err(|e| io("read audit log", &path, e))?;
+                        match last_line(&bytes) {
+                            Some(line) => sha256(line).map_err(io::Error::other)?,
+                            None => Hash32::ZERO,
+                        }
+                    }
+                }
+            };
+            (prev, report.next_seq)
+        };
+
+        // 10. Hand back the writer.
+        Ok(Opened {
+            writer: Writer {
+                lock_file: lock,
+                path: dir.to_path_buf(),
+                latest,
+                prev,
+                next_seq,
+                stopped: false,
+            },
+            recovered,
+            accepted: report.failure.map(Box::new),
+        })
+    }
+
+    /// Append a record at `time`, in the right day's file. The seq it writes is returned.
+    pub fn append(&mut self, time: Timestamp, event: AuditEvent) -> Result<u64, AuditError> {
+        if self.stopped {
+            return Err(AuditError::Stopped);
+        }
+        let target = target_for(&self.latest, time);
+        let (seq, hash) = match write_record(
+            &self.path,
+            &target,
+            false,
+            self.next_seq,
+            self.prev,
+            event,
+            time,
+        ) {
+            Ok(r) => r,
+            Err(e) => {
+                self.stopped = true;
+                return Err(e);
+            }
+        };
+        self.prev = hash;
+        self.next_seq = seq.checked_add(1).unwrap_or(self.next_seq);
+        if self.latest.as_deref() != Some(target.as_str()) {
+            self.latest = Some(target);
+        }
+        Ok(seq)
+    }
+
+    /// The seq the next append writes.
+    pub fn next_seq(&self) -> u64 {
+        self.next_seq
+    }
+}
+
+impl Drop for Writer {
+    fn drop(&mut self) {
+        // This field holds the .lock handle open, so the lock lasts as long as the writer.
+        let _ = &self.lock_file;
+        let _ = fs::remove_file(self.path.join(".lock"));
+    }
+}
+
+/// What `open` hands back: the writer plus what it recovered or accepted.
+#[derive(Debug)]
+pub struct Opened {
+    pub writer: Writer,
+    pub recovered: bool,
+    pub accepted: Option<Box<ChainFailure>>,
+}

crates/brokerd/src/lib.rs

@@ -1,6 +1,7 @@
 //! The broker: the only role that holds authority.
 
 pub mod args;
+pub mod audit;
 pub mod config;
 pub mod grants;
 pub mod policy;

crates/brokerd/tests/audit.rs

@@ -0,0 +1,258 @@
+//! The audit writer: the chain it writes, the lock, rollover, and stopping after a failed
+//! write. Do not edit. Startup checks, recovery and accepted breaks are in `audit_startup.rs`.
+
+#[path = "support/audit_dir.rs"]
+mod audit_dir;
+
+use std::os::unix::fs::PermissionsExt;
+
+use audit_dir::{D1, D2, TempDir, denied, lines, ts};
+use brokerd::audit::{AuditError, RECOVERED_NOTICE, Writer, verify_dir};
+use proto::{AuditRecord, Hash32, sha256};
+
+fn mode(path: &std::path::Path) -> u32 {
+    std::fs::metadata(path).unwrap().permissions().mode() & 0o777
+}
+
+#[test]
+fn the_first_record_starts_the_chain() {
+    let dir = TempDir::unmade("first");
+    let opened = Writer::open(&dir.path, false).unwrap();
+    assert!(!opened.recovered);
+    assert!(opened.accepted.is_none());
+    let mut writer = opened.writer;
+    assert_eq!(writer.next_seq(), 0);
+
+    let seq = writer
+        .append(ts("2026-09-17T08:00:00.000Z"), denied(1))
+        .unwrap();
+    assert_eq!(seq, 0);
+    assert_eq!(writer.next_seq(), 1);
+
+    let text = std::fs::read_to_string(dir.path.join(D1)).unwrap();
+    assert!(text.ends_with('\n'), "a record is one line and its newline");
+    assert_eq!(text.lines().count(), 1);
+    let record: AuditRecord = serde_json::from_str(text.lines().next().unwrap()).unwrap();
+    assert_eq!((record.seq, record.prev), (0, Hash32::ZERO));
+    assert_eq!(record.time, ts("2026-09-17T08:00:00.000Z"));
+    assert_eq!(record.event, denied(1));
+
+    assert_eq!(mode(&dir.path), 0o700, "the directory open() made");
+    assert_eq!(mode(&dir.path.join(D1)), 0o600);
+}
+
+#[test]
+fn the_chain_runs_across_a_day_boundary() {
+    let dir = TempDir::unmade("days");
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    let times = [
+        "2026-09-17T23:59:58.000Z",
+        "2026-09-17T23:59:59.999Z",
+        "2026-09-18T00:00:00.000Z",
+        "2026-09-18T00:00:01.000Z",
+    ];
+    for (i, time) in times.iter().enumerate() {
+        assert_eq!(writer.append(ts(time), denied(i as u64)).unwrap(), i as u64);
+    }
+    let (day1, day2) = (lines(&dir.path, D1), lines(&dir.path, D2));
+    assert_eq!((day1.len(), day2.len()), (2, 2));
+
+    // seq goes on across files, and the new file chains from the last line of the old one.
+    let first: AuditRecord = serde_json::from_str(&day2[0]).unwrap();
+    assert_eq!(first.seq, 2);
+    assert_eq!(first.prev, sha256(day1[1].as_bytes()).unwrap());
+
+    let report = verify_dir(&dir.path).unwrap();
+    assert_eq!(report.failure, None);
+    assert_eq!((report.records, report.next_seq), (4, 4));
+    assert!(report.clock_warnings.is_empty());
+}
+
+#[test]
+fn reopening_continues_the_chain() {
+    let dir = TempDir::unmade("reopen");
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    writer
+        .append(ts("2026-09-17T08:00:00.000Z"), denied(1))
+        .unwrap();
+    writer
+        .append(ts("2026-09-17T08:00:01.000Z"), denied(2))
+        .unwrap();
+    drop(writer);
+
+    // One file: the whole of it is checked.
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    assert_eq!(writer.next_seq(), 2);
+    assert_eq!(
+        writer
+            .append(ts("2026-09-18T08:00:00.000Z"), denied(3))
+            .unwrap(),
+        2
+    );
+    drop(writer);
+
+    // Two files: the latest is checked, resumed from the last line of the one before.
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    assert_eq!(writer.next_seq(), 3);
+    assert_eq!(
+        writer
+            .append(ts("2026-09-18T08:00:01.000Z"), denied(4))
+            .unwrap(),
+        3
+    );
+    drop(writer);
+
+    let report = verify_dir(&dir.path).unwrap();
+    assert_eq!(report.failure, None);
+    assert_eq!(report.records, 4);
+}
+
+#[test]
+fn an_empty_latest_file_gets_the_next_record_as_its_first_line() {
+    let dir = TempDir::case("empty-latest", None);
+    let opened = Writer::open(&dir.path, false).unwrap();
+    assert!(!opened.recovered);
+    let mut writer = opened.writer;
+    assert_eq!(
+        writer
+            .append(ts("2026-09-18T08:00:00.000Z"), denied(9))
+            .unwrap(),
+        5
+    );
+
+    let day2 = lines(&dir.path, D2);
+    assert_eq!(day2.len(), 1);
+    let record: AuditRecord = serde_json::from_str(&day2[0]).unwrap();
+    assert_eq!(
+        record.prev,
+        sha256(lines(&dir.path, D1)[4].as_bytes()).unwrap()
+    );
+    assert_eq!(verify_dir(&dir.path).unwrap().failure, None);
+}
+
+#[test]
+fn a_second_writer_is_refused() {
+    let dir = TempDir::unmade("lock");
+    let first = Writer::open(&dir.path, false).unwrap();
+    let error = Writer::open(&dir.path, false).unwrap_err();
+    assert!(matches!(error, AuditError::Locked), "{error}");
+    let text = error.to_string();
+    assert!(text.starts_with("brokerd is already running"), "{text}");
+    assert!(
+        text.ends_with("see docs/runbook.md#brokerd-already-running"),
+        "{text}"
+    );
+
+    // The lock goes when the writer goes, however that happens.
+    drop(first);
+    assert!(Writer::open(&dir.path, false).is_ok());
+}
+
+#[test]
+fn the_writer_never_goes_back_to_an_earlier_file() {
+    let dir = TempDir::unmade("clock");
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    writer
+        .append(ts("2026-09-18T00:00:05.000Z"), denied(1))
+        .unwrap();
+    // The clock is stepped back over midnight.
+    writer
+        .append(ts("2026-09-17T23:59:50.000Z"), denied(2))
+        .unwrap();
+
+    assert!(
+        !dir.path.join(D1).exists(),
+        "a record went into an earlier file"
+    );
+    assert_eq!(lines(&dir.path, D2).len(), 2);
+    let report = verify_dir(&dir.path).unwrap();
+    assert_eq!(report.failure, None);
+    assert_eq!(report.clock_warnings.len(), 1);
+
+    // It holds across a restart too.
+    drop(writer);
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    writer
+        .append(ts("2026-09-17T23:59:55.000Z"), denied(3))
+        .unwrap();
+    assert!(!dir.path.join(D1).exists());
+    assert_eq!(verify_dir(&dir.path).unwrap().failure, None);
+}
+
+#[test]
+fn files_that_are_not_log_files_are_ignored() {
+    let dir = TempDir::case("good", None);
+    std::fs::write(dir.path.join("notes.txt"), "not a log\n").unwrap();
+    std::fs::write(dir.path.join("2026-09-19.jsonl.bak"), "not a log\n").unwrap();
+    std::fs::write(dir.path.join("latest.jsonl"), "not a log\n").unwrap();
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    assert_eq!(writer.next_seq(), 10);
+    writer
+        .append(ts("2026-09-18T10:00:00.000Z"), denied(9))
+        .unwrap();
+    assert_eq!(verify_dir(&dir.path).unwrap().records, 11);
+}
+
+/// After one failed write the writer writes nothing more, even when the cause has gone: part of
+/// a line may be on disk, and only the next start deals with that.
+#[test]
+fn a_failed_write_stops_the_writer() {
+    let dir = TempDir::unmade("sticky");
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    writer
+        .append(ts("2026-09-17T08:00:00.000Z"), denied(1))
+        .unwrap();
+
+    // A new day needs a new file, and the directory no longer allows one.
+    std::fs::set_permissions(&dir.path, std::fs::Permissions::from_mode(0o500)).unwrap();
+    if std::fs::write(dir.path.join("probe"), "").is_ok() {
+        eprintln!("skipped: this user can write to a read-only directory (root?)");
+        std::fs::set_permissions(&dir.path, std::fs::Permissions::from_mode(0o700)).unwrap();
+        return;
+    }
+    let error = writer
+        .append(ts("2026-09-18T08:00:00.000Z"), denied(2))
+        .unwrap_err();
+    assert!(matches!(error, AuditError::Io { .. }), "{error}");
+    assert!(
+        error
+            .to_string()
+            .ends_with("see docs/runbook.md#audit-unavailable"),
+        "{error}"
+    );
+
+    std::fs::set_permissions(&dir.path, std::fs::Permissions::from_mode(0o700)).unwrap();
+    for time in ["2026-09-18T08:00:01.000Z", "2026-09-17T08:00:02.000Z"] {
+        let error = writer.append(ts(time), denied(3)).unwrap_err();
+        assert!(matches!(error, AuditError::Stopped), "{error}");
+        assert!(
+            error
+                .to_string()
+                .ends_with("see docs/runbook.md#audit-unavailable"),
+            "{error}"
+        );
+    }
+    assert_eq!(
+        lines(&dir.path, D1).len(),
+        1,
+        "a stopped writer wrote something"
+    );
+    assert!(!dir.path.join(D2).exists());
+
+    // A restart puts it right.
+    drop(writer);
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    assert_eq!(
+        writer
+            .append(ts("2026-09-18T08:00:03.000Z"), denied(4))
+            .unwrap(),
+        1
+    );
+    assert_eq!(verify_dir(&dir.path).unwrap().failure, None);
+}
+
+#[test]
+fn the_recovered_notice_names_its_runbook_entry() {
+    assert!(RECOVERED_NOTICE.starts_with("audit: recovered a torn final line"));
+    assert!(RECOVERED_NOTICE.ends_with("see docs/runbook.md#audit-recovered"));
+}

crates/brokerd/tests/audit_startup.rs

@@ -0,0 +1,300 @@
+//! What `Writer::open` does with the log it finds: refuse a broken chain, recover a torn tail,
+//! accept a break when told to. Do not edit. The fixture logs are in
+//! `crates/proto/tests/fixtures/audit/`; each test works on a copy.
+
+#[path = "support/audit_dir.rs"]
+mod audit_dir;
+
+use audit_dir::{D1, D2, TempDir, denied, lines, snapshot};
+use brokerd::audit::{AuditError, Writer, verify_dir};
+use proto::{AuditEvent, AuditRecord, Location, Timestamp};
+
+type Case = (
+    &'static str,
+    Option<&'static [&'static str]>,
+    &'static str,
+    u64,
+    &'static str,
+);
+
+fn at(file: &str, line: u64) -> Location {
+    Location {
+        file: file.to_string(),
+        line,
+    }
+}
+
+/// An ordinary start checks the latest file only, so each damaged file is copied alone: it is
+/// then the latest. Nothing may be written to a log that does not verify.
+#[test]
+fn a_broken_chain_refuses_to_start_and_writes_nothing() {
+    let parse = "does not parse as an audit record";
+    // (case, the files to copy, then the failure's file, line and text)
+    let cases: [Case; 9] = [
+        (
+            "changed-byte",
+            Some(&[D1]),
+            D1,
+            4,
+            "prev is not the hash of the line before",
+        ),
+        ("deleted-line", Some(&[D1]), D1, 3, "seq is 3, expected 2"),
+        ("swapped-lines", Some(&[D1]), D1, 2, "seq is 2, expected 1"),
+        ("seq-gap", None, D1, 3, "seq is 3, expected 2"),
+        ("cut-short", Some(&[D1]), D1, 3, parse),
+        // Both files: the latest does not chain from the last line of the one before.
+        (
+            "file-not-chained",
+            None,
+            D2,
+            1,
+            "does not chain from the last line of the file before",
+        ),
+        (
+            "break-without-failure",
+            None,
+            D2,
+            6,
+            "an accepted break with no failure before it",
+        ),
+        ("recovery-wrong-hash", None, D2, 6, parse),
+        ("torn-recovery", None, D2, 6, parse),
+    ];
+    for (case, only, file, line, what) in cases {
+        let dir = TempDir::case(case, only);
+        let before = snapshot(&dir.path);
+        let error = Writer::open(&dir.path, false)
+            .err()
+            .unwrap_or_else(|| panic!("{case}: started"));
+        let AuditError::Broken(failure) = &error else {
+            panic!("{case}: {error}");
+        };
+        assert_eq!(
+            (failure.file.as_str(), failure.line, failure.what.as_str()),
+            (file, line, what),
+            "{case}"
+        );
+        let text = error.to_string();
+        assert!(
+            text.starts_with(&format!("{file}:{line}: {what}")),
+            "{case}: {text}"
+        );
+        assert!(
+            text.ends_with("see docs/runbook.md#audit-chain-broken"),
+            "{case}: {text}"
+        );
+        assert_eq!(
+            snapshot(&dir.path),
+            before,
+            "{case}: the log was written to"
+        );
+    }
+}
+
+/// A torn tail is recovered: the torn bytes stay, a newline ends them if one is missing, and a
+/// `Recovery` record follows in the same file, whatever today's date is.
+#[test]
+fn a_torn_tail_is_recovered() {
+    // (case, the torn line's file and number, newline already there)
+    let cases = [
+        ("torn-tail", D2, 6, false),
+        ("torn-tail-complete-json", D2, 6, false),
+        ("torn-unparseable-newline", D2, 6, true),
+        ("torn-first-line", D2, 1, false),
+    ];
+    for (case, file, line, has_newline) in cases {
+        let dir = TempDir::case(case, None);
+        let before = snapshot(&dir.path);
+        let opened = Writer::open(&dir.path, false).unwrap_or_else(|e| panic!("{case}: {e}"));
+        assert!(opened.recovered, "{case}");
+        assert!(opened.accepted.is_none(), "{case}");
+
+        let after = snapshot(&dir.path);
+        assert_eq!(
+            after.len(),
+            before.len(),
+            "{case}: the Recovery went into a new file"
+        );
+        let (old, new) = (&before[file], &after[file]);
+        assert!(
+            new.starts_with(old),
+            "{case}: bytes already on disk were changed"
+        );
+        let added = &new[old.len()..];
+        // One newline to end the torn line if it had none, then one line.
+        let added = if has_newline {
+            added
+        } else {
+            added.strip_prefix(b"\n").expect(case)
+        };
+        assert_eq!(added.iter().filter(|b| **b == b'\n').count(), 1, "{case}");
+        let record: AuditRecord =
+            serde_json::from_slice(added.strip_suffix(b"\n").expect(case)).expect(case);
+        assert!(
+            matches!(record.event, AuditEvent::Recovery { .. }),
+            "{case}"
+        );
+
+        let report = verify_dir(&dir.path).unwrap();
+        assert_eq!(report.failure, None, "{case}");
+        assert_eq!(report.torn_tail, None, "{case}");
+        assert_eq!(report.recoveries, vec![at(file, line)], "{case}");
+
+        // The chain goes on from the Recovery, and the next start finds nothing to recover.
+        let mut writer = opened.writer;
+        assert_eq!(
+            writer.append(Timestamp::now(), denied(9)).unwrap(),
+            record.seq + 1,
+            "{case}"
+        );
+        drop(writer);
+        let opened = Writer::open(&dir.path, false).unwrap();
+        assert!(!opened.recovered, "{case}");
+        assert_eq!(verify_dir(&dir.path).unwrap().failure, None, "{case}");
+    }
+}
+
+/// Damage in an older file is not seen by an ordinary start. `bxctl audit verify` sees it, and
+/// `--accept-break` must too: it verifies the whole log.
+#[test]
+fn a_break_in_an_older_file_can_be_accepted() {
+    let dir = TempDir::case("changed-byte", None);
+    let before = snapshot(&dir.path);
+    drop(Writer::open(&dir.path, false).expect("the latest file verifies"));
+    assert_eq!(snapshot(&dir.path), before);
+    let failure = verify_dir(&dir.path).unwrap().failure.unwrap();
+    assert_eq!((failure.file.as_str(), failure.line), (D1, 4));
+
+    let opened = Writer::open(&dir.path, true).unwrap();
+    assert!(!opened.recovered);
+    assert_eq!(
+        opened.accepted.as_ref().map(|f| (f.file.as_str(), f.line)),
+        Some((D1, 4))
+    );
+    let after = snapshot(&dir.path);
+    assert_eq!(after[D1], before[D1], "nothing is repaired");
+    assert!(after[D2].starts_with(&before[D2]));
+    let last: AuditRecord = serde_json::from_str(lines(&dir.path, D2).last().unwrap()).unwrap();
+    assert_eq!(
+        last.event,
+        AuditEvent::AcceptedBreak {
+            file: D1.to_string(),
+            line: 4,
+            last_good: failure.last_good,
+        }
+    );
+    assert_eq!(
+        last.seq, 10,
+        "seq 3 for the failing line, and seven lines to the break"
+    );
+
+    let report = verify_dir(&dir.path).unwrap();
+    assert_eq!(report.failure, None);
+    assert_eq!(report.accepted_breaks, vec![at(D2, 6)]);
+
+    let mut writer = opened.writer;
+    assert_eq!(writer.append(Timestamp::now(), denied(9)).unwrap(), 11);
+    drop(writer);
+
+    // The next ordinary start resumes at the latest file and meets a break that names a file
+    // it has not read.
+    let mut writer = Writer::open(&dir.path, false).unwrap().writer;
+    assert_eq!(writer.append(Timestamp::now(), denied(10)).unwrap(), 12);
+    assert_eq!(verify_dir(&dir.path).unwrap().failure, None);
+}
+
+#[test]
+fn a_break_in_the_latest_file_can_be_accepted() {
+    // (case, failing line, seq of the break record)
+    for (case, line, seq) in [("recovery-wrong-hash", 6, 12), ("torn-recovery", 6, 12)] {
+        let dir = TempDir::case(case, None);
+        let before = snapshot(&dir.path);
+        let opened = Writer::open(&dir.path, true).unwrap_or_else(|e| panic!("{case}: {e}"));
+        assert_eq!(
+            opened.accepted.as_ref().map(|f| f.line),
+            Some(line),
+            "{case}"
+        );
+        let after = snapshot(&dir.path);
+        assert!(after[D2].starts_with(&before[D2]), "{case}");
+
+        // torn-recovery ends without a newline: the break record must start on its own line.
+        let all = lines(&dir.path, D2);
+        let last: AuditRecord = serde_json::from_str(all.last().unwrap()).expect(case);
+        assert!(
+            matches!(last.event, AuditEvent::AcceptedBreak { .. }),
+            "{case}"
+        );
+        assert_eq!(last.seq, seq, "{case}");
+        assert_eq!(all.len(), 8, "{case}");
+
+        let report = verify_dir(&dir.path).unwrap();
+        assert_eq!(report.failure, None, "{case}");
+        assert_eq!(report.accepted_breaks, vec![at(D2, 8)], "{case}");
+        drop(opened);
+        assert!(Writer::open(&dir.path, false).is_ok(), "{case}");
+    }
+}
+
+/// The short check is a shortcut and never the last word: when it cannot be made, or fails, the
+/// whole log is verified and that verdict stands. Here the last line of the older file is the
+/// damage, so there is nothing to resume from.
+#[test]
+fn an_accepted_break_at_the_end_of_an_older_file_does_not_stop_later_starts() {
+    let dir = TempDir::case("good", None);
+    let day1 = std::fs::read_to_string(dir.path.join(D1)).unwrap();
+    let cut = format!("{}\n", &day1[..day1.len() - 40]);
+    std::fs::write(dir.path.join(D1), cut).unwrap();
+
+    let error = Writer::open(&dir.path, false).unwrap_err();
+    let AuditError::Broken(failure) = &error else {
+        panic!("{error}");
+    };
+    assert_eq!((failure.file.as_str(), failure.line), (D1, 5));
+
+    drop(Writer::open(&dir.path, true).unwrap());
+    let mut writer = Writer::open(&dir.path, false)
+        .expect("the break was accepted")
+        .writer;
+    writer.append(Timestamp::now(), denied(9)).unwrap();
+    assert_eq!(verify_dir(&dir.path).unwrap().failure, None);
+}
+
+#[test]
+fn accept_break_with_nothing_to_accept_is_an_error() {
+    for case in ["good", "torn-tail"] {
+        let dir = TempDir::case(case, None);
+        let before = snapshot(&dir.path);
+        let error = Writer::open(&dir.path, true).unwrap_err();
+        assert!(
+            matches!(error, AuditError::NothingToAccept),
+            "{case}: {error}"
+        );
+        assert!(error.to_string().starts_with("nothing to accept"), "{case}");
+        assert_eq!(
+            snapshot(&dir.path),
+            before,
+            "{case}: the log was written to"
+        );
+    }
+}
+
+/// A second failure after an accepted break needs its own break.
+#[test]
+fn damage_after_a_break_is_a_new_failure() {
+    let dir = TempDir::case("accepted-break", None);
+    drop(Writer::open(&dir.path, false).expect("the fixture verifies"));
+    let mut text = std::fs::read_to_string(dir.path.join(D1)).unwrap();
+    text.push_str("{}\n{}\n");
+    std::fs::write(dir.path.join(D1), text).unwrap();
+
+    let error = Writer::open(&dir.path, false).unwrap_err();
+    let AuditError::Broken(failure) = &error else {
+        panic!("{error}");
+    };
+    assert_eq!((failure.file.as_str(), failure.line), (D1, 8));
+    drop(Writer::open(&dir.path, true).unwrap());
+    let report = verify_dir(&dir.path).unwrap();
+    assert_eq!(report.failure, None);
+    assert_eq!(report.accepted_breaks, vec![at(D1, 6), at(D1, 10)]);
+}

crates/brokerd/tests/support/audit_dir.rs

@@ -0,0 +1,98 @@
+//! Temporary audit directories for the audit tests. Do not edit.
+
+#![allow(dead_code)] // each test file uses its own part of this
+
+use std::collections::BTreeMap;
+use std::path::{Path, PathBuf};
+use std::sync::atomic::{AtomicU32, Ordering};
+
+use proto::{AuditEvent, CallId, DataClass, DecisionRecord, SessionId, Timestamp};
+
+pub const D1: &str = "2026-09-17.jsonl";
+pub const D2: &str = "2026-09-18.jsonl";
+
+static NEXT: AtomicU32 = AtomicU32::new(0);
+
+/// A directory under the system's temporary directory, removed when dropped.
+pub struct TempDir {
+    pub path: PathBuf,
+}
+
+impl TempDir {
+    /// A path that does not exist yet.
+    pub fn unmade(tag: &str) -> TempDir {
+        let n = NEXT.fetch_add(1, Ordering::SeqCst);
+        let name = format!("brokerd-{tag}-{}-{n}", std::process::id());
+        let path = std::env::temp_dir().join(name);
+        let _ = std::fs::remove_dir_all(&path);
+        TempDir { path }
+    }
+
+    /// A copy of the fixture log `case` from `crates/proto/tests/fixtures/audit/`. With `only`,
+    /// just those files: damage in an older file is not seen by an ordinary start, so tests of
+    /// the startup check copy the damaged file alone.
+    pub fn case(case: &str, only: Option<&[&str]>) -> TempDir {
+        let dir = TempDir::unmade(case);
+        std::fs::create_dir_all(&dir.path).unwrap();
+        let from = format!(
+            "{}/../proto/tests/fixtures/audit/{case}",
+            env!("CARGO_MANIFEST_DIR")
+        );
+        let mut copied = 0;
+        for entry in std::fs::read_dir(&from).unwrap_or_else(|e| panic!("{from}: {e}")) {
+            let entry = entry.unwrap();
+            let name = entry.file_name().into_string().unwrap();
+            if only.is_none_or(|names| names.contains(&name.as_str())) {
+                std::fs::copy(entry.path(), dir.path.join(&name)).unwrap();
+                copied += 1;
+            }
+        }
+        assert!(copied > 0, "{from}: nothing copied");
+        dir
+    }
+}
+
+impl Drop for TempDir {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.path);
+    }
+}
+
+/// Every log file in `dir` with its bytes.
+pub fn snapshot(dir: &Path) -> BTreeMap<String, Vec<u8>> {
+    std::fs::read_dir(dir)
+        .unwrap()
+        .map(|entry| entry.unwrap())
+        .filter(|entry| entry.file_name().to_string_lossy().ends_with(".jsonl"))
+        .map(|entry| {
+            let name = entry.file_name().into_string().unwrap();
+            (name, std::fs::read(entry.path()).unwrap())
+        })
+        .collect()
+}
+
+pub fn lines(dir: &Path, file: &str) -> Vec<String> {
+    let text = std::fs::read_to_string(dir.join(file)).unwrap();
+    text.lines().map(str::to_string).collect()
+}
+
+pub fn ts(s: &str) -> Timestamp {
+    Timestamp::parse(s).unwrap()
+}
+
+/// A denied decision for call `call`: an event that leaves nothing open in the report.
+pub fn denied(call: u64) -> AuditEvent {
+    AuditEvent::Decision {
+        session: SessionId::new("chat-1").unwrap(),
+        call: CallId(call),
+        tool: "read_file".to_string(),
+        arguments: r#"{"path":"/etc/hosts"}"#.to_string(),
+        outcome: DecisionRecord::Denied {
+            reason: proto::DenyReason::NoGrant,
+        },
+        grant: None,
+        grant_sha256: None,
+        taint: DataClass::Private,
+        untrusted: false,
+    }
+}