kyle/boxmaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

brokerd: recover a torn line in place, real dates only, bounded ttl, EMFILE

Browse Source 
From the independent review of task 23. A torn last line followed by an empty
later file had its recovery written into the later file, which broke the
chain for good; the line is now ended in its own file. The log-name rule
takes months 01 to 12 and days 01 to 31 only. [approvals] ttl_ms is limited
to a day, the longest loopd waits after a pending frame. Running out of file
descriptors or memory pauses the listener instead of stopping brokerd (the
errors the previous fix skipped do not occur on Linux). args.rs's doc fixed.

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
kyle
2026-09-22 21:48:24 -07:00
co-authored by Claude Opus 5.5
parent ba369f82ba
commit e08deb39a6
9 changed files with 157 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brokerd/src/args.rs
+1 -1
View File
@@ -6,8 +6,8 @@
use serde::{Deserialize, Serialize};
use std::fmt;
/// Maximum length, in bytes, of a path.
/// Maximum length, in bytes, of a path or of a URL.
pub const MAX_PATH: usize = 4096;
/// Maximum length, in bytes, of a URL.
pub const MAX_URL: usize = 2048;
crates/brokerd/src/audit.rs
+38 -21
View File
@@ -191,12 +191,35 @@ fn tail_of_file_before_latest(dir: &Path) -> Result<Hash32, AuditError> {
}
}
/// End the last line of the log with a newline, in the file that holds it: the last log file that
/// is not empty. A torn line is always there, even when an empty later file exists, and it must be
/// ended in place, or the next record would join it on one line.
fn end_last_line(dir: &Path) -> Result<(), AuditError> {
for name in log_files(dir)?.iter().rev() {
let path = dir.join(name);
let bytes = fs::read(&path).map_err(|e| io("read audit log", &path, e))?;
match bytes.last() {
None => continue,
Some(b'\n') => return Ok(()),
Some(_) => {
let mut file = fs::OpenOptions::new()
.append(true)
.open(&path)
.map_err(|e| io("open audit log", &path, e))?;
file.write_all(b"\n").map_err(io::Error::other)?;
file.sync_all().map_err(io::Error::other)?;
return Ok(());
}
}
}
Ok(())
}
/// Append one record: serialise it, open the target file, write the line, and return the seq and
/// the hash of the line it wrote. A fresh file is created mode 0600 and the directory synced.
fn write_record(
dir: &Path,
target: &str,
newline_first: bool,
seq: u64,
prev: Hash32,
event: AuditEvent,
@@ -216,9 +239,6 @@ fn write_record(
.mode(0o600)
.open(&path)
.map_err(|e| io("open audit log", &path, e))?;
if newline_first {
file.write_all(b"\n").map_err(io::Error::other)?;
}
file.write_all(json.as_bytes()).map_err(io::Error::other)?;
file.write_all(b"\n").map_err(io::Error::other)?;
file.sync_all().map_err(io::Error::other)?;
@@ -305,10 +325,12 @@ impl Writer {
if !accept_break {
return Err(AuditError::Broken(Box::new(failure.clone())));
}
if failure.tail_torn {
end_last_line(dir)?;
}
let (seq, hash) = write_record(
dir,
latest.as_deref().unwrap_or(&day_name(now)),
failure.tail_torn,
failure.break_seq,
failure.break_prev,
AuditEvent::AcceptedBreak {
@@ -322,10 +344,12 @@ impl Writer {
} else if accept_break {
return Err(AuditError::NothingToAccept);
} else if let Some(torn) = report.torn_tail.as_ref() {
if !torn.has_newline {
end_last_line(dir)?;
}
let (seq, hash) = write_record(
dir,
latest.as_deref().unwrap_or(&day_name(now)),
!torn.has_newline,
torn.recovery_seq,
torn.recovery_prev,
AuditEvent::Recovery {
@@ -366,21 +390,14 @@ impl Writer {
return Err(AuditError::Stopped);
}
let target = target_for(&self.latest, time);
let (seq, hash) = match write_record(
&self.path,
&target,
false,
self.next_seq,
self.prev,
event,
time,
) {
Ok(r) => r,
Err(e) => {
self.stopped = true;
return Err(e);
}
};
let (seq, hash) =
match write_record(&self.path, &target, self.next_seq, self.prev, event, time) {
Ok(r) => r,
Err(e) => {
self.stopped = true;
return Err(e);
}
};
self.prev = hash;
self.next_seq = seq.checked_add(1).unwrap_or(self.next_seq);
if self.latest.as_deref() != Some(target.as_str()) {
crates/brokerd/src/config.rs
+17
View File
@@ -56,10 +56,15 @@ pub struct Config {
pub approvals: Approvals,
}
/// The longest an approval may wait: a day, the longest `loopd` waits after a pending frame.
pub const MAX_TTL_MS: u64 = 86_400_000;
#[derive(Debug)]
pub enum ConfigError {
Read(PathBuf, std::io::Error),
Parse(PathBuf, toml::de::Error),
/// The file parses, but a value is outside what brokerd accepts.
Invalid(PathBuf, String),
}
impl std::fmt::Display for ConfigError {
@@ -67,6 +72,7 @@ impl std::fmt::Display for ConfigError {
match self {
ConfigError::Read(path, err) => write!(f, "{}: {err}", path.display()),
ConfigError::Parse(path, err) => write!(f, "{}: {err}", path.display()),
ConfigError::Invalid(path, why) => write!(f, "{}: {why}", path.display()),
}
}
}
@@ -82,6 +88,17 @@ impl Config {
std::fs::read_to_string(path).map_err(|e| ConfigError::Read(path.to_path_buf(), e))?;
let config: Config =
toml::from_str(&text).map_err(|e| ConfigError::Parse(path.to_path_buf(), e))?;
// `loopd` waits at most a day after a pending frame, so a longer approval would be given up
// by the caller while `bxctl approvals` still listed it.
if !(1..=MAX_TTL_MS).contains(&config.approvals.ttl_ms) {
return Err(ConfigError::Invalid(
path.to_path_buf(),
format!(
"[approvals] ttl_ms is {}; it must be from 1 to {MAX_TTL_MS} (a day)",
config.approvals.ttl_ms
),
));
}
Ok(config)
}
pub fn broker_socket(&self) -> PathBuf {
crates/brokerd/src/serve.rs
+25 -11
View File
@@ -197,9 +197,10 @@ impl Started {
}
}
/// Accept streams from one listener forever, starting a handler thread per stream. An error that
/// only concerns one connection (the peer aborted it, a signal interrupted `accept`) is skipped;
/// any other stops the daemon through `tx`.
/// Accept streams from one listener forever, starting a handler thread per stream. When the
/// system is out of file descriptors or memory, the listener pauses and tries again instead of
/// stopping: otherwise a peer holding many idle connections could stop the daemon, and the other
/// socket with it. Any other error stops the daemon through `tx`.
fn accept_loop(
listener: UnixListener,
broker: Arc<Broker>,
@@ -213,10 +214,22 @@ fn accept_loop(
std::thread::Builder::new()
.name(name.to_string())
.spawn(move || {
let mut reported = false;
for stream in listener.incoming() {
match stream {
Ok(stream) => serve_one(stream, &broker, which),
Err(e) if one_connection_only(&e) => {}
Ok(stream) => {
reported = false;
serve_one(stream, &broker, which);
}
Err(e) if out_of_resources(&e) => {
if !reported {
broker.log(&format!(
"brokerd: cannot accept on {name} for now, retrying: {e}\nsee docs/runbook.md#brokerd-listener-lost"
));
reported = true;
}
std::thread::sleep(Duration::from_millis(200));
}
Err(e) => {
// Once `run` has returned, nobody receives; dropping this is correct.
let _ = tx.send(e);
@@ -243,10 +256,11 @@ fn serve_one(stream: UnixStream, broker: &Arc<Broker>, which: Which) {
}
}
/// Errors from `accept` that end one connection, not the listener.
fn one_connection_only(e: &io::Error) -> bool {
matches!(
e.kind(),
io::ErrorKind::ConnectionAborted | io::ErrorKind::Interrupted | io::ErrorKind::WouldBlock
)
/// Errors from `accept` that mean "not now" rather than "never": out of file descriptors
/// (`EMFILE`, `ENFILE`, the same numbers on Linux and macOS) or out of memory. `accept` already
/// retries `EINTR` itself.
fn out_of_resources(e: &io::Error) -> bool {
const ENFILE: i32 = 23;
const EMFILE: i32 = 24;
matches!(e.raw_os_error(), Some(ENFILE | EMFILE)) || e.kind() == io::ErrorKind::OutOfMemory
}