review: a session log that cannot be read now is waited out
The worker marked a turn failed on any error reading its log. A missing log, a damaged line or a turn not found still fail it; any other I/O error is now waited out with the outage's doubling wait, logged once when it starts and once when the log reads again (pointer session-log-damaged, whose runbook entry now covers it), shown by bxctl memory sync, and the turn stays waiting. read_records tells an unreadable file from a damaged line (read_log is unchanged). The message helpers move from worker.rs to honcho/messages.rs, re-exported, to keep worker.rs under 500 lines. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -42,6 +42,7 @@ fn at(minute: u32) -> Timestamp {
|
||||
const ALL: &str = "honcho sync: 0 decided: 0 sent, 0 waiting, 0 failed, 0 skipped
|
||||
brokerd unanswered since 2026-09-28T10:01:00.000Z: turns wait undecided
|
||||
honcho unreachable since 2026-09-28T10:02:00.000Z: sends wait and are retried
|
||||
session log unreadable since 2026-09-28T10:03:00.000Z: sends wait and are retried
|
||||
key refused (HTTP 401): trusted workspace boxmaker0-trusted: its sends wait until memoryd restarts
|
||||
key refused (HTTP 403): fleet workspace fleet: its sends wait until memoryd restarts";
|
||||
|
||||
@@ -54,6 +55,7 @@ fn the_status_lines_are_exactly_these() {
|
||||
let mut status = Status {
|
||||
broker_down_since: Some(at(1)),
|
||||
outage_since: Some(at(2)),
|
||||
log_unreadable_since: Some(at(3)),
|
||||
..Status::default()
|
||||
};
|
||||
status
|
||||
|
||||
@@ -0,0 +1,128 @@
|
||||
//! A session log the worker cannot read for now (review of M5c-1, finding 6): an I/O error other
|
||||
//! than a missing file is waited out like an outage, with the same doubling wait, logged once when
|
||||
//! it starts and once when it ends, and shown by `bxctl memory sync`; the turn stays waiting and
|
||||
//! is sent once the log reads. A missing log or a damaged line still marks the turn failed. The
|
||||
//! log is made unreadable by a directory in its place (EISDIR), which holds for root too.
|
||||
|
||||
#[path = "support/honcho.rs"]
|
||||
mod honcho;
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
#[path = "support/worker.rs"]
|
||||
mod worker;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use honcho::FakeHoncho;
|
||||
use logs::assistant;
|
||||
use memoryd::honcho::ledger::{Item, Record};
|
||||
use memoryd::honcho::status;
|
||||
use memoryd::honcho::turns::log_path;
|
||||
use memoryd::honcho::worker::Step;
|
||||
use turns::{sid, spoken};
|
||||
use worker::{FAST, Home};
|
||||
|
||||
fn turn(text: &str) -> Vec<bxcore::LogRecord> {
|
||||
vec![spoken(text, Some("kyle")), assistant(2, Some("ok"), &[])]
|
||||
}
|
||||
|
||||
fn is_sent(records: &[Record]) -> bool {
|
||||
let item = Item::Turn {
|
||||
session: sid("mm-a"),
|
||||
turn: 1,
|
||||
};
|
||||
records
|
||||
.iter()
|
||||
.any(|r| matches!(r, Record::Sent { item: i, .. } if *i == item))
|
||||
}
|
||||
|
||||
fn has_failed(records: &[Record]) -> bool {
|
||||
records.iter().any(|r| matches!(r, Record::Failed { .. }))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unreadable_log_is_waited_out() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-log-io");
|
||||
home.log("mm-a", &[turn("hello")]);
|
||||
home.report("mm-a", 1);
|
||||
let path = log_path(&home.dir.path().join("sessions"), &sid("mm-a"));
|
||||
let kept = std::fs::read(&path).unwrap();
|
||||
std::fs::remove_file(&path).unwrap();
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Backoff(Duration::from_millis(1)));
|
||||
assert_eq!(w.step(), Step::Backoff(Duration::from_millis(2)));
|
||||
assert_eq!(w.step(), Step::Backoff(Duration::from_millis(4)));
|
||||
let records = home.records();
|
||||
assert!(!has_failed(&records), "{records:?}");
|
||||
let lines = home.lines();
|
||||
assert_eq!(lines.len(), 1, "logged once: {lines:?}");
|
||||
assert!(
|
||||
lines[0].starts_with("memoryd: turn 1 of session mm-a cannot be read now ("),
|
||||
"{}",
|
||||
lines[0]
|
||||
);
|
||||
assert!(
|
||||
lines[0]
|
||||
.ends_with("); sends wait and are retried; see docs/runbook.md#session-log-damaged"),
|
||||
"{}",
|
||||
lines[0]
|
||||
);
|
||||
assert!(!lines[0].contains("hello"));
|
||||
let shared = home.sync.lock().unwrap().shared_status();
|
||||
assert!(status::read(&shared).log_unreadable_since.is_some());
|
||||
let shown = status::lines(&status::read(&shared));
|
||||
assert!(
|
||||
shown[0].starts_with("session log unreadable since "),
|
||||
"{shown:?}"
|
||||
);
|
||||
assert!(
|
||||
shown[0].ends_with(": sends wait and are retried"),
|
||||
"{shown:?}"
|
||||
);
|
||||
|
||||
std::fs::remove_dir(&path).unwrap();
|
||||
std::fs::write(&path, kept).unwrap();
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert!(is_sent(&home.records()));
|
||||
let lines = home.lines();
|
||||
assert_eq!(lines.len(), 2, "{lines:?}");
|
||||
assert_eq!(
|
||||
lines[1],
|
||||
format!(
|
||||
"memoryd: {} reads again; waiting sends go on",
|
||||
path.display()
|
||||
)
|
||||
);
|
||||
assert_eq!(status::read(&shared), status::Status::default());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_or_damaged_log_still_fails_the_turn() {
|
||||
let fake = FakeHoncho::start();
|
||||
for damage in ["missing", "damaged"] {
|
||||
let home = Home::new("worker-log-gone");
|
||||
home.log("mm-a", &[turn("hello")]);
|
||||
home.report("mm-a", 1);
|
||||
let path = log_path(&home.dir.path().join("sessions"), &sid("mm-a"));
|
||||
match damage {
|
||||
"missing" => std::fs::remove_file(&path).unwrap(),
|
||||
_ => std::fs::write(&path, "not a record\n").unwrap(),
|
||||
}
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done, "{damage}");
|
||||
let records = home.records();
|
||||
assert!(
|
||||
records
|
||||
.iter()
|
||||
.any(|r| matches!(r, Record::Failed { kind, .. } if kind == "log")),
|
||||
"{damage}: {records:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user