review: bxctl memory sync shows what the Honcho side waits on
A shared status, kept by the worker and the walk and read by the report under its own leaf lock, holds the workspaces whose key was refused, when an outage of Honcho began, and when brokerd stopped answering. The report adds a line for each while it lasts, and is unchanged when all is well. The runbook's Honcho entries and broker-state-unavailable now say what sync shows. Tuning and Wake move to honcho/wake.rs (re-exported from worker) to keep worker.rs under 500 lines. honcho_sync.rs's serve test now allows the outage line: the write wakes the worker, which finds Honcho unreachable. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -16,7 +16,7 @@ use serde::{Deserialize, Serialize};
|
||||
pub const RUNBOOK: &str = "see docs/runbook.md#honcho-ledger-damaged";
|
||||
|
||||
/// One of this agent's three workspaces.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord, Hash, Serialize, Deserialize)]
|
||||
#[serde(rename_all = "snake_case")]
|
||||
pub enum Workspace {
|
||||
Trusted,
|
||||
|
||||
@@ -9,6 +9,7 @@ use bxcore::{DataClass, SessionId, Timestamp};
|
||||
use crate::files::Store;
|
||||
use crate::honcho::ledger::{self, Action, Item, Ledger, Record, Workspace};
|
||||
use crate::honcho::report;
|
||||
use crate::honcho::status::{self, Shared};
|
||||
use crate::journal;
|
||||
use crate::notes::{self, Kind};
|
||||
|
||||
@@ -204,6 +205,8 @@ fn brought_back(ledger: &Ledger, action: &Action) -> bool {
|
||||
/// With `[honcho]`: decides each journalled edit into the ledger.
|
||||
pub struct Mirror {
|
||||
ledger: Arc<Mutex<Ledger>>,
|
||||
/// What the worker and the walk wait on, for the report; empty unless `with_status` is given.
|
||||
status: Shared,
|
||||
agent: String,
|
||||
owner: String,
|
||||
}
|
||||
@@ -212,14 +215,25 @@ impl Mirror {
|
||||
pub fn new(ledger: Arc<Mutex<Ledger>>, agent: &str, owner: &str) -> Mirror {
|
||||
Mirror {
|
||||
ledger,
|
||||
status: Shared::default(),
|
||||
agent: agent.to_string(),
|
||||
owner: owner.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
/// `bxctl memory sync`'s report of the ledger, under the ledger's lock alone.
|
||||
/// The same, reporting `status` too.
|
||||
pub fn with_status(self, status: Shared) -> Mirror {
|
||||
Mirror { status, ..self }
|
||||
}
|
||||
|
||||
/// `bxctl memory sync`'s report: the status copied under its lock alone, then the ledger
|
||||
/// under its own.
|
||||
pub fn report(&self) -> String {
|
||||
report::report(&self.ledger.lock().unwrap_or_else(|p| p.into_inner()))
|
||||
let status = status::read(&self.status);
|
||||
report::with_status(
|
||||
&self.ledger.lock().unwrap_or_else(|p| p.into_inner()),
|
||||
&status,
|
||||
)
|
||||
}
|
||||
|
||||
/// Decide the journalled edit `id` (the latest when `None`), called under the index lock
|
||||
|
||||
@@ -11,5 +11,7 @@ pub mod naming;
|
||||
pub mod report;
|
||||
pub mod start;
|
||||
pub mod state;
|
||||
pub mod status;
|
||||
pub mod turns;
|
||||
pub mod wake;
|
||||
pub mod worker;
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
//! What `bxctl memory sync` prints (M5c spec, section 5): how many items the ledger holds in each
|
||||
//! state, and every failed item, oldest first, by its label, never by its text.
|
||||
//! What `bxctl memory sync` prints (M5c spec, sections 5 and 10): how many items the ledger holds
|
||||
//! in each state, every failed item, oldest first, by its label, never by its text, and what the
|
||||
//! worker and the walk are waiting on.
|
||||
|
||||
use crate::honcho::ledger::{Ledger, label};
|
||||
use crate::honcho::status::{self, Status};
|
||||
|
||||
/// The answer without `[honcho]`: not an error.
|
||||
pub const NOT_CONFIGURED: &str = "honcho sync: not configured (memoryd.toml has no [honcho])";
|
||||
@@ -31,3 +33,14 @@ pub fn report(ledger: &Ledger) -> String {
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// The report of `ledger`, then a line for each state in `status` that lasts (`status::lines`).
|
||||
/// When all is well, the same as `report`.
|
||||
pub fn with_status(ledger: &Ledger, status: &Status) -> String {
|
||||
let mut out = report(ledger);
|
||||
for line in status::lines(status) {
|
||||
out.push('\n');
|
||||
out.push_str(&line);
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
@@ -41,12 +41,11 @@ impl Opened {
|
||||
|
||||
/// The memory writes' side of the ledger (task 12).
|
||||
pub fn mirror(&self) -> Mirror {
|
||||
let ledger = self
|
||||
.sync
|
||||
.lock()
|
||||
.unwrap_or_else(|p| p.into_inner())
|
||||
.shared_ledger();
|
||||
Mirror::new(ledger, &self.agent, &self.owner)
|
||||
let (ledger, status) = {
|
||||
let sync = self.sync.lock().unwrap_or_else(|p| p.into_inner());
|
||||
(sync.shared_ledger(), sync.shared_status())
|
||||
};
|
||||
Mirror::new(ledger, &self.agent, &self.owner).with_status(status)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
//! What the Honcho side is waiting on, beyond the ledger, for `bxctl memory sync` (M5c spec,
|
||||
//! sections 10, 12 and 13): workspaces whose key Honcho refused, an outage of Honcho, and a
|
||||
//! `brokerd` that does not answer. The worker and the walk write it; the report reads it. Its lock
|
||||
//! is a leaf: nothing else is locked while it is held.
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::sync::{Arc, Mutex};
|
||||
|
||||
use bxcore::Timestamp;
|
||||
|
||||
use crate::honcho::ledger::Workspace;
|
||||
|
||||
#[derive(Debug, Clone, Default, PartialEq, Eq)]
|
||||
pub struct Status {
|
||||
/// Each workspace whose key was refused: its name and the HTTP status. Nothing more is sent
|
||||
/// to it until `memoryd` restarts.
|
||||
pub refused: BTreeMap<Workspace, (String, u16)>,
|
||||
/// When the outage of Honcho that lasts began.
|
||||
pub outage_since: Option<Timestamp>,
|
||||
/// When `brokerd` stopped answering on its state socket, while it does not.
|
||||
pub broker_down_since: Option<Timestamp>,
|
||||
}
|
||||
|
||||
/// The status, shared between the worker, the walk and the report.
|
||||
pub type Shared = Arc<Mutex<Status>>;
|
||||
|
||||
/// Change the shared status under its lock alone.
|
||||
pub fn update(shared: &Shared, change: impl FnOnce(&mut Status)) {
|
||||
change(&mut shared.lock().unwrap_or_else(|p| p.into_inner()));
|
||||
}
|
||||
|
||||
/// A copy of the shared status, taken under its lock alone.
|
||||
pub fn read(shared: &Shared) -> Status {
|
||||
shared.lock().unwrap_or_else(|p| p.into_inner()).clone()
|
||||
}
|
||||
|
||||
fn role(workspace: Workspace) -> &'static str {
|
||||
match workspace {
|
||||
Workspace::Trusted => "trusted",
|
||||
Workspace::Untrusted => "untrusted",
|
||||
Workspace::Fleet => "fleet",
|
||||
}
|
||||
}
|
||||
|
||||
/// One line for each state that lasts, in this order: `brokerd` silent, Honcho unreachable, then
|
||||
/// each refused key by workspace. None when all is well.
|
||||
pub fn lines(status: &Status) -> Vec<String> {
|
||||
let mut out = Vec::new();
|
||||
if let Some(since) = status.broker_down_since {
|
||||
out.push(format!(
|
||||
"brokerd unanswered since {}: turns wait undecided",
|
||||
since.to_rfc3339()
|
||||
));
|
||||
}
|
||||
if let Some(since) = status.outage_since {
|
||||
out.push(format!(
|
||||
"honcho unreachable since {}: sends wait and are retried",
|
||||
since.to_rfc3339()
|
||||
));
|
||||
}
|
||||
for (workspace, (name, code)) in &status.refused {
|
||||
out.push(format!(
|
||||
"key refused (HTTP {code}): {} workspace {name}: its sends wait until memoryd restarts",
|
||||
role(*workspace)
|
||||
));
|
||||
}
|
||||
out
|
||||
}
|
||||
@@ -12,6 +12,7 @@ use bxcore::{LogRecord, SessionId, SessionStateReply, Timestamp};
|
||||
use crate::honcho::decide::{decide, turn_records, turns};
|
||||
use crate::honcho::ledger::{self, Item, Ledger, Record};
|
||||
use crate::honcho::state;
|
||||
use crate::honcho::status::{self, Shared};
|
||||
|
||||
/// The ledger's file name, in `memory/`.
|
||||
pub const LEDGER: &str = "honcho.jsonl";
|
||||
@@ -47,6 +48,8 @@ pub struct TurnSync {
|
||||
broker_down: bool,
|
||||
/// Sessions whose log last had a damaged line: logged once each.
|
||||
unreadable: BTreeSet<String>,
|
||||
/// What `bxctl memory sync` shows beyond the ledger, shared with the worker and the report.
|
||||
status: Shared,
|
||||
}
|
||||
|
||||
/// The path of `session`'s log under `sessions`: epoch 0, the only one `loopd` writes.
|
||||
@@ -108,6 +111,7 @@ impl TurnSync {
|
||||
reported: BTreeMap::new(),
|
||||
broker_down: false,
|
||||
unreadable: BTreeSet::new(),
|
||||
status: Shared::default(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -121,6 +125,11 @@ impl TurnSync {
|
||||
Arc::clone(&self.ledger)
|
||||
}
|
||||
|
||||
/// The status `bxctl memory sync` shows, for the worker and the report.
|
||||
pub fn shared_status(&self) -> Shared {
|
||||
Arc::clone(&self.status)
|
||||
}
|
||||
|
||||
/// The directory of `loopd`'s session logs.
|
||||
pub fn sessions(&self) -> &Path {
|
||||
&self.sessions
|
||||
@@ -214,7 +223,7 @@ impl TurnSync {
|
||||
if undecided.is_empty() {
|
||||
return if held { Walk::Waiting } else { Walk::Done };
|
||||
}
|
||||
let Some(state) = self.state(session, ask, log) else {
|
||||
let Some(state) = self.state(session, ask, now, log) else {
|
||||
return Walk::Stop;
|
||||
};
|
||||
for turn in undecided {
|
||||
@@ -258,12 +267,14 @@ impl TurnSync {
|
||||
&mut self,
|
||||
session: &SessionId,
|
||||
ask: Ask<'_>,
|
||||
now: Timestamp,
|
||||
log: &dyn Fn(&str),
|
||||
) -> Option<SessionStateReply> {
|
||||
match ask(session) {
|
||||
Ok(state) => {
|
||||
if self.broker_down {
|
||||
self.broker_down = false;
|
||||
status::update(&self.status, |s| s.broker_down_since = None);
|
||||
log("memoryd: brokerd answers on its state socket again; turns are decided");
|
||||
}
|
||||
Some(state)
|
||||
@@ -271,6 +282,7 @@ impl TurnSync {
|
||||
Err(e) => {
|
||||
if !self.broker_down {
|
||||
self.broker_down = true;
|
||||
status::update(&self.status, |s| s.broker_down_since = Some(now));
|
||||
log(&format!(
|
||||
"memoryd: {e}; turns wait undecided; {}",
|
||||
state::RUNBOOK
|
||||
|
||||
@@ -0,0 +1,49 @@
|
||||
//! The Honcho worker's waits (M5c spec, section 5): the doubling wait of an outage and the idle
|
||||
//! sleep, and what wakes it early.
|
||||
|
||||
use std::sync::{Condvar, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
/// The worker's waits. Tests shorten them.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Tuning {
|
||||
/// The first wait in an outage; each next one doubles, up to `backoff_max`.
|
||||
pub backoff: Duration,
|
||||
pub backoff_max: Duration,
|
||||
/// How long the worker sleeps with nothing to send, unless a turn ends first.
|
||||
pub idle: Duration,
|
||||
}
|
||||
|
||||
impl Tuning {
|
||||
pub const DEFAULT: Tuning = Tuning {
|
||||
backoff: Duration::from_secs(5),
|
||||
backoff_max: Duration::from_secs(600),
|
||||
idle: Duration::from_secs(30),
|
||||
};
|
||||
}
|
||||
|
||||
/// Wakes the worker from its idle sleep when a turn ends.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Wake {
|
||||
woken: Mutex<bool>,
|
||||
ready: Condvar,
|
||||
}
|
||||
|
||||
impl Wake {
|
||||
/// Wake the worker if it sleeps, or make its next sleep return at once.
|
||||
pub fn notify(&self) {
|
||||
let mut woken = self.woken.lock().unwrap_or_else(|p| p.into_inner());
|
||||
*woken = true;
|
||||
self.ready.notify_one();
|
||||
}
|
||||
|
||||
/// Sleep until `notify`, or at most `timeout`.
|
||||
pub fn wait(&self, timeout: Duration) {
|
||||
let woken = self.woken.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let (mut woken, _) = self
|
||||
.ready
|
||||
.wait_timeout_while(woken, timeout, |woken| !*woken)
|
||||
.unwrap_or_else(|p| p.into_inner());
|
||||
*woken = false;
|
||||
}
|
||||
}
|
||||
@@ -7,7 +7,7 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use bxcore::sanitize::neutralise;
|
||||
@@ -20,10 +20,12 @@ use crate::honcho::decide::{final_reply, turn_records, user_text};
|
||||
use crate::honcho::edits;
|
||||
use crate::honcho::ledger::{self, Action, Item, Ledger, Record, Workspace};
|
||||
use crate::honcho::naming::{HonchoSession, honcho_session};
|
||||
use crate::honcho::status::{self, Shared};
|
||||
use crate::honcho::turns::{TurnSync, log_path, read_log};
|
||||
use crate::serve::Log;
|
||||
|
||||
pub use crate::honcho::start::{Opened, open, spawn};
|
||||
pub use crate::honcho::wake::{Tuning, Wake};
|
||||
|
||||
/// The most characters Honcho takes in one message (recorded: 25,000 is taken, 25,001 refused).
|
||||
pub const MAX_PART: usize = 25_000;
|
||||
@@ -35,50 +37,6 @@ pub const SEND_REFUSED: &str = "see docs/runbook.md#honcho-send-refused";
|
||||
pub const KEY_REFUSED: &str = "see docs/runbook.md#honcho-key-refused";
|
||||
pub const LOG_DAMAGED: &str = "see docs/runbook.md#session-log-damaged";
|
||||
|
||||
/// The worker's waits. Tests shorten them.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Tuning {
|
||||
/// The first wait in an outage; each next one doubles, up to `backoff_max`.
|
||||
pub backoff: Duration,
|
||||
pub backoff_max: Duration,
|
||||
/// How long the worker sleeps with nothing to send, unless a turn ends first.
|
||||
pub idle: Duration,
|
||||
}
|
||||
|
||||
impl Tuning {
|
||||
pub const DEFAULT: Tuning = Tuning {
|
||||
backoff: Duration::from_secs(5),
|
||||
backoff_max: Duration::from_secs(600),
|
||||
idle: Duration::from_secs(30),
|
||||
};
|
||||
}
|
||||
|
||||
/// Wakes the worker from its idle sleep when a turn ends.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Wake {
|
||||
woken: Mutex<bool>,
|
||||
ready: Condvar,
|
||||
}
|
||||
|
||||
impl Wake {
|
||||
/// Wake the worker if it sleeps, or make its next sleep return at once.
|
||||
pub fn notify(&self) {
|
||||
let mut woken = self.woken.lock().unwrap_or_else(|p| p.into_inner());
|
||||
*woken = true;
|
||||
self.ready.notify_one();
|
||||
}
|
||||
|
||||
/// Sleep until `notify`, or at most `timeout`.
|
||||
pub fn wait(&self, timeout: Duration) {
|
||||
let woken = self.woken.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let (mut woken, _) = self
|
||||
.ready
|
||||
.wait_timeout_while(woken, timeout, |woken| !*woken)
|
||||
.unwrap_or_else(|p| p.into_inner());
|
||||
*woken = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// What one step of the worker did, and so what the loop does next.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Step {
|
||||
@@ -149,6 +107,8 @@ pub struct Worker {
|
||||
outage: Option<Duration>,
|
||||
/// Workspaces whose key was refused: nothing more is sent to them until restart.
|
||||
refused: HashSet<Workspace>,
|
||||
/// What `bxctl memory sync` shows; its lock is taken alone.
|
||||
status: Shared,
|
||||
stopped: bool,
|
||||
}
|
||||
|
||||
@@ -194,14 +154,6 @@ pub fn batch(
|
||||
messages
|
||||
}
|
||||
|
||||
/// An item as the log lines name it: `turn 3 of session mm-a`, `part 0 of edit 2026-09-28-0001`.
|
||||
fn label(item: &Item) -> String {
|
||||
match item {
|
||||
Item::Turn { session, turn } => format!("turn {turn} of session {}", session.as_str()),
|
||||
Item::Edit { id, part } => format!("part {part} of edit {id}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The ids of the messages Honcho made, in order.
|
||||
fn ids(messages: &[roncho::Message]) -> Vec<String> {
|
||||
messages.iter().map(|m| m.id.clone()).collect()
|
||||
@@ -218,9 +170,13 @@ impl Worker {
|
||||
tuning: Tuning,
|
||||
agent: &str,
|
||||
) -> Worker {
|
||||
let (ledger, sessions) = {
|
||||
let (ledger, sessions, status) = {
|
||||
let sync = sync.lock().unwrap_or_else(|p| p.into_inner());
|
||||
(sync.shared_ledger(), sync.sessions().to_path_buf())
|
||||
(
|
||||
sync.shared_ledger(),
|
||||
sync.sessions().to_path_buf(),
|
||||
sync.shared_status(),
|
||||
)
|
||||
};
|
||||
Worker {
|
||||
sync,
|
||||
@@ -234,6 +190,7 @@ impl Worker {
|
||||
agent: agent.to_string(),
|
||||
outage: None,
|
||||
refused: HashSet::new(),
|
||||
status,
|
||||
stopped: false,
|
||||
}
|
||||
}
|
||||
@@ -275,17 +232,24 @@ impl Worker {
|
||||
Err(Unsent::Honcho(HonchoError::KeyRefused { status })) => {
|
||||
self.recovered();
|
||||
self.refused.insert(next.workspace);
|
||||
let name = self
|
||||
.clients
|
||||
.client(next.workspace)
|
||||
.workspace_id()
|
||||
.to_string();
|
||||
(self.log)(&format!(
|
||||
"memoryd: Honcho refused the key of workspace {} (HTTP {status}); nothing more is sent to it until memoryd restarts; {KEY_REFUSED}",
|
||||
self.clients.client(next.workspace).workspace_id()
|
||||
"memoryd: Honcho refused the key of workspace {name} (HTTP {status}); nothing more is sent to it until memoryd restarts; {KEY_REFUSED}"
|
||||
));
|
||||
status::update(&self.status, |s| {
|
||||
s.refused.insert(next.workspace, (name, status));
|
||||
});
|
||||
Step::Done
|
||||
}
|
||||
Err(Unsent::Honcho(e)) => {
|
||||
self.recovered();
|
||||
(self.log)(&format!(
|
||||
"memoryd: {} was not sent: {e}; it is marked failed; {SEND_REFUSED}",
|
||||
label(&next.item)
|
||||
ledger::label(&next.item)
|
||||
));
|
||||
self.record(Record::Failed {
|
||||
time: Timestamp::now(),
|
||||
@@ -302,7 +266,7 @@ impl Worker {
|
||||
};
|
||||
(self.log)(&format!(
|
||||
"memoryd: {} cannot be sent ({kind}); it is marked failed; {pointer}",
|
||||
label(&next.item)
|
||||
ledger::label(&next.item)
|
||||
));
|
||||
self.record(Record::Failed {
|
||||
time: Timestamp::now(),
|
||||
@@ -446,6 +410,7 @@ impl Worker {
|
||||
(self.log)(
|
||||
format!("memoryd: {error}; sends wait and are retried; {UNREACHABLE}").as_str(),
|
||||
);
|
||||
status::update(&self.status, |s| s.outage_since = Some(Timestamp::now()));
|
||||
self.tuning.backoff
|
||||
}
|
||||
};
|
||||
@@ -457,6 +422,7 @@ impl Worker {
|
||||
fn recovered(&mut self) {
|
||||
if self.outage.take().is_some() {
|
||||
(self.log)("memoryd: Honcho answers again; waiting sends go on");
|
||||
status::update(&self.status, |s| s.outage_since = None);
|
||||
self.retry();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
//! What `bxctl memory sync` shows beyond the ledger (M5c spec, sections 10, 12 and 13; review of
|
||||
//! M5c-1, finding 4): a workspace whose key was refused, an outage of Honcho and a silent
|
||||
//! `brokerd`, each while it lasts, in exact lines; nothing more when all is well. The worker and
|
||||
//! the walk keep the status; `memoryd` answers it.
|
||||
|
||||
#[path = "support/brokerd.rs"]
|
||||
mod brokerd;
|
||||
#[path = "support/honcho.rs"]
|
||||
mod honcho;
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
#[path = "support/worker.rs"]
|
||||
mod worker;
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bxcore::{
|
||||
CallId, DataClass, Envelope, MemoryAdmin, MemoryCall, MemoryCommand, Message, SessionId,
|
||||
Timestamp, ToolRequest, read_frame, write_frame,
|
||||
};
|
||||
use honcho::{FakeHoncho, Mode};
|
||||
use logs::assistant;
|
||||
use memoryd::config::Config;
|
||||
use memoryd::honcho::ledger::{Ledger, Record, Workspace};
|
||||
use memoryd::honcho::report::{report, with_status};
|
||||
use memoryd::honcho::status::Status;
|
||||
use memoryd::serve;
|
||||
use tmp::TempDir;
|
||||
use turns::spoken;
|
||||
use worker::{FAST, Home};
|
||||
|
||||
fn at(minute: u32) -> Timestamp {
|
||||
Timestamp::parse(&format!("2026-09-28T10:{minute:02}:00.000Z")).unwrap()
|
||||
}
|
||||
|
||||
const ALL: &str = "honcho sync: 0 decided: 0 sent, 0 waiting, 0 failed, 0 skipped
|
||||
brokerd unanswered since 2026-09-28T10:01:00.000Z: turns wait undecided
|
||||
honcho unreachable since 2026-09-28T10:02:00.000Z: sends wait and are retried
|
||||
key refused (HTTP 401): trusted workspace boxmaker0-trusted: its sends wait until memoryd restarts
|
||||
key refused (HTTP 403): fleet workspace fleet: its sends wait until memoryd restarts";
|
||||
|
||||
#[test]
|
||||
fn the_status_lines_are_exactly_these() {
|
||||
let dir = TempDir::new("status-report");
|
||||
let mut ledger = Ledger::open(&dir.path().join("honcho.jsonl")).unwrap();
|
||||
ledger.append(Record::Began { time: at(0) }).unwrap();
|
||||
assert_eq!(with_status(&ledger, &Status::default()), report(&ledger));
|
||||
let mut status = Status {
|
||||
broker_down_since: Some(at(1)),
|
||||
outage_since: Some(at(2)),
|
||||
..Status::default()
|
||||
};
|
||||
status
|
||||
.refused
|
||||
.insert(Workspace::Fleet, ("fleet".to_string(), 403));
|
||||
status
|
||||
.refused
|
||||
.insert(Workspace::Trusted, ("boxmaker0-trusted".to_string(), 401));
|
||||
assert_eq!(with_status(&ledger, &status), ALL);
|
||||
}
|
||||
|
||||
fn turn(text: &str) -> Vec<bxcore::LogRecord> {
|
||||
vec![spoken(text, Some("kyle")), assistant(2, Some("ok"), &[])]
|
||||
}
|
||||
|
||||
fn status(home: &Home) -> Status {
|
||||
let shared = home.sync.lock().unwrap().shared_status();
|
||||
memoryd::honcho::status::read(&shared)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_worker_keeps_the_outage_and_the_refused_keys() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("status-worker");
|
||||
home.log("mm-a", &[turn("a")]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(503));
|
||||
let before = Timestamp::now();
|
||||
w.step();
|
||||
let since = status(&home).outage_since.expect("an outage");
|
||||
assert!(since >= before);
|
||||
w.step();
|
||||
assert_eq!(status(&home).outage_since, Some(since), "from its start");
|
||||
fake.set(Mode::Fixtures);
|
||||
w.step();
|
||||
assert_eq!(status(&home), Status::default(), "the outage ended");
|
||||
|
||||
home.log("mm-b", &[turn("b")]);
|
||||
home.report("mm-b", 1);
|
||||
fake.set(Mode::Status(401));
|
||||
w.step();
|
||||
let refused = status(&home).refused;
|
||||
assert_eq!(
|
||||
refused.get(&Workspace::Trusted),
|
||||
Some(&("boxmaker0-trusted".to_string(), 401))
|
||||
);
|
||||
assert_eq!(refused.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_walk_keeps_a_silent_brokerd() {
|
||||
let home = Home::new("status-broker");
|
||||
home.log("mm-a", &[turn("a")]);
|
||||
*home.answer.lock().unwrap() = Err("brokerd is down".to_string());
|
||||
home.report("mm-a", 1);
|
||||
assert_eq!(
|
||||
status(&home).broker_down_since,
|
||||
Some(Timestamp::parse("2026-09-28T10:00:00.000Z").unwrap())
|
||||
);
|
||||
*home.answer.lock().unwrap() = Ok(turns::clean());
|
||||
home.report("mm-a", 1);
|
||||
assert_eq!(status(&home), Status::default());
|
||||
}
|
||||
|
||||
fn ask(socket: &std::path::Path, msg: Message) -> Message {
|
||||
let mut stream = UnixStream::connect(socket).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.unwrap();
|
||||
let env = Envelope {
|
||||
v: 1,
|
||||
id: 1,
|
||||
r#final: true,
|
||||
msg,
|
||||
};
|
||||
write_frame(&mut stream, &env).unwrap();
|
||||
read_frame(&mut stream).unwrap().msg
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memoryd_shows_a_refused_key() {
|
||||
let fake = FakeHoncho::start();
|
||||
fake.set(Mode::Status(401));
|
||||
let dir = TempDir::new("status-serve");
|
||||
let soul = dir.write("etc/soul.md", "You are Boxmaker.\n");
|
||||
let text = format!(
|
||||
"[paths]\nhome = \"{}\"\n[memory]\nsoul_starter = \"{}\"\n[honcho]\nurl = \"{}\"\nagent = \"boxmaker0\"\nowner = \"kyle\"\ntrusted = \"boxmaker0-trusted\"\nuntrusted = \"boxmaker0-untrusted\"\nfleet = \"fleet\"\n",
|
||||
dir.path().display(),
|
||||
soul.display(),
|
||||
fake.url
|
||||
);
|
||||
let config = Config::parse(&text).unwrap();
|
||||
let log: serve::Log = Arc::new(|_: &str| {});
|
||||
let running = serve::start(config.clone(), log).unwrap();
|
||||
std::thread::spawn(move || running.run());
|
||||
let call = Message::MemoryCall(MemoryCall {
|
||||
request: ToolRequest {
|
||||
session: SessionId::new("mm-a").unwrap(),
|
||||
call: CallId(1),
|
||||
tool: "memory".to_string(),
|
||||
arguments: r#"{"action":"add","file":"user","text":"Prefers UTC."}"#.to_string(),
|
||||
},
|
||||
taint: DataClass::Private,
|
||||
untrusted: false,
|
||||
});
|
||||
ask(&config.broker_socket(), call);
|
||||
let sync = || match ask(
|
||||
&config.admin_socket(),
|
||||
Message::MemoryAdmin(MemoryAdmin {
|
||||
command: MemoryCommand::Sync,
|
||||
arg: String::new(),
|
||||
}),
|
||||
) {
|
||||
Message::MemoryAdminReply(reply) => reply.text,
|
||||
other => panic!("{other:?}"),
|
||||
};
|
||||
let shown = Mutex::new(String::new());
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline {
|
||||
*shown.lock().unwrap() = sync();
|
||||
if shown.lock().unwrap().contains("key refused") {
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
assert_eq!(
|
||||
*shown.lock().unwrap(),
|
||||
"honcho sync: 1 decided: 0 sent, 1 waiting, 0 failed, 0 skipped\n\
|
||||
key refused (HTTP 401): trusted workspace boxmaker0-trusted: its sends wait until memoryd restarts"
|
||||
);
|
||||
}
|
||||
@@ -221,11 +221,19 @@ fn memoryd_answers_with_the_report() {
|
||||
untrusted: false,
|
||||
});
|
||||
ask(&config.broker_socket(), call);
|
||||
// Honcho is not reachable here: the conclusion waits.
|
||||
// Honcho is not reachable here: the conclusion waits. The write wakes the worker, so once it
|
||||
// has tried, the report also shows the outage (review of M5c-1, findings 3 and 4).
|
||||
let shown = sync(&config);
|
||||
let mut lines = shown.lines();
|
||||
assert_eq!(
|
||||
sync(&config),
|
||||
"honcho sync: 1 decided: 0 sent, 1 waiting, 0 failed, 0 skipped"
|
||||
lines.next(),
|
||||
Some("honcho sync: 1 decided: 0 sent, 1 waiting, 0 failed, 0 skipped")
|
||||
);
|
||||
if let Some(outage) = lines.next() {
|
||||
assert!(outage.starts_with("honcho unreachable since "), "{shown}");
|
||||
assert!(outage.ends_with(": sends wait and are retried"), "{shown}");
|
||||
}
|
||||
assert_eq!(lines.next(), None, "{shown}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user