review: bxctl memory sync shows what the Honcho side waits on
A shared status, kept by the worker and the walk and read by the report under its own leaf lock, holds the workspaces whose key was refused, when an outage of Honcho began, and when brokerd stopped answering. The report adds a line for each while it lasts, and is unchanged when all is well. The runbook's Honcho entries and broker-state-unavailable now say what sync shows. Tuning and Wake move to honcho/wake.rs (re-exported from worker) to keep worker.rs under 500 lines. honcho_sync.rs's serve test now allows the outage line: the write wakes the worker, which finds Honcho unreachable. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -7,7 +7,7 @@
|
||||
|
||||
use std::collections::HashSet;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::{Arc, Condvar, Mutex};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use bxcore::sanitize::neutralise;
|
||||
@@ -20,10 +20,12 @@ use crate::honcho::decide::{final_reply, turn_records, user_text};
|
||||
use crate::honcho::edits;
|
||||
use crate::honcho::ledger::{self, Action, Item, Ledger, Record, Workspace};
|
||||
use crate::honcho::naming::{HonchoSession, honcho_session};
|
||||
use crate::honcho::status::{self, Shared};
|
||||
use crate::honcho::turns::{TurnSync, log_path, read_log};
|
||||
use crate::serve::Log;
|
||||
|
||||
pub use crate::honcho::start::{Opened, open, spawn};
|
||||
pub use crate::honcho::wake::{Tuning, Wake};
|
||||
|
||||
/// The most characters Honcho takes in one message (recorded: 25,000 is taken, 25,001 refused).
|
||||
pub const MAX_PART: usize = 25_000;
|
||||
@@ -35,50 +37,6 @@ pub const SEND_REFUSED: &str = "see docs/runbook.md#honcho-send-refused";
|
||||
pub const KEY_REFUSED: &str = "see docs/runbook.md#honcho-key-refused";
|
||||
pub const LOG_DAMAGED: &str = "see docs/runbook.md#session-log-damaged";
|
||||
|
||||
/// The worker's waits. Tests shorten them.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct Tuning {
|
||||
/// The first wait in an outage; each next one doubles, up to `backoff_max`.
|
||||
pub backoff: Duration,
|
||||
pub backoff_max: Duration,
|
||||
/// How long the worker sleeps with nothing to send, unless a turn ends first.
|
||||
pub idle: Duration,
|
||||
}
|
||||
|
||||
impl Tuning {
|
||||
pub const DEFAULT: Tuning = Tuning {
|
||||
backoff: Duration::from_secs(5),
|
||||
backoff_max: Duration::from_secs(600),
|
||||
idle: Duration::from_secs(30),
|
||||
};
|
||||
}
|
||||
|
||||
/// Wakes the worker from its idle sleep when a turn ends.
|
||||
#[derive(Debug, Default)]
|
||||
pub struct Wake {
|
||||
woken: Mutex<bool>,
|
||||
ready: Condvar,
|
||||
}
|
||||
|
||||
impl Wake {
|
||||
/// Wake the worker if it sleeps, or make its next sleep return at once.
|
||||
pub fn notify(&self) {
|
||||
let mut woken = self.woken.lock().unwrap_or_else(|p| p.into_inner());
|
||||
*woken = true;
|
||||
self.ready.notify_one();
|
||||
}
|
||||
|
||||
/// Sleep until `notify`, or at most `timeout`.
|
||||
pub fn wait(&self, timeout: Duration) {
|
||||
let woken = self.woken.lock().unwrap_or_else(|p| p.into_inner());
|
||||
let (mut woken, _) = self
|
||||
.ready
|
||||
.wait_timeout_while(woken, timeout, |woken| !*woken)
|
||||
.unwrap_or_else(|p| p.into_inner());
|
||||
*woken = false;
|
||||
}
|
||||
}
|
||||
|
||||
/// What one step of the worker did, and so what the loop does next.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Step {
|
||||
@@ -149,6 +107,8 @@ pub struct Worker {
|
||||
outage: Option<Duration>,
|
||||
/// Workspaces whose key was refused: nothing more is sent to them until restart.
|
||||
refused: HashSet<Workspace>,
|
||||
/// What `bxctl memory sync` shows; its lock is taken alone.
|
||||
status: Shared,
|
||||
stopped: bool,
|
||||
}
|
||||
|
||||
@@ -194,14 +154,6 @@ pub fn batch(
|
||||
messages
|
||||
}
|
||||
|
||||
/// An item as the log lines name it: `turn 3 of session mm-a`, `part 0 of edit 2026-09-28-0001`.
|
||||
fn label(item: &Item) -> String {
|
||||
match item {
|
||||
Item::Turn { session, turn } => format!("turn {turn} of session {}", session.as_str()),
|
||||
Item::Edit { id, part } => format!("part {part} of edit {id}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// The ids of the messages Honcho made, in order.
|
||||
fn ids(messages: &[roncho::Message]) -> Vec<String> {
|
||||
messages.iter().map(|m| m.id.clone()).collect()
|
||||
@@ -218,9 +170,13 @@ impl Worker {
|
||||
tuning: Tuning,
|
||||
agent: &str,
|
||||
) -> Worker {
|
||||
let (ledger, sessions) = {
|
||||
let (ledger, sessions, status) = {
|
||||
let sync = sync.lock().unwrap_or_else(|p| p.into_inner());
|
||||
(sync.shared_ledger(), sync.sessions().to_path_buf())
|
||||
(
|
||||
sync.shared_ledger(),
|
||||
sync.sessions().to_path_buf(),
|
||||
sync.shared_status(),
|
||||
)
|
||||
};
|
||||
Worker {
|
||||
sync,
|
||||
@@ -234,6 +190,7 @@ impl Worker {
|
||||
agent: agent.to_string(),
|
||||
outage: None,
|
||||
refused: HashSet::new(),
|
||||
status,
|
||||
stopped: false,
|
||||
}
|
||||
}
|
||||
@@ -275,17 +232,24 @@ impl Worker {
|
||||
Err(Unsent::Honcho(HonchoError::KeyRefused { status })) => {
|
||||
self.recovered();
|
||||
self.refused.insert(next.workspace);
|
||||
let name = self
|
||||
.clients
|
||||
.client(next.workspace)
|
||||
.workspace_id()
|
||||
.to_string();
|
||||
(self.log)(&format!(
|
||||
"memoryd: Honcho refused the key of workspace {} (HTTP {status}); nothing more is sent to it until memoryd restarts; {KEY_REFUSED}",
|
||||
self.clients.client(next.workspace).workspace_id()
|
||||
"memoryd: Honcho refused the key of workspace {name} (HTTP {status}); nothing more is sent to it until memoryd restarts; {KEY_REFUSED}"
|
||||
));
|
||||
status::update(&self.status, |s| {
|
||||
s.refused.insert(next.workspace, (name, status));
|
||||
});
|
||||
Step::Done
|
||||
}
|
||||
Err(Unsent::Honcho(e)) => {
|
||||
self.recovered();
|
||||
(self.log)(&format!(
|
||||
"memoryd: {} was not sent: {e}; it is marked failed; {SEND_REFUSED}",
|
||||
label(&next.item)
|
||||
ledger::label(&next.item)
|
||||
));
|
||||
self.record(Record::Failed {
|
||||
time: Timestamp::now(),
|
||||
@@ -302,7 +266,7 @@ impl Worker {
|
||||
};
|
||||
(self.log)(&format!(
|
||||
"memoryd: {} cannot be sent ({kind}); it is marked failed; {pointer}",
|
||||
label(&next.item)
|
||||
ledger::label(&next.item)
|
||||
));
|
||||
self.record(Record::Failed {
|
||||
time: Timestamp::now(),
|
||||
@@ -446,6 +410,7 @@ impl Worker {
|
||||
(self.log)(
|
||||
format!("memoryd: {error}; sends wait and are retried; {UNREACHABLE}").as_str(),
|
||||
);
|
||||
status::update(&self.status, |s| s.outage_since = Some(Timestamp::now()));
|
||||
self.tuning.backoff
|
||||
}
|
||||
};
|
||||
@@ -457,6 +422,7 @@ impl Worker {
|
||||
fn recovered(&mut self) {
|
||||
if self.outage.take().is_some() {
|
||||
(self.log)("memoryd: Honcho answers again; waiting sends go on");
|
||||
status::update(&self.status, |s| s.outage_since = None);
|
||||
self.retry();
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user