memoryd: a reported turn decided after loopd has its answer, from brokerd and the log
Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>
This commit is contained in:
ornith-1.5-35b-a3b
@@ -3,3 +3,5 @@
|
||||
pub mod config;
|
||||
pub mod decide;
|
||||
pub mod ledger;
|
||||
pub mod state;
|
||||
pub mod turns;
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
//! Asking `brokerd` for a session's state on its `state.sock` (M5c spec, section 7): one question
|
||||
//! per connection, one final answer, as on every socket between the roles.
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::Path;
|
||||
use std::time::Duration;
|
||||
|
||||
use bxcore::{
|
||||
Envelope, Message, PROTOCOL_VERSION, SessionId, SessionState, SessionStateReply, read_frame,
|
||||
write_frame,
|
||||
};
|
||||
|
||||
/// How long `brokerd` may leave the connection without a byte, sending or answering.
|
||||
pub const WAIT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The runbook entry for a `brokerd` that does not answer.
|
||||
pub const RUNBOOK: &str = "see docs/runbook.md#broker-state-unavailable";
|
||||
|
||||
/// `brokerd`'s answer for `session`, asked on `socket`. `wait` is the read and the write timeout;
|
||||
/// the connect has none, and fails at once when nothing listens. Any error, an error frame
|
||||
/// included, is `Err` with one line that names the socket or `brokerd`.
|
||||
pub fn ask(
|
||||
socket: &Path,
|
||||
session: &SessionId,
|
||||
wait: Duration,
|
||||
) -> Result<SessionStateReply, String> {
|
||||
let line = |e: &dyn std::fmt::Display| format!("{}: {e}", socket.display());
|
||||
let mut stream = UnixStream::connect(socket).map_err(|e| line(&e))?;
|
||||
stream.set_read_timeout(Some(wait)).map_err(|e| line(&e))?;
|
||||
stream.set_write_timeout(Some(wait)).map_err(|e| line(&e))?;
|
||||
let question = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
id: 1,
|
||||
r#final: true,
|
||||
msg: Message::SessionState(SessionState {
|
||||
session: session.clone(),
|
||||
}),
|
||||
};
|
||||
write_frame(&mut stream, &question).map_err(|e| line(&e))?;
|
||||
match read_frame(&mut stream).map_err(|e| line(&e))?.msg {
|
||||
Message::SessionStateReply(reply) => Ok(reply),
|
||||
Message::Error(e) => Err(format!("brokerd: {}", e.detail)),
|
||||
_ => Err("brokerd answered with the wrong message".to_string()),
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,247 @@
|
||||
//! Deciding finished turns when `loopd` reports one (M5c spec, section 5). At each report every
|
||||
//! session with undecided turns is walked: its log is read, `brokerd` is asked once for its state,
|
||||
//! and each undecided turn is decided and appended to the ledger at once. A turn that cannot be
|
||||
//! decided now (`brokerd` silent, the log unreadable) waits for the next report of any session.
|
||||
|
||||
use std::collections::{BTreeMap, BTreeSet};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
use bxcore::{LogRecord, SessionId, SessionStateReply, Timestamp};
|
||||
|
||||
use crate::honcho::decide::{decide, turn_records, turns};
|
||||
use crate::honcho::ledger::{self, Item, Ledger, Record};
|
||||
use crate::honcho::state;
|
||||
|
||||
/// The ledger's file name, in `memory/`.
|
||||
pub const LEDGER: &str = "honcho.jsonl";
|
||||
|
||||
/// The runbook entry for a session log with a damaged line.
|
||||
pub const LOG_RUNBOOK: &str = "see docs/runbook.md#session-log-damaged";
|
||||
|
||||
/// `brokerd`'s answer for a session, or why there is none.
|
||||
pub type Ask<'a> = &'a dyn Fn(&SessionId) -> Result<SessionStateReply, String>;
|
||||
|
||||
/// How one session's walk ended.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum Walk {
|
||||
/// Every turn up to the reported one is decided: the session leaves the map.
|
||||
Done,
|
||||
/// Some turn waits: the session stays in the map.
|
||||
Waiting,
|
||||
/// `brokerd` did not answer, or the ledger could not be written: the walk stops.
|
||||
Stop,
|
||||
}
|
||||
|
||||
/// The turns reported and not yet decided, and the ledger their decisions go to.
|
||||
pub struct TurnSync {
|
||||
ledger: Ledger,
|
||||
agent: String,
|
||||
sessions: PathBuf,
|
||||
/// Each session with undecided turns, by its id, and the highest turn `loopd` reported for
|
||||
/// it. `SessionId` has no order, so the map is keyed by its text.
|
||||
reported: BTreeMap<String, (SessionId, u64)>,
|
||||
/// Whether the last question to `brokerd` went unanswered: an outage is logged once.
|
||||
broker_down: bool,
|
||||
/// Sessions whose log last had a damaged line: logged once each.
|
||||
unreadable: BTreeSet<String>,
|
||||
}
|
||||
|
||||
/// The path of `session`'s log under `sessions`: epoch 0, the only one `loopd` writes.
|
||||
pub fn log_path(sessions: &Path, session: &SessionId) -> PathBuf {
|
||||
sessions.join(session.as_str()).join("0.jsonl")
|
||||
}
|
||||
|
||||
/// A session log's complete lines as records. A missing file has none. A last line without its
|
||||
/// newline is one `loopd` is still writing, and is left out. A complete line that does not parse
|
||||
/// is an error naming the file and the line, with the runbook pointer.
|
||||
pub fn read_log(path: &Path) -> Result<Vec<LogRecord>, String> {
|
||||
let text = match std::fs::read(path) {
|
||||
Ok(bytes) => bytes,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
|
||||
Err(e) => return Err(format!("{}: {e}; {LOG_RUNBOOK}", path.display())),
|
||||
};
|
||||
let whole = match text.iter().rposition(|b| *b == b'\n') {
|
||||
Some(last) => last + 1,
|
||||
None => 0,
|
||||
};
|
||||
let mut records = Vec::new();
|
||||
for (index, line) in text
|
||||
.get(..whole)
|
||||
.unwrap_or_default()
|
||||
.split_inclusive(|b| *b == b'\n')
|
||||
.enumerate()
|
||||
{
|
||||
match serde_json::from_slice::<LogRecord>(line) {
|
||||
Ok(record) => records.push(record),
|
||||
Err(e) => {
|
||||
return Err(format!(
|
||||
"{}:{}: {e}; {LOG_RUNBOOK}",
|
||||
path.display(),
|
||||
index + 1
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(records)
|
||||
}
|
||||
|
||||
/// Whether a turn has ended with an answer: its last `Assistant` record asks for no tool.
|
||||
pub fn finished(turn: &[LogRecord]) -> bool {
|
||||
let last = turn
|
||||
.iter()
|
||||
.rev()
|
||||
.find(|r| matches!(r, LogRecord::Assistant { .. }));
|
||||
matches!(last, Some(LogRecord::Assistant { tool_calls, .. }) if tool_calls.is_empty())
|
||||
}
|
||||
|
||||
impl TurnSync {
|
||||
/// `ledger` read through at start; `agent` is `[honcho] agent`; `sessions` the directory of
|
||||
/// `loopd`'s session logs.
|
||||
pub fn new(ledger: Ledger, agent: &str, sessions: &Path) -> TurnSync {
|
||||
TurnSync {
|
||||
ledger,
|
||||
agent: agent.to_string(),
|
||||
sessions: sessions.to_path_buf(),
|
||||
reported: BTreeMap::new(),
|
||||
broker_down: false,
|
||||
unreadable: BTreeSet::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ledger(&self) -> &Ledger {
|
||||
&self.ledger
|
||||
}
|
||||
|
||||
/// The sessions that still have undecided turns, with the highest turn reported for each.
|
||||
pub fn waiting(&self) -> Vec<(SessionId, u64)> {
|
||||
self.reported.values().cloned().collect()
|
||||
}
|
||||
|
||||
/// `loopd` reported that turn `turn` of `session` ended. Remember it, then walk every session
|
||||
/// with undecided turns, in the order of their ids, until one stops the walk.
|
||||
pub fn updated(
|
||||
&mut self,
|
||||
session: &SessionId,
|
||||
turn: u64,
|
||||
ask: Ask<'_>,
|
||||
now: Timestamp,
|
||||
log: &dyn Fn(&str),
|
||||
) {
|
||||
let entry = self
|
||||
.reported
|
||||
.entry(session.as_str().to_string())
|
||||
.or_insert((session.clone(), 0));
|
||||
entry.1 = entry.1.max(turn);
|
||||
for (session, turn) in self.waiting() {
|
||||
match self.walk(&session, turn, ask, now, log) {
|
||||
Walk::Done => {
|
||||
self.reported.remove(session.as_str());
|
||||
}
|
||||
Walk::Waiting => {}
|
||||
Walk::Stop => return,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Decide `session`'s undecided turns from 1 to `reported`, as far as its log goes. The log's
|
||||
/// last turn waits until it has an answer: it may still be running.
|
||||
fn walk(
|
||||
&mut self,
|
||||
session: &SessionId,
|
||||
reported: u64,
|
||||
ask: Ask<'_>,
|
||||
now: Timestamp,
|
||||
log: &dyn Fn(&str),
|
||||
) -> Walk {
|
||||
let Some(records) = self.records(session, log) else {
|
||||
return Walk::Waiting;
|
||||
};
|
||||
let (limit, held) = decidable(&records, reported);
|
||||
let item = |turn: u64| Item::Turn {
|
||||
session: session.clone(),
|
||||
turn,
|
||||
};
|
||||
let undecided: Vec<u64> = (1..=limit)
|
||||
.filter(|turn| !self.ledger.is_decided(&item(*turn)))
|
||||
.collect();
|
||||
if undecided.is_empty() {
|
||||
return if held { Walk::Waiting } else { Walk::Done };
|
||||
}
|
||||
let Some(state) = self.state(session, ask, log) else {
|
||||
return Walk::Stop;
|
||||
};
|
||||
for turn in undecided {
|
||||
let Ok(action) = decide(&state, &records, turn, &self.agent) else {
|
||||
continue;
|
||||
};
|
||||
let record = Record::Decided {
|
||||
time: now,
|
||||
item: item(turn),
|
||||
action,
|
||||
};
|
||||
if let Err(e) = self.ledger.append(record) {
|
||||
log(&format!("memoryd: {e}; {}", ledger::RUNBOOK));
|
||||
return Walk::Stop;
|
||||
}
|
||||
}
|
||||
if held { Walk::Waiting } else { Walk::Done }
|
||||
}
|
||||
|
||||
/// `session`'s log, read. A log that does not read is logged once, until it reads again.
|
||||
fn records(&mut self, session: &SessionId, log: &dyn Fn(&str)) -> Option<Vec<LogRecord>> {
|
||||
match read_log(&log_path(&self.sessions, session)) {
|
||||
Ok(records) => {
|
||||
self.unreadable.remove(session.as_str());
|
||||
Some(records)
|
||||
}
|
||||
Err(e) => {
|
||||
if self.unreadable.insert(session.as_str().to_string()) {
|
||||
log(&format!("memoryd: {e}"));
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// `brokerd`'s answer for `session`. The first failure after an answer is logged, and so is
|
||||
/// the first answer after a failure.
|
||||
fn state(
|
||||
&mut self,
|
||||
session: &SessionId,
|
||||
ask: Ask<'_>,
|
||||
log: &dyn Fn(&str),
|
||||
) -> Option<SessionStateReply> {
|
||||
match ask(session) {
|
||||
Ok(state) => {
|
||||
if self.broker_down {
|
||||
self.broker_down = false;
|
||||
log("memoryd: brokerd answers on its state socket again; turns are decided");
|
||||
}
|
||||
Some(state)
|
||||
}
|
||||
Err(e) => {
|
||||
if !self.broker_down {
|
||||
self.broker_down = true;
|
||||
log(&format!(
|
||||
"memoryd: {e}; turns wait undecided; {}",
|
||||
state::RUNBOOK
|
||||
));
|
||||
}
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// How far a log's turns may be decided for a report of turn `reported`: up to the smaller of
|
||||
/// `reported` and the log's turns, but for the log's last turn when it has no answer yet. The
|
||||
/// limit, and whether that last turn was held back.
|
||||
pub fn decidable(records: &[LogRecord], reported: u64) -> (u64, bool) {
|
||||
let count = turns(records);
|
||||
let limit = reported.min(count);
|
||||
if limit > 0 && limit == count && !turn_records(records, limit).is_some_and(finished) {
|
||||
(limit - 1, true)
|
||||
} else {
|
||||
(limit, false)
|
||||
}
|
||||
}
|
||||
@@ -44,6 +44,11 @@ fn main() -> ExitCode {
|
||||
let log: serve::Log = Arc::new(|line: &str| eprintln!("{line}"));
|
||||
let running = match serve::start(config, log) {
|
||||
Ok(running) => running,
|
||||
// The ledger's line names its own pointer, `honcho-ledger-damaged`.
|
||||
Err(e @ serve::StartError::Ledger(_)) => {
|
||||
eprintln!("memoryd: {e}");
|
||||
return ExitCode::from(1);
|
||||
}
|
||||
Err(e) => {
|
||||
eprintln!("memoryd: {e}\n{START_FAILED}");
|
||||
return ExitCode::from(1);
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Serving: three sockets, one request per connection, each answered by one frame (M5a spec,
|
||||
//! section 7). One lock covers the files and the index, so writes never interleave.
|
||||
//! section 7). One lock covers the files and the index, so writes never interleave. With
|
||||
//! `[honcho]`, a reported turn is decided after `loopd` has its answer, under a lock of its own
|
||||
//! (M5c spec, section 5).
|
||||
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::sync::{Arc, Mutex, MutexGuard};
|
||||
@@ -7,13 +9,16 @@ use std::time::Duration;
|
||||
|
||||
use bxcore::{
|
||||
DataClass, Empty, Envelope, ErrorCode, MemoryAdminReply, MemoryArgs, MemoryBlockReply,
|
||||
MemoryCall, MemoryCommand, Message, PROTOCOL_VERSION, RecallArgs, Reindexed, Timestamp,
|
||||
ToolResponse, WireError, read_frame, write_frame,
|
||||
MemoryCall, MemoryCommand, Message, PROTOCOL_VERSION, RecallArgs, Reindexed, SessionId,
|
||||
SessionUpdated, Timestamp, ToolResponse, WireError, read_frame, write_frame,
|
||||
};
|
||||
|
||||
use crate::config::Config;
|
||||
use crate::edit::{self, Context};
|
||||
use crate::files::Store;
|
||||
use crate::honcho::ledger::{self, Ledger, LedgerError};
|
||||
use crate::honcho::state;
|
||||
use crate::honcho::turns::{self, TurnSync};
|
||||
use crate::index::{self, Index};
|
||||
use crate::recall::Recalled;
|
||||
use crate::{journal, recall, review};
|
||||
@@ -32,9 +37,39 @@ pub struct Memoryd {
|
||||
config: Config,
|
||||
store: Store,
|
||||
state: Mutex<State>,
|
||||
/// With `[honcho]`: the turns reported and not yet decided, and the ledger. Never taken
|
||||
/// while `state` is held, and never held by a request that waits for its answer.
|
||||
honcho: Option<Mutex<TurnSync>>,
|
||||
log: Log,
|
||||
}
|
||||
|
||||
/// Why `start` failed. `main` prints a start failure with the pointer `memoryd-start-failed`, and
|
||||
/// a ledger that does not read through with its own.
|
||||
#[derive(Debug)]
|
||||
pub enum StartError {
|
||||
Failed(String),
|
||||
Ledger(LedgerError),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for StartError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
StartError::Failed(why) => write!(f, "{why}"),
|
||||
// `Damaged` names its pointer itself.
|
||||
StartError::Ledger(e @ LedgerError::Damaged { .. }) => write!(f, "{e}"),
|
||||
StartError::Ledger(e) => write!(f, "{e}; {}", ledger::RUNBOOK),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for StartError {}
|
||||
|
||||
impl From<String> for StartError {
|
||||
fn from(why: String) -> StartError {
|
||||
StartError::Failed(why)
|
||||
}
|
||||
}
|
||||
|
||||
/// A started `memoryd`: its three listeners, bound and waiting.
|
||||
pub struct Running {
|
||||
memoryd: Arc<Memoryd>,
|
||||
@@ -48,9 +83,30 @@ enum Which {
|
||||
Admin,
|
||||
}
|
||||
|
||||
/// Create what is missing under `memory/`, open the index and catch it up, then bind the three
|
||||
/// sockets. The error names what failed; the caller points at `memoryd-start-failed`.
|
||||
pub fn start(config: Config, log: Log) -> Result<Running, String> {
|
||||
/// With `[honcho]`, read the ledger through first. Then create what is missing under `memory/`,
|
||||
/// open the index and catch it up, and bind the three sockets. The error names what failed.
|
||||
pub fn start(config: Config, log: Log) -> Result<Running, StartError> {
|
||||
let honcho = match config.honcho_config()? {
|
||||
Some(honcho) => {
|
||||
let path = config.memory_dir().join(turns::LEDGER);
|
||||
let ledger = Ledger::open(&path).map_err(StartError::Ledger)?;
|
||||
if ledger.repaired() {
|
||||
(log)(
|
||||
format!(
|
||||
"memoryd: {}: a last line cut off by a crash was dropped",
|
||||
path.display()
|
||||
)
|
||||
.as_str(),
|
||||
);
|
||||
}
|
||||
Some(Mutex::new(TurnSync::new(
|
||||
ledger,
|
||||
&honcho.agent,
|
||||
&config.sessions_dir(),
|
||||
)))
|
||||
}
|
||||
None => None,
|
||||
};
|
||||
let store = Store::new(&config.memory_dir());
|
||||
store.init(&config.memory.soul_starter)?;
|
||||
let index = Index::open(&config.index_path(), config.index.max_class)?;
|
||||
@@ -71,6 +127,7 @@ pub fn start(config: Config, log: Log) -> Result<Running, String> {
|
||||
config,
|
||||
store,
|
||||
state: Mutex::new(State { index }),
|
||||
honcho,
|
||||
log,
|
||||
}),
|
||||
listeners,
|
||||
@@ -141,6 +198,10 @@ impl Memoryd {
|
||||
Ok(request) => request,
|
||||
Err(_) => return,
|
||||
};
|
||||
let update = match (which, &request.msg) {
|
||||
(Which::Loop, Message::SessionUpdated(update)) => Some(update.clone()),
|
||||
_ => None,
|
||||
};
|
||||
let reply = self.handle(request.msg, which);
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
@@ -149,6 +210,29 @@ impl Memoryd {
|
||||
msg: reply,
|
||||
};
|
||||
let _ = write_frame(&mut stream, &env);
|
||||
// `loopd` has its answer before `brokerd` is asked: it sends the turn's reply only after
|
||||
// this frame, and `brokerd` may take seconds.
|
||||
drop(stream);
|
||||
if let Some(update) = update {
|
||||
self.decide_turns(&update);
|
||||
}
|
||||
}
|
||||
|
||||
/// With `[honcho]`: decide the reported turn, and every other that waits (`TurnSync`).
|
||||
fn decide_turns(&self, update: &SessionUpdated) {
|
||||
let Some(honcho) = &self.honcho else {
|
||||
return;
|
||||
};
|
||||
let socket = self.config.state_socket();
|
||||
let ask = |session: &SessionId| state::ask(&socket, session, state::WAIT);
|
||||
let mut sync = honcho.lock().unwrap_or_else(|p| p.into_inner());
|
||||
sync.updated(
|
||||
&update.session,
|
||||
update.turn,
|
||||
&ask,
|
||||
Timestamp::now(),
|
||||
&*self.log,
|
||||
);
|
||||
}
|
||||
|
||||
/// The answer to one request on one socket. A message sent to the wrong socket is refused.
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
//! `memoryd` deciding turns as it serves (M5c spec, section 5): `loopd` has its answer to
|
||||
//! `session_updated` before `brokerd` is asked, and a silent `brokerd` holds up neither `loopd`
|
||||
//! nor a memory call; without `[honcho]` nothing is asked or written; a damaged ledger stops the
|
||||
//! start with its pointer, and a cut-off last line is dropped and logged. Every wait has a
|
||||
//! deadline. Do not edit.
|
||||
|
||||
#[path = "support/brokerd.rs"]
|
||||
mod brokerd;
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use brokerd::{Answer, FakeBrokerd};
|
||||
use bxcore::{
|
||||
CallId, DataClass, Envelope, MemoryCall, Message, SessionId, SessionUpdated, ToolRequest,
|
||||
read_frame, write_frame,
|
||||
};
|
||||
use memoryd::config::Config;
|
||||
use memoryd::honcho::ledger::LedgerError;
|
||||
use memoryd::serve::{self, StartError};
|
||||
use tmp::TempDir;
|
||||
use turns::{clean, write_turns};
|
||||
|
||||
const SECTION: &str = "[honcho]
|
||||
url = \"http://127.0.0.1:9\"
|
||||
agent = \"boxmaker0\"
|
||||
owner = \"kyle\"
|
||||
trusted = \"boxmaker0-trusted\"
|
||||
untrusted = \"boxmaker0-untrusted\"
|
||||
fleet = \"fleet\"
|
||||
";
|
||||
|
||||
const DECIDED: &str = r#"{"type":"decided","time":"2026-09-28T10:00:00.000Z","item":{"type":"turn","session":"mm-a","turn":1},"action":{"type":"turn","workspace":"trusted","speaker":"kyle"}}"#;
|
||||
|
||||
struct Home {
|
||||
config: Config,
|
||||
text: String,
|
||||
dir: TempDir,
|
||||
}
|
||||
|
||||
impl Home {
|
||||
fn new(tag: &str, honcho: bool) -> Home {
|
||||
let dir = TempDir::new(tag);
|
||||
let soul = dir.write("etc/soul.md", "You are Boxmaker.\n");
|
||||
let text = format!(
|
||||
"[paths]\nhome = \"{}\"\n[memory]\nsoul_starter = \"{}\"\n{}",
|
||||
dir.path().display(),
|
||||
soul.display(),
|
||||
if honcho { SECTION } else { "" }
|
||||
);
|
||||
Home {
|
||||
config: Config::parse(&text).unwrap(),
|
||||
text,
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn ledger(&self) -> PathBuf {
|
||||
self.dir.path().join("memory/honcho.jsonl")
|
||||
}
|
||||
|
||||
fn ledger_text(&self) -> String {
|
||||
std::fs::read_to_string(self.ledger()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// `serve::start`, run on a thread; its log lines are collected.
|
||||
fn serve(&self) -> Arc<Mutex<Vec<String>>> {
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&lines);
|
||||
let log: serve::Log = Arc::new(move |l: &str| sink.lock().unwrap().push(l.to_string()));
|
||||
let running = serve::start(self.config.clone(), log).unwrap();
|
||||
std::thread::spawn(move || running.run());
|
||||
lines
|
||||
}
|
||||
}
|
||||
|
||||
/// One request and its answer, with a 10 s limit; how long the answer took.
|
||||
fn ask(socket: &Path, msg: Message) -> (Message, Duration) {
|
||||
let started = Instant::now();
|
||||
let mut stream = UnixStream::connect(socket).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.unwrap();
|
||||
let env = Envelope {
|
||||
v: 1,
|
||||
id: 7,
|
||||
r#final: true,
|
||||
msg,
|
||||
};
|
||||
write_frame(&mut stream, &env).unwrap();
|
||||
let reply = read_frame(&mut stream).unwrap();
|
||||
(reply.msg, started.elapsed())
|
||||
}
|
||||
|
||||
fn updated(home: &Home, session: &str, turn: u64) -> (Message, Duration) {
|
||||
let msg = Message::SessionUpdated(SessionUpdated {
|
||||
session: SessionId::new(session).unwrap(),
|
||||
turn,
|
||||
});
|
||||
ask(&home.config.loop_socket(), msg)
|
||||
}
|
||||
|
||||
/// Waits up to 10 s for `done`; true if it came.
|
||||
fn within_10s(done: impl Fn() -> bool) -> bool {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline {
|
||||
if done() {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reported_turn_is_decided_after_loopd_has_its_answer() {
|
||||
let home = Home::new("serve-decide", true);
|
||||
write_turns(home.dir.path(), "mm-a", &[turns::finished(Some("kyle"))]);
|
||||
write_turns(home.dir.path(), "mm-z", &[turns::finished(Some("kyle"))]);
|
||||
let fake = FakeBrokerd::start(&home.config.state_socket(), Answer::State(clean()));
|
||||
let _lines = home.serve();
|
||||
// A report on another socket is refused, and decides nothing.
|
||||
let wrong = Message::SessionUpdated(SessionUpdated {
|
||||
session: SessionId::new("mm-z").unwrap(),
|
||||
turn: 1,
|
||||
});
|
||||
let (answer, _) = ask(&home.config.admin_socket(), wrong);
|
||||
assert!(matches!(answer, Message::Error(_)), "{answer:?}");
|
||||
let (answer, _) = updated(&home, "mm-a", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
assert!(within_10s(|| home.ledger_text().contains("\"turn\":1}")));
|
||||
// The line as `DECIDED`, but for its time, which is the decision's.
|
||||
let text = home.ledger_text();
|
||||
assert_eq!(text.lines().count(), 1, "{text}");
|
||||
let mut got: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
let mut want: serde_json::Value = serde_json::from_str(DECIDED).unwrap();
|
||||
got["time"] = serde_json::Value::Null;
|
||||
want["time"] = serde_json::Value::Null;
|
||||
assert_eq!(got, want);
|
||||
assert_eq!(fake.seen().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_silent_brokerd_holds_up_neither_loopd_nor_memory() {
|
||||
let home = Home::new("serve-silent", true);
|
||||
write_turns(home.dir.path(), "mm-a", &[turns::finished(Some("kyle"))]);
|
||||
write_turns(home.dir.path(), "mm-b", &[turns::finished(Some("kyle"))]);
|
||||
let fake = FakeBrokerd::start(&home.config.state_socket(), Answer::Hang);
|
||||
let lines = home.serve();
|
||||
let (answer, took) = updated(&home, "mm-a", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
assert!(took < Duration::from_secs(1), "loopd waited {took:?}");
|
||||
assert!(fake.wait_for(1), "brokerd was never asked");
|
||||
// While the question hangs: another report, and a memory call.
|
||||
let (answer, took) = updated(&home, "mm-b", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
assert!(took < Duration::from_secs(1), "loopd waited {took:?}");
|
||||
let call = Message::MemoryCall(MemoryCall {
|
||||
request: ToolRequest {
|
||||
session: SessionId::new("mm-b").unwrap(),
|
||||
call: CallId(1),
|
||||
tool: "recall".to_string(),
|
||||
arguments: r#"{"query":"hello"}"#.to_string(),
|
||||
},
|
||||
taint: DataClass::Private,
|
||||
untrusted: false,
|
||||
});
|
||||
let (answer, took) = ask(&home.config.broker_socket(), call);
|
||||
assert!(matches!(answer, Message::ToolResponse(_)), "{answer:?}");
|
||||
assert!(
|
||||
took < Duration::from_secs(1),
|
||||
"the memory call waited {took:?}"
|
||||
);
|
||||
// After 5 s without a byte, the outage is logged once, and nothing is decided.
|
||||
let pointer = "turns wait undecided; see docs/runbook.md#broker-state-unavailable";
|
||||
assert!(within_10s(|| lines
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|l| l.contains(pointer))));
|
||||
let logged = lines.lock().unwrap().clone();
|
||||
let outage: Vec<&String> = logged.iter().filter(|l| l.contains(pointer)).collect();
|
||||
assert_eq!(outage.len(), 1, "{logged:?}");
|
||||
let start = format!("memoryd: {}: ", home.config.state_socket().display());
|
||||
assert!(outage[0].starts_with(&start), "{}", outage[0]);
|
||||
assert_eq!(home.ledger_text(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_honcho_nothing_is_asked_or_written() {
|
||||
let home = Home::new("serve-off", false);
|
||||
write_turns(home.dir.path(), "mm-a", &[turns::finished(Some("kyle"))]);
|
||||
let fake = FakeBrokerd::start(&home.config.state_socket(), Answer::State(clean()));
|
||||
let lines = home.serve();
|
||||
let (answer, _) = updated(&home, "mm-a", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
// A negative: nothing may happen, so there is nothing to wait for but a while.
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
assert!(fake.seen().is_empty());
|
||||
assert!(!home.ledger().exists());
|
||||
assert!(
|
||||
lines.lock().unwrap().is_empty(),
|
||||
"{:?}",
|
||||
lines.lock().unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_ledger_stops_the_start() {
|
||||
let home = Home::new("serve-damaged", true);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), "not json\n").unwrap();
|
||||
let log: serve::Log = Arc::new(|_: &str| {});
|
||||
let err = match serve::start(home.config.clone(), log) {
|
||||
Err(err) => err,
|
||||
Ok(_) => panic!("memoryd started"),
|
||||
};
|
||||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
StartError::Ledger(LedgerError::Damaged { line: 1, .. })
|
||||
),
|
||||
"{err:?}"
|
||||
);
|
||||
let shown = err.to_string();
|
||||
assert!(shown.starts_with(&format!("{}:1: ", home.ledger().display())));
|
||||
assert!(
|
||||
shown.ends_with("; see docs/runbook.md#honcho-ledger-damaged"),
|
||||
"{shown}"
|
||||
);
|
||||
assert_eq!(shown.matches("see docs/runbook.md").count(), 1, "{shown}");
|
||||
assert!(!home.config.loop_socket().exists(), "a socket was bound");
|
||||
// A ledger that cannot be read is named with the same pointer.
|
||||
std::fs::remove_file(home.ledger()).unwrap();
|
||||
std::fs::create_dir(home.ledger()).unwrap();
|
||||
let log: serve::Log = Arc::new(|_: &str| {});
|
||||
let err = match serve::start(home.config.clone(), log) {
|
||||
Err(err) => err,
|
||||
Ok(_) => panic!("memoryd started"),
|
||||
};
|
||||
assert!(
|
||||
matches!(err, StartError::Ledger(LedgerError::Io(..))),
|
||||
"{err:?}"
|
||||
);
|
||||
let shown = err.to_string();
|
||||
assert!(shown.starts_with(&format!("{}: ", home.ledger().display())));
|
||||
assert!(
|
||||
shown.ends_with("; see docs/runbook.md#honcho-ledger-damaged"),
|
||||
"{shown}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cut_off_last_line_is_dropped_and_logged() {
|
||||
let home = Home::new("serve-repaired", true);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), format!("{DECIDED}\n{{\"type\":\"se")).unwrap();
|
||||
let lines = home.serve();
|
||||
assert_eq!(
|
||||
*lines.lock().unwrap(),
|
||||
vec![format!(
|
||||
"memoryd: {}: a last line cut off by a crash was dropped",
|
||||
home.ledger().display()
|
||||
)]
|
||||
);
|
||||
assert_eq!(home.ledger_text(), format!("{DECIDED}\n"));
|
||||
}
|
||||
|
||||
/// A `memoryd` process, killed and waited for when dropped: declared after its `Home`, so it
|
||||
/// drops first.
|
||||
struct Daemon(std::process::Child);
|
||||
|
||||
impl Drop for Daemon {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memoryd_exits_with_the_ledgers_line() {
|
||||
let home = Home::new("serve-exit", true);
|
||||
let config = home.dir.write("memoryd.toml", &home.text);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), "not json\n").unwrap();
|
||||
let mut child = Daemon(
|
||||
Command::new(env!("CARGO_BIN_EXE_memoryd"))
|
||||
.args(["serve", "--config"])
|
||||
.arg(&config)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
);
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let mut code = None;
|
||||
while Instant::now() < deadline {
|
||||
if let Ok(Some(status)) = child.0.try_wait() {
|
||||
code = status.code();
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
let _ = child.0.kill();
|
||||
let _ = child.0.wait();
|
||||
let mut stderr = String::new();
|
||||
if let Some(mut pipe) = child.0.stderr.take() {
|
||||
let _ = std::io::Read::read_to_string(&mut pipe, &mut stderr);
|
||||
}
|
||||
assert_eq!(code, Some(1), "{stderr}");
|
||||
let start = format!("memoryd: {}:1: ", home.ledger().display());
|
||||
assert!(stderr.starts_with(&start), "{stderr}");
|
||||
assert!(
|
||||
stderr.ends_with("; see docs/runbook.md#honcho-ledger-damaged\n"),
|
||||
"{stderr}"
|
||||
);
|
||||
assert_eq!(stderr.lines().count(), 1, "{stderr}");
|
||||
assert!(!home.config.loop_socket().exists(), "a socket was bound");
|
||||
}
|
||||
@@ -0,0 +1,87 @@
|
||||
//! Asking `brokerd` for a session's state on its `state.sock` (M5c spec, section 7): the
|
||||
//! question as `brokerd` reads it, its answer, an error frame, silence and no listener, each an
|
||||
//! error with one line. Against a fake `brokerd` in the test's own directory. Do not edit.
|
||||
|
||||
#[path = "support/brokerd.rs"]
|
||||
mod brokerd;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use brokerd::{Answer, FakeBrokerd};
|
||||
use bxcore::{DataClass, Message, PROTOCOL_VERSION, SessionId, SessionState, SessionStateReply};
|
||||
use memoryd::honcho::state::{RUNBOOK, WAIT, ask};
|
||||
use tmp::TempDir;
|
||||
|
||||
fn session() -> SessionId {
|
||||
SessionId::new("mm-chat-1").unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_constants() {
|
||||
assert_eq!(WAIT, Duration::from_secs(5));
|
||||
assert_eq!(RUNBOOK, "see docs/runbook.md#broker-state-unavailable");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_question_and_its_answer() {
|
||||
let dir = TempDir::new("state-answer");
|
||||
let socket = dir.path().join("run/memory-broker/state.sock");
|
||||
let reply = SessionStateReply {
|
||||
class: DataClass::Secret,
|
||||
untrusted: true,
|
||||
};
|
||||
let fake = FakeBrokerd::start(&socket, Answer::State(reply.clone()));
|
||||
assert_eq!(ask(&socket, &session(), WAIT), Ok(reply));
|
||||
let seen = fake.seen();
|
||||
assert_eq!(seen.len(), 1);
|
||||
assert_eq!(
|
||||
(seen[0].v, seen[0].id, seen[0].r#final),
|
||||
(PROTOCOL_VERSION, 1, true)
|
||||
);
|
||||
assert_eq!(
|
||||
seen[0].msg,
|
||||
Message::SessionState(SessionState { session: session() })
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_frame_is_an_error() {
|
||||
let dir = TempDir::new("state-error");
|
||||
let socket = dir.path().join("state.sock");
|
||||
let _fake = FakeBrokerd::start(&socket, Answer::Error("the state file is damaged".into()));
|
||||
assert_eq!(
|
||||
ask(&socket, &session(), WAIT),
|
||||
Err("brokerd: the state file is damaged".to_string())
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_listener_is_an_error_at_once() {
|
||||
let dir = TempDir::new("state-none");
|
||||
let socket = dir.path().join("state.sock");
|
||||
let started = Instant::now();
|
||||
let err = ask(&socket, &session(), WAIT).unwrap_err();
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_secs(1),
|
||||
"{:?}",
|
||||
started.elapsed()
|
||||
);
|
||||
assert!(err.starts_with(&format!("{}: ", socket.display())), "{err}");
|
||||
assert!(!err.contains('\n'), "{err}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silence_is_an_error_after_the_wait() {
|
||||
let dir = TempDir::new("state-hang");
|
||||
let socket = dir.path().join("state.sock");
|
||||
let fake = FakeBrokerd::start(&socket, Answer::Hang);
|
||||
let started = Instant::now();
|
||||
let err = ask(&socket, &session(), Duration::from_millis(300)).unwrap_err();
|
||||
let waited = started.elapsed();
|
||||
assert!(waited >= Duration::from_millis(300), "{waited:?}");
|
||||
assert!(waited < Duration::from_secs(5), "{waited:?}");
|
||||
assert!(err.starts_with(&format!("{}: ", socket.display())), "{err}");
|
||||
assert_eq!(fake.seen().len(), 1, "the question was not sent");
|
||||
}
|
||||
@@ -0,0 +1,428 @@
|
||||
//! Deciding reported turns (M5c spec, section 5): every undecided turn of every waiting session up
|
||||
//! to the one reported, as far as the log goes; the log's last turn waits for its reply; one
|
||||
//! question to `brokerd` per session per walk, and none when nothing waits; a silent `brokerd`
|
||||
//! stops the walk and is logged once, and so is its return; a damaged log and a ledger that cannot
|
||||
//! be written. `brokerd` is a closure here; the socket is in `honcho_state.rs`. Do not edit.
|
||||
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
|
||||
use std::cell::RefCell;
|
||||
|
||||
use bxcore::{DataClass, LogRecord, SessionId, SessionStateReply, Timestamp};
|
||||
use logs::{assistant, write_log};
|
||||
use memoryd::honcho::decide::{NO_REPLY, NO_SPEAKER, SECRET_CLASS};
|
||||
use memoryd::honcho::ledger::{Action, Item, Ledger, Workspace};
|
||||
use memoryd::honcho::turns::{
|
||||
LEDGER, LOG_RUNBOOK, TurnSync, decidable, finished, log_path, read_log,
|
||||
};
|
||||
use tmp::TempDir;
|
||||
use turns::{clean, fetched, running, sid, spoken, write_turns};
|
||||
|
||||
const BROKER_DOWN: &str = "see docs/runbook.md#broker-state-unavailable";
|
||||
const BACK: &str = "memoryd: brokerd answers on its state socket again; turns are decided";
|
||||
|
||||
/// A home with a ledger, a `TurnSync`, and a `brokerd` whose answer the test sets.
|
||||
struct World {
|
||||
sync: TurnSync,
|
||||
answer: RefCell<Result<SessionStateReply, String>>,
|
||||
asked: RefCell<Vec<String>>,
|
||||
lines: RefCell<Vec<String>>,
|
||||
dir: TempDir,
|
||||
}
|
||||
|
||||
impl World {
|
||||
fn new(tag: &str) -> World {
|
||||
let dir = TempDir::new(tag);
|
||||
std::fs::create_dir_all(dir.path().join("memory")).unwrap();
|
||||
let ledger = Ledger::open(&dir.path().join("memory").join(LEDGER)).unwrap();
|
||||
World {
|
||||
sync: TurnSync::new(ledger, "boxmaker0", &dir.path().join("sessions")),
|
||||
answer: RefCell::new(Ok(clean())),
|
||||
asked: RefCell::new(Vec::new()),
|
||||
lines: RefCell::new(Vec::new()),
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn log(&self, session: &str, turns: &[Vec<LogRecord>]) {
|
||||
write_turns(self.dir.path(), session, turns);
|
||||
}
|
||||
|
||||
fn updated(&mut self, session: &str, turn: u64) {
|
||||
let ask = |s: &SessionId| {
|
||||
self.asked.borrow_mut().push(s.as_str().to_string());
|
||||
self.answer.borrow().clone()
|
||||
};
|
||||
let log = |line: &str| self.lines.borrow_mut().push(line.to_string());
|
||||
let now = Timestamp::parse("2026-09-28T10:00:00.000Z").unwrap();
|
||||
self.sync.updated(&sid(session), turn, &ask, now, &log);
|
||||
}
|
||||
|
||||
fn action(&self, session: &str, turn: u64) -> Option<Action> {
|
||||
let item = Item::Turn {
|
||||
session: sid(session),
|
||||
turn,
|
||||
};
|
||||
let path = self.dir.path().join("memory").join(LEDGER);
|
||||
let text = std::fs::read_to_string(path).unwrap_or_default();
|
||||
let key = serde_json::to_string(&item).unwrap();
|
||||
let line = text.lines().find(|l| l.contains(&key))?;
|
||||
let value: serde_json::Value = serde_json::from_str(line).unwrap();
|
||||
Some(serde_json::from_value(value["action"].clone()).unwrap())
|
||||
}
|
||||
|
||||
fn waiting(&self) -> Vec<(String, u64)> {
|
||||
let waiting = self.sync.waiting();
|
||||
waiting
|
||||
.into_iter()
|
||||
.map(|(s, n)| (s.as_str().to_string(), n))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn to(workspace: Workspace, speaker: &str) -> Option<Action> {
|
||||
Some(Action::Turn {
|
||||
workspace,
|
||||
speaker: speaker.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
fn skip(why: &str) -> Option<Action> {
|
||||
Some(Action::Skip {
|
||||
why: why.to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_reported_turn_is_decided_and_written_at_once() {
|
||||
let mut w = World::new("turns-one");
|
||||
w.log("mm-a", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.action("mm-a", 1), to(Workspace::Trusted, "kyle"));
|
||||
assert_eq!(*w.asked.borrow(), vec!["mm-a"]);
|
||||
assert!(w.waiting().is_empty());
|
||||
assert!(w.lines.borrow().is_empty(), "{:?}", w.lines.borrow());
|
||||
let reopened = Ledger::open(&w.dir.path().join("memory").join(LEDGER)).unwrap();
|
||||
assert_eq!(reopened.counts().decided, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_undecided_turn_up_to_the_reported_one_is_decided() {
|
||||
let mut w = World::new("turns-all");
|
||||
w.log(
|
||||
"mm-a",
|
||||
&[
|
||||
turns::finished(None),
|
||||
running(Some("kyle")),
|
||||
fetched(Some("kyle")),
|
||||
turns::finished(Some("kyle")),
|
||||
],
|
||||
);
|
||||
w.updated("mm-a", 4);
|
||||
assert_eq!(w.action("mm-a", 1), skip(NO_SPEAKER));
|
||||
assert_eq!(w.action("mm-a", 2), skip(NO_REPLY));
|
||||
assert_eq!(w.action("mm-a", 3), to(Workspace::Untrusted, "kyle"));
|
||||
assert_eq!(w.action("mm-a", 4), to(Workspace::Untrusted, "kyle"));
|
||||
assert_eq!(*w.asked.borrow(), vec!["mm-a"], "one question per session");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_as_far_as_the_report_and_the_log_go() {
|
||||
let mut w = World::new("turns-limit");
|
||||
w.log(
|
||||
"mm-a",
|
||||
&[turns::finished(Some("kyle")), turns::finished(Some("kyle"))],
|
||||
);
|
||||
w.updated("mm-a", 9);
|
||||
assert!(w.action("mm-a", 2).is_some());
|
||||
assert!(w.waiting().is_empty(), "turns not in the log do not wait");
|
||||
w.updated("mm-a", 9);
|
||||
assert_eq!(
|
||||
w.asked.borrow().len(),
|
||||
1,
|
||||
"asked about turns not in the log"
|
||||
);
|
||||
w.log(
|
||||
"mm-b",
|
||||
&[
|
||||
turns::finished(Some("kyle")),
|
||||
turns::finished(Some("kyle")),
|
||||
turns::finished(Some("kyle")),
|
||||
],
|
||||
);
|
||||
w.updated("mm-b", 1);
|
||||
assert!(w.action("mm-b", 1).is_some());
|
||||
assert_eq!(w.action("mm-b", 2), None, "turn 2 was not reported");
|
||||
assert!(w.waiting().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_logs_last_turn_waits_for_its_reply() {
|
||||
let mut w = World::new("turns-running");
|
||||
w.log(
|
||||
"mm-a",
|
||||
&[turns::finished(Some("kyle")), running(Some("kyle"))],
|
||||
);
|
||||
w.updated("mm-a", 2);
|
||||
assert!(w.action("mm-a", 1).is_some());
|
||||
assert_eq!(w.action("mm-a", 2), None, "a running turn was decided");
|
||||
assert_eq!(w.waiting(), vec![("mm-a".to_string(), 2)]);
|
||||
// Another report while it runs: it still waits, and nothing is asked for it.
|
||||
w.log("mm-c", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-c", 1);
|
||||
assert_eq!(w.action("mm-a", 2), None);
|
||||
assert_eq!(w.waiting(), vec![("mm-a".to_string(), 2)]);
|
||||
assert_eq!(*w.asked.borrow(), vec!["mm-a", "mm-c"]);
|
||||
// It ends; the next report of any session decides it.
|
||||
let mut ended = running(Some("kyle"));
|
||||
ended.push(assistant(3, Some("found it"), &[]));
|
||||
w.log("mm-a", &[turns::finished(Some("kyle")), ended]);
|
||||
w.log("mm-b", &[turns::finished(Some("ana"))]);
|
||||
w.updated("mm-b", 1);
|
||||
assert_eq!(w.action("mm-a", 2), to(Workspace::Trusted, "kyle"));
|
||||
assert_eq!(w.action("mm-b", 1), to(Workspace::Trusted, "ana"));
|
||||
assert!(w.waiting().is_empty());
|
||||
assert_eq!(w.sync.ledger().counts().skipped, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_report_past_the_log_does_not_decide_a_running_turn() {
|
||||
let mut w = World::new("turns-inflated");
|
||||
w.log(
|
||||
"mm-a",
|
||||
&[turns::finished(Some("kyle")), running(Some("kyle"))],
|
||||
);
|
||||
w.updated("mm-a", 5);
|
||||
assert!(w.action("mm-a", 1).is_some());
|
||||
assert_eq!(w.action("mm-a", 2), None, "a running turn was decided");
|
||||
assert_eq!(w.waiting(), vec![("mm-a".to_string(), 5)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_earlier_report_does_not_lower_the_turn() {
|
||||
let mut w = World::new("turns-order");
|
||||
*w.answer.borrow_mut() = Err("no".to_string());
|
||||
w.log(
|
||||
"mm-a",
|
||||
&[
|
||||
turns::finished(Some("kyle")),
|
||||
turns::finished(Some("kyle")),
|
||||
turns::finished(Some("kyle")),
|
||||
],
|
||||
);
|
||||
w.updated("mm-a", 3);
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.waiting(), vec![("mm-a".to_string(), 3)]);
|
||||
*w.answer.borrow_mut() = Ok(clean());
|
||||
w.log("mm-b", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-b", 1);
|
||||
assert_eq!(w.action("mm-a", 3), to(Workspace::Trusted, "kyle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_silent_brokerd_is_logged_once_and_its_return_once() {
|
||||
let mut w = World::new("turns-down");
|
||||
let why = "/h/run/memory-broker/state.sock: Connection refused (os error 111)";
|
||||
*w.answer.borrow_mut() = Err(why.to_string());
|
||||
w.log("mm-a", &[turns::finished(Some("kyle"))]);
|
||||
w.log("mm-b", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-a", 1);
|
||||
w.updated("mm-b", 1);
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.action("mm-a", 1), None);
|
||||
assert_eq!(w.action("mm-b", 1), None);
|
||||
assert_eq!(
|
||||
*w.lines.borrow(),
|
||||
vec![format!(
|
||||
"memoryd: {why}; turns wait undecided; {BROKER_DOWN}"
|
||||
)]
|
||||
);
|
||||
assert_eq!(w.waiting().len(), 2);
|
||||
*w.answer.borrow_mut() = Ok(clean());
|
||||
w.log("mm-c", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-c", 1);
|
||||
for session in ["mm-a", "mm-b", "mm-c"] {
|
||||
assert_eq!(
|
||||
w.action(session, 1),
|
||||
to(Workspace::Trusted, "kyle"),
|
||||
"{session}"
|
||||
);
|
||||
}
|
||||
assert_eq!(w.lines.borrow().len(), 2);
|
||||
assert_eq!(w.lines.borrow()[1], BACK);
|
||||
assert!(w.waiting().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_silent_brokerd_stops_the_walk_at_the_first_question() {
|
||||
let mut w = World::new("turns-stop");
|
||||
*w.answer.borrow_mut() = Err("no".to_string());
|
||||
for session in ["mm-a", "mm-b", "mm-c"] {
|
||||
w.log(session, &[turns::finished(Some("kyle"))]);
|
||||
}
|
||||
w.updated("mm-a", 1);
|
||||
w.updated("mm-b", 1);
|
||||
w.updated("mm-c", 1);
|
||||
// One question per walk, to the first waiting session in the order of the ids.
|
||||
assert_eq!(*w.asked.borrow(), vec!["mm-a", "mm-a", "mm-a"]);
|
||||
*w.answer.borrow_mut() = Ok(clean());
|
||||
w.updated("mm-c", 1);
|
||||
assert_eq!(w.asked.borrow()[3..], ["mm-a", "mm-b", "mm-c"]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn nothing_undecided_asks_nothing() {
|
||||
let mut w = World::new("turns-quiet");
|
||||
w.log("mm-a", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-a", 1);
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(*w.asked.borrow(), vec!["mm-a"]);
|
||||
let lines = std::fs::read_to_string(w.dir.path().join("memory").join(LEDGER)).unwrap();
|
||||
assert_eq!(lines.lines().count(), 1);
|
||||
// A restart reads the ledger through: nothing is decided twice.
|
||||
let ledger = Ledger::open(&w.dir.path().join("memory").join(LEDGER)).unwrap();
|
||||
w.sync = TurnSync::new(ledger, "boxmaker0", &w.dir.path().join("sessions"));
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.asked.borrow().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn brokerds_state_decides_the_workspace() {
|
||||
let mut w = World::new("turns-state");
|
||||
w.log("mm-a", &[turns::finished(Some("kyle"))]);
|
||||
*w.answer.borrow_mut() = Ok(SessionStateReply {
|
||||
class: DataClass::Private,
|
||||
untrusted: true,
|
||||
});
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.action("mm-a", 1), to(Workspace::Untrusted, "kyle"));
|
||||
w.log("mm-b", &[turns::finished(Some("kyle"))]);
|
||||
*w.answer.borrow_mut() = Ok(SessionStateReply {
|
||||
class: DataClass::Secret,
|
||||
untrusted: false,
|
||||
});
|
||||
w.updated("mm-b", 1);
|
||||
assert_eq!(w.action("mm-b", 1), skip(SECRET_CLASS));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_log_waits_and_is_logged_once() {
|
||||
let mut w = World::new("turns-damaged");
|
||||
let path = write_log(w.dir.path(), "mm-a", &turns::finished(Some("kyle")));
|
||||
let text = std::fs::read_to_string(&path).unwrap();
|
||||
std::fs::write(&path, format!("not json\n{text}")).unwrap();
|
||||
w.log("mm-b", &[turns::finished(Some("kyle"))]);
|
||||
w.updated("mm-a", 1);
|
||||
w.updated("mm-b", 1);
|
||||
assert_eq!(w.action("mm-a", 1), None);
|
||||
assert_eq!(w.action("mm-b", 1), to(Workspace::Trusted, "kyle"));
|
||||
let lines = w.lines.borrow().clone();
|
||||
assert_eq!(lines.len(), 1, "{lines:?}");
|
||||
let start = format!("memoryd: {}:1: ", path.display());
|
||||
assert!(lines[0].starts_with(&start), "{}", lines[0]);
|
||||
assert!(
|
||||
lines[0].ends_with(&format!("; {LOG_RUNBOOK}")),
|
||||
"{}",
|
||||
lines[0]
|
||||
);
|
||||
assert_eq!(w.waiting(), vec![("mm-a".to_string(), 1)]);
|
||||
assert_eq!(*w.asked.borrow(), vec!["mm-b"]);
|
||||
// Mended, it is decided at the next report.
|
||||
std::fs::write(&path, text).unwrap();
|
||||
w.updated("mm-b", 1);
|
||||
assert_eq!(w.action("mm-a", 1), to(Workspace::Trusted, "kyle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_line_loopd_is_still_writing_is_left_out() {
|
||||
let mut w = World::new("turns-partial");
|
||||
let path = write_log(w.dir.path(), "mm-a", &turns::finished(Some("kyle")));
|
||||
let half = serde_json::to_string(&spoken("next", Some("kyle"))).unwrap();
|
||||
let mut text = std::fs::read_to_string(&path).unwrap();
|
||||
text.push_str(&half[..half.len() / 2]);
|
||||
std::fs::write(&path, text).unwrap();
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.action("mm-a", 1), to(Workspace::Trusted, "kyle"));
|
||||
assert!(w.lines.borrow().is_empty(), "{:?}", w.lines.borrow());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_ledger_that_cannot_be_written_stops_the_walk() {
|
||||
let mut w = World::new("turns-ledger");
|
||||
w.log("mm-a", &[turns::finished(Some("kyle"))]);
|
||||
let ledger = w.dir.path().join("memory").join(LEDGER);
|
||||
std::fs::create_dir(&ledger).unwrap();
|
||||
w.updated("mm-a", 1);
|
||||
let lines = w.lines.borrow().clone();
|
||||
assert_eq!(lines.len(), 1, "{lines:?}");
|
||||
assert!(lines[0].starts_with(&format!("memoryd: {}: ", ledger.display())));
|
||||
assert!(lines[0].ends_with("; see docs/runbook.md#honcho-ledger-damaged"));
|
||||
assert!(!w.sync.ledger().is_decided(&Item::Turn {
|
||||
session: sid("mm-a"),
|
||||
turn: 1
|
||||
}));
|
||||
assert_eq!(w.waiting(), vec![("mm-a".to_string(), 1)]);
|
||||
std::fs::remove_dir(&ledger).unwrap();
|
||||
w.updated("mm-a", 1);
|
||||
assert_eq!(w.action("mm-a", 1), to(Workspace::Trusted, "kyle"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reading_a_log() {
|
||||
let dir = TempDir::new("turns-read");
|
||||
let path = log_path(&dir.path().join("sessions"), &sid("mm-a"));
|
||||
assert_eq!(path, dir.path().join("sessions/mm-a/0.jsonl"));
|
||||
assert_eq!(read_log(&path), Ok(Vec::new()), "a missing log");
|
||||
let records = turns::finished(Some("kyle"));
|
||||
write_log(dir.path(), "mm-a", &records);
|
||||
assert_eq!(read_log(&path), Ok(records.clone()));
|
||||
let mut text = std::fs::read_to_string(&path).unwrap();
|
||||
text.push_str("{\"type\":\"us");
|
||||
std::fs::write(&path, &text).unwrap();
|
||||
assert_eq!(read_log(&path), Ok(records));
|
||||
std::fs::write(&path, "{\"type\":\"user\"}\n").unwrap();
|
||||
let err = read_log(&path).unwrap_err();
|
||||
assert!(err.starts_with(&format!("{}:1: ", path.display())), "{err}");
|
||||
assert!(err.ends_with(&format!("; {LOG_RUNBOOK}")), "{err}");
|
||||
std::fs::remove_file(&path).unwrap();
|
||||
std::fs::create_dir(&path).unwrap();
|
||||
let err = read_log(&path).unwrap_err();
|
||||
assert!(err.starts_with(&format!("{}: ", path.display())), "{err}");
|
||||
assert_eq!(LOG_RUNBOOK, "see docs/runbook.md#session-log-damaged");
|
||||
assert_eq!(LEDGER, "honcho.jsonl");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_is_finished_by_an_answer_without_tool_calls() {
|
||||
assert!(finished(&turns::finished(Some("kyle"))));
|
||||
assert!(finished(&fetched(None)));
|
||||
assert!(!finished(&running(None)));
|
||||
assert!(!finished(&[spoken("hello", None)]));
|
||||
let mut empty = vec![spoken("hello", None), assistant(2, None, &[])];
|
||||
assert!(finished(&empty), "an empty answer still ends the turn");
|
||||
empty.push(assistant(3, Some("and"), &[("fetch", "{}")]));
|
||||
assert!(!finished(&empty));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn how_far_a_log_may_be_decided() {
|
||||
let done = |n: usize| -> Vec<LogRecord> {
|
||||
(0..n).flat_map(|_| turns::finished(Some("kyle"))).collect()
|
||||
};
|
||||
assert_eq!(decidable(&[], 3), (0, false));
|
||||
assert_eq!(decidable(&done(3), 0), (0, false));
|
||||
assert_eq!(decidable(&done(3), 2), (2, false));
|
||||
assert_eq!(decidable(&done(3), 3), (3, false));
|
||||
assert_eq!(decidable(&done(3), 7), (3, false));
|
||||
let mut open = done(2);
|
||||
open.extend(running(Some("kyle")));
|
||||
assert_eq!(decidable(&open, 2), (2, false));
|
||||
assert_eq!(decidable(&open, 3), (2, true));
|
||||
assert_eq!(decidable(&open, 9), (2, true));
|
||||
assert_eq!(decidable(&running(None), 1), (0, true));
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
//! A fake `brokerd` state socket for tests: it records each question and answers with a fixed
|
||||
//! state, an error frame, or nothing at all (it holds the connection open without a byte). It
|
||||
//! stops, and releases every held connection, when dropped; every wait has a deadline. Do not
|
||||
//! edit.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bxcore::{
|
||||
Envelope, ErrorCode, Message, PROTOCOL_VERSION, SessionStateReply, WireError, read_frame,
|
||||
write_frame,
|
||||
};
|
||||
|
||||
/// How the fake answers.
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum Answer {
|
||||
State(SessionStateReply),
|
||||
Error(String),
|
||||
/// Read the question, then send nothing until the fake is dropped (at most 30 s).
|
||||
Hang,
|
||||
}
|
||||
|
||||
pub struct FakeBrokerd {
|
||||
path: PathBuf,
|
||||
stop: Arc<AtomicBool>,
|
||||
seen: Arc<Mutex<Vec<Envelope>>>,
|
||||
accept: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
impl FakeBrokerd {
|
||||
/// Listen on `path` (its directory is made) and answer every connection with `answer`.
|
||||
pub fn start(path: &Path, answer: Answer) -> FakeBrokerd {
|
||||
if let Some(dir) = path.parent() {
|
||||
std::fs::create_dir_all(dir).unwrap();
|
||||
}
|
||||
let _ = std::fs::remove_file(path);
|
||||
let listener = UnixListener::bind(path).unwrap();
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let seen = Arc::new(Mutex::new(Vec::new()));
|
||||
let (stop_accept, seen_accept) = (Arc::clone(&stop), Arc::clone(&seen));
|
||||
let accept = std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
if stop_accept.load(Ordering::SeqCst) {
|
||||
return;
|
||||
}
|
||||
let Ok(stream) = stream else { return };
|
||||
let (stop, seen, answer) = (
|
||||
Arc::clone(&stop_accept),
|
||||
Arc::clone(&seen_accept),
|
||||
answer.clone(),
|
||||
);
|
||||
std::thread::spawn(move || serve(stream, &answer, &seen, &stop));
|
||||
}
|
||||
});
|
||||
FakeBrokerd {
|
||||
path: path.to_path_buf(),
|
||||
stop,
|
||||
seen,
|
||||
accept: Some(accept),
|
||||
}
|
||||
}
|
||||
|
||||
/// Every question read so far, in order.
|
||||
pub fn seen(&self) -> Vec<Envelope> {
|
||||
self.seen.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Waits up to 10 s until at least `n` questions were read; true if they were.
|
||||
pub fn wait_for(&self, n: usize) -> bool {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline {
|
||||
if self.seen().len() >= n {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
fn serve(mut stream: UnixStream, answer: &Answer, seen: &Mutex<Vec<Envelope>>, stop: &AtomicBool) {
|
||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
|
||||
let Ok(question) = read_frame(&mut stream) else {
|
||||
return;
|
||||
};
|
||||
let id = question.id;
|
||||
seen.lock().unwrap().push(question);
|
||||
let msg = match answer {
|
||||
Answer::State(state) => Message::SessionStateReply(state.clone()),
|
||||
Answer::Error(detail) => Message::Error(WireError {
|
||||
code: ErrorCode::Internal,
|
||||
detail: detail.clone(),
|
||||
}),
|
||||
Answer::Hang => {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
while !stop.load(Ordering::SeqCst) && Instant::now() < deadline {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
let env = Envelope {
|
||||
v: PROTOCOL_VERSION,
|
||||
id,
|
||||
r#final: true,
|
||||
msg,
|
||||
};
|
||||
let _ = write_frame(&mut stream, &env);
|
||||
}
|
||||
|
||||
impl Drop for FakeBrokerd {
|
||||
fn drop(&mut self) {
|
||||
self.stop.store(true, Ordering::SeqCst);
|
||||
// Wakes the accept loop, which then sees `stop`.
|
||||
let _ = UnixStream::connect(&self.path);
|
||||
if let Some(accept) = self.accept.take() {
|
||||
let _ = accept.join();
|
||||
}
|
||||
let _ = std::fs::remove_file(&self.path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,57 @@
|
||||
//! Session logs of turns with speakers, for the tests of deciding turns. Do not edit.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::path::Path;
|
||||
|
||||
use bxcore::{DataClass, LogRecord, SessionId, SessionStateReply};
|
||||
|
||||
use crate::logs::{assistant, at, result};
|
||||
|
||||
pub fn sid(s: &str) -> SessionId {
|
||||
SessionId::new(s).unwrap()
|
||||
}
|
||||
|
||||
pub fn clean() -> SessionStateReply {
|
||||
SessionStateReply {
|
||||
class: DataClass::Private,
|
||||
untrusted: false,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn spoken(content: &str, speaker: Option<&str>) -> LogRecord {
|
||||
LogRecord::User {
|
||||
time: at(1),
|
||||
content: content.to_string(),
|
||||
speaker: speaker.map(str::to_string),
|
||||
}
|
||||
}
|
||||
|
||||
/// A finished turn: the speaker's message and the reply.
|
||||
pub fn finished(speaker: Option<&str>) -> Vec<LogRecord> {
|
||||
vec![spoken("hello", speaker), assistant(2, Some("hi"), &[])]
|
||||
}
|
||||
|
||||
/// A finished turn that read an untrusted page.
|
||||
pub fn fetched(speaker: Option<&str>) -> Vec<LogRecord> {
|
||||
vec![
|
||||
spoken("read this page", speaker),
|
||||
assistant(2, None, &[("fetch", "{}")]),
|
||||
result(3, "the page", DataClass::Public, true),
|
||||
assistant(4, Some("it says hello"), &[]),
|
||||
]
|
||||
}
|
||||
|
||||
/// A turn still running, or one that failed: no reply.
|
||||
pub fn running(speaker: Option<&str>) -> Vec<LogRecord> {
|
||||
vec![
|
||||
spoken("look it up", speaker),
|
||||
assistant(2, None, &[("fetch", "{}")]),
|
||||
]
|
||||
}
|
||||
|
||||
/// `sessions/<session>/0.jsonl` under `home`, the turns one after another.
|
||||
pub fn write_turns(home: &Path, session: &str, turns: &[Vec<LogRecord>]) {
|
||||
let records: Vec<LogRecord> = turns.iter().flatten().cloned().collect();
|
||||
crate::logs::write_log(home, session, &records);
|
||||
}
|
||||
@@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone.
|
||||
|
||||
| Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model |
|
||||
|---|---|---|---|---|---|---|---|
|
||||
| M5c1/09-memoryd-honcho-turns | 2026-09-28 | done | 1 | pass | none | Copied the five given files from `docs/plans/M5c1/files/`: the tests `crates/memoryd/tests/honcho_state.rs`, `honcho_turns.rs` (from `honcho_turns.rs-task09`) and `honcho_serve.rs` (from `honcho_serve.rs-task09`) and the support `crates/memoryd/tests/support/brokerd.rs` and `support/turns.rs` (from `turns.rs-task09`). Step 2 showed exactly the named failures: three test targets (`honcho_state`, `honcho_turns`, `honcho_serve`) with one E0432 each (`unresolved import memoryd::honcho::state`, `...::turns` and `memoryd::serve::StartError`); every other target compiled. Made H1 (`pub mod state;` and `pub mod turns;` after `pub mod ledger;` in `honcho/mod.rs`) and copied the two skeletons; `cargo check` gave 19 warnings (four unused imports: `UnixStream`; the five `bxcore` frame items plus `crate::honcho::state` and `turn_records`/`turns`; fourteen unused variables: `session` four, `log` twice, the rest once each; three fields never read: `sessions`, `broker_down`, `unreadable`). Filled its seven bodies in the order of the file, each from its comment and `cargo check` after: `ask` (`state.rs`) — connect, set read and write timeout to `wait`, write an `Envelope { v: PROTOCOL_VERSION, id: 1, r#final: true, msg: Message::SessionState(SessionState { session }) }`, match `read_frame`'s msg: `SessionStateReply` -> Ok, `Error` -> `Err(format!("brokerd: {}", e.detail))`, anything else the wrong-message error, every io error through `socket: <e>`; `log_path` (`sessions.join(id).join("0.jsonl")`); `read_log` (`fs::read`, NotFound -> empty, else Io with the runbook, walk the complete lines up to the last newline so a partial last line is left out, each line parsed and a bad line named with its number and the runbook); `finished` (last `Assistant` with empty `tool_calls`); `records` (`read_log`, remove from `unreadable` on Ok, log a damaged log once on Err); `state` (`ask`, log the outage and its return once each); `decidable` (`reported.min(count)`, hold the log's last turn when it is not finished). After the seventh `cargo check` is clean with no warning; `cargo test -p memoryd --test honcho_state --test honcho_turns` gives all 21 pass. Step 6 all pass: `honcho_serve` 6, `honcho_turns` 16, `honcho_state` 5, `serve` 8, `honcho_start` 4, the existing suites unchanged; the three step-8 greps print as expected and the other-crates `git diff --stat` is empty. `make gate` printed `gate: ok` with 1257 tests (1256 in `cargo test --workspace`, plus the 1 ignored `end_to_end` the gate runs separately with `BOXMAKER_BROKERD`). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
|
||||
| M5c1/08-memoryd-honcho-decide | 2026-09-28 | done | 1 | pass | none | Copied the given test `crates/memoryd/tests/honcho_decide.rs` and the skeleton `crates/memoryd/src/honcho/decide.rs` from `docs/plans/M5c1/files/`. Step 2 showed exactly the named failure: `honcho_decide` with one E0432 (`unresolved import memoryd::honcho::decide`); every other target compiled. Made H1 (`pub mod decide;` after `pub mod config;` in `honcho/mod.rs`) and copied the skeleton; `cargo check` gave 11 warnings (one unused import `bxcore::sanitize::refuse`, ten unused variables: `records` four, `turn` four, `speaker` and `text` once each). Step 3 `cargo test -p memoryd --test honcho_decide` gave 1 pass (`the_skip_reasons_are_these_texts`) and 17 fail with `not yet implemented`. Filled its eight bodies in the order of the file, deleting each instruction comment as it went and keeping the doc comments, each from its comment and `cargo check` after: `turns` (filter `User`, `u64::try_from(count).unwrap_or(u64::MAX)`); `bounds` (walk `enumerate`, when a second `User` follows the tracked turn-th one return `(start, index)`, else count and set `start` at `seen == turn`, then `start.map(|s| (s, len))`); `turn_records` (`bounds` then `get`); `peer_name` (`replace('.', "_")`); `has_secret` (map each char `refuse` refuses on its own to a space via a 4-byte `encode_utf8` buffer, then keep only a `secret-` rule); `user_text` (first record's `User`, `speaker.as_deref()`); `final_reply` (last `Assistant` with empty `tool_calls`, `content.as_deref()`); `labelled_untrusted` (any `ToolResult { untrusted: true }`). After the eighth `cargo check` is clean with no warning; `cargo test` gives all 18 pass. Step 6 all pass: the grep for `fs::`/`std::io`/`Timestamp::now` prints 0, the two `refuse(` lines (87, 94) are both in `has_secret`, and the other-crates `git diff --stat` is empty. `make gate` printed `gate: ok` with 1230 tests (1229 in `cargo test --workspace`, plus the 1 ignored `end_to_end` the gate runs separately with `BOXMAKER_BROKERD` set). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
|
||||
| M5c1/07-memoryd-honcho-ledger | 2026-09-28 | done | 1 | pass | none | Copied the three given files from `docs/plans/M5c1/files/`: the tests `crates/memoryd/tests/honcho_ledger.rs`, `honcho_ledger_file.rs` and the support `crates/memoryd/tests/support/ledger.rs`. Step 2 showed exactly the named failures: two test targets (`honcho_ledger` and `honcho_ledger_file`) with two E0432 each (`unresolved import memoryd::honcho::ledger`), one in each test file and one in `support/ledger.rs`; every other target compiled. Made H1 (`pub mod ledger;` after `pub mod config;` in `honcho/mod.rs`) and copied the skeleton `crates/memoryd/src/honcho/ledger.rs`; `cargo check` gave 20 warnings from the unfilled bodies. Step 3 `cargo test -p memoryd --test honcho_ledger` gave 2 pass (`each_record_is_one_line_exactly_as_given`, `an_unknown_field_or_value_is_refused`) and 10 fail with `not yet implemented`. Filled its twelve bodies in the order of the file, each from its comment and `cargo check` after: `is_decided` (`contains_key`); `outbox` (walk `order` with `.iter().filter_map`, keeping `(item, action)` when `state == Waiting`); `sent_ids` (match `State::Sent(ids)` -> `ids.as_slice()`); `conclusion` (walk `sends` in order, pushing a sent `Conclude`'s first id and popping on a matching sent `Unconclude`, returning `live.pop()` so only workspace and text match and a `Sent` with no ids pushes nothing); `counts` (one per entry, matching `&entry.state` since `State` is not `Copy`); `check` (`AlreadyDecided` for a second `Decided`, `NotWaiting` unless a `Sent`/`Failed` finds its entry `Waiting`); `apply` (`Skip` -> `Skipped` else `Waiting`, push to `order` and `entries`; `Sent` sets `Sent(ids)` and pushes to `sends`, `Failed` sets `Failed` without pushing); `write_line` (append, create, mode 0600, one `write_all` then `sync_data`); then `read` (NotFound -> None, else Io), `complete` (rposition of `\n`), `parse` (serde error -> Damaged), `cut` (`set_len` then `sync_data`). Removed the `mut` in `cut` that clippy flagged (`set_len` and `sync_data` take `&self`); after the twelfth `cargo check` was clean and `cargo fmt` left the file. Both test files passed 12 and 12; appended `runbook-honcho-ledger-damaged.md` (`grep -c '^## honcho-ledger-damaged$'` printed 1). Step 7: `sync_data` 2, `write_all` 1, one runbook pointer line in `src`, no `create_dir`, no diff in the other crates. `make gate` printed `gate: ok` with 1212 tests (1211 in `cargo test --workspace`, plus the ignored `end_to_end` the gate runs with `BOXMAKER_BROKERD` set). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
|
||||
| M5c1/06-memoryd-honcho-config | 2026-09-28 | done | 1 | pass | none | Copied the three given tests from `docs/plans/M5c1/files/crates/memoryd/tests/`: `honcho_config.rs`, `honcho_reasons.rs`, `honcho_start.rs`. Step 2 showed exactly the named failures: `honcho_config` with eight (E0433 cannot find `honcho` in `memoryd`, E0432 unresolved import `bxcore::secrets`, two E0599 for `honcho_config`, three for `state_socket`, one for the `Honcho` variant) and `honcho_reasons` with two (the same E0433 and E0432); `honcho_start` compiled. Made K1 (`bxcore` gains `features = ["secrets"]`) and L1 (`pub mod honcho;` after `pub mod files;`), created `crates/memoryd/src/honcho/mod.rs` with the given one-line module, and copied the skeleton `crates/memoryd/src/honcho/config.rs`. Filled its seven bodies in the given order, each from its comment and `cargo check` after: `is_name` (not empty, byte length `<= MAX_NAME`, every byte `is_ascii_alphanumeric()` or `_` or `-`, bytes not chars so `é` and U+0660 fail); `check_url` (strip `https://` then `http://`, `split_once(':')` into host and port, a `host_ok` helper for the non-empty ASCII-alnum/`.`/`-` host, a port that is non-empty, all digits and parses to `u16` `> 0` via `ok().is_some_and`, the `URL_REASON` which never quotes the URL); `check_own_workspaces` (the three pairs in order, first equal pair is R5 with its name); `trust_of` (match on `as_deref`, `None`/`Some("untrusted")` is `Untrusted`, `Some("trusted")` is `Trusted`, else R9 with the written text); `class_of` (same shape, `Private`/`Public`/`Secret`, else R10); `read_workspace` (check agent then workspace names, R6 the agent is this agent, R7 the agent is listed in `earlier`, R8 the workspace is one of the three, then the `ReadWorkspace` with `trust_of`/`class_of`); `check_keys` (a name other than the three is a read key only when `strip_prefix(READ_KEY)` names an agent in `read`, else R11 with the suffix or R12 without the prefix, then `spec.source()` with R13). Made C1-C8 in `config.rs` (module comment, imports with `BTreeMap`/`SecretSpec`/`honcho`, `Sockets.state`, `Config.honcho` and `.secrets`, `ConfigError::Honcho` and its Display arm, `load` calls `honcho_config` before returning, `honcho_config` and `state_socket`), and M1-M2 in `main.rs` (imports and the pointer matched by error kind on the `Config::load` arm); the pointer edit first landed on the `instance::lock` arm and I moved it to the `Config::load` arm, the diff being exactly M1-M2. Appended `runbook-honcho-config-invalid.md` to `docs/runbook.md` (`grep -c '^## honcho-config-invalid$'` printed 1) and made D1 in `docs/dependencies.md`. Step 7: `cargo build -p memoryd` builds, the runbook grep prints one line, the `secrets::load`/`read_checked` grep prints nothing, the other-crates diff is empty. `cargo test -p memoryd` every suite passes (config 4, honcho_config 8, honcho_reasons 5, honcho_start 4). `make gate` printed `gate: ok` with 1188 tests (1187 in `cargo test --workspace` plus the ignored `end_to_end` the gate runs separately with `BOXMAKER_BROKERD`; 1171 before this task plus the 17 new). No new dependency, so `Cargo.lock` is unchanged. | ornith-1.5-35b-a3b |
|
||||
|
||||
+65
-16
@@ -368,7 +368,9 @@ that does not parse tool calls, or whose cache does not work.
|
||||
## session-log-damaged
|
||||
|
||||
**What you see.** A turn on an existing session fails with `<file>:<line>: <reason>` naming
|
||||
`sessions/<id>/<epoch>.jsonl`, and this entry. Other sessions work.
|
||||
`sessions/<id>/<epoch>.jsonl`, and this entry. Other sessions work. With `[honcho]`, `memoryd`
|
||||
logs the same line once, starting `memoryd: `, and that session's turns wait undecided for Honcho
|
||||
until the log reads again.
|
||||
|
||||
**Why.** `loopd` replays the log exactly to rebuild the conversation, so a line it cannot read
|
||||
would change what the model sees. It refuses rather than guess. `last line does not end in a
|
||||
@@ -926,23 +928,27 @@ tables and every `[secrets.honcho_…]`.
|
||||
|
||||
## honcho-ledger-damaged
|
||||
|
||||
**What you see.** `memoryd` logs a line naming `memory/honcho.jsonl`, a line number and a reason,
|
||||
then this entry:
|
||||
**What you see.** `memoryd` exits 1 at start, before it binds its sockets, with one line naming
|
||||
`memory/honcho.jsonl`, a line number and a reason, then this entry:
|
||||
|
||||
```text
|
||||
/srv/bx/memory/honcho.jsonl:12: turn 3 of session chat-1: decided already; see docs/runbook.md#honcho-ledger-damaged
|
||||
memoryd: /srv/bx/memory/honcho.jsonl:12: turn 3 of session chat-1: decided already; see docs/runbook.md#honcho-ledger-damaged
|
||||
```
|
||||
|
||||
Or, while it runs, `memoryd: <path>: <error>; see docs/runbook.md#honcho-ledger-damaged` when the
|
||||
ledger cannot be written (a full disk, a wrong owner). The turn waits undecided, and is decided at
|
||||
a later turn's end once the ledger can be written.
|
||||
|
||||
**Why.** The ledger is `memoryd`'s record of what it sends to Honcho: one line when a turn or a
|
||||
memory write is decided (where it goes, or that it goes nowhere), and one when it is sent or
|
||||
refused (M5c spec, section 5). What is decided and not yet sent is the outbox. `memoryd` reads the
|
||||
ledger through at start so that it sends nothing twice and nothing to a workspace it was not
|
||||
decided for. A line it cannot read, or one that breaks the ledger's rules, could do either, so it
|
||||
does not guess: it does not start its Honcho sync, and nothing is decided or sent to Honcho until
|
||||
the ledger reads through. The rules: an item is decided once (`decided already`), and it is sent
|
||||
or refused once, only after it is decided and only when it was not decided to go nowhere (`not
|
||||
waiting to be sent`). A last line cut off by a crash is not damage: `memoryd` drops it and says so
|
||||
in its log.
|
||||
does not guess: it does not start. While it is stopped no memory is served either, and `loopd`
|
||||
cannot start a session (`memory-unavailable`). The rules: an item is decided once (`decided
|
||||
already`), and it is sent or refused once, only after it is decided and only when it was not
|
||||
decided to go nowhere (`not waiting to be sent`). A last line cut off by a crash is not damage:
|
||||
`memoryd` drops it and says so in its log.
|
||||
|
||||
**Confirm.** Look at the line named, and at every line of the same item:
|
||||
|
||||
@@ -952,12 +958,55 @@ sed -n '<line>p' "$f"
|
||||
grep -n '"session":"<id>","turn":<n>}' "$f" # a turn; for an edit: '"id":"<id>","part":<n>}'
|
||||
```
|
||||
|
||||
**Fix.** Keep a copy first: `cp -a "$f" "$f.damaged-$(date +%s)"`. The ledger is written only by
|
||||
`memoryd`, so a damaged line was changed by hand or by the disk. If you know what the line should
|
||||
say, correct it. Otherwise remove it (`sed -i '<line>d' "$f"`), knowing what that costs: a removed
|
||||
`sent` or `failed` line means the item is sent again (a turn's messages appear twice in Honcho, a
|
||||
conclusion is made twice); a removed `decided` line means the item is decided again, and any
|
||||
later `sent` or `failed` line of the same item then breaks the rules, so remove those too. Honcho
|
||||
cannot delete a message, so a turn sent twice stays twice.
|
||||
**Fix.** If the ledger cannot be written, free space or correct the owner of `memory/`
|
||||
(`df -h`, `ls -l "$f"`); nothing else is needed. For a damaged line, keep a copy first:
|
||||
`cp -a "$f" "$f.damaged-$(date +%s)"`. The ledger is written only by `memoryd`, so a damaged line
|
||||
was changed by hand or by the disk. If you know what the line should say, correct it. Otherwise
|
||||
remove it (`sed -i '<line>d' "$f"`), knowing what that costs: a removed `sent` or `failed` line
|
||||
means the item is sent again (a turn's messages appear twice in Honcho, a conclusion is made
|
||||
twice); a removed `decided` line means the item is decided again, and any later `sent` or
|
||||
`failed` line of the same item then breaks the rules, so remove those too. Honcho cannot delete a
|
||||
message, so a turn sent twice stays twice.
|
||||
|
||||
**Check.** `memoryd` starts, and its log has no line with this entry.
|
||||
|
||||
## broker-state-unavailable
|
||||
|
||||
**What you see.** `memoryd` logs one line when it cannot get a session's state from `brokerd`,
|
||||
then this entry:
|
||||
|
||||
```text
|
||||
memoryd: /srv/bx/run/memory-broker/state.sock: Connection refused (os error 111); turns wait undecided; see docs/runbook.md#broker-state-unavailable
|
||||
```
|
||||
|
||||
The reason may instead be `No such file or directory` (nothing listens), a timeout
|
||||
(`frame I/O error: Resource temporarily unavailable`, after 5 seconds without a byte), or
|
||||
`brokerd: <its error>` when `brokerd` answered with an error. When `brokerd` answers again,
|
||||
`memoryd` logs `memoryd: brokerd answers on its state socket again; turns are decided`.
|
||||
Conversations are not affected.
|
||||
|
||||
**Why.** With `[honcho]`, `memoryd` decides where each finished turn goes in Honcho: the trusted
|
||||
workspace, the untrusted one, or nowhere (M5c spec, section 5). It needs `brokerd`'s record of the
|
||||
session's class and trust for that, and never guesses it: a guess could put a turn from a session
|
||||
that read a web page in the trusted workspace. So nothing is decided while `brokerd` does not
|
||||
answer. The turns wait, in memory, and are decided at the next turn's end of any session once
|
||||
`brokerd` answers, from the session's state then (which can only have moved towards untrusted).
|
||||
The outage is logged once, not at every turn. A restart of `memoryd` forgets the waiting turns
|
||||
until each of their sessions has another turn.
|
||||
|
||||
**Confirm.**
|
||||
|
||||
```sh
|
||||
ls -l "$BOXMAKER_HOME/run/memory-broker/state.sock"
|
||||
grep -n 'state' /etc/boxmaker/memoryd.toml /etc/boxmaker/brokerd.toml
|
||||
```
|
||||
|
||||
`memoryd`'s `[sockets] state` and `brokerd`'s `[sockets] state` must name the same path (both
|
||||
default to `<home>/run/memory-broker/state.sock`). If `brokerd` answered with an error, its own
|
||||
log names the problem, usually `broker-state-damaged`.
|
||||
|
||||
**Fix.** Start `brokerd` if it is stopped; make the two `[sockets] state` paths agree; or, for
|
||||
`brokerd`'s error, follow the entry its log names.
|
||||
|
||||
**Check.** At the next turn's end `memoryd` logs that `brokerd` answers again, and
|
||||
`memory/honcho.jsonl` gains a `decided` line for each turn that waited.
|
||||
|
||||
Reference in New Issue
Block a user