M5c-1 plan: task 11, the Honcho worker that sends turns
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -2,7 +2,8 @@
|
||||
//! `session_updated` before `brokerd` is asked, and a silent `brokerd` holds up neither `loopd`
|
||||
//! nor a memory call; without `[honcho]` nothing is asked or written; a damaged ledger stops the
|
||||
//! start with its pointer, and a cut-off last line is dropped and logged. Every wait has a
|
||||
//! deadline. Do not edit.
|
||||
//! deadline. (From M5c-1 task 11 a Honcho worker runs beside it; nothing here waits for it, and the
|
||||
//! cut-off line is a skipped turn, so that it has nothing to send.) Do not edit.
|
||||
|
||||
#[path = "support/brokerd.rs"]
|
||||
mod brokerd;
|
||||
@@ -41,6 +42,9 @@ fleet = \"fleet\"
|
||||
|
||||
const DECIDED: &str = r#"{"type":"decided","time":"2026-09-28T10:00:00.000Z","item":{"type":"turn","session":"mm-a","turn":1},"action":{"type":"turn","workspace":"trusted","speaker":"kyle"}}"#;
|
||||
|
||||
/// A turn decided to go nowhere: the worker has nothing to send for it.
|
||||
const SKIPPED: &str = r#"{"type":"decided","time":"2026-09-28T10:00:00.000Z","item":{"type":"turn","session":"mm-a","turn":1},"action":{"type":"skip","why":"the turn has no single speaker"}}"#;
|
||||
|
||||
struct Home {
|
||||
config: Config,
|
||||
text: String,
|
||||
@@ -263,7 +267,7 @@ fn a_damaged_ledger_stops_the_start() {
|
||||
fn a_cut_off_last_line_is_dropped_and_logged() {
|
||||
let home = Home::new("serve-repaired", true);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), format!("{DECIDED}\n{{\"type\":\"se")).unwrap();
|
||||
std::fs::write(home.ledger(), format!("{SKIPPED}\n{{\"type\":\"se")).unwrap();
|
||||
let lines = home.serve();
|
||||
assert_eq!(
|
||||
*lines.lock().unwrap(),
|
||||
@@ -272,7 +276,7 @@ fn a_cut_off_last_line_is_dropped_and_logged() {
|
||||
home.ledger().display()
|
||||
)]
|
||||
);
|
||||
assert_eq!(home.ledger_text(), format!("{DECIDED}\n"));
|
||||
assert_eq!(home.ledger_text(), format!("{SKIPPED}\n"));
|
||||
}
|
||||
|
||||
/// A `memoryd` process, killed and waited for when dropped: declared after its `Home`, so it
|
||||
|
||||
@@ -0,0 +1,329 @@
|
||||
//! `memoryd` deciding turns as it serves (M5c spec, section 5): `loopd` has its answer to
|
||||
//! `session_updated` before `brokerd` is asked, and a silent `brokerd` holds up neither `loopd`
|
||||
//! nor a memory call; without `[honcho]` nothing is asked or written; a damaged ledger stops the
|
||||
//! start with its pointer, and a cut-off last line is dropped and logged. Every wait has a
|
||||
//! deadline. Do not edit.
|
||||
|
||||
#[path = "support/brokerd.rs"]
|
||||
mod brokerd;
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
|
||||
use std::os::unix::net::UnixStream;
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use brokerd::{Answer, FakeBrokerd};
|
||||
use bxcore::{
|
||||
CallId, DataClass, Envelope, MemoryCall, Message, SessionId, SessionUpdated, ToolRequest,
|
||||
read_frame, write_frame,
|
||||
};
|
||||
use memoryd::config::Config;
|
||||
use memoryd::honcho::ledger::LedgerError;
|
||||
use memoryd::serve::{self, StartError};
|
||||
use tmp::TempDir;
|
||||
use turns::{clean, write_turns};
|
||||
|
||||
const SECTION: &str = "[honcho]
|
||||
url = \"http://127.0.0.1:9\"
|
||||
agent = \"boxmaker0\"
|
||||
owner = \"kyle\"
|
||||
trusted = \"boxmaker0-trusted\"
|
||||
untrusted = \"boxmaker0-untrusted\"
|
||||
fleet = \"fleet\"
|
||||
";
|
||||
|
||||
const DECIDED: &str = r#"{"type":"decided","time":"2026-09-28T10:00:00.000Z","item":{"type":"turn","session":"mm-a","turn":1},"action":{"type":"turn","workspace":"trusted","speaker":"kyle"}}"#;
|
||||
|
||||
struct Home {
|
||||
config: Config,
|
||||
text: String,
|
||||
dir: TempDir,
|
||||
}
|
||||
|
||||
impl Home {
|
||||
fn new(tag: &str, honcho: bool) -> Home {
|
||||
let dir = TempDir::new(tag);
|
||||
let soul = dir.write("etc/soul.md", "You are Boxmaker.\n");
|
||||
let text = format!(
|
||||
"[paths]\nhome = \"{}\"\n[memory]\nsoul_starter = \"{}\"\n{}",
|
||||
dir.path().display(),
|
||||
soul.display(),
|
||||
if honcho { SECTION } else { "" }
|
||||
);
|
||||
Home {
|
||||
config: Config::parse(&text).unwrap(),
|
||||
text,
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
fn ledger(&self) -> PathBuf {
|
||||
self.dir.path().join("memory/honcho.jsonl")
|
||||
}
|
||||
|
||||
fn ledger_text(&self) -> String {
|
||||
std::fs::read_to_string(self.ledger()).unwrap_or_default()
|
||||
}
|
||||
|
||||
/// `serve::start`, run on a thread; its log lines are collected.
|
||||
fn serve(&self) -> Arc<Mutex<Vec<String>>> {
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&lines);
|
||||
let log: serve::Log = Arc::new(move |l: &str| sink.lock().unwrap().push(l.to_string()));
|
||||
let running = serve::start(self.config.clone(), log).unwrap();
|
||||
std::thread::spawn(move || running.run());
|
||||
lines
|
||||
}
|
||||
}
|
||||
|
||||
/// One request and its answer, with a 10 s limit; how long the answer took.
|
||||
fn ask(socket: &Path, msg: Message) -> (Message, Duration) {
|
||||
let started = Instant::now();
|
||||
let mut stream = UnixStream::connect(socket).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.unwrap();
|
||||
let env = Envelope {
|
||||
v: 1,
|
||||
id: 7,
|
||||
r#final: true,
|
||||
msg,
|
||||
};
|
||||
write_frame(&mut stream, &env).unwrap();
|
||||
let reply = read_frame(&mut stream).unwrap();
|
||||
(reply.msg, started.elapsed())
|
||||
}
|
||||
|
||||
fn updated(home: &Home, session: &str, turn: u64) -> (Message, Duration) {
|
||||
let msg = Message::SessionUpdated(SessionUpdated {
|
||||
session: SessionId::new(session).unwrap(),
|
||||
turn,
|
||||
});
|
||||
ask(&home.config.loop_socket(), msg)
|
||||
}
|
||||
|
||||
/// Waits up to 10 s for `done`; true if it came.
|
||||
fn within_10s(done: impl Fn() -> bool) -> bool {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline {
|
||||
if done() {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_reported_turn_is_decided_after_loopd_has_its_answer() {
|
||||
let home = Home::new("serve-decide", true);
|
||||
write_turns(home.dir.path(), "mm-a", &[turns::finished(Some("kyle"))]);
|
||||
write_turns(home.dir.path(), "mm-z", &[turns::finished(Some("kyle"))]);
|
||||
let fake = FakeBrokerd::start(&home.config.state_socket(), Answer::State(clean()));
|
||||
let _lines = home.serve();
|
||||
// A report on another socket is refused, and decides nothing.
|
||||
let wrong = Message::SessionUpdated(SessionUpdated {
|
||||
session: SessionId::new("mm-z").unwrap(),
|
||||
turn: 1,
|
||||
});
|
||||
let (answer, _) = ask(&home.config.admin_socket(), wrong);
|
||||
assert!(matches!(answer, Message::Error(_)), "{answer:?}");
|
||||
let (answer, _) = updated(&home, "mm-a", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
assert!(within_10s(|| home.ledger_text().contains("\"turn\":1}")));
|
||||
// The line as `DECIDED`, but for its time, which is the decision's.
|
||||
let text = home.ledger_text();
|
||||
assert_eq!(text.lines().count(), 1, "{text}");
|
||||
let mut got: serde_json::Value = serde_json::from_str(&text).unwrap();
|
||||
let mut want: serde_json::Value = serde_json::from_str(DECIDED).unwrap();
|
||||
got["time"] = serde_json::Value::Null;
|
||||
want["time"] = serde_json::Value::Null;
|
||||
assert_eq!(got, want);
|
||||
assert_eq!(fake.seen().len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_silent_brokerd_holds_up_neither_loopd_nor_memory() {
|
||||
let home = Home::new("serve-silent", true);
|
||||
write_turns(home.dir.path(), "mm-a", &[turns::finished(Some("kyle"))]);
|
||||
write_turns(home.dir.path(), "mm-b", &[turns::finished(Some("kyle"))]);
|
||||
let fake = FakeBrokerd::start(&home.config.state_socket(), Answer::Hang);
|
||||
let lines = home.serve();
|
||||
let (answer, took) = updated(&home, "mm-a", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
assert!(took < Duration::from_secs(1), "loopd waited {took:?}");
|
||||
assert!(fake.wait_for(1), "brokerd was never asked");
|
||||
// While the question hangs: another report, and a memory call.
|
||||
let (answer, took) = updated(&home, "mm-b", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
assert!(took < Duration::from_secs(1), "loopd waited {took:?}");
|
||||
let call = Message::MemoryCall(MemoryCall {
|
||||
request: ToolRequest {
|
||||
session: SessionId::new("mm-b").unwrap(),
|
||||
call: CallId(1),
|
||||
tool: "recall".to_string(),
|
||||
arguments: r#"{"query":"hello"}"#.to_string(),
|
||||
},
|
||||
taint: DataClass::Private,
|
||||
untrusted: false,
|
||||
});
|
||||
let (answer, took) = ask(&home.config.broker_socket(), call);
|
||||
assert!(matches!(answer, Message::ToolResponse(_)), "{answer:?}");
|
||||
assert!(
|
||||
took < Duration::from_secs(1),
|
||||
"the memory call waited {took:?}"
|
||||
);
|
||||
// After 5 s without a byte, the outage is logged once, and nothing is decided.
|
||||
let pointer = "turns wait undecided; see docs/runbook.md#broker-state-unavailable";
|
||||
assert!(within_10s(|| lines
|
||||
.lock()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.any(|l| l.contains(pointer))));
|
||||
let logged = lines.lock().unwrap().clone();
|
||||
let outage: Vec<&String> = logged.iter().filter(|l| l.contains(pointer)).collect();
|
||||
assert_eq!(outage.len(), 1, "{logged:?}");
|
||||
let start = format!("memoryd: {}: ", home.config.state_socket().display());
|
||||
assert!(outage[0].starts_with(&start), "{}", outage[0]);
|
||||
assert_eq!(home.ledger_text(), "");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn without_honcho_nothing_is_asked_or_written() {
|
||||
let home = Home::new("serve-off", false);
|
||||
write_turns(home.dir.path(), "mm-a", &[turns::finished(Some("kyle"))]);
|
||||
let fake = FakeBrokerd::start(&home.config.state_socket(), Answer::State(clean()));
|
||||
let lines = home.serve();
|
||||
let (answer, _) = updated(&home, "mm-a", 1);
|
||||
assert!(matches!(answer, Message::Ok(_)), "{answer:?}");
|
||||
// A negative: nothing may happen, so there is nothing to wait for but a while.
|
||||
std::thread::sleep(Duration::from_millis(300));
|
||||
assert!(fake.seen().is_empty());
|
||||
assert!(!home.ledger().exists());
|
||||
assert!(
|
||||
lines.lock().unwrap().is_empty(),
|
||||
"{:?}",
|
||||
lines.lock().unwrap()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_damaged_ledger_stops_the_start() {
|
||||
let home = Home::new("serve-damaged", true);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), "not json\n").unwrap();
|
||||
let log: serve::Log = Arc::new(|_: &str| {});
|
||||
let err = match serve::start(home.config.clone(), log) {
|
||||
Err(err) => err,
|
||||
Ok(_) => panic!("memoryd started"),
|
||||
};
|
||||
assert!(
|
||||
matches!(
|
||||
err,
|
||||
StartError::Ledger(LedgerError::Damaged { line: 1, .. })
|
||||
),
|
||||
"{err:?}"
|
||||
);
|
||||
let shown = err.to_string();
|
||||
assert!(shown.starts_with(&format!("{}:1: ", home.ledger().display())));
|
||||
assert!(
|
||||
shown.ends_with("; see docs/runbook.md#honcho-ledger-damaged"),
|
||||
"{shown}"
|
||||
);
|
||||
assert_eq!(shown.matches("see docs/runbook.md").count(), 1, "{shown}");
|
||||
assert!(!home.config.loop_socket().exists(), "a socket was bound");
|
||||
// A ledger that cannot be read is named with the same pointer.
|
||||
std::fs::remove_file(home.ledger()).unwrap();
|
||||
std::fs::create_dir(home.ledger()).unwrap();
|
||||
let log: serve::Log = Arc::new(|_: &str| {});
|
||||
let err = match serve::start(home.config.clone(), log) {
|
||||
Err(err) => err,
|
||||
Ok(_) => panic!("memoryd started"),
|
||||
};
|
||||
assert!(
|
||||
matches!(err, StartError::Ledger(LedgerError::Io(..))),
|
||||
"{err:?}"
|
||||
);
|
||||
let shown = err.to_string();
|
||||
assert!(shown.starts_with(&format!("{}: ", home.ledger().display())));
|
||||
assert!(
|
||||
shown.ends_with("; see docs/runbook.md#honcho-ledger-damaged"),
|
||||
"{shown}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cut_off_last_line_is_dropped_and_logged() {
|
||||
let home = Home::new("serve-repaired", true);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), format!("{DECIDED}\n{{\"type\":\"se")).unwrap();
|
||||
let lines = home.serve();
|
||||
assert_eq!(
|
||||
*lines.lock().unwrap(),
|
||||
vec![format!(
|
||||
"memoryd: {}: a last line cut off by a crash was dropped",
|
||||
home.ledger().display()
|
||||
)]
|
||||
);
|
||||
assert_eq!(home.ledger_text(), format!("{DECIDED}\n"));
|
||||
}
|
||||
|
||||
/// A `memoryd` process, killed and waited for when dropped: declared after its `Home`, so it
|
||||
/// drops first.
|
||||
struct Daemon(std::process::Child);
|
||||
|
||||
impl Drop for Daemon {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memoryd_exits_with_the_ledgers_line() {
|
||||
let home = Home::new("serve-exit", true);
|
||||
let config = home.dir.write("memoryd.toml", &home.text);
|
||||
std::fs::create_dir_all(home.dir.path().join("memory")).unwrap();
|
||||
std::fs::write(home.ledger(), "not json\n").unwrap();
|
||||
let mut child = Daemon(
|
||||
Command::new(env!("CARGO_BIN_EXE_memoryd"))
|
||||
.args(["serve", "--config"])
|
||||
.arg(&config)
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
);
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let mut code = None;
|
||||
while Instant::now() < deadline {
|
||||
if let Ok(Some(status)) = child.0.try_wait() {
|
||||
code = status.code();
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
let _ = child.0.kill();
|
||||
let _ = child.0.wait();
|
||||
let mut stderr = String::new();
|
||||
if let Some(mut pipe) = child.0.stderr.take() {
|
||||
let _ = std::io::Read::read_to_string(&mut pipe, &mut stderr);
|
||||
}
|
||||
assert_eq!(code, Some(1), "{stderr}");
|
||||
let start = format!("memoryd: {}:1: ", home.ledger().display());
|
||||
assert!(stderr.starts_with(&start), "{stderr}");
|
||||
assert!(
|
||||
stderr.ends_with("; see docs/runbook.md#honcho-ledger-damaged\n"),
|
||||
"{stderr}"
|
||||
);
|
||||
assert_eq!(stderr.lines().count(), 1, "{stderr}");
|
||||
assert!(!home.config.loop_socket().exists(), "a socket was bound");
|
||||
}
|
||||
@@ -0,0 +1,380 @@
|
||||
//! The Honcho worker, stepped by hand against the fake Honcho (M5c spec, section 5): a turn sent
|
||||
//! once as one batch, the speaker's parts then the agent's, neutralised, in parts of 25,000
|
||||
//! characters, to the `-2` session for a new speaker; an outage waited out with a doubling wait
|
||||
//! and logged twice in all; a refused send marked failed; a refused key stopping its workspace
|
||||
//! only; a turn the log no longer holds marked failed; other actions left waiting. The waits and
|
||||
//! a restart are in `honcho_worker_more.rs`. Do not edit.
|
||||
|
||||
#[path = "support/honcho.rs"]
|
||||
mod honcho;
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
#[path = "support/worker.rs"]
|
||||
mod worker;
|
||||
|
||||
use std::time::Duration;
|
||||
|
||||
use bxcore::sanitize::TAGS;
|
||||
use bxcore::{DataClass, SessionStateReply};
|
||||
use honcho::{FakeHoncho, Mode, Request};
|
||||
use logs::assistant;
|
||||
use memoryd::honcho::ledger::{Item, Record};
|
||||
use memoryd::honcho::worker::{MAX_BATCH, MAX_PART, Step, parts};
|
||||
use serde_json::{Value, json};
|
||||
use turns::{sid, spoken};
|
||||
use worker::{FAST, Home};
|
||||
|
||||
/// The ids fixture 08 answers with.
|
||||
const IDS: [&str; 2] = ["7HqDWzXcKAdw9tnElCNFg", "mjsyhwuGsXTh1_zG_tWRx"];
|
||||
|
||||
fn message_posts(fake: &FakeHoncho) -> Vec<Request> {
|
||||
fake.requests()
|
||||
.into_iter()
|
||||
.filter(|r| r.path.ends_with("/messages"))
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn turn(text: &str, speaker: &str, reply: &str) -> Vec<bxcore::LogRecord> {
|
||||
vec![spoken(text, Some(speaker)), assistant(2, Some(reply), &[])]
|
||||
}
|
||||
|
||||
fn meta(session: &str, turn: u64) -> Value {
|
||||
json!({ "agent": "boxmaker0", "boxmaker_session": session, "turn": turn })
|
||||
}
|
||||
|
||||
fn sent(session: &str, turn: u64) -> impl Fn(&Record) -> bool {
|
||||
let item = Item::Turn {
|
||||
session: sid(session),
|
||||
turn,
|
||||
};
|
||||
move |r| matches!(r, Record::Sent { item: i, .. } if *i == item)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_limits() {
|
||||
assert_eq!(MAX_PART, 25_000);
|
||||
assert_eq!(MAX_BATCH, 100);
|
||||
assert!(parts("").is_empty());
|
||||
assert_eq!(parts("abc"), vec!["abc".to_string()]);
|
||||
let long = "é".repeat(MAX_PART);
|
||||
assert_eq!(parts(&long), vec![long.clone()], "characters, not bytes");
|
||||
let over = format!("{long}xy");
|
||||
assert_eq!(parts(&over), vec![long, "xy".to_string()]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_is_sent_once_as_one_batch() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-once");
|
||||
home.log("mm-a", &[turn("hello", "kyle", "hi")]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let posts = message_posts(&fake);
|
||||
assert_eq!(posts.len(), 1);
|
||||
assert_eq!(
|
||||
posts[0].path,
|
||||
"/v3/workspaces/boxmaker0-trusted/sessions/mm-a/messages"
|
||||
);
|
||||
assert_eq!(
|
||||
posts[0].body,
|
||||
Some(json!({ "messages": [
|
||||
{ "content": "hello", "peer_id": "kyle", "metadata": meta("mm-a", 1) },
|
||||
{ "content": "hi", "peer_id": "boxmaker0", "metadata": meta("mm-a", 1) },
|
||||
] }))
|
||||
);
|
||||
// The session was made first, with both peers.
|
||||
assert_eq!(fake.requests().len(), 5);
|
||||
assert!(fake.requests()[3].path.ends_with("/sessions"));
|
||||
let records = home.records();
|
||||
assert_eq!(records.len(), 2);
|
||||
assert!(matches!(&records[1], Record::Sent { ids, .. } if *ids == IDS));
|
||||
fake.clear();
|
||||
assert_eq!(w.step(), Step::Idle);
|
||||
assert!(fake.requests().is_empty());
|
||||
assert!(home.lines().is_empty(), "{:?}", home.lines());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_text_over_the_limit_is_sent_in_parts() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-parts");
|
||||
let text = "a".repeat(MAX_PART + 1);
|
||||
let reply = "é".repeat(MAX_PART);
|
||||
home.log("mm-a", &[turn(&text, "kyle", &reply)]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let body = message_posts(&fake)[0].body.clone().unwrap();
|
||||
let got: Vec<(usize, String)> = body["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|m| {
|
||||
(
|
||||
m["content"].as_str().unwrap().chars().count(),
|
||||
m["peer_id"].as_str().unwrap().to_string(),
|
||||
)
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
got,
|
||||
vec![
|
||||
(MAX_PART, "kyle".to_string()),
|
||||
(1, "kyle".to_string()),
|
||||
(MAX_PART, "boxmaker0".to_string()),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn both_texts_are_neutralised() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-neutral");
|
||||
let text = format!("a{}b", TAGS[0].text);
|
||||
let reply = format!("c{}d", TAGS[1].text);
|
||||
home.log("mm-a", &[turn(&text, "kyle", &reply)]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let body = message_posts(&fake)[0].body.clone().unwrap();
|
||||
let contents: Vec<String> = body["messages"]
|
||||
.as_array()
|
||||
.unwrap()
|
||||
.iter()
|
||||
.map(|m| m["content"].as_str().unwrap().to_string())
|
||||
.collect();
|
||||
assert_eq!(contents.len(), 2);
|
||||
for (content, tag) in contents.iter().zip([TAGS[0].text, TAGS[1].text]) {
|
||||
assert!(!content.contains(tag));
|
||||
assert!(content.contains("<"), "not neutralised");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_new_speaker_goes_to_the_next_session() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-second");
|
||||
home.log(
|
||||
"mm-a",
|
||||
&[turn("one", "kyle", "a"), turn("two", "ana_b", "b")],
|
||||
);
|
||||
home.report("mm-a", 2);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let posts = message_posts(&fake);
|
||||
assert_eq!(posts.len(), 2);
|
||||
assert!(posts[0].path.ends_with("/sessions/mm-a/messages"));
|
||||
assert!(posts[1].path.ends_with("/sessions/mm-a-2/messages"));
|
||||
let second = fake
|
||||
.requests()
|
||||
.into_iter()
|
||||
.find(|r| r.path.ends_with("/sessions") && r.body.as_ref().unwrap()["id"] == "mm-a-2")
|
||||
.unwrap();
|
||||
let peers = second.body.unwrap()["peers"].clone();
|
||||
assert_eq!(peers["kyle"]["observe_me"], true);
|
||||
assert_eq!(peers["ana_b"]["observe_me"], true);
|
||||
assert_eq!(peers["boxmaker0"]["observe_me"], false);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_outage_is_waited_out_and_logged_twice_in_all() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-outage");
|
||||
home.log("mm-a", &[turn("private-alpha", "kyle", "a")]);
|
||||
home.log("mm-b", &[turn("private-beta", "kyle", "b")]);
|
||||
home.report("mm-a", 1);
|
||||
home.report("mm-b", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(503));
|
||||
let waits: Vec<Step> = (0..6).map(|_| w.step()).collect();
|
||||
let ms = |n| Step::Backoff(Duration::from_millis(n));
|
||||
assert_eq!(waits, vec![ms(1), ms(2), ms(4), ms(8), ms(8), ms(8)]);
|
||||
assert_eq!(home.lines().len(), 1, "{:?}", home.lines());
|
||||
fake.set(Mode::Fixtures);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let lines = home.lines();
|
||||
assert_eq!(lines.len(), 2, "{lines:?}");
|
||||
assert_eq!(
|
||||
lines[0],
|
||||
"memoryd: Honcho is unreachable (server, HTTP 503); sends wait and are retried; see docs/runbook.md#honcho-unreachable"
|
||||
);
|
||||
assert_eq!(
|
||||
lines[1],
|
||||
"memoryd: Honcho answers again; waiting sends go on"
|
||||
);
|
||||
// In order: mm-a, then mm-b.
|
||||
let records = home.records();
|
||||
let a = records.iter().position(sent("mm-a", 1)).unwrap();
|
||||
let b = records.iter().position(sent("mm-b", 1)).unwrap();
|
||||
assert!(a < b);
|
||||
// A new outage starts from the first wait again.
|
||||
home.log("mm-c", &[turn("c", "kyle", "c")]);
|
||||
home.report("mm-c", 1);
|
||||
fake.set(Mode::Hang);
|
||||
assert_eq!(w.step(), ms(1));
|
||||
assert!(home.lines()[2].contains("(timeout)"), "{:?}", home.lines());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_send_is_marked_failed_and_the_next_goes_on() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-refused");
|
||||
home.log("mm-a", &[turn("private-alpha", "kyle", "a")]);
|
||||
home.log("mm-b", &[turn("private-beta", "kyle", "b")]);
|
||||
home.report("mm-a", 1);
|
||||
home.report("mm-b", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(422));
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let failed = home.records().into_iter().find_map(|r| match r {
|
||||
Record::Failed {
|
||||
item, status, kind, ..
|
||||
} => Some((item, status, kind)),
|
||||
_ => None,
|
||||
});
|
||||
assert_eq!(
|
||||
failed,
|
||||
Some((
|
||||
Item::Turn {
|
||||
session: sid("mm-a"),
|
||||
turn: 1
|
||||
},
|
||||
Some(422),
|
||||
"status".to_string()
|
||||
))
|
||||
);
|
||||
let lines = home.lines();
|
||||
assert_eq!(
|
||||
lines,
|
||||
vec![
|
||||
"memoryd: turn 1 of session mm-a was not sent: Honcho refused the request (status, HTTP 422); it is marked failed; see docs/runbook.md#honcho-send-refused"
|
||||
.to_string()
|
||||
]
|
||||
);
|
||||
fake.set(Mode::Fixtures);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert!(home.records().iter().any(sent("mm-b", 1)));
|
||||
assert_eq!(w.step(), Step::Idle, "the failed turn is not tried again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_key_stops_its_workspace_only() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-key");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.log("mm-b", &[turn("b", "kyle", "b")]);
|
||||
home.report("mm-a", 1);
|
||||
*home.answer.lock().unwrap() = Ok(SessionStateReply {
|
||||
class: DataClass::Private,
|
||||
untrusted: true,
|
||||
});
|
||||
home.report("mm-b", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(401));
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert_eq!(
|
||||
home.lines(),
|
||||
vec![
|
||||
"memoryd: Honcho refused the key of workspace boxmaker0-trusted (HTTP 401); nothing more is sent to it until memoryd restarts; see docs/runbook.md#honcho-key-refused"
|
||||
.to_string()
|
||||
]
|
||||
);
|
||||
fake.set(Mode::Fixtures);
|
||||
fake.clear();
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert!(home.records().iter().any(sent("mm-b", 1)));
|
||||
assert!(
|
||||
fake.requests()
|
||||
.iter()
|
||||
.all(|r| !r.path.contains("boxmaker0-trusted"))
|
||||
);
|
||||
assert_eq!(w.step(), Step::Idle);
|
||||
// The trusted turn waits: neither sent nor failed.
|
||||
let sync = home.sync.lock().unwrap();
|
||||
assert_eq!(sync.ledger().outbox().len(), 1);
|
||||
assert_eq!(sync.ledger().counts().failed, 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_the_log_no_longer_holds_is_marked_failed() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-gone");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
let path = home.dir.path().join("sessions/mm-a/0.jsonl");
|
||||
std::fs::write(&path, "not json\n").unwrap();
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert!(matches!(
|
||||
home.records().last(),
|
||||
Some(Record::Failed { status: None, kind, .. }) if kind == "log"
|
||||
));
|
||||
assert_eq!(
|
||||
home.lines(),
|
||||
vec![
|
||||
"memoryd: turn 1 of session mm-a cannot be sent (log); it is marked failed; see docs/runbook.md#session-log-damaged"
|
||||
.to_string()
|
||||
]
|
||||
);
|
||||
assert!(message_posts(&fake).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_without_its_reply_any_more_is_marked_failed() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-noreply");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
// The log now ends in a tool round: the reply is gone.
|
||||
home.log("mm-a", &[turns::running(Some("kyle"))]);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert!(matches!(
|
||||
home.records().last(),
|
||||
Some(Record::Failed { status: None, kind, .. }) if kind == "log"
|
||||
));
|
||||
assert!(message_posts(&fake).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_refused_key_ends_an_outage() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-key-outage");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(503));
|
||||
assert!(matches!(w.step(), Step::Backoff(_)));
|
||||
fake.set(Mode::Status(403));
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
let lines = home.lines();
|
||||
assert_eq!(lines.len(), 3, "{lines:?}");
|
||||
assert_eq!(
|
||||
lines[1],
|
||||
"memoryd: Honcho answers again; waiting sends go on"
|
||||
);
|
||||
assert!(lines[2].ends_with("see docs/runbook.md#honcho-key-refused"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_turn_too_long_for_one_batch_is_marked_failed() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-long");
|
||||
let text = "a".repeat(MAX_PART * MAX_BATCH);
|
||||
home.log("mm-a", &[turn(&text, "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
assert!(matches!(
|
||||
home.records().last(),
|
||||
Some(Record::Failed { status: None, kind, .. }) if kind == "too-long"
|
||||
));
|
||||
assert!(fake.requests().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,327 @@
|
||||
//! The Honcho worker's other duties (M5c spec, section 5): with nothing to send it decides what
|
||||
//! `brokerd` could not answer for; after a restart nothing is sent again, and a session with a
|
||||
//! decided turn waits again (a session without one does not); a running turn is never decided;
|
||||
//! actions of later tasks wait; a ledger that cannot be written stops it; no log line holds a
|
||||
//! turn's text. Then `memoryd` itself: a reported turn reaches Honcho, woken at once, and a key
|
||||
//! that cannot be read stops the start. Do not edit.
|
||||
|
||||
#[path = "support/brokerd.rs"]
|
||||
mod brokerd;
|
||||
#[path = "support/honcho.rs"]
|
||||
mod honcho;
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
#[path = "support/turns.rs"]
|
||||
mod turns;
|
||||
#[path = "support/worker.rs"]
|
||||
mod worker;
|
||||
|
||||
use std::process::{Command, Stdio};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use brokerd::{Answer, FakeBrokerd};
|
||||
use bxcore::{Envelope, Message, SessionId, SessionUpdated, read_frame, write_frame};
|
||||
use honcho::{FakeHoncho, Mode};
|
||||
use logs::assistant;
|
||||
use memoryd::config::Config;
|
||||
use memoryd::honcho::ledger::{Item, Record};
|
||||
use memoryd::honcho::worker::{Step, Tuning};
|
||||
use memoryd::serve::{self, StartError};
|
||||
use tmp::TempDir;
|
||||
use turns::{clean, running, sid, spoken};
|
||||
use worker::{FAST, Home};
|
||||
|
||||
fn turn(text: &str, speaker: &str, reply: &str) -> Vec<bxcore::LogRecord> {
|
||||
vec![spoken(text, Some(speaker)), assistant(2, Some(reply), &[])]
|
||||
}
|
||||
|
||||
fn messages(fake: &FakeHoncho) -> usize {
|
||||
fake.requests()
|
||||
.iter()
|
||||
.filter(|r| r.path.ends_with("/messages"))
|
||||
.count()
|
||||
}
|
||||
|
||||
fn is_sent(records: &[Record], session: &str, turn: u64) -> bool {
|
||||
let item = Item::Turn {
|
||||
session: sid(session),
|
||||
turn,
|
||||
};
|
||||
records
|
||||
.iter()
|
||||
.any(|r| matches!(r, Record::Sent { item: i, .. } if *i == item))
|
||||
}
|
||||
|
||||
fn decided(records: &[Record], session: &str, turn: u64) -> bool {
|
||||
let item = Item::Turn {
|
||||
session: sid(session),
|
||||
turn,
|
||||
};
|
||||
records
|
||||
.iter()
|
||||
.any(|r| matches!(r, Record::Decided { item: i, .. } if *i == item))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_default_waits() {
|
||||
assert_eq!(
|
||||
Tuning::DEFAULT,
|
||||
Tuning {
|
||||
backoff: Duration::from_secs(5),
|
||||
backoff_max: Duration::from_secs(600),
|
||||
idle: Duration::from_secs(30),
|
||||
}
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn with_nothing_to_send_undecided_turns_are_decided() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-retry");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
*home.answer.lock().unwrap() = Err("brokerd is down".to_string());
|
||||
home.report("mm-a", 1);
|
||||
assert!(!decided(&home.records(), "mm-a", 1));
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Idle, "brokerd still down");
|
||||
*home.answer.lock().unwrap() = Ok(clean());
|
||||
assert_eq!(w.step(), Step::Done, "decided now");
|
||||
assert!(decided(&home.records(), "mm-a", 1));
|
||||
assert_eq!(w.step(), Step::Done, "sent");
|
||||
assert!(is_sent(&home.records(), "mm-a", 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_outage_that_ends_is_followed_by_the_retry() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-after-outage");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.log("mm-b", &[turn("b", "kyle", "b")]);
|
||||
home.report("mm-a", 1);
|
||||
*home.answer.lock().unwrap() = Err("brokerd is down".to_string());
|
||||
home.report("mm-b", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(502));
|
||||
assert!(matches!(w.step(), Step::Backoff(_)));
|
||||
*home.answer.lock().unwrap() = Ok(clean());
|
||||
fake.set(Mode::Fixtures);
|
||||
assert_eq!(w.step(), Step::Done);
|
||||
// mm-a is sent, and the outage's end decided mm-b before any idle step.
|
||||
assert!(decided(&home.records(), "mm-b", 1));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_restart_sends_nothing_again_and_resumes_the_sessions_it_knows() {
|
||||
let fake = FakeHoncho::start();
|
||||
let mut home = Home::new("worker-restart");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
home.worker(&fake, FAST).step();
|
||||
assert_eq!(messages(&fake), 1);
|
||||
// While memoryd is down: mm-a has a second turn, never reported; mm-z, never seen, one.
|
||||
home.log("mm-a", &[turn("a", "kyle", "a"), turn("b", "kyle", "b")]);
|
||||
home.log("mm-z", &[turn("z", "kyle", "z")]);
|
||||
home.restart();
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Done, "mm-a's second turn decided");
|
||||
assert_eq!(w.step(), Step::Done, "and sent");
|
||||
assert_eq!(w.step(), Step::Idle);
|
||||
let records = home.records();
|
||||
assert!(is_sent(&records, "mm-a", 2));
|
||||
assert!(!decided(&records, "mm-z", 1), "a session never reported");
|
||||
assert_eq!(messages(&fake), 2, "turn 1 was sent again");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_restart_never_decides_a_running_turn() {
|
||||
let fake = FakeHoncho::start();
|
||||
let mut home = Home::new("worker-running");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
home.worker(&fake, FAST).step();
|
||||
home.log("mm-a", &[turn("a", "kyle", "a"), running(Some("kyle"))]);
|
||||
home.restart();
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Idle);
|
||||
assert!(!decided(&home.records(), "mm-a", 2));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn actions_of_later_tasks_wait() {
|
||||
let fake = FakeHoncho::start();
|
||||
let mut home = Home::new("worker-later");
|
||||
let conclude = r#"{"type":"decided","time":"2026-09-28T10:00:00.000Z","item":{"type":"edit","id":"2026-09-28-0001","part":0},"action":{"type":"conclude","workspace":"trusted","peer":"kyle","session":"mm-a","text":"a fact"}}"#;
|
||||
std::fs::write(home.ledger_path(), format!("{conclude}\n")).unwrap();
|
||||
home.restart();
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
assert_eq!(w.step(), Step::Idle);
|
||||
assert_eq!(home.records().len(), 1, "the conclusion was failed or sent");
|
||||
assert!(fake.requests().is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_ledger_that_cannot_be_written_stops_the_worker() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-ledger");
|
||||
home.log("mm-a", &[turn("a", "kyle", "a")]);
|
||||
home.report("mm-a", 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
let ledger = home.ledger_path();
|
||||
let kept = std::fs::read_to_string(&ledger).unwrap();
|
||||
std::fs::remove_file(&ledger).unwrap();
|
||||
std::fs::create_dir(&ledger).unwrap();
|
||||
assert_eq!(w.step(), Step::Stopped);
|
||||
assert_eq!(w.step(), Step::Stopped);
|
||||
assert_eq!(
|
||||
messages(&fake),
|
||||
1,
|
||||
"sent once; its sent line could not be written"
|
||||
);
|
||||
let lines = home.lines();
|
||||
assert_eq!(lines.len(), 1, "{lines:?}");
|
||||
assert!(lines[0].ends_with(
|
||||
"; the Honcho worker stops until memoryd restarts; see docs/runbook.md#honcho-ledger-damaged"
|
||||
));
|
||||
std::fs::remove_dir(&ledger).unwrap();
|
||||
std::fs::write(&ledger, kept).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_log_line_holds_a_turns_text() {
|
||||
let fake = FakeHoncho::start();
|
||||
let home = Home::new("worker-quiet");
|
||||
for (session, status) in [("mm-a", 503), ("mm-b", 422), ("mm-c", 401)] {
|
||||
home.log(
|
||||
session,
|
||||
&[turn("private-question", "kyle", "private-answer")],
|
||||
);
|
||||
home.report(session, 1);
|
||||
let mut w = home.worker(&fake, FAST);
|
||||
fake.set(Mode::Status(status));
|
||||
w.step();
|
||||
}
|
||||
let lines = home.lines();
|
||||
assert_eq!(lines.len(), 3, "{lines:?}");
|
||||
for line in lines {
|
||||
assert!(!line.contains("private"), "{line}");
|
||||
}
|
||||
}
|
||||
|
||||
/// `memoryd serve`'s config for `home`, with `[honcho]` at `url`.
|
||||
fn serve_config(home: &TempDir, url: &str, extra: &str) -> String {
|
||||
let soul = home.write("etc/soul.md", "You are Boxmaker.\n");
|
||||
format!(
|
||||
"[paths]\nhome = \"{}\"\n[memory]\nsoul_starter = \"{}\"\n[honcho]\nurl = \"{url}\"\nagent = \"boxmaker0\"\nowner = \"kyle\"\ntrusted = \"boxmaker0-trusted\"\nuntrusted = \"boxmaker0-untrusted\"\nfleet = \"fleet\"\n{extra}",
|
||||
home.path().display(),
|
||||
soul.display()
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn memoryd_sends_a_reported_turn_at_once() {
|
||||
let fake = FakeHoncho::start();
|
||||
let dir = TempDir::new("worker-serve");
|
||||
turns::write_turns(dir.path(), "mm-a", &[turn("a", "kyle", "a")]);
|
||||
let config = Config::parse(&serve_config(&dir, &fake.url, "")).unwrap();
|
||||
let _brokerd = FakeBrokerd::start(&config.state_socket(), Answer::State(clean()));
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&lines);
|
||||
let log: serve::Log = Arc::new(move |l: &str| sink.lock().unwrap().push(l.to_string()));
|
||||
let running = serve::start(config.clone(), log).unwrap();
|
||||
std::thread::spawn(move || running.run());
|
||||
// The worker has slept since start (30 s when idle): the report wakes it.
|
||||
std::thread::sleep(Duration::from_millis(100));
|
||||
let mut stream = std::os::unix::net::UnixStream::connect(config.loop_socket()).unwrap();
|
||||
stream
|
||||
.set_read_timeout(Some(Duration::from_secs(10)))
|
||||
.unwrap();
|
||||
let report = Envelope {
|
||||
v: 1,
|
||||
id: 1,
|
||||
r#final: true,
|
||||
msg: Message::SessionUpdated(SessionUpdated {
|
||||
session: SessionId::new("mm-a").unwrap(),
|
||||
turn: 1,
|
||||
}),
|
||||
};
|
||||
write_frame(&mut stream, &report).unwrap();
|
||||
assert!(matches!(
|
||||
read_frame(&mut stream).unwrap().msg,
|
||||
Message::Ok(_)
|
||||
));
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline && messages(&fake) == 0 {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
assert_eq!(messages(&fake), 1, "{:?}", lines.lock().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_key_that_cannot_be_read_stops_the_start() {
|
||||
let fake = FakeHoncho::start();
|
||||
let dir = TempDir::new("worker-key-start");
|
||||
let text = serve_config(
|
||||
&dir,
|
||||
&fake.url,
|
||||
"[secrets.honcho_trusted]\nenv = \"MEMORYD_TEST_UNSET_KEY\"\n",
|
||||
);
|
||||
let config = Config::parse(&text).unwrap();
|
||||
let log: serve::Log = Arc::new(|_: &str| {});
|
||||
let err = match serve::start(config.clone(), log) {
|
||||
Err(err) => err,
|
||||
Ok(_) => panic!("memoryd started"),
|
||||
};
|
||||
assert!(matches!(err, StartError::Secret(_)), "{err:?}");
|
||||
assert!(!config.loop_socket().exists(), "a socket was bound");
|
||||
// The process: exit 1 with the secret's own lines and pointer, not the start pointer.
|
||||
let path = dir.write("memoryd.toml", &text);
|
||||
let mut child = Daemon(
|
||||
Command::new(env!("CARGO_BIN_EXE_memoryd"))
|
||||
.args(["serve", "--config"])
|
||||
.arg(&path)
|
||||
.env_remove("MEMORYD_TEST_UNSET_KEY")
|
||||
.stdin(Stdio::null())
|
||||
.stdout(Stdio::null())
|
||||
.stderr(Stdio::piped())
|
||||
.spawn()
|
||||
.unwrap(),
|
||||
);
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
let mut code = None;
|
||||
while Instant::now() < deadline {
|
||||
if let Ok(Some(status)) = child.0.try_wait() {
|
||||
code = status.code();
|
||||
break;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
let _ = child.0.kill();
|
||||
let _ = child.0.wait();
|
||||
let mut stderr = String::new();
|
||||
if let Some(mut pipe) = child.0.stderr.take() {
|
||||
let _ = std::io::Read::read_to_string(&mut pipe, &mut stderr);
|
||||
}
|
||||
assert_eq!(code, Some(1), "{stderr}");
|
||||
assert!(
|
||||
stderr.starts_with("memoryd: secret honcho_trusted: "),
|
||||
"{stderr}"
|
||||
);
|
||||
assert!(
|
||||
stderr.ends_with("\nsee docs/runbook.md#secret-unavailable\n"),
|
||||
"{stderr}"
|
||||
);
|
||||
assert!(!stderr.contains("memoryd-start-failed"), "{stderr}");
|
||||
}
|
||||
|
||||
/// A `memoryd` process, killed and waited for when dropped.
|
||||
struct Daemon(std::process::Child);
|
||||
|
||||
impl Drop for Daemon {
|
||||
fn drop(&mut self) {
|
||||
let _ = self.0.kill();
|
||||
let _ = self.0.wait();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,133 @@
|
||||
//! A Honcho worker for tests, stepped by hand: a home with session logs and a ledger, the fake
|
||||
//! Honcho, `brokerd` as a closure whose answer the test sets, and the log lines collected. Do not
|
||||
//! edit.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::ffi::OsString;
|
||||
use std::path::PathBuf;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::Duration;
|
||||
|
||||
use bxcore::{LogRecord, SessionStateReply, Timestamp};
|
||||
use memoryd::honcho::client::{Clients, Timeouts};
|
||||
use memoryd::honcho::config::HonchoConfig;
|
||||
use memoryd::honcho::ledger::{Ledger, Record};
|
||||
use memoryd::honcho::turns::{LEDGER, TurnSync};
|
||||
use memoryd::honcho::worker::{Tuning, Wake, Worker};
|
||||
use memoryd::serve::Log;
|
||||
|
||||
use crate::honcho::FakeHoncho;
|
||||
use crate::tmp::TempDir;
|
||||
use crate::turns::{clean, sid, write_turns};
|
||||
|
||||
pub const FAST: Tuning = Tuning {
|
||||
backoff: Duration::from_millis(1),
|
||||
backoff_max: Duration::from_millis(8),
|
||||
idle: Duration::from_millis(1),
|
||||
};
|
||||
|
||||
pub fn config(url: &str) -> HonchoConfig {
|
||||
HonchoConfig {
|
||||
url: url.to_string(),
|
||||
agent: "boxmaker0".to_string(),
|
||||
owner: "kyle".to_string(),
|
||||
trusted: "boxmaker0-trusted".to_string(),
|
||||
untrusted: "boxmaker0-untrusted".to_string(),
|
||||
fleet: "fleet".to_string(),
|
||||
read: Vec::new(),
|
||||
keys: BTreeMap::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub struct Home {
|
||||
pub sync: Arc<Mutex<TurnSync>>,
|
||||
/// What `brokerd` answers.
|
||||
pub answer: Arc<Mutex<Result<SessionStateReply, String>>>,
|
||||
pub lines: Arc<Mutex<Vec<String>>>,
|
||||
pub dir: TempDir,
|
||||
}
|
||||
|
||||
impl Home {
|
||||
pub fn new(tag: &str) -> Home {
|
||||
let dir = TempDir::new(tag);
|
||||
std::fs::create_dir_all(dir.path().join("memory")).unwrap();
|
||||
Home {
|
||||
sync: Arc::new(Mutex::new(TurnSync::new(
|
||||
Ledger::open(&dir.path().join("memory").join(LEDGER)).unwrap(),
|
||||
"boxmaker0",
|
||||
&dir.path().join("sessions"),
|
||||
))),
|
||||
answer: Arc::new(Mutex::new(Ok(clean()))),
|
||||
lines: Arc::new(Mutex::new(Vec::new())),
|
||||
dir,
|
||||
}
|
||||
}
|
||||
|
||||
pub fn ledger_path(&self) -> PathBuf {
|
||||
self.dir.path().join("memory").join(LEDGER)
|
||||
}
|
||||
|
||||
/// A new `TurnSync` over the ledger as it is on disk, resumed: what a restart makes.
|
||||
pub fn restart(&mut self) {
|
||||
let mut sync = TurnSync::new(
|
||||
Ledger::open(&self.ledger_path()).unwrap(),
|
||||
"boxmaker0",
|
||||
&self.dir.path().join("sessions"),
|
||||
);
|
||||
sync.resume();
|
||||
self.sync = Arc::new(Mutex::new(sync));
|
||||
}
|
||||
|
||||
pub fn log(&self, session: &str, turns: &[Vec<LogRecord>]) {
|
||||
write_turns(self.dir.path(), session, turns);
|
||||
}
|
||||
|
||||
/// `loopd` reports turn `turn` of `session`: decided now, with `answer`.
|
||||
pub fn report(&self, session: &str, turn: u64) {
|
||||
let answer = Arc::clone(&self.answer);
|
||||
let ask = move |_: &bxcore::SessionId| answer.lock().unwrap().clone();
|
||||
let lines = Arc::clone(&self.lines);
|
||||
let log = move |line: &str| lines.lock().unwrap().push(line.to_string());
|
||||
let now = Timestamp::parse("2026-09-28T10:00:00.000Z").unwrap();
|
||||
self.sync
|
||||
.lock()
|
||||
.unwrap()
|
||||
.updated(&sid(session), turn, &ask, now, &log);
|
||||
}
|
||||
|
||||
pub fn worker(&self, fake: &FakeHoncho, tuning: Tuning) -> Worker {
|
||||
let no_env = |_: &str| -> Option<OsString> { None };
|
||||
let timeouts = Timeouts {
|
||||
connect: Duration::from_secs(5),
|
||||
read: Duration::from_millis(300),
|
||||
};
|
||||
let (clients, _) = Clients::new(&config(&fake.url), &no_env, timeouts).unwrap();
|
||||
let answer = Arc::clone(&self.answer);
|
||||
let lines = Arc::clone(&self.lines);
|
||||
let log: Log = Arc::new(move |line: &str| lines.lock().unwrap().push(line.to_string()));
|
||||
Worker::new(
|
||||
Arc::clone(&self.sync),
|
||||
clients,
|
||||
Arc::new(Wake::default()),
|
||||
Box::new(move |_: &bxcore::SessionId| answer.lock().unwrap().clone()),
|
||||
log,
|
||||
tuning,
|
||||
"boxmaker0",
|
||||
)
|
||||
}
|
||||
|
||||
/// Every record of the ledger on disk, in order.
|
||||
pub fn records(&self) -> Vec<Record> {
|
||||
std::fs::read_to_string(self.ledger_path())
|
||||
.unwrap_or_default()
|
||||
.lines()
|
||||
.map(|l| serde_json::from_str(l).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub fn lines(&self) -> Vec<String> {
|
||||
self.lines.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user