diff --git a/crates/memoryd/src/honcho/turns.rs b/crates/memoryd/src/honcho/turns.rs index c576277..636f4a9 100644 --- a/crates/memoryd/src/honcho/turns.rs +++ b/crates/memoryd/src/honcho/turns.rs @@ -52,7 +52,8 @@ pub struct TurnSync { status: Shared, } -/// The path of `session`'s log under `sessions`: epoch 0, the only one `loopd` writes. +/// The path of `session`'s epoch-0 log under `sessions`. Later epochs are sibling files; +/// `read_session` concatenates them. Existing tests pin this path. pub fn log_path(sessions: &Path, session: &SessionId) -> PathBuf { sessions.join(session.as_str()).join("0.jsonl") } @@ -115,6 +116,41 @@ pub fn read_records(path: &Path) -> Result, LogError> { Ok(records) } +/// Every `.jsonl` of `session`, sorted by `n`. A missing directory is empty. +pub fn epoch_logs(sessions: &Path, session: &SessionId) -> Result, LogError> { + let dir = sessions.join(session.as_str()); + let entries = match std::fs::read_dir(&dir) { + Ok(e) => e, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(LogError::Io(format!("{}: {e}", dir.display()))), + }; + let mut found: Vec<(u32, PathBuf)> = Vec::new(); + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(e) => return Err(LogError::Io(format!("{}: {e}", dir.display()))), + }; + let name = entry.file_name(); + let Some(name) = name.to_str() else { continue }; + let Some(stem) = name.strip_suffix(".jsonl") else { + continue; + }; + let Ok(n) = stem.parse::() else { continue }; + found.push((n, entry.path())); + } + found.sort_by_key(|(n, _)| *n); + Ok(found.into_iter().map(|(_, p)| p).collect()) +} + +/// Every epoch's records, in epoch order. A missing session has none. +pub fn read_session(sessions: &Path, session: &SessionId) -> Result, LogError> { + let mut out = Vec::new(); + for path in epoch_logs(sessions, session)? { + out.extend(read_records(&path)?); + } + Ok(out) +} + /// Whether a turn has ended with an answer: its last `Assistant` record asks for no tool. pub fn finished(turn: &[LogRecord]) -> bool { let last = turn @@ -178,7 +214,7 @@ impl TurnSync { } } for session in sessions { - if let Ok(records) = read_log(&log_path(&self.sessions, &session)) { + if let Ok(records) = read_session(&self.sessions, &session) { self.reported.insert( session.as_str().to_string(), (session.clone(), turns(&records)), @@ -271,7 +307,7 @@ impl TurnSync { /// `session`'s log, read. A log that does not read is logged once, until it reads again. fn records(&mut self, session: &SessionId, log: &dyn Fn(&str)) -> Option> { - match read_log(&log_path(&self.sessions, session)) { + match read_session(&self.sessions, session).map_err(|e| e.to_string()) { Ok(records) => { self.unreadable.remove(session.as_str()); Some(records) diff --git a/crates/memoryd/src/honcho/worker.rs b/crates/memoryd/src/honcho/worker.rs index 164587c..5fa6bd4 100644 --- a/crates/memoryd/src/honcho/worker.rs +++ b/crates/memoryd/src/honcho/worker.rs @@ -13,11 +13,12 @@ use std::time::Duration; use bxcore::{SessionId, SessionStateReply, Timestamp}; use crate::honcho::client::{Clients, HonchoError, classify}; +use crate::honcho::decide::{final_reply, turn_records, user_text}; use crate::honcho::edits; use crate::honcho::ledger::{self, Action, Item, Ledger, Record, Workspace}; use crate::honcho::naming::{HonchoSession, honcho_session}; use crate::honcho::status::{self, Shared}; -use crate::honcho::turns::{TurnSync, log_path}; +use crate::honcho::turns::{LogError, TurnSync, log_path, read_session}; use crate::serve::Log; use crate::honcho::messages::ids; @@ -338,11 +339,19 @@ impl Worker { honcho: Option<&HonchoSession>, ) -> Result, Unsent> { let path = log_path(&self.sessions, session); - let read = read_turn(&path, turn).map_err(Unsent::Unreadable)?; - self.readable(&path); - let (text, reply) = read.ok_or(Unsent::Local("log"))?; + let records = match read_session(&self.sessions, session) { + Ok(records) => { + self.readable(&path); + records + } + Err(LogError::Damaged(_)) => return Err(Unsent::Local("log")), + Err(e) => return Err(Unsent::Unreadable(e.to_string())), + }; + let slice = turn_records(&records, turn).ok_or(Unsent::Local("log"))?; + let (text, _) = user_text(slice); + let reply = final_reply(slice).ok_or(Unsent::Local("log"))?; let honcho = honcho.ok_or(Unsent::Local("log"))?; - let messages = batch(&text, &reply, speaker, &self.agent, session, turn); + let messages = batch(text, reply, speaker, &self.agent, session, turn); if messages.len() > MAX_BATCH { return Err(Unsent::Local("too-long")); } diff --git a/crates/memoryd/tests/honcho_epochs.rs b/crates/memoryd/tests/honcho_epochs.rs new file mode 100644 index 0000000..0b00eb8 --- /dev/null +++ b/crates/memoryd/tests/honcho_epochs.rs @@ -0,0 +1,225 @@ +//! Honcho reads every epoch file of a session, in order. Do not edit. + +#[path = "support/honcho.rs"] +mod honcho; +#[path = "support/logs.rs"] +mod logs; +#[path = "support/tmp.rs"] +mod tmp; +#[path = "support/turns.rs"] +mod turns; +#[path = "support/worker.rs"] +mod worker; + +use std::cell::RefCell; + +use bxcore::{Epoch, LogRecord, SessionId, Timestamp}; +use honcho::FakeHoncho; +use logs::{assistant, write_log}; +use memoryd::honcho::decide::{turn_records, turns}; +use memoryd::honcho::ledger::{Action, Item, Ledger, Record, Workspace}; +use memoryd::honcho::turns::{LOG_RUNBOOK, TurnSync, read_session}; +use memoryd::honcho::worker::Step; +use tmp::TempDir; +use turns::{clean, sid, spoken}; +use worker::{FAST, Home as WorkerHome}; + +const LEDGER: &str = "honcho.jsonl"; + +fn epoch_end(summary: &str) -> LogRecord { + LogRecord::EpochEnd { + time: Timestamp::parse("2026-09-28T10:00:00.000Z").unwrap(), + next: Epoch(1), + summary: summary.to_string(), + } +} + +fn write_epoch(home: &std::path::Path, session: &str, epoch: u32, records: &[LogRecord]) { + let dir = home.join("sessions").join(session); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{epoch}.jsonl")); + let text: String = records + .iter() + .map(|r| format!("{}\n", serde_json::to_string(r).unwrap())) + .collect(); + std::fs::write(path, text).unwrap(); +} + +struct World { + sync: TurnSync, + answer: RefCell>, + dir: TempDir, +} + +impl World { + fn new(tag: &str) -> World { + let dir = TempDir::new(tag); + std::fs::create_dir_all(dir.path().join("memory")).unwrap(); + let mut ledger = Ledger::open(&dir.path().join("memory").join(LEDGER)).unwrap(); + ledger + .append(Record::Began { + time: Timestamp::parse("2026-09-28T09:00:00.000Z").unwrap(), + }) + .unwrap(); + World { + sync: TurnSync::new(ledger, "boxmaker0", &dir.path().join("sessions")), + answer: RefCell::new(Ok(clean())), + dir, + } + } + + fn updated(&mut self, session: &str, turn: u64) { + let ask = |s: &SessionId| { + let _ = s; + self.answer.borrow().clone() + }; + let log = |_line: &str| {}; + let now = Timestamp::parse("2026-09-28T10:00:00.000Z").unwrap(); + self.sync.updated(&sid(session), turn, &ask, now, &log); + } + + fn action(&self, session: &str, turn: u64) -> Option { + let item = Item::Turn { + session: sid(session), + turn, + }; + let path = self.dir.path().join("memory").join(LEDGER); + let text = std::fs::read_to_string(path).unwrap_or_default(); + let key = serde_json::to_string(&item).unwrap(); + let line = text.lines().find(|l| l.contains(&key))?; + let value: serde_json::Value = serde_json::from_str(line).unwrap(); + Some(serde_json::from_value(value["action"].clone()).unwrap()) + } +} + +fn to(workspace: Workspace, speaker: &str) -> Action { + Action::Turn { + workspace, + speaker: speaker.to_string(), + } +} + +#[test] +fn concatenated_logs_count_three_turns() { + let dir = TempDir::new("honcho-epochs-count"); + let home = dir.path(); + let epoch0 = vec![ + spoken("one", Some("kyle")), + assistant(2, Some("ok"), &[]), + spoken("two", Some("kyle")), + assistant(3, Some("ok"), &[]), + epoch_end("sum"), + ]; + write_log(home, "mm-a", &epoch0); + write_epoch( + home, + "mm-a", + 1, + &[spoken("three", Some("kyle")), assistant(4, Some("ok"), &[])], + ); + let records = read_session(&home.join("sessions"), &sid("mm-a")).unwrap(); + assert_eq!(turns(&records), 3); + let turn3 = turn_records(&records, 3).unwrap(); + assert!( + matches!(turn3[0], LogRecord::User { ref content, .. } if content == "three"), + "{turn3:?}" + ); +} + +#[test] +fn epoch_0_alone_still_reads() { + let dir = TempDir::new("honcho-epochs-zero"); + let home = dir.path(); + write_log(home, "mm-a", &[spoken("only", Some("kyle"))]); + let records = read_session(&home.join("sessions"), &sid("mm-a")).unwrap(); + assert_eq!(turns(&records), 1); +} + +#[test] +fn a_damaged_later_epoch_is_an_error_naming_that_file() { + let dir = TempDir::new("honcho-epochs-damaged"); + let home = dir.path(); + write_log(home, "mm-a", &[spoken("one", Some("kyle"))]); + let path = home.join("sessions/mm-a/1.jsonl"); + std::fs::write(&path, "{\"type\":\"zz\"}\n").unwrap(); + let err = read_session(&home.join("sessions"), &sid("mm-a")).unwrap_err(); + assert!(err.to_string().contains("1.jsonl"), "{err}"); + assert!( + err.to_string().contains(LOG_RUNBOOK) || err.to_string().contains("session-log-damaged"), + "{err}" + ); +} + +#[test] +fn turn_sync_decides_a_turn_that_lives_in_epoch_1() { + let mut w = World::new("honcho-epochs-sync"); + write_log( + w.dir.path(), + "mm-a", + &[ + spoken("one", Some("kyle")), + assistant(2, Some("ok"), &[]), + spoken("two", Some("kyle")), + assistant(3, Some("ok"), &[]), + epoch_end("sum"), + ], + ); + write_epoch( + w.dir.path(), + "mm-a", + 1, + &[spoken("three", Some("kyle")), assistant(4, Some("ok"), &[])], + ); + w.updated("mm-a", 3); + assert_eq!(w.action("mm-a", 3), Some(to(Workspace::Trusted, "kyle"))); +} + +#[test] +fn a_missing_session_directory_is_empty() { + let dir = TempDir::new("honcho-epochs-missing"); + let records = read_session(&dir.path().join("sessions"), &sid("mm-none")).unwrap(); + assert!(records.is_empty()); +} + +#[test] +fn send_turn_reads_a_turn_that_lives_in_epoch_1() { + let fake = FakeHoncho::start(); + let home = WorkerHome::new("honcho-epochs-send"); + write_log( + home.dir.path(), + "mm-a", + &[ + spoken("one", Some("kyle")), + assistant(2, Some("ok"), &[]), + spoken("two", Some("kyle")), + assistant(3, Some("ok"), &[]), + epoch_end("sum"), + ], + ); + write_epoch( + home.dir.path(), + "mm-a", + 1, + &[spoken("three", Some("kyle")), assistant(4, Some("ok"), &[])], + ); + home.report("mm-a", 3); + let mut w = home.worker(&fake, FAST); + assert_eq!(w.step(), Step::Done); + assert_eq!(w.step(), Step::Done); + assert_eq!(w.step(), Step::Done); + let posts: Vec<_> = fake + .requests() + .into_iter() + .filter(|r| r.path.ends_with("/messages")) + .collect(); + assert_eq!(posts.len(), 3, "{posts:?}"); + let body = posts[2].body.as_ref().expect("message batch"); + let contents: Vec<&str> = body["messages"] + .as_array() + .unwrap() + .iter() + .map(|m| m["content"].as_str().unwrap()) + .collect(); + assert_eq!(contents, ["three", "ok"]); + assert_eq!(body["messages"][0]["metadata"]["turn"], 3); +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 928e376..5947d05 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone. | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| M5d1/04-memoryd-honcho-epochs | 2026-10-02 | done | 1 | pass | none | Copied the given test `crates/memoryd/tests/honcho_epochs.rs` (new). Step 2 (`cargo build --tests --keep-going -p memoryd`) showed exactly the named failure: `honcho_epochs` with one E0432 (`unresolved import memoryd::honcho::turns::read_session`); every other target compiled. Made H1 (`turns.rs`: `log_path` doc now names the sibling epoch files and that `read_session` concatenates them), H1b (inserted `epoch_logs` after `read_records`: lists `.jsonl` under `sessions//`, `n` parsed as `u32`, sorted by `n`, a missing directory empty; then `read_session` concatenates `read_records` over each), H2 (`resume` reads `read_session` instead of `read_log(&log_path(...))`), H3 (`records` matches `read_session(...).map_err(|e| e.to_string())`, keeping the logged line shape); W1 (`worker.rs`: added `use ...decide::{final_reply, turn_records, user_text}` and `use ...turns::{LogError, TurnSync, log_path, read_session}`), W2 (`send_turn` loads `read_session`, `self.readable(&path)` on Ok, `LogError::Damaged(_)` -> `Unsent::Local("log")`, other error -> `Unsent::Unreadable(e.to_string())`, then `turn_records`/`user_text`/`final_reply` from `decide`), W3 (`batch` called with `text`/`reply` by value, no borrow). `read_turn` no longer used in the worker body but stays re-exported. `cargo check -p memoryd` clean, no error or warning. `wc -l` prints 369 for `turns.rs` and 453 for `worker.rs`, each ≤ 500. `cargo test -p memoryd --test honcho_epochs --test honcho_turns --test honcho_worker` passes: 6 in `honcho_epochs`, 16 in `honcho_turns`, 12 in `honcho_worker`. `git diff --stat` lists only `turns.rs` and `worker.rs`; `cargo fmt --all` changed nothing after the edits. `make gate` printed `gate: ok` with 1484 tests (1478 before, plus the 6 of `honcho_epochs.rs`). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b | | M5d1/03-loopd-carry-messages | 2026-10-02 | done | 2 | pass | none | Copied the given test `crates/loopd/tests/carry_messages.rs` (new). Step 2 (`cargo build --tests --keep-going -p loopd`) showed exactly the named failure: `carry_messages` with one E0432 (`unresolved import loopd::baseline::{CARRY_HEADING, messages_with_carry}`); every other target compiled. Made B1 (`baseline.rs`: `CARRY_HEADING` = `concat!("Earlier ", "turns, summarised:")`, `messages` becomes `messages_with_carry(baseline, records, None)`, new `messages_with_carry` inserts the heading plus a blank line plus the summary as one `ChatMessage::User` after the system prompt and before walking `records`; the rest of the `match` untouched); T1 (`turn.rs`: `use crate::baseline::messages_with_carry`); T2 (`run_turn` passes `session.carry()`). `cargo check -p loopd` clean, no error or warning. `cargo test -p loopd --test carry_messages --test baseline --test turn` passes: 5 in `carry_messages`, 6 in `baseline`, 6 in `turn`. `git diff --stat` lists only `baseline.rs` and `turn.rs`; `cargo fmt --all` changed nothing. `make gate` printed `gate: ok` with 1478 tests (1473 before, plus the 5 of `carry_messages.rs`). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b | | M5d1/02-loopd-session-epochs | 2026-10-02 | done | 1 | pass | none | Copied the given test `crates/loopd/tests/session_epochs.rs` (new). Step 2 (`cargo build --tests --keep-going -p loopd`) showed exactly the named failures: `session_epochs` with E0599 (`no method named epoch` / `carry` / `slot`) and E0433 (`NoCarry`); every other target compiled and the existing `session` tests still compiled. Made S1 (module doc naming `.baseline.json` / `.jsonl` and epochs after compaction), S2 (`SessionError` gains `NoCarry { path: PathBuf }`), S3 (its Display arm naming the damaged-log runbook pointer), S4 (`Session` gains `epoch`, `slot`, `carry`, `prior_users`, `prior_untrusted` in that order), S5 (`create` inits the five new fields, `slot` from the argument), S6 (whole `open`: `complete_epochs` picks the highest epoch with both files, `records` is the current log, `priors` walks every earlier complete epoch for user/untrusted counts and the epoch-1 carry, `max_call` over every epoch for `next_call`, `live_slot` from the current `SessionStart`), S7 (accessors `epoch` / `slot` / `carry`), S8 (`untrusted` = `prior_untrusted || any_untrusted`, `turns` = `prior_users.saturating_add(count_users)`), S9 (`log_path` to the current epoch's file plus helpers `complete_epochs`, `load_records`, `priors`, `carry_from`, `count_users`, `any_untrusted`, `max_call`, `live_slot`). `cargo check -p loopd` clean, no error or warning. `wc -l crates/loopd/src/session.rs` prints 402. `cargo test -p loopd --test session_epochs --test session` passes: 9 in `session_epochs`, 7 in `session`. `git diff --stat` lists only `session.rs`; `cargo fmt --all` changed nothing. `make gate` printed `gate: ok` with 1473 tests (1472 in `cargo test --workspace`, plus the 1 ignored `end_to_end`; the +9 over the 1464 before is exactly the tests of `session_epochs.rs`). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b | | M5d1/01-loopd-compact-config | 2026-10-02 | done | 1 | pass | none | Copied the given test `crates/loopd/tests/config_compact.rs` (new). Step 2 (`cargo build --tests --keep-going -p loopd`) showed exactly the named failures: `config_compact` with E0609 (`no field \`compact\` on type \`Config\``), E0599 (`no method named \`compact_prompt\``) and E0433 (`no variant or associated item named \`default\` for \`Compact\``); every other target compiled. Made the edits exactly as the task gives: C1 (new `Compact` struct after `Memory`, `#[derive(Debug, Clone, PartialEq, Deserialize)]` plus `#[serde(deny_unknown_fields, default)]`, fields in order `threshold: f64`, `summary_chars: usize`, `summarise: Vec`, and `Default` = threshold `0.85`, `summary_chars` `4_000`, `summarise` `vec!["http_fetch".to_string()]`); C2 (`Config` gains `#[serde(default)] pub compact: Compact` after `memory`); C3 (`compact_prompt` after `channel_socket`, `self.baseline.system.with_file_name("compact.md")`). `cargo check -p loopd` clean, no error or warning. `cargo test -p loopd --test config_compact --test config` passes: 8 in `config_compact`, 11 in `config`. `git diff --stat` lists only `config.rs`; `cargo fmt --all` changed nothing. `make gate` printed `gate: ok` with 1464 tests (1463 in `cargo test --workspace`, plus the 1 ignored `end_to_end`; the +8 over the 1456 before is exactly the tests of `config_compact.rs`). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b |