From 3bea6c24c9dd630b239cf28690a12ba91fbdc224 Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Sat, 3 Oct 2026 17:54:18 -0700 Subject: [PATCH] memoryd: Tiers listing; bxctl memory tiers Co-Authored-By: ornith-1.5-35b-a3b --- config/jobs/tier.toml.example | 6 ++ crates/bxcore/src/memory.rs | 2 + crates/bxctl/src/cli.rs | 2 +- crates/bxctl/src/memory.rs | 1 + crates/bxctl/tests/cli_memory_sync.rs | 2 +- crates/bxctl/tests/cli_memory_tiers.rs | 86 +++++++++++++++ crates/memoryd/src/lib.rs | 1 + crates/memoryd/src/serve/admin.rs | 1 + crates/memoryd/src/tiers.rs | 143 +++++++++++++++++++++++++ crates/memoryd/tests/tiers.rs | 122 +++++++++++++++++++++ docs/implementer-log.md | 1 + 11 files changed, 365 insertions(+), 2 deletions(-) create mode 100644 config/jobs/tier.toml.example create mode 100644 crates/bxctl/tests/cli_memory_tiers.rs create mode 100644 crates/memoryd/src/tiers.rs create mode 100644 crates/memoryd/tests/tiers.rs diff --git a/config/jobs/tier.toml.example b/config/jobs/tier.toml.example new file mode 100644 index 0000000..3596b1d --- /dev/null +++ b/config/jobs/tier.toml.example @@ -0,0 +1,6 @@ +# Copy to {BOXMAKER_HOME}/jobs/tier.toml and edit the schedule. +# Five-field UTC crontab. This example is Sunday 07:00. +schedule = "0 7 * * 0" +prompt = """Read the listing in this message. Promote only trusted notes into identity +(memory.md / user.md) with add or consolidate. Do not promote an untrusted note. +Do not fetch. Write a note of what you changed, or that nothing needed changing.""" diff --git a/crates/bxcore/src/memory.rs b/crates/bxcore/src/memory.rs index 2e9d77b..8b20472 100644 --- a/crates/bxcore/src/memory.rs +++ b/crates/bxcore/src/memory.rs @@ -365,6 +365,8 @@ pub enum MemoryCommand { Reject, /// What the Honcho ledger holds: its counts and failed items (M5c spec, section 5). Sync, + /// Identity entries and notes with recall-hit counts (M5d spec, section 4.4). + Tiers, } /// `bxctl` → `memoryd`: one command; `arg` is the count for `log`, the edit or pending id for diff --git a/crates/bxctl/src/cli.rs b/crates/bxctl/src/cli.rs index fccd08c..c53f567 100644 --- a/crates/bxctl/src/cli.rs +++ b/crates/bxctl/src/cli.rs @@ -34,7 +34,7 @@ usage: bxctl [options] bxctl reindex [--memory-socket PATH] Rebuild memoryd's index from the session logs and notes. - bxctl memory log [N] | undo | pending | accept | reject | sync [--memory-socket PATH] + bxctl memory log [N] | undo | pending | accept | reject | sync | tiers [--memory-socket PATH] List memory edits, undo the latest edit of a file, or answer writes waiting for review. sync shows what memoryd sends to Honcho: counts, and each failed item by its label. diff --git a/crates/bxctl/src/memory.rs b/crates/bxctl/src/memory.rs index 8e4bfb8..c5b870d 100644 --- a/crates/bxctl/src/memory.rs +++ b/crates/bxctl/src/memory.rs @@ -57,6 +57,7 @@ pub fn parse_memory(args: &[String], home: &Path) -> Result ["accept", id] => Ask::Admin(MemoryCommand::Accept, id.to_string()), ["reject", id] => Ask::Admin(MemoryCommand::Reject, id.to_string()), ["sync"] => Ask::Admin(MemoryCommand::Sync, String::new()), + ["tiers"] => Ask::Admin(MemoryCommand::Tiers, String::new()), _ => return Err(UsageError), }; Ok(Command::Memory { socket, ask }) diff --git a/crates/bxctl/tests/cli_memory_sync.rs b/crates/bxctl/tests/cli_memory_sync.rs index f83a8df..49f42f8 100644 --- a/crates/bxctl/tests/cli_memory_sync.rs +++ b/crates/bxctl/tests/cli_memory_sync.rs @@ -61,7 +61,7 @@ fn sync_parses() { ); assert_eq!(parse(&args("memory sync now"), &home), Err(UsageError)); assert!( - USAGE.contains("| reject | sync [--memory-socket PATH]\n"), + USAGE.contains("| reject | sync | tiers [--memory-socket PATH]\n"), "{USAGE}" ); } diff --git a/crates/bxctl/tests/cli_memory_tiers.rs b/crates/bxctl/tests/cli_memory_tiers.rs new file mode 100644 index 0000000..1477e9b --- /dev/null +++ b/crates/bxctl/tests/cli_memory_tiers.rs @@ -0,0 +1,86 @@ +//! Tests for `bxctl memory tiers`. Do not edit. + +#[path = "support/temp.rs"] +mod temp; +use std::os::unix::net::UnixListener; +use std::path::{Path, PathBuf}; +use std::sync::mpsc; +use std::time::Duration; + +use bxcore::{Envelope, MemoryAdminReply, MemoryCommand, Message, read_frame, write_frame}; +use bxctl::cli::{Command, UsageError, parse}; +use bxctl::memory::{Ask, run}; + +fn args(line: &str) -> Vec { + line.split_whitespace().map(str::to_string).collect() +} + +fn home() -> PathBuf { + PathBuf::from("/srv/bx") +} + +fn fake(socket: &Path, answer: Message) -> mpsc::Receiver { + let listener = UnixListener::bind(socket).unwrap(); + let (tx, rx) = mpsc::channel(); + std::thread::spawn(move || { + let (mut stream, _) = listener.accept().unwrap(); + let env = read_frame(&mut stream).unwrap(); + tx.send(env.msg).unwrap(); + let reply = Envelope { + v: 1, + id: env.id, + r#final: true, + msg: answer, + }; + write_frame(&mut stream, &reply).unwrap(); + }); + rx +} + +#[test] +fn memory_tiers_parses() { + let h = home(); + assert_eq!( + parse(&args("memory tiers"), &h), + Ok(Command::Memory { + socket: PathBuf::from("/srv/bx/run/owner-memory/admin.sock"), + ask: Ask::Admin(MemoryCommand::Tiers, String::new()) + }) + ); + assert_eq!( + parse(&args("memory --memory-socket /r/m.sock tiers"), &h), + Ok(Command::Memory { + socket: PathBuf::from("/r/m.sock"), + ask: Ask::Admin(MemoryCommand::Tiers, String::new()) + }) + ); + assert_eq!(parse(&args("memory tiers extra"), &h), Err(UsageError)); +} + +#[test] +fn memory_tiers_prints_the_reply() { + let d = temp::TempDir::new("bx-tiers"); + let socket = d.join("admin.sock"); + let listing = "identity memory.md hits 0 last -\n I like tea.\n"; + let seen = fake( + &socket, + Message::MemoryAdminReply(MemoryAdminReply { + text: listing.to_string(), + }), + ); + let mut out = Vec::new(); + run( + &socket, + &Ask::Admin(MemoryCommand::Tiers, String::new()), + Duration::from_secs(5), + &mut out, + ) + .unwrap(); + assert_eq!(String::from_utf8(out).unwrap(), format!("{listing}\n")); + match seen.recv().unwrap() { + Message::MemoryAdmin(m) => { + assert_eq!((m.command, m.arg.as_str()), (MemoryCommand::Tiers, "")) + } + other => panic!("{other:?}"), + } +} diff --git a/crates/memoryd/src/lib.rs b/crates/memoryd/src/lib.rs index 7ccbe43..6087389 100644 --- a/crates/memoryd/src/lib.rs +++ b/crates/memoryd/src/lib.rs @@ -12,3 +12,4 @@ pub mod recall; pub mod review; pub mod serve; pub mod socket; +pub mod tiers; diff --git a/crates/memoryd/src/serve/admin.rs b/crates/memoryd/src/serve/admin.rs index 8e13e2f..cbc8446 100644 --- a/crates/memoryd/src/serve/admin.rs +++ b/crates/memoryd/src/serve/admin.rs @@ -62,6 +62,7 @@ impl Memoryd { Some(mirror) => mirror.report(), None => report::NOT_CONFIGURED.to_string(), }), + MemoryCommand::Tiers => crate::tiers::list(&self.store, now), }; if result.is_ok() && matches!(command, MemoryCommand::Undo | MemoryCommand::Accept) { self.mirror(None); diff --git a/crates/memoryd/src/tiers.rs b/crates/memoryd/src/tiers.rs new file mode 100644 index 0000000..cce29c0 --- /dev/null +++ b/crates/memoryd/src/tiers.rs @@ -0,0 +1,143 @@ +//! The tiering listing: identity entries and notes with recall-hit counts (M5d spec, 4.4). + +use std::collections::{HashMap, HashSet}; +use std::io::ErrorKind; + +use bxcore::{MemoryFile, Timestamp}; +use serde::Deserialize; + +use crate::files::Store; +use crate::notes::{self, Kind, Note}; + +#[derive(Deserialize)] +struct Logged { + time: String, + hits: Vec, +} + +#[derive(Deserialize)] +struct Hit { + kind: String, + session: String, + record: i64, +} + +/// How many recall lines named one note, and the latest of those lines' times. +#[derive(Default)] +struct Tally { + n: u64, + last: String, +} + +fn fail(path: &std::path::Path, e: std::io::Error) -> String { + format!("{}: {e}", path.display()) +} + +fn class_of(note: &Note) -> &'static str { + match ¬e.kind { + Kind::Noted { class, .. } => notes::class_name(*class), + Kind::Moved(_) | Kind::Dropped(_) => "private", + } +} + +fn trust_of(note: &Note) -> &'static str { + match ¬e.kind { + Kind::Noted { untrusted, .. } => { + if *untrusted { + "untrusted" + } else { + "trusted" + } + } + Kind::Moved(_) | Kind::Dropped(_) => "trusted", + } +} + +fn read_notes(store: &Store) -> Result { + let path = store.path("notes.md"); + match std::fs::read_to_string(&path) { + Ok(text) => Ok(text), + Err(e) if e.kind() == ErrorKind::NotFound => Ok(String::new()), + Err(e) => Err(fail(&path, e)), + } +} + +/// The recall hits per `(session, record)`: how many lines named it and the latest of their times +/// (M5d spec, section 4.4). A missing `recalls/` is an empty map; a line that does not parse is +/// skipped, and a line counts once for each note it names. +fn tally(store: &Store) -> Result, String> { + let dir = store.path("recalls"); + let files = match std::fs::read_dir(&dir) { + Ok(files) => files, + Err(e) if e.kind() == ErrorKind::NotFound => return Ok(HashMap::new()), + Err(e) => return Err(fail(&dir, e)), + }; + let mut hits: HashMap<(String, i64), Tally> = HashMap::new(); + for file in files { + let file = file.map_err(|e| fail(&dir, e))?; + if !file.file_type().map_err(|e| fail(&dir, e))?.is_file() { + continue; + } + let path = file.path(); + let text = match std::fs::read_to_string(&path) { + Ok(text) => text, + Err(e) if e.kind() == ErrorKind::NotFound => continue, + Err(e) => return Err(fail(&path, e)), + }; + for line in text.lines() { + let logged: Logged = match serde_json::from_str(line) { + Ok(logged) => logged, + Err(_) => continue, + }; + let mut seen: HashSet<(String, i64)> = HashSet::new(); + for hit in &logged.hits { + if hit.kind == "noted" { + seen.insert((hit.session.clone(), hit.record)); + } + } + for key in seen { + let tally = hits.entry(key).or_default(); + tally.n += 1; + if logged.time > tally.last { + tally.last = logged.time.clone(); + } + } + } + } + Ok(hits) +} + +/// One listing for `bxctl memory tiers` and for the tiering job's user text. +pub fn list(store: &Store, _now: Timestamp) -> Result { + let mut out = String::new(); + + for file in [MemoryFile::Memory, MemoryFile::User] { + for entry in store.entries(file)? { + out.push_str(&format!( + "identity {} hits 0 last -\n {}\n", + file.name(), + entry + )); + } + } + + let hits = tally(store)?; + for (i, note) in notes::parse(&read_notes(store)?).into_iter().enumerate() { + let key = (note.session.clone(), i as i64); + let (n, last) = match hits.get(&key) { + Some(t) => (t.n, t.last.clone()), + None => (0, "-".to_string()), + }; + out.push_str(&format!( + "note session {} hits {} last {} {} {}\n {}\n", + note.session, + n, + last, + class_of(¬e), + trust_of(¬e), + note.text + )); + } + + Ok(out) +} diff --git a/crates/memoryd/tests/tiers.rs b/crates/memoryd/tests/tiers.rs new file mode 100644 index 0000000..71ff8f5 --- /dev/null +++ b/crates/memoryd/tests/tiers.rs @@ -0,0 +1,122 @@ +//! Tests for `MemoryCommand::Tiers`: identity entries and notes with recall-hit counts. Do not edit. + +#[path = "support/tmp.rs"] +mod tmp; + +use bxcore::{MemoryFile, Timestamp}; +use memoryd::files::Store; +use memoryd::tiers; +use tmp::TempDir; + +fn now() -> Timestamp { + Timestamp::parse("2026-10-03T08:00:00.000Z").unwrap() +} + +struct Setup { + _dir: TempDir, + store: Store, +} + +fn setup(tag: &str) -> Setup { + let dir = TempDir::new(tag); + let starter = dir.write("etc/soul.md", "Soul.\n"); + let store = Store::new(&dir.path().join("memory")); + store.init(&starter).unwrap(); + Setup { _dir: dir, store } +} + +fn write_recall(store: &Store, line: &str) { + std::fs::create_dir_all(store.path("recalls")).unwrap(); + store + .append("recalls/2026-10.jsonl", &format!("{line}\n")) + .unwrap(); +} + +#[test] +fn identity_entries_list_with_zero_hits() { + let s = setup("tiers-id"); + s.store + .write_entries(MemoryFile::Memory, &["I like tea.".to_string()]) + .unwrap(); + s.store + .write_entries(MemoryFile::User, &["The owner is Kyle.".to_string()]) + .unwrap(); + let text = tiers::list(&s.store, now()).unwrap(); + assert!( + text.contains("identity memory.md hits 0 last -"), + "{text}" + ); + assert!(text.contains("I like tea."), "{text}"); + assert!(text.contains("identity user.md hits 0 last -"), "{text}"); + assert!(text.contains("The owner is Kyle."), "{text}"); +} + +#[test] +fn notes_carry_counts_from_recalls() { + let s = setup("tiers-notes"); + std::fs::write( + s.store.path("notes.md"), + "§ noted 2026-09-22T18:20Z in session mm-a · private\nlaunch window moved.\n", + ) + .unwrap(); + write_recall( + &s.store, + r#"{"time":"2026-10-03T07:00:00.000Z","session":"mm-now","query":"launch","hits":[{"kind":"noted","session":"mm-a","epoch":0,"record":0}]}"#, + ); + write_recall( + &s.store, + r#"{"time":"2026-10-03T07:30:00.000Z","session":"mm-now","query":"window","hits":[{"kind":"noted","session":"mm-a","epoch":0,"record":0}]}"#, + ); + let text = tiers::list(&s.store, now()).unwrap(); + assert!( + text.contains( + "note session mm-a hits 2 last 2026-10-03T07:30:00.000Z private trusted" + ), + "{text}" + ); + assert!(text.contains("launch window moved."), "{text}"); +} + +#[test] +fn untrusted_notes_are_listed_and_marked() { + let s = setup("tiers-untrusted"); + std::fs::write( + s.store.path("notes.md"), + "§ noted 2026-09-22T18:20Z in session mm-c · private · untrusted\nexample.org says the launch moved.\n", + ) + .unwrap(); + let text = tiers::list(&s.store, now()).unwrap(); + assert!( + text.contains("note session mm-c hits 0 last - private untrusted"), + "{text}" + ); + assert!( + text.contains("example.org says the launch moved."), + "{text}" + ); +} + +#[test] +fn a_recall_of_a_different_note_does_not_count() { + let s = setup("tiers-other"); + std::fs::write( + s.store.path("notes.md"), + "§ noted 2026-09-22T18:20Z in session mm-a · private\nfirst note.\n§ noted 2026-09-22T18:21Z in session mm-b · private\nsecond note.\n", + ) + .unwrap(); + write_recall( + &s.store, + r#"{"time":"2026-10-03T07:00:00.000Z","session":"mm-now","query":"second","hits":[{"kind":"noted","session":"mm-b","epoch":0,"record":1}]}"#, + ); + let text = tiers::list(&s.store, now()).unwrap(); + assert!( + text.contains("note session mm-a hits 0 last - private trusted"), + "{text}" + ); + assert!( + text.contains( + "note session mm-b hits 1 last 2026-10-03T07:00:00.000Z private trusted" + ), + "{text}" + ); +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 12098fb..f0eb956 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone. | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| M5d2/10-memoryd-tiers | 2026-10-03 | done | 2 | fail | none | Copied the given tests `crates/memoryd/tests/tiers.rs` and `crates/bxctl/tests/cli_memory_tiers.rs` and the starter `config/jobs/tier.toml.example` from `docs/plans/M5d2/files/`. Added `MemoryCommand::Tiers` after `Sync` in `bxcore/src/memory.rs` with the doc comment from M1. Created `crates/memoryd/src/tiers.rs` with `list`: identity entries first (memory.md then user.md via `store.entries`, each `hits 0 last -`), then notes from `notes::parse`, each joined to `recalls/` on `(noted, session, record)` where `hits` counts the lines holding a matching hit and `last` is the max `time` among them or `-`; moved/dropped notes are still listed as `note` with class `private` and `trusted`, and untrusted notes are marked. `recalls/` is read via `store.path("recalls")`: a missing directory yields an empty tally (all zeros), a file that exists but cannot be read is an error, and a line that does not parse is skipped. `Logged` and `Hit` deserialize without `deny_unknown_fields` so `epoch` and `query` on a line are ignored. Used a small `Tally { n: u64, last: String }` so the map is `HashMap<(String, i64), Tally>` and clippy's `type_complexity` stays off. Added `pub mod tiers;` after `pub mod socket;` in lib.rs, the `Tiers` arm in `serve/admin.rs`, `["tiers"]` in bxctl/src/memory.rs, `| tiers` in the USAGE line, and the edited USAGE pin in cli_memory_sync.rs (left cli_memory.rs untouched). First `make gate` failed on a third leading space I had put on the USAGE memory line, which the `usage_explains_the_speaker_flag` indentation check caught; restored it to two spaces. `cargo check` clean, no warning; `wc -l` prints 139 for tiers.rs. All suites pass: `tiers` 4, `cli_memory_tiers` 2 (plus `cli_memory` 4 and `cli_memory_sync` 4). `make gate` printed `gate: ok` with 1574 tests (1568 before, plus 6). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b | | M5d2/09-memoryd-summary-index | 2026-10-03 | done | 1 | pass | none | Copied the given test `crates/memoryd/tests/tool_summary_index.rs` from `docs/plans/M5d2/files/`. Edited `index_record` in `crates/memoryd/src/index.rs`, adding the `ToolSummary` arm before `_ => {}`: `Source { kind: "summary", role: "summary", class: *class, untrusted: *untrusted, ..base.clone() }` with `time` from the record, guarded by `if !content.trim().is_empty() && *class <= self.max_class`; `DataClass` implements `Ord` so `<=` compiles. The existing `if base.class > self.max_class` guard at the top of `index_record` stays, and the rolling-label update in `index_log` still looks at `ToolResult` only (no `ToolSummary` added there). Updated the module doc "never a tool's result" to say summaries of opted-in results are indexed. `cargo check -p memoryd` clean, no error or warning. All suites pass: `tool_summary_index` 5, `recall` 6. `cargo fmt --all` changed nothing after the edits. `make gate` printed `gate: ok` with 1568 tests (1563 before, plus 5). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b | | M5d2/08-loopd-tool-summary | 2026-10-03 | done | 1 | pass | none | Copied the given test `crates/loopd/tests/tool_summary_call.rs` and the starter `config/summarise.md` from `docs/plans/M5d2/files/`. Created `crates/loopd/src/summarise.rs` with the `missing` function: walks `session.records()` once, remembering `tool_calls` by `id -> name` and each `ToolSummary`'s `call` in `done`, queueing opted-in `ToolResult`s during the walk (an in-walk `done.contains` still queues), then dropping any whose call is already done after the walk; for each pending it builds `messages_with_carry` plus a `User` message of the trimmed `summarise.md` prompt and the result content capped by `tool_result_cap`, sends on `slots.background` with `tools: Vec::new()` and thinking on, neutralises and caps the completion by `summary_chars`, and appends a `ToolSummary` copying call/tool_call_id/class/untrusted; context-full or other infer errors log `loopd: tool summary: {e}` and skip. Added `pub mod summarise;` after `pub mod sse;` in `lib.rs` (rustfmt order). Edited `finish_turn` in `compact.rs` (F1) to call `summarise::missing` as the first statement of both the `Ok(done)` arm and the `Err(SessionFull)` arm, before `compact_session`. `cargo check -p loopd` clean, no error or warning; `wc -l` prints 96 for summarise.rs and 194 for compact.rs, both under 500. All suites pass: `tool_summary_call` 6, `tool_summary_messages` 3, `compact_turn` 6, `turn` 6. `CallId(pub u64)` `.0` field compiled. `cargo fmt --all` changed nothing after the edits. `make gate` printed `gate: ok` with 1563 tests (1557 before, plus 6). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b | | M5d2/07-loopd-scheduler-cron | 2026-10-03 | done | 1 | pass | none | Copied the given test `crates/loopd/tests/scheduler_cron.rs` from `docs/plans/M5d2/files/`. Added the cron loop into `tick` in `crates/loopd/src/scheduler.rs`, right after `self.heartbeat(...)` and before `work` is returned (heartbeat untouched): load `jobs` via `jobs::load(&ctx.cfg.paths.home.join("jobs"))`, on `Err` log `loopd: {e}` and return; for each job in order check `cron::due` (`Err` log+skip, `Ok(false)` skip), then `jobs::live_session` on `{home}/jobs/runs.jsonl` (`Some` push `SkippedJob` and continue), then `session_id` (`Err` log+skip), then `ctx.hold` (`Busy` push `SkippedJob`). On a hold: `Baseline::assemble`, `Session::create` on `slots.background` (on `Exists`, `open` instead), append `Started`, run `run_turn` then `compact::finish_turn` with user text `format!("It is {}.\n\n{}", now.to_rfc3339(), job.prompt)` and speaker `None`, append `Finished` on an ok turn else `Failed`, drop the hold, push `TickWork::Job { name, session }`. Added `use crate::cron;` to the imports. `cargo check -p loopd` clean, no error or warning; `wc -l` prints 304, under 500. All four suites pass: `cron` 11, `jobs` 8, `scheduler_cron` 6, `scheduler_heartbeat` 6. `cargo fmt --all` wrapped the `Session::create` call and its `Exists` arm as a nested match; `make gate` printed `gate: ok` with 1557 tests (1551 before, plus the 6 of `scheduler_cron.rs`). No new dependency, so `Cargo.lock` unchanged. | ornith-1.5-35b-a3b |