diff --git a/crates/memoryd/src/index.rs b/crates/memoryd/src/index.rs new file mode 100644 index 0000000..350b599 --- /dev/null +++ b/crates/memoryd/src/index.rs @@ -0,0 +1,421 @@ +//! The index: SQLite FTS5 over the session logs and `notes.md` (M5a spec, section 6). Disposable: +//! it is rebuilt from those files. What is indexed: the owner's messages, the assistant's +//! replies, and each tool call by name and arguments; never a tool's result. + +use std::os::unix::fs::PermissionsExt; +use std::path::{Path, PathBuf}; + +use proto::{DataClass, LogRecord}; +use rusqlite::{Connection, params}; + +use crate::notes::{self, Kind}; + +const SCHEMA: &str = " +CREATE VIRTUAL TABLE IF NOT EXISTS hits USING fts5(text, tokenize = 'unicode61'); +CREATE TABLE IF NOT EXISTS sources ( + rowid INTEGER PRIMARY KEY, kind TEXT NOT NULL, session TEXT NOT NULL, epoch INTEGER NOT NULL, + record INTEGER NOT NULL, time TEXT NOT NULL, role TEXT NOT NULL, class TEXT NOT NULL, + untrusted INTEGER NOT NULL); +CREATE TABLE IF NOT EXISTS progress (log TEXT PRIMARY KEY, records INTEGER NOT NULL); +"; + +/// One indexed item. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct Source { + /// `session`, `noted`, `moved` or `dropped`. + pub kind: String, + pub session: String, + pub epoch: i64, + /// The record's line in its log, or the note's place in `notes.md` (from 0). + pub record: i64, + pub time: String, + /// `owner`, `assistant` or `call` for a session record; the file (`memory.md`, `user.md`) for + /// a moved or dropped note; empty for a noted one. + pub role: String, + pub class: DataClass, + pub untrusted: bool, +} + +/// How much a reindex read. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub struct Counts { + pub logs: u64, + pub records: u64, + pub notes: u64, +} + +pub struct Index { + conn: Connection, + max_class: DataClass, +} + +fn db(e: rusqlite::Error) -> String { + format!("index: {e}") +} + +pub fn class_name(class: DataClass) -> &'static str { + match class { + DataClass::Public => "public", + DataClass::Private => "private", + DataClass::Secret => "secret", + } +} + +pub fn class_of(name: &str) -> DataClass { + match name { + "public" => DataClass::Public, + "secret" => DataClass::Secret, + _ => DataClass::Private, + } +} + +fn int(n: usize) -> i64 { + i64::try_from(n).unwrap_or(i64::MAX) +} + +impl Index { + /// Open or create the index at `path` (its directory 0700, the file 0600). + pub fn open(path: &Path, max_class: DataClass) -> Result { + if let Some(dir) = path.parent() { + std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?; + std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700)) + .map_err(|e| format!("{}: {e}", dir.display()))?; + } + let conn = Connection::open(path).map_err(|e| format!("{}: {e}", path.display()))?; + std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600)) + .map_err(|e| format!("{}: {e}", path.display()))?; + conn.execute_batch(SCHEMA) + .map_err(|e| format!("{}: {e}", path.display()))?; + Ok(Index { conn, max_class }) + } + + /// The connection, for `recall`. + pub fn conn(&self) -> &Connection { + &self.conn + } + + fn progress(&self, log: &str) -> Result { + // `self.conn.query_row("SELECT records FROM progress WHERE log = ?1", params![log], |row| + // row.get(0))` as an Option: Err(rusqlite::Error::QueryReturnedNoRows) means None, + // another error goes through `db`. Return it as a usize (`usize::try_from`, 0 when it does + // not fit), 0 for None. + let value: Option = match self.conn.query_row( + "SELECT records FROM progress WHERE log = ?1", + params![log], + |row| row.get(0), + ) { + Ok(v) => Some(v), + Err(rusqlite::Error::QueryReturnedNoRows) => None, + Err(e) => return Err(db(e)), + }; + let n = match value { + Some(v) => usize::try_from(v).unwrap_or(0), + None => 0, + }; + Ok(n) + } + + fn add(&self, text: &str, source: &Source) -> Result<(), String> { + // 1. `execute("INSERT INTO hits (text) VALUES (?1)", params![text])`. + // 2. `let rowid = self.conn.last_insert_rowid();` + // 3. `execute("INSERT INTO sources (rowid, kind, session, epoch, record, time, role, class, + // untrusted) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", params![rowid, kind, session, + // epoch, record, time, role, class_name(class), untrusted])`. Errors through `db`. + self.conn + .execute("INSERT INTO hits (text) VALUES (?1)", params![text]) + .map_err(db)?; + let rowid = self.conn.last_insert_rowid(); + self.conn + .execute( + "INSERT INTO sources (rowid, kind, session, epoch, record, time, role, class, untrusted) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + rowid, + source.kind.clone(), + source.session.clone(), + source.epoch, + source.record, + source.time.clone(), + source.role.clone(), + class_name(source.class), + source.untrusted, + ], + ) + .map_err(db)?; + Ok(()) + } + + fn set_progress(&self, log: &str, records: usize) -> Result<(), String> { + // `execute("INSERT INTO progress (log, records) VALUES (?1, ?2) ON CONFLICT(log) DO UPDATE + // SET records = excluded.records", params![log, int(records)])`, errors through `db`. + self.conn + .execute( + "INSERT INTO progress (log, records) VALUES (?1, ?2) ON CONFLICT(log) DO UPDATE SET records = excluded.records", + params![log, int(records)], + ) + .map_err(db)?; + Ok(()) + } + + /// Index the records of one session log past what was indexed before; how many were read. + /// The label is walked from the log's first record: the highest class of any tool result + /// before a record (at least `private`), and whether any was untrusted. + pub fn index_log(&self, path: &Path, session: &str, epoch: i64) -> Result { + // 1. `read_to_string(path)` (an error names the path). `key` = + // format!("{session}/{epoch}"); `done = self.progress(&key)?`. + // 2. The label starts at class Private, untrusted false; `read` at 0. + // 3. For each line with its place n (from 0): parse it as a LogRecord (an error -> + // Err(format!("{}: line {}: {e}", path.display(), n + 1))). When n >= done: read += 1, + // and `self.index_record(&record, &base)?` where base is a Source with kind "session", + // session, epoch, record int(n), time and role empty, and the label as it stands. Then, + // for every record (indexed or not): a ToolResult raises the label: class = + // class.max(its class), untrusted |= its untrusted. + // 4. `self.set_progress(&key, done + read)?`; Ok(read). + let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?; + let key = format!("{session}/{epoch}"); + let done = self.progress(&key)?; + let mut class = DataClass::Private; + let mut untrusted = false; + let mut read = 0; + for (n, line) in text.lines().enumerate() { + let record: LogRecord = serde_json::from_str(line) + .map_err(|e| format!("{}: line {}: {e}", path.display(), n + 1))?; + if n >= done { + read += 1; + let base = Source { + kind: "session".to_string(), + session: session.to_string(), + epoch, + record: int(n), + time: String::new(), + role: String::new(), + class, + untrusted, + }; + self.index_record(&record, &base)?; + } + if let LogRecord::ToolResult { + class: c, + untrusted: u, + .. + } = &record + { + class = class.max(*c); + untrusted |= *u; + } + } + self.set_progress(&key, done + read)?; + Ok(read) + } + + /// Index what one log record holds: the owner's message, the assistant's reply, and each tool + /// call by name and arguments. `base` carries the kind, session, epoch, line and label; each item + /// gets its own role and time. Nothing when the label's class is above `max_class`, and nothing + /// for an item with no text. + fn index_record(&self, record: &LogRecord, base: &Source) -> Result<(), String> { + // When base.class > self.max_class: Ok(()) and nothing indexed. Otherwise each item is + // `self.add(text, &Source { time: time.to_rfc3339(), role, ..base.clone() })`, skipped when + // text.trim() is empty. User -> its content, role "owner". Assistant -> its content when + // Some, role "assistant"; then each tool call as format!("{} {}", name, arguments), role + // "call"; all with the record's time. Every other record: nothing. + if base.class > self.max_class { + return Ok(()); + } + match record { + LogRecord::User { time, content } => { + if !content.trim().is_empty() { + let src = Source { + time: time.to_rfc3339(), + role: "owner".to_string(), + ..base.clone() + }; + self.add(content, &src)?; + } + } + LogRecord::Assistant { + time, + content, + tool_calls, + .. + } => { + if let Some(c) = content + && !c.trim().is_empty() + { + let src = Source { + time: time.to_rfc3339(), + role: "assistant".to_string(), + ..base.clone() + }; + self.add(c, &src)?; + } + for call in tool_calls { + let text = format!("{} {}", call.name, call.arguments); + if !text.trim().is_empty() { + let src = Source { + time: time.to_rfc3339(), + role: "call".to_string(), + ..base.clone() + }; + self.add(&text, &src)?; + } + } + } + _ => {} + } + Ok(()) + } + + /// Index the entries of `notes.md` past what was indexed before; how many were read. + pub fn index_notes(&self, text: &str) -> Result { + // `done = self.progress("notes.md")?`. Parse `text` with notes::parse and skip the first + // `done`. For each remaining note (with its place n): read += 1; its kind, role, class and + // untrusted: Noted -> ("noted", "", its class, its untrusted); Moved(file) -> ("moved", + // file.name(), Private, false); Dropped(file) -> ("dropped", file.name(), Private, false). + // Skip it when class > self.max_class. Otherwise `self.add(¬e.text, &Source { kind, + // session: note.session, epoch: 0, record: int(n), time: note.time, role, class, untrusted + // })?`. Then `self.set_progress("notes.md", done + read)?`; Ok(read). + let done = self.progress("notes.md")?; + let notes = notes::parse(text); + let mut read = 0; + for (n, note) in notes.iter().enumerate() { + if n < done { + continue; + } + read += 1; + let (kind, role, class, untrusted) = match ¬e.kind { + Kind::Noted { class, untrusted } => { + ("noted".to_string(), String::new(), *class, *untrusted) + } + Kind::Moved(file) => ( + "moved".to_string(), + file.name().to_string(), + DataClass::Private, + false, + ), + Kind::Dropped(file) => ( + "dropped".to_string(), + file.name().to_string(), + DataClass::Private, + false, + ), + }; + if class > self.max_class { + continue; + } + self.add( + ¬e.text, + &Source { + kind, + session: note.session.clone(), + epoch: 0, + record: int(n), + time: note.time.clone(), + role, + class, + untrusted, + }, + )?; + } + self.set_progress("notes.md", done + read)?; + Ok(read) + } + + /// Index everything new: every `sessions//.jsonl`, then `notes.md`. A log that + /// cannot be read or parsed is skipped and named in the returned problems; it is `loopd`'s. + pub fn catch_up( + &self, + sessions: &Path, + notes_md: &Path, + ) -> Result<(Counts, Vec), String> { + // For each (session, epoch, path) of `logs(sessions)?`: `self.index_log(&path, &session, + // epoch)`; Ok(n) -> counts.logs += 1 and counts.records += n (u64::try_from, u64::MAX when + // it does not fit); Err(e) -> push e onto `problems` and go on. Then read `notes_md`: + // NotFound is an empty text, another error -> Err(format!("{}: {e}", notes_md.display())); + // counts.notes = `self.index_notes(&text)?` as u64 (u64::try_from). Ok((counts, problems)). + let mut counts = Counts::default(); + let mut problems: Vec = Vec::new(); + for (session, epoch, path) in logs(sessions)? { + match self.index_log(&path, &session, epoch) { + Ok(n) => { + counts.logs += 1; + counts.records += u64::try_from(n).unwrap_or(u64::MAX); + } + Err(e) => problems.push(e), + } + } + let text = match std::fs::read_to_string(notes_md) { + Ok(t) => t, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(), + Err(e) => return Err(format!("{}: {e}", notes_md.display())), + }; + counts.notes = u64::try_from(self.index_notes(&text)?).unwrap_or(u64::MAX); + Ok((counts, problems)) + } +} + +/// Every session log under `sessions`: `(session, epoch, path)`, sorted. +pub fn logs(sessions: &Path) -> Result, String> { + // Read the directory `sessions` (NotFound -> Ok(empty); another error -> Err naming it). For + // each entry, its name is the session; read that directory (skip it when that fails); every + // file named `.jsonl` where n parses as an i64 gives (session, n, the file's path). Sort the + // list and return it. + let entries = match std::fs::read_dir(sessions) { + Ok(e) => e, + Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()), + Err(e) => return Err(format!("{}: {e}", sessions.display())), + }; + let mut out: Vec<(String, i64, PathBuf)> = Vec::new(); + for entry in entries { + let entry = match entry { + Ok(e) => e, + Err(_) => continue, + }; + let name = entry.file_name(); + let session = match name.to_str() { + Some(s) => s, + None => continue, + }; + let dir = entry.path(); + let sub = match std::fs::read_dir(&dir) { + Ok(d) => d, + Err(_) => continue, + }; + for sub_entry in sub { + let sub_entry = match sub_entry { + Ok(e) => e, + Err(_) => continue, + }; + let sub_name = sub_entry.file_name(); + let file_name = match sub_name.to_str() { + Some(s) => s, + None => continue, + }; + if let Some(stem) = file_name.strip_suffix(".jsonl") + && let Ok(epoch) = stem.parse::() + { + out.push((session.to_string(), epoch, sub_entry.path())); + } + } + } + out.sort(); + Ok(out) +} + +/// Build a fresh index at `path` from `sessions` and `notes.md`: into a new file beside it, +/// renamed over the old one, so a running `recall` keeps its index until the new one is whole. +pub fn rebuild( + path: &Path, + max_class: DataClass, + sessions: &Path, + notes_md: &Path, +) -> Result<(Counts, Vec), String> { + // `fresh = path.with_extension("rebuilding")`; remove it if it is there (ignore the result). + // `Index::open(&fresh, max_class)?`, then `catch_up(sessions, notes_md)?` on it, and drop that + // Index. `std::fs::rename(&fresh, path)` (an error names path). Return the result of catch_up. + let fresh = path.with_extension("rebuilding"); + let _ = std::fs::remove_file(&fresh); + let index = Index::open(&fresh, max_class)?; + let result = index.catch_up(sessions, notes_md)?; + drop(index); + std::fs::rename(&fresh, path).map_err(|e| format!("{}: {e}", path.display()))?; + Ok(result) +} diff --git a/crates/memoryd/src/lib.rs b/crates/memoryd/src/lib.rs index f72ab85..d91c777 100644 --- a/crates/memoryd/src/lib.rs +++ b/crates/memoryd/src/lib.rs @@ -5,6 +5,7 @@ pub mod config; pub mod edit; pub mod entries; pub mod files; +pub mod index; pub mod journal; pub mod notes; pub mod review; diff --git a/crates/memoryd/tests/index.rs b/crates/memoryd/tests/index.rs new file mode 100644 index 0000000..5ef2a44 --- /dev/null +++ b/crates/memoryd/tests/index.rs @@ -0,0 +1,229 @@ +//! The index: what is indexed from a session log and `notes.md`, each record's label, catching up, +//! `max_class`, and a rebuild equal to indexing as it went (M5a spec, section 6). Do not edit. + +#[path = "support/logs.rs"] +mod logs; +#[path = "support/tmp.rs"] +mod tmp; + +use logs::{append, assistant, result, user, write_log}; +use memoryd::index::{Index, rebuild}; +use proto::DataClass; +use tmp::TempDir; + +/// Every indexed item, in order: (kind, role, text, class, untrusted, record). +fn items(index: &Index) -> Vec<(String, String, String, String, bool, i64)> { + let mut stmt = index + .conn() + .prepare("SELECT s.kind, s.role, h.text, s.class, s.untrusted, s.record FROM sources s JOIN hits h ON h.rowid = s.rowid ORDER BY s.rowid") + .unwrap(); + stmt.query_map([], |r| { + Ok(( + r.get(0)?, + r.get(1)?, + r.get(2)?, + r.get(3)?, + r.get(4)?, + r.get(5)?, + )) + }) + .unwrap() + .map(|r| r.unwrap()) + .collect() +} + +fn item( + kind: &str, + role: &str, + text: &str, + class: &str, + untrusted: bool, + record: i64, +) -> (String, String, String, String, bool, i64) { + ( + kind.to_string(), + role.to_string(), + text.to_string(), + class.to_string(), + untrusted, + record, + ) +} + +#[test] +fn owner_assistant_and_calls_are_indexed_with_the_label_in_force() { + let dir = TempDir::new("index-log"); + let log = write_log( + dir.path(), + "mm-a", + &[ + user(1, "What is the launch window?"), + assistant( + 2, + None, + &[("http_fetch", r#"{"url":"https://example.org"}"#)], + ), + result(3, "PAGE TEXT never indexed", DataClass::Private, true), + assistant(4, Some("The page says Thursday."), &[]), + user(5, " "), + ], + ); + let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); + assert_eq!(index.index_log(&log, "mm-a", 0), Ok(5)); + assert_eq!( + items(&index), + [ + item( + "session", + "owner", + "What is the launch window?", + "private", + false, + 0 + ), + item( + "session", + "call", + r#"http_fetch {"url":"https://example.org"}"#, + "private", + false, + 1 + ), + item( + "session", + "assistant", + "The page says Thursday.", + "private", + true, + 3 + ), + ] + ); +} + +#[test] +fn catching_up_reads_only_new_records_and_keeps_the_label() { + let dir = TempDir::new("index-catch"); + let log = write_log( + dir.path(), + "mm-b", + &[user(1, "first"), result(2, "x", DataClass::Private, true)], + ); + let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); + assert_eq!(index.index_log(&log, "mm-b", 0), Ok(2)); + append(&log, &user(3, "second")); + assert_eq!(index.index_log(&log, "mm-b", 0), Ok(1)); + assert_eq!(index.index_log(&log, "mm-b", 0), Ok(0)); + let got = items(&index); + assert_eq!(got.len(), 2); + assert_eq!( + got[1], + item("session", "owner", "second", "private", true, 2), + "tainted by the earlier result" + ); +} + +#[test] +fn records_above_max_class_are_not_indexed() { + let dir = TempDir::new("index-class"); + let log = write_log( + dir.path(), + "mm-c", + &[ + user(1, "before"), + result(2, "s", DataClass::Secret, false), + user(3, "after the secret"), + ], + ); + let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); + index.index_log(&log, "mm-c", 0).unwrap(); + assert_eq!( + items(&index), + [item("session", "owner", "before", "private", false, 0)] + ); + let open = Index::open(&dir.path().join("index/secret.sqlite3"), DataClass::Secret).unwrap(); + open.index_log(&log, "mm-c", 0).unwrap(); + assert_eq!( + items(&open)[1], + item("session", "owner", "after the secret", "secret", false, 2) + ); +} + +#[test] +fn notes_are_indexed_with_their_labels_and_only_once() { + let dir = TempDir::new("index-notes"); + let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); + let mut text = "§ noted 2026-09-24T20:14Z in session mm-a · private · untrusted\nexample.org says March.\n§ moved 2026-09-24T20:15Z from memory.md in session mm-b\nOwner prefers UTC.\n".to_string(); + assert_eq!(index.index_notes(&text), Ok(2)); + text.push_str("§ dropped 2026-09-24T20:16Z from user.md in session mm-b\nLives in Oslo.\n§ noted 2026-09-24T20:17Z in session mm-c · secret\nThe code.\n"); + assert_eq!(index.index_notes(&text), Ok(2)); + assert_eq!( + items(&index), + [ + item("noted", "", "example.org says March.", "private", true, 0), + item( + "moved", + "memory.md", + "Owner prefers UTC.", + "private", + false, + 1 + ), + item("dropped", "user.md", "Lives in Oslo.", "private", false, 2), + ] + ); +} + +#[test] +fn a_rebuild_equals_the_incremental_index_and_skips_damaged_logs() { + let dir = TempDir::new("index-rebuild"); + let home = dir.path(); + let log = write_log(home, "mm-a", &[user(1, "one")]); + write_log( + home, + "mm-b", + &[ + user(2, "two"), + result(3, "r", DataClass::Private, true), + assistant(4, Some("reply"), &[]), + ], + ); + std::fs::create_dir_all(home.join("sessions/mm-bad")).unwrap(); + std::fs::write(home.join("sessions/mm-bad/0.jsonl"), "{not json\n").unwrap(); + let notes = dir.write( + "memory/notes.md", + "§ noted 2026-09-24T20:14Z in session mm-a · private\nseen\n", + ); + let path = home.join("index/memory.sqlite3"); + let index = Index::open(&path, DataClass::Private).unwrap(); + let (counts, problems) = index.catch_up(&home.join("sessions"), ¬es).unwrap(); + assert_eq!((counts.logs, counts.records, counts.notes), (2, 4, 1)); + assert_eq!(problems.len(), 1); + assert!(problems[0].contains("mm-bad"), "{problems:?}"); + append(&log, &user(9, "later")); + index.catch_up(&home.join("sessions"), ¬es).unwrap(); + let incremental = items(&index); + drop(index); + let (counts, _) = rebuild(&path, DataClass::Private, &home.join("sessions"), ¬es).unwrap(); + assert_eq!((counts.logs, counts.records, counts.notes), (2, 5, 1)); + let rebuilt = Index::open(&path, DataClass::Private).unwrap(); + let mut a = incremental.clone(); + let mut b = items(&rebuilt); + a.sort(); + b.sort(); + assert_eq!(a, b); + assert!(!home.join("index/memory.rebuilding").exists()); +} + +#[test] +fn the_index_file_and_directory_are_private() { + use std::os::unix::fs::PermissionsExt; + let dir = TempDir::new("index-mode"); + let path = dir.path().join("index/memory.sqlite3"); + Index::open(&path, DataClass::Private).unwrap(); + let mode = |p: &std::path::Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; + assert_eq!( + (mode(&dir.path().join("index")), mode(&path)), + (0o700, 0o600) + ); +} diff --git a/crates/memoryd/tests/support/logs.rs b/crates/memoryd/tests/support/logs.rs new file mode 100644 index 0000000..37bbc33 --- /dev/null +++ b/crates/memoryd/tests/support/logs.rs @@ -0,0 +1,67 @@ +//! Session logs for tests: records written as `loopd` writes them. Do not edit. + +#![allow(dead_code)] // each test file uses a different part of this module + +use std::path::{Path, PathBuf}; + +use proto::{CallId, DataClass, LogRecord, Timestamp, ToolCall}; + +pub fn at(minute: u32) -> Timestamp { + Timestamp::parse(&format!("2026-09-20T14:{minute:02}:00.000Z")).unwrap() +} + +pub fn user(minute: u32, content: &str) -> LogRecord { + LogRecord::User { + time: at(minute), + content: content.to_string(), + } +} + +pub fn assistant(minute: u32, content: Option<&str>, calls: &[(&str, &str)]) -> LogRecord { + LogRecord::Assistant { + time: at(minute), + content: content.map(str::to_string), + reasoning_content: Some("thinking about it".to_string()), + tool_calls: calls + .iter() + .enumerate() + .map(|(i, (name, arguments))| ToolCall { + id: format!("c{i}"), + name: name.to_string(), + arguments: arguments.to_string(), + }) + .collect(), + } +} + +pub fn result(minute: u32, content: &str, class: DataClass, untrusted: bool) -> LogRecord { + LogRecord::ToolResult { + time: at(minute), + call: CallId(1), + tool_call_id: "c0".to_string(), + content: content.to_string(), + class, + untrusted, + truncated: false, + } +} + +/// Write `records` as `sessions//0.jsonl` under `home`. +pub fn write_log(home: &Path, session: &str, records: &[LogRecord]) -> PathBuf { + let dir = home.join("sessions").join(session); + std::fs::create_dir_all(&dir).unwrap(); + let path = dir.join("0.jsonl"); + let text: String = records + .iter() + .map(|r| format!("{}\n", serde_json::to_string(r).unwrap())) + .collect(); + std::fs::write(&path, text).unwrap(); + path +} + +/// Append one record to a log. +pub fn append(path: &Path, record: &LogRecord) { + use std::io::Write; + let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap(); + writeln!(file, "{}", serde_json::to_string(record).unwrap()).unwrap(); +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 27b5cdc..025546d 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone. | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| M5a/08-memoryd-index | 2026-09-24 | done | 2 | fail | none | Copied the skeleton `crates/memoryd/src/index.rs`, the test `crates/memoryd/tests/index.rs` and the `crates/memoryd/tests/support/logs.rs` into place, and added `pub mod index;` after `pub mod files;` in `lib.rs`. Filled the ten functions the skeleton left as `todo!()`: `open` (create the parent dir, make it 0700, `Connection::open`, make the file 0600, `execute_batch(SCHEMA)`); `progress` (`query_row` as `Option`, `QueryReturnedNoRows` -> None, other error through `db`, `usize::try_from` with 0 on overflow); `add` (insert into `hits`, `last_insert_rowid`, insert the `sources` row with `class_name(class)`); `set_progress` (upsert on `log`); `index_log` (`read_to_string`, `progress`, walk the label from Private with untrusted false, `index_record` each record at n >= done, every `ToolResult` raising the label to `class.max`/`untrusted |=`); `index_record` (nothing when `base.class > max_class`, User -> "owner", Assistant -> "assistant" then each call as `"{} {}"` -> "call", each item skipped when its text trims empty); `index_notes` (`notes::parse`, skip the first `done`, label per `Kind` — Noted/Moved/Dropped, skip when class > max_class, `record` is the note's place n); `catch_up` (`logs` then `notes.md`, a missing notes.md is empty text, a log that errors is pushed onto `problems`); `logs` (`read_dir`, NotFound -> empty, each session dir read and skipped on failure, `.jsonl` with n as i64, sort the list); `rebuild` (fresh file at `path.with_extension("rebuilding")`, open, `catch_up`, rename over the old). `cargo fmt --all`; the six `index` tests passed three runs; `make gate` printed `gate: ok` on the second run — the first failed on clippy `collapsible_if`, fixed by collapsing the nested `if let` into a let-chain guard. No new dependency, so `Cargo.lock` is unchanged. | ? | | M5a/07-memoryd-journal | 2026-09-24 | done | 2 | fail | none | Copied the skeletons `crates/memoryd/src/journal.rs` and `crates/memoryd/src/review.rs` and the test `crates/memoryd/tests/journal.rs` into place, and added `pub mod journal;` after `pub mod files;` and `pub mod review;` after `pub mod notes;` in `lib.rs`. Filled `journal.rs`: `records` (read_dir the `edits` directory, NotFound -> empty, other error by path, keep the `.jsonl` names, sort, read each and parse every line as a Record with a bad line reported `path: line N: e`); `append` (day = first 10 chars of `to_rfc3339()` or "0000-00-00", `taken` = records whose id starts `"{day}-"`, id `"{day}-{:04}"`, `create_dir_all` the edits dir, `serde_json::to_string`, append to `edits/{month}.jsonl`); `record` (append a Record with id empty, the time, session/call from ctx, action, file as its name or None, `to` that name or "notes.md", before/after, each note `render()`ed, untrusted, undoes None); `log` (last `count` of records, each line `id time session action to` plus " untrusted" when untrusted, "no memory edits yet" when empty); `undo` (find by id, file must be Some and known to `file_of`, the latest record of that file must be this one, `entries` must equal `after`, write `before`, append an undo record with session "owner"). Filled `review.rs`: `lines` (read `pending.jsonl`, NotFound -> empty, parse each line); `add` (to_string, append); `waiting` (ids with an Accepted or Rejected line are done, return the not-done Waiting lines in file order); `submit` (check, the daily id `p-{day}-{:04}`, add a Waiting line); `list` (one line per waiting with `escape_model_text` on the arguments, "nothing is waiting for review"); `accept` (find the Waiting line, parse its arguments, a Context from its session/call/taint with untrusted false, apply, `journal::record`, add an Accepted line); `reject` (find the Waiting line, add a Rejected line, `journal::append` a reject Record with its session/call). Removed the skeleton's unused `use crate::notes::Note;` import. `cargo fmt --all`; the journal test went 3 pass/1 fail then 4 passed; `make gate` printed `gate: ok` on the second run -- the first failed on clippy `double_ended_iterator_last` (fixed with `.rev().find()`) and `unnecessary_map_or` (fixed with `is_none_or`). No new dependency, so `Cargo.lock` is unchanged. | ? | | M5a/06-memoryd-edits | 2026-09-24 | done | 3 | fail | The task pointed at `docs/plans/M5a/06-memoryd-edits/files/` which does not exist; the four files were copied from `docs/plans/M5a/files/` instead. | Filled `notes.rs` first: `minute` (first 16 chars of `to_rfc3339()` plus "Z"), `header` (Noted = "§ noted {time} in session {session} · {class}" with " · untrusted" appended when untrusted; Moved/Dropped = "§ {moved|dropped} {time} from {file} in session {session}"), `render` (header, text, newline), `parse_header` (strip "§ ", split word then time, then "noted" parses session/class/[untrusted] on " · " with the empty-and-space session rules, "moved"/"dropped" parses file via `file_of` then "in session {session}", anything else None) and `parse` (a header starts a note with empty text, other lines append to the note above, text trimmed at the end). Then `edit.rs`: `noted` (checked text, a Noted note labelled by the session's taint and untrusted flag, appended to notes.md, answer "noted" or "noted (untrusted)"), `consolidate` (walk the lines, each old matched by `position` and each place named once or "twice", Keep/rename to after, Move/Drop to notes, New checked, a missing name reported), `edited` (add/replace/remove/consolidate with the duplicate, stale and not-an-identity-edit refusals, the Note arm returning an error the comment says `apply` never reaches), and `apply` (check, take the file, route an untrusted Add to `noted` with SAVED_AS_NOTE and every other untrusted action to `untrusted_refusal`, then before/after via `edited`, the duplicate check, the limit, write then append the moved notes). Added `pub mod edit;` after `pub mod config;` and `pub mod notes;` after `pub mod files;` in `lib.rs`. `cargo fmt --all`; `notes` 3 and `edit` 8 passed on the third run; `make gate` printed `gate: ok` — the first run failed on rustfmt import order, the second on clippy `manual_contains` (fixed by `before.contains(&text)`). No new dependency, so `Cargo.lock` is unchanged. | ? | | M5a/05-memoryd-files | 2026-09-24 | done | 2 | fail | none | Copied the skeleton `crates/memoryd/src/files.rs` and the test `crates/memoryd/tests/files.rs` into place, and added `pub mod files;` after `pub mod entries;` in `lib.rs`. Filled the seven functions the skeleton left as `todo!()`: `grouped` (walk the digits' chars with their index, before each char where i > 0 and `(len - i)` is a multiple of 3 push a comma), `init` (recursive `DirBuilder` at 0700 then `set_permissions` to 0700, then `soul.md` from the starter when missing, then each of memory/user/notes empty when missing, nothing existing changed), `create` (`create_new` at 0600 so an existing file errors and is never overwritten), `entries` (read through `fail`, then `entries::parse`), `write_entries` (remove any stale temp, `create` the rendered list at 0600, then rename over the file), `append` (append+create at 0600, one write), and `block` (soul read and trimmed, over SOUL_LIMIT errors, the two identity files each: over limit errors, entries checked with `check` and the refused ones pushed onto `left_out`, the size counting the whole file). `cargo fmt --all`; `files` 8 passed; `make gate` printed `gate: ok` on the second run — the first failed on clippy `useless_format`, fixed by using `.to_string()` for the empty-soul line. No new dependency, so `Cargo.lock` is unchanged. | ? |