memoryd: the FTS5 index over session logs and notes, with labels

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/memoryd/src/index.rs

@@ -0,0 +1,421 @@
+//! The index: SQLite FTS5 over the session logs and `notes.md` (M5a spec, section 6). Disposable:
+//! it is rebuilt from those files. What is indexed: the owner's messages, the assistant's
+//! replies, and each tool call by name and arguments; never a tool's result.
+
+use std::os::unix::fs::PermissionsExt;
+use std::path::{Path, PathBuf};
+
+use proto::{DataClass, LogRecord};
+use rusqlite::{Connection, params};
+
+use crate::notes::{self, Kind};
+
+const SCHEMA: &str = "
+CREATE VIRTUAL TABLE IF NOT EXISTS hits USING fts5(text, tokenize = 'unicode61');
+CREATE TABLE IF NOT EXISTS sources (
+  rowid INTEGER PRIMARY KEY, kind TEXT NOT NULL, session TEXT NOT NULL, epoch INTEGER NOT NULL,
+  record INTEGER NOT NULL, time TEXT NOT NULL, role TEXT NOT NULL, class TEXT NOT NULL,
+  untrusted INTEGER NOT NULL);
+CREATE TABLE IF NOT EXISTS progress (log TEXT PRIMARY KEY, records INTEGER NOT NULL);
+";
+
+/// One indexed item.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Source {
+    /// `session`, `noted`, `moved` or `dropped`.
+    pub kind: String,
+    pub session: String,
+    pub epoch: i64,
+    /// The record's line in its log, or the note's place in `notes.md` (from 0).
+    pub record: i64,
+    pub time: String,
+    /// `owner`, `assistant` or `call` for a session record; the file (`memory.md`, `user.md`) for
+    /// a moved or dropped note; empty for a noted one.
+    pub role: String,
+    pub class: DataClass,
+    pub untrusted: bool,
+}
+
+/// How much a reindex read.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub struct Counts {
+    pub logs: u64,
+    pub records: u64,
+    pub notes: u64,
+}
+
+pub struct Index {
+    conn: Connection,
+    max_class: DataClass,
+}
+
+fn db(e: rusqlite::Error) -> String {
+    format!("index: {e}")
+}
+
+pub fn class_name(class: DataClass) -> &'static str {
+    match class {
+        DataClass::Public => "public",
+        DataClass::Private => "private",
+        DataClass::Secret => "secret",
+    }
+}
+
+pub fn class_of(name: &str) -> DataClass {
+    match name {
+        "public" => DataClass::Public,
+        "secret" => DataClass::Secret,
+        _ => DataClass::Private,
+    }
+}
+
+fn int(n: usize) -> i64 {
+    i64::try_from(n).unwrap_or(i64::MAX)
+}
+
+impl Index {
+    /// Open or create the index at `path` (its directory 0700, the file 0600).
+    pub fn open(path: &Path, max_class: DataClass) -> Result<Index, String> {
+        if let Some(dir) = path.parent() {
+            std::fs::create_dir_all(dir).map_err(|e| format!("{}: {e}", dir.display()))?;
+            std::fs::set_permissions(dir, std::fs::Permissions::from_mode(0o700))
+                .map_err(|e| format!("{}: {e}", dir.display()))?;
+        }
+        let conn = Connection::open(path).map_err(|e| format!("{}: {e}", path.display()))?;
+        std::fs::set_permissions(path, std::fs::Permissions::from_mode(0o600))
+            .map_err(|e| format!("{}: {e}", path.display()))?;
+        conn.execute_batch(SCHEMA)
+            .map_err(|e| format!("{}: {e}", path.display()))?;
+        Ok(Index { conn, max_class })
+    }
+
+    /// The connection, for `recall`.
+    pub fn conn(&self) -> &Connection {
+        &self.conn
+    }
+
+    fn progress(&self, log: &str) -> Result<usize, String> {
+        // `self.conn.query_row("SELECT records FROM progress WHERE log = ?1", params![log], |row|
+        // row.get(0))` as an Option<i64>: Err(rusqlite::Error::QueryReturnedNoRows) means None,
+        // another error goes through `db`. Return it as a usize (`usize::try_from`, 0 when it does
+        // not fit), 0 for None.
+        let value: Option<i64> = match self.conn.query_row(
+            "SELECT records FROM progress WHERE log = ?1",
+            params![log],
+            |row| row.get(0),
+        ) {
+            Ok(v) => Some(v),
+            Err(rusqlite::Error::QueryReturnedNoRows) => None,
+            Err(e) => return Err(db(e)),
+        };
+        let n = match value {
+            Some(v) => usize::try_from(v).unwrap_or(0),
+            None => 0,
+        };
+        Ok(n)
+    }
+
+    fn add(&self, text: &str, source: &Source) -> Result<(), String> {
+        // 1. `execute("INSERT INTO hits (text) VALUES (?1)", params![text])`.
+        // 2. `let rowid = self.conn.last_insert_rowid();`
+        // 3. `execute("INSERT INTO sources (rowid, kind, session, epoch, record, time, role, class,
+        //    untrusted) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", params![rowid, kind, session,
+        //    epoch, record, time, role, class_name(class), untrusted])`. Errors through `db`.
+        self.conn
+            .execute("INSERT INTO hits (text) VALUES (?1)", params![text])
+            .map_err(db)?;
+        let rowid = self.conn.last_insert_rowid();
+        self.conn
+            .execute(
+                "INSERT INTO sources (rowid, kind, session, epoch, record, time, role, class, untrusted) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)",
+                params![
+                    rowid,
+                    source.kind.clone(),
+                    source.session.clone(),
+                    source.epoch,
+                    source.record,
+                    source.time.clone(),
+                    source.role.clone(),
+                    class_name(source.class),
+                    source.untrusted,
+                ],
+            )
+            .map_err(db)?;
+        Ok(())
+    }
+
+    fn set_progress(&self, log: &str, records: usize) -> Result<(), String> {
+        // `execute("INSERT INTO progress (log, records) VALUES (?1, ?2) ON CONFLICT(log) DO UPDATE
+        //    SET records = excluded.records", params![log, int(records)])`, errors through `db`.
+        self.conn
+            .execute(
+                "INSERT INTO progress (log, records) VALUES (?1, ?2) ON CONFLICT(log) DO UPDATE SET records = excluded.records",
+                params![log, int(records)],
+            )
+            .map_err(db)?;
+        Ok(())
+    }
+
+    /// Index the records of one session log past what was indexed before; how many were read.
+    /// The label is walked from the log's first record: the highest class of any tool result
+    /// before a record (at least `private`), and whether any was untrusted.
+    pub fn index_log(&self, path: &Path, session: &str, epoch: i64) -> Result<usize, String> {
+        // 1. `read_to_string(path)` (an error names the path). `key` =
+        //    format!("{session}/{epoch}"); `done = self.progress(&key)?`.
+        // 2. The label starts at class Private, untrusted false; `read` at 0.
+        // 3. For each line with its place n (from 0): parse it as a LogRecord (an error ->
+        //    Err(format!("{}: line {}: {e}", path.display(), n + 1))). When n >= done: read += 1,
+        //    and `self.index_record(&record, &base)?` where base is a Source with kind "session",
+        //    session, epoch, record int(n), time and role empty, and the label as it stands. Then,
+        //    for every record (indexed or not): a ToolResult raises the label: class =
+        //    class.max(its class), untrusted |= its untrusted.
+        // 4. `self.set_progress(&key, done + read)?`; Ok(read).
+        let text = std::fs::read_to_string(path).map_err(|e| format!("{}: {e}", path.display()))?;
+        let key = format!("{session}/{epoch}");
+        let done = self.progress(&key)?;
+        let mut class = DataClass::Private;
+        let mut untrusted = false;
+        let mut read = 0;
+        for (n, line) in text.lines().enumerate() {
+            let record: LogRecord = serde_json::from_str(line)
+                .map_err(|e| format!("{}: line {}: {e}", path.display(), n + 1))?;
+            if n >= done {
+                read += 1;
+                let base = Source {
+                    kind: "session".to_string(),
+                    session: session.to_string(),
+                    epoch,
+                    record: int(n),
+                    time: String::new(),
+                    role: String::new(),
+                    class,
+                    untrusted,
+                };
+                self.index_record(&record, &base)?;
+            }
+            if let LogRecord::ToolResult {
+                class: c,
+                untrusted: u,
+                ..
+            } = &record
+            {
+                class = class.max(*c);
+                untrusted |= *u;
+            }
+        }
+        self.set_progress(&key, done + read)?;
+        Ok(read)
+    }
+
+    /// Index what one log record holds: the owner's message, the assistant's reply, and each tool
+    /// call by name and arguments. `base` carries the kind, session, epoch, line and label; each item
+    /// gets its own role and time. Nothing when the label's class is above `max_class`, and nothing
+    /// for an item with no text.
+    fn index_record(&self, record: &LogRecord, base: &Source) -> Result<(), String> {
+        // When base.class > self.max_class: Ok(()) and nothing indexed. Otherwise each item is
+        // `self.add(text, &Source { time: time.to_rfc3339(), role, ..base.clone() })`, skipped when
+        // text.trim() is empty. User -> its content, role "owner". Assistant -> its content when
+        // Some, role "assistant"; then each tool call as format!("{} {}", name, arguments), role
+        //    "call"; all with the record's time. Every other record: nothing.
+        if base.class > self.max_class {
+            return Ok(());
+        }
+        match record {
+            LogRecord::User { time, content } => {
+                if !content.trim().is_empty() {
+                    let src = Source {
+                        time: time.to_rfc3339(),
+                        role: "owner".to_string(),
+                        ..base.clone()
+                    };
+                    self.add(content, &src)?;
+                }
+            }
+            LogRecord::Assistant {
+                time,
+                content,
+                tool_calls,
+                ..
+            } => {
+                if let Some(c) = content
+                    && !c.trim().is_empty()
+                {
+                    let src = Source {
+                        time: time.to_rfc3339(),
+                        role: "assistant".to_string(),
+                        ..base.clone()
+                    };
+                    self.add(c, &src)?;
+                }
+                for call in tool_calls {
+                    let text = format!("{} {}", call.name, call.arguments);
+                    if !text.trim().is_empty() {
+                        let src = Source {
+                            time: time.to_rfc3339(),
+                            role: "call".to_string(),
+                            ..base.clone()
+                        };
+                        self.add(&text, &src)?;
+                    }
+                }
+            }
+            _ => {}
+        }
+        Ok(())
+    }
+
+    /// Index the entries of `notes.md` past what was indexed before; how many were read.
+    pub fn index_notes(&self, text: &str) -> Result<usize, String> {
+        // `done = self.progress("notes.md")?`. Parse `text` with notes::parse and skip the first
+        // `done`. For each remaining note (with its place n): read += 1; its kind, role, class and
+        // untrusted: Noted -> ("noted", "", its class, its untrusted); Moved(file) -> ("moved",
+        // file.name(), Private, false); Dropped(file) -> ("dropped", file.name(), Private, false).
+        // Skip it when class > self.max_class. Otherwise `self.add(&note.text, &Source { kind,
+        // session: note.session, epoch: 0, record: int(n), time: note.time, role, class, untrusted
+        //    })?`. Then `self.set_progress("notes.md", done + read)?`; Ok(read).
+        let done = self.progress("notes.md")?;
+        let notes = notes::parse(text);
+        let mut read = 0;
+        for (n, note) in notes.iter().enumerate() {
+            if n < done {
+                continue;
+            }
+            read += 1;
+            let (kind, role, class, untrusted) = match &note.kind {
+                Kind::Noted { class, untrusted } => {
+                    ("noted".to_string(), String::new(), *class, *untrusted)
+                }
+                Kind::Moved(file) => (
+                    "moved".to_string(),
+                    file.name().to_string(),
+                    DataClass::Private,
+                    false,
+                ),
+                Kind::Dropped(file) => (
+                    "dropped".to_string(),
+                    file.name().to_string(),
+                    DataClass::Private,
+                    false,
+                ),
+            };
+            if class > self.max_class {
+                continue;
+            }
+            self.add(
+                &note.text,
+                &Source {
+                    kind,
+                    session: note.session.clone(),
+                    epoch: 0,
+                    record: int(n),
+                    time: note.time.clone(),
+                    role,
+                    class,
+                    untrusted,
+                },
+            )?;
+        }
+        self.set_progress("notes.md", done + read)?;
+        Ok(read)
+    }
+
+    /// Index everything new: every `sessions/<id>/<epoch>.jsonl`, then `notes.md`. A log that
+    /// cannot be read or parsed is skipped and named in the returned problems; it is `loopd`'s.
+    pub fn catch_up(
+        &self,
+        sessions: &Path,
+        notes_md: &Path,
+    ) -> Result<(Counts, Vec<String>), String> {
+        // For each (session, epoch, path) of `logs(sessions)?`: `self.index_log(&path, &session,
+        // epoch)`; Ok(n) -> counts.logs += 1 and counts.records += n (u64::try_from, u64::MAX when
+        // it does not fit); Err(e) -> push e onto `problems` and go on. Then read `notes_md`:
+        //    NotFound is an empty text, another error -> Err(format!("{}: {e}", notes_md.display()));
+        //    counts.notes = `self.index_notes(&text)?` as u64 (u64::try_from). Ok((counts, problems)).
+        let mut counts = Counts::default();
+        let mut problems: Vec<String> = Vec::new();
+        for (session, epoch, path) in logs(sessions)? {
+            match self.index_log(&path, &session, epoch) {
+                Ok(n) => {
+                    counts.logs += 1;
+                    counts.records += u64::try_from(n).unwrap_or(u64::MAX);
+                }
+                Err(e) => problems.push(e),
+            }
+        }
+        let text = match std::fs::read_to_string(notes_md) {
+            Ok(t) => t,
+            Err(e) if e.kind() == std::io::ErrorKind::NotFound => String::new(),
+            Err(e) => return Err(format!("{}: {e}", notes_md.display())),
+        };
+        counts.notes = u64::try_from(self.index_notes(&text)?).unwrap_or(u64::MAX);
+        Ok((counts, problems))
+    }
+}
+
+/// Every session log under `sessions`: `(session, epoch, path)`, sorted.
+pub fn logs(sessions: &Path) -> Result<Vec<(String, i64, PathBuf)>, String> {
+    // Read the directory `sessions` (NotFound -> Ok(empty); another error -> Err naming it). For
+    // each entry, its name is the session; read that directory (skip it when that fails); every
+    //    file named `<n>.jsonl` where n parses as an i64 gives (session, n, the file's path). Sort the
+    //    list and return it.
+    let entries = match std::fs::read_dir(sessions) {
+        Ok(e) => e,
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => return Ok(Vec::new()),
+        Err(e) => return Err(format!("{}: {e}", sessions.display())),
+    };
+    let mut out: Vec<(String, i64, PathBuf)> = Vec::new();
+    for entry in entries {
+        let entry = match entry {
+            Ok(e) => e,
+            Err(_) => continue,
+        };
+        let name = entry.file_name();
+        let session = match name.to_str() {
+            Some(s) => s,
+            None => continue,
+        };
+        let dir = entry.path();
+        let sub = match std::fs::read_dir(&dir) {
+            Ok(d) => d,
+            Err(_) => continue,
+        };
+        for sub_entry in sub {
+            let sub_entry = match sub_entry {
+                Ok(e) => e,
+                Err(_) => continue,
+            };
+            let sub_name = sub_entry.file_name();
+            let file_name = match sub_name.to_str() {
+                Some(s) => s,
+                None => continue,
+            };
+            if let Some(stem) = file_name.strip_suffix(".jsonl")
+                && let Ok(epoch) = stem.parse::<i64>()
+            {
+                out.push((session.to_string(), epoch, sub_entry.path()));
+            }
+        }
+    }
+    out.sort();
+    Ok(out)
+}
+
+/// Build a fresh index at `path` from `sessions` and `notes.md`: into a new file beside it,
+/// renamed over the old one, so a running `recall` keeps its index until the new one is whole.
+pub fn rebuild(
+    path: &Path,
+    max_class: DataClass,
+    sessions: &Path,
+    notes_md: &Path,
+) -> Result<(Counts, Vec<String>), String> {
+    // `fresh = path.with_extension("rebuilding")`; remove it if it is there (ignore the result).
+    //    `Index::open(&fresh, max_class)?`, then `catch_up(sessions, notes_md)?` on it, and drop that
+    //    Index. `std::fs::rename(&fresh, path)` (an error names path). Return the result of catch_up.
+    let fresh = path.with_extension("rebuilding");
+    let _ = std::fs::remove_file(&fresh);
+    let index = Index::open(&fresh, max_class)?;
+    let result = index.catch_up(sessions, notes_md)?;
+    drop(index);
+    std::fs::rename(&fresh, path).map_err(|e| format!("{}: {e}", path.display()))?;
+    Ok(result)
+}

crates/memoryd/src/lib.rs

@@ -5,6 +5,7 @@ pub mod config;
 pub mod edit;
 pub mod entries;
 pub mod files;
+pub mod index;
 pub mod journal;
 pub mod notes;
 pub mod review;

crates/memoryd/tests/index.rs

@@ -0,0 +1,229 @@
+//! The index: what is indexed from a session log and `notes.md`, each record's label, catching up,
+//! `max_class`, and a rebuild equal to indexing as it went (M5a spec, section 6). Do not edit.
+
+#[path = "support/logs.rs"]
+mod logs;
+#[path = "support/tmp.rs"]
+mod tmp;
+
+use logs::{append, assistant, result, user, write_log};
+use memoryd::index::{Index, rebuild};
+use proto::DataClass;
+use tmp::TempDir;
+
+/// Every indexed item, in order: (kind, role, text, class, untrusted, record).
+fn items(index: &Index) -> Vec<(String, String, String, String, bool, i64)> {
+    let mut stmt = index
+        .conn()
+        .prepare("SELECT s.kind, s.role, h.text, s.class, s.untrusted, s.record FROM sources s JOIN hits h ON h.rowid = s.rowid ORDER BY s.rowid")
+        .unwrap();
+    stmt.query_map([], |r| {
+        Ok((
+            r.get(0)?,
+            r.get(1)?,
+            r.get(2)?,
+            r.get(3)?,
+            r.get(4)?,
+            r.get(5)?,
+        ))
+    })
+    .unwrap()
+    .map(|r| r.unwrap())
+    .collect()
+}
+
+fn item(
+    kind: &str,
+    role: &str,
+    text: &str,
+    class: &str,
+    untrusted: bool,
+    record: i64,
+) -> (String, String, String, String, bool, i64) {
+    (
+        kind.to_string(),
+        role.to_string(),
+        text.to_string(),
+        class.to_string(),
+        untrusted,
+        record,
+    )
+}
+
+#[test]
+fn owner_assistant_and_calls_are_indexed_with_the_label_in_force() {
+    let dir = TempDir::new("index-log");
+    let log = write_log(
+        dir.path(),
+        "mm-a",
+        &[
+            user(1, "What is the launch window?"),
+            assistant(
+                2,
+                None,
+                &[("http_fetch", r#"{"url":"https://example.org"}"#)],
+            ),
+            result(3, "PAGE TEXT never indexed", DataClass::Private, true),
+            assistant(4, Some("The page says Thursday."), &[]),
+            user(5, "   "),
+        ],
+    );
+    let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
+    assert_eq!(index.index_log(&log, "mm-a", 0), Ok(5));
+    assert_eq!(
+        items(&index),
+        [
+            item(
+                "session",
+                "owner",
+                "What is the launch window?",
+                "private",
+                false,
+                0
+            ),
+            item(
+                "session",
+                "call",
+                r#"http_fetch {"url":"https://example.org"}"#,
+                "private",
+                false,
+                1
+            ),
+            item(
+                "session",
+                "assistant",
+                "The page says Thursday.",
+                "private",
+                true,
+                3
+            ),
+        ]
+    );
+}
+
+#[test]
+fn catching_up_reads_only_new_records_and_keeps_the_label() {
+    let dir = TempDir::new("index-catch");
+    let log = write_log(
+        dir.path(),
+        "mm-b",
+        &[user(1, "first"), result(2, "x", DataClass::Private, true)],
+    );
+    let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
+    assert_eq!(index.index_log(&log, "mm-b", 0), Ok(2));
+    append(&log, &user(3, "second"));
+    assert_eq!(index.index_log(&log, "mm-b", 0), Ok(1));
+    assert_eq!(index.index_log(&log, "mm-b", 0), Ok(0));
+    let got = items(&index);
+    assert_eq!(got.len(), 2);
+    assert_eq!(
+        got[1],
+        item("session", "owner", "second", "private", true, 2),
+        "tainted by the earlier result"
+    );
+}
+
+#[test]
+fn records_above_max_class_are_not_indexed() {
+    let dir = TempDir::new("index-class");
+    let log = write_log(
+        dir.path(),
+        "mm-c",
+        &[
+            user(1, "before"),
+            result(2, "s", DataClass::Secret, false),
+            user(3, "after the secret"),
+        ],
+    );
+    let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
+    index.index_log(&log, "mm-c", 0).unwrap();
+    assert_eq!(
+        items(&index),
+        [item("session", "owner", "before", "private", false, 0)]
+    );
+    let open = Index::open(&dir.path().join("index/secret.sqlite3"), DataClass::Secret).unwrap();
+    open.index_log(&log, "mm-c", 0).unwrap();
+    assert_eq!(
+        items(&open)[1],
+        item("session", "owner", "after the secret", "secret", false, 2)
+    );
+}
+
+#[test]
+fn notes_are_indexed_with_their_labels_and_only_once() {
+    let dir = TempDir::new("index-notes");
+    let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
+    let mut text = "§ noted 2026-09-24T20:14Z in session mm-a · private · untrusted\nexample.org says March.\n§ moved 2026-09-24T20:15Z from memory.md in session mm-b\nOwner prefers UTC.\n".to_string();
+    assert_eq!(index.index_notes(&text), Ok(2));
+    text.push_str("§ dropped 2026-09-24T20:16Z from user.md in session mm-b\nLives in Oslo.\n§ noted 2026-09-24T20:17Z in session mm-c · secret\nThe code.\n");
+    assert_eq!(index.index_notes(&text), Ok(2));
+    assert_eq!(
+        items(&index),
+        [
+            item("noted", "", "example.org says March.", "private", true, 0),
+            item(
+                "moved",
+                "memory.md",
+                "Owner prefers UTC.",
+                "private",
+                false,
+                1
+            ),
+            item("dropped", "user.md", "Lives in Oslo.", "private", false, 2),
+        ]
+    );
+}
+
+#[test]
+fn a_rebuild_equals_the_incremental_index_and_skips_damaged_logs() {
+    let dir = TempDir::new("index-rebuild");
+    let home = dir.path();
+    let log = write_log(home, "mm-a", &[user(1, "one")]);
+    write_log(
+        home,
+        "mm-b",
+        &[
+            user(2, "two"),
+            result(3, "r", DataClass::Private, true),
+            assistant(4, Some("reply"), &[]),
+        ],
+    );
+    std::fs::create_dir_all(home.join("sessions/mm-bad")).unwrap();
+    std::fs::write(home.join("sessions/mm-bad/0.jsonl"), "{not json\n").unwrap();
+    let notes = dir.write(
+        "memory/notes.md",
+        "§ noted 2026-09-24T20:14Z in session mm-a · private\nseen\n",
+    );
+    let path = home.join("index/memory.sqlite3");
+    let index = Index::open(&path, DataClass::Private).unwrap();
+    let (counts, problems) = index.catch_up(&home.join("sessions"), &notes).unwrap();
+    assert_eq!((counts.logs, counts.records, counts.notes), (2, 4, 1));
+    assert_eq!(problems.len(), 1);
+    assert!(problems[0].contains("mm-bad"), "{problems:?}");
+    append(&log, &user(9, "later"));
+    index.catch_up(&home.join("sessions"), &notes).unwrap();
+    let incremental = items(&index);
+    drop(index);
+    let (counts, _) = rebuild(&path, DataClass::Private, &home.join("sessions"), &notes).unwrap();
+    assert_eq!((counts.logs, counts.records, counts.notes), (2, 5, 1));
+    let rebuilt = Index::open(&path, DataClass::Private).unwrap();
+    let mut a = incremental.clone();
+    let mut b = items(&rebuilt);
+    a.sort();
+    b.sort();
+    assert_eq!(a, b);
+    assert!(!home.join("index/memory.rebuilding").exists());
+}
+
+#[test]
+fn the_index_file_and_directory_are_private() {
+    use std::os::unix::fs::PermissionsExt;
+    let dir = TempDir::new("index-mode");
+    let path = dir.path().join("index/memory.sqlite3");
+    Index::open(&path, DataClass::Private).unwrap();
+    let mode = |p: &std::path::Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777;
+    assert_eq!(
+        (mode(&dir.path().join("index")), mode(&path)),
+        (0o700, 0o600)
+    );
+}

crates/memoryd/tests/support/logs.rs

@@ -0,0 +1,67 @@
+//! Session logs for tests: records written as `loopd` writes them. Do not edit.
+
+#![allow(dead_code)] // each test file uses a different part of this module
+
+use std::path::{Path, PathBuf};
+
+use proto::{CallId, DataClass, LogRecord, Timestamp, ToolCall};
+
+pub fn at(minute: u32) -> Timestamp {
+    Timestamp::parse(&format!("2026-09-20T14:{minute:02}:00.000Z")).unwrap()
+}
+
+pub fn user(minute: u32, content: &str) -> LogRecord {
+    LogRecord::User {
+        time: at(minute),
+        content: content.to_string(),
+    }
+}
+
+pub fn assistant(minute: u32, content: Option<&str>, calls: &[(&str, &str)]) -> LogRecord {
+    LogRecord::Assistant {
+        time: at(minute),
+        content: content.map(str::to_string),
+        reasoning_content: Some("thinking about it".to_string()),
+        tool_calls: calls
+            .iter()
+            .enumerate()
+            .map(|(i, (name, arguments))| ToolCall {
+                id: format!("c{i}"),
+                name: name.to_string(),
+                arguments: arguments.to_string(),
+            })
+            .collect(),
+    }
+}
+
+pub fn result(minute: u32, content: &str, class: DataClass, untrusted: bool) -> LogRecord {
+    LogRecord::ToolResult {
+        time: at(minute),
+        call: CallId(1),
+        tool_call_id: "c0".to_string(),
+        content: content.to_string(),
+        class,
+        untrusted,
+        truncated: false,
+    }
+}
+
+/// Write `records` as `sessions/<session>/0.jsonl` under `home`.
+pub fn write_log(home: &Path, session: &str, records: &[LogRecord]) -> PathBuf {
+    let dir = home.join("sessions").join(session);
+    std::fs::create_dir_all(&dir).unwrap();
+    let path = dir.join("0.jsonl");
+    let text: String = records
+        .iter()
+        .map(|r| format!("{}\n", serde_json::to_string(r).unwrap()))
+        .collect();
+    std::fs::write(&path, text).unwrap();
+    path
+}
+
+/// Append one record to a log.
+pub fn append(path: &Path, record: &LogRecord) {
+    use std::io::Write;
+    let mut file = std::fs::OpenOptions::new().append(true).open(path).unwrap();
+    writeln!(file, "{}", serde_json::to_string(record).unwrap()).unwrap();
+}

docs/implementer-log.md

@@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone.
 
 | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model |
 |---|---|---|---|---|---|---|---|
+| M5a/08-memoryd-index | 2026-09-24 | done | 2 | fail | none | Copied the skeleton `crates/memoryd/src/index.rs`, the test `crates/memoryd/tests/index.rs` and the `crates/memoryd/tests/support/logs.rs` into place, and added `pub mod index;` after `pub mod files;` in `lib.rs`. Filled the ten functions the skeleton left as `todo!()`: `open` (create the parent dir, make it 0700, `Connection::open`, make the file 0600, `execute_batch(SCHEMA)`); `progress` (`query_row` as `Option<i64>`, `QueryReturnedNoRows` -> None, other error through `db`, `usize::try_from` with 0 on overflow); `add` (insert into `hits`, `last_insert_rowid`, insert the `sources` row with `class_name(class)`); `set_progress` (upsert on `log`); `index_log` (`read_to_string`, `progress`, walk the label from Private with untrusted false, `index_record` each record at n >= done, every `ToolResult` raising the label to `class.max`/`untrusted |=`); `index_record` (nothing when `base.class > max_class`, User -> "owner", Assistant -> "assistant" then each call as `"{} {}"` -> "call", each item skipped when its text trims empty); `index_notes` (`notes::parse`, skip the first `done`, label per `Kind` — Noted/Moved/Dropped, skip when class > max_class, `record` is the note's place n); `catch_up` (`logs` then `notes.md`, a missing notes.md is empty text, a log that errors is pushed onto `problems`); `logs` (`read_dir`, NotFound -> empty, each session dir read and skipped on failure, `<n>.jsonl` with n as i64, sort the list); `rebuild` (fresh file at `path.with_extension("rebuilding")`, open, `catch_up`, rename over the old). `cargo fmt --all`; the six `index` tests passed three runs; `make gate` printed `gate: ok` on the second run — the first failed on clippy `collapsible_if`, fixed by collapsing the nested `if let` into a let-chain guard. No new dependency, so `Cargo.lock` is unchanged. | ? |
 | M5a/07-memoryd-journal | 2026-09-24 | done | 2 | fail | none | Copied the skeletons `crates/memoryd/src/journal.rs` and `crates/memoryd/src/review.rs` and the test `crates/memoryd/tests/journal.rs` into place, and added `pub mod journal;` after `pub mod files;` and `pub mod review;` after `pub mod notes;` in `lib.rs`. Filled `journal.rs`: `records` (read_dir the `edits` directory, NotFound -> empty, other error by path, keep the `.jsonl` names, sort, read each and parse every line as a Record with a bad line reported `path: line N: e`); `append` (day = first 10 chars of `to_rfc3339()` or "0000-00-00", `taken` = records whose id starts `"{day}-"`, id `"{day}-{:04}"`, `create_dir_all` the edits dir, `serde_json::to_string`, append to `edits/{month}.jsonl`); `record` (append a Record with id empty, the time, session/call from ctx, action, file as its name or None, `to` that name or "notes.md", before/after, each note `render()`ed, untrusted, undoes None); `log` (last `count` of records, each line `id  time  session  action  to` plus "  untrusted" when untrusted, "no memory edits yet" when empty); `undo` (find by id, file must be Some and known to `file_of`, the latest record of that file must be this one, `entries` must equal `after`, write `before`, append an undo record with session "owner"). Filled `review.rs`: `lines` (read `pending.jsonl`, NotFound -> empty, parse each line); `add` (to_string, append); `waiting` (ids with an Accepted or Rejected line are done, return the not-done Waiting lines in file order); `submit` (check, the daily id `p-{day}-{:04}`, add a Waiting line); `list` (one line per waiting with `escape_model_text` on the arguments, "nothing is waiting for review"); `accept` (find the Waiting line, parse its arguments, a Context from its session/call/taint with untrusted false, apply, `journal::record`, add an Accepted line); `reject` (find the Waiting line, add a Rejected line, `journal::append` a reject Record with its session/call). Removed the skeleton's unused `use crate::notes::Note;` import. `cargo fmt --all`; the journal test went 3 pass/1 fail then 4 passed; `make gate` printed `gate: ok` on the second run -- the first failed on clippy `double_ended_iterator_last` (fixed with `.rev().find()`) and `unnecessary_map_or` (fixed with `is_none_or`). No new dependency, so `Cargo.lock` is unchanged. | ? |
 | M5a/06-memoryd-edits | 2026-09-24 | done | 3 | fail | The task pointed at `docs/plans/M5a/06-memoryd-edits/files/` which does not exist; the four files were copied from `docs/plans/M5a/files/` instead. | Filled `notes.rs` first: `minute` (first 16 chars of `to_rfc3339()` plus "Z"), `header` (Noted = "§ noted {time} in session {session} · {class}" with " · untrusted" appended when untrusted; Moved/Dropped = "§ {moved|dropped} {time} from {file} in session {session}"), `render` (header, text, newline), `parse_header` (strip "§ ", split word then time, then "noted" parses session/class/[untrusted] on " · " with the empty-and-space session rules, "moved"/"dropped" parses file via `file_of` then "in session {session}", anything else None) and `parse` (a header starts a note with empty text, other lines append to the note above, text trimmed at the end). Then `edit.rs`: `noted` (checked text, a Noted note labelled by the session's taint and untrusted flag, appended to notes.md, answer "noted" or "noted (untrusted)"), `consolidate` (walk the lines, each old matched by `position` and each place named once or "twice", Keep/rename to after, Move/Drop to notes, New checked, a missing name reported), `edited` (add/replace/remove/consolidate with the duplicate, stale and not-an-identity-edit refusals, the Note arm returning an error the comment says `apply` never reaches), and `apply` (check, take the file, route an untrusted Add to `noted` with SAVED_AS_NOTE and every other untrusted action to `untrusted_refusal`, then before/after via `edited`, the duplicate check, the limit, write then append the moved notes). Added `pub mod edit;` after `pub mod config;` and `pub mod notes;` after `pub mod files;` in `lib.rs`. `cargo fmt --all`; `notes` 3 and `edit` 8 passed on the third run; `make gate` printed `gate: ok` — the first run failed on rustfmt import order, the second on clippy `manual_contains` (fixed by `before.contains(&text)`). No new dependency, so `Cargo.lock` is unchanged. | ? |
 | M5a/05-memoryd-files | 2026-09-24 | done | 2 | fail | none | Copied the skeleton `crates/memoryd/src/files.rs` and the test `crates/memoryd/tests/files.rs` into place, and added `pub mod files;` after `pub mod entries;` in `lib.rs`. Filled the seven functions the skeleton left as `todo!()`: `grouped` (walk the digits' chars with their index, before each char where i > 0 and `(len - i)` is a multiple of 3 push a comma), `init` (recursive `DirBuilder` at 0700 then `set_permissions` to 0700, then `soul.md` from the starter when missing, then each of memory/user/notes empty when missing, nothing existing changed), `create` (`create_new` at 0600 so an existing file errors and is never overwritten), `entries` (read through `fail`, then `entries::parse`), `write_entries` (remove any stale temp, `create` the rendered list at 0600, then rename over the file), `append` (append+create at 0600, one write), and `block` (soul read and trimmed, over SOUL_LIMIT errors, the two identity files each: over limit errors, entries checked with `check` and the refused ones pushed onto `left_out`, the size counting the whole file). `cargo fmt --all`; `files` 8 passed; `make gate` printed `gate: ok` on the second run — the first failed on clippy `useless_format`, fixed by using `.to_string()` for the empty-soul line. No new dependency, so `Cargo.lock` is unchanged. | ? |