//! The index: what is indexed from a session log and `notes.md`, each record's label, catching up, //! `max_class`, and a rebuild equal to indexing as it went (M5a spec, section 6). Do not edit. #[path = "support/logs.rs"] mod logs; #[path = "support/tmp.rs"] mod tmp; use bxcore::DataClass; use logs::{append, assistant, result, user, write_log}; use memoryd::index::{Index, rebuild}; use tmp::TempDir; /// Every indexed item, in order: (kind, role, text, class, untrusted, record). fn items(index: &Index) -> Vec<(String, String, String, String, bool, i64)> { let mut stmt = index .conn() .prepare("SELECT s.kind, s.role, h.text, s.class, s.untrusted, s.record FROM sources s JOIN hits h ON h.rowid = s.rowid ORDER BY s.rowid") .unwrap(); stmt.query_map([], |r| { Ok(( r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?, r.get(5)?, )) }) .unwrap() .map(|r| r.unwrap()) .collect() } fn item( kind: &str, role: &str, text: &str, class: &str, untrusted: bool, record: i64, ) -> (String, String, String, String, bool, i64) { ( kind.to_string(), role.to_string(), text.to_string(), class.to_string(), untrusted, record, ) } #[test] fn owner_assistant_and_calls_are_indexed_with_the_label_in_force() { let dir = TempDir::new("index-log"); let log = write_log( dir.path(), "mm-a", &[ user(1, "What is the launch window?"), assistant( 2, None, &[("http_fetch", r#"{"url":"https://example.org"}"#)], ), result(3, "PAGE TEXT never indexed", DataClass::Private, true), assistant(4, Some("The page says Thursday."), &[]), user(5, " "), ], ); let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); assert_eq!(index.index_log(&log, "mm-a", 0), Ok(5)); assert_eq!( items(&index), [ item( "session", "owner", "What is the launch window?", "private", false, 0 ), item( "session", "call", r#"http_fetch {"url":"https://example.org"}"#, "private", false, 1 ), item( "session", "assistant", "The page says Thursday.", "private", true, 3 ), ] ); } #[test] fn catching_up_reads_only_new_records_and_keeps_the_label() { let dir = TempDir::new("index-catch"); let log = write_log( dir.path(), "mm-b", &[user(1, "first"), result(2, "x", DataClass::Private, true)], ); let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); assert_eq!(index.index_log(&log, "mm-b", 0), Ok(2)); append(&log, &user(3, "second")); assert_eq!(index.index_log(&log, "mm-b", 0), Ok(1)); assert_eq!(index.index_log(&log, "mm-b", 0), Ok(0)); let got = items(&index); assert_eq!(got.len(), 2); assert_eq!( got[1], item("session", "owner", "second", "private", true, 2), "tainted by the earlier result" ); } #[test] fn records_above_max_class_are_not_indexed() { let dir = TempDir::new("index-class"); let log = write_log( dir.path(), "mm-c", &[ user(1, "before"), result(2, "s", DataClass::Secret, false), user(3, "after the secret"), ], ); let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); index.index_log(&log, "mm-c", 0).unwrap(); assert_eq!( items(&index), [item("session", "owner", "before", "private", false, 0)] ); let open = Index::open(&dir.path().join("index/secret.sqlite3"), DataClass::Secret).unwrap(); open.index_log(&log, "mm-c", 0).unwrap(); assert_eq!( items(&open)[1], item("session", "owner", "after the secret", "secret", false, 2) ); } #[test] fn notes_are_indexed_with_their_labels_and_only_once() { let dir = TempDir::new("index-notes"); let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap(); let mut text = "§ noted 2026-09-24T20:14Z in session mm-a · private · untrusted\nexample.org says March.\n§ moved 2026-09-24T20:15Z from memory.md in session mm-b\nOwner prefers UTC.\n".to_string(); assert_eq!(index.index_notes(&text), Ok(2)); text.push_str("§ dropped 2026-09-24T20:16Z from user.md in session mm-b\nLives in Oslo.\n§ noted 2026-09-24T20:17Z in session mm-c · secret\nThe code.\n"); assert_eq!(index.index_notes(&text), Ok(2)); assert_eq!( items(&index), [ item("noted", "", "example.org says March.", "private", true, 0), item( "moved", "memory.md", "Owner prefers UTC.", "private", false, 1 ), item("dropped", "user.md", "Lives in Oslo.", "private", false, 2), ] ); } #[test] fn a_rebuild_equals_the_incremental_index_and_skips_damaged_logs() { let dir = TempDir::new("index-rebuild"); let home = dir.path(); let log = write_log(home, "mm-a", &[user(1, "one")]); write_log( home, "mm-b", &[ user(2, "two"), result(3, "r", DataClass::Private, true), assistant(4, Some("reply"), &[]), ], ); std::fs::create_dir_all(home.join("sessions/mm-bad")).unwrap(); std::fs::write(home.join("sessions/mm-bad/0.jsonl"), "{not json\n").unwrap(); let notes = dir.write( "memory/notes.md", "§ noted 2026-09-24T20:14Z in session mm-a · private\nseen\n", ); let path = home.join("index/memory.sqlite3"); let index = Index::open(&path, DataClass::Private).unwrap(); let (counts, problems) = index.catch_up(&home.join("sessions"), ¬es).unwrap(); assert_eq!((counts.logs, counts.records, counts.notes), (2, 4, 1)); assert_eq!(problems.len(), 1); assert!(problems[0].contains("mm-bad"), "{problems:?}"); append(&log, &user(9, "later")); index.catch_up(&home.join("sessions"), ¬es).unwrap(); let incremental = items(&index); drop(index); let (counts, _) = rebuild(&path, DataClass::Private, &home.join("sessions"), ¬es).unwrap(); assert_eq!((counts.logs, counts.records, counts.notes), (2, 5, 1)); let rebuilt = Index::open(&path, DataClass::Private).unwrap(); let mut a = incremental.clone(); let mut b = items(&rebuilt); a.sort(); b.sort(); assert_eq!(a, b); assert!(!home.join("index/memory.rebuilding").exists()); } #[test] fn the_index_file_and_directory_are_private() { use std::os::unix::fs::PermissionsExt; let dir = TempDir::new("index-mode"); let path = dir.path().join("index/memory.sqlite3"); Index::open(&path, DataClass::Private).unwrap(); let mode = |p: &std::path::Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777; assert_eq!( (mode(&dir.path().join("index")), mode(&path)), (0o700, 0o600) ); }