memoryd: the FTS5 index over session logs and notes, with labels
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -0,0 +1,229 @@
|
||||
//! The index: what is indexed from a session log and `notes.md`, each record's label, catching up,
|
||||
//! `max_class`, and a rebuild equal to indexing as it went (M5a spec, section 6). Do not edit.
|
||||
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
|
||||
use logs::{append, assistant, result, user, write_log};
|
||||
use memoryd::index::{Index, rebuild};
|
||||
use proto::DataClass;
|
||||
use tmp::TempDir;
|
||||
|
||||
/// Every indexed item, in order: (kind, role, text, class, untrusted, record).
|
||||
fn items(index: &Index) -> Vec<(String, String, String, String, bool, i64)> {
|
||||
let mut stmt = index
|
||||
.conn()
|
||||
.prepare("SELECT s.kind, s.role, h.text, s.class, s.untrusted, s.record FROM sources s JOIN hits h ON h.rowid = s.rowid ORDER BY s.rowid")
|
||||
.unwrap();
|
||||
stmt.query_map([], |r| {
|
||||
Ok((
|
||||
r.get(0)?,
|
||||
r.get(1)?,
|
||||
r.get(2)?,
|
||||
r.get(3)?,
|
||||
r.get(4)?,
|
||||
r.get(5)?,
|
||||
))
|
||||
})
|
||||
.unwrap()
|
||||
.map(|r| r.unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn item(
|
||||
kind: &str,
|
||||
role: &str,
|
||||
text: &str,
|
||||
class: &str,
|
||||
untrusted: bool,
|
||||
record: i64,
|
||||
) -> (String, String, String, String, bool, i64) {
|
||||
(
|
||||
kind.to_string(),
|
||||
role.to_string(),
|
||||
text.to_string(),
|
||||
class.to_string(),
|
||||
untrusted,
|
||||
record,
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn owner_assistant_and_calls_are_indexed_with_the_label_in_force() {
|
||||
let dir = TempDir::new("index-log");
|
||||
let log = write_log(
|
||||
dir.path(),
|
||||
"mm-a",
|
||||
&[
|
||||
user(1, "What is the launch window?"),
|
||||
assistant(
|
||||
2,
|
||||
None,
|
||||
&[("http_fetch", r#"{"url":"https://example.org"}"#)],
|
||||
),
|
||||
result(3, "PAGE TEXT never indexed", DataClass::Private, true),
|
||||
assistant(4, Some("The page says Thursday."), &[]),
|
||||
user(5, " "),
|
||||
],
|
||||
);
|
||||
let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
|
||||
assert_eq!(index.index_log(&log, "mm-a", 0), Ok(5));
|
||||
assert_eq!(
|
||||
items(&index),
|
||||
[
|
||||
item(
|
||||
"session",
|
||||
"owner",
|
||||
"What is the launch window?",
|
||||
"private",
|
||||
false,
|
||||
0
|
||||
),
|
||||
item(
|
||||
"session",
|
||||
"call",
|
||||
r#"http_fetch {"url":"https://example.org"}"#,
|
||||
"private",
|
||||
false,
|
||||
1
|
||||
),
|
||||
item(
|
||||
"session",
|
||||
"assistant",
|
||||
"The page says Thursday.",
|
||||
"private",
|
||||
true,
|
||||
3
|
||||
),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn catching_up_reads_only_new_records_and_keeps_the_label() {
|
||||
let dir = TempDir::new("index-catch");
|
||||
let log = write_log(
|
||||
dir.path(),
|
||||
"mm-b",
|
||||
&[user(1, "first"), result(2, "x", DataClass::Private, true)],
|
||||
);
|
||||
let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
|
||||
assert_eq!(index.index_log(&log, "mm-b", 0), Ok(2));
|
||||
append(&log, &user(3, "second"));
|
||||
assert_eq!(index.index_log(&log, "mm-b", 0), Ok(1));
|
||||
assert_eq!(index.index_log(&log, "mm-b", 0), Ok(0));
|
||||
let got = items(&index);
|
||||
assert_eq!(got.len(), 2);
|
||||
assert_eq!(
|
||||
got[1],
|
||||
item("session", "owner", "second", "private", true, 2),
|
||||
"tainted by the earlier result"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn records_above_max_class_are_not_indexed() {
|
||||
let dir = TempDir::new("index-class");
|
||||
let log = write_log(
|
||||
dir.path(),
|
||||
"mm-c",
|
||||
&[
|
||||
user(1, "before"),
|
||||
result(2, "s", DataClass::Secret, false),
|
||||
user(3, "after the secret"),
|
||||
],
|
||||
);
|
||||
let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
|
||||
index.index_log(&log, "mm-c", 0).unwrap();
|
||||
assert_eq!(
|
||||
items(&index),
|
||||
[item("session", "owner", "before", "private", false, 0)]
|
||||
);
|
||||
let open = Index::open(&dir.path().join("index/secret.sqlite3"), DataClass::Secret).unwrap();
|
||||
open.index_log(&log, "mm-c", 0).unwrap();
|
||||
assert_eq!(
|
||||
items(&open)[1],
|
||||
item("session", "owner", "after the secret", "secret", false, 2)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn notes_are_indexed_with_their_labels_and_only_once() {
|
||||
let dir = TempDir::new("index-notes");
|
||||
let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
|
||||
let mut text = "§ noted 2026-09-24T20:14Z in session mm-a · private · untrusted\nexample.org says March.\n§ moved 2026-09-24T20:15Z from memory.md in session mm-b\nOwner prefers UTC.\n".to_string();
|
||||
assert_eq!(index.index_notes(&text), Ok(2));
|
||||
text.push_str("§ dropped 2026-09-24T20:16Z from user.md in session mm-b\nLives in Oslo.\n§ noted 2026-09-24T20:17Z in session mm-c · secret\nThe code.\n");
|
||||
assert_eq!(index.index_notes(&text), Ok(2));
|
||||
assert_eq!(
|
||||
items(&index),
|
||||
[
|
||||
item("noted", "", "example.org says March.", "private", true, 0),
|
||||
item(
|
||||
"moved",
|
||||
"memory.md",
|
||||
"Owner prefers UTC.",
|
||||
"private",
|
||||
false,
|
||||
1
|
||||
),
|
||||
item("dropped", "user.md", "Lives in Oslo.", "private", false, 2),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_rebuild_equals_the_incremental_index_and_skips_damaged_logs() {
|
||||
let dir = TempDir::new("index-rebuild");
|
||||
let home = dir.path();
|
||||
let log = write_log(home, "mm-a", &[user(1, "one")]);
|
||||
write_log(
|
||||
home,
|
||||
"mm-b",
|
||||
&[
|
||||
user(2, "two"),
|
||||
result(3, "r", DataClass::Private, true),
|
||||
assistant(4, Some("reply"), &[]),
|
||||
],
|
||||
);
|
||||
std::fs::create_dir_all(home.join("sessions/mm-bad")).unwrap();
|
||||
std::fs::write(home.join("sessions/mm-bad/0.jsonl"), "{not json\n").unwrap();
|
||||
let notes = dir.write(
|
||||
"memory/notes.md",
|
||||
"§ noted 2026-09-24T20:14Z in session mm-a · private\nseen\n",
|
||||
);
|
||||
let path = home.join("index/memory.sqlite3");
|
||||
let index = Index::open(&path, DataClass::Private).unwrap();
|
||||
let (counts, problems) = index.catch_up(&home.join("sessions"), ¬es).unwrap();
|
||||
assert_eq!((counts.logs, counts.records, counts.notes), (2, 4, 1));
|
||||
assert_eq!(problems.len(), 1);
|
||||
assert!(problems[0].contains("mm-bad"), "{problems:?}");
|
||||
append(&log, &user(9, "later"));
|
||||
index.catch_up(&home.join("sessions"), ¬es).unwrap();
|
||||
let incremental = items(&index);
|
||||
drop(index);
|
||||
let (counts, _) = rebuild(&path, DataClass::Private, &home.join("sessions"), ¬es).unwrap();
|
||||
assert_eq!((counts.logs, counts.records, counts.notes), (2, 5, 1));
|
||||
let rebuilt = Index::open(&path, DataClass::Private).unwrap();
|
||||
let mut a = incremental.clone();
|
||||
let mut b = items(&rebuilt);
|
||||
a.sort();
|
||||
b.sort();
|
||||
assert_eq!(a, b);
|
||||
assert!(!home.join("index/memory.rebuilding").exists());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_index_file_and_directory_are_private() {
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let dir = TempDir::new("index-mode");
|
||||
let path = dir.path().join("index/memory.sqlite3");
|
||||
Index::open(&path, DataClass::Private).unwrap();
|
||||
let mode = |p: &std::path::Path| std::fs::metadata(p).unwrap().permissions().mode() & 0o777;
|
||||
assert_eq!(
|
||||
(mode(&dir.path().join("index")), mode(&path)),
|
||||
(0o700, 0o600)
|
||||
);
|
||||
}
|
||||
Reference in New Issue
Block a user