memoryd: recall, with source tags and the recall log
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -8,5 +8,6 @@ pub mod files;
|
||||
pub mod index;
|
||||
pub mod journal;
|
||||
pub mod notes;
|
||||
pub mod recall;
|
||||
pub mod review;
|
||||
pub mod socket;
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
//! `recall`: search the index and answer with tagged hits (M5a spec, section 5). The query
|
||||
//! reaches SQLite only as quoted terms, so no FTS5 operator from the model is ever parsed.
|
||||
|
||||
use serde::Serialize;
|
||||
|
||||
use proto::{DataClass, RecallArgs, Timestamp};
|
||||
use rusqlite::params;
|
||||
|
||||
use crate::files::Store;
|
||||
use crate::index::{Index, class_name, class_of};
|
||||
|
||||
/// The most characters of one hit's text.
|
||||
pub const SNIPPET: usize = 600;
|
||||
|
||||
/// The answer to the model, and the label the result carries.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Recalled {
|
||||
pub text: String,
|
||||
pub class: DataClass,
|
||||
pub untrusted: bool,
|
||||
}
|
||||
|
||||
/// The FTS5 query for `query`: every run of letters and digits, double-quoted, joined by `OR`.
|
||||
/// `None` when there is no such run.
|
||||
pub fn terms(query: &str) -> Option<String> {
|
||||
// Split `query` on every char that is not alphanumeric (`!c.is_alphanumeric()`), drop the empty
|
||||
// pieces, wrap each in double quotes (format!("\"{t}\"")) and join them with " OR ". None when
|
||||
// there is no piece. So `a NEAR(b` is `"a" OR "NEAR" OR "b"`: no FTS5 operator from the model
|
||||
// is ever parsed.
|
||||
let pieces: Vec<&str> = query
|
||||
.split(|c: char| !c.is_alphanumeric())
|
||||
.filter(|p| !p.is_empty())
|
||||
.collect();
|
||||
match pieces.is_empty() {
|
||||
true => None,
|
||||
false => Some(
|
||||
pieces
|
||||
.iter()
|
||||
.map(|t| format!("\"{t}\""))
|
||||
.collect::<Vec<_>>()
|
||||
.join(" OR "),
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
/// `2026-09-20 14:02 UTC` from `2026-09-20T14:02:00.000Z` or `2026-09-20T14:02Z`.
|
||||
fn when(time: &str) -> String {
|
||||
// The first 10 characters (the day) and characters 11 to 16 (the clock), as format!("{day}
|
||||
// {clock} UTC"): both 2026-09-20T14:02:00.000Z and 2026-09-20T14:02Z give 2026-09-20 14:02 UTC.
|
||||
// When either `get` is None, the time unchanged.
|
||||
let (Some(day), Some(clock)) = (time.get(0..10), time.get(11..16)) else {
|
||||
return time.to_string();
|
||||
};
|
||||
format!("{day} {clock} UTC")
|
||||
}
|
||||
|
||||
fn day(time: &str) -> &str {
|
||||
// The first 10 characters of `time` (`get(..10)`), or all of it when shorter.
|
||||
match time.get(..10) {
|
||||
Some(d) => d,
|
||||
None => time,
|
||||
}
|
||||
}
|
||||
|
||||
struct Hit {
|
||||
kind: String,
|
||||
session: String,
|
||||
epoch: i64,
|
||||
record: i64,
|
||||
time: String,
|
||||
role: String,
|
||||
class: DataClass,
|
||||
untrusted: bool,
|
||||
text: String,
|
||||
}
|
||||
|
||||
impl Hit {
|
||||
/// `identity`, `memory` or `memory-tainted`.
|
||||
fn tag(&self) -> &'static str {
|
||||
// kind "moved" -> "identity"; "dropped" -> "memory"; any other kind: "memory-tainted" when
|
||||
// untrusted, else "memory".
|
||||
match self.kind.as_str() {
|
||||
"moved" => "identity",
|
||||
"dropped" => "memory",
|
||||
_ => {
|
||||
if self.untrusted {
|
||||
"memory-tainted"
|
||||
} else {
|
||||
"memory"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn header(&self, n: usize) -> String {
|
||||
// `from` by kind: "moved" -> format!("moved {} from {}", day(time), role); "dropped" -> the
|
||||
// same with "dropped"; "noted" -> format!("note, {}, session {}", when(time), session); a
|
||||
// session record -> format!("session {}, {}, {said}", session, when(time)) where said is
|
||||
// "the owner said" for role "owner", "the assistant called" for "call", and "the assistant
|
||||
// said" otherwise. Return format!("[{n}] {} · {from} · {}", self.tag(),
|
||||
// class_name(self.class)).
|
||||
let from = match self.kind.as_str() {
|
||||
"moved" => format!("moved {} from {}", day(&self.time), self.role),
|
||||
"dropped" => format!("dropped {} from {}", day(&self.time), self.role),
|
||||
"noted" => format!("note, {}, session {}", when(&self.time), self.session),
|
||||
_ => {
|
||||
let said = match self.role.as_str() {
|
||||
"owner" => "the owner said",
|
||||
"call" => "the assistant called",
|
||||
_ => "the assistant said",
|
||||
};
|
||||
format!("session {}, {}, {said}", self.session, when(&self.time))
|
||||
}
|
||||
};
|
||||
format!("[{n}] {} · {from} · {}", self.tag(), class_name(self.class))
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct LoggedHit<'a> {
|
||||
kind: &'a str,
|
||||
session: &'a str,
|
||||
epoch: i64,
|
||||
record: i64,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
struct Logged<'a> {
|
||||
time: String,
|
||||
session: &'a str,
|
||||
query: &'a str,
|
||||
hits: Vec<LoggedHit<'a>>,
|
||||
}
|
||||
|
||||
/// Search for `args` on behalf of `session`, log the recall, and answer.
|
||||
pub fn recall(
|
||||
index: &Index,
|
||||
store: &Store,
|
||||
args: &RecallArgs,
|
||||
session: &str,
|
||||
now: Timestamp,
|
||||
) -> Result<Recalled, String> {
|
||||
// 1. `args.check()?`.
|
||||
// 2. hits: `terms(&args.query)` None -> none; Some(fts) -> `search(index, &fts, args.k)?`.
|
||||
// 3. Log it: a Logged { time: now.to_rfc3339(), session, query, hits: a LoggedHit (kind,
|
||||
// session, epoch, record) per hit } as one JSON line appended to
|
||||
// format!("recalls/{month}.jsonl") (month: the first 7 characters of now.to_rfc3339()),
|
||||
// after `create_dir_all(store.path("recalls"))`.
|
||||
// 4. No hits: Ok(Recalled { text: format!("(no matches for \"{}\")", args.query), class
|
||||
// Private, untrusted false }).
|
||||
// 5. Otherwise, for each hit with i from 0: class = class.max(its class) starting at Private;
|
||||
// untrusted |= its tag is "memory-tainted"; its text cut to its first SNIPPET characters;
|
||||
// the item is format!("{}\n{text}", hit.header(i + 1)). Ok(Recalled { the items joined by
|
||||
// newlines, class, untrusted }).
|
||||
args.check()?;
|
||||
let hits: Vec<Hit> = match terms(&args.query) {
|
||||
None => Vec::new(),
|
||||
Some(fts) => search(index, &fts, args.k)?,
|
||||
};
|
||||
let rfc = now.to_rfc3339();
|
||||
let month = &rfc[..7];
|
||||
std::fs::create_dir_all(store.path("recalls")).map_err(|e| format!("recalls: {e}"))?;
|
||||
let logged = Logged {
|
||||
time: rfc.clone(),
|
||||
session,
|
||||
query: &args.query,
|
||||
hits: hits
|
||||
.iter()
|
||||
.map(|h| LoggedHit {
|
||||
kind: h.kind.as_str(),
|
||||
session: &h.session,
|
||||
epoch: h.epoch,
|
||||
record: h.record,
|
||||
})
|
||||
.collect(),
|
||||
};
|
||||
let line = serde_json::to_string(&logged).map_err(|e| format!("recall: {e}"))?;
|
||||
store.append(&format!("recalls/{month}.jsonl"), &format!("{line}\n"))?;
|
||||
if hits.is_empty() {
|
||||
return Ok(Recalled {
|
||||
text: format!("(no matches for \"{}\")", args.query),
|
||||
class: DataClass::Private,
|
||||
untrusted: false,
|
||||
});
|
||||
}
|
||||
let mut class = DataClass::Private;
|
||||
let mut untrusted = false;
|
||||
let mut items: Vec<String> = Vec::with_capacity(hits.len());
|
||||
for (i, hit) in hits.iter().enumerate() {
|
||||
class = class.max(hit.class);
|
||||
untrusted |= hit.tag() == "memory-tainted";
|
||||
let text: String = hit.text.chars().take(SNIPPET).collect();
|
||||
items.push(format!("{}\n{text}", hit.header(i + 1)));
|
||||
}
|
||||
Ok(Recalled {
|
||||
text: items.join("\n"),
|
||||
class,
|
||||
untrusted,
|
||||
})
|
||||
}
|
||||
|
||||
fn search(index: &Index, fts: &str, k: u64) -> Result<Vec<Hit>, String> {
|
||||
// `index.conn().prepare("SELECT s.kind, s.session, s.epoch, s.record, s.time, s.role, s.class,
|
||||
// s.untrusted, snippet(hits, 0, '', '', '…', 64) FROM hits JOIN sources s ON s.rowid =
|
||||
// hits.rowid WHERE hits MATCH ?1 ORDER BY bm25(hits) LIMIT ?2")`, then `query_map(params![fts,
|
||||
// limit], ..)` building a Hit from columns 0 to 8 in that order (the class through `class_of`);
|
||||
// limit is k as an i64 (`i64::try_from`, 10 when it does not fit). Collect into Vec<Hit>. Every
|
||||
// rusqlite error as format!("index: {e}").
|
||||
let limit = i64::try_from(k).unwrap_or(10);
|
||||
let mut stmt = index
|
||||
.conn()
|
||||
.prepare("SELECT s.kind, s.session, s.epoch, s.record, s.time, s.role, s.class, s.untrusted, snippet(hits, 0, '', '', '…', 64) FROM hits JOIN sources s ON s.rowid = hits.rowid WHERE hits MATCH ?1 ORDER BY bm25(hits) LIMIT ?2")
|
||||
.map_err(|e| format!("index: {e}"))?;
|
||||
let rows = stmt
|
||||
.query_map(params![fts, limit], |r| {
|
||||
Ok(Hit {
|
||||
kind: r.get::<_, String>(0)?,
|
||||
session: r.get::<_, String>(1)?,
|
||||
epoch: r.get::<_, i64>(2)?,
|
||||
record: r.get::<_, i64>(3)?,
|
||||
time: r.get::<_, String>(4)?,
|
||||
role: r.get::<_, String>(5)?,
|
||||
class: class_of(&r.get::<_, String>(6)?),
|
||||
untrusted: r.get::<_, bool>(7)?,
|
||||
text: r.get::<_, String>(8)?,
|
||||
})
|
||||
})
|
||||
.map_err(|e| format!("index: {e}"))?
|
||||
.collect::<Result<Vec<_>, _>>()
|
||||
.map_err(|e| format!("index: {e}"))?;
|
||||
Ok(rows)
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
//! `recall`: quoted terms only, ranked hits with source tags and labels, the result's label, and
|
||||
//! the recall log (M5a spec, section 5). Do not edit.
|
||||
|
||||
#[path = "support/logs.rs"]
|
||||
mod logs;
|
||||
#[path = "support/tmp.rs"]
|
||||
mod tmp;
|
||||
|
||||
use logs::{assistant, result, user, write_log};
|
||||
use memoryd::files::Store;
|
||||
use memoryd::index::Index;
|
||||
use memoryd::recall::{recall, terms};
|
||||
use proto::{DataClass, RecallArgs, Timestamp};
|
||||
use tmp::TempDir;
|
||||
|
||||
fn now() -> Timestamp {
|
||||
Timestamp::parse("2026-09-24T21:00:00.000Z").unwrap()
|
||||
}
|
||||
|
||||
struct Setup {
|
||||
dir: TempDir,
|
||||
store: Store,
|
||||
index: Index,
|
||||
}
|
||||
|
||||
fn setup(tag: &str) -> Setup {
|
||||
let dir = TempDir::new(tag);
|
||||
let starter = dir.write("etc/soul.md", "Soul.\n");
|
||||
let store = Store::new(&dir.path().join("memory"));
|
||||
store.init(&starter).unwrap();
|
||||
let index = Index::open(&dir.path().join("index/memory.sqlite3"), DataClass::Private).unwrap();
|
||||
Setup { dir, store, index }
|
||||
}
|
||||
|
||||
fn ask(s: &Setup, query: &str, k: u64) -> memoryd::recall::Recalled {
|
||||
let args = RecallArgs {
|
||||
query: query.to_string(),
|
||||
k,
|
||||
};
|
||||
recall(&s.index, &s.store, &args, "mm-now", now()).unwrap()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn terms_are_quoted_and_operators_never_pass() {
|
||||
assert_eq!(
|
||||
terms("launch window"),
|
||||
Some("\"launch\" OR \"window\"".to_string())
|
||||
);
|
||||
assert_eq!(
|
||||
terms(r#"a NEAR(b c) OR "x" title:y z* ^w -v"#),
|
||||
Some(
|
||||
r#""a" OR "NEAR" OR "b" OR "c" OR "OR" OR "x" OR "title" OR "y" OR "z" OR "w" OR "v""#
|
||||
.to_string()
|
||||
)
|
||||
);
|
||||
assert_eq!(terms("Київ 東京"), Some("\"Київ\" OR \"東京\"".to_string()));
|
||||
assert_eq!(terms("*** ::: \"\""), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn hits_are_tagged_and_labelled() {
|
||||
let s = setup("recall-tags");
|
||||
write_log(
|
||||
s.dir.path(),
|
||||
"mm-a",
|
||||
&[
|
||||
user(2, "The launch window moved to Thursday."),
|
||||
assistant(
|
||||
3,
|
||||
None,
|
||||
&[("http_fetch", r#"{"url":"https://example.org/launch"}"#)],
|
||||
),
|
||||
result(4, "page", DataClass::Private, true),
|
||||
assistant(5, Some("The page says the launch is in March."), &[]),
|
||||
],
|
||||
);
|
||||
s.index
|
||||
.catch_up(&s.dir.path().join("sessions"), &s.store.path("notes.md"))
|
||||
.unwrap();
|
||||
std::fs::write(
|
||||
s.store.path("notes.md"),
|
||||
"§ moved 2026-09-24T20:14Z from memory.md in session mm-b\nOwner cares about the launch.\n§ dropped 2026-09-24T20:15Z from user.md in session mm-b\nOld launch plan.\n§ noted 2026-09-22T18:20Z in session mm-c · private · untrusted\nexample.org says the launch moved.\n",
|
||||
)
|
||||
.unwrap();
|
||||
s.index
|
||||
.catch_up(&s.dir.path().join("sessions"), &s.store.path("notes.md"))
|
||||
.unwrap();
|
||||
let got = ask(&s, "launch", 10);
|
||||
let headers: Vec<&str> = got.text.lines().filter(|l| l.starts_with('[')).collect();
|
||||
let mut sorted: Vec<String> = headers
|
||||
.iter()
|
||||
.map(|h| h.split_once("] ").unwrap().1.to_string())
|
||||
.collect();
|
||||
sorted.sort();
|
||||
assert_eq!(
|
||||
sorted,
|
||||
[
|
||||
"identity · moved 2026-09-24 from memory.md · private",
|
||||
"memory · dropped 2026-09-24 from user.md · private",
|
||||
"memory · session mm-a, 2026-09-20 14:02 UTC, the owner said · private",
|
||||
"memory · session mm-a, 2026-09-20 14:03 UTC, the assistant called · private",
|
||||
"memory-tainted · note, 2026-09-22 18:20 UTC, session mm-c · private",
|
||||
"memory-tainted · session mm-a, 2026-09-20 14:05 UTC, the assistant said · private",
|
||||
]
|
||||
);
|
||||
assert!(got.text.starts_with("[1] "));
|
||||
assert!(got.text.contains("\nThe launch window moved to Thursday."));
|
||||
assert!(!got.text.contains("page\n"), "tool results are not indexed");
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, true));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_clean_result_is_trusted_and_k_limits_the_hits() {
|
||||
let s = setup("recall-clean");
|
||||
write_log(
|
||||
s.dir.path(),
|
||||
"mm-a",
|
||||
&[
|
||||
user(1, "alpha one"),
|
||||
user(2, "alpha two"),
|
||||
user(3, "alpha three"),
|
||||
],
|
||||
);
|
||||
s.index
|
||||
.catch_up(&s.dir.path().join("sessions"), &s.store.path("notes.md"))
|
||||
.unwrap();
|
||||
let got = ask(&s, "alpha", 2);
|
||||
assert_eq!(got.text.lines().filter(|l| l.starts_with('[')).count(), 2);
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, false));
|
||||
let none = ask(&s, "zebra", 5);
|
||||
assert_eq!(none.text, "(no matches for \"zebra\")");
|
||||
assert_eq!(ask(&s, "NEAR(", 5).text, "(no matches for \"NEAR(\")");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_hits_are_cut() {
|
||||
let s = setup("recall-long");
|
||||
// 64 words of 40 letters: longer than 600 characters even after `snippet` keeps 64 words.
|
||||
let long = format!("needle {}", format!("{} ", "w".repeat(40)).repeat(100));
|
||||
write_log(s.dir.path(), "mm-a", &[user(1, &long)]);
|
||||
s.index
|
||||
.catch_up(&s.dir.path().join("sessions"), &s.store.path("notes.md"))
|
||||
.unwrap();
|
||||
let got = ask(&s, "needle", 1);
|
||||
let body = got.text.split_once('\n').unwrap().1;
|
||||
assert!(body.chars().count() <= 600, "{}", body.chars().count());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_recall_is_logged_with_its_hits() {
|
||||
let s = setup("recall-log");
|
||||
write_log(s.dir.path(), "mm-a", &[user(1, "found me")]);
|
||||
s.index
|
||||
.catch_up(&s.dir.path().join("sessions"), &s.store.path("notes.md"))
|
||||
.unwrap();
|
||||
ask(&s, "found", 5);
|
||||
ask(&s, "missing", 5);
|
||||
let log = std::fs::read_to_string(s.store.path("recalls/2026-09.jsonl")).unwrap();
|
||||
let lines: Vec<serde_json::Value> = log
|
||||
.lines()
|
||||
.map(|l| serde_json::from_str(l).unwrap())
|
||||
.collect();
|
||||
assert_eq!(lines.len(), 2);
|
||||
assert_eq!(lines[0]["query"], "found");
|
||||
assert_eq!(lines[0]["session"], "mm-now");
|
||||
assert_eq!(lines[0]["hits"][0]["session"], "mm-a");
|
||||
assert_eq!(lines[0]["hits"][0]["record"], 0);
|
||||
assert_eq!(lines[1]["hits"], serde_json::json!([]));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_arguments_are_refused() {
|
||||
let s = setup("recall-bad");
|
||||
let args = RecallArgs {
|
||||
query: String::new(),
|
||||
k: 5,
|
||||
};
|
||||
assert!(recall(&s.index, &s.store, &args, "mm-now", now()).is_err());
|
||||
}
|
||||
Reference in New Issue
Block a user