kyle/boxmaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

memoryd: recall, with source tags and the recall log

Browse Source 
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
kyle
2026-09-25 16:41:44 -07:00
committed by kyle
parent f329d23bd9
commit 49af057c81
4 changed files with 413 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/memoryd/src/lib.rs
+1
View File
@@ -8,5 +8,6 @@ pub mod files;
pub mod index;
pub mod journal;
pub mod notes;
pub mod recall;
pub mod review;
pub mod socket;
crates/memoryd/src/recall.rs
+232
View File
@@ -0,0 +1,232 @@
//! `recall`: search the index and answer with tagged hits (M5a spec, section 5). The query
//! reaches SQLite only as quoted terms, so no FTS5 operator from the model is ever parsed.
use serde::Serialize;
use proto::{DataClass, RecallArgs, Timestamp};
use rusqlite::params;
use crate::files::Store;
use crate::index::{Index, class_name, class_of};
/// The most characters of one hit's text.
pub const SNIPPET: usize = 600;
/// The answer to the model, and the label the result carries.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Recalled {
pub text: String,
pub class: DataClass,
pub untrusted: bool,
}
/// The FTS5 query for `query`: every run of letters and digits, double-quoted, joined by `OR`.
/// `None` when there is no such run.
pub fn terms(query: &str) -> Option<String> {
// Split `query` on every char that is not alphanumeric (`!c.is_alphanumeric()`), drop the empty
// pieces, wrap each in double quotes (format!("\"{t}\"")) and join them with " OR ". None when
// there is no piece. So `a NEAR(b` is `"a" OR "NEAR" OR "b"`: no FTS5 operator from the model
// is ever parsed.
let pieces: Vec<&str> = query
.split(|c: char| !c.is_alphanumeric())
.filter(|p| !p.is_empty())
.collect();
match pieces.is_empty() {
true => None,
false => Some(
pieces
.iter()
.map(|t| format!("\"{t}\""))
.collect::<Vec<_>>()
.join(" OR "),
),
}
}
/// `2026-09-20 14:02 UTC` from `2026-09-20T14:02:00.000Z` or `2026-09-20T14:02Z`.
fn when(time: &str) -> String {
// The first 10 characters (the day) and characters 11 to 16 (the clock), as format!("{day}
// {clock} UTC"): both 2026-09-20T14:02:00.000Z and 2026-09-20T14:02Z give 2026-09-20 14:02 UTC.
// When either `get` is None, the time unchanged.
let (Some(day), Some(clock)) = (time.get(0..10), time.get(11..16)) else {
return time.to_string();
};
format!("{day} {clock} UTC")
}
fn day(time: &str) -> &str {
// The first 10 characters of `time` (`get(..10)`), or all of it when shorter.
match time.get(..10) {
Some(d) => d,
None => time,
}
}
struct Hit {
kind: String,
session: String,
epoch: i64,
record: i64,
time: String,
role: String,
class: DataClass,
untrusted: bool,
text: String,
}
impl Hit {
/// `identity`, `memory` or `memory-tainted`.
fn tag(&self) -> &'static str {
// kind "moved" -> "identity"; "dropped" -> "memory"; any other kind: "memory-tainted" when
// untrusted, else "memory".
match self.kind.as_str() {
"moved" => "identity",
"dropped" => "memory",
_ => {
if self.untrusted {
"memory-tainted"
} else {
"memory"
}
}
}
}
fn header(&self, n: usize) -> String {
// `from` by kind: "moved" -> format!("moved {} from {}", day(time), role); "dropped" -> the
// same with "dropped"; "noted" -> format!("note, {}, session {}", when(time), session); a
// session record -> format!("session {}, {}, {said}", session, when(time)) where said is
// "the owner said" for role "owner", "the assistant called" for "call", and "the assistant
// said" otherwise. Return format!("[{n}] {} · {from} · {}", self.tag(),
// class_name(self.class)).
let from = match self.kind.as_str() {
"moved" => format!("moved {} from {}", day(&self.time), self.role),
"dropped" => format!("dropped {} from {}", day(&self.time), self.role),
"noted" => format!("note, {}, session {}", when(&self.time), self.session),
_ => {
let said = match self.role.as_str() {
"owner" => "the owner said",
"call" => "the assistant called",
_ => "the assistant said",
};
format!("session {}, {}, {said}", self.session, when(&self.time))
}
};
format!("[{n}] {} · {from} · {}", self.tag(), class_name(self.class))
}
}
#[derive(Serialize)]
struct LoggedHit<'a> {
kind: &'a str,
session: &'a str,
epoch: i64,
record: i64,
}
#[derive(Serialize)]
struct Logged<'a> {
time: String,
session: &'a str,
query: &'a str,
hits: Vec<LoggedHit<'a>>,
}
/// Search for `args` on behalf of `session`, log the recall, and answer.
pub fn recall(
index: &Index,
store: &Store,
args: &RecallArgs,
session: &str,
now: Timestamp,
) -> Result<Recalled, String> {
// 1. `args.check()?`.
// 2. hits: `terms(&args.query)` None -> none; Some(fts) -> `search(index, &fts, args.k)?`.
// 3. Log it: a Logged { time: now.to_rfc3339(), session, query, hits: a LoggedHit (kind,
// session, epoch, record) per hit } as one JSON line appended to
// format!("recalls/{month}.jsonl") (month: the first 7 characters of now.to_rfc3339()),
// after `create_dir_all(store.path("recalls"))`.
// 4. No hits: Ok(Recalled { text: format!("(no matches for \"{}\")", args.query), class
// Private, untrusted false }).
// 5. Otherwise, for each hit with i from 0: class = class.max(its class) starting at Private;
// untrusted |= its tag is "memory-tainted"; its text cut to its first SNIPPET characters;
// the item is format!("{}\n{text}", hit.header(i + 1)). Ok(Recalled { the items joined by
// newlines, class, untrusted }).
args.check()?;
let hits: Vec<Hit> = match terms(&args.query) {
None => Vec::new(),
Some(fts) => search(index, &fts, args.k)?,
};
let rfc = now.to_rfc3339();
let month = &rfc[..7];
std::fs::create_dir_all(store.path("recalls")).map_err(|e| format!("recalls: {e}"))?;
let logged = Logged {
time: rfc.clone(),
session,
query: &args.query,
hits: hits
.iter()
.map(|h| LoggedHit {
kind: h.kind.as_str(),
session: &h.session,
epoch: h.epoch,
record: h.record,
})
.collect(),
};
let line = serde_json::to_string(&logged).map_err(|e| format!("recall: {e}"))?;
store.append(&format!("recalls/{month}.jsonl"), &format!("{line}\n"))?;
if hits.is_empty() {
return Ok(Recalled {
text: format!("(no matches for \"{}\")", args.query),
class: DataClass::Private,
untrusted: false,
});
}
let mut class = DataClass::Private;
let mut untrusted = false;
let mut items: Vec<String> = Vec::with_capacity(hits.len());
for (i, hit) in hits.iter().enumerate() {
class = class.max(hit.class);
untrusted |= hit.tag() == "memory-tainted";
let text: String = hit.text.chars().take(SNIPPET).collect();
items.push(format!("{}\n{text}", hit.header(i + 1)));
}
Ok(Recalled {
text: items.join("\n"),
class,
untrusted,
})
}
fn search(index: &Index, fts: &str, k: u64) -> Result<Vec<Hit>, String> {
// `index.conn().prepare("SELECT s.kind, s.session, s.epoch, s.record, s.time, s.role, s.class,
// s.untrusted, snippet(hits, 0, '', '', '…', 64) FROM hits JOIN sources s ON s.rowid =
// hits.rowid WHERE hits MATCH ?1 ORDER BY bm25(hits) LIMIT ?2")`, then `query_map(params![fts,
// limit], ..)` building a Hit from columns 0 to 8 in that order (the class through `class_of`);
// limit is k as an i64 (`i64::try_from`, 10 when it does not fit). Collect into Vec<Hit>. Every
// rusqlite error as format!("index: {e}").
let limit = i64::try_from(k).unwrap_or(10);
let mut stmt = index
.conn()
.prepare("SELECT s.kind, s.session, s.epoch, s.record, s.time, s.role, s.class, s.untrusted, snippet(hits, 0, '', '', '…', 64) FROM hits JOIN sources s ON s.rowid = hits.rowid WHERE hits MATCH ?1 ORDER BY bm25(hits) LIMIT ?2")
.map_err(|e| format!("index: {e}"))?;
let rows = stmt
.query_map(params![fts, limit], |r| {
Ok(Hit {
kind: r.get::<_, String>(0)?,
session: r.get::<_, String>(1)?,
epoch: r.get::<_, i64>(2)?,
record: r.get::<_, i64>(3)?,
time: r.get::<_, String>(4)?,
role: r.get::<_, String>(5)?,
class: class_of(&r.get::<_, String>(6)?),
untrusted: r.get::<_, bool>(7)?,
text: r.get::<_, String>(8)?,
})
})
.map_err(|e| format!("index: {e}"))?
.collect::<Result<Vec<_>, _>>()
.map_err(|e| format!("index: {e}"))?;
Ok(rows)
}