285 lines
10 KiB
Rust
285 lines
10 KiB
Rust
//! `recall`: search the index and answer with tagged hits (M5a spec, section 5). The query
|
|
//! reaches SQLite only as quoted terms, so no FTS5 operator from the model is ever parsed.
|
|
|
|
use serde::Serialize;
|
|
|
|
use bxcore::sanitize::neutralise;
|
|
use bxcore::{DataClass, RecallArgs, Timestamp};
|
|
use rusqlite::params;
|
|
|
|
use crate::files::Store;
|
|
use crate::index::{Index, class_name, class_of};
|
|
|
|
/// The most characters of one hit's text.
|
|
pub const SNIPPET: usize = 600;
|
|
|
|
/// The answer to the model, and the label the result carries.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct Recalled {
|
|
pub text: String,
|
|
pub class: DataClass,
|
|
pub untrusted: bool,
|
|
}
|
|
|
|
/// The local index's hits, each rendered as an item, numbered from 1, and the label they give
|
|
/// the answer: `Private` and trusted when there are none.
|
|
#[derive(Debug, Clone, PartialEq, Eq)]
|
|
pub struct Local {
|
|
pub items: Vec<String>,
|
|
pub class: DataClass,
|
|
pub untrusted: bool,
|
|
}
|
|
|
|
/// A hit's text as an item holds it, from the local index or from Honcho: `one_line`, cut to
|
|
/// `SNIPPET` characters.
|
|
pub fn snippet(text: &str) -> String {
|
|
one_line(text, SNIPPET)
|
|
}
|
|
|
|
/// Text from memory as an item holds it: neutralised, every control character (a line break
|
|
/// among them) and every Unicode line or paragraph separator a space, then cut to its first
|
|
/// `cap` characters, with no mark.
|
|
pub fn one_line(text: &str, cap: usize) -> String {
|
|
neutralise(text)
|
|
.chars()
|
|
.map(|c| {
|
|
if c.is_control() || c == '\u{2028}' || c == '\u{2029}' {
|
|
' '
|
|
} else {
|
|
c
|
|
}
|
|
})
|
|
.take(cap)
|
|
.collect()
|
|
}
|
|
|
|
/// An item as the model reads it: its header, then its text on one line indented by two spaces,
|
|
/// so that no text can begin a line the way a header does (M5c spec, section 8; tip T37).
|
|
pub fn item(header: &str, text: &str) -> String {
|
|
format!("{header}\n {text}")
|
|
}
|
|
|
|
/// The FTS5 query for `query`: every run of letters and digits, double-quoted, joined by `OR`.
|
|
/// `None` when there is no such run.
|
|
pub fn terms(query: &str) -> Option<String> {
|
|
// Split `query` on every char that is not alphanumeric (`!c.is_alphanumeric()`), drop the empty
|
|
// pieces, wrap each in double quotes (format!("\"{t}\"")) and join them with " OR ". None when
|
|
// there is no piece. So `a NEAR(b` is `"a" OR "NEAR" OR "b"`: no FTS5 operator from the model
|
|
// is ever parsed.
|
|
let pieces: Vec<&str> = query
|
|
.split(|c: char| !c.is_alphanumeric())
|
|
.filter(|p| !p.is_empty())
|
|
.collect();
|
|
match pieces.is_empty() {
|
|
true => None,
|
|
false => Some(
|
|
pieces
|
|
.iter()
|
|
.map(|t| format!("\"{t}\""))
|
|
.collect::<Vec<_>>()
|
|
.join(" OR "),
|
|
),
|
|
}
|
|
}
|
|
|
|
/// `2026-09-20 14:02 UTC` from `2026-09-20T14:02:00.000Z` or `2026-09-20T14:02Z`.
|
|
pub(crate) fn when(time: &str) -> String {
|
|
// The first 10 characters (the day) and characters 11 to 16 (the clock), as format!("{day}
|
|
// {clock} UTC"): both 2026-09-20T14:02:00.000Z and 2026-09-20T14:02Z give 2026-09-20 14:02 UTC.
|
|
// When either `get` is None, the time unchanged.
|
|
let (Some(day), Some(clock)) = (time.get(0..10), time.get(11..16)) else {
|
|
return time.to_string();
|
|
};
|
|
format!("{day} {clock} UTC")
|
|
}
|
|
|
|
fn day(time: &str) -> &str {
|
|
// The first 10 characters of `time` (`get(..10)`), or all of it when shorter.
|
|
match time.get(..10) {
|
|
Some(d) => d,
|
|
None => time,
|
|
}
|
|
}
|
|
|
|
struct Hit {
|
|
kind: String,
|
|
session: String,
|
|
epoch: i64,
|
|
record: i64,
|
|
time: String,
|
|
role: String,
|
|
class: DataClass,
|
|
untrusted: bool,
|
|
text: String,
|
|
}
|
|
|
|
impl Hit {
|
|
/// `identity`, `memory` or `memory-tainted`.
|
|
fn tag(&self) -> &'static str {
|
|
// kind "moved" -> "identity"; "dropped" -> "memory"; any other kind: "memory-tainted" when
|
|
// untrusted, else "memory".
|
|
match self.kind.as_str() {
|
|
"moved" => "identity",
|
|
"dropped" => "memory",
|
|
_ => {
|
|
if self.untrusted {
|
|
"memory-tainted"
|
|
} else {
|
|
"memory"
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
fn header(&self, n: usize) -> String {
|
|
// `from` by kind: "moved" -> format!("moved {} from {}", day(time), role); "dropped" -> the
|
|
// same with "dropped"; "noted" -> format!("note, {}, session {}", when(time), session); a
|
|
// session record -> format!("session {}, {}, {said}", session, when(time)) where said is
|
|
// "the owner said" for role "owner", "the assistant called" for "call", and "the assistant
|
|
// said" otherwise. Return format!("[{n}] {} · {from} · {}", self.tag(),
|
|
// class_name(self.class)).
|
|
let from = match self.kind.as_str() {
|
|
"moved" => format!("moved {} from {}", day(&self.time), self.role),
|
|
"dropped" => format!("dropped {} from {}", day(&self.time), self.role),
|
|
"noted" => format!("note, {}, session {}", when(&self.time), self.session),
|
|
_ => {
|
|
let said = match self.role.as_str() {
|
|
"owner" => "the owner said",
|
|
"call" => "the assistant called",
|
|
_ => "the assistant said",
|
|
};
|
|
format!("session {}, {}, {said}", self.session, when(&self.time))
|
|
}
|
|
};
|
|
format!("[{n}] {} · {from} · {}", self.tag(), class_name(self.class))
|
|
}
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct LoggedHit<'a> {
|
|
kind: &'a str,
|
|
session: &'a str,
|
|
epoch: i64,
|
|
record: i64,
|
|
}
|
|
|
|
#[derive(Serialize)]
|
|
struct Logged<'a> {
|
|
time: String,
|
|
session: &'a str,
|
|
query: &'a str,
|
|
hits: Vec<LoggedHit<'a>>,
|
|
}
|
|
|
|
/// Search for `args` on behalf of `session`, log the recall, and answer.
|
|
pub fn recall(
|
|
index: &Index,
|
|
store: &Store,
|
|
args: &RecallArgs,
|
|
session: &str,
|
|
now: Timestamp,
|
|
) -> Result<Recalled, String> {
|
|
let found = local(index, store, args, session, now)?;
|
|
if found.items.is_empty() {
|
|
return Ok(Recalled {
|
|
text: format!("(no matches for \"{}\")", args.query),
|
|
class: DataClass::Private,
|
|
untrusted: false,
|
|
});
|
|
}
|
|
Ok(Recalled {
|
|
text: found.items.join("\n"),
|
|
class: found.class,
|
|
untrusted: found.untrusted,
|
|
})
|
|
}
|
|
|
|
/// Search the local index for `args` on behalf of `session`, log the recall, and give its hits
|
|
/// as items with their label. `recall` answers with them alone; with Honcho, its hits follow.
|
|
pub fn local(
|
|
index: &Index,
|
|
store: &Store,
|
|
args: &RecallArgs,
|
|
session: &str,
|
|
now: Timestamp,
|
|
) -> Result<Local, String> {
|
|
// 1. `args.check()?`.
|
|
// 2. hits: `terms(&args.query)` None -> none; Some(fts) -> `search(index, &fts, args.k)?`.
|
|
// 3. Log it: a Logged { time: now.to_rfc3339(), session, query, hits: a LoggedHit (kind,
|
|
// session, epoch, record) per hit } as one JSON line appended to
|
|
// format!("recalls/{month}.jsonl") (month: the first 7 characters of now.to_rfc3339()),
|
|
// after `create_dir_all(store.path("recalls"))`.
|
|
// 4. For each hit with i from 0: class = class.max(its class) starting at Private;
|
|
// untrusted |= its tag is "memory-tainted"; the item is `item(&hit.header(i + 1),
|
|
// &snippet(&hit.text))`. Ok(Local { items, class, untrusted }).
|
|
args.check()?;
|
|
let hits: Vec<Hit> = match terms(&args.query) {
|
|
None => Vec::new(),
|
|
Some(fts) => search(index, &fts, args.k)?,
|
|
};
|
|
let rfc = now.to_rfc3339();
|
|
let month = rfc.get(..7).unwrap_or("0000-00");
|
|
std::fs::create_dir_all(store.path("recalls")).map_err(|e| format!("recalls: {e}"))?;
|
|
let logged = Logged {
|
|
time: rfc.clone(),
|
|
session,
|
|
query: &args.query,
|
|
hits: hits
|
|
.iter()
|
|
.map(|h| LoggedHit {
|
|
kind: h.kind.as_str(),
|
|
session: &h.session,
|
|
epoch: h.epoch,
|
|
record: h.record,
|
|
})
|
|
.collect(),
|
|
};
|
|
let line = serde_json::to_string(&logged).map_err(|e| format!("recall: {e}"))?;
|
|
store.append(&format!("recalls/{month}.jsonl"), &format!("{line}\n"))?;
|
|
let mut class = DataClass::Private;
|
|
let mut untrusted = false;
|
|
let mut items: Vec<String> = Vec::with_capacity(hits.len());
|
|
for (i, hit) in hits.iter().enumerate() {
|
|
class = class.max(hit.class);
|
|
untrusted |= hit.tag() == "memory-tainted";
|
|
items.push(item(&hit.header(i + 1), &snippet(&hit.text)));
|
|
}
|
|
Ok(Local {
|
|
items,
|
|
class,
|
|
untrusted,
|
|
})
|
|
}
|
|
|
|
fn search(index: &Index, fts: &str, k: u64) -> Result<Vec<Hit>, String> {
|
|
// `index.conn().prepare("SELECT s.kind, s.session, s.epoch, s.record, s.time, s.role, s.class,
|
|
// s.untrusted, snippet(hits, 0, '', '', '…', 64) FROM hits JOIN sources s ON s.rowid =
|
|
// hits.rowid WHERE hits MATCH ?1 ORDER BY bm25(hits) LIMIT ?2")`, then `query_map(params![fts,
|
|
// limit], ..)` building a Hit from columns 0 to 8 in that order (the class through `class_of`);
|
|
// limit is k as an i64 (`i64::try_from`, 10 when it does not fit). Collect into Vec<Hit>. Every
|
|
// rusqlite error as format!("index: {e}").
|
|
let limit = i64::try_from(k).unwrap_or(10);
|
|
let mut stmt = index
|
|
.conn()
|
|
.prepare("SELECT s.kind, s.session, s.epoch, s.record, s.time, s.role, s.class, s.untrusted, snippet(hits, 0, '', '', '…', 64) FROM hits JOIN sources s ON s.rowid = hits.rowid WHERE hits MATCH ?1 ORDER BY bm25(hits) LIMIT ?2")
|
|
.map_err(|e| format!("index: {e}"))?;
|
|
let rows = stmt
|
|
.query_map(params![fts, limit], |r| {
|
|
Ok(Hit {
|
|
kind: r.get::<_, String>(0)?,
|
|
session: r.get::<_, String>(1)?,
|
|
epoch: r.get::<_, i64>(2)?,
|
|
record: r.get::<_, i64>(3)?,
|
|
time: r.get::<_, String>(4)?,
|
|
role: r.get::<_, String>(5)?,
|
|
class: class_of(&r.get::<_, String>(6)?),
|
|
untrusted: r.get::<_, bool>(7)?,
|
|
text: r.get::<_, String>(8)?,
|
|
})
|
|
})
|
|
.map_err(|e| format!("index: {e}"))?
|
|
.collect::<Result<Vec<_>, _>>()
|
|
.map_err(|e| format!("index: {e}"))?;
|
|
Ok(rows)
|
|
}
|