memoryd: recall's Honcho part, every source at once with its own limit

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/memoryd/src/honcho/mod.rs

@@ -12,6 +12,7 @@ pub mod naming;
 pub mod read;
 pub mod recalled;
 pub mod report;
+pub mod search;
 pub mod start;
 pub mod state;
 pub mod status;

crates/memoryd/src/honcho/search.rs

@@ -0,0 +1,195 @@
+//! `recall`'s Honcho part (M5c spec, section 8): every source `Readers::recall_sources` names,
+//! searched at once, each on its own thread with its own limit, then its answer made from the
+//! local index's items followed by Honcho's. A query that holds a secret asks nothing. Runs
+//! outside `memoryd`'s state lock; a source still running at its limit is left to finish, and
+//! what it finds is dropped.
+
+use std::sync::{Arc, mpsc};
+use std::time::{Duration, Instant};
+
+use bxcore::RecallArgs;
+use roncho::{Conclusion, ConclusionQuery, MessageSearch};
+use serde_json::{Map, json};
+
+use crate::honcho::client::{HonchoError, classify};
+use crate::honcho::decide::has_secret;
+use crate::honcho::read::{Reader, Readers, Source};
+use crate::honcho::recalled::{Item, interleave, items, label, render, unavailable};
+use crate::recall::{Local, Recalled};
+use crate::serve::Log;
+
+/// How long each source has, counted from when the Honcho part starts (M5c spec, section 8).
+pub const SOURCE_WAIT: Duration = Duration::from_secs(5);
+
+/// The line in place of Honcho's items when the query holds a known-prefix secret.
+pub const SECRET: &str = "honcho: not asked; the query looks like it holds a secret";
+
+/// What the Honcho part found.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Searched {
+    /// The query holds a secret: no source was asked.
+    Secret,
+    /// Each source's kept items, in `recall_sources` order (none for a source that did not
+    /// answer), and the names of those that did not, in the same order.
+    Asked {
+        found: Vec<Vec<Item>>,
+        unavailable: Vec<String>,
+    },
+}
+
+/// The conclusions about `peer` that `peer` drew, as `memoryd` writes them: the observer and the
+/// observed are the same peer (M5c spec, section 8).
+fn conclusions(
+    reader: &Reader,
+    peer: &str,
+    query: &str,
+    k: u32,
+) -> Result<Vec<Conclusion>, HonchoError> {
+    let mut filters = Map::new();
+    filters.insert("observer_id".to_string(), json!(peer));
+    filters.insert("observed_id".to_string(), json!(peer));
+    let asked = ConclusionQuery::new(query).top_k(k).filters(filters);
+    reader
+        .search()
+        .query_conclusions(&asked)
+        .map_err(|e| classify(&e))
+}
+
+/// One source, its calls one after another: the workspace's messages, then for another agent's
+/// workspace the conclusions about that agent's peer and about the owner's. Its kept items, or
+/// the first call's failure.
+pub fn source(reader: &Reader, owner: &str, query: &str, k: u32) -> Result<Vec<Item>, HonchoError> {
+    let messages = reader
+        .search()
+        .search_workspace(&MessageSearch {
+            limit: Some(k),
+            ..MessageSearch::new(query)
+        })
+        .map_err(|e| classify(&e))?;
+    let (about_agent, about_owner) = match &reader.source {
+        Source::Agent(peer) => (
+            conclusions(reader, peer, query, k)?,
+            conclusions(reader, owner, query, k)?,
+        ),
+        _ => (Vec::new(), Vec::new()),
+    };
+    Ok(items(
+        &reader.label,
+        query,
+        &messages,
+        &about_agent,
+        &about_owner,
+    ))
+}
+
+/// Ask every source of `readers.recall_sources()` at once for `args`, each on a thread of its
+/// own, and wait for each at most `wait` from now. A source that failed, or that has not answered
+/// by then, is unavailable; each is logged by its name and the error's kind and status, never the
+/// query. `Secret`, with nothing sent, when the query holds a known-prefix secret.
+pub fn search(
+    readers: &Arc<Readers>,
+    owner: &str,
+    args: &RecallArgs,
+    wait: Duration,
+    log: &Log,
+) -> Searched {
+    if has_secret(&args.query) {
+        return Searched::Secret;
+    }
+    let deadline = Instant::now() + wait;
+    let k = u32::try_from(args.k).unwrap_or(u32::MAX);
+    let names: Vec<String> = readers
+        .recall_sources()
+        .iter()
+        .map(|r| r.label.name.clone())
+        .collect();
+    let (tx, rx) = mpsc::channel::<(usize, Result<Vec<Item>, HonchoError>)>();
+    let mut answers: Vec<Option<Result<Vec<Item>, HonchoError>>> = vec![None; names.len()];
+    for (i, slot) in answers.iter_mut().enumerate() {
+        let (readers, sender) = (Arc::clone(readers), tx.clone());
+        let (owner, query) = (owner.to_string(), args.query.clone());
+        let spawned = std::thread::Builder::new()
+            .name("honcho-recall".to_string())
+            .spawn(move || {
+                if let Some(reader) = readers.recall_sources().get(i) {
+                    let _ = sender.send((i, source(reader, &owner, &query, k)));
+                }
+            });
+        if spawned.is_err() {
+            *slot = Some(Err(HonchoError::Unreachable {
+                kind: "thread",
+                status: None,
+            }));
+        }
+    }
+    drop(tx);
+    while answers.iter().any(Option::is_none) {
+        let left = deadline.saturating_duration_since(Instant::now());
+        match rx.recv_timeout(left) {
+            Ok((i, answer)) => {
+                if let Some(slot) = answers.get_mut(i) {
+                    *slot = Some(answer);
+                }
+            }
+            Err(_) => break,
+        }
+    }
+    let mut found = Vec::with_capacity(names.len());
+    let mut missing = Vec::new();
+    for (name, answer) in names.into_iter().zip(answers) {
+        match answer {
+            Some(Ok(kept)) => found.push(kept),
+            Some(Err(e)) => {
+                (log)(&format!("memoryd: recall: {name} unavailable: {e}"));
+                found.push(Vec::new());
+                missing.push(name);
+            }
+            None => {
+                (log)(&format!(
+                    "memoryd: recall: {name} unavailable: no answer in {} ms",
+                    wait.as_millis()
+                ));
+                found.push(Vec::new());
+                missing.push(name);
+            }
+        }
+    }
+    Searched::Asked {
+        found,
+        unavailable: missing,
+    }
+}
+
+/// `recall`'s answer from the local index's items and what Honcho found: the local items, then
+/// Honcho's up to `k` in all, numbered after them; `(no matches for "…")` when there is neither;
+/// then the `unavailable` lines, or the secret line. The label is the stricter of the two parts'.
+pub fn answer(local: Local, query: &str, k: u64, searched: Searched) -> Recalled {
+    let k = usize::try_from(k).unwrap_or(usize::MAX);
+    let (shown, tail) = match searched {
+        Searched::Secret => (Vec::new(), SECRET.to_string()),
+        Searched::Asked {
+            found,
+            unavailable: missing,
+        } => {
+            let names: Vec<&str> = missing.iter().map(|s| s.as_str()).collect();
+            (interleave(local.items.len(), k, found), unavailable(&names))
+        }
+    };
+    let (class, untrusted) = label(&shown);
+    let first = local.items.len() + 1;
+    let mut parts: Vec<String> = local.items;
+    if !shown.is_empty() {
+        parts.push(render(first, &shown));
+    }
+    if parts.is_empty() {
+        parts.push(format!("(no matches for \"{query}\")"));
+    }
+    if !tail.is_empty() {
+        parts.push(tail);
+    }
+    Recalled {
+        text: parts.join("\n"),
+        class: local.class.max(class),
+        untrusted: local.untrusted || untrusted,
+    }
+}