M5c-2 plan: task 08, recall's Honcho part; the wiring split out as task 09
Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
@@ -0,0 +1,165 @@
|
||||
//! `recall`'s Honcho part (M5c spec, section 8): every source `Readers::recall_sources` names,
|
||||
//! searched at once, each on its own thread with its own limit, then its answer made from the
|
||||
//! local index's items followed by Honcho's. A query that holds a secret asks nothing. Runs
|
||||
//! outside `memoryd`'s state lock; a source still running at its limit is left to finish, and
|
||||
//! what it finds is dropped.
|
||||
|
||||
use std::sync::{Arc, mpsc};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bxcore::RecallArgs;
|
||||
use roncho::{Conclusion, ConclusionQuery, MessageSearch};
|
||||
use serde_json::{Map, json};
|
||||
|
||||
use crate::honcho::client::{HonchoError, classify};
|
||||
use crate::honcho::decide::has_secret;
|
||||
use crate::honcho::read::{Reader, Readers, Source};
|
||||
use crate::honcho::recalled::{Item, interleave, items, label, render, unavailable};
|
||||
use crate::recall::{Local, Recalled};
|
||||
use crate::serve::Log;
|
||||
|
||||
/// How long each source has, counted from when the Honcho part starts (M5c spec, section 8).
|
||||
pub const SOURCE_WAIT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// The line in place of Honcho's items when the query holds a known-prefix secret.
|
||||
pub const SECRET: &str = "honcho: not asked; the query looks like it holds a secret";
|
||||
|
||||
/// What the Honcho part found.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum Searched {
|
||||
/// The query holds a secret: no source was asked.
|
||||
Secret,
|
||||
/// Each source's kept items, in `recall_sources` order (none for a source that did not
|
||||
/// answer), and the names of those that did not, in the same order.
|
||||
Asked {
|
||||
found: Vec<Vec<Item>>,
|
||||
unavailable: Vec<String>,
|
||||
},
|
||||
}
|
||||
|
||||
/// The conclusions about `peer` that `peer` drew, as `memoryd` writes them: the observer and the
|
||||
/// observed are the same peer (M5c spec, section 8).
|
||||
fn conclusions(
|
||||
reader: &Reader,
|
||||
peer: &str,
|
||||
query: &str,
|
||||
k: u32,
|
||||
) -> Result<Vec<Conclusion>, HonchoError> {
|
||||
let mut filters = Map::new();
|
||||
filters.insert("observer_id".to_string(), json!(peer));
|
||||
filters.insert("observed_id".to_string(), json!(peer));
|
||||
let asked = ConclusionQuery::new(query).top_k(k).filters(filters);
|
||||
reader
|
||||
.search()
|
||||
.query_conclusions(&asked)
|
||||
.map_err(|e| classify(&e))
|
||||
}
|
||||
|
||||
/// One source, its calls one after another: the workspace's messages, then for another agent's
|
||||
/// workspace the conclusions about that agent's peer and about the owner's. Its kept items, or
|
||||
/// the first call's failure.
|
||||
pub fn source(reader: &Reader, owner: &str, query: &str, k: u32) -> Result<Vec<Item>, HonchoError> {
|
||||
// 1. messages: `reader.search().search_workspace(&MessageSearch { limit: Some(k),
|
||||
// ..MessageSearch::new(query) })`, its error mapped with `classify(&e)`, then `?`.
|
||||
// 2. For `Source::Agent(peer)`: about_agent = `conclusions(reader, peer, query, k)?`, then
|
||||
// about_owner = `conclusions(reader, owner, query, k)?`, in that order. Any other source:
|
||||
// both `Vec::new()`, and no conclusion is asked for.
|
||||
// 3. `Ok(items(&reader.label, query, &messages, &about_agent, &about_owner))`.
|
||||
todo!()
|
||||
}
|
||||
|
||||
/// Ask every source of `readers.recall_sources()` at once for `args`, each on a thread of its
|
||||
/// own, and wait for each at most `wait` from now. A source that failed, or that has not answered
|
||||
/// by then, is unavailable; each is logged by its name and the error's kind and status, never the
|
||||
/// query. `Secret`, with nothing sent, when the query holds a known-prefix secret.
|
||||
pub fn search(
|
||||
readers: &Arc<Readers>,
|
||||
owner: &str,
|
||||
args: &RecallArgs,
|
||||
wait: Duration,
|
||||
log: &Log,
|
||||
) -> Searched {
|
||||
if has_secret(&args.query) {
|
||||
return Searched::Secret;
|
||||
}
|
||||
let deadline = Instant::now() + wait;
|
||||
let k = u32::try_from(args.k).unwrap_or(u32::MAX);
|
||||
let names: Vec<String> = readers
|
||||
.recall_sources()
|
||||
.iter()
|
||||
.map(|r| r.label.name.clone())
|
||||
.collect();
|
||||
let (tx, rx) = mpsc::channel::<(usize, Result<Vec<Item>, HonchoError>)>();
|
||||
let mut answers: Vec<Option<Result<Vec<Item>, HonchoError>>> = vec![None; names.len()];
|
||||
for (i, slot) in answers.iter_mut().enumerate() {
|
||||
let (readers, sender) = (Arc::clone(readers), tx.clone());
|
||||
let (owner, query) = (owner.to_string(), args.query.clone());
|
||||
let spawned = std::thread::Builder::new()
|
||||
.name("honcho-recall".to_string())
|
||||
.spawn(move || {
|
||||
if let Some(reader) = readers.recall_sources().get(i) {
|
||||
let _ = sender.send((i, source(reader, &owner, &query, k)));
|
||||
}
|
||||
});
|
||||
if spawned.is_err() {
|
||||
*slot = Some(Err(HonchoError::Unreachable {
|
||||
kind: "thread",
|
||||
status: None,
|
||||
}));
|
||||
}
|
||||
}
|
||||
drop(tx);
|
||||
while answers.iter().any(Option::is_none) {
|
||||
let left = deadline.saturating_duration_since(Instant::now());
|
||||
match rx.recv_timeout(left) {
|
||||
Ok((i, answer)) => {
|
||||
if let Some(slot) = answers.get_mut(i) {
|
||||
*slot = Some(answer);
|
||||
}
|
||||
}
|
||||
Err(_) => break,
|
||||
}
|
||||
}
|
||||
let mut found = Vec::with_capacity(names.len());
|
||||
let mut missing = Vec::new();
|
||||
for (name, answer) in names.into_iter().zip(answers) {
|
||||
match answer {
|
||||
Some(Ok(kept)) => found.push(kept),
|
||||
Some(Err(e)) => {
|
||||
(log)(&format!("memoryd: recall: {name} unavailable: {e}"));
|
||||
found.push(Vec::new());
|
||||
missing.push(name);
|
||||
}
|
||||
None => {
|
||||
(log)(&format!(
|
||||
"memoryd: recall: {name} unavailable: no answer in {} ms",
|
||||
wait.as_millis()
|
||||
));
|
||||
found.push(Vec::new());
|
||||
missing.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
Searched::Asked {
|
||||
found,
|
||||
unavailable: missing,
|
||||
}
|
||||
}
|
||||
|
||||
/// `recall`'s answer from the local index's items and what Honcho found: the local items, then
|
||||
/// Honcho's up to `k` in all, numbered after them; `(no matches for "…")` when there is neither;
|
||||
/// then the `unavailable` lines, or the secret line. The label is the stricter of the two parts'.
|
||||
pub fn answer(local: Local, query: &str, k: u64, searched: Searched) -> Recalled {
|
||||
// 1. k as a usize (`usize::try_from(k).unwrap_or(usize::MAX)`).
|
||||
// 2. (shown, tail): `Secret` -> (no items, `SECRET.to_string()`); `Asked { found,
|
||||
// unavailable: missing }` (bound as `missing`: the field has the function's name) ->
|
||||
// (`interleave(local.items.len(), k, found)`, `unavailable(&names)` where names is
|
||||
// `missing` as a `Vec<&str>`).
|
||||
// 3. (class, untrusted) = `label(&shown)`; `first` = local.items.len() + 1, taken before the
|
||||
// items move. parts: `local.items`; then, if `shown` is not empty, `render(first,
|
||||
// &shown)`; then, if parts is still empty, format!("(no matches for \"{query}\")"); then
|
||||
// `tail` if it is not empty.
|
||||
// 4. Recalled { text: parts joined by "\n", class: `local.class.max(class)`, untrusted:
|
||||
// `local.untrusted || untrusted` }.
|
||||
todo!()
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
//! `recall`'s Honcho part (M5c spec, section 8), against a fake Honcho that answers from the
|
||||
//! responses recorded from rift: every read workspace and the fleet's searched at once, never the
|
||||
//! agent's own; a source that hangs or fails is one `unavailable` line after its limit while the
|
||||
//! others answer; a missing workspace is no hits; a query holding a secret asks nothing; and the
|
||||
//! answer made from the local items and Honcho's, with the stricter label. Do not edit.
|
||||
|
||||
#[path = "support/honcho.rs"]
|
||||
mod honcho;
|
||||
|
||||
use std::collections::BTreeMap;
|
||||
use std::ffi::OsString;
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use bxcore::{DataClass, RecallArgs};
|
||||
use honcho::{FakeHoncho, Mode};
|
||||
use memoryd::honcho::config::{HonchoConfig, ReadWorkspace, Trust};
|
||||
use memoryd::honcho::read::{ReadTimeouts, Readers};
|
||||
use memoryd::honcho::search::{SECRET, SOURCE_WAIT, Searched, answer, search};
|
||||
use memoryd::recall::Local;
|
||||
use memoryd::serve::Log;
|
||||
|
||||
const FAST: ReadTimeouts = ReadTimeouts {
|
||||
connect: Duration::from_secs(5),
|
||||
search: Duration::from_secs(5),
|
||||
chat: Duration::from_secs(5),
|
||||
};
|
||||
/// The limit the tests give each source: long, so that a loaded machine never makes a source
|
||||
/// that answers late; a search where every source answers returns at once.
|
||||
const WAIT: Duration = Duration::from_secs(10);
|
||||
/// The limit when a source hangs: the others still answer well inside it.
|
||||
const HANG: Duration = Duration::from_millis(1_500);
|
||||
|
||||
fn read(agent: &str, trust: Trust) -> ReadWorkspace {
|
||||
ReadWorkspace {
|
||||
agent: agent.to_string(),
|
||||
workspace: format!("{agent}-trusted"),
|
||||
trust,
|
||||
class: DataClass::Private,
|
||||
}
|
||||
}
|
||||
|
||||
/// `boxmaker0`, owned by `kyle`, reading `boxmaker1` (labelled trusted) and `boxmaker2`.
|
||||
fn readers(fake: &FakeHoncho, read: Vec<ReadWorkspace>) -> Arc<Readers> {
|
||||
let config = HonchoConfig {
|
||||
url: fake.url.clone(),
|
||||
agent: "boxmaker0".to_string(),
|
||||
owner: "kyle".to_string(),
|
||||
trusted: "boxmaker0-trusted".to_string(),
|
||||
untrusted: "boxmaker0-untrusted".to_string(),
|
||||
fleet: "fleet".to_string(),
|
||||
read,
|
||||
keys: BTreeMap::new(),
|
||||
};
|
||||
let env = |_: &str| -> Option<OsString> { None };
|
||||
Arc::new(Readers::new(&config, &env, FAST).unwrap().0)
|
||||
}
|
||||
|
||||
fn two(fake: &FakeHoncho) -> Arc<Readers> {
|
||||
readers(
|
||||
fake,
|
||||
vec![
|
||||
read("boxmaker1", Trust::Trusted),
|
||||
read("boxmaker2", Trust::Untrusted),
|
||||
],
|
||||
)
|
||||
}
|
||||
|
||||
/// A log that keeps its lines.
|
||||
fn log() -> (Log, Arc<Mutex<Vec<String>>>) {
|
||||
let lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let sink = Arc::clone(&lines);
|
||||
let log: Log = Arc::new(move |line: &str| sink.lock().unwrap().push(line.to_string()));
|
||||
(log, lines)
|
||||
}
|
||||
|
||||
fn args(query: &str, k: u64) -> RecallArgs {
|
||||
RecallArgs {
|
||||
query: query.to_string(),
|
||||
k,
|
||||
}
|
||||
}
|
||||
|
||||
fn nothing_local() -> Local {
|
||||
Local {
|
||||
items: Vec::new(),
|
||||
class: DataClass::Private,
|
||||
untrusted: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn asked(searched: &Searched) -> (Vec<Vec<String>>, Vec<String>) {
|
||||
match searched {
|
||||
Searched::Asked { found, unavailable } => (
|
||||
found
|
||||
.iter()
|
||||
.map(|items| items.iter().map(|i| i.text.clone()).collect())
|
||||
.collect(),
|
||||
unavailable.clone(),
|
||||
),
|
||||
Searched::Secret => panic!("not asked"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_limit() {
|
||||
assert_eq!(SOURCE_WAIT, Duration::from_secs(5));
|
||||
assert_eq!(
|
||||
SECRET,
|
||||
"honcho: not asked; the query looks like it holds a secret"
|
||||
);
|
||||
}
|
||||
|
||||
/// A read agent's workspace: its messages, the conclusions about its peer and about the owner's;
|
||||
/// the fleet's: its messages. Each kept only with a word of the query, once.
|
||||
#[test]
|
||||
fn every_source_is_searched_with_its_calls() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, lines) = log();
|
||||
let searched = search(&two(&fake), "kyle", &args("tea", 7), WAIT, &log);
|
||||
let (found, unavailable) = asked(&searched);
|
||||
let tea = [
|
||||
"I prefer tea to coffee in the afternoon.",
|
||||
"Noted: tea in the afternoon.",
|
||||
];
|
||||
let with_owner = [tea[0], tea[1], "Kyle prefers tea in the afternoon."];
|
||||
assert_eq!(found, [&with_owner[..], &with_owner[..], &tea[..]]);
|
||||
assert_eq!(unavailable, Vec::<String>::new());
|
||||
assert!(lines.lock().unwrap().is_empty());
|
||||
let mut seen: Vec<(String, serde_json::Value)> = fake
|
||||
.requests()
|
||||
.into_iter()
|
||||
.map(|r| (r.path, r.body.unwrap_or_default()))
|
||||
.collect();
|
||||
seen.sort_by(|a, b| a.0.cmp(&b.0).then(a.1.to_string().cmp(&b.1.to_string())));
|
||||
let query = |peer: &str| {
|
||||
serde_json::json!({ "query": "tea", "top_k": 7,
|
||||
"filters": { "observer_id": peer, "observed_id": peer } })
|
||||
};
|
||||
let search = serde_json::json!({ "query": "tea", "limit": 7 });
|
||||
let w = "/v3/workspaces";
|
||||
assert_eq!(
|
||||
seen,
|
||||
[
|
||||
(
|
||||
format!("{w}/boxmaker1-trusted/conclusions/query"),
|
||||
query("boxmaker1")
|
||||
),
|
||||
(
|
||||
format!("{w}/boxmaker1-trusted/conclusions/query"),
|
||||
query("kyle")
|
||||
),
|
||||
(format!("{w}/boxmaker1-trusted/search"), search.clone()),
|
||||
(
|
||||
format!("{w}/boxmaker2-trusted/conclusions/query"),
|
||||
query("boxmaker2")
|
||||
),
|
||||
(
|
||||
format!("{w}/boxmaker2-trusted/conclusions/query"),
|
||||
query("kyle")
|
||||
),
|
||||
(format!("{w}/boxmaker2-trusted/search"), search.clone()),
|
||||
(format!("{w}/fleet/search"), search),
|
||||
]
|
||||
);
|
||||
}
|
||||
|
||||
/// The agent's own workspaces hold what the local index holds: `recall` never searches them.
|
||||
#[test]
|
||||
fn the_agents_own_workspaces_are_never_searched() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, _) = log();
|
||||
search(&two(&fake), "kyle", &args("tea", 5), WAIT, &log);
|
||||
let requests = fake.requests();
|
||||
assert_eq!(requests.len(), 7);
|
||||
for request in requests {
|
||||
assert!(!request.path.contains("boxmaker0"), "{}", request.path);
|
||||
}
|
||||
}
|
||||
|
||||
/// The answer shows each source's hits with its tag, numbered after the local items, one from
|
||||
/// each source in turn.
|
||||
#[test]
|
||||
fn the_answer_shows_each_sources_hits_with_its_tag() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, _) = log();
|
||||
let searched = search(&two(&fake), "kyle", &args("tea", 4), WAIT, &log);
|
||||
let local = Local {
|
||||
items: vec![
|
||||
"[1] memory · session mm-a, 2026-09-20 14:02 UTC, the owner said · private\n tea time"
|
||||
.to_string(),
|
||||
],
|
||||
class: DataClass::Private,
|
||||
untrusted: false,
|
||||
};
|
||||
let got = answer(local, "tea", 4, searched);
|
||||
let headers: Vec<&str> = got
|
||||
.text
|
||||
.lines()
|
||||
.filter(|l| l.starts_with('['))
|
||||
.map(|l| l.split(" · ").next().unwrap())
|
||||
.collect();
|
||||
assert_eq!(
|
||||
headers,
|
||||
[
|
||||
"[1] memory",
|
||||
"[2] memory",
|
||||
"[3] memory-tainted",
|
||||
"[4] memory-tainted"
|
||||
]
|
||||
);
|
||||
assert!(
|
||||
got.text.contains("\n[2] memory · boxmaker1: kyle said, "),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
assert!(
|
||||
got.text
|
||||
.contains("\n[4] memory-tainted · fleet: kyle said, "),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, true));
|
||||
}
|
||||
|
||||
/// T58: a fleet hit makes the answer untrusted; a hit from a workspace the owner labelled trusted
|
||||
/// does not.
|
||||
#[test]
|
||||
fn a_fleet_hit_taints_the_answer_and_a_trusted_workspaces_does_not() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, _) = log();
|
||||
let one = readers(&fake, vec![read("boxmaker1", Trust::Trusted)]);
|
||||
fake.set_workspace("fleet", Mode::Missing);
|
||||
let got = answer(
|
||||
nothing_local(),
|
||||
"tea",
|
||||
5,
|
||||
search(&one, "kyle", &args("tea", 5), WAIT, &log),
|
||||
);
|
||||
assert!(
|
||||
got.text.contains("[1] memory · boxmaker1: "),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, false));
|
||||
fake.set_workspace("fleet", Mode::Fixtures);
|
||||
fake.set_workspace("boxmaker1-trusted", Mode::Missing);
|
||||
let got = answer(
|
||||
nothing_local(),
|
||||
"tea",
|
||||
5,
|
||||
search(&one, "kyle", &args("tea", 5), WAIT, &log),
|
||||
);
|
||||
assert!(
|
||||
got.text.starts_with("[1] memory-tainted · fleet: "),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, true));
|
||||
}
|
||||
|
||||
/// A source that does not answer is one `unavailable` line after its limit; the others answer.
|
||||
#[test]
|
||||
fn a_hanging_source_is_unavailable_and_the_others_still_answer() {
|
||||
let fake = FakeHoncho::start();
|
||||
fake.set_workspace("boxmaker2-trusted", Mode::Hang);
|
||||
let (log, lines) = log();
|
||||
let started = Instant::now();
|
||||
let searched = search(&two(&fake), "kyle", &args("tea", 10), HANG, &log);
|
||||
let took = started.elapsed();
|
||||
assert!(took >= HANG, "{took:?}");
|
||||
assert!(took < Duration::from_secs(6), "{took:?}");
|
||||
let (found, unavailable) = asked(&searched);
|
||||
assert_eq!(unavailable, ["boxmaker2"]);
|
||||
assert_eq!(found.iter().map(Vec::len).collect::<Vec<_>>(), [3, 0, 2]);
|
||||
assert_eq!(
|
||||
*lines.lock().unwrap(),
|
||||
["memoryd: recall: boxmaker2 unavailable: no answer in 1500 ms"]
|
||||
);
|
||||
let got = answer(nothing_local(), "tea", 10, searched);
|
||||
assert!(
|
||||
got.text.ends_with("\nboxmaker2: unavailable"),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
}
|
||||
|
||||
/// The sources are asked at once: when every one hangs, the answer comes after one limit, not
|
||||
/// one per source.
|
||||
#[test]
|
||||
fn every_source_is_asked_at_once() {
|
||||
let fake = FakeHoncho::start();
|
||||
fake.set(Mode::Hang);
|
||||
let (log, _) = log();
|
||||
let started = Instant::now();
|
||||
let (found, unavailable) = asked(&search(&two(&fake), "kyle", &args("tea", 5), HANG, &log));
|
||||
let took = started.elapsed();
|
||||
assert!(took < Duration::from_millis(4_500), "{took:?}");
|
||||
assert_eq!(unavailable, ["boxmaker1", "boxmaker2", "fleet"]);
|
||||
assert!(found.iter().all(Vec::is_empty));
|
||||
}
|
||||
|
||||
/// A source whose call fails is unavailable, logged by the error's kind and status only.
|
||||
#[test]
|
||||
fn a_failing_source_is_unavailable_and_logged_without_the_query() {
|
||||
let fake = FakeHoncho::start();
|
||||
fake.set_workspace("fleet", Mode::Status(503));
|
||||
fake.set_workspace("boxmaker1-trusted", Mode::Status(422));
|
||||
let (log, lines) = log();
|
||||
let searched = search(&two(&fake), "kyle", &args("tea", 5), WAIT, &log);
|
||||
let (found, unavailable) = asked(&searched);
|
||||
assert_eq!(unavailable, ["boxmaker1", "fleet"]);
|
||||
assert_eq!(found[1].len(), 3);
|
||||
assert_eq!(
|
||||
*lines.lock().unwrap(),
|
||||
[
|
||||
"memoryd: recall: boxmaker1 unavailable: Honcho refused the request (status, HTTP 422)",
|
||||
"memoryd: recall: fleet unavailable: Honcho is unreachable (server, HTTP 503)",
|
||||
]
|
||||
);
|
||||
let got = answer(nothing_local(), "tea", 5, searched);
|
||||
assert!(
|
||||
got.text
|
||||
.ends_with("\nboxmaker1: unavailable\nfleet: unavailable"),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
}
|
||||
|
||||
/// Honcho answers a search or a conclusion query in a workspace that does not exist with 200 and
|
||||
/// no hits: the source answered, with nothing.
|
||||
#[test]
|
||||
fn a_missing_workspace_is_no_hits() {
|
||||
let fake = FakeHoncho::start();
|
||||
fake.set_workspace("boxmaker1-trusted", Mode::Missing);
|
||||
let (log, lines) = log();
|
||||
let (found, unavailable) = asked(&search(&two(&fake), "kyle", &args("tea", 5), WAIT, &log));
|
||||
assert_eq!(found[0], Vec::<String>::new());
|
||||
assert_eq!(unavailable, Vec::<String>::new());
|
||||
assert!(lines.lock().unwrap().is_empty());
|
||||
}
|
||||
|
||||
/// A query that holds a known-prefix secret is sent to no source (M5c spec, section 8).
|
||||
#[test]
|
||||
fn a_query_holding_a_secret_asks_nothing() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, _) = log();
|
||||
let token = format!("{}{}", ["gh", "p_"].concat(), "A1b2".repeat(9));
|
||||
let searched = search(
|
||||
&two(&fake),
|
||||
"kyle",
|
||||
&args(&format!("tea {token}"), 5),
|
||||
WAIT,
|
||||
&log,
|
||||
);
|
||||
assert_eq!(searched, Searched::Secret);
|
||||
assert_eq!(fake.requests().len(), 0);
|
||||
let local = Local {
|
||||
items: vec!["[1] memory · x · private\n tea".to_string()],
|
||||
class: DataClass::Private,
|
||||
untrusted: false,
|
||||
};
|
||||
let got = answer(local, "tea", 5, Searched::Secret);
|
||||
assert_eq!(
|
||||
got.text,
|
||||
format!("[1] memory · x · private\n tea\n{SECRET}")
|
||||
);
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, false));
|
||||
}
|
||||
|
||||
/// Neither the local index nor Honcho has anything: the sentence local `recall` gives, then any
|
||||
/// `unavailable` lines.
|
||||
#[test]
|
||||
fn when_nothing_is_found_the_answer_says_so() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, _) = log();
|
||||
let searched = search(&two(&fake), "kyle", &args("zebra", 5), WAIT, &log);
|
||||
let got = answer(nothing_local(), "zebra", 5, searched);
|
||||
assert_eq!(got.text, "(no matches for \"zebra\")");
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Private, false));
|
||||
let got = answer(
|
||||
nothing_local(),
|
||||
"zebra",
|
||||
5,
|
||||
Searched::Asked {
|
||||
found: vec![Vec::new()],
|
||||
unavailable: vec!["fleet".to_string()],
|
||||
},
|
||||
);
|
||||
assert_eq!(got.text, "(no matches for \"zebra\")\nfleet: unavailable");
|
||||
}
|
||||
|
||||
/// The local index first: when its hits fill `k`, no Honcho hit is shown and none labels the
|
||||
/// answer. The label is the stricter of the two parts'.
|
||||
#[test]
|
||||
fn local_hits_come_first_and_the_labels_join() {
|
||||
let fake = FakeHoncho::start();
|
||||
let (log, _) = log();
|
||||
let local = Local {
|
||||
items: vec![
|
||||
"[1] memory · a · secret\n tea".to_string(),
|
||||
"[2] memory · b · private\n tea".to_string(),
|
||||
],
|
||||
class: DataClass::Secret,
|
||||
untrusted: false,
|
||||
};
|
||||
let searched = search(&two(&fake), "kyle", &args("tea", 2), WAIT, &log);
|
||||
let got = answer(local.clone(), "tea", 2, searched.clone());
|
||||
assert_eq!(got.text, local.items.join("\n"));
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Secret, false));
|
||||
let got = answer(local, "tea", 3, searched);
|
||||
assert_eq!(got.text.lines().filter(|l| l.starts_with('[')).count(), 3);
|
||||
assert!(
|
||||
got.text.contains("\n[3] memory · boxmaker1: "),
|
||||
"{}",
|
||||
got.text
|
||||
);
|
||||
assert_eq!((got.class, got.untrusted), (DataClass::Secret, false));
|
||||
}
|
||||
@@ -0,0 +1,321 @@
|
||||
//! A fake Honcho for tests, on 127.0.0.1 and a port of its own: it answers each request from the
|
||||
//! responses recorded from rift (`tests/fixtures/honcho/`), by method and path (a batch of
|
||||
//! messages or conclusions answered with as many as it asked for, up to the two recorded), and
|
||||
//! records every request, body included. It can be set to answer every request with a status
|
||||
//! instead, to answer nothing, or to answer as a workspace that does not exist, for every
|
||||
//! workspace or for one. Stopped, and its held connections released, when dropped; every wait has
|
||||
//! a deadline. Do not edit.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::io::{Read, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::path::PathBuf;
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread::JoinHandle;
|
||||
use std::time::{Duration, Instant};
|
||||
|
||||
use serde_json::Value;
|
||||
|
||||
/// One request as the fake read it.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct Request {
|
||||
pub method: String,
|
||||
pub path: String,
|
||||
/// Header names in lower case.
|
||||
pub headers: Vec<(String, String)>,
|
||||
pub body: Option<Value>,
|
||||
}
|
||||
|
||||
impl Request {
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(k, _)| k == name)
|
||||
.map(|(_, v)| v.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
/// How the fake answers.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum Mode {
|
||||
/// From the fixtures.
|
||||
Fixtures,
|
||||
/// Every request with this status. A 422 echoes the request's body, as Honcho does.
|
||||
Status(u16),
|
||||
/// Read the request, then answer nothing until the fake is dropped (at most 30 s).
|
||||
Hang,
|
||||
/// As a workspace that does not exist: a search or a conclusion query answers 200 with no
|
||||
/// hits (fixtures 34, 35), chat 404 (36).
|
||||
Missing,
|
||||
}
|
||||
|
||||
struct Shared {
|
||||
mode: Mutex<Mode>,
|
||||
/// A mode for one workspace's requests, by its name, instead of `mode`.
|
||||
workspaces: Mutex<HashMap<String, Mode>>,
|
||||
requests: Mutex<Vec<Request>>,
|
||||
/// What was made, by method and path and id: a second create answers 200, a second delete
|
||||
/// 404.
|
||||
made: Mutex<HashSet<String>>,
|
||||
stop: AtomicBool,
|
||||
}
|
||||
|
||||
pub struct FakeHoncho {
|
||||
pub url: String,
|
||||
port: u16,
|
||||
shared: Arc<Shared>,
|
||||
accept: Option<JoinHandle<()>>,
|
||||
}
|
||||
|
||||
/// A recorded fixture: its response's status and body.
|
||||
fn fixture(name: &str) -> (u16, String) {
|
||||
let path = PathBuf::from(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/honcho")
|
||||
.join(name);
|
||||
let text = std::fs::read_to_string(&path).unwrap();
|
||||
let value: Value = serde_json::from_str(&text).unwrap();
|
||||
let status = u16::try_from(value["response"]["status"].as_u64().unwrap()).unwrap();
|
||||
let body = match &value["response"]["body"] {
|
||||
Value::String(s) if s.is_empty() => String::new(),
|
||||
other => other.to_string(),
|
||||
};
|
||||
(status, body)
|
||||
}
|
||||
|
||||
impl FakeHoncho {
|
||||
pub fn start() -> FakeHoncho {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").unwrap();
|
||||
let port = listener.local_addr().unwrap().port();
|
||||
let shared = Arc::new(Shared {
|
||||
mode: Mutex::new(Mode::Fixtures),
|
||||
workspaces: Mutex::new(HashMap::new()),
|
||||
requests: Mutex::new(Vec::new()),
|
||||
made: Mutex::new(HashSet::new()),
|
||||
stop: AtomicBool::new(false),
|
||||
});
|
||||
let for_accept = Arc::clone(&shared);
|
||||
let accept = std::thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
if for_accept.stop.load(Ordering::SeqCst) {
|
||||
return;
|
||||
}
|
||||
let Ok(stream) = stream else { return };
|
||||
let shared = Arc::clone(&for_accept);
|
||||
std::thread::spawn(move || serve(stream, &shared));
|
||||
}
|
||||
});
|
||||
FakeHoncho {
|
||||
url: format!("http://127.0.0.1:{port}"),
|
||||
port,
|
||||
shared,
|
||||
accept: Some(accept),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn set(&self, mode: Mode) {
|
||||
*self.shared.mode.lock().unwrap() = mode;
|
||||
}
|
||||
|
||||
/// Answer `workspace`'s requests in `mode`, whatever `set` says for the others.
|
||||
pub fn set_workspace(&self, workspace: &str, mode: Mode) {
|
||||
self.shared
|
||||
.workspaces
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(workspace.to_string(), mode);
|
||||
}
|
||||
|
||||
/// Every request read so far, in order.
|
||||
pub fn requests(&self) -> Vec<Request> {
|
||||
self.shared.requests.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
/// Forget the requests read so far (not what was made).
|
||||
pub fn clear(&self) {
|
||||
self.shared.requests.lock().unwrap().clear();
|
||||
}
|
||||
|
||||
/// Waits up to 10 s until at least `n` requests were read; true if they were.
|
||||
pub fn wait_for(&self, n: usize) -> bool {
|
||||
let deadline = Instant::now() + Duration::from_secs(10);
|
||||
while Instant::now() < deadline {
|
||||
if self.requests().len() >= n {
|
||||
return true;
|
||||
}
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
false
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for FakeHoncho {
|
||||
fn drop(&mut self) {
|
||||
self.shared.stop.store(true, Ordering::SeqCst);
|
||||
let _ = TcpStream::connect(("127.0.0.1", self.port));
|
||||
if let Some(accept) = self.accept.take() {
|
||||
let _ = accept.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Reads one request: the head up to its blank line, then `Content-Length` bytes of body.
|
||||
fn read_request(stream: &mut TcpStream) -> Option<Request> {
|
||||
let mut buf = Vec::new();
|
||||
let mut byte = [0u8; 1];
|
||||
while !buf.ends_with(b"\r\n\r\n") {
|
||||
if stream.read(&mut byte).ok()? == 0 {
|
||||
return None;
|
||||
}
|
||||
buf.push(byte[0]);
|
||||
}
|
||||
let head = String::from_utf8(buf).ok()?;
|
||||
let mut lines = head.split("\r\n");
|
||||
let mut first = lines.next()?.split(' ');
|
||||
let (method, path) = (first.next()?.to_string(), first.next()?.to_string());
|
||||
let headers: Vec<(String, String)> = lines
|
||||
.filter_map(|l| l.split_once(": "))
|
||||
.map(|(k, v)| (k.to_ascii_lowercase(), v.to_string()))
|
||||
.collect();
|
||||
let length: usize = headers
|
||||
.iter()
|
||||
.find(|(k, _)| k == "content-length")
|
||||
.map_or(Some(0), |(_, v)| v.parse().ok())?;
|
||||
let mut body = vec![0u8; length];
|
||||
stream.read_exact(&mut body).ok()?;
|
||||
let body = if length == 0 {
|
||||
None
|
||||
} else {
|
||||
serde_json::from_slice(&body).ok()
|
||||
};
|
||||
Some(Request {
|
||||
method,
|
||||
path,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
/// The fixture that answers `request`, by its method and the shape of its path; the second of a
|
||||
/// pair when the same thing was made (or deleted) before.
|
||||
fn answer(request: &Request, made: &Mutex<HashSet<String>>) -> (u16, String) {
|
||||
let parts: Vec<&str> = request.path.trim_start_matches('/').split('/').collect();
|
||||
let id = request
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|b| b["id"].as_str())
|
||||
.unwrap_or_default()
|
||||
.to_string();
|
||||
let again = !made
|
||||
.lock()
|
||||
.unwrap()
|
||||
.insert(format!("{} {} {id}", request.method, request.path));
|
||||
let configured = request
|
||||
.body
|
||||
.as_ref()
|
||||
.is_some_and(|b| b.get("configuration").is_some());
|
||||
match (request.method.as_str(), parts.as_slice()) {
|
||||
("POST", ["v3", "workspaces"]) if again => fixture("02-workspace-create-again.json"),
|
||||
("POST", ["v3", "workspaces"]) => fixture("01-workspace-create.json"),
|
||||
("POST", ["v3", "workspaces", _, "peers"]) if again => {
|
||||
fixture("05-peer-create-boxmaker0-again.json")
|
||||
}
|
||||
("POST", ["v3", "workspaces", _, "peers"]) if configured => {
|
||||
fixture("04-peer-create-boxmaker0.json")
|
||||
}
|
||||
("POST", ["v3", "workspaces", _, "peers"]) => fixture("03-peer-create-kyle.json"),
|
||||
("POST", ["v3", "workspaces", _, "sessions"]) if again => {
|
||||
fixture("07-session-create-again.json")
|
||||
}
|
||||
("POST", ["v3", "workspaces", _, "sessions"]) => fixture("06-session-create.json"),
|
||||
("POST", ["v3", "workspaces", _, "sessions", _, "messages"]) => {
|
||||
first(fixture("08-messages-add-batch.json"), request, "messages")
|
||||
}
|
||||
("POST", ["v3", "workspaces", _, "conclusions"]) => first(
|
||||
fixture("11-conclusions-create-batch.json"),
|
||||
request,
|
||||
"conclusions",
|
||||
),
|
||||
// A search answers the three recorded messages about tea (fixture 30), whatever it asked.
|
||||
("POST", ["v3", "workspaces", _, "search"]) => fixture("30-workspace-search-limit.json"),
|
||||
// A conclusion query answers the recorded one about the owner (32) when it asks about
|
||||
// `kyle`, else the one about an agent (33).
|
||||
("POST", ["v3", "workspaces", _, "conclusions", "query"]) => {
|
||||
let about = request
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|b| b["filters"]["observed_id"].as_str().map(str::to_string));
|
||||
if about.as_deref() == Some("kyle") {
|
||||
fixture("32-conclusions-query-owner.json")
|
||||
} else {
|
||||
fixture("33-conclusions-query-agent.json")
|
||||
}
|
||||
}
|
||||
("DELETE", ["v3", "workspaces", _, "conclusions", _]) if again => {
|
||||
fixture("16-conclusion-delete-again.json")
|
||||
}
|
||||
("DELETE", ["v3", "workspaces", _, "conclusions", _]) => {
|
||||
fixture("15-conclusion-delete.json")
|
||||
}
|
||||
_ => (404, r#"{"detail":"Not Found"}"#.to_string()),
|
||||
}
|
||||
}
|
||||
|
||||
/// A recorded batch answer cut to as many items as the request's `field` holds (the recorded ones
|
||||
/// hold two), so that one message or conclusion gets one id.
|
||||
fn first(recorded: (u16, String), request: &Request, field: &str) -> (u16, String) {
|
||||
let asked = request
|
||||
.body
|
||||
.as_ref()
|
||||
.and_then(|b| b[field].as_array())
|
||||
.map_or(0, Vec::len);
|
||||
let mut items: Vec<Value> = serde_json::from_str(&recorded.1).unwrap();
|
||||
items.truncate(asked);
|
||||
(recorded.0, Value::Array(items).to_string())
|
||||
}
|
||||
|
||||
fn serve(mut stream: TcpStream, shared: &Shared) {
|
||||
let _ = stream.set_read_timeout(Some(Duration::from_secs(10)));
|
||||
let Some(request) = read_request(&mut stream) else {
|
||||
return;
|
||||
};
|
||||
shared.requests.lock().unwrap().push(request.clone());
|
||||
let workspace = request.path.split('/').nth(3).unwrap_or("").to_string();
|
||||
let mode = match shared.workspaces.lock().unwrap().get(&workspace) {
|
||||
Some(mode) => *mode,
|
||||
None => *shared.mode.lock().unwrap(),
|
||||
};
|
||||
let (status, body) = match mode {
|
||||
Mode::Fixtures => answer(&request, &shared.made),
|
||||
Mode::Missing if request.path.ends_with("/search") => {
|
||||
fixture("34-workspace-search-missing-workspace.json")
|
||||
}
|
||||
Mode::Missing if request.path.ends_with("/conclusions/query") => {
|
||||
fixture("35-conclusions-query-missing-workspace.json")
|
||||
}
|
||||
Mode::Missing => fixture("36-workspace-chat-missing-workspace.json"),
|
||||
Mode::Status(422) => {
|
||||
let input = request.body.clone().unwrap_or(Value::Null);
|
||||
(
|
||||
422,
|
||||
serde_json::json!({ "detail": [{ "input": input }] }).to_string(),
|
||||
)
|
||||
}
|
||||
Mode::Status(status) => (status, r#"{"detail":"refused"}"#.to_string()),
|
||||
Mode::Hang => {
|
||||
let deadline = Instant::now() + Duration::from_secs(30);
|
||||
while !shared.stop.load(Ordering::SeqCst) && Instant::now() < deadline {
|
||||
std::thread::sleep(Duration::from_millis(10));
|
||||
}
|
||||
return;
|
||||
}
|
||||
};
|
||||
let head = format!(
|
||||
"HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n",
|
||||
body.len()
|
||||
);
|
||||
let _ = stream.write_all(head.as_bytes());
|
||||
let _ = stream.write_all(body.as_bytes());
|
||||
}
|
||||
Reference in New Issue
Block a user