boxmaker/crates/memoryd/tests/honcho_ask_serve.rs

//! `memoryd` answering `ask_memory` through its `brokerd` socket (M5c spec, section 9), against a
//! fake Honcho: `own`'s answer with its labels, a fleet answer making the result untrusted (T58);
//! a chat that hangs holds up no memory call, because `ask_memory` never takes the state lock;
//! without `[honcho]`, a sentence that says so; bad arguments refused with `bxcore`'s sentences;
//! an unknown name's sentence as the result; an unknown tool refused. Every wait has a deadline.
//! Do not edit.

#[path = "support/honcho.rs"]
mod honcho;
#[path = "support/tmp.rs"]
mod tmp;

use std::os::unix::net::UnixStream;
use std::path::Path;
use std::sync::Arc;
use std::time::{Duration, Instant};

use bxcore::{
    CallId, DataClass, Envelope, MemoryCall, Message, SessionId, ToolRequest, ToolResponse,
    read_frame, write_frame,
};
use honcho::{FakeHoncho, Mode};
use memoryd::config::Config;
use memoryd::serve;
use tmp::TempDir;

const QUESTION: &str = r#"{"question":"What does the owner drink in the afternoon?"}"#;

/// `[honcho]` for `url`, reading `boxmaker1`'s workspace.
fn section(url: &str) -> String {
    format!(
        "[honcho]\nurl = \"{url}\"\nagent = \"boxmaker0\"\nowner = \"kyle\"\ntrusted = \"boxmaker0-trusted\"\nuntrusted = \"boxmaker0-untrusted\"\nfleet = \"fleet\"\n[[honcho.read]]\nagent = \"boxmaker1\"\nworkspace = \"boxmaker1-trusted\"\n"
    )
}

struct Home {
    config: Config,
    /// Removed when the test ends; declared last, so it outlives what writes into it (T44).
    _dir: TempDir,
}

impl Home {
    /// A `memoryd` serving on a thread, with `[honcho]` at `url` when there is one.
    fn serve(tag: &str, url: Option<&str>) -> Home {
        let dir = TempDir::new(tag);
        let soul = dir.write("etc/soul.md", "You are Boxmaker.\n");
        let text = format!(
            "[paths]\nhome = \"{}\"\n[memory]\nsoul_starter = \"{}\"\n{}",
            dir.path().display(),
            soul.display(),
            url.map(section).unwrap_or_default()
        );
        let config = Config::parse(&text).unwrap();
        let log: serve::Log = Arc::new(|_: &str| {});
        let running = serve::start(config.clone(), log).unwrap();
        std::thread::spawn(move || running.run());
        Home { config, _dir: dir }
    }
}

/// One call on `brokerd`'s socket and its answer, within 20 s; how long it took.
fn call(socket: &Path, tool: &str, arguments: &str) -> (ToolResponse, Duration) {
    call_at(socket, tool, arguments, DataClass::Private)
}

/// `call`, with the session's class on the `MemoryCall`.
fn call_at(
    socket: &Path,
    tool: &str,
    arguments: &str,
    taint: DataClass,
) -> (ToolResponse, Duration) {
    let started = Instant::now();
    let mut stream = UnixStream::connect(socket).unwrap();
    stream
        .set_read_timeout(Some(Duration::from_secs(20)))
        .unwrap();
    let env = Envelope {
        v: 1,
        id: 5,
        r#final: true,
        msg: Message::MemoryCall(MemoryCall {
            request: ToolRequest {
                session: SessionId::new("mm-now").unwrap(),
                call: CallId(5),
                tool: tool.to_string(),
                arguments: arguments.to_string(),
            },
            taint,
            untrusted: false,
        }),
    };
    write_frame(&mut stream, &env).unwrap();
    match read_frame(&mut stream).unwrap().msg {
        Message::ToolResponse(response) => (response, started.elapsed()),
        other => panic!("{other:?}"),
    }
}

fn result(response: ToolResponse) -> (String, DataClass, bool) {
    match response {
        ToolResponse::Result {
            content,
            class,
            untrusted,
            truncated,
        } => {
            assert!(!truncated);
            (content, class, untrusted)
        }
        other => panic!("{other:?}"),
    }
}

fn failed(response: ToolResponse) -> String {
    match response {
        ToolResponse::Failed { message } => message,
        other => panic!("{other:?}"),
    }
}

/// `own`'s answer, trusted; a fleet answer, untrusted, as `brokerd` will label the session
/// (T58, end to end).
#[test]
fn each_answer_comes_back_with_its_workspaces_label() {
    let fake = FakeHoncho::start();
    let home = Home::serve("ask-serve-labels", Some(&fake.url));
    let socket = home.config.broker_socket();
    let (content, class, untrusted) = result(call(&socket, "ask_memory", QUESTION).0);
    assert!(
        content.starts_with("[1] memory · own · private\n  Based on what I found in memory, "),
        "{content}"
    );
    assert_eq!((class, untrusted), (DataClass::Private, false));
    let fleet = r#"{"question":"What does the owner drink?","workspaces":["fleet"]}"#;
    let (content, class, untrusted) = result(call(&socket, "ask_memory", fleet).0);
    assert!(
        content.starts_with("[1] memory-tainted · fleet · private\n  "),
        "{content}"
    );
    assert_eq!((class, untrusted), (DataClass::Private, true));
}

/// `ask_memory` never takes the state lock: while its chat hangs, a memory call is answered at
/// once. Stopping the fake Honcho ends the chat, and the answer says the workspace was
/// unavailable.
#[test]
fn a_hanging_chat_holds_up_no_memory_call() {
    let fake = FakeHoncho::start();
    fake.set(Mode::Hang);
    let home = Home::serve("ask-serve-hang", Some(&fake.url));
    let socket = home.config.broker_socket();
    let for_ask = socket.clone();
    let asking = std::thread::spawn(move || call(&for_ask, "ask_memory", QUESTION));
    let deadline = Instant::now() + Duration::from_secs(10);
    while !fake.requests().iter().any(|r| r.path.ends_with("/chat")) {
        assert!(Instant::now() < deadline, "the chat was never sent");
        std::thread::sleep(Duration::from_millis(10));
    }
    let note = r#"{"action":"note","text":"tea is kept in the blue tin"}"#;
    let (answer, took) = call(&socket, "memory", note);
    assert!(matches!(answer, ToolResponse::Result { .. }), "{answer:?}");
    assert!(
        took < Duration::from_secs(2),
        "the memory call waited {took:?}"
    );
    assert!(!asking.is_finished(), "the chat is still waiting");
    drop(fake);
    let (answer, _) = asking.join().unwrap();
    let (content, class, untrusted) = result(answer);
    assert_eq!(content, "own: unavailable");
    assert_eq!((class, untrusted), (DataClass::Private, false));
}

/// Without `[honcho]`, a result the model reads and moves on from, private and trusted.
#[test]
fn without_honcho_the_answer_says_so() {
    let home = Home::serve("ask-serve-off", None);
    let (content, class, untrusted) =
        result(call(&home.config.broker_socket(), "ask_memory", QUESTION).0);
    assert_eq!(
        content,
        "ask_memory: Honcho is not configured for this agent; recall searches its own memory"
    );
    assert_eq!((class, untrusted), (DataClass::Private, false));
}

/// `memoryd` checks the arguments again, as it does `recall`'s: a refusal is `bxcore`'s sentence.
#[test]
fn bad_arguments_are_refused_with_the_checks_sentences() {
    let fake = FakeHoncho::start();
    let home = Home::serve("ask-serve-bad", Some(&fake.url));
    let socket = home.config.broker_socket();
    assert_eq!(
        failed(call(&socket, "ask_memory", r#"{"question":"  "}"#).0),
        "question is empty; write the question to ask"
    );
    let three = r#"{"question":"q","workspaces":["own","untrusted","fleet"]}"#;
    assert_eq!(
        failed(call(&socket, "ask_memory", three).0),
        "workspaces names 3, over the limit of 2; ask about the others in another call"
    );
    let unknown = failed(
        call(
            &socket,
            "ask_memory",
            r#"{"question":"q","reasoning":"low"}"#,
        )
        .0,
    );
    assert!(
        unknown.starts_with("ask_memory's arguments: unknown field `reasoning`"),
        "{unknown}"
    );
    assert_eq!(fake.requests().len(), 0, "nothing was sent");
}

/// A name that is none of this agent's workspaces: the sentence listing the names it may use,
/// as the result, and nothing sent.
#[test]
fn an_unknown_names_sentence_is_the_result() {
    let fake = FakeHoncho::start();
    let home = Home::serve("ask-serve-unknown", Some(&fake.url));
    let asked = r#"{"question":"q","workspaces":["boxmaker9"]}"#;
    let (content, class, untrusted) =
        result(call(&home.config.broker_socket(), "ask_memory", asked).0);
    assert_eq!(
        content,
        "no workspace is named boxmaker9; ask own, untrusted, fleet or boxmaker1"
    );
    assert_eq!((class, untrusted), (DataClass::Private, false));
    assert_eq!(fake.requests().len(), 0);
}

#[test]
fn an_unknown_tool_is_refused() {
    let home = Home::serve("ask-serve-tool", None);
    assert_eq!(
        failed(call(&home.config.broker_socket(), "forget", "{}").0),
        "memoryd runs only recall, memory and ask_memory"
    );
}

/// Spec 13: a `secret` session's question still reaches Honcho; only a question holding a secret
/// is refused.
#[test]
fn a_secret_session_still_asks_honcho() {
    let fake = FakeHoncho::start();
    let home = Home::serve("ask-serve-secret", Some(&fake.url));
    let (content, class, untrusted) = result(
        call_at(
            &home.config.broker_socket(),
            "ask_memory",
            QUESTION,
            DataClass::Secret,
        )
        .0,
    );
    assert!(
        fake.requests().iter().any(|r| r.path.ends_with("/chat")),
        "Honcho was not asked: {content}"
    );
    assert!(
        content.starts_with("[1] memory · own · private\n  Based on what I found in memory, "),
        "{content}"
    );
    assert_eq!((class, untrusted), (DataClass::Private, false));
}