memoryd: the Honcho workspaces it reads, with their clients and labels

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/memoryd/src/honcho/client.rs

@@ -19,8 +19,8 @@ use crate::honcho::naming::HonchoSession;
 /// How long a connection may take, and a read may go without a byte.
 pub const CONNECT: Duration = Duration::from_secs(5);
 pub const READ: Duration = Duration::from_secs(30);
-/// A chat's read: a non-streamed chat sends nothing until it is done (M5c-2's `ask_memory`).
-pub const CHAT_READ: Duration = Duration::from_secs(180);
+/// A chat's read: a non-streamed chat sends nothing until it is done (M5c spec, section 9).
+pub const CHAT_READ: Duration = Duration::from_secs(150);
 
 /// The timeouts a client is built with. Tests shorten them.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
@@ -176,7 +176,7 @@ fn created<T>(result: Result<T, Error>) -> Result<(), HonchoError> {
 }
 
 /// One workspace's client: no `roncho.toml` read, the timeouts given, the key when there is one.
-fn client(
+pub(crate) fn client(
     url: &str,
     workspace: &str,
     key: Option<&secrets::Secret>,
@@ -197,7 +197,7 @@ fn client(
 }
 
 /// The key named `name` in `[secrets]`, if one is configured, read now.
-fn key(
+pub(crate) fn key(
     config: &HonchoConfig,
     name: &str,
     env: &dyn Fn(&str) -> Option<OsString>,

crates/memoryd/src/honcho/config.rs

@@ -14,6 +14,10 @@ pub const RUNBOOK: &str = "see docs/runbook.md#honcho-config-invalid";
 /// The longest name Honcho accepts for a workspace, a peer or a session (M5c spec, section 6).
 pub const MAX_NAME: usize = 512;
 
+/// The names `ask_memory` gives this agent's own workspaces and the fleet's (M5c spec, section 9),
+/// which no `[[honcho.read]]` agent may have.
+pub const RESERVED: [&str; 3] = ["own", "untrusted", "fleet"];
+
 /// The names of the keys in `[secrets.<name>]`, one per workspace. A read workspace's key is
 /// `READ_KEY` and its agent's name (`read_key`).
 pub const TRUSTED_KEY: &str = "honcho_trusted";
@@ -233,6 +237,13 @@ fn read_workspace(
             entry.agent
         ));
     }
+    if RESERVED.contains(&entry.agent.as_str()) {
+        return Err(format!(
+            "[[honcho.read]] agent {:?} is a name ask_memory gives this agent's own workspaces: \
+             own, untrusted and fleet cannot be agents' names",
+            entry.agent
+        ));
+    }
     if earlier.iter().any(|r| r.agent == entry.agent) {
         return Err(format!(
             "[[honcho.read]] agent {:?} is listed twice",

crates/memoryd/src/honcho/mod.rs

@@ -9,6 +9,7 @@ pub mod ledger;
 pub mod messages;
 pub mod mirror;
 pub mod naming;
+pub mod read;
 pub mod report;
 pub mod start;
 pub mod state;

crates/memoryd/src/honcho/read.rs

@@ -0,0 +1,226 @@
+//! The Honcho workspaces this agent reads (M5c spec, sections 3, 8 and 9): each `[[honcho.read]]`
+//! workspace, the fleet's, and the agent's own two, which only `ask_memory` asks. Each has a
+//! client for searches and one for chat, whose answers take far longer, and the label its results
+//! carry. Nothing here writes to Honcho, keeps a response or shows a key.
+
+use std::ffi::OsString;
+use std::time::Duration;
+
+use bxcore::DataClass;
+use bxcore::memory::OWN_WORKSPACE;
+use roncho::blocking::Client;
+
+use crate::honcho::client::{CHAT_READ, CONNECT, Timeouts, client, key};
+use crate::honcho::config::{FLEET_KEY, HonchoConfig, TRUSTED_KEY, Trust, UNTRUSTED_KEY, read_key};
+
+/// How long a search may go without a byte (M5c spec, section 8).
+pub const SEARCH_READ: Duration = Duration::from_secs(5);
+/// The names `ask_memory` gives the agent's own untrusted workspace and the fleet's. The own
+/// trusted one is `bxcore::memory::OWN_WORKSPACE`, `own`.
+pub const UNTRUSTED: &str = "untrusted";
+pub const FLEET: &str = "fleet";
+
+/// The timeouts the read clients are built with. Tests shorten them.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct ReadTimeouts {
+    pub connect: Duration,
+    pub search: Duration,
+    pub chat: Duration,
+}
+
+impl ReadTimeouts {
+    pub const DEFAULT: ReadTimeouts = ReadTimeouts {
+        connect: CONNECT,
+        search: SEARCH_READ,
+        chat: CHAT_READ,
+    };
+}
+
+/// What a reader reads.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum Source {
+    /// Another agent's workspace, from `[[honcho.read]]`, by the agent's name.
+    Agent(String),
+    Fleet,
+    /// This agent's own trusted workspace, and its own untrusted one: `ask_memory` only.
+    Own,
+    OwnUntrusted,
+}
+
+/// What a reader's results are labelled with: the name each hit shows, whether the owner trusts
+/// the workspace, and its class.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Label {
+    pub name: String,
+    pub trust: Trust,
+    pub class: DataClass,
+}
+
+impl Label {
+    /// The tag on each hit, as a local `recall` hit has: `memory` when trusted, else
+    /// `memory-tainted`.
+    pub fn tag(&self) -> &'static str {
+        match self.trust {
+            Trust::Trusted => "memory",
+            Trust::Untrusted => "memory-tainted",
+        }
+    }
+
+    /// Whether a result makes the session untrusted.
+    pub fn untrusted(&self) -> bool {
+        self.trust == Trust::Untrusted
+    }
+}
+
+/// One workspace this agent reads: what it is, its label, and its two clients.
+#[derive(Debug, Clone)]
+pub struct Reader {
+    pub source: Source,
+    pub label: Label,
+    search: Client,
+    chat: Client,
+}
+
+impl Reader {
+    /// The client for searches and conclusion queries, with the short read timeout.
+    pub fn search(&self) -> &Client {
+        &self.search
+    }
+
+    /// The client for workspace chat, with the long read timeout.
+    pub fn chat(&self) -> &Client {
+        &self.chat
+    }
+}
+
+/// Every workspace this agent reads, built at start with `[honcho]`. Shared by the threads that
+/// search and ask (`roncho`'s blocking client is `Send + Sync`).
+#[derive(Debug)]
+pub struct Readers {
+    read: Vec<Reader>,
+    fleet: Reader,
+    own: Reader,
+    own_untrusted: Reader,
+}
+
+/// What every reader is built from, and the key files' warnings so far.
+struct Build<'a> {
+    config: &'a HonchoConfig,
+    env: &'a dyn Fn(&str) -> Option<OsString>,
+    timeouts: ReadTimeouts,
+    warnings: Vec<String>,
+}
+
+impl Build<'_> {
+    /// A reader of `workspace`: both clients, with the key named `key_name` when one is
+    /// configured, read once.
+    fn reader(
+        &mut self,
+        workspace: &str,
+        key_name: &str,
+        source: Source,
+        label: Label,
+    ) -> Result<Reader, String> {
+        let secret = key(self.config, key_name, self.env, &mut self.warnings)?;
+        let search = Timeouts {
+            connect: self.timeouts.connect,
+            read: self.timeouts.search,
+        };
+        let chat = Timeouts {
+            connect: self.timeouts.connect,
+            read: self.timeouts.chat,
+        };
+        let url = &self.config.url;
+        Ok(Reader {
+            source,
+            label,
+            search: client(url, workspace, secret.as_ref(), search)?,
+            chat: client(url, workspace, secret.as_ref(), chat)?,
+        })
+    }
+}
+
+/// A `private` label.
+fn private(name: &str, trust: Trust) -> Label {
+    Label {
+        name: name.to_string(),
+        trust,
+        class: DataClass::Private,
+    }
+}
+
+impl Readers {
+    /// A reader for each `[[honcho.read]]` workspace, in the order written, then the fleet's and
+    /// the agent's own two. The warnings are the key files' plaintext warnings, for the caller to
+    /// log. An error names the key or the workspace, never a key's value.
+    pub fn new(
+        config: &HonchoConfig,
+        env: &dyn Fn(&str) -> Option<OsString>,
+        timeouts: ReadTimeouts,
+    ) -> Result<(Readers, Vec<String>), String> {
+        let mut build = Build {
+            config,
+            env,
+            timeouts,
+            warnings: Vec::new(),
+        };
+        let mut read = Vec::new();
+        for entry in &config.read {
+            let label = Label {
+                name: entry.agent.clone(),
+                trust: entry.trust,
+                class: entry.class,
+            };
+            let source = Source::Agent(entry.agent.clone());
+            read.push(build.reader(&entry.workspace, &read_key(&entry.agent), source, label)?);
+        }
+        let fleet = private(FLEET, Trust::Untrusted);
+        let fleet = build.reader(&config.fleet, FLEET_KEY, Source::Fleet, fleet)?;
+        let own = private(OWN_WORKSPACE, Trust::Trusted);
+        let own = build.reader(&config.trusted, TRUSTED_KEY, Source::Own, own)?;
+        let own_untrusted = private(UNTRUSTED, Trust::Untrusted);
+        let own_untrusted = build.reader(
+            &config.untrusted,
+            UNTRUSTED_KEY,
+            Source::OwnUntrusted,
+            own_untrusted,
+        )?;
+        let readers = Readers {
+            read,
+            fleet,
+            own,
+            own_untrusted,
+        };
+        Ok((readers, build.warnings))
+    }
+
+    /// What `recall` searches: the `[[honcho.read]]` workspaces in the order written, then the
+    /// fleet's. Never the agent's own (M5c spec, section 8): the local index holds their text.
+    pub fn recall_sources(&self) -> Vec<&Reader> {
+        self.read
+            .iter()
+            .chain(std::iter::once(&self.fleet))
+            .collect()
+    }
+
+    /// The reader `ask_memory` names `name`: `own`, `untrusted`, `fleet`, or a `[[honcho.read]]`
+    /// agent. Any other name is the sentence the model is shown, listing the names it may use.
+    pub fn resolve(&self, name: &str) -> Result<&Reader, String> {
+        match name {
+            OWN_WORKSPACE => return Ok(&self.own),
+            UNTRUSTED => return Ok(&self.own_untrusted),
+            FLEET => return Ok(&self.fleet),
+            _ => {}
+        }
+        if let Some(reader) = self.read.iter().find(|r| r.label.name == name) {
+            return Ok(reader);
+        }
+        let mut names = vec![OWN_WORKSPACE, UNTRUSTED, FLEET];
+        names.extend(self.read.iter().map(|r| r.label.name.as_str()));
+        let (last, rest) = names.split_last().unwrap_or((&FLEET, &[]));
+        Err(format!(
+            "no workspace is named {name}; ask {} or {last}",
+            rest.join(", ")
+        ))
+    }
+}

crates/memoryd/src/honcho/start.rs

@@ -10,6 +10,7 @@ use crate::honcho::client::{Clients, Timeouts};
 use crate::honcho::config::HonchoConfig;
 use crate::honcho::ledger::{Ledger, Record};
 use crate::honcho::mirror::Mirror;
+use crate::honcho::read::{ReadTimeouts, Readers};
 use crate::honcho::state;
 use crate::honcho::turns::{LEDGER, TurnSync};
 use crate::honcho::worker::{AskBox, Tuning, Wake, Worker};
@@ -19,6 +20,8 @@ use crate::serve::{Log, StartError};
 /// clients with their keys read. Nothing is bound or started yet.
 pub struct Opened {
     pub sync: Arc<Mutex<TurnSync>>,
+    /// The workspaces `recall` and `ask_memory` read (M5c spec, sections 8 and 9).
+    pub readers: Arc<Readers>,
     clients: Clients,
     agent: String,
     owner: String,
@@ -65,11 +68,19 @@ pub fn open(config: &Config, honcho: &HonchoConfig, log: &Log) -> Result<Opened,
     let env = |name: &str| std::env::var_os(name);
     let (clients, warnings) =
         Clients::new(honcho, &env, Timeouts::WRITES).map_err(StartError::Secret)?;
-    for warning in warnings {
+    let (readers, read_warnings) =
+        Readers::new(honcho, &env, ReadTimeouts::DEFAULT).map_err(StartError::Secret)?;
+    // The readers load the keys of the three written workspaces again: each warning once.
+    let read_warnings: Vec<String> = read_warnings
+        .into_iter()
+        .filter(|w| !warnings.contains(w))
+        .collect();
+    for warning in warnings.into_iter().chain(read_warnings) {
         (log)(&warning);
     }
     Ok(Opened {
         sync: Arc::new(Mutex::new(sync)),
+        readers: Arc::new(readers),
         clients,
         agent: honcho.agent.clone(),
         owner: honcho.owner.clone(),