memoryd: a Honcho client per workspace, and its workspace, peers and sessions made sure of

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/memoryd/src/honcho/client.rs

@@ -0,0 +1,322 @@
+//! This agent's three Honcho workspaces through `roncho`'s blocking client (M5c spec, section 6):
+//! one client per workspace, with its own key, and the steps that make sure a workspace, a peer
+//! and a session exist before anything is written to them. Nothing here keeps or logs a response
+//! body or a key: an error is its kind and its status.
+
+use std::collections::{HashMap, HashSet};
+use std::ffi::OsString;
+use std::time::Duration;
+
+use bxcore::secrets;
+use roncho::blocking::{Client, Error};
+use roncho::{PeerCreate, SessionCreate, SessionPeerConfig, WorkspaceCreate};
+use serde_json::{Map, Value, json};
+
+use crate::honcho::config::{FLEET_KEY, HonchoConfig, TRUSTED_KEY, UNTRUSTED_KEY};
+use crate::honcho::ledger::Workspace;
+use crate::honcho::naming::HonchoSession;
+
+/// How long a connection may take, and a read may go without a byte.
+pub const CONNECT: Duration = Duration::from_secs(5);
+pub const READ: Duration = Duration::from_secs(30);
+/// A chat's read: a non-streamed chat sends nothing until it is done (M5c-2's `ask_memory`).
+pub const CHAT_READ: Duration = Duration::from_secs(180);
+
+/// The timeouts a client is built with. Tests shorten them.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct Timeouts {
+    pub connect: Duration,
+    pub read: Duration,
+}
+
+impl Timeouts {
+    pub const WRITES: Timeouts = Timeouts {
+        connect: CONNECT,
+        read: READ,
+    };
+}
+
+/// Why a call to Honcho failed: its kind and status only, never the response's body.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub enum HonchoError {
+    /// No answer that counts: no connection, a timeout, a closed socket, or a 5xx. Try again later.
+    Unreachable {
+        kind: &'static str,
+        status: Option<u16>,
+    },
+    /// Honcho answered, and will answer the same again: a 4xx, a redirect, or a body that is not
+    /// what `roncho` expects.
+    Refused {
+        kind: &'static str,
+        status: Option<u16>,
+    },
+    /// The workspace's key was refused (401 or 403).
+    KeyRefused { status: u16 },
+}
+
+impl HonchoError {
+    pub fn kind(&self) -> &'static str {
+        match self {
+            HonchoError::Unreachable { kind, .. } | HonchoError::Refused { kind, .. } => kind,
+            HonchoError::KeyRefused { .. } => "key-refused",
+        }
+    }
+
+    pub fn status(&self) -> Option<u16> {
+        match self {
+            HonchoError::Unreachable { status, .. } | HonchoError::Refused { status, .. } => {
+                *status
+            }
+            HonchoError::KeyRefused { status } => Some(*status),
+        }
+    }
+}
+
+impl std::fmt::Display for HonchoError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let what = match self {
+            HonchoError::Unreachable { .. } => "Honcho is unreachable",
+            HonchoError::Refused { .. } => "Honcho refused the request",
+            HonchoError::KeyRefused { .. } => "Honcho refused the key",
+        };
+        match self.status() {
+            Some(status) => write!(f, "{what} ({}, HTTP {status})", self.kind()),
+            None => write!(f, "{what} ({})", self.kind()),
+        }
+    }
+}
+
+impl std::error::Error for HonchoError {}
+
+/// `roncho`'s error as a `HonchoError`. Its text is dropped: a `Status` or `Decode` error carries
+/// up to 4 KiB of the response, which can echo memory text (a 422 echoes the whole input).
+pub fn classify(error: &Error) -> HonchoError {
+    let unreachable = |kind, status| HonchoError::Unreachable { kind, status };
+    let refused = |kind, status| HonchoError::Refused { kind, status };
+    match error {
+        Error::Connect(_) => unreachable("connect", None),
+        Error::Timeout => unreachable("timeout", None),
+        Error::Closed => unreachable("closed", None),
+        Error::Io(_) => unreachable("io", None),
+        Error::Status { status, .. } if *status >= 500 => unreachable("server", Some(*status)),
+        Error::Status { status, .. } => refused("status", Some(*status)),
+        Error::Unauthorized { status } => HonchoError::KeyRefused { status: *status },
+        Error::Redirect { status } => refused("redirect", Some(*status)),
+        Error::ActiveSessions { .. } => refused("active-sessions", Some(409)),
+        Error::TooLarge => refused("too-large", None),
+        Error::Decode(_) => refused("decode", None),
+        Error::Stream(_) => refused("stream", None),
+        Error::Config(_) => refused("config", None),
+        _ => refused("other", None),
+    }
+}
+
+/// What has been made sure of in one workspace during this run.
+#[derive(Debug, Clone, PartialEq, Eq, Hash)]
+enum Ensured {
+    Workspace,
+    Peer(String),
+    Session(String),
+}
+
+/// The three clients, and what they have made sure of since `memoryd` started. After a restart
+/// everything is made sure of again: each step is get-or-create, so that costs a request, no more.
+#[derive(Debug)]
+pub struct Clients {
+    trusted: Client,
+    untrusted: Client,
+    fleet: Client,
+    agent: String,
+    ensured: HashSet<(Workspace, Ensured)>,
+}
+
+/// The fleet workspace's configuration: nothing reasons over the fleet's notes (M5c spec,
+/// section 6). All three are set, since Honcho checks each on its own.
+pub fn fleet_configuration() -> Map<String, Value> {
+    let mut map = Map::new();
+    for feature in ["reasoning", "summary", "dream"] {
+        map.insert(feature.to_string(), json!({ "enabled": false }));
+    }
+    map
+}
+
+/// A session's peers: the agent's, unobserved, and each speaker's, observed; no peer forms a
+/// picture of the others.
+pub fn session_peers(agent: &str, speakers: &[String]) -> HashMap<String, SessionPeerConfig> {
+    let mut peers = HashMap::new();
+    peers.insert(
+        agent.to_string(),
+        SessionPeerConfig::new()
+            .observe_me(false)
+            .observe_others(false),
+    );
+    for speaker in speakers {
+        peers.insert(
+            speaker.clone(),
+            SessionPeerConfig::new()
+                .observe_me(true)
+                .observe_others(false),
+        );
+    }
+    peers
+}
+
+/// A create that found the thing there (409) did what it was for.
+fn created<T>(result: Result<T, Error>) -> Result<(), HonchoError> {
+    match result {
+        Ok(_) => Ok(()),
+        Err(e) => match classify(&e) {
+            HonchoError::Refused {
+                status: Some(409), ..
+            } => Ok(()),
+            other => Err(other),
+        },
+    }
+}
+
+/// One workspace's client: no `roncho.toml` read, the timeouts given, the key when there is one.
+fn client(
+    url: &str,
+    workspace: &str,
+    key: Option<&secrets::Secret>,
+    timeouts: Timeouts,
+) -> Result<Client, String> {
+    let mut builder = Client::builder()
+        .without_config_file()
+        .base_url(url)
+        .workspace_id(workspace)
+        .connect_timeout(timeouts.connect)
+        .read_timeout(timeouts.read);
+    if let Some(key) = key {
+        builder = builder.api_key(key.expose());
+    }
+    builder
+        .build()
+        .map_err(|e| format!("the Honcho client for {workspace}: {e}"))
+}
+
+/// The key named `name` in `[secrets]`, if one is configured, read now.
+fn key(
+    config: &HonchoConfig,
+    name: &str,
+    env: &dyn Fn(&str) -> Option<OsString>,
+    warnings: &mut Vec<String>,
+) -> Result<Option<secrets::Secret>, String> {
+    let Some(source) = config.keys.get(name) else {
+        return Ok(None);
+    };
+    let loaded = secrets::load("memoryd", name, source, env).map_err(|e| e.to_string())?;
+    warnings.extend(loaded.warning);
+    Ok(Some(loaded.secret))
+}
+
+impl Clients {
+    /// A client for each of the three workspaces, each with its key when one is configured. The
+    /// warnings are the key files' plaintext warnings, for the caller to log. An error names the
+    /// key or the workspace, never a key's value.
+    pub fn new(
+        config: &HonchoConfig,
+        env: &dyn Fn(&str) -> Option<OsString>,
+        timeouts: Timeouts,
+    ) -> Result<(Clients, Vec<String>), String> {
+        let mut warnings = Vec::new();
+        let trusted_key = key(config, TRUSTED_KEY, env, &mut warnings)?;
+        let untrusted_key = key(config, UNTRUSTED_KEY, env, &mut warnings)?;
+        let fleet_key = key(config, FLEET_KEY, env, &mut warnings)?;
+        let clients = Clients {
+            trusted: client(&config.url, &config.trusted, trusted_key.as_ref(), timeouts)?,
+            untrusted: client(
+                &config.url,
+                &config.untrusted,
+                untrusted_key.as_ref(),
+                timeouts,
+            )?,
+            fleet: client(&config.url, &config.fleet, fleet_key.as_ref(), timeouts)?,
+            agent: config.agent.clone(),
+            ensured: HashSet::new(),
+        };
+        Ok((clients, warnings))
+    }
+
+    /// The client of `workspace`.
+    pub fn client(&self, workspace: Workspace) -> &Client {
+        match workspace {
+            Workspace::Trusted => &self.trusted,
+            Workspace::Untrusted => &self.untrusted,
+            Workspace::Fleet => &self.fleet,
+        }
+    }
+
+    /// Make sure `workspace` exists; the fleet one is made with `fleet_configuration`, the other
+    /// two with Honcho's defaults.
+    pub fn ensure_workspace(&mut self, workspace: Workspace) -> Result<(), HonchoError> {
+        let key = (workspace, Ensured::Workspace);
+        if self.ensured.contains(&key) {
+            return Ok(());
+        }
+        let client = self.client(workspace);
+        let mut create = WorkspaceCreate::new(client.workspace_id());
+        if workspace == Workspace::Fleet {
+            create = create.with_configuration(fleet_configuration());
+        }
+        created(client.get_or_create_workspace(&create))?;
+        self.ensured.insert(key);
+        Ok(())
+    }
+
+    /// Make sure peer `peer` exists in `workspace`. The agent's own peer is made unobserved
+    /// (`observe_me` false): Honcho draws no conclusions from the agent's own output.
+    pub fn ensure_peer(&mut self, workspace: Workspace, peer: &str) -> Result<(), HonchoError> {
+        let key = (workspace, Ensured::Peer(peer.to_string()));
+        if self.ensured.contains(&key) {
+            return Ok(());
+        }
+        let mut create = PeerCreate::new(peer);
+        if peer == self.agent {
+            let mut configuration = Map::new();
+            configuration.insert("observe_me".to_string(), json!(false));
+            create = create.with_configuration(configuration);
+        }
+        created(self.client(workspace).peer(&create))?;
+        self.ensured.insert(key);
+        Ok(())
+    }
+
+    /// Make sure `session` exists in `workspace`, with the agent's peer and its speakers' peers,
+    /// after the workspace and every peer. A session must exist before a message is added to
+    /// it: Honcho makes a missing one on the fly, with no peers configured.
+    pub fn ensure_session(
+        &mut self,
+        workspace: Workspace,
+        session: &HonchoSession,
+    ) -> Result<(), HonchoError> {
+        self.ensure_workspace(workspace)?;
+        let agent = self.agent.clone();
+        self.ensure_peer(workspace, &agent)?;
+        for speaker in &session.speakers {
+            self.ensure_peer(workspace, speaker)?;
+        }
+        let key = (workspace, Ensured::Session(session.name.clone()));
+        if self.ensured.contains(&key) {
+            return Ok(());
+        }
+        let peers = session_peers(&agent, &session.speakers);
+        let create = SessionCreate::new(session.name.as_str()).with_peers(peers);
+        created(self.client(workspace).session(&create))?;
+        self.ensured.insert(key);
+        Ok(())
+    }
+
+    /// Delete conclusion `id` in `workspace`. Gone already (404) counts as done.
+    pub fn delete_conclusion(&self, workspace: Workspace, id: &str) -> Result<(), HonchoError> {
+        match self.client(workspace).delete_conclusion(id) {
+            Ok(()) => Ok(()),
+            Err(e) => match classify(&e) {
+                HonchoError::Refused {
+                    status: Some(404), ..
+                } => Ok(()),
+                other => Err(other),
+            },
+        }
+    }
+}

crates/memoryd/src/honcho/ledger.rs

@@ -208,6 +208,13 @@ impl Ledger {
         self.entries.contains_key(item)
     }
 
+    /// Every decided item with its action, `Skip` included, in the order they were decided.
+    pub fn decisions(&self) -> impl Iterator<Item = (&Item, &Action)> + '_ {
+        self.order
+            .iter()
+            .filter_map(|item| Some((item, &self.entries.get(item)?.action)))
+    }
+
     /// The items waiting to be sent, with their actions, in the order they were decided.
     pub fn outbox(&self) -> Vec<(Item, Action)> {
         self.order

crates/memoryd/src/honcho/mod.rs

@@ -1,7 +1,9 @@
 //! Honcho, the memory this agent shares with the fleet, reached through `roncho` (M5c spec).
 
+pub mod client;
 pub mod config;
 pub mod decide;
 pub mod ledger;
+pub mod naming;
 pub mod state;
 pub mod turns;

crates/memoryd/src/honcho/naming.rs

@@ -0,0 +1,61 @@
+//! Which Honcho session a turn goes to (M5c spec, section 3). A Boxmaker session maps to the Honcho
+//! session of its own id; a speaker new to it starts the next one, `<id>-2`, `<id>-3` and so on,
+//! holding every speaker so far, since a Honcho session's peers are fixed when it is made. The
+//! numbering is read from the ledger alone, so a restart numbers the same way.
+
+use bxcore::SessionId;
+
+use crate::honcho::ledger::{Action, Item};
+
+/// A Honcho session: its name, and the speakers' peers it holds besides the agent's.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct HonchoSession {
+    pub name: String,
+    pub speakers: Vec<String>,
+}
+
+/// The name of the Honcho session for the `count`-th speaker of `session`: the id itself for the
+/// first (and for none), then `<id>-<count>`.
+pub fn session_name(session: &SessionId, count: usize) -> String {
+    if count <= 1 {
+        session.as_str().to_string()
+    } else {
+        format!("{}-{count}", session.as_str())
+    }
+}
+
+/// The Honcho session of turn `turn` of `session`, from the ledger's decisions in their order:
+/// the speakers of that session's `Turn` actions (in any workspace) up to and including this
+/// turn, each counted once, in the order they first spoke. `None` when the turn has no `Turn`
+/// action: undecided, or decided to go nowhere.
+pub fn honcho_session<'a, I>(decisions: I, session: &SessionId, turn: u64) -> Option<HonchoSession>
+where
+    I: IntoIterator<Item = (&'a Item, &'a Action)>,
+{
+    let mut speakers: Vec<String> = Vec::new();
+    for (item, action) in decisions {
+        let Item::Turn {
+            session: s,
+            turn: t,
+        } = item
+        else {
+            continue;
+        };
+        let Action::Turn { speaker, .. } = action else {
+            continue;
+        };
+        if s != session {
+            continue;
+        }
+        if !speakers.contains(speaker) {
+            speakers.push(speaker.clone());
+        }
+        if *t == turn {
+            return Some(HonchoSession {
+                name: session_name(session, speakers.len()),
+                speakers,
+            });
+        }
+    }
+    None
+}