brokerd: recall and memory, decided under built-in grants

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/brokerd/src/args.rs

@@ -5,6 +5,7 @@
 //! was written by the model, so it is treated as hostile.
 
 use proto::tools::{HttpFetchArgs, ReadFileArgs, ShellArgs, WriteFileArgs};
+use proto::{MemoryArgs, RecallArgs};
 use std::fmt;
 
 /// Maximum length, in bytes, of a path.
@@ -12,17 +13,20 @@ pub const MAX_PATH: usize = 4096;
 /// Maximum length, in bytes, of a URL.
 pub const MAX_URL: usize = 2048;
 
-/// The four tools the broker can dispatch to.
+/// The tools the broker can dispatch to: four that run in a container, and the two memory
+/// tools that `memoryd` runs (M5a spec, section 8).
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ToolName {
     ReadFile,
     WriteFile,
     Shell,
     HttpFetch,
+    Recall,
+    Memory,
 }
 
 impl ToolName {
-    /// The four tools, in the order listed in the interface.
+    /// The four container tools, in the order listed in the interface.
     pub const ALL: [ToolName; 4] = [
         ToolName::ReadFile,
         ToolName::WriteFile,
@@ -37,6 +41,8 @@ impl ToolName {
             "write_file" => Some(ToolName::WriteFile),
             "shell" => Some(ToolName::Shell),
             "http_fetch" => Some(ToolName::HttpFetch),
+            "recall" => Some(ToolName::Recall),
+            "memory" => Some(ToolName::Memory),
             _ => None,
         }
     }
@@ -48,8 +54,15 @@ impl ToolName {
             ToolName::WriteFile => "write_file",
             ToolName::Shell => "shell",
             ToolName::HttpFetch => "http_fetch",
+            ToolName::Recall => "recall",
+            ToolName::Memory => "memory",
         }
     }
+
+    /// Whether `memoryd` runs this tool rather than a container.
+    pub fn is_memory(self) -> bool {
+        matches!(self, ToolName::Recall | ToolName::Memory)
+    }
 }
 
 /// A parsed set of tool arguments, one variant per tool.
@@ -67,6 +80,8 @@ pub enum ToolArgs {
         cwd: Option<String>,
     },
     HttpFetch(FetchUrl),
+    Recall(RecallArgs),
+    Memory(MemoryArgs),
 }
 
 /// A URL that passed the checks, and its host. Only `parse` makes one, so the host policy matched
@@ -109,6 +124,8 @@ impl ToolArgs {
             ToolArgs::WriteFile { .. } => ToolName::WriteFile,
             ToolArgs::Shell { .. } => ToolName::Shell,
             ToolArgs::HttpFetch(_) => ToolName::HttpFetch,
+            ToolArgs::Recall(_) => ToolName::Recall,
+            ToolArgs::Memory(_) => ToolName::Memory,
         }
     }
 
@@ -142,6 +159,12 @@ impl ToolArgs {
                 };
                 serde_json::to_string(&value).unwrap_or_else(|_| "{}".to_string())
             }
+            ToolArgs::Recall(value) => {
+                serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string())
+            }
+            ToolArgs::Memory(value) => {
+                serde_json::to_string(value).unwrap_or_else(|_| "{}".to_string())
+            }
         }
     }
 }
@@ -228,6 +251,18 @@ pub fn parse(tool: ToolName, arguments: &str) -> Result<ToolArgs, ArgsError> {
                 host,
             }))
         }
+        ToolName::Recall => {
+            let value: RecallArgs = serde_json::from_str(arguments)
+                .map_err(|error| ArgsError::Shape(error.to_string()))?;
+            value.check().map_err(ArgsError::Shape)?;
+            Ok(ToolArgs::Recall(value))
+        }
+        ToolName::Memory => {
+            let value: MemoryArgs = serde_json::from_str(arguments)
+                .map_err(|error| ArgsError::Shape(error.to_string()))?;
+            value.check().map_err(ArgsError::Shape)?;
+            Ok(ToolArgs::Memory(value))
+        }
     }
 }
 

crates/brokerd/src/builtin.rs

@@ -0,0 +1,45 @@
+//! The built-in grants of `recall` and `memory` (M5a spec, section 8). Reading and writing memory
+//! is what an agent is, so no grant file can turn it off or put questions in front of it; the
+//! prompt is protected by `memoryd`, which lets only clean sessions edit identity.
+
+use proto::{Grant, GrantProblem, sha256};
+
+use crate::grants::LoadedGrant;
+
+/// Ids starting with this are the built-ins'; an owner's grant file may not use one.
+pub const RESERVED: &str = "builtin-";
+
+/// The built-in grants' text, hashed for the audit log like a grant file's bytes.
+pub const RECALL: &str = "tool = \"recall\"\nmode = \"auto\"\nmax_taint = \"secret\"\nresult_class = \"private\"\nuntrusted = false\n";
+pub const MEMORY: &str = "tool = \"memory\"\nmode = \"auto\"\nmax_taint = \"secret\"\nresult_class = \"private\"\nuntrusted = false\n";
+
+/// The sentence for an owner's grant naming a memory tool.
+pub const BUILT_IN: &str = "the policy of recall and memory is built in; remove this grant";
+
+/// The two built-in grants, `builtin-memory` and `builtin-recall`.
+pub fn grants() -> Result<Vec<LoadedGrant>, GrantProblem> {
+    // For each (id, text) of [("builtin-memory", MEMORY), ("builtin-recall", RECALL)]:
+    // `toml::from_str::<Grant>(text)` and `sha256(text.as_bytes())`, each error as a GrantProblem {
+    // file: id, line: None, problem: e.to_string() }; build LoadedGrant { id, grant, sha256 }.
+    // Collect into Result<Vec<LoadedGrant>, GrantProblem> (the first error wins).
+    let sources: &[(&str, &str)] = &[("builtin-memory", MEMORY), ("builtin-recall", RECALL)];
+    let mut out: Vec<LoadedGrant> = Vec::new();
+    for (id, text) in sources {
+        let grant = toml::from_str::<Grant>(text).map_err(|error| GrantProblem {
+            file: id.to_string(),
+            line: None,
+            problem: error.to_string(),
+        })?;
+        let sha256 = sha256(text.as_bytes()).map_err(|error| GrantProblem {
+            file: id.to_string(),
+            line: None,
+            problem: error.to_string(),
+        })?;
+        out.push(LoadedGrant {
+            id: id.to_string(),
+            grant,
+            sha256,
+        });
+    }
+    Ok(out)
+}

crates/brokerd/src/config.rs

@@ -113,6 +113,8 @@ impl Runner {
             crate::args::ToolName::WriteFile => self.write_file_ms,
             crate::args::ToolName::Shell => self.shell_ms,
             crate::args::ToolName::HttpFetch => self.http_fetch_ms,
+            // `memoryd` runs these; they never reach a container.
+            crate::args::ToolName::Recall | crate::args::ToolName::Memory => self.read_file_ms,
         };
         std::time::Duration::from_millis(ms)
     }

crates/brokerd/src/grants.rs

@@ -26,6 +26,9 @@ pub struct LoadedGrant {
 #[derive(Debug, Clone, PartialEq, Eq, Default)]
 pub struct GrantSet {
     grants: Vec<LoadedGrant>,
+    /// The built-in grants of the memory tools, which every set from `from_grants` has. They are
+    /// not the owner's, so `grants` does not list them; matching sees them (`matchable`).
+    builtin: Vec<LoadedGrant>,
 }
 
 impl GrantSet {
@@ -49,17 +52,24 @@ impl GrantSet {
             check_grant(grant, &mut problems);
         }
 
-        if problems.is_empty() {
-            Ok(GrantSet { grants })
-        } else {
-            Err(problems)
+        if !problems.is_empty() {
+            return Err(problems);
         }
+        // The built-ins join only a set that passed: an owner's grant for a memory tool is a
+        // problem above, never a replacement.
+        let builtin = crate::builtin::grants().map_err(|problem| vec![problem])?;
+        Ok(GrantSet { grants, builtin })
     }
 
-    /// The grants in id order.
+    /// The owner's grants in id order.
     pub fn grants(&self) -> &[LoadedGrant] {
         &self.grants
     }
+
+    /// Every grant a call is matched against: the owner's, then the built-ins.
+    pub fn matchable(&self) -> impl Iterator<Item = &LoadedGrant> {
+        self.grants.iter().chain(self.builtin.iter())
+    }
 }
 
 /// A file name is a valid grant id when it is 1 to 64 characters of `a-z`, `0-9` and `-`.
@@ -220,7 +230,18 @@ fn check_grant(grant: &LoadedGrant, problems: &mut Vec<GrantProblem>) {
         );
     }
 
+    // Rule 2b: ids starting `builtin-` are the built-in grants'.
+    if id.starts_with(crate::builtin::RESERVED) {
+        push(
+            problems,
+            file.clone(),
+            None,
+            "grant ids starting builtin- are reserved",
+        );
+    }
+
     // Rule 3: the tool must be one of the four; an unknown tool skips rule 6 but not the others.
+    // The memory tools' grants are built in (M5a spec, section 8).
     match ToolName::parse(&grant.grant.tool) {
         None => push(
             problems,
@@ -228,6 +249,9 @@ fn check_grant(grant: &LoadedGrant, problems: &mut Vec<GrantProblem>) {
             None,
             "unknown tool; only read_file, write_file, shell and http_fetch",
         ),
+        Some(tool) if tool.is_memory() => {
+            push(problems, file.clone(), None, crate::builtin::BUILT_IN)
+        }
         Some(tool) => check_tool_constraints(tool, &file, inner, problems),
     }
 

crates/brokerd/src/lib.rs

@@ -5,6 +5,7 @@ pub mod approvals;
 pub mod args;
 pub mod audit;
 pub mod broker;
+pub mod builtin;
 pub mod config;
 pub mod container;
 pub mod courier;

crates/brokerd/src/policy/matching.rs

@@ -37,7 +37,7 @@ pub(super) fn match_grants(
     let mut expired = false;
     let mut tainted = false;
 
-    for loaded in grants.grants() {
+    for loaded in grants.matchable() {
         // M1: only grants for this tool play a part.
         if loaded.grant.tool != args.tool().as_str() {
             continue;
@@ -93,6 +93,8 @@ fn covers(args: &ToolArgs, grant: &Grant) -> Option<Option<String>> {
             }
         }
         ToolArgs::Shell { cwd: Some(cwd), .. } => best_path(grant, cwd, true),
+        // The memory tools take no constraints: their grants are built in (M5a spec, section 8).
+        ToolArgs::Recall(_) | ToolArgs::Memory(_) => Some(None),
         ToolArgs::HttpFetch(target) => {
             if grant
                 .constraints

crates/brokerd/src/runner.rs

@@ -117,6 +117,7 @@ pub fn run(decision: Decision, runtime: &dyn Runtime) -> proto::ToolResponse {
             None,
         ),
         ToolArgs::HttpFetch(_) => (Vec::new(), Some(decision.hosts().to_vec())),
+        ToolArgs::Recall(_) | ToolArgs::Memory(_) => (Vec::new(), None),
     };
     let spec = RunSpec {
         session: decision.request().session.clone(),

crates/brokerd/tests/grants.rs

@@ -92,7 +92,12 @@ fn files_that_do_not_end_in_toml_are_ignored() {
 fn an_empty_directory_is_a_valid_empty_set() {
     let set = load(&fixture("empty")).unwrap();
     assert!(set.grants().is_empty());
-    assert_eq!(set, GrantSet::default());
+    let builtin: Vec<&str> = set.matchable().map(|g| g.id.as_str()).collect();
+    assert_eq!(
+        builtin,
+        ["builtin-memory", "builtin-recall"],
+        "only the built-ins"
+    );
 }
 
 #[test]

crates/brokerd/tests/memory_tools.rs

@@ -0,0 +1,142 @@
+//! `recall` and `memory` in `brokerd`: their arguments, the built-in grants that allow them in
+//! every session, and grant files that name them refused (M5a spec, section 8). Do not edit.
+
+#[path = "support/build.rs"]
+mod build;
+#[path = "support/tmp.rs"]
+mod tmp;
+
+use brokerd::args::{ToolArgs, ToolName, parse};
+use brokerd::builtin::{BUILT_IN, MEMORY, RECALL};
+use brokerd::grants::{GrantSet, load};
+use brokerd::policy::{Label, Outcome, decide};
+use build::{allowed, grant, now, private, request, secret, set};
+use proto::{DataClass, DenyReason, Mode, sha256};
+use tmp::TempDir;
+
+#[test]
+fn the_names_and_arguments_of_the_memory_tools() {
+    assert_eq!(ToolName::parse("recall"), Some(ToolName::Recall));
+    assert_eq!(ToolName::parse("memory"), Some(ToolName::Memory));
+    assert_eq!(
+        (ToolName::Recall.as_str(), ToolName::Memory.as_str()),
+        ("recall", "memory")
+    );
+    assert!(ToolName::Recall.is_memory() && ToolName::Memory.is_memory());
+    assert!(
+        ToolName::ALL.iter().all(|t| !t.is_memory()),
+        "ALL is the container tools"
+    );
+    let recall = parse(ToolName::Recall, r#"{ "k": 3, "query": "launch" }"#).unwrap();
+    assert_eq!(recall.tool(), ToolName::Recall);
+    assert_eq!(recall.canonical_json(), r#"{"query":"launch","k":3}"#);
+    let memory = parse(ToolName::Memory, r#"{"text":"x","action":"note"}"#).unwrap();
+    assert!(matches!(memory, ToolArgs::Memory(_)));
+    assert_eq!(memory.canonical_json(), r#"{"action":"note","text":"x"}"#);
+    for (tool, bad) in [
+        (ToolName::Recall, r#"{"query":""}"#),
+        (ToolName::Recall, r#"{"query":"x","k":11}"#),
+        (
+            ToolName::Memory,
+            r#"{"action":"add","file":"soul","text":"x"}"#,
+        ),
+        (
+            ToolName::Memory,
+            r#"{"action":"add","file":"memory","text":"a\n§\nb"}"#,
+        ),
+        (
+            ToolName::Memory,
+            r#"{"action":"consolidate","file":"memory","entries":[]}"#,
+        ),
+    ] {
+        assert!(parse(tool, bad).is_err(), "{bad}");
+    }
+}
+
+#[test]
+fn both_are_allowed_in_every_session_by_the_built_in_grants() {
+    let grants = set(vec![]);
+    for state in [private(), secret()] {
+        let d = allowed(decide(
+            request("recall", r#"{"query":"x"}"#),
+            &grants,
+            state,
+            now(),
+        ));
+        assert_eq!(d.grant(), "builtin-recall");
+        assert_eq!(d.grant_sha256(), sha256(RECALL.as_bytes()).unwrap());
+        assert_eq!(
+            d.label(),
+            Label {
+                class: DataClass::Private,
+                untrusted: false
+            }
+        );
+        let d = allowed(decide(
+            request("memory", r#"{"action":"note","text":"x"}"#),
+            &grants,
+            state,
+            now(),
+        ));
+        assert_eq!(d.grant(), "builtin-memory");
+        assert_eq!(d.grant_sha256(), sha256(MEMORY.as_bytes()).unwrap());
+    }
+}
+
+#[test]
+fn a_set_without_built_ins_has_no_memory_tools() {
+    let outcome = decide(
+        request("recall", r#"{"query":"x"}"#),
+        &GrantSet::default(),
+        private(),
+        now(),
+    );
+    assert!(
+        matches!(outcome, Outcome::Denied(ref d) if d.reason == DenyReason::NoGrant),
+        "{outcome:?}"
+    );
+}
+
+#[test]
+fn grant_files_cannot_name_the_memory_tools_or_take_a_built_in_id() {
+    let dir = TempDir::new("memory-grants");
+    std::fs::write(
+        dir.path().join("mine.toml"),
+        "tool = \"memory\"\nmode = \"ask\"\nmax_taint = \"secret\"\n",
+    )
+    .unwrap();
+    std::fs::write(
+        dir.path().join("off.toml"),
+        "tool = \"recall\"\nmode = \"deny\"\nmax_taint = \"secret\"\n",
+    )
+    .unwrap();
+    std::fs::write(
+        dir.path().join("builtin-x.toml"),
+        "tool = \"shell\"\nmode = \"ask\"\nmax_taint = \"private\"\n",
+    )
+    .unwrap();
+    let problems = load(dir.path()).unwrap_err();
+    let got: Vec<(String, String)> = problems.into_iter().map(|p| (p.file, p.problem)).collect();
+    assert!(
+        got.contains(&("mine.toml".to_string(), BUILT_IN.to_string())),
+        "{got:?}"
+    );
+    assert!(
+        got.contains(&("off.toml".to_string(), BUILT_IN.to_string())),
+        "{got:?}"
+    );
+    assert!(
+        got.contains(&(
+            "builtin-x.toml".to_string(),
+            "grant ids starting builtin- are reserved".to_string()
+        )),
+        "{got:?}"
+    );
+    let with_shell = set(vec![grant("s", "shell", Mode::Ask)]);
+    assert_eq!(
+        with_shell.grants().len(),
+        1,
+        "the built-ins are not listed as the owner's"
+    );
+    assert_eq!(with_shell.matchable().count(), 3);
+}