toolkit: the tool program, read_file and write_file

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

Cargo.lock

@@ -226,6 +226,8 @@ name = "toolkit"
 version = "0.1.0"
 dependencies = [
  "proto",
+ "serde",
+ "serde_json",
 ]
 
 [[package]]

crates/toolkit/Cargo.toml

@@ -10,3 +10,5 @@ workspace = true
 
 [dependencies]
 proto.workspace = true
+serde.workspace = true
+serde_json.workspace = true

crates/toolkit/src/files.rs

@@ -0,0 +1,68 @@
+//! The first two tools, `read_file` and `write_file`. `shell` and `http_fetch` come later; until
+//! then `run` never reaches them, so they stay unknown tools.
+
+use std::fs;
+use std::io::Read;
+use std::path::Path;
+
+use proto::tools::{ReadFileArgs, WriteFileArgs};
+
+use crate::Outcome;
+
+pub const MAX_READ: usize = 1024 * 1024;
+
+fn tool_err(tool: &str, path: &str, why: &str) -> Outcome {
+    Outcome::tool_error(format!("{tool}: {path}: {why}"))
+}
+
+pub fn read_file(args: &ReadFileArgs) -> Outcome {
+    let path = args.path.as_str();
+
+    let is_dir = match fs::metadata(path) {
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+            return tool_err("read_file", path, "no such file");
+        }
+        Err(e) => return tool_err("read_file", path, &e.to_string()),
+        Ok(meta) => meta.is_dir(),
+    };
+    if is_dir {
+        return tool_err("read_file", path, "is a directory");
+    }
+
+    let file = match fs::File::open(path) {
+        Err(e) => return tool_err("read_file", path, &e.to_string()),
+        Ok(f) => f,
+    };
+
+    let mut buf = Vec::new();
+    let n = match file.take(MAX_READ as u64 + 1).read_to_end(&mut buf) {
+        Err(e) => return tool_err("read_file", path, &e.to_string()),
+        Ok(n) => n,
+    };
+    if n > MAX_READ {
+        return tool_err("read_file", path, &format!("larger than {MAX_READ} bytes"));
+    }
+
+    match String::from_utf8(buf) {
+        Ok(text) => Outcome::done(text),
+        Err(_) => tool_err("read_file", path, "not UTF-8 text"),
+    }
+}
+
+pub fn write_file(args: &WriteFileArgs) -> Outcome {
+    let path = args.path.as_str();
+    let content = args.content.as_str();
+
+    match Path::new(path).parent() {
+        Some(p) if p.is_dir() => {}
+        _ => return tool_err("write_file", path, "the directory does not exist"),
+    }
+    if Path::new(path).is_dir() {
+        return tool_err("write_file", path, "is a directory");
+    }
+    if let Err(e) = fs::write(path, content) {
+        return tool_err("write_file", path, &e.to_string());
+    }
+
+    Outcome::done(format!("wrote {} bytes to {path}", content.len()))
+}

crates/toolkit/src/input.rs

@@ -0,0 +1,48 @@
+//! Reading and parsing the arguments `brokerd` puts on `toolkit`'s standard input.
+
+use std::io::Read;
+
+pub const MAX_INPUT: usize = 2 * 1024 * 1024;
+
+#[derive(Debug)]
+pub enum InputError {
+    TooLarge,
+    NotUtf8,
+    Io(std::io::Error),
+}
+
+impl std::fmt::Display for InputError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            InputError::TooLarge => write!(f, "the arguments are larger than {MAX_INPUT} bytes"),
+            InputError::NotUtf8 => write!(f, "the arguments are not UTF-8"),
+            InputError::Io(e) => write!(f, "cannot read the arguments: {e}"),
+        }
+    }
+}
+
+impl std::error::Error for InputError {}
+
+impl From<std::io::Error> for InputError {
+    fn from(e: std::io::Error) -> Self {
+        InputError::Io(e)
+    }
+}
+
+/// All of `stdin`, at most MAX_INPUT bytes, as UTF-8. Read at most MAX_INPUT + 1 bytes
+/// (`Read::take`); more than MAX_INPUT is TooLarge.
+pub fn read_input(stdin: &mut dyn std::io::Read) -> Result<String, InputError> {
+    let mut buf = Vec::new();
+    let n = stdin.take(MAX_INPUT as u64 + 1).read_to_end(&mut buf)?;
+    if n > MAX_INPUT {
+        return Err(InputError::TooLarge);
+    }
+    String::from_utf8(buf).map_err(|_| InputError::NotUtf8)
+}
+
+/// Parse `text` as the arguments of tool `name`; the error is the whole line for stderr:
+/// "toolkit: {name}: the arguments do not parse: {serde's error}".
+pub fn parse<T: serde::de::DeserializeOwned>(name: &str, text: &str) -> Result<T, String> {
+    serde_json::from_str(text)
+        .map_err(|e| format!("toolkit: {name}: the arguments do not parse: {e}"))
+}

crates/toolkit/src/lib.rs

@@ -1 +1,82 @@
-//! Entry points that run inside tool containers.
+//! The programs that run inside tool containers.
+
+use proto::tools::{ReadFileArgs, WriteFileArgs};
+
+pub mod files;
+pub mod input;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Exit {
+    Done,
+    ToolError,
+    Misuse,
+}
+
+impl Exit {
+    /// 0, 1, 2. No `as` cast: a `match`.
+    pub fn code(self) -> u8 {
+        match self {
+            Exit::Done => 0,
+            Exit::ToolError => 1,
+            Exit::Misuse => 2,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Outcome {
+    pub exit: Exit,
+    pub stdout: String,
+    pub stderr: String,
+}
+
+impl Outcome {
+    pub fn done(stdout: String) -> Outcome {
+        Outcome {
+            exit: Exit::Done,
+            stdout,
+            stderr: String::new(),
+        }
+    }
+
+    pub fn tool_error(stdout: String) -> Outcome {
+        Outcome {
+            exit: Exit::ToolError,
+            stdout,
+            stderr: String::new(),
+        }
+    }
+
+    pub fn misuse(stderr: String) -> Outcome {
+        Outcome {
+            exit: Exit::Misuse,
+            stdout: String::new(),
+            stderr,
+        }
+    }
+}
+
+/// Run tool `name` with the arguments on `stdin`.
+pub fn run(name: &str, stdin: &mut dyn std::io::Read) -> Outcome {
+    let text = match input::read_input(stdin) {
+        Ok(t) => t,
+        Err(e) => return Outcome::misuse(format!("toolkit: {e}")),
+    };
+    match name {
+        "read_file" => {
+            let args = match input::parse::<ReadFileArgs>(name, &text) {
+                Ok(a) => a,
+                Err(e) => return Outcome::misuse(e),
+            };
+            files::read_file(&args)
+        }
+        "write_file" => {
+            let args = match input::parse::<WriteFileArgs>(name, &text) {
+                Ok(a) => a,
+                Err(e) => return Outcome::misuse(e),
+            };
+            files::write_file(&args)
+        }
+        _ => Outcome::misuse(format!("toolkit: unknown tool {name:?}")),
+    }
+}

crates/toolkit/src/main.rs

@@ -1,4 +1,15 @@
-fn main() {
+use std::io::Write;
-    eprintln!("toolkit: not implemented until M3");
+use std::process::ExitCode;
-    std::process::exit(2);
+
+fn main() -> ExitCode {
+    let mut rest = std::env::args_os().skip(1);
+    let name = match (rest.next(), rest.next()) {
+        (Some(arg), None) => arg.to_str().unwrap_or("").to_string(),
+        _ => String::new(),
+    };
+
+    let outcome = toolkit::run(&name, &mut std::io::stdin().lock());
+    let _ = std::io::stdout().write_all(outcome.stdout.as_bytes());
+    let _ = std::io::stderr().write_all(outcome.stderr.as_bytes());
+    ExitCode::from(outcome.exit.code())
 }

crates/toolkit/tests/files.rs

@@ -0,0 +1,159 @@
+//! `toolkit read_file` and `toolkit write_file`, run as `brokerd` runs them. Do not edit.
+
+mod support;
+
+use std::os::unix::fs::PermissionsExt;
+
+use proto::tools::{ReadFileArgs, WriteFileArgs};
+use support::{TempDir, json, toolkit};
+use toolkit::files::MAX_READ;
+
+fn read(path: &str) -> support::Ran {
+    toolkit(
+        &["read_file"],
+        &json(&ReadFileArgs {
+            path: path.to_string(),
+        }),
+    )
+}
+
+fn write(path: &str, content: &str) -> support::Ran {
+    let args = WriteFileArgs {
+        path: path.to_string(),
+        content: content.to_string(),
+    };
+    toolkit(&["write_file"], &json(&args))
+}
+
+#[test]
+fn a_file_is_read_exactly() {
+    let dir = TempDir::new("read");
+    let text = "line one\nline two, no newline at the end: ✓";
+    std::fs::write(dir.at("a.md"), text).unwrap();
+    let ran = read(&dir.at("a.md"));
+    assert_eq!((ran.code, ran.stdout.as_str()), (0, text));
+    assert_eq!(ran.stderr, "");
+}
+
+#[test]
+fn an_empty_file_is_empty_output() {
+    let dir = TempDir::new("read-empty");
+    std::fs::write(dir.at("e"), "").unwrap();
+    let ran = read(&dir.at("e"));
+    assert_eq!((ran.code, ran.stdout.as_str()), (0, ""));
+}
+
+#[test]
+fn what_cannot_be_read_is_exit_1_with_one_line_for_the_model() {
+    let dir = TempDir::new("read-bad");
+    std::fs::write(dir.at("bin"), [0xff, 0xfe, 0x00]).unwrap();
+    std::fs::write(dir.at("big"), vec![b'a'; MAX_READ + 1]).unwrap();
+    std::fs::write(dir.at("exact"), vec![b'a'; MAX_READ]).unwrap();
+    let cases = [
+        (dir.at("missing"), "no such file"),
+        (dir.at(""), "is a directory"),
+        (dir.at("bin"), "not UTF-8 text"),
+        (dir.at("big"), "larger than 1048576 bytes"),
+    ];
+    for (path, why) in cases {
+        let ran = read(&path);
+        assert_eq!(ran.code, 1, "{path}");
+        assert_eq!(ran.stdout, format!("read_file: {path}: {why}"));
+    }
+    let ran = read(&dir.at("exact"));
+    assert_eq!(
+        (ran.code, ran.stdout.len()),
+        (0, MAX_READ),
+        "exactly the limit is fine"
+    );
+}
+
+#[test]
+fn an_unreadable_file_names_the_error() {
+    if is_root() {
+        return; // root reads anything
+    }
+    let dir = TempDir::new("read-perm");
+    std::fs::write(dir.at("secret"), "x").unwrap();
+    std::fs::set_permissions(dir.at("secret"), std::fs::Permissions::from_mode(0o000)).unwrap();
+    let ran = read(&dir.at("secret"));
+    assert_eq!(ran.code, 1);
+    assert!(
+        ran.stdout
+            .starts_with(&format!("read_file: {}: ", dir.at("secret"))),
+        "{}",
+        ran.stdout
+    );
+    assert!(ran.stdout.contains("ermission denied"), "{}", ran.stdout);
+}
+
+#[test]
+fn a_file_is_written_created_or_replaced() {
+    let dir = TempDir::new("write");
+    let ran = write(&dir.at("new.md"), "hello ✓\n");
+    assert_eq!(ran.code, 0);
+    assert_eq!(
+        ran.stdout,
+        format!("wrote 10 bytes to {}", dir.at("new.md"))
+    );
+    assert_eq!(
+        std::fs::read_to_string(dir.at("new.md")).unwrap(),
+        "hello ✓\n"
+    );
+    let ran = write(&dir.at("new.md"), "");
+    assert_eq!(ran.stdout, format!("wrote 0 bytes to {}", dir.at("new.md")));
+    assert_eq!(std::fs::read_to_string(dir.at("new.md")).unwrap(), "");
+}
+
+#[test]
+fn what_cannot_be_written_is_exit_1() {
+    let dir = TempDir::new("write-bad");
+    std::fs::create_dir(dir.at("sub")).unwrap();
+    let cases = [
+        (dir.at("nope/a.md"), "the directory does not exist"),
+        (dir.at("sub"), "is a directory"),
+    ];
+    for (path, why) in cases {
+        let ran = write(&path, "x");
+        assert_eq!(ran.code, 1, "{path}");
+        assert_eq!(ran.stdout, format!("write_file: {path}: {why}"));
+    }
+    assert!(!dir.path().join("nope").exists(), "no directory is created");
+}
+
+#[test]
+fn misuse_is_exit_2_with_nothing_on_standard_output() {
+    let long = vec![b' '; toolkit::input::MAX_INPUT + 1];
+    let cases: [(&[&str], &[u8], &str); 7] = [
+        (
+            &["read_file"],
+            br#"{"path":"/a","mode":1}"#,
+            "the arguments do not parse",
+        ),
+        (&["read_file"], b"", "the arguments do not parse"),
+        (
+            &["write_file"],
+            br#"{"path":"/a"}"#,
+            "the arguments do not parse",
+        ),
+        (&["read_file"], &[0xff, 0xfe], "not UTF-8"),
+        (&["read_file"], &long, "larger than 2097152 bytes"),
+        (&["format_disk"], b"{}", "unknown tool"),
+        (&[], b"{}", "unknown tool"),
+    ];
+    for (args, input, why) in cases {
+        let ran = toolkit(args, input);
+        assert_eq!(ran.code, 2, "{args:?}");
+        assert_eq!(
+            ran.stdout, "",
+            "{args:?}: the model sees nothing of a misuse"
+        );
+        assert!(ran.stderr.contains(why), "{args:?}: {}", ran.stderr);
+    }
+}
+
+fn is_root() -> bool {
+    std::fs::read_to_string("/proc/self/status")
+        .map(|s| s.lines().any(|l| l.starts_with("Uid:\t0\t")))
+        .unwrap_or(false)
+}

crates/toolkit/tests/support/mod.rs

@@ -0,0 +1,68 @@
+//! Running the `toolkit` binary as `brokerd` does: arguments on standard input. Do not edit.
+
+#![allow(dead_code)] // each test file uses its own part
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::atomic::{AtomicU32, Ordering};
+
+static NEXT: AtomicU32 = AtomicU32::new(0);
+
+/// A temporary directory, removed when dropped.
+pub struct TempDir(pub PathBuf);
+
+impl TempDir {
+    pub fn new(tag: &str) -> TempDir {
+        let n = NEXT.fetch_add(1, Ordering::SeqCst);
+        let path = std::env::temp_dir().join(format!("tk-{tag}-{}-{n}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&path);
+        std::fs::create_dir_all(&path).unwrap();
+        TempDir(path)
+    }
+    pub fn path(&self) -> &Path {
+        &self.0
+    }
+    /// The path of `name` inside, as a string (the tools take strings).
+    pub fn at(&self, name: &str) -> String {
+        self.0.join(name).to_str().unwrap().to_string()
+    }
+}
+
+impl Drop for TempDir {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.0);
+    }
+}
+
+pub struct Ran {
+    pub code: i32,
+    pub stdout: String,
+    pub stderr: String,
+}
+
+/// Run `toolkit <args…>` with `input` on standard input.
+pub fn toolkit(args: &[&str], input: &[u8]) -> Ran {
+    let mut child = Command::new(env!("CARGO_BIN_EXE_toolkit"))
+        .args(args)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .unwrap();
+    let mut stdin = child.stdin.take().unwrap();
+    // A broken pipe here only means toolkit stopped reading, which some tests expect.
+    let _ = stdin.write_all(input);
+    drop(stdin);
+    let out = child.wait_with_output().unwrap();
+    Ran {
+        code: out.status.code().unwrap_or(-1),
+        stdout: String::from_utf8(out.stdout).unwrap(),
+        stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
+    }
+}
+
+/// The JSON for one argument struct.
+pub fn json<T: serde::Serialize>(value: &T) -> Vec<u8> {
+    serde_json::to_vec(value).unwrap()
+}

docs/dependencies.md

@@ -4,8 +4,8 @@ Every external crate has a row here. `scripts/check-dep-docs.sh` enforces it.
 
 | Crate | Version | Used by | Why |
 |---|---|---|---|
-| `serde` | 1.0.229 | `proto`, `brokerd` | Derives serialization for every shared type. MIT OR Apache-2.0. |
+| `serde` | 1.0.229 | `proto`, `brokerd`, `toolkit` | Derives serialization for every shared type. MIT OR Apache-2.0. |
-| `serde_json` | 1.0.151 | `proto`, `brokerd` | JSON for frames and log files. MIT OR Apache-2.0. |
+| `serde_json` | 1.0.151 | `proto`, `brokerd`, `toolkit` | JSON for frames and log files. MIT OR Apache-2.0. |
 | `humantime` | 2.4.0 | `proto` | RFC 3339 timestamps. No dependencies of its own. MIT OR Apache-2.0. |
 | `toml` | 1.1.6 | `proto` tests; `brokerd` and `bxctl` from M3 | Parses grant files. MIT OR Apache-2.0. |
 | `emsha` | 1.0.4 | `proto` | SHA-256. Written by the owner; no dependencies, no `unsafe`. Checked against `sha256sum` on 3,204 inputs. |

docs/implementer-log.md

@@ -6,6 +6,7 @@ reviewer adds findings under "Reviews" once per milestone.
 
 | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model |
 |---|---|---|---|---|---|---|---|
+| M3b/04-toolkit-files | 2026-09-22 | done | 1 | pass | none | Wrote `crates/toolkit/src/input.rs` (`MAX_INPUT`, `InputError` with hand-written `Display` + `std::error::Error` + `From<io::Error>`, `read_input` reading at most `MAX_INPUT+1` bytes via `Read::take` and flagging `TooLarge`, `parse` returning the whole stderr line) and `crates/toolkit/src/files.rs` (`MAX_READ`, `read_file` and `write_file`). `read_file` walks the exits in order: metadata NotFound→"no such file", other metadata error→Display text, directory→"is a directory", open failure→text, `take(MAX_READ+1)` read failure→text, over-limit→"larger than {MAX_READ} bytes", non-UTF-8→"not UTF-8 text", else `done`. `write_file` checks the parent is an existing directory before writing so no directory is ever created, then the path-is-directory case, then `fs::write`. Replaced the `lib.rs`/`main.rs` stubs: `Exit` via a `match` (no `as`), `Outcome` with `done`/`tool_error`/`misuse` (misuse stdout empty), `run` reading input before the name lookup for every name. `Cargo.toml` gained `serde` and `serde_json` workspace deps; `dependencies.md` lists `toolkit` under both. Copied the two given test files. `an_unreadable_file_names_the_error` ran for real (UID 1000, not root). 7 `files.rs` tests pass; `make gate` prints `gate: ok` on the first run. | ? |
 | M3b/02-brokerd-fetch-url | 2026-09-22 | done | 1 | pass | none | Sealed the fetch target: replaced the two-field `ToolArgs::HttpFetch { url, host }` variant with a tuple variant `HttpFetch(FetchUrl)` holding a new `FetchUrl { url, host }` struct whose fields are private and exposed only through `url()`/`host()`; `parse` is the only constructor. Added the two doctests word for word (a `compile_fail` proving the struct cannot be built outside `args`, and a passing one reading the parsed value through getters). Changed the four match sites that built or read the old variant: `tool` (`HttpFetch(_)`), `canonical_json` (`HttpFetch(target)` serialising `HttpFetchArgs { url: target.url.clone() }`, still only the URL), `parse` (`HttpFetch(FetchUrl { url: value.url, host })`), and `policy::covers` (`HttpFetch(target)` matching with `host_matches(pattern, target.host())`); `runner.rs` `HttpFetch { .. }` became `HttpFetch(_)`. Copied the new `tests/args.rs` (one test now reads the parsed value through getters). `grep "FetchUrl {"` shows the struct def, impl opener, the `parse` line and the doctest only; `grep "HttpFetch {"` shows nothing. All brokerd tests and doctests pass; `make gate` prints `gate: ok` on the first run. | ? |
 | M3b/01-proto-tools-hosts | 2026-09-22 | done | 1 | pass | none | Copied `tools.rs` and `hosts.rs` from the plan's `files/` into `crates/proto/tests/`; they fail to compile until the modules exist. Created `crates/proto/src/tools.rs` with the four public arg structs (ReadFile/Write/Shell/HttpFetch), each `#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]` and `#[serde(deny_unknown_fields)]`, `ShellArgs` keeping its `#[serde(default, skip_serializing_if = "Option::is_none")]` cwd; and `crates/proto/src/hosts.rs` with private `valid_label` plus `valid_host`, `valid_host_pattern`, `host_matches`, bodies and doc comments unchanged. Added `pub mod hosts;` and `pub mod tools;` to `proto/src/lib.rs` in alphabetical order (no re-exports). Deleted the private structs from `brokerd/src/args.rs`, added `use proto::tools::{...}` at the top and the `pub use proto::hosts::{...}` re-export where the functions were, and dropped the now-unused `use serde::{Deserialize, Serialize}`; `url_host`, `valid_path`, `inside` and `MAX_URL` stay. New suites 3 and 3 pass; brokerd args/grants/policy pass unchanged. `grep "fn valid_host\|struct ShellArgs"` shows one extra line, `brokerd/tests/args.rs:91`, the pre-existing test `valid_hosts_and_patterns` (substring match, not a duplicate definition). Gate passed on the first run. | ? |
 | M3a/23-review-fixes | 2026-09-22 | done | 1 | pass | none | Done by the design model, not an implementer run: Ornith was under heavy contention. Findings 1 to 13 and 16 fixed. Tests that failed without their fix cover 1, 2, 3, 4, 7, 8, 9, 10 and 16; two more (a torn first record, an empty later file) passed before the fix and are regression guards; 6, 11 and 13 were checked by reading. 14 moves to M3b's first task; of 15, the log-name rule, `MAX_PATH`'s doc and the unused push are fixed, and the two that change the baseline wait for an epoch change. The server expectations in the device tests were updated to the new layout. | Claude Opus 5.5 |