toolkit: the tool program, read_file and write_file

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/toolkit/Cargo.toml

@@ -10,3 +10,5 @@ workspace = true
 
 [dependencies]
 proto.workspace = true
+serde.workspace = true
+serde_json.workspace = true

crates/toolkit/src/files.rs

@@ -0,0 +1,68 @@
+//! The first two tools, `read_file` and `write_file`. `shell` and `http_fetch` come later; until
+//! then `run` never reaches them, so they stay unknown tools.
+
+use std::fs;
+use std::io::Read;
+use std::path::Path;
+
+use proto::tools::{ReadFileArgs, WriteFileArgs};
+
+use crate::Outcome;
+
+pub const MAX_READ: usize = 1024 * 1024;
+
+fn tool_err(tool: &str, path: &str, why: &str) -> Outcome {
+    Outcome::tool_error(format!("{tool}: {path}: {why}"))
+}
+
+pub fn read_file(args: &ReadFileArgs) -> Outcome {
+    let path = args.path.as_str();
+
+    let is_dir = match fs::metadata(path) {
+        Err(e) if e.kind() == std::io::ErrorKind::NotFound => {
+            return tool_err("read_file", path, "no such file");
+        }
+        Err(e) => return tool_err("read_file", path, &e.to_string()),
+        Ok(meta) => meta.is_dir(),
+    };
+    if is_dir {
+        return tool_err("read_file", path, "is a directory");
+    }
+
+    let file = match fs::File::open(path) {
+        Err(e) => return tool_err("read_file", path, &e.to_string()),
+        Ok(f) => f,
+    };
+
+    let mut buf = Vec::new();
+    let n = match file.take(MAX_READ as u64 + 1).read_to_end(&mut buf) {
+        Err(e) => return tool_err("read_file", path, &e.to_string()),
+        Ok(n) => n,
+    };
+    if n > MAX_READ {
+        return tool_err("read_file", path, &format!("larger than {MAX_READ} bytes"));
+    }
+
+    match String::from_utf8(buf) {
+        Ok(text) => Outcome::done(text),
+        Err(_) => tool_err("read_file", path, "not UTF-8 text"),
+    }
+}
+
+pub fn write_file(args: &WriteFileArgs) -> Outcome {
+    let path = args.path.as_str();
+    let content = args.content.as_str();
+
+    match Path::new(path).parent() {
+        Some(p) if p.is_dir() => {}
+        _ => return tool_err("write_file", path, "the directory does not exist"),
+    }
+    if Path::new(path).is_dir() {
+        return tool_err("write_file", path, "is a directory");
+    }
+    if let Err(e) = fs::write(path, content) {
+        return tool_err("write_file", path, &e.to_string());
+    }
+
+    Outcome::done(format!("wrote {} bytes to {path}", content.len()))
+}

crates/toolkit/src/input.rs

@@ -0,0 +1,48 @@
+//! Reading and parsing the arguments `brokerd` puts on `toolkit`'s standard input.
+
+use std::io::Read;
+
+pub const MAX_INPUT: usize = 2 * 1024 * 1024;
+
+#[derive(Debug)]
+pub enum InputError {
+    TooLarge,
+    NotUtf8,
+    Io(std::io::Error),
+}
+
+impl std::fmt::Display for InputError {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            InputError::TooLarge => write!(f, "the arguments are larger than {MAX_INPUT} bytes"),
+            InputError::NotUtf8 => write!(f, "the arguments are not UTF-8"),
+            InputError::Io(e) => write!(f, "cannot read the arguments: {e}"),
+        }
+    }
+}
+
+impl std::error::Error for InputError {}
+
+impl From<std::io::Error> for InputError {
+    fn from(e: std::io::Error) -> Self {
+        InputError::Io(e)
+    }
+}
+
+/// All of `stdin`, at most MAX_INPUT bytes, as UTF-8. Read at most MAX_INPUT + 1 bytes
+/// (`Read::take`); more than MAX_INPUT is TooLarge.
+pub fn read_input(stdin: &mut dyn std::io::Read) -> Result<String, InputError> {
+    let mut buf = Vec::new();
+    let n = stdin.take(MAX_INPUT as u64 + 1).read_to_end(&mut buf)?;
+    if n > MAX_INPUT {
+        return Err(InputError::TooLarge);
+    }
+    String::from_utf8(buf).map_err(|_| InputError::NotUtf8)
+}
+
+/// Parse `text` as the arguments of tool `name`; the error is the whole line for stderr:
+/// "toolkit: {name}: the arguments do not parse: {serde's error}".
+pub fn parse<T: serde::de::DeserializeOwned>(name: &str, text: &str) -> Result<T, String> {
+    serde_json::from_str(text)
+        .map_err(|e| format!("toolkit: {name}: the arguments do not parse: {e}"))
+}

crates/toolkit/src/lib.rs

@@ -1 +1,82 @@
-//! Entry points that run inside tool containers.
+//! The programs that run inside tool containers.
+
+use proto::tools::{ReadFileArgs, WriteFileArgs};
+
+pub mod files;
+pub mod input;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Exit {
+    Done,
+    ToolError,
+    Misuse,
+}
+
+impl Exit {
+    /// 0, 1, 2. No `as` cast: a `match`.
+    pub fn code(self) -> u8 {
+        match self {
+            Exit::Done => 0,
+            Exit::ToolError => 1,
+            Exit::Misuse => 2,
+        }
+    }
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Outcome {
+    pub exit: Exit,
+    pub stdout: String,
+    pub stderr: String,
+}
+
+impl Outcome {
+    pub fn done(stdout: String) -> Outcome {
+        Outcome {
+            exit: Exit::Done,
+            stdout,
+            stderr: String::new(),
+        }
+    }
+
+    pub fn tool_error(stdout: String) -> Outcome {
+        Outcome {
+            exit: Exit::ToolError,
+            stdout,
+            stderr: String::new(),
+        }
+    }
+
+    pub fn misuse(stderr: String) -> Outcome {
+        Outcome {
+            exit: Exit::Misuse,
+            stdout: String::new(),
+            stderr,
+        }
+    }
+}
+
+/// Run tool `name` with the arguments on `stdin`.
+pub fn run(name: &str, stdin: &mut dyn std::io::Read) -> Outcome {
+    let text = match input::read_input(stdin) {
+        Ok(t) => t,
+        Err(e) => return Outcome::misuse(format!("toolkit: {e}")),
+    };
+    match name {
+        "read_file" => {
+            let args = match input::parse::<ReadFileArgs>(name, &text) {
+                Ok(a) => a,
+                Err(e) => return Outcome::misuse(e),
+            };
+            files::read_file(&args)
+        }
+        "write_file" => {
+            let args = match input::parse::<WriteFileArgs>(name, &text) {
+                Ok(a) => a,
+                Err(e) => return Outcome::misuse(e),
+            };
+            files::write_file(&args)
+        }
+        _ => Outcome::misuse(format!("toolkit: unknown tool {name:?}")),
+    }
+}

crates/toolkit/src/main.rs

@@ -1,4 +1,15 @@
-fn main() {
-    eprintln!("toolkit: not implemented until M3");
-    std::process::exit(2);
+use std::io::Write;
+use std::process::ExitCode;
+
+fn main() -> ExitCode {
+    let mut rest = std::env::args_os().skip(1);
+    let name = match (rest.next(), rest.next()) {
+        (Some(arg), None) => arg.to_str().unwrap_or("").to_string(),
+        _ => String::new(),
+    };
+
+    let outcome = toolkit::run(&name, &mut std::io::stdin().lock());
+    let _ = std::io::stdout().write_all(outcome.stdout.as_bytes());
+    let _ = std::io::stderr().write_all(outcome.stderr.as_bytes());
+    ExitCode::from(outcome.exit.code())
 }

crates/toolkit/tests/files.rs

@@ -0,0 +1,159 @@
+//! `toolkit read_file` and `toolkit write_file`, run as `brokerd` runs them. Do not edit.
+
+mod support;
+
+use std::os::unix::fs::PermissionsExt;
+
+use proto::tools::{ReadFileArgs, WriteFileArgs};
+use support::{TempDir, json, toolkit};
+use toolkit::files::MAX_READ;
+
+fn read(path: &str) -> support::Ran {
+    toolkit(
+        &["read_file"],
+        &json(&ReadFileArgs {
+            path: path.to_string(),
+        }),
+    )
+}
+
+fn write(path: &str, content: &str) -> support::Ran {
+    let args = WriteFileArgs {
+        path: path.to_string(),
+        content: content.to_string(),
+    };
+    toolkit(&["write_file"], &json(&args))
+}
+
+#[test]
+fn a_file_is_read_exactly() {
+    let dir = TempDir::new("read");
+    let text = "line one\nline two, no newline at the end: ✓";
+    std::fs::write(dir.at("a.md"), text).unwrap();
+    let ran = read(&dir.at("a.md"));
+    assert_eq!((ran.code, ran.stdout.as_str()), (0, text));
+    assert_eq!(ran.stderr, "");
+}
+
+#[test]
+fn an_empty_file_is_empty_output() {
+    let dir = TempDir::new("read-empty");
+    std::fs::write(dir.at("e"), "").unwrap();
+    let ran = read(&dir.at("e"));
+    assert_eq!((ran.code, ran.stdout.as_str()), (0, ""));
+}
+
+#[test]
+fn what_cannot_be_read_is_exit_1_with_one_line_for_the_model() {
+    let dir = TempDir::new("read-bad");
+    std::fs::write(dir.at("bin"), [0xff, 0xfe, 0x00]).unwrap();
+    std::fs::write(dir.at("big"), vec![b'a'; MAX_READ + 1]).unwrap();
+    std::fs::write(dir.at("exact"), vec![b'a'; MAX_READ]).unwrap();
+    let cases = [
+        (dir.at("missing"), "no such file"),
+        (dir.at(""), "is a directory"),
+        (dir.at("bin"), "not UTF-8 text"),
+        (dir.at("big"), "larger than 1048576 bytes"),
+    ];
+    for (path, why) in cases {
+        let ran = read(&path);
+        assert_eq!(ran.code, 1, "{path}");
+        assert_eq!(ran.stdout, format!("read_file: {path}: {why}"));
+    }
+    let ran = read(&dir.at("exact"));
+    assert_eq!(
+        (ran.code, ran.stdout.len()),
+        (0, MAX_READ),
+        "exactly the limit is fine"
+    );
+}
+
+#[test]
+fn an_unreadable_file_names_the_error() {
+    if is_root() {
+        return; // root reads anything
+    }
+    let dir = TempDir::new("read-perm");
+    std::fs::write(dir.at("secret"), "x").unwrap();
+    std::fs::set_permissions(dir.at("secret"), std::fs::Permissions::from_mode(0o000)).unwrap();
+    let ran = read(&dir.at("secret"));
+    assert_eq!(ran.code, 1);
+    assert!(
+        ran.stdout
+            .starts_with(&format!("read_file: {}: ", dir.at("secret"))),
+        "{}",
+        ran.stdout
+    );
+    assert!(ran.stdout.contains("ermission denied"), "{}", ran.stdout);
+}
+
+#[test]
+fn a_file_is_written_created_or_replaced() {
+    let dir = TempDir::new("write");
+    let ran = write(&dir.at("new.md"), "hello ✓\n");
+    assert_eq!(ran.code, 0);
+    assert_eq!(
+        ran.stdout,
+        format!("wrote 10 bytes to {}", dir.at("new.md"))
+    );
+    assert_eq!(
+        std::fs::read_to_string(dir.at("new.md")).unwrap(),
+        "hello ✓\n"
+    );
+    let ran = write(&dir.at("new.md"), "");
+    assert_eq!(ran.stdout, format!("wrote 0 bytes to {}", dir.at("new.md")));
+    assert_eq!(std::fs::read_to_string(dir.at("new.md")).unwrap(), "");
+}
+
+#[test]
+fn what_cannot_be_written_is_exit_1() {
+    let dir = TempDir::new("write-bad");
+    std::fs::create_dir(dir.at("sub")).unwrap();
+    let cases = [
+        (dir.at("nope/a.md"), "the directory does not exist"),
+        (dir.at("sub"), "is a directory"),
+    ];
+    for (path, why) in cases {
+        let ran = write(&path, "x");
+        assert_eq!(ran.code, 1, "{path}");
+        assert_eq!(ran.stdout, format!("write_file: {path}: {why}"));
+    }
+    assert!(!dir.path().join("nope").exists(), "no directory is created");
+}
+
+#[test]
+fn misuse_is_exit_2_with_nothing_on_standard_output() {
+    let long = vec![b' '; toolkit::input::MAX_INPUT + 1];
+    let cases: [(&[&str], &[u8], &str); 7] = [
+        (
+            &["read_file"],
+            br#"{"path":"/a","mode":1}"#,
+            "the arguments do not parse",
+        ),
+        (&["read_file"], b"", "the arguments do not parse"),
+        (
+            &["write_file"],
+            br#"{"path":"/a"}"#,
+            "the arguments do not parse",
+        ),
+        (&["read_file"], &[0xff, 0xfe], "not UTF-8"),
+        (&["read_file"], &long, "larger than 2097152 bytes"),
+        (&["format_disk"], b"{}", "unknown tool"),
+        (&[], b"{}", "unknown tool"),
+    ];
+    for (args, input, why) in cases {
+        let ran = toolkit(args, input);
+        assert_eq!(ran.code, 2, "{args:?}");
+        assert_eq!(
+            ran.stdout, "",
+            "{args:?}: the model sees nothing of a misuse"
+        );
+        assert!(ran.stderr.contains(why), "{args:?}: {}", ran.stderr);
+    }
+}
+
+fn is_root() -> bool {
+    std::fs::read_to_string("/proc/self/status")
+        .map(|s| s.lines().any(|l| l.starts_with("Uid:\t0\t")))
+        .unwrap_or(false)
+}

crates/toolkit/tests/support/mod.rs

@@ -0,0 +1,68 @@
+//! Running the `toolkit` binary as `brokerd` does: arguments on standard input. Do not edit.
+
+#![allow(dead_code)] // each test file uses its own part
+
+use std::io::Write;
+use std::path::{Path, PathBuf};
+use std::process::{Command, Stdio};
+use std::sync::atomic::{AtomicU32, Ordering};
+
+static NEXT: AtomicU32 = AtomicU32::new(0);
+
+/// A temporary directory, removed when dropped.
+pub struct TempDir(pub PathBuf);
+
+impl TempDir {
+    pub fn new(tag: &str) -> TempDir {
+        let n = NEXT.fetch_add(1, Ordering::SeqCst);
+        let path = std::env::temp_dir().join(format!("tk-{tag}-{}-{n}", std::process::id()));
+        let _ = std::fs::remove_dir_all(&path);
+        std::fs::create_dir_all(&path).unwrap();
+        TempDir(path)
+    }
+    pub fn path(&self) -> &Path {
+        &self.0
+    }
+    /// The path of `name` inside, as a string (the tools take strings).
+    pub fn at(&self, name: &str) -> String {
+        self.0.join(name).to_str().unwrap().to_string()
+    }
+}
+
+impl Drop for TempDir {
+    fn drop(&mut self) {
+        let _ = std::fs::remove_dir_all(&self.0);
+    }
+}
+
+pub struct Ran {
+    pub code: i32,
+    pub stdout: String,
+    pub stderr: String,
+}
+
+/// Run `toolkit <args…>` with `input` on standard input.
+pub fn toolkit(args: &[&str], input: &[u8]) -> Ran {
+    let mut child = Command::new(env!("CARGO_BIN_EXE_toolkit"))
+        .args(args)
+        .stdin(Stdio::piped())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped())
+        .spawn()
+        .unwrap();
+    let mut stdin = child.stdin.take().unwrap();
+    // A broken pipe here only means toolkit stopped reading, which some tests expect.
+    let _ = stdin.write_all(input);
+    drop(stdin);
+    let out = child.wait_with_output().unwrap();
+    Ran {
+        code: out.status.code().unwrap_or(-1),
+        stdout: String::from_utf8(out.stdout).unwrap(),
+        stderr: String::from_utf8_lossy(&out.stderr).into_owned(),
+    }
+}
+
+/// The JSON for one argument struct.
+pub fn json<T: serde::Serialize>(value: &T) -> Vec<u8> {
+    serde_json::to_vec(value).unwrap()
+}