toolkit: http_fetch through curl

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/toolkit/src/fetch.rs

@@ -0,0 +1,146 @@
+//! The fourth tool, `http_fetch`: run `/bin/curl` with a fixed argument list against `args.url`,
+//! through the egress proxy's socket `brokerd` mounts for this call, and report what curl printed
+//! and how it ended. toolkit writes no HTTP or TLS client of its own; curl does that inside the
+//! container. Spec sections 4 and 6.
+
+use std::io::{self, Read};
+use std::path::Path;
+use std::process::{Command, Stdio};
+
+use proto::tools::HttpFetchArgs;
+
+use crate::Outcome;
+
+pub const CURL: &str = "/bin/curl";
+pub const PROXY: &str = "socks5h://localhost/run/egress/egress.sock";
+pub const CA_BUNDLE: &str = "/etc/ssl/certs/ca-certificates.crt";
+
+/// The most to keep from curl's standard error. curl can still run past it; the rest is drained
+/// and discarded, never shown.
+const MAX_STDERR: usize = 64 * 1024;
+
+/// `curl`'s arguments for `url`, in order, without the program name.
+pub fn curl_args(url: &str) -> Vec<String> {
+    vec![
+        "--silent".to_string(),
+        "--show-error".to_string(),
+        "--proto".to_string(),
+        "=https".to_string(),
+        "--proto-redir".to_string(),
+        "=https".to_string(),
+        "--location".to_string(),
+        "--max-redirs".to_string(),
+        "5".to_string(),
+        "--max-time".to_string(),
+        "50".to_string(),
+        "--max-filesize".to_string(),
+        "8388608".to_string(),
+        "--cacert".to_string(),
+        CA_BUNDLE.to_string(),
+        "--proxy".to_string(),
+        PROXY.to_string(),
+        "--write-out".to_string(),
+        "\n[http %{response_code}]".to_string(),
+        "--url".to_string(),
+        url.to_string(),
+    ]
+}
+
+pub fn fetch(args: &HttpFetchArgs) -> Outcome {
+    fetch_with(Path::new(CURL), args)
+}
+
+pub fn fetch_with(curl: &Path, args: &HttpFetchArgs) -> Outcome {
+    let url = &args.url;
+    let args = curl_args(url);
+
+    let mut command = Command::new(curl);
+    command
+        .args(args)
+        .stdin(Stdio::null())
+        .stdout(Stdio::piped())
+        .stderr(Stdio::piped());
+    let mut child = match command.spawn() {
+        Ok(c) => c,
+        Err(e) => {
+            return Outcome::tool_error(format!(
+                "http_fetch: cannot start {}: {e}",
+                curl.display()
+            ));
+        }
+    };
+
+    // Read standard error on its own thread while the main thread reads standard output to the
+    // end: two full pipes block each other, and the given test fills both.
+    let stderr_reader = match child.stderr.take() {
+        Some(r) => r,
+        None => return Outcome::tool_error("http_fetch: curl has no standard error".to_string()),
+    };
+    let stderr_handle = std::thread::spawn(move || read_capped(stderr_reader));
+
+    let mut body = Vec::new();
+    let mut stdout = match child.stdout.take() {
+        Some(r) => r,
+        None => return Outcome::tool_error("http_fetch: curl has no standard output".to_string()),
+    };
+    if let Err(e) = stdout.read_to_end(&mut body) {
+        return Outcome::tool_error(format!("http_fetch: cannot read curl output: {e}"));
+    }
+
+    let status = match child.wait() {
+        Ok(s) => s,
+        Err(e) => return Outcome::tool_error(format!("http_fetch: cannot wait for curl: {e}")),
+    };
+    // A panicked reader lost its data; the body and status are still what curl gave us.
+    let stderr_bytes = stderr_handle.join().unwrap_or_default();
+
+    if status.success() {
+        return Outcome::done(String::from_utf8_lossy(&body).into_owned());
+    }
+
+    let why = match first_non_blank_line(&stderr_bytes) {
+        Some(line) => line,
+        None => match status.code() {
+            Some(code) => format!("curl exited {code}"),
+            None => "curl was killed".to_string(),
+        },
+    };
+    Outcome::tool_error(format!("http_fetch: {url}: {why}"))
+}
+
+/// Read `reader` to the end, keeping at most `MAX_STDERR` bytes; past the cap it keeps draining so
+/// curl never blocks on a full pipe, but the rest is discarded.
+fn read_capped(mut reader: impl Read) -> Vec<u8> {
+    let mut buf = [0u8; 8 * 1024];
+    let mut out: Vec<u8> = Vec::new();
+    loop {
+        let n = match reader.read(&mut buf) {
+            Ok(0) => break,
+            Ok(n) => n,
+            Err(e) if e.kind() == io::ErrorKind::Interrupted => continue,
+            Err(_) => break,
+        };
+        if out.len() >= MAX_STDERR {
+            continue;
+        }
+        let take = n.min(MAX_STDERR - out.len());
+        match buf.get(..take) {
+            Some(chunk) => out.extend_from_slice(chunk),
+            None => break,
+        }
+    }
+    out
+}
+
+/// The first line of `bytes` that is not blank after trimming, or `None` if every line is blank or
+/// the bytes are not UTF-8 text.
+fn first_non_blank_line(bytes: &[u8]) -> Option<String> {
+    let text = String::from_utf8_lossy(bytes);
+    for line in text.lines() {
+        let trimmed = line.trim();
+        if !trimmed.is_empty() {
+            return Some(trimmed.to_string());
+        }
+    }
+    None
+}