kyle/boxmaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

brokerd: start and remove the egress proxy for http_fetch

Browse Source 
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
kyle
2026-09-23 09:22:24 -07:00
parent cfe13787b0
commit e6081a2177
3 changed files with 379 additions and 9 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/brokerd/src/container.rs
+165 -9
View File
@@ -4,6 +4,7 @@
use std::ffi::OsString;
use std::io::{Read, Write};
use std::os::unix::fs::{DirBuilderExt, PermissionsExt};
use std::path::{Path, PathBuf};
use std::process::{Child, Command, ExitStatus, Stdio};
use std::sync::Arc;
@@ -25,6 +26,10 @@ pub const UNEXPECTED: &str = "the tool failed with an unexpected status";
pub const POLL: Duration = Duration::from_millis(50);
/// How much of Podman's standard error is kept for the log.
pub const STDERR_KEPT: usize = 4096;
/// How long `run` waits for the egress proxy to make its socket.
pub const EGRESS_WAIT: Duration = Duration::from_secs(5);
/// How often the egress proxy's socket is checked while waiting.
pub const EGRESS_POLL: Duration = Duration::from_millis(20);
/// A log sink. Call it as `(self.log)("a line")`.
pub type Log = Arc<dyn Fn(&str) + Send + Sync>;
@@ -32,6 +37,7 @@ pub type Log = Arc<dyn Fn(&str) + Send + Sync>;
pub struct Podman {
runner: Runner,
egress_dir: PathBuf,
egress_wait: Duration,
log: Log,
next: AtomicU64,
}
@@ -42,11 +48,20 @@ impl Podman {
Podman {
runner,
egress_dir,
egress_wait: EGRESS_WAIT,
log,
next: AtomicU64::new(0),
}
}
/// The same runtime with another wait for the proxy's socket, for tests.
pub fn with_egress_wait(self, egress_wait: Duration) -> Podman {
Podman {
egress_wait,
..self
}
}
/// Where `http_fetch` calls get their directories. Public, so the field counts as read.
pub fn egress_dir(&self) -> &Path {
&self.egress_dir
@@ -69,6 +84,26 @@ impl Podman {
}
}
/// The log for a `podman` that cannot be launched at all (task 11 step 3).
fn cannot_launch(&self, podman: &Path, e: std::io::Error) {
(self.log)(&format!(
"brokerd: cannot start {}: {}\n{}",
podman.display(),
e,
RUNBOOK
));
}
/// The log for a directory that cannot be made (task 12 step 1).
fn cannot_make(&self, dir: &Path, e: std::io::Error) -> RunError {
(self.log)(&format!(
"brokerd: cannot make {}: {e}\n{}",
dir.display(),
RUNBOOK
));
RunError::Unavailable(CANNOT_START.to_string())
}
/// Wait for `child` until `limit` has passed since `started`. `Some(status)` if it ended;
/// `None` if it ran too long, after `podman kill <name>`, `podman rm -f <name>`,
/// `child.kill()` and `child.wait()` (step 5).
@@ -129,12 +164,7 @@ impl Podman {
{
Ok(child) => Some(child),
Err(e) => {
(self.log)(&format!(
"brokerd: cannot start {}: {}\n{}",
podman.display(),
e,
RUNBOOK
));
self.cannot_launch(podman, e);
None
}
}
@@ -183,14 +213,140 @@ impl Podman {
}
impl Runtime for Podman {
/// Steps 1 and 2, then `run_container`. Already written.
/// Step 1 (the container number), then step 2's choice: no egress runs the tool as before,
/// an egress starts the proxy first and gives the tool the directory. `run_container` is step 3
/// to 7 for either. Already written.
fn run(&self, spec: &RunSpec) -> Result<RunOutput, RunError> {
let n = self.next.fetch_add(1, Ordering::SeqCst);
let name = podman::container_name(spec.session(), spec.call(), n);
let args = podman::tool_args(spec, &self.runner, &name, None);
let input = spec.arguments().canonical_json();
let limit = self.runner.time_limit(spec.tool());
self.run_container(&name, args, input, limit)
match spec.egress() {
None => {
let args = podman::tool_args(spec, &self.runner, &name, None);
self.run_container(&name, args, input, limit)
}
Some(hosts) => {
let dir = self.egress_dir.join(&name);
let _guard = self.start_egress(&name, &self.egress_dir, &dir, hosts)?;
let args = podman::tool_args(spec, &self.runner, &name, Some(&dir));
self.run_container(&name, args, input, limit)
}
}
}
}
impl Podman {
/// Steps 1 to 4 of the egress proxy's start (task 12). The guard it returns cleans up the
/// proxy and directory on every path; `run` keeps it alive until the tool's container ends.
fn start_egress(
&self,
name: &str,
egress_dir: &Path,
dir: &Path,
hosts: &[String],
) -> Result<EgressGuard<'_>, RunError> {
// The guard is created before anything can fail, so every return below
// cleans up the proxy and directory by dropping it.
let _guard = EgressGuard::new(self, name, dir.to_path_buf());
// 1. Make the egress directory (and its parents) 0700, then replace the
// call's directory if a crash left one behind.
std::fs::DirBuilder::new()
.recursive(true)
.mode(0o700)
.create(egress_dir)
.map_err(|e| self.cannot_make(dir, e))?;
let _ = std::fs::set_permissions(egress_dir, std::fs::Permissions::from_mode(0o700));
if dir.exists() {
match std::fs::remove_dir_all(dir) {
Ok(()) => {}
Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => return Err(self.cannot_make(dir, e)),
}
}
std::fs::DirBuilder::new()
.mode(0o700)
.create(dir)
.map_err(|e| self.cannot_make(dir, e))?;
// 2. Start the proxy, detached. A run that exits non-zero is logged and
// unavailable; one that cannot be launched at all uses task 11's log.
let args = podman::egress_args(&self.runner, name, dir, hosts);
match Command::new(&self.runner.podman)
.args(args)
.stdin(Stdio::null())
.stdout(Stdio::null())
.stderr(Stdio::piped())
.output()
{
Ok(output) if output.status.success() => {}
Ok(output) => {
let stderr = String::from_utf8_lossy(&output.stderr).into_owned();
(self.log)(&format!(
"brokerd: podman could not start {name}-egress: {stderr}\n{RUNBOOK}"
));
return Err(RunError::Unavailable(CANNOT_START.to_string()));
}
Err(e) => {
self.cannot_launch(&self.runner.podman, e);
return Err(RunError::Unavailable(CANNOT_START.to_string()));
}
}
// 3. Wait for the proxy to make its socket, or give up.
let socket = dir.join("egress.sock");
let deadline = Instant::now() + self.egress_wait;
loop {
if socket.exists() {
break;
}
if Instant::now() >= deadline {
let ms = self.egress_wait.as_millis();
(self.log)(&format!(
"brokerd: {name}-egress did not make its socket within {ms} ms\n{RUNBOOK}"
));
return Err(RunError::Unavailable(CANNOT_START.to_string()));
}
std::thread::sleep(EGRESS_POLL);
}
// 4. The tool's container may now run; `run` holds the guard until it
// returns, then drops it to remove the proxy and directory.
Ok(_guard)
}
}
/// Removes the egress proxy's container and its directory when `run` returns, on every path.
///
/// `run` creates it before anything can fail and holds it until the tool's container has ended, so
/// the proxy and directory never survive the call that started it.
struct EgressGuard<'a> {
podman: &'a Podman,
name: String,
dir: PathBuf,
}
impl<'a> Drop for EgressGuard<'a> {
fn drop(&mut self) {
self.podman.podman(&["rm", "-f", self.name.as_str()]);
match std::fs::remove_dir_all(&self.dir) {
Ok(()) => {}
Err(ref e) if e.kind() == std::io::ErrorKind::NotFound => {}
Err(e) => (self.podman.log)(&format!(
"brokerd: cannot remove {}: {e}",
self.dir.display()
)),
}
}
}
impl<'a> EgressGuard<'a> {
fn new(podman: &'a Podman, name: &str, dir: PathBuf) -> EgressGuard<'a> {
EgressGuard {
podman,
name: format!("{name}-egress"),
dir,
}
}
}
crates/brokerd/tests/container_egress.rs
+213
View File
@@ -0,0 +1,213 @@
//! `http_fetch` through the Podman runtime: the proxy is started first, its socket awaited, the tool
//! run with the directory mounted, and the proxy and directory removed afterwards on every path
//! (M3b spec, section 6, "http_fetch"). Against a fake `podman`. Do not edit.
#[path = "support/build.rs"]
mod build;
#[path = "support/fake_podman.rs"]
mod fake_podman;
use std::os::unix::fs::PermissionsExt;
use std::time::{Duration, Instant};
use brokerd::container::{CANNOT_START, Podman, RUNBOOK, TIMED_OUT};
use brokerd::policy::{Outcome, SessionState, decide};
use brokerd::runner::run;
use build::{fetch, grant, now, set};
use fake_podman::{Fake, Lines, path, serial};
use proto::{Mode, ToolResponse};
/// For `run -d` (the proxy): make the socket file in the mounted directory, as the proxy does.
const PROXY_OK: &str = r#"for a in "$@"; do case "$a" in --volume=*:/run/egress:rw) v=${a#--volume=}; v=${v%:/run/egress:rw};; esac; done"#;
fn body(proxy: &str, tool: &str) -> String {
format!("{PROXY_OK}\nif [ \"$2\" = -d ]; then\n{proxy}\nfi\n{tool}")
}
fn call(podman: &Podman) -> ToolResponse {
let grants = set(vec![
grant("web", "http_fetch", Mode::Auto).hosts(&["example.com", "*.example.org"]),
]);
let decision = match decide(
fetch("https://example.com/a"),
&grants,
SessionState::default(),
now(),
) {
Outcome::Allowed(d) => d,
other => panic!("not allowed: {other:?}"),
};
run(decision, podman)
}
fn failed(message: &str) -> ToolResponse {
ToolResponse::Failed {
message: message.to_string(),
}
}
#[test]
fn the_proxy_starts_first_the_tool_gets_its_socket_and_both_are_cleaned_up() {
let _s = serial();
let fake = Fake::new(
"eg-ok",
&body(
r#": > "$v/egress.sock"; exit 0"#,
r#"cat > "$D/stdin"; printf 'body\n[http 200]'; exit 0"#,
),
);
let log = Lines::default();
let egress = fake.dir.join("egress");
let got = call(&Podman::new(fake.runner(""), egress.clone(), log.sink()));
assert!(
matches!(&got, ToolResponse::Result { content, .. } if content == "body\n[http 200]"),
"{got:?} {}",
log.all()
);
let calls = fake.calls();
assert_eq!(calls.len(), 3, "{calls:?}");
let dir = egress.join("boxmaker-s1-1-0");
// 1. The proxy, detached, with the call's hosts.
assert_eq!(&calls[0][..2], ["run", "-d"]);
assert!(calls[0].contains(&"--name=boxmaker-s1-1-0-egress".to_string()));
assert!(calls[0].contains(&format!("--volume={}:/run/egress:rw", path(&dir))));
assert_eq!(calls[0].last().unwrap(), "example.com,*.example.org");
// 2. The tool, with the same directory and no network.
assert_eq!(&calls[1][..3], ["run", "--rm", "-i"]);
assert!(calls[1].contains(&format!("--volume={}:/run/egress:rw", path(&dir))));
assert!(calls[1].contains(&"--network=none".to_string()));
assert_eq!(fake.stdin(), r#"{"url":"https://example.com/a"}"#);
// 3. The proxy removed, and its directory with it.
assert_eq!(calls[2], ["rm", "-f", "boxmaker-s1-1-0-egress"]);
assert!(!dir.exists(), "the call's directory is removed");
let mode = std::fs::metadata(&egress).unwrap().permissions().mode() & 0o777;
assert_eq!(mode, 0o700);
}
#[test]
fn a_proxy_that_podman_cannot_start_means_no_tool_and_nothing_left() {
let _s = serial();
let fake = Fake::new(
"eg-fail",
&body(
"echo 'Error: network pasta not found' >&2; exit 125",
"cat > /dev/null; exit 0",
),
);
let log = Lines::default();
let egress = fake.dir.join("egress");
let got = call(&Podman::new(fake.runner(""), egress.clone(), log.sink()));
assert_eq!(got, failed(CANNOT_START));
let calls = fake.calls();
assert!(
calls
.iter()
.all(|c| c.get(2).map(String::as_str) != Some("-i")),
"no tool ran: {calls:?}"
);
assert_eq!(
calls.last().unwrap(),
&["rm", "-f", "boxmaker-s1-1-0-egress"]
);
assert!(!egress.join("boxmaker-s1-1-0").exists());
assert!(
log.all().contains("network pasta not found"),
"{}",
log.all()
);
assert!(log.all().contains(RUNBOOK));
}
#[test]
fn a_proxy_that_makes_no_socket_in_time_means_no_tool_and_nothing_left() {
let _s = serial();
let fake = Fake::new("eg-nosock", &body("exit 0", "cat > /dev/null; exit 0"));
let log = Lines::default();
let egress = fake.dir.join("egress");
let podman = Podman::new(fake.runner(""), egress.clone(), log.sink())
.with_egress_wait(Duration::from_millis(200));
let started = Instant::now();
assert_eq!(call(&podman), failed(CANNOT_START));
assert!(started.elapsed() < Duration::from_secs(3));
let calls = fake.calls();
assert_eq!(calls.len(), 2, "{calls:?}");
assert_eq!(calls[1], ["rm", "-f", "boxmaker-s1-1-0-egress"]);
assert!(!egress.join("boxmaker-s1-1-0").exists());
assert!(log.all().contains(RUNBOOK));
}
#[test]
fn a_tool_that_runs_too_long_still_leaves_nothing_behind() {
let _s = serial();
let fake = Fake::new(
"eg-slow",
&body(
r#": > "$v/egress.sock"; exit 0"#,
"cat > /dev/null; exec sleep 30",
),
);
let log = Lines::default();
let egress = fake.dir.join("egress");
let got = call(&Podman::new(
fake.runner("http_fetch_ms = 300"),
egress.clone(),
log.sink(),
));
assert_eq!(got, failed(TIMED_OUT));
let calls = fake.calls();
let tail: Vec<Vec<String>> = calls[2..].to_vec();
assert_eq!(
tail,
[
vec!["kill", "boxmaker-s1-1-0"],
vec!["rm", "-f", "boxmaker-s1-1-0"],
vec!["rm", "-f", "boxmaker-s1-1-0-egress"],
]
.map(|c| c.into_iter().map(String::from).collect::<Vec<_>>())
);
assert!(!egress.join("boxmaker-s1-1-0").exists());
}
#[test]
fn a_tool_that_podman_cannot_start_still_leaves_nothing_behind() {
let _s = serial();
let fake = Fake::new(
"eg-toolfail",
&body(
r#": > "$v/egress.sock"; exit 0"#,
"cat > /dev/null; exit 125",
),
);
let log = Lines::default();
let egress = fake.dir.join("egress");
assert_eq!(
call(&Podman::new(fake.runner(""), egress.clone(), log.sink())),
failed(CANNOT_START)
);
assert_eq!(
fake.calls().last().unwrap(),
&["rm", "-f", "boxmaker-s1-1-0-egress"]
);
assert!(!egress.join("boxmaker-s1-1-0").exists());
}
#[test]
fn a_directory_left_by_a_crash_is_replaced() {
let _s = serial();
let fake = Fake::new(
"eg-stale",
&body(
r#"[ -e "$v/old.sock" ] && exit 9; : > "$v/egress.sock"; exit 0"#,
"cat > /dev/null; printf ok; exit 0",
),
);
let log = Lines::default();
let egress = fake.dir.join("egress");
std::fs::create_dir_all(egress.join("boxmaker-s1-1-0")).unwrap();
std::fs::write(egress.join("boxmaker-s1-1-0/old.sock"), "").unwrap();
let got = call(&Podman::new(fake.runner(""), egress.clone(), log.sink()));
assert!(
matches!(&got, ToolResponse::Result { content, .. } if content == "ok"),
"{got:?}"
);
}