gatewayd: http, requests over a stream with size caps and rate-limit waits
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -0,0 +1,357 @@
|
||||
//! HTTP/1.1 over a connected stream: one request, one response, `Connection: close` (M4a spec,
|
||||
//! section 5). The response head is read a byte at a time, so nothing past it is consumed: the
|
||||
//! WebSocket handshake reads its frames after it from the same stream.
|
||||
|
||||
use std::io::{Read, Write};
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
pub const MAX_HEAD: usize = 16 * 1024;
|
||||
pub const MAX_BODY: usize = 4 * 1024 * 1024;
|
||||
/// The longest a rate limit is waited out, whatever the server says.
|
||||
pub const MAX_RATE_WAIT: Duration = Duration::from_secs(60);
|
||||
|
||||
#[derive(Debug)]
|
||||
pub enum HttpError {
|
||||
Io(std::io::Error),
|
||||
/// The response is not HTTP/1.1 as we read it.
|
||||
Protocol(String),
|
||||
/// A head or body over its cap.
|
||||
TooLarge(&'static str),
|
||||
}
|
||||
|
||||
impl std::fmt::Display for HttpError {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
match self {
|
||||
HttpError::Io(e) => write!(f, "{e}"),
|
||||
HttpError::Protocol(why) => write!(f, "bad HTTP response: {why}"),
|
||||
HttpError::TooLarge(what) => write!(f, "the response {what} is too large"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl std::error::Error for HttpError {}
|
||||
|
||||
impl From<std::io::Error> for HttpError {
|
||||
fn from(e: std::io::Error) -> Self {
|
||||
HttpError::Io(e)
|
||||
}
|
||||
}
|
||||
|
||||
/// A status line and headers.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Head {
|
||||
pub status: u16,
|
||||
pub headers: Vec<(String, String)>,
|
||||
}
|
||||
|
||||
impl Head {
|
||||
/// The first header named `name`, compared case-insensitively.
|
||||
pub fn header(&self, name: &str) -> Option<&str> {
|
||||
// The value of the first header whose name matches, ignoring ASCII case.
|
||||
self.headers
|
||||
.iter()
|
||||
.find(|(k, _)| k.eq_ignore_ascii_case(name))
|
||||
.map(|(_, v)| v.as_str())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct Response {
|
||||
pub head: Head,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
/// Write one request. `host` is the `Host` header; `headers` come after it, then
|
||||
/// `Content-Length` when there is a body, then `Connection: close`.
|
||||
pub fn write_request(
|
||||
stream: &mut dyn Write,
|
||||
method: &str,
|
||||
host: &str,
|
||||
path: &str,
|
||||
headers: &[(&str, &str)],
|
||||
body: Option<&[u8]>,
|
||||
) -> Result<(), HttpError> {
|
||||
// Exactly: "<method> <path> HTTP/1.1\r\nHost: <host>\r\n", each header as "<k>: <v>\r\n",
|
||||
// "Content-Length: <n>\r\n" when there is a body, "Connection: close\r\n\r\n", then the body.
|
||||
// Flush.
|
||||
let mut head = Vec::new();
|
||||
let body_len = body.map(|b| b.len()).unwrap_or(0);
|
||||
head.extend_from_slice(format!("{method} {path} HTTP/1.1\r\n").as_bytes());
|
||||
head.extend_from_slice(format!("Host: {host}\r\n").as_bytes());
|
||||
for (k, v) in headers {
|
||||
head.extend_from_slice(format!("{k}: {v}\r\n").as_bytes());
|
||||
}
|
||||
if body.is_some() {
|
||||
head.extend_from_slice(format!("Content-Length: {body_len}\r\n").as_bytes());
|
||||
}
|
||||
head.extend_from_slice(b"Connection: close\r\n\r\n");
|
||||
stream.write_all(&head)?;
|
||||
if let Some(b) = body {
|
||||
stream.write_all(b)?;
|
||||
}
|
||||
stream.flush()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// One request and its whole response.
|
||||
pub fn request(
|
||||
stream: &mut (impl Read + Write),
|
||||
method: &str,
|
||||
host: &str,
|
||||
path: &str,
|
||||
headers: &[(&str, &str)],
|
||||
body: Option<&[u8]>,
|
||||
) -> Result<Response, HttpError> {
|
||||
// `write_request`, then `read_head`, then `read_body`.
|
||||
write_request(stream, method, host, path, headers, body)?;
|
||||
let head = read_head(stream)?;
|
||||
let body = read_body(stream, &head)?;
|
||||
Ok(Response { head, body })
|
||||
}
|
||||
|
||||
/// The status line and headers, up to and including the blank line, and not a byte more.
|
||||
pub fn read_head(stream: &mut dyn Read) -> Result<Head, HttpError> {
|
||||
// One byte at a time until CRLF CRLF, never more; over MAX_HEAD is TooLarge("head"); end of
|
||||
// stream is Protocol; retry Interrupted. Then: UTF-8; status line "HTTP/1.1" or "HTTP/1.0", a
|
||||
// 3-digit status in 100..=599; each header line "name: value" with a non-empty name without
|
||||
// spaces, the value trimmed. Anything else is Protocol.
|
||||
let mut raw = Vec::new();
|
||||
let mut byte = [0u8; 1];
|
||||
loop {
|
||||
let n = loop {
|
||||
match stream.read(&mut byte) {
|
||||
Ok(0) => {
|
||||
return Err(HttpError::Protocol(
|
||||
"the head ended before the blank line".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(n) => break n,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(HttpError::Io(e)),
|
||||
}
|
||||
};
|
||||
debug_assert_eq!(n, 1);
|
||||
raw.push(byte[0]);
|
||||
if raw.len() > MAX_HEAD {
|
||||
return Err(HttpError::TooLarge("head"));
|
||||
}
|
||||
if raw.ends_with(b"\r\n\r\n") {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let text = std::str::from_utf8(&raw)
|
||||
.map_err(|_| HttpError::Protocol("the head is not UTF-8".to_string()))?;
|
||||
|
||||
let mut lines = text.split("\r\n");
|
||||
let status_line = lines.next().unwrap_or("");
|
||||
let (version, code) = parse_status_line(status_line)?;
|
||||
let _ = version;
|
||||
|
||||
let mut headers = Vec::new();
|
||||
for line in lines {
|
||||
if line.is_empty() {
|
||||
break;
|
||||
}
|
||||
let (name, value) = line
|
||||
.split_once(':')
|
||||
.ok_or_else(|| HttpError::Protocol("a header line has no colon".to_string()))?;
|
||||
if name.is_empty() || name.bytes().any(|b| b == b' ') {
|
||||
return Err(HttpError::Protocol(
|
||||
"a header name is empty or has a space".to_string(),
|
||||
));
|
||||
}
|
||||
headers.push((name.to_string(), value.trim().to_string()));
|
||||
}
|
||||
|
||||
Ok(Head {
|
||||
status: code,
|
||||
headers,
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_status_line(line: &str) -> Result<(&str, u16), HttpError> {
|
||||
// "HTTP/1.1 200 OK": a 1.1 or 1.0 version, then a 3-digit status in 100..=599, then anything.
|
||||
let mut parts = line.split_whitespace();
|
||||
let version = parts.next().unwrap_or("");
|
||||
let code = parts.next().unwrap_or("");
|
||||
if version != "HTTP/1.1" && version != "HTTP/1.0" {
|
||||
return Err(HttpError::Protocol(
|
||||
"not an HTTP/1.1 status line".to_string(),
|
||||
));
|
||||
}
|
||||
if code.len() != 3 || !code.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(HttpError::Protocol(
|
||||
"the status code is not three digits".to_string(),
|
||||
));
|
||||
}
|
||||
let code: u16 = code
|
||||
.parse()
|
||||
.map_err(|_| HttpError::Protocol("the status code will not parse".to_string()))?;
|
||||
if !(100..=599).contains(&code) {
|
||||
return Err(HttpError::Protocol(
|
||||
"the status code is out of range".to_string(),
|
||||
));
|
||||
}
|
||||
Ok((version, code))
|
||||
}
|
||||
|
||||
/// The body after `head`: chunked, `Content-Length`, or to the end; at most `MAX_BODY`.
|
||||
pub fn read_body(stream: &mut dyn Read, head: &Head) -> Result<Vec<u8>, HttpError> {
|
||||
// Transfer-Encoding: chunked (any case) -> `read_chunked`. Else Content-Length: parse (else
|
||||
// Protocol), over MAX_BODY is TooLarge("body"), then read_exact that many. Else read to the end
|
||||
// through `take(MAX_BODY + 1)`; more than MAX_BODY is TooLarge("body").
|
||||
if head
|
||||
.header("transfer-encoding")
|
||||
.is_some_and(|v| v.eq_ignore_ascii_case("chunked"))
|
||||
{
|
||||
return read_chunked(stream);
|
||||
}
|
||||
match head.header("content-length") {
|
||||
Some(s) => {
|
||||
let len = parse_content_length(s)?;
|
||||
if len > MAX_BODY {
|
||||
return Err(HttpError::TooLarge("body"));
|
||||
}
|
||||
let mut buf = vec![0u8; len];
|
||||
stream.read_exact(&mut buf)?;
|
||||
Ok(buf)
|
||||
}
|
||||
None => {
|
||||
let mut buf = Vec::new();
|
||||
stream.take((MAX_BODY + 1) as u64).read_to_end(&mut buf)?;
|
||||
if buf.len() > MAX_BODY {
|
||||
return Err(HttpError::TooLarge("body"));
|
||||
}
|
||||
Ok(buf)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_content_length(s: &str) -> Result<usize, HttpError> {
|
||||
// Its own digits only: a non-number is Protocol, an overflowing one is Protocol too.
|
||||
if s.is_empty() || !s.bytes().all(|b| b.is_ascii_digit()) {
|
||||
return Err(HttpError::Protocol(
|
||||
"the Content-Length is not a number".to_string(),
|
||||
));
|
||||
}
|
||||
s.parse::<usize>()
|
||||
.map_err(|_| HttpError::Protocol("the Content-Length is too large".to_string()))
|
||||
}
|
||||
|
||||
fn read_line(stream: &mut dyn Read, cap: usize) -> Result<String, HttpError> {
|
||||
// Bytes up to CRLF (dropped), at most `cap` (else TooLarge("chunk header")); end of stream is
|
||||
// Protocol.
|
||||
let mut bytes = Vec::new();
|
||||
let mut byte = [0u8; 1];
|
||||
loop {
|
||||
let n = loop {
|
||||
match stream.read(&mut byte) {
|
||||
Ok(0) => {
|
||||
return Err(HttpError::Protocol(
|
||||
"the line ended before its CRLF".to_string(),
|
||||
));
|
||||
}
|
||||
Ok(n) => break n,
|
||||
Err(e) if e.kind() == std::io::ErrorKind::Interrupted => continue,
|
||||
Err(e) => return Err(HttpError::Io(e)),
|
||||
}
|
||||
};
|
||||
debug_assert_eq!(n, 1);
|
||||
match byte[0] {
|
||||
b'\n' => {
|
||||
if bytes.last() == Some(&b'\r') {
|
||||
bytes.pop();
|
||||
}
|
||||
return String::from_utf8(bytes)
|
||||
.map_err(|_| HttpError::Protocol("a line is not UTF-8".to_string()));
|
||||
}
|
||||
other => {
|
||||
bytes.push(other);
|
||||
if bytes.len() > cap {
|
||||
return Err(HttpError::TooLarge("chunk header"));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn read_chunked(stream: &mut dyn Read) -> Result<Vec<u8>, HttpError> {
|
||||
// Loop: a size line (hex, before any ";"), at most 1024 bytes. Size 0: read trailer lines (8
|
||||
// KiB each) until an empty one, and return. Otherwise the size must fit in MAX_BODY minus what
|
||||
// is already read (else TooLarge("body")), read it, then exactly CRLF (else Protocol).
|
||||
let mut out = Vec::new();
|
||||
loop {
|
||||
let size_line = read_line(stream, 1024)?.trim().to_string();
|
||||
let hex = match size_line.split_once(';') {
|
||||
Some((h, _)) => h.trim(),
|
||||
None => &size_line,
|
||||
};
|
||||
let size = parse_hex(hex)?;
|
||||
if size == 0 {
|
||||
loop {
|
||||
let trailer = read_line(stream, 8 * 1024)?;
|
||||
if trailer.is_empty() {
|
||||
return Ok(out);
|
||||
}
|
||||
}
|
||||
}
|
||||
if size > MAX_BODY as u128 - out.len() as u128 {
|
||||
return Err(HttpError::TooLarge("body"));
|
||||
}
|
||||
let size = size as usize;
|
||||
let mut chunk = vec![0u8; size];
|
||||
stream.read_exact(&mut chunk)?;
|
||||
out.extend_from_slice(&chunk);
|
||||
let terminator = read_line(stream, 2)?;
|
||||
if !terminator.is_empty() {
|
||||
return Err(HttpError::Protocol(
|
||||
"a chunk is not ended by a blank line".to_string(),
|
||||
));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn parse_hex(s: &str) -> Result<u128, HttpError> {
|
||||
// A hex chunk size; an overflow past u128 is refused here, so a size larger than the body cap
|
||||
// still comes back as an error to the caller.
|
||||
let mut value: u128 = 0;
|
||||
for b in s.bytes() {
|
||||
let d = match b {
|
||||
b'0'..=b'9' => (b - b'0') as u128,
|
||||
b'a'..=b'f' => (b - b'a' + 10) as u128,
|
||||
b'A'..=b'F' => (b - b'A' + 10) as u128,
|
||||
_ => return Err(HttpError::Protocol("a chunk size is not hex".to_string())),
|
||||
};
|
||||
value = value
|
||||
.checked_mul(16)
|
||||
.and_then(|v| v.checked_add(d))
|
||||
.ok_or_else(|| HttpError::Protocol("a chunk size is too large".to_string()))?;
|
||||
}
|
||||
Ok(value)
|
||||
}
|
||||
|
||||
/// How long a 429 asks us to wait, from `X-Ratelimit-Reset`: a Unix time if it is one, else a
|
||||
/// number of seconds; never more than `MAX_RATE_WAIT`. One second if the header is missing or bad.
|
||||
pub fn rate_limit_wait(head: &Head, now: SystemTime) -> Duration {
|
||||
// X-Ratelimit-Reset as u64: above 1_000_000_000 it is a Unix time (wait = it - now, at least 1
|
||||
// s), otherwise seconds (at least 1). Missing or not a number: 1 s. Never more than
|
||||
// MAX_RATE_WAIT.
|
||||
let reset = match head.header("x-ratelimit-reset") {
|
||||
Some(v) => v,
|
||||
None => return Duration::from_secs(1),
|
||||
};
|
||||
let value: u64 = match reset.parse() {
|
||||
Ok(v) => v,
|
||||
Err(_) => return Duration::from_secs(1),
|
||||
};
|
||||
let secs = if value > 1_000_000_000 {
|
||||
match now.duration_since(UNIX_EPOCH) {
|
||||
Ok(elapsed) => value.saturating_sub(elapsed.as_secs()).max(1),
|
||||
Err(_) => 1,
|
||||
}
|
||||
} else {
|
||||
value.max(1)
|
||||
};
|
||||
Duration::from_secs(secs.min(MAX_RATE_WAIT.as_secs()))
|
||||
}
|
||||
Reference in New Issue
Block a user