Add the per-slot gate and chat_with_retry

Implemented SlotGate (one request in flight per server slot, FIFO queue,
bounded) and chat_with_retry (retry only server-gone errors, jittered
backoff within a window and attempt budget). chat acquires the gate and
maps GateFull to Busy; Client gained a gate field.

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
2026-09-18 14:45:45 -07:00
parent a07c341cbb
commit 59cf89e7ba
6 changed files with 597 additions and 2 deletions
+9
View File
@@ -35,6 +35,15 @@ impl Client {
req: &ChatRequest,
on_event: &mut dyn FnMut(&ChatEvent),
) -> Result<Completion, InferError> {
// The gate holds one request per slot from before it is sent until chat returns; a second
// request on the same slot queues here, bounded, and is told (Queued) or refused (Busy).
let _permit = self
.gate
.acquire(req.slot, self.cfg.limits.queue_len, &mut |ahead| {
on_event(&ChatEvent::Queued { ahead })
})
.map_err(|_| InferError::Busy)?;
// 1. Send.
let body_text =
build_body(&self.cfg, req).map_err(|e| InferError::Protocol(e.to_string()))?;
+139
View File
@@ -0,0 +1,139 @@
//! One request in flight per server slot at a time, in `loopd`.
//!
//! The inference server is shared, so two requests on the same slot would pile up on it and queue
//! invisibly. The gate bounds that queue here, where the caller is told it is waiting. The gate is
//! held for one request: from before it is sent until `chat` returns.
//!
//! A slot is free only when nobody holds it and nobody is waiting for it; a free slot is taken at
//! once. Otherwise the acquirer joins a first-in-first-out queue, is told how many are ahead, and
//! waits. The queue is bounded: past `max_queue` waiters the slot is reported full at once.
use std::collections::{HashMap, VecDeque};
use std::sync::{Condvar, Mutex};
/// Per-slot state, behind the mutex.
#[derive(Default)]
struct State {
slots: HashMap<u32, Slot>,
}
#[derive(Default)]
struct Slot {
/// Held by exactly one permit at a time.
holder: bool,
/// Tickets of the waiters, in the order they arrived.
queue: VecDeque<u32>,
/// A fresh ticket for each waiter that joins this slot's queue.
tickets: u32,
}
impl State {
fn slot_mut(&mut self, slot: u32) -> &mut Slot {
self.slots.entry(slot).or_default()
}
}
#[derive(Default)]
pub struct SlotGate {
state: Mutex<State>,
cond: Condvar,
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub struct GateFull;
/// The right to have a request in flight on the slot. Dropping it passes the slot on.
pub struct Permit<'a> {
gate: &'a SlotGate,
slot: u32,
}
impl SlotGate {
pub fn new() -> Self {
Self {
state: Mutex::new(State::default()),
cond: Condvar::new(),
}
}
pub fn acquire(
&self,
slot: u32,
max_queue: usize,
on_queued: &mut dyn FnMut(usize),
) -> Result<Permit<'_>, GateFull> {
// Rule 1: a free slot with nobody waiting is taken at once.
{
let mut state = self.lock();
let slot_ref = state.slot_mut(slot);
if !slot_ref.holder && slot_ref.queue.is_empty() {
slot_ref.holder = true;
return Ok(Permit { gate: self, slot });
}
}
// Rules 2 and 3: the slot is taken or someone is waiting. Join the queue, bounded.
let (ticket, ahead): (u32, usize);
{
let mut state = self.lock();
let slot_ref = state.slot_mut(slot);
if slot_ref.queue.len() >= max_queue {
return Err(GateFull);
}
slot_ref.tickets = slot_ref.tickets.wrapping_add(1);
ticket = slot_ref.tickets;
ahead = usize::from(slot_ref.holder) + slot_ref.queue.len();
slot_ref.queue.push_back(ticket);
}
on_queued(ahead);
// Rule 4: wait, in arrival order, for the slot to reach the front of the queue.
loop {
let mut state = self.lock();
if self.take_if_front(&mut state, slot, ticket) {
return Ok(Permit { gate: self, slot });
}
state = self
.cond
.wait(state)
.unwrap_or_else(|poisoned| poisoned.into_inner());
}
}
}
impl Drop for Permit<'_> {
fn drop(&mut self) {
let mut state = self.gate.lock();
state.slot_mut(self.slot).holder = false;
// Wake every waiter; only the front one with a free slot takes it (rule 4).
self.gate.cond.notify_all();
}
}
impl SlotGate {
fn lock(&self) -> std::sync::MutexGuard<'_, State> {
self.state
.lock()
.unwrap_or_else(|poisoned| poisoned.into_inner())
}
fn take_if_front(
&self,
state: &mut std::sync::MutexGuard<'_, State>,
slot: u32,
ticket: u32,
) -> bool {
let slot_ref = state.slot_mut(slot);
if slot_ref.holder {
return false;
}
match slot_ref.queue.front() {
Some(&front) if front == ticket => {
slot_ref.queue.pop_front();
slot_ref.holder = true;
true
}
_ => false,
}
}
}
+7 -1
View File
@@ -2,8 +2,10 @@
pub mod assemble;
pub mod chat;
pub mod gate;
pub mod info;
pub mod request;
pub mod retry;
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum ChatMessage {
@@ -136,11 +138,15 @@ impl std::error::Error for InferError {}
pub struct Client {
pub(crate) cfg: crate::config::Config,
pub(crate) gate: gate::SlotGate,
}
impl Client {
pub fn new(cfg: crate::config::Config) -> Self {
Self { cfg }
Self {
cfg,
gate: gate::SlotGate::new(),
}
}
pub fn config(&self) -> &crate::config::Config {
&self.cfg
+85
View File
@@ -0,0 +1,85 @@
//! Retrying a request whose server went away, with a bounded, jittered backoff.
//!
//! Retrying is safe because nothing is recorded until a completion is final: a retry sends the same
//! bytes again. Only errors that mean "the server went away" are retried, and only within the first
//! window and attempt budget, so a dead server is not chased forever.
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
use super::{ChatEvent, ChatRequest, Client, Completion, InferError};
pub fn is_retryable(e: &InferError) -> bool {
match e {
InferError::Connect(_) => true, // the server went away and may be back
InferError::StreamClosedEarly => true,
InferError::Stalled => true,
InferError::LoadTimeout => true,
InferError::Http { status: 503, .. } => true,
InferError::Busy => false, // the slot is busy; the gate already said so
InferError::WaitTimeout => false, // it has already waited
InferError::ThinkingOverrun => false,
InferError::Protocol(_) => false,
InferError::Http { .. } => false, // any other status would fail again
}
}
pub fn backoff_ms(schedule: &[u64], retry: u32, jitter: i32) -> u64 {
// Retry numbers start at 1; 0 is treated as 1.
let retry = retry.max(1);
// The base is schedule[retry - 1], or the last entry past the end, or 0 for an empty schedule.
let index = usize::try_from(retry - 1).unwrap_or(usize::MAX);
let base = schedule
.get(index)
.copied()
.or_else(|| schedule.last().copied())
.unwrap_or(0);
// Jitter is clamped to -1000..=1000 and moves the wait by up to a quarter of the base.
let jitter = jitter.clamp(-1000, 1000);
let value = i128::from(base) + i128::from(base / 4) * i128::from(jitter) / 1000;
u64::try_from(value.clamp(0, i128::from(u64::MAX))).unwrap_or(u64::MAX)
}
impl Client {
pub fn chat_with_retry(
&self,
req: &ChatRequest,
on_event: &mut dyn FnMut(&ChatEvent),
) -> Result<Completion, InferError> {
let start = Instant::now();
let mut attempt: u32 = 1;
loop {
match self.chat(req, on_event) {
Ok(completion) => return Ok(completion),
Err(e) => {
if !is_retryable(&e) || attempt >= self.cfg.limits.retry_attempts {
return Err(e);
}
let wait = backoff_ms(&self.cfg.limits.retry_backoff_ms, attempt, now_jitter());
if elapsed_ms(start) + wait > self.cfg.limits.retry_window_ms {
return Err(e);
}
on_event(&ChatEvent::Retrying {
attempt: attempt + 1,
after_ms: wait,
error: e.to_string(),
});
std::thread::sleep(Duration::from_millis(wait));
attempt += 1;
}
}
}
}
}
fn now_jitter() -> i32 {
let nanos = SystemTime::now()
.duration_since(UNIX_EPOCH)
.map(|since| since.subsec_nanos())
.unwrap_or(0);
// A nanosecond count reduced to -1000..=1000 is random enough for jitter.
i32::try_from(nanos % 2000).unwrap_or(999) - 1000
}
fn elapsed_ms(start: Instant) -> u64 {
start.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
}
+356
View File
@@ -0,0 +1,356 @@
//! Tests for the slot gate and for retrying. Do not edit.
mod support;
use loopd::llama::gate::{GateFull, SlotGate};
use loopd::llama::retry::{backoff_ms, is_retryable};
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client, InferError};
use std::sync::atomic::{AtomicUsize, Ordering};
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, Instant};
use support::{FakeServer, Reply};
const CHAT: &str = "/v1/chat/completions";
fn request(slot: u32) -> ChatRequest {
ChatRequest {
slot,
messages: vec![ChatMessage::User {
content: "hi".to_string(),
}],
tools: vec![],
thinking: false,
}
}
fn io(kind: std::io::ErrorKind) -> std::io::Error {
std::io::Error::from(kind)
}
// ---- the gate on its own ----
#[test]
fn a_free_slot_is_taken_at_once_and_slots_are_independent() {
let gate = SlotGate::new();
let mut queued = Vec::new();
let a = gate.acquire(0, 8, &mut |n| queued.push(n)).unwrap();
let b = gate.acquire(1, 8, &mut |n| queued.push(n)).unwrap();
assert!(queued.is_empty(), "nobody had to wait");
drop(a);
drop(b);
let _again = gate.acquire(0, 8, &mut |n| queued.push(n)).unwrap();
assert!(queued.is_empty());
}
#[test]
fn waiters_are_served_in_order_one_at_a_time() {
let gate = Arc::new(SlotGate::new());
let order = Arc::new(Mutex::new(Vec::new()));
let inside = Arc::new(AtomicUsize::new(0));
let first = gate.acquire(0, 8, &mut |_| {}).unwrap();
let mut threads = Vec::new();
for i in 0..4usize {
let (gate, order, inside) = (Arc::clone(&gate), Arc::clone(&order), Arc::clone(&inside));
threads.push(thread::spawn(move || {
let mut ahead = None;
let permit = gate.acquire(0, 8, &mut |n| ahead = Some(n)).unwrap();
assert_eq!(
inside.fetch_add(1, Ordering::SeqCst),
0,
"two permits for one slot at once"
);
order.lock().unwrap().push((i, ahead));
thread::sleep(Duration::from_millis(20));
inside.fetch_sub(1, Ordering::SeqCst);
drop(permit);
}));
thread::sleep(Duration::from_millis(30)); // so that the arrival order is known
}
drop(first);
for t in threads {
t.join().unwrap();
}
// Each was told how many were ahead of it: the holder plus the earlier waiters.
let want: Vec<(usize, Option<usize>)> = (0..4).map(|i| (i, Some(i + 1))).collect();
assert_eq!(*order.lock().unwrap(), want);
}
#[test]
fn a_full_queue_refuses_at_once() {
let gate = Arc::new(SlotGate::new());
let _holder = gate.acquire(0, 1, &mut |_| {}).unwrap();
let waiter = {
let gate = Arc::clone(&gate);
thread::spawn(move || drop(gate.acquire(0, 1, &mut |_| {}).unwrap()))
};
thread::sleep(Duration::from_millis(50));
let started = Instant::now();
assert!(
matches!(gate.acquire(0, 1, &mut |_| {}), Err(GateFull)),
"one holder and one waiter is the limit"
);
assert!(
started.elapsed() < Duration::from_millis(50),
"refusal must not wait"
);
assert!(
gate.acquire(1, 1, &mut |_| {}).is_ok(),
"another slot is unaffected"
);
drop(_holder);
waiter.join().unwrap();
}
#[test]
fn a_queue_of_zero_means_no_waiting_at_all() {
let gate = SlotGate::new();
let _holder = gate.acquire(0, 0, &mut |_| {}).unwrap();
assert!(matches!(gate.acquire(0, 0, &mut |_| {}), Err(GateFull)));
}
// ---- the gate inside the client ----
#[test]
fn two_requests_for_one_slot_never_overlap_at_the_server() {
let server = FakeServer::start();
let size = support::fixture_bytes("http", "turn1.http").len();
// Each reply takes about 200 ms to arrive.
server.route(CHAT, vec![Reply::fixture("turn1").trickle(size / 4, 50)]);
let client = Arc::new(Client::new(support::test_config(&server.socket)));
let started = Instant::now();
let second = {
let client = Arc::clone(&client);
thread::spawn(move || {
thread::sleep(Duration::from_millis(30));
let mut events = Vec::new();
let done = client.chat(&request(0), &mut |e| events.push(e.clone()));
(done.map(|d| d.content), events, Instant::now())
})
};
let first = client.chat(&request(0), &mut |_| {}).unwrap();
let first_done = Instant::now();
let (second_result, second_events, second_done) = second.join().unwrap();
assert_eq!(first.content.as_deref(), Some("Blue"));
assert_eq!(second_result.unwrap().as_deref(), Some("Blue"));
assert_eq!(second_events.first(), Some(&ChatEvent::Queued { ahead: 1 }));
assert!(second_done > first_done);
assert!(
started.elapsed() >= Duration::from_millis(280),
"the two ran one after the other"
);
}
#[test]
fn a_full_queue_is_busy() {
let server = FakeServer::start();
server.route(CHAT, vec![Reply::fixture("turn1").head_delay(300)]);
// Both slots report busy, so that the 300 ms of silence reads as a queue and not a stall.
let busy = r#"[{"id":0,"is_processing":true},{"id":1,"is_processing":true}]"#;
server.route("/slots", vec![Reply::json(200, busy)]);
let mut cfg = support::test_config(&server.socket);
cfg.limits.queue_len = 0;
let client = Arc::new(Client::new(cfg));
let holder = {
let client = Arc::clone(&client);
thread::spawn(move || client.chat(&request(0), &mut |_| {}).map(|_| ()))
};
thread::sleep(Duration::from_millis(80));
assert!(matches!(
client.chat(&request(0), &mut |_| {}),
Err(InferError::Busy)
));
assert!(
client.chat(&request(1), &mut |_| {}).is_ok(),
"another slot is free"
);
holder.join().unwrap().unwrap();
assert_eq!(
server.requests_to(CHAT).len(),
2,
"the refused request was never sent"
);
}
// ---- retry ----
#[test]
fn which_errors_are_retried() {
use std::io::ErrorKind::ConnectionRefused;
let http = |status| InferError::Http {
status,
body: String::new(),
};
let retried = [
InferError::Connect(io(ConnectionRefused)),
InferError::StreamClosedEarly,
InferError::Stalled,
InferError::LoadTimeout,
http(503),
];
for e in &retried {
assert!(
is_retryable(e),
"{e:?} means the server went away and may be back"
);
}
let not_retried = [
InferError::Busy,
InferError::WaitTimeout,
InferError::ThinkingOverrun,
InferError::Protocol("x".to_string()),
http(400),
http(404),
http(500),
];
for e in &not_retried {
assert!(
!is_retryable(e),
"{e:?} would fail again, or has already waited"
);
}
}
#[test]
fn backoff_follows_the_schedule_with_a_quarter_of_jitter() {
let schedule = [2_000, 8_000, 30_000];
assert_eq!(backoff_ms(&schedule, 1, 0), 2_000);
assert_eq!(backoff_ms(&schedule, 2, 0), 8_000);
assert_eq!(backoff_ms(&schedule, 3, 0), 30_000);
assert_eq!(
backoff_ms(&schedule, 4, 0),
30_000,
"past the end, the last entry"
);
assert_eq!(
backoff_ms(&schedule, 0, 0),
2_000,
"retry numbers start at 1; 0 is treated as 1"
);
assert_eq!(backoff_ms(&schedule, 1, 1000), 2_500);
assert_eq!(backoff_ms(&schedule, 1, -1000), 1_500);
assert_eq!(backoff_ms(&schedule, 2, 500), 9_000);
assert_eq!(
backoff_ms(&schedule, 1, 99_999),
2_500,
"jitter is clamped to -1000..=1000"
);
assert_eq!(backoff_ms(&schedule, 1, i32::MIN), 1_500);
assert_eq!(backoff_ms(&[], 1, 1000), 0, "no schedule, no wait");
// Huge values must not overflow or panic.
let _ = backoff_ms(&[u64::MAX], 1, 1000);
let _ = backoff_ms(&[u64::MAX], u32::MAX, -1000);
}
fn run_retry(
server: &FakeServer,
attempts: u32,
window_ms: u64,
) -> (Result<Option<String>, InferError>, Vec<ChatEvent>) {
let mut cfg = support::test_config(&server.socket);
cfg.limits.retry_attempts = attempts;
cfg.limits.retry_window_ms = window_ms;
let mut events = Vec::new();
let result = Client::new(cfg).chat_with_retry(&request(0), &mut |e| events.push(e.clone()));
(result.map(|d| d.content), events)
}
fn retrying(events: &[ChatEvent]) -> Vec<(u32, u64)> {
events
.iter()
.filter_map(|e| match e {
ChatEvent::Retrying {
attempt, after_ms, ..
} => Some((*attempt, *after_ms)),
_ => None,
})
.collect()
}
#[test]
fn a_server_that_comes_back_is_survived() {
let server = FakeServer::start();
let dead = Reply::fixture("turn1").cut_after(400);
server.route(CHAT, vec![dead.clone(), dead, Reply::fixture("turn1")]);
let (result, events) = run_retry(&server, 4, 5_000);
assert_eq!(result.unwrap().as_deref(), Some("Blue"));
let retries = retrying(&events);
assert_eq!(
retries
.iter()
.map(|(attempt, _)| *attempt)
.collect::<Vec<_>>(),
vec![2, 3]
);
// The test schedule is 10, 20, 30 ms, each moved by at most a quarter.
assert!((8..=12).contains(&retries[0].1), "{retries:?}");
assert!((15..=25).contains(&retries[1].1), "{retries:?}");
let sent = server.requests_to(CHAT);
assert_eq!(sent.len(), 3);
assert!(
sent.iter().all(|r| r.body == sent[0].body),
"a retry sends the same bytes again"
);
}
#[test]
fn retries_stop_at_the_attempt_limit() {
let server = FakeServer::start();
server.route(CHAT, vec![Reply::fixture("turn1").cut_after(400)]);
let (result, events) = run_retry(&server, 3, 5_000);
assert!(
matches!(result, Err(InferError::StreamClosedEarly)),
"{result:?}"
);
assert_eq!(server.requests_to(CHAT).len(), 3, "three attempts in all");
assert_eq!(retrying(&events).len(), 2);
}
#[test]
fn retries_stop_at_the_time_window() {
let server = FakeServer::start();
server.route(CHAT, vec![Reply::fixture("turn1").cut_after(400)]);
// The first retry would wait about 10 ms, which does not fit in a 5 ms window.
let (result, events) = run_retry(&server, 10, 5);
assert!(
matches!(result, Err(InferError::StreamClosedEarly)),
"{result:?}"
);
assert_eq!(server.requests_to(CHAT).len(), 1);
assert!(retrying(&events).is_empty());
}
#[test]
fn errors_that_would_fail_again_are_not_retried() {
let server = FakeServer::start();
server.route(CHAT, vec![Reply::fixture("bad_request")]);
let (result, events) = run_retry(&server, 4, 5_000);
assert!(
matches!(result, Err(InferError::Http { status: 400, .. })),
"{result:?}"
);
assert_eq!(server.requests_to(CHAT).len(), 1);
assert!(retrying(&events).is_empty());
}
#[test]
fn a_loading_server_is_retried() {
let server = FakeServer::start();
server.route(
CHAT,
vec![
Reply::json(503, r#"{"error":{"message":"Loading model"}}"#),
Reply::fixture("turn1"),
],
);
let (result, events) = run_retry(&server, 4, 5_000);
assert_eq!(result.unwrap().as_deref(), Some("Blue"));
let ChatEvent::Retrying { error, .. } = events
.iter()
.find(|e| matches!(e, ChatEvent::Retrying { .. }))
.unwrap()
else {
unreachable!()
};
assert!(error.contains("503"), "the event says why: {error}");
}