Add the per-slot gate and chat_with_retry

Implemented SlotGate (one request in flight per server slot, FIFO queue,
bounded) and chat_with_retry (retry only server-gone errors, jittered
backoff within a window and attempt budget). chat acquires the gate and
maps GateFull to Busy; Client gained a gate field.

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

crates/loopd/src/llama/chat.rs

@@ -35,6 +35,15 @@ impl Client {
         req: &ChatRequest,
         on_event: &mut dyn FnMut(&ChatEvent),
     ) -> Result<Completion, InferError> {
+        // The gate holds one request per slot from before it is sent until chat returns; a second
+        // request on the same slot queues here, bounded, and is told (Queued) or refused (Busy).
+        let _permit = self
+            .gate
+            .acquire(req.slot, self.cfg.limits.queue_len, &mut |ahead| {
+                on_event(&ChatEvent::Queued { ahead })
+            })
+            .map_err(|_| InferError::Busy)?;
+
         // 1. Send.
         let body_text =
             build_body(&self.cfg, req).map_err(|e| InferError::Protocol(e.to_string()))?;

crates/loopd/src/llama/gate.rs

@@ -0,0 +1,139 @@
+//! One request in flight per server slot at a time, in `loopd`.
+//!
+//! The inference server is shared, so two requests on the same slot would pile up on it and queue
+//! invisibly. The gate bounds that queue here, where the caller is told it is waiting. The gate is
+//! held for one request: from before it is sent until `chat` returns.
+//!
+//! A slot is free only when nobody holds it and nobody is waiting for it; a free slot is taken at
+//! once. Otherwise the acquirer joins a first-in-first-out queue, is told how many are ahead, and
+//! waits. The queue is bounded: past `max_queue` waiters the slot is reported full at once.
+
+use std::collections::{HashMap, VecDeque};
+use std::sync::{Condvar, Mutex};
+
+/// Per-slot state, behind the mutex.
+#[derive(Default)]
+struct State {
+    slots: HashMap<u32, Slot>,
+}
+
+#[derive(Default)]
+struct Slot {
+    /// Held by exactly one permit at a time.
+    holder: bool,
+    /// Tickets of the waiters, in the order they arrived.
+    queue: VecDeque<u32>,
+    /// A fresh ticket for each waiter that joins this slot's queue.
+    tickets: u32,
+}
+
+impl State {
+    fn slot_mut(&mut self, slot: u32) -> &mut Slot {
+        self.slots.entry(slot).or_default()
+    }
+}
+
+#[derive(Default)]
+pub struct SlotGate {
+    state: Mutex<State>,
+    cond: Condvar,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub struct GateFull;
+
+/// The right to have a request in flight on the slot. Dropping it passes the slot on.
+pub struct Permit<'a> {
+    gate: &'a SlotGate,
+    slot: u32,
+}
+
+impl SlotGate {
+    pub fn new() -> Self {
+        Self {
+            state: Mutex::new(State::default()),
+            cond: Condvar::new(),
+        }
+    }
+
+    pub fn acquire(
+        &self,
+        slot: u32,
+        max_queue: usize,
+        on_queued: &mut dyn FnMut(usize),
+    ) -> Result<Permit<'_>, GateFull> {
+        // Rule 1: a free slot with nobody waiting is taken at once.
+        {
+            let mut state = self.lock();
+            let slot_ref = state.slot_mut(slot);
+            if !slot_ref.holder && slot_ref.queue.is_empty() {
+                slot_ref.holder = true;
+                return Ok(Permit { gate: self, slot });
+            }
+        }
+
+        // Rules 2 and 3: the slot is taken or someone is waiting. Join the queue, bounded.
+        let (ticket, ahead): (u32, usize);
+        {
+            let mut state = self.lock();
+            let slot_ref = state.slot_mut(slot);
+            if slot_ref.queue.len() >= max_queue {
+                return Err(GateFull);
+            }
+            slot_ref.tickets = slot_ref.tickets.wrapping_add(1);
+            ticket = slot_ref.tickets;
+            ahead = usize::from(slot_ref.holder) + slot_ref.queue.len();
+            slot_ref.queue.push_back(ticket);
+        }
+        on_queued(ahead);
+
+        // Rule 4: wait, in arrival order, for the slot to reach the front of the queue.
+        loop {
+            let mut state = self.lock();
+            if self.take_if_front(&mut state, slot, ticket) {
+                return Ok(Permit { gate: self, slot });
+            }
+            state = self
+                .cond
+                .wait(state)
+                .unwrap_or_else(|poisoned| poisoned.into_inner());
+        }
+    }
+}
+
+impl Drop for Permit<'_> {
+    fn drop(&mut self) {
+        let mut state = self.gate.lock();
+        state.slot_mut(self.slot).holder = false;
+        // Wake every waiter; only the front one with a free slot takes it (rule 4).
+        self.gate.cond.notify_all();
+    }
+}
+
+impl SlotGate {
+    fn lock(&self) -> std::sync::MutexGuard<'_, State> {
+        self.state
+            .lock()
+            .unwrap_or_else(|poisoned| poisoned.into_inner())
+    }
+
+    fn take_if_front(
+        &self,
+        state: &mut std::sync::MutexGuard<'_, State>,
+        slot: u32,
+        ticket: u32,
+    ) -> bool {
+        let slot_ref = state.slot_mut(slot);
+        if slot_ref.holder {
+            return false;
+        }
+        match slot_ref.queue.front() {
+            Some(&front) if front == ticket => {
+                slot_ref.queue.pop_front();
+                slot_ref.holder = true;
+                true
+            }
+            _ => false,
+        }
+    }
+}

crates/loopd/src/llama/mod.rs

@@ -2,8 +2,10 @@
 
 pub mod assemble;
 pub mod chat;
+pub mod gate;
 pub mod info;
 pub mod request;
+pub mod retry;
 
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum ChatMessage {
@@ -136,11 +138,15 @@ impl std::error::Error for InferError {}
 
 pub struct Client {
     pub(crate) cfg: crate::config::Config,
+    pub(crate) gate: gate::SlotGate,
 }
 
 impl Client {
     pub fn new(cfg: crate::config::Config) -> Self {
-        Self { cfg }
+        Self {
+            cfg,
+            gate: gate::SlotGate::new(),
+        }
     }
     pub fn config(&self) -> &crate::config::Config {
         &self.cfg

crates/loopd/src/llama/retry.rs

@@ -0,0 +1,85 @@
+//! Retrying a request whose server went away, with a bounded, jittered backoff.
+//!
+//! Retrying is safe because nothing is recorded until a completion is final: a retry sends the same
+//! bytes again. Only errors that mean "the server went away" are retried, and only within the first
+//! window and attempt budget, so a dead server is not chased forever.
+
+use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
+
+use super::{ChatEvent, ChatRequest, Client, Completion, InferError};
+
+pub fn is_retryable(e: &InferError) -> bool {
+    match e {
+        InferError::Connect(_) => true, // the server went away and may be back
+        InferError::StreamClosedEarly => true,
+        InferError::Stalled => true,
+        InferError::LoadTimeout => true,
+        InferError::Http { status: 503, .. } => true,
+        InferError::Busy => false, // the slot is busy; the gate already said so
+        InferError::WaitTimeout => false, // it has already waited
+        InferError::ThinkingOverrun => false,
+        InferError::Protocol(_) => false,
+        InferError::Http { .. } => false, // any other status would fail again
+    }
+}
+
+pub fn backoff_ms(schedule: &[u64], retry: u32, jitter: i32) -> u64 {
+    // Retry numbers start at 1; 0 is treated as 1.
+    let retry = retry.max(1);
+    // The base is schedule[retry - 1], or the last entry past the end, or 0 for an empty schedule.
+    let index = usize::try_from(retry - 1).unwrap_or(usize::MAX);
+    let base = schedule
+        .get(index)
+        .copied()
+        .or_else(|| schedule.last().copied())
+        .unwrap_or(0);
+    // Jitter is clamped to -1000..=1000 and moves the wait by up to a quarter of the base.
+    let jitter = jitter.clamp(-1000, 1000);
+    let value = i128::from(base) + i128::from(base / 4) * i128::from(jitter) / 1000;
+    u64::try_from(value.clamp(0, i128::from(u64::MAX))).unwrap_or(u64::MAX)
+}
+
+impl Client {
+    pub fn chat_with_retry(
+        &self,
+        req: &ChatRequest,
+        on_event: &mut dyn FnMut(&ChatEvent),
+    ) -> Result<Completion, InferError> {
+        let start = Instant::now();
+        let mut attempt: u32 = 1;
+        loop {
+            match self.chat(req, on_event) {
+                Ok(completion) => return Ok(completion),
+                Err(e) => {
+                    if !is_retryable(&e) || attempt >= self.cfg.limits.retry_attempts {
+                        return Err(e);
+                    }
+                    let wait = backoff_ms(&self.cfg.limits.retry_backoff_ms, attempt, now_jitter());
+                    if elapsed_ms(start) + wait > self.cfg.limits.retry_window_ms {
+                        return Err(e);
+                    }
+                    on_event(&ChatEvent::Retrying {
+                        attempt: attempt + 1,
+                        after_ms: wait,
+                        error: e.to_string(),
+                    });
+                    std::thread::sleep(Duration::from_millis(wait));
+                    attempt += 1;
+                }
+            }
+        }
+    }
+}
+
+fn now_jitter() -> i32 {
+    let nanos = SystemTime::now()
+        .duration_since(UNIX_EPOCH)
+        .map(|since| since.subsec_nanos())
+        .unwrap_or(0);
+    // A nanosecond count reduced to -1000..=1000 is random enough for jitter.
+    i32::try_from(nanos % 2000).unwrap_or(999) - 1000
+}
+
+fn elapsed_ms(start: Instant) -> u64 {
+    start.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
+}