Add the per-slot gate and chat_with_retry
Implemented SlotGate (one request in flight per server slot, FIFO queue, bounded) and chat_with_retry (retry only server-gone errors, jittered backoff within a window and attempt budget). chat acquires the gate and maps GateFull to Busy; Client gained a gate field. Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -35,6 +35,15 @@ impl Client {
|
||||
req: &ChatRequest,
|
||||
on_event: &mut dyn FnMut(&ChatEvent),
|
||||
) -> Result<Completion, InferError> {
|
||||
// The gate holds one request per slot from before it is sent until chat returns; a second
|
||||
// request on the same slot queues here, bounded, and is told (Queued) or refused (Busy).
|
||||
let _permit = self
|
||||
.gate
|
||||
.acquire(req.slot, self.cfg.limits.queue_len, &mut |ahead| {
|
||||
on_event(&ChatEvent::Queued { ahead })
|
||||
})
|
||||
.map_err(|_| InferError::Busy)?;
|
||||
|
||||
// 1. Send.
|
||||
let body_text =
|
||||
build_body(&self.cfg, req).map_err(|e| InferError::Protocol(e.to_string()))?;
|
||||
|
||||
@@ -0,0 +1,139 @@
|
||||
//! One request in flight per server slot at a time, in `loopd`.
|
||||
//!
|
||||
//! The inference server is shared, so two requests on the same slot would pile up on it and queue
|
||||
//! invisibly. The gate bounds that queue here, where the caller is told it is waiting. The gate is
|
||||
//! held for one request: from before it is sent until `chat` returns.
|
||||
//!
|
||||
//! A slot is free only when nobody holds it and nobody is waiting for it; a free slot is taken at
|
||||
//! once. Otherwise the acquirer joins a first-in-first-out queue, is told how many are ahead, and
|
||||
//! waits. The queue is bounded: past `max_queue` waiters the slot is reported full at once.
|
||||
|
||||
use std::collections::{HashMap, VecDeque};
|
||||
use std::sync::{Condvar, Mutex};
|
||||
|
||||
/// Per-slot state, behind the mutex.
|
||||
#[derive(Default)]
|
||||
struct State {
|
||||
slots: HashMap<u32, Slot>,
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
struct Slot {
|
||||
/// Held by exactly one permit at a time.
|
||||
holder: bool,
|
||||
/// Tickets of the waiters, in the order they arrived.
|
||||
queue: VecDeque<u32>,
|
||||
/// A fresh ticket for each waiter that joins this slot's queue.
|
||||
tickets: u32,
|
||||
}
|
||||
|
||||
impl State {
|
||||
fn slot_mut(&mut self, slot: u32) -> &mut Slot {
|
||||
self.slots.entry(slot).or_default()
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Default)]
|
||||
pub struct SlotGate {
|
||||
state: Mutex<State>,
|
||||
cond: Condvar,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub struct GateFull;
|
||||
|
||||
/// The right to have a request in flight on the slot. Dropping it passes the slot on.
|
||||
pub struct Permit<'a> {
|
||||
gate: &'a SlotGate,
|
||||
slot: u32,
|
||||
}
|
||||
|
||||
impl SlotGate {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
state: Mutex::new(State::default()),
|
||||
cond: Condvar::new(),
|
||||
}
|
||||
}
|
||||
|
||||
pub fn acquire(
|
||||
&self,
|
||||
slot: u32,
|
||||
max_queue: usize,
|
||||
on_queued: &mut dyn FnMut(usize),
|
||||
) -> Result<Permit<'_>, GateFull> {
|
||||
// Rule 1: a free slot with nobody waiting is taken at once.
|
||||
{
|
||||
let mut state = self.lock();
|
||||
let slot_ref = state.slot_mut(slot);
|
||||
if !slot_ref.holder && slot_ref.queue.is_empty() {
|
||||
slot_ref.holder = true;
|
||||
return Ok(Permit { gate: self, slot });
|
||||
}
|
||||
}
|
||||
|
||||
// Rules 2 and 3: the slot is taken or someone is waiting. Join the queue, bounded.
|
||||
let (ticket, ahead): (u32, usize);
|
||||
{
|
||||
let mut state = self.lock();
|
||||
let slot_ref = state.slot_mut(slot);
|
||||
if slot_ref.queue.len() >= max_queue {
|
||||
return Err(GateFull);
|
||||
}
|
||||
slot_ref.tickets = slot_ref.tickets.wrapping_add(1);
|
||||
ticket = slot_ref.tickets;
|
||||
ahead = usize::from(slot_ref.holder) + slot_ref.queue.len();
|
||||
slot_ref.queue.push_back(ticket);
|
||||
}
|
||||
on_queued(ahead);
|
||||
|
||||
// Rule 4: wait, in arrival order, for the slot to reach the front of the queue.
|
||||
loop {
|
||||
let mut state = self.lock();
|
||||
if self.take_if_front(&mut state, slot, ticket) {
|
||||
return Ok(Permit { gate: self, slot });
|
||||
}
|
||||
state = self
|
||||
.cond
|
||||
.wait(state)
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Permit<'_> {
|
||||
fn drop(&mut self) {
|
||||
let mut state = self.gate.lock();
|
||||
state.slot_mut(self.slot).holder = false;
|
||||
// Wake every waiter; only the front one with a free slot takes it (rule 4).
|
||||
self.gate.cond.notify_all();
|
||||
}
|
||||
}
|
||||
|
||||
impl SlotGate {
|
||||
fn lock(&self) -> std::sync::MutexGuard<'_, State> {
|
||||
self.state
|
||||
.lock()
|
||||
.unwrap_or_else(|poisoned| poisoned.into_inner())
|
||||
}
|
||||
|
||||
fn take_if_front(
|
||||
&self,
|
||||
state: &mut std::sync::MutexGuard<'_, State>,
|
||||
slot: u32,
|
||||
ticket: u32,
|
||||
) -> bool {
|
||||
let slot_ref = state.slot_mut(slot);
|
||||
if slot_ref.holder {
|
||||
return false;
|
||||
}
|
||||
match slot_ref.queue.front() {
|
||||
Some(&front) if front == ticket => {
|
||||
slot_ref.queue.pop_front();
|
||||
slot_ref.holder = true;
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,8 +2,10 @@
|
||||
|
||||
pub mod assemble;
|
||||
pub mod chat;
|
||||
pub mod gate;
|
||||
pub mod info;
|
||||
pub mod request;
|
||||
pub mod retry;
|
||||
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum ChatMessage {
|
||||
@@ -136,11 +138,15 @@ impl std::error::Error for InferError {}
|
||||
|
||||
pub struct Client {
|
||||
pub(crate) cfg: crate::config::Config,
|
||||
pub(crate) gate: gate::SlotGate,
|
||||
}
|
||||
|
||||
impl Client {
|
||||
pub fn new(cfg: crate::config::Config) -> Self {
|
||||
Self { cfg }
|
||||
Self {
|
||||
cfg,
|
||||
gate: gate::SlotGate::new(),
|
||||
}
|
||||
}
|
||||
pub fn config(&self) -> &crate::config::Config {
|
||||
&self.cfg
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
//! Retrying a request whose server went away, with a bounded, jittered backoff.
|
||||
//!
|
||||
//! Retrying is safe because nothing is recorded until a completion is final: a retry sends the same
|
||||
//! bytes again. Only errors that mean "the server went away" are retried, and only within the first
|
||||
//! window and attempt budget, so a dead server is not chased forever.
|
||||
|
||||
use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH};
|
||||
|
||||
use super::{ChatEvent, ChatRequest, Client, Completion, InferError};
|
||||
|
||||
pub fn is_retryable(e: &InferError) -> bool {
|
||||
match e {
|
||||
InferError::Connect(_) => true, // the server went away and may be back
|
||||
InferError::StreamClosedEarly => true,
|
||||
InferError::Stalled => true,
|
||||
InferError::LoadTimeout => true,
|
||||
InferError::Http { status: 503, .. } => true,
|
||||
InferError::Busy => false, // the slot is busy; the gate already said so
|
||||
InferError::WaitTimeout => false, // it has already waited
|
||||
InferError::ThinkingOverrun => false,
|
||||
InferError::Protocol(_) => false,
|
||||
InferError::Http { .. } => false, // any other status would fail again
|
||||
}
|
||||
}
|
||||
|
||||
pub fn backoff_ms(schedule: &[u64], retry: u32, jitter: i32) -> u64 {
|
||||
// Retry numbers start at 1; 0 is treated as 1.
|
||||
let retry = retry.max(1);
|
||||
// The base is schedule[retry - 1], or the last entry past the end, or 0 for an empty schedule.
|
||||
let index = usize::try_from(retry - 1).unwrap_or(usize::MAX);
|
||||
let base = schedule
|
||||
.get(index)
|
||||
.copied()
|
||||
.or_else(|| schedule.last().copied())
|
||||
.unwrap_or(0);
|
||||
// Jitter is clamped to -1000..=1000 and moves the wait by up to a quarter of the base.
|
||||
let jitter = jitter.clamp(-1000, 1000);
|
||||
let value = i128::from(base) + i128::from(base / 4) * i128::from(jitter) / 1000;
|
||||
u64::try_from(value.clamp(0, i128::from(u64::MAX))).unwrap_or(u64::MAX)
|
||||
}
|
||||
|
||||
impl Client {
|
||||
pub fn chat_with_retry(
|
||||
&self,
|
||||
req: &ChatRequest,
|
||||
on_event: &mut dyn FnMut(&ChatEvent),
|
||||
) -> Result<Completion, InferError> {
|
||||
let start = Instant::now();
|
||||
let mut attempt: u32 = 1;
|
||||
loop {
|
||||
match self.chat(req, on_event) {
|
||||
Ok(completion) => return Ok(completion),
|
||||
Err(e) => {
|
||||
if !is_retryable(&e) || attempt >= self.cfg.limits.retry_attempts {
|
||||
return Err(e);
|
||||
}
|
||||
let wait = backoff_ms(&self.cfg.limits.retry_backoff_ms, attempt, now_jitter());
|
||||
if elapsed_ms(start) + wait > self.cfg.limits.retry_window_ms {
|
||||
return Err(e);
|
||||
}
|
||||
on_event(&ChatEvent::Retrying {
|
||||
attempt: attempt + 1,
|
||||
after_ms: wait,
|
||||
error: e.to_string(),
|
||||
});
|
||||
std::thread::sleep(Duration::from_millis(wait));
|
||||
attempt += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn now_jitter() -> i32 {
|
||||
let nanos = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.map(|since| since.subsec_nanos())
|
||||
.unwrap_or(0);
|
||||
// A nanosecond count reduced to -1000..=1000 is random enough for jitter.
|
||||
i32::try_from(nanos % 2000).unwrap_or(999) - 1000
|
||||
}
|
||||
|
||||
fn elapsed_ms(start: Instant) -> u64 {
|
||||
start.elapsed().as_millis().try_into().unwrap_or(u64::MAX)
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
//! Tests for the slot gate and for retrying. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::gate::{GateFull, SlotGate};
|
||||
use loopd::llama::retry::{backoff_ms, is_retryable};
|
||||
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client, InferError};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
fn request(slot: u32) -> ChatRequest {
|
||||
ChatRequest {
|
||||
slot,
|
||||
messages: vec![ChatMessage::User {
|
||||
content: "hi".to_string(),
|
||||
}],
|
||||
tools: vec![],
|
||||
thinking: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn io(kind: std::io::ErrorKind) -> std::io::Error {
|
||||
std::io::Error::from(kind)
|
||||
}
|
||||
|
||||
// ---- the gate on its own ----
|
||||
|
||||
#[test]
|
||||
fn a_free_slot_is_taken_at_once_and_slots_are_independent() {
|
||||
let gate = SlotGate::new();
|
||||
let mut queued = Vec::new();
|
||||
let a = gate.acquire(0, 8, &mut |n| queued.push(n)).unwrap();
|
||||
let b = gate.acquire(1, 8, &mut |n| queued.push(n)).unwrap();
|
||||
assert!(queued.is_empty(), "nobody had to wait");
|
||||
drop(a);
|
||||
drop(b);
|
||||
let _again = gate.acquire(0, 8, &mut |n| queued.push(n)).unwrap();
|
||||
assert!(queued.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiters_are_served_in_order_one_at_a_time() {
|
||||
let gate = Arc::new(SlotGate::new());
|
||||
let order = Arc::new(Mutex::new(Vec::new()));
|
||||
let inside = Arc::new(AtomicUsize::new(0));
|
||||
let first = gate.acquire(0, 8, &mut |_| {}).unwrap();
|
||||
let mut threads = Vec::new();
|
||||
for i in 0..4usize {
|
||||
let (gate, order, inside) = (Arc::clone(&gate), Arc::clone(&order), Arc::clone(&inside));
|
||||
threads.push(thread::spawn(move || {
|
||||
let mut ahead = None;
|
||||
let permit = gate.acquire(0, 8, &mut |n| ahead = Some(n)).unwrap();
|
||||
assert_eq!(
|
||||
inside.fetch_add(1, Ordering::SeqCst),
|
||||
0,
|
||||
"two permits for one slot at once"
|
||||
);
|
||||
order.lock().unwrap().push((i, ahead));
|
||||
thread::sleep(Duration::from_millis(20));
|
||||
inside.fetch_sub(1, Ordering::SeqCst);
|
||||
drop(permit);
|
||||
}));
|
||||
thread::sleep(Duration::from_millis(30)); // so that the arrival order is known
|
||||
}
|
||||
drop(first);
|
||||
for t in threads {
|
||||
t.join().unwrap();
|
||||
}
|
||||
// Each was told how many were ahead of it: the holder plus the earlier waiters.
|
||||
let want: Vec<(usize, Option<usize>)> = (0..4).map(|i| (i, Some(i + 1))).collect();
|
||||
assert_eq!(*order.lock().unwrap(), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_queue_refuses_at_once() {
|
||||
let gate = Arc::new(SlotGate::new());
|
||||
let _holder = gate.acquire(0, 1, &mut |_| {}).unwrap();
|
||||
let waiter = {
|
||||
let gate = Arc::clone(&gate);
|
||||
thread::spawn(move || drop(gate.acquire(0, 1, &mut |_| {}).unwrap()))
|
||||
};
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
let started = Instant::now();
|
||||
assert!(
|
||||
matches!(gate.acquire(0, 1, &mut |_| {}), Err(GateFull)),
|
||||
"one holder and one waiter is the limit"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_millis(50),
|
||||
"refusal must not wait"
|
||||
);
|
||||
assert!(
|
||||
gate.acquire(1, 1, &mut |_| {}).is_ok(),
|
||||
"another slot is unaffected"
|
||||
);
|
||||
drop(_holder);
|
||||
waiter.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_queue_of_zero_means_no_waiting_at_all() {
|
||||
let gate = SlotGate::new();
|
||||
let _holder = gate.acquire(0, 0, &mut |_| {}).unwrap();
|
||||
assert!(matches!(gate.acquire(0, 0, &mut |_| {}), Err(GateFull)));
|
||||
}
|
||||
|
||||
// ---- the gate inside the client ----
|
||||
|
||||
#[test]
|
||||
fn two_requests_for_one_slot_never_overlap_at_the_server() {
|
||||
let server = FakeServer::start();
|
||||
let size = support::fixture_bytes("http", "turn1.http").len();
|
||||
// Each reply takes about 200 ms to arrive.
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").trickle(size / 4, 50)]);
|
||||
let client = Arc::new(Client::new(support::test_config(&server.socket)));
|
||||
let started = Instant::now();
|
||||
let second = {
|
||||
let client = Arc::clone(&client);
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(30));
|
||||
let mut events = Vec::new();
|
||||
let done = client.chat(&request(0), &mut |e| events.push(e.clone()));
|
||||
(done.map(|d| d.content), events, Instant::now())
|
||||
})
|
||||
};
|
||||
let first = client.chat(&request(0), &mut |_| {}).unwrap();
|
||||
let first_done = Instant::now();
|
||||
let (second_result, second_events, second_done) = second.join().unwrap();
|
||||
assert_eq!(first.content.as_deref(), Some("Blue"));
|
||||
assert_eq!(second_result.unwrap().as_deref(), Some("Blue"));
|
||||
assert_eq!(second_events.first(), Some(&ChatEvent::Queued { ahead: 1 }));
|
||||
assert!(second_done > first_done);
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(280),
|
||||
"the two ran one after the other"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_queue_is_busy() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").head_delay(300)]);
|
||||
// Both slots report busy, so that the 300 ms of silence reads as a queue and not a stall.
|
||||
let busy = r#"[{"id":0,"is_processing":true},{"id":1,"is_processing":true}]"#;
|
||||
server.route("/slots", vec![Reply::json(200, busy)]);
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.limits.queue_len = 0;
|
||||
let client = Arc::new(Client::new(cfg));
|
||||
let holder = {
|
||||
let client = Arc::clone(&client);
|
||||
thread::spawn(move || client.chat(&request(0), &mut |_| {}).map(|_| ()))
|
||||
};
|
||||
thread::sleep(Duration::from_millis(80));
|
||||
assert!(matches!(
|
||||
client.chat(&request(0), &mut |_| {}),
|
||||
Err(InferError::Busy)
|
||||
));
|
||||
assert!(
|
||||
client.chat(&request(1), &mut |_| {}).is_ok(),
|
||||
"another slot is free"
|
||||
);
|
||||
holder.join().unwrap().unwrap();
|
||||
assert_eq!(
|
||||
server.requests_to(CHAT).len(),
|
||||
2,
|
||||
"the refused request was never sent"
|
||||
);
|
||||
}
|
||||
|
||||
// ---- retry ----
|
||||
|
||||
#[test]
|
||||
fn which_errors_are_retried() {
|
||||
use std::io::ErrorKind::ConnectionRefused;
|
||||
let http = |status| InferError::Http {
|
||||
status,
|
||||
body: String::new(),
|
||||
};
|
||||
let retried = [
|
||||
InferError::Connect(io(ConnectionRefused)),
|
||||
InferError::StreamClosedEarly,
|
||||
InferError::Stalled,
|
||||
InferError::LoadTimeout,
|
||||
http(503),
|
||||
];
|
||||
for e in &retried {
|
||||
assert!(
|
||||
is_retryable(e),
|
||||
"{e:?} means the server went away and may be back"
|
||||
);
|
||||
}
|
||||
let not_retried = [
|
||||
InferError::Busy,
|
||||
InferError::WaitTimeout,
|
||||
InferError::ThinkingOverrun,
|
||||
InferError::Protocol("x".to_string()),
|
||||
http(400),
|
||||
http(404),
|
||||
http(500),
|
||||
];
|
||||
for e in ¬_retried {
|
||||
assert!(
|
||||
!is_retryable(e),
|
||||
"{e:?} would fail again, or has already waited"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_follows_the_schedule_with_a_quarter_of_jitter() {
|
||||
let schedule = [2_000, 8_000, 30_000];
|
||||
assert_eq!(backoff_ms(&schedule, 1, 0), 2_000);
|
||||
assert_eq!(backoff_ms(&schedule, 2, 0), 8_000);
|
||||
assert_eq!(backoff_ms(&schedule, 3, 0), 30_000);
|
||||
assert_eq!(
|
||||
backoff_ms(&schedule, 4, 0),
|
||||
30_000,
|
||||
"past the end, the last entry"
|
||||
);
|
||||
assert_eq!(
|
||||
backoff_ms(&schedule, 0, 0),
|
||||
2_000,
|
||||
"retry numbers start at 1; 0 is treated as 1"
|
||||
);
|
||||
assert_eq!(backoff_ms(&schedule, 1, 1000), 2_500);
|
||||
assert_eq!(backoff_ms(&schedule, 1, -1000), 1_500);
|
||||
assert_eq!(backoff_ms(&schedule, 2, 500), 9_000);
|
||||
assert_eq!(
|
||||
backoff_ms(&schedule, 1, 99_999),
|
||||
2_500,
|
||||
"jitter is clamped to -1000..=1000"
|
||||
);
|
||||
assert_eq!(backoff_ms(&schedule, 1, i32::MIN), 1_500);
|
||||
assert_eq!(backoff_ms(&[], 1, 1000), 0, "no schedule, no wait");
|
||||
// Huge values must not overflow or panic.
|
||||
let _ = backoff_ms(&[u64::MAX], 1, 1000);
|
||||
let _ = backoff_ms(&[u64::MAX], u32::MAX, -1000);
|
||||
}
|
||||
|
||||
fn run_retry(
|
||||
server: &FakeServer,
|
||||
attempts: u32,
|
||||
window_ms: u64,
|
||||
) -> (Result<Option<String>, InferError>, Vec<ChatEvent>) {
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.limits.retry_attempts = attempts;
|
||||
cfg.limits.retry_window_ms = window_ms;
|
||||
let mut events = Vec::new();
|
||||
let result = Client::new(cfg).chat_with_retry(&request(0), &mut |e| events.push(e.clone()));
|
||||
(result.map(|d| d.content), events)
|
||||
}
|
||||
|
||||
fn retrying(events: &[ChatEvent]) -> Vec<(u32, u64)> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
ChatEvent::Retrying {
|
||||
attempt, after_ms, ..
|
||||
} => Some((*attempt, *after_ms)),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_that_comes_back_is_survived() {
|
||||
let server = FakeServer::start();
|
||||
let dead = Reply::fixture("turn1").cut_after(400);
|
||||
server.route(CHAT, vec![dead.clone(), dead, Reply::fixture("turn1")]);
|
||||
let (result, events) = run_retry(&server, 4, 5_000);
|
||||
assert_eq!(result.unwrap().as_deref(), Some("Blue"));
|
||||
let retries = retrying(&events);
|
||||
assert_eq!(
|
||||
retries
|
||||
.iter()
|
||||
.map(|(attempt, _)| *attempt)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![2, 3]
|
||||
);
|
||||
// The test schedule is 10, 20, 30 ms, each moved by at most a quarter.
|
||||
assert!((8..=12).contains(&retries[0].1), "{retries:?}");
|
||||
assert!((15..=25).contains(&retries[1].1), "{retries:?}");
|
||||
let sent = server.requests_to(CHAT);
|
||||
assert_eq!(sent.len(), 3);
|
||||
assert!(
|
||||
sent.iter().all(|r| r.body == sent[0].body),
|
||||
"a retry sends the same bytes again"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retries_stop_at_the_attempt_limit() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").cut_after(400)]);
|
||||
let (result, events) = run_retry(&server, 3, 5_000);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::StreamClosedEarly)),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(server.requests_to(CHAT).len(), 3, "three attempts in all");
|
||||
assert_eq!(retrying(&events).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retries_stop_at_the_time_window() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").cut_after(400)]);
|
||||
// The first retry would wait about 10 ms, which does not fit in a 5 ms window.
|
||||
let (result, events) = run_retry(&server, 10, 5);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::StreamClosedEarly)),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(server.requests_to(CHAT).len(), 1);
|
||||
assert!(retrying(&events).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_that_would_fail_again_are_not_retried() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("bad_request")]);
|
||||
let (result, events) = run_retry(&server, 4, 5_000);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::Http { status: 400, .. })),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(server.requests_to(CHAT).len(), 1);
|
||||
assert!(retrying(&events).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_loading_server_is_retried() {
|
||||
let server = FakeServer::start();
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::json(503, r#"{"error":{"message":"Loading model"}}"#),
|
||||
Reply::fixture("turn1"),
|
||||
],
|
||||
);
|
||||
let (result, events) = run_retry(&server, 4, 5_000);
|
||||
assert_eq!(result.unwrap().as_deref(), Some("Blue"));
|
||||
let ChatEvent::Retrying { error, .. } = events
|
||||
.iter()
|
||||
.find(|e| matches!(e, ChatEvent::Retrying { .. }))
|
||||
.unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert!(error.contains("503"), "the event says why: {error}");
|
||||
}
|
||||
Reference in New Issue
Block a user