Add the turn loop with its limits
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -0,0 +1,303 @@
|
||||
//! Tests for the turn limits and for the append-only property over generated conversations.
|
||||
//! Do not edit.
|
||||
|
||||
mod support;
|
||||
#[path = "support/turn.rs"]
|
||||
mod turn_support;
|
||||
|
||||
use loopd::session::Session;
|
||||
use loopd::turn::TurnError;
|
||||
use proto::{LogRecord, SessionId, TurnEvent};
|
||||
use support::{Reply, ScriptedPort, ok_result};
|
||||
use turn_support::{setup, types, without_cache_loss};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
#[test]
|
||||
fn too_many_tool_iterations_end_the_turn() {
|
||||
let mut s = setup(vec![]);
|
||||
s.cfg.r#loop.tool_iterations = 2;
|
||||
s.cfg.r#loop.repeat_detection = false;
|
||||
s.server.route(CHAT, vec![Reply::fixture("tool_call")]);
|
||||
let mut session = s.session("a");
|
||||
let (result, _) = s.turn(&mut session, "x");
|
||||
assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}");
|
||||
// Two iterations ran their tool; the third completion was recorded and then stopped.
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
[
|
||||
"start",
|
||||
"user",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage",
|
||||
"tool_result",
|
||||
"assistant",
|
||||
"usage"
|
||||
]
|
||||
);
|
||||
assert_eq!(s.port.calls().len(), 2);
|
||||
assert_eq!(s.server.requests_to(CHAT).len(), 3);
|
||||
// The session is still usable: the log ends at a record boundary and a new turn works.
|
||||
s.server.route(CHAT, vec![Reply::fixture("plain")]);
|
||||
assert!(s.turn(&mut session, "again").0.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_repeated_identical_call_is_not_run_and_a_second_repeat_ends_the_turn() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(CHAT, vec![Reply::fixture("tool_call")]);
|
||||
let mut session = s.session("a");
|
||||
let (result, _) = s.turn(&mut session, "x");
|
||||
assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}");
|
||||
assert_eq!(
|
||||
s.port.calls().len(),
|
||||
1,
|
||||
"the first call ran; the repeat did not"
|
||||
);
|
||||
match &without_cache_loss(session.records())[7] {
|
||||
LogRecord::ToolResult { content, .. } => {
|
||||
assert!(content.contains("already called"), "{content}")
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
s.server.requests_to(CHAT).len(),
|
||||
3,
|
||||
"one more completion after the first repeat, then stop"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn repeat_detection_can_be_turned_off() {
|
||||
let mut s = setup(vec![]);
|
||||
s.cfg.r#loop.repeat_detection = false;
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("plain"),
|
||||
],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
assert!(s.turn(&mut session, "x").0.is_ok());
|
||||
assert_eq!(s.port.calls().len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_context_ends_the_turn_with_nothing_appended_for_the_request() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(CHAT, vec![Reply::fixture("context_full")]);
|
||||
let mut session = s.session("a");
|
||||
let (result, _) = s.turn(&mut session, "x");
|
||||
assert!(matches!(result, Err(TurnError::SessionFull)), "{result:?}");
|
||||
assert_eq!(types(session.records()), ["start", "user"]);
|
||||
let e: Box<dyn std::error::Error> = Box::new(result.unwrap_err());
|
||||
assert!(e.to_string().contains("full"), "{e}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn other_server_errors_are_inference_errors() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(CHAT, vec![Reply::fixture("bad_request")]);
|
||||
let mut session = s.session("a");
|
||||
let (result, _) = s.turn(&mut session, "x");
|
||||
assert!(
|
||||
matches!(
|
||||
result,
|
||||
Err(TurnError::Infer(loopd::llama::InferError::Http {
|
||||
status: 400,
|
||||
..
|
||||
}))
|
||||
),
|
||||
"{result:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cache_loss_is_recorded_and_reported() {
|
||||
let s = setup(vec![]);
|
||||
// progress leaves 15 + 7029 + 8 = 7052 tokens in the slot; plain reports 0 reused.
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![Reply::fixture("progress"), Reply::fixture("plain")],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
assert!(s.turn(&mut session, "one").0.is_ok());
|
||||
let (result, events) = s.turn(&mut session, "two");
|
||||
assert!(result.is_ok());
|
||||
assert_eq!(session.records().len(), 8);
|
||||
assert!(
|
||||
matches!(
|
||||
session.records()[7],
|
||||
LogRecord::CacheLoss {
|
||||
expected: 7052,
|
||||
got: 0,
|
||||
..
|
||||
}
|
||||
),
|
||||
"{:?}",
|
||||
session.records()[7]
|
||||
);
|
||||
assert!(events.contains(&TurnEvent::CacheLoss {
|
||||
expected: 7052,
|
||||
got: 0
|
||||
}));
|
||||
|
||||
// turn1 then turn2 are a real consecutive pair: 15 + 29 + 2 = 46 left, 45 reused. No loss.
|
||||
let s = setup(vec![]);
|
||||
s.server
|
||||
.route(CHAT, vec![Reply::fixture("turn1"), Reply::fixture("turn2")]);
|
||||
let mut session = s.session("b");
|
||||
assert!(s.turn(&mut session, "one").0.is_ok());
|
||||
let (result, events) = s.turn(&mut session, "two");
|
||||
assert!(result.is_ok());
|
||||
assert!(
|
||||
!session
|
||||
.records()
|
||||
.iter()
|
||||
.any(|r| matches!(r, LogRecord::CacheLoss { .. }))
|
||||
);
|
||||
assert!(
|
||||
!events
|
||||
.iter()
|
||||
.any(|e| matches!(e, TurnEvent::CacheLoss { .. }))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retries_are_reported_and_leave_no_trace_in_the_log() {
|
||||
let s = setup(vec![]);
|
||||
s.server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::fixture("plain").cut_after(400),
|
||||
Reply::fixture("plain"),
|
||||
],
|
||||
);
|
||||
let mut session = s.session("a");
|
||||
let (result, events) = s.turn(&mut session, "x");
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert!(
|
||||
events
|
||||
.iter()
|
||||
.any(|e| matches!(e, TurnEvent::Retrying { attempt: 2, .. }))
|
||||
);
|
||||
assert_eq!(
|
||||
types(session.records()),
|
||||
["start", "user", "assistant", "usage"]
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn resumed_sessions_continue_the_same_conversation() {
|
||||
let s = setup(vec![]);
|
||||
s.server
|
||||
.route(CHAT, vec![Reply::fixture("plain"), Reply::fixture("turn2")]);
|
||||
let mut session = s.session("a");
|
||||
assert!(s.turn(&mut session, "one").0.is_ok());
|
||||
drop(session);
|
||||
let mut session = Session::open(&s.home.dir, SessionId::new("a").unwrap()).unwrap();
|
||||
assert!(s.turn(&mut session, "two").0.is_ok());
|
||||
let sent = s.server.requests_to(CHAT);
|
||||
let m1 = sent[0].json()["messages"].as_array().unwrap().clone();
|
||||
let m2 = sent[1].json()["messages"].as_array().unwrap().clone();
|
||||
assert_eq!(
|
||||
m2[..m1.len()],
|
||||
m1[..],
|
||||
"the second request extends the first"
|
||||
);
|
||||
assert_eq!(m2.len(), m1.len() + 2);
|
||||
}
|
||||
|
||||
/// A small xorshift, so that the test needs no crate and a failure can be replayed by seed.
|
||||
struct Rng(u64);
|
||||
|
||||
impl Rng {
|
||||
fn next(&mut self) -> u64 {
|
||||
self.0 ^= self.0 << 13;
|
||||
self.0 ^= self.0 >> 7;
|
||||
self.0 ^= self.0 << 17;
|
||||
self.0
|
||||
}
|
||||
|
||||
fn below(&mut self, n: u64) -> u64 {
|
||||
self.next() % n
|
||||
}
|
||||
}
|
||||
|
||||
/// Every request of a generated conversation is a strict extension of the one before, and the
|
||||
/// baseline (first message and tools) never changes.
|
||||
#[test]
|
||||
fn every_request_extends_the_previous_one() {
|
||||
for seed in [1u64, 2, 3, 0xdead_beef, 0x9e37_79b9_7f4a_7c15] {
|
||||
let mut rng = Rng(seed);
|
||||
let mut s = setup(vec![]);
|
||||
s.cfg.r#loop.repeat_detection = false;
|
||||
s.cfg.r#loop.tool_result_cap = 64;
|
||||
let mut replies = Vec::new();
|
||||
let mut port_replies = Vec::new();
|
||||
let turns = 2 + rng.below(4);
|
||||
for _ in 0..turns {
|
||||
let iterations = rng.below(3);
|
||||
for _ in 0..iterations {
|
||||
if rng.below(2) == 0 {
|
||||
replies.push(Reply::fixture("tool_call"));
|
||||
} else {
|
||||
replies.push(Reply::fixture("find_tool"));
|
||||
replies.push(Reply::fixture("call_tool"));
|
||||
}
|
||||
let size = rng.below(120) as usize;
|
||||
port_replies.push(ok_result(&"r".repeat(size)));
|
||||
port_replies.push(ok_result(&"r".repeat(size)));
|
||||
}
|
||||
replies.push(Reply::fixture(
|
||||
["plain", "turn1", "turn2"][rng.below(3) as usize],
|
||||
));
|
||||
}
|
||||
s.port = ScriptedPort::new(port_replies);
|
||||
s.server.route(CHAT, replies);
|
||||
let mut session = s.session("a");
|
||||
for t in 0..turns {
|
||||
let text = format!("turn {t} {}", "u".repeat(rng.below(30) as usize));
|
||||
let (result, _) = s.turn(&mut session, &text);
|
||||
assert!(result.is_ok(), "seed {seed}, turn {t}: {result:?}");
|
||||
}
|
||||
let sent = s.server.requests_to(CHAT);
|
||||
assert!(sent.len() >= turns as usize, "seed {seed}");
|
||||
let first = sent[0].json();
|
||||
for (i, pair) in sent.windows(2).enumerate() {
|
||||
let a = pair[0].json();
|
||||
let b = pair[1].json();
|
||||
let ma = a["messages"].as_array().unwrap();
|
||||
let mb = b["messages"].as_array().unwrap();
|
||||
assert!(
|
||||
mb.len() > ma.len(),
|
||||
"seed {seed}, request {}: not longer",
|
||||
i + 1
|
||||
);
|
||||
assert_eq!(
|
||||
mb[..ma.len()],
|
||||
ma[..],
|
||||
"seed {seed}, request {}: not an extension",
|
||||
i + 1
|
||||
);
|
||||
assert_eq!(b["tools"], first["tools"], "seed {seed}: tools changed");
|
||||
assert_eq!(
|
||||
b["messages"][0], first["messages"][0],
|
||||
"seed {seed}: system message changed"
|
||||
);
|
||||
}
|
||||
// And the log replays to exactly the last request.
|
||||
let replayed = loopd::baseline::messages(session.baseline(), session.records());
|
||||
let last = sent.last().unwrap().json();
|
||||
assert_eq!(
|
||||
replayed.len(),
|
||||
last["messages"].as_array().unwrap().len() + 1,
|
||||
"seed {seed}: the last request plus the final answer"
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user