//! Tests for the turn limits and for the append-only property over generated conversations. //! Do not edit. mod support; #[path = "support/turn.rs"] mod turn_support; use loopd::session::Session; use loopd::turn::TurnError; use proto::{LogRecord, SessionId, TurnEvent}; use support::{Reply, ScriptedPort, ok_result}; use turn_support::{setup, types, without_cache_loss}; const CHAT: &str = "/v1/chat/completions"; #[test] fn too_many_tool_iterations_end_the_turn() { let mut s = setup(vec![]); s.cfg.r#loop.tool_iterations = 2; s.cfg.r#loop.repeat_detection = false; s.server.route(CHAT, vec![Reply::fixture("tool_call")]); let mut session = s.session("a"); let (result, _) = s.turn(&mut session, "x"); assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}"); // Two iterations ran their tool; the third completion was recorded and then stopped. assert_eq!( types(session.records()), [ "start", "user", "assistant", "usage", "tool_result", "assistant", "usage", "tool_result", "assistant", "usage" ] ); assert_eq!(s.port.calls().len(), 2); assert_eq!(s.server.requests_to(CHAT).len(), 3); // The session is still usable: the log ends at a record boundary and a new turn works. s.server.route(CHAT, vec![Reply::fixture("plain")]); assert!(s.turn(&mut session, "again").0.is_ok()); } #[test] fn a_repeated_identical_call_is_not_run_and_a_second_repeat_ends_the_turn() { let s = setup(vec![]); s.server.route(CHAT, vec![Reply::fixture("tool_call")]); let mut session = s.session("a"); let (result, _) = s.turn(&mut session, "x"); assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}"); assert_eq!( s.port.calls().len(), 1, "the first call ran; the repeat did not" ); match &without_cache_loss(session.records())[7] { LogRecord::ToolResult { content, .. } => { assert!(content.contains("already called"), "{content}") } other => panic!("{other:?}"), } assert_eq!( s.server.requests_to(CHAT).len(), 3, "one more completion after the first repeat, then stop" ); } #[test] fn repeat_detection_can_be_turned_off() { let mut s = setup(vec![]); s.cfg.r#loop.repeat_detection = false; s.server.route( CHAT, vec![ Reply::fixture("tool_call"), Reply::fixture("tool_call"), Reply::fixture("plain"), ], ); let mut session = s.session("a"); assert!(s.turn(&mut session, "x").0.is_ok()); assert_eq!(s.port.calls().len(), 2); } #[test] fn a_full_context_ends_the_turn_with_nothing_appended_for_the_request() { let s = setup(vec![]); s.server.route(CHAT, vec![Reply::fixture("context_full")]); let mut session = s.session("a"); let (result, _) = s.turn(&mut session, "x"); assert!(matches!(result, Err(TurnError::SessionFull)), "{result:?}"); assert_eq!(types(session.records()), ["start", "user"]); let e: Box = Box::new(result.unwrap_err()); assert!(e.to_string().contains("full"), "{e}"); } #[test] fn other_server_errors_are_inference_errors() { let s = setup(vec![]); s.server.route(CHAT, vec![Reply::fixture("bad_request")]); let mut session = s.session("a"); let (result, _) = s.turn(&mut session, "x"); assert!( matches!( result, Err(TurnError::Infer(loopd::llama::InferError::Http { status: 400, .. })) ), "{result:?}" ); } #[test] fn a_cache_loss_is_recorded_and_reported() { let s = setup(vec![]); // progress leaves 15 + 7029 + 8 = 7052 tokens in the slot; plain reports 0 reused. s.server.route( CHAT, vec![Reply::fixture("progress"), Reply::fixture("plain")], ); let mut session = s.session("a"); assert!(s.turn(&mut session, "one").0.is_ok()); let (result, events) = s.turn(&mut session, "two"); assert!(result.is_ok()); assert_eq!(session.records().len(), 8); assert!( matches!( session.records()[7], LogRecord::CacheLoss { expected: 7052, got: 0, .. } ), "{:?}", session.records()[7] ); assert!(events.contains(&TurnEvent::CacheLoss { expected: 7052, got: 0 })); // turn1 then turn2 are a real consecutive pair: 15 + 29 + 2 = 46 left, 45 reused. No loss. let s = setup(vec![]); s.server .route(CHAT, vec![Reply::fixture("turn1"), Reply::fixture("turn2")]); let mut session = s.session("b"); assert!(s.turn(&mut session, "one").0.is_ok()); let (result, events) = s.turn(&mut session, "two"); assert!(result.is_ok()); assert!( !session .records() .iter() .any(|r| matches!(r, LogRecord::CacheLoss { .. })) ); assert!( !events .iter() .any(|e| matches!(e, TurnEvent::CacheLoss { .. })) ); } #[test] fn retries_are_reported_and_leave_no_trace_in_the_log() { let s = setup(vec![]); s.server.route( CHAT, vec![ Reply::fixture("plain").cut_after(400), Reply::fixture("plain"), ], ); let mut session = s.session("a"); let (result, events) = s.turn(&mut session, "x"); assert!(result.is_ok(), "{result:?}"); assert!( events .iter() .any(|e| matches!(e, TurnEvent::Retrying { attempt: 2, .. })) ); assert_eq!( types(session.records()), ["start", "user", "assistant", "usage"] ); } #[test] fn resumed_sessions_continue_the_same_conversation() { let s = setup(vec![]); s.server .route(CHAT, vec![Reply::fixture("plain"), Reply::fixture("turn2")]); let mut session = s.session("a"); assert!(s.turn(&mut session, "one").0.is_ok()); drop(session); let mut session = Session::open(&s.home.dir, SessionId::new("a").unwrap()).unwrap(); assert!(s.turn(&mut session, "two").0.is_ok()); let sent = s.server.requests_to(CHAT); let m1 = sent[0].json()["messages"].as_array().unwrap().clone(); let m2 = sent[1].json()["messages"].as_array().unwrap().clone(); assert_eq!( m2[..m1.len()], m1[..], "the second request extends the first" ); assert_eq!(m2.len(), m1.len() + 2); } /// A small xorshift, so that the test needs no crate and a failure can be replayed by seed. struct Rng(u64); impl Rng { fn next(&mut self) -> u64 { self.0 ^= self.0 << 13; self.0 ^= self.0 >> 7; self.0 ^= self.0 << 17; self.0 } fn below(&mut self, n: u64) -> u64 { self.next() % n } } /// Every request of a generated conversation is a strict extension of the one before, and the /// baseline (first message and tools) never changes. #[test] fn every_request_extends_the_previous_one() { for seed in [1u64, 2, 3, 0xdead_beef, 0x9e37_79b9_7f4a_7c15] { let mut rng = Rng(seed); let mut s = setup(vec![]); s.cfg.r#loop.repeat_detection = false; s.cfg.r#loop.tool_result_cap = 64; let mut replies = Vec::new(); let mut port_replies = Vec::new(); let turns = 2 + rng.below(4); for _ in 0..turns { let iterations = rng.below(3); for _ in 0..iterations { if rng.below(2) == 0 { replies.push(Reply::fixture("tool_call")); } else { replies.push(Reply::fixture("find_tool")); replies.push(Reply::fixture("call_tool")); } let size = rng.below(120) as usize; port_replies.push(ok_result(&"r".repeat(size))); port_replies.push(ok_result(&"r".repeat(size))); } replies.push(Reply::fixture( ["plain", "turn1", "turn2"][rng.below(3) as usize], )); } s.port = ScriptedPort::new(port_replies); s.server.route(CHAT, replies); let mut session = s.session("a"); for t in 0..turns { let text = format!("turn {t} {}", "u".repeat(rng.below(30) as usize)); let (result, _) = s.turn(&mut session, &text); assert!(result.is_ok(), "seed {seed}, turn {t}: {result:?}"); } let sent = s.server.requests_to(CHAT); assert!(sent.len() >= turns as usize, "seed {seed}"); let first = sent[0].json(); for (i, pair) in sent.windows(2).enumerate() { let a = pair[0].json(); let b = pair[1].json(); let ma = a["messages"].as_array().unwrap(); let mb = b["messages"].as_array().unwrap(); assert!( mb.len() > ma.len(), "seed {seed}, request {}: not longer", i + 1 ); assert_eq!( mb[..ma.len()], ma[..], "seed {seed}, request {}: not an extension", i + 1 ); assert_eq!(b["tools"], first["tools"], "seed {seed}: tools changed"); assert_eq!( b["messages"][0], first["messages"][0], "seed {seed}: system message changed" ); } // And the log replays to exactly the last request. let replayed = loopd::baseline::messages(session.baseline(), session.records()); let last = sent.last().unwrap().json(); assert_eq!( replayed.len(), last["messages"].as_array().unwrap().len() + 1, "seed {seed}: the last request plus the final answer" ); } }