diff --git a/crates/loopd/src/lib.rs b/crates/loopd/src/lib.rs index b8f7bc0..5c74b73 100644 --- a/crates/loopd/src/lib.rs +++ b/crates/loopd/src/lib.rs @@ -8,3 +8,4 @@ pub mod selftest; pub mod session; pub mod sse; pub mod tools; +pub mod turn; diff --git a/crates/loopd/src/turn.rs b/crates/loopd/src/turn.rs new file mode 100644 index 0000000..a12d1b2 --- /dev/null +++ b/crates/loopd/src/turn.rs @@ -0,0 +1,299 @@ +//! One turn: ask, run any tool calls under their limits, and return. +//! The loop is bounded every way a runaway can go: a cap on tool iterations, a repeated-call +//! detector, the two M2a limits (thinking tokens, a full context), and a tool-result cap. + +use crate::baseline::messages; +use crate::config::Config; +use crate::llama::info::{CacheOutcome, cache_outcome}; +use crate::llama::{ChatEvent, ChatRequest, Client, InferError}; +use crate::session::{Session, SessionError}; +use crate::tools::{Registry, ToolPort, cap_result, dispatch}; +use proto::{CallId, DataClass, LogRecord, Timestamp, ToolRequest, ToolResponse, TurnEvent, Usage}; + +/// What one turn can fail with, beyond the inference server. +#[derive(Debug)] +pub enum TurnError { + SessionFull, + TurnLimit, + Infer(InferError), + Session(SessionError), +} + +impl std::fmt::Display for TurnError { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match self { + TurnError::SessionFull => write!(f, "the context is full"), + TurnError::TurnLimit => write!(f, "the turn limit was reached"), + TurnError::Infer(e) => write!(f, "{e}"), + TurnError::Session(e) => write!(f, "{e}"), + } + } +} + +impl std::error::Error for TurnError {} + +impl From for TurnError { + fn from(e: SessionError) -> Self { + TurnError::Session(e) + } +} + +/// The text to show and what the turn cost. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct TurnOutcome { + pub content: String, + pub usage: Usage, +} + +/// What a turn needs besides the session. +pub struct Runtime<'a> { + pub cfg: &'a Config, + pub client: &'a Client, + pub port: &'a dyn ToolPort, + pub registry: &'a Registry, +} + +/// True for the one 400 the server sends when the prompt does not fit. +pub fn is_context_full(error: &InferError) -> bool { + match error { + InferError::Http { status: 400, body } => { + let Ok(value) = serde_json::from_str::(body) else { + return false; + }; + matches!( + value + .get("error") + .and_then(|e| e.get("type")) + .and_then(|t| t.as_str()), + Some("exceed_context_size_error") + ) + } + _ => false, + } +} + +fn map_event(event: &ChatEvent, on_event: &mut dyn FnMut(&TurnEvent)) { + match event { + ChatEvent::Queued { ahead } => on_event(&TurnEvent::Queued { + ahead: u64::try_from(*ahead).unwrap_or(u64::MAX), + }), + ChatEvent::Waiting { slot_busy } => on_event(&TurnEvent::Waiting { + slot_busy: *slot_busy, + }), + ChatEvent::Progress { + total, + cache, + processed, + } => on_event(&TurnEvent::Progress { + total: *total, + cache: *cache, + processed: *processed, + }), + ChatEvent::Reasoning(text) => on_event(&TurnEvent::Reasoning { text: text.clone() }), + ChatEvent::Content(text) => on_event(&TurnEvent::Content { text: text.clone() }), + ChatEvent::ThinkingCapped { tokens } => { + on_event(&TurnEvent::ThinkingCapped { tokens: *tokens }) + } + ChatEvent::Retrying { + attempt, + after_ms, + error, + } => on_event(&TurnEvent::Retrying { + attempt: *attempt, + after_ms: *after_ms, + error: error.clone(), + }), + ChatEvent::ToolCallDelta { .. } => {} + } +} + +pub fn run_turn( + session: &mut Session, + rt: &Runtime<'_>, + content: &str, + on_event: &mut dyn FnMut(&TurnEvent), +) -> Result { + session + .append(LogRecord::User { + time: Timestamp::now(), + content: content.to_string(), + }) + .map_err(TurnError::Session)?; + + let mut seen: Vec<(String, String)> = Vec::new(); + let mut iterations: u32 = 0; + + loop { + let baseline = session.baseline(); + let request = ChatRequest { + slot: rt.cfg.slots.main, + messages: messages(baseline, session.records()), + tools: baseline.tools.clone(), + thinking: true, + }; + let previous = session.last_usage(); + + let completion = match rt + .client + .chat_with_retry(&request, &mut |event| map_event(event, on_event)) + { + Ok(completion) => completion, + Err(e) => { + if is_context_full(&e) { + return Err(TurnError::SessionFull); + } + return Err(TurnError::Infer(e)); + } + }; + + session + .append(LogRecord::Assistant { + time: Timestamp::now(), + content: completion.content.clone(), + reasoning_content: completion.reasoning_content.clone(), + tool_calls: completion.tool_calls.clone(), + }) + .map_err(TurnError::Session)?; + + let usage = Usage { + cache_n: completion.timings.cache_n, + prompt_n: completion.timings.prompt_n, + predicted_n: completion.timings.predicted_n, + reasoning_tokens: completion.reasoning_tokens, + thinking_capped: completion.thinking_capped, + }; + session + .append(LogRecord::Usage { + time: Timestamp::now(), + cache_n: usage.cache_n, + prompt_n: usage.prompt_n, + predicted_n: usage.predicted_n, + reasoning_tokens: usage.reasoning_tokens, + thinking_capped: usage.thinking_capped, + }) + .map_err(TurnError::Session)?; + + // The recordings were made in separate conversations, so their cache numbers do not line up. + if let Some(previous) = previous { + let previous = crate::llama::Timings { + cache_n: previous.cache_n, + prompt_n: previous.prompt_n, + predicted_n: previous.predicted_n, + }; + if let CacheOutcome::Loss { expected, got } = + cache_outcome(&previous, &completion.timings) + { + session + .append(LogRecord::CacheLoss { + time: Timestamp::now(), + expected, + got, + }) + .map_err(TurnError::Session)?; + on_event(&TurnEvent::CacheLoss { expected, got }); + } + } + + if completion.tool_calls.is_empty() { + return Ok(TurnOutcome { + content: completion.content.clone().unwrap_or_default(), + usage, + }); + } + + iterations += 1; + if iterations > rt.cfg.r#loop.tool_iterations { + return Err(TurnError::TurnLimit); + } + + for call in &completion.tool_calls { + on_event(&TurnEvent::ToolCallStarted { + name: call.name.clone(), + }); + let call_id = session.next_call(); + + // A repeated identical (name, arguments) pair is not run the first time; a second repeat stops the turn. + let key = (call.name.clone(), call.arguments.clone()); + let times = seen.iter().filter(|pair| **pair == key).count(); + if rt.cfg.r#loop.repeat_detection && times >= 2 { + return Err(TurnError::TurnLimit); + } + seen.push(key); + + let (text, class, untrusted) = if rt.cfg.r#loop.repeat_detection && times == 1 { + ( + format!( + "The `{}` tool call was already called with these arguments this turn.", + call.name + ), + DataClass::Public, + false, + ) + } else { + run_call(rt, session, call, call_id) + }; + + let (text, truncated) = cap_result(&text, rt.cfg.r#loop.tool_result_cap); + session + .append(LogRecord::ToolResult { + time: Timestamp::now(), + call: call_id, + tool_call_id: call.id.clone(), + content: text, + class, + untrusted, + truncated, + }) + .map_err(TurnError::Session)?; + on_event(&TurnEvent::ToolResult { + name: call.name.clone(), + class, + truncated, + }); + } + } +} + +/// Resolves one tool call to the text, class and trust flag of its result. +fn run_call( + rt: &Runtime<'_>, + session: &Session, + call: &proto::ToolCall, + call_id: CallId, +) -> (String, DataClass, bool) { + match dispatch(rt.registry, &call.name, &call.arguments) { + crate::tools::Dispatch::Local(text) => (text, DataClass::Public, false), + crate::tools::Dispatch::Port { tool, arguments } => { + let request = ToolRequest { + session: session.id().clone(), + call: call_id, + tool, + arguments, + }; + match rt.port.call(&request) { + ToolResponse::Result { + content, + class, + untrusted, + .. + } => (content, class, untrusted), + ToolResponse::Failed { message } => ( + format!("The tool failed: {message}"), + DataClass::Public, + false, + ), + ToolResponse::Denied { reason } => ( + format!("The call was denied: {reason:?}"), + DataClass::Public, + false, + ), + ToolResponse::PendingApproval { .. } => ( + "This version of loopd cannot wait for approval; the call was rejected." + .to_string(), + DataClass::Public, + false, + ), + } + } + } +} diff --git a/crates/loopd/tests/fixtures/http/call_tool.http b/crates/loopd/tests/fixtures/http/call_tool.http new file mode 100644 index 0000000..dadc46e --- /dev/null +++ b/crates/loopd/tests/fixtures/http/call_tool.http @@ -0,0 +1,83 @@ +HTTP/1.1 200 OK +Server: llama.cpp +Access-Control-Allow-Origin: +Connection: close +X-Accel-Buffering: no +Content-Type: text/event-stream +Transfer-Encoding: chunked +Connection: close + +218 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":657,"cache":582,"processed":582,"time_ms":0}} + + +23c +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":71,"prompt_ms":527.742,"prompt_per_token_ms":7.432985915492957,"prompt_per_second":134.53543587586358,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":657,"cache":582,"processed":653,"time_ms":527}} + + +23b +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.13,"prompt_per_token_ms":11.655066666666666,"prompt_per_second":85.79959502591149,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":657,"cache":582,"processed":657,"time_ms":874}} + + +1ea +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}} + + +25e +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"id":"3E643m2jnS3ahaAM6cU6YyRjVa4kOtBV","type":"function","function":{"name":"call_tool","arguments":"{"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":9,"predicted_ms":113.464,"predicted_per_token_ms":14.183,"predicted_per_second":70.506944934076}} + + +22a +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"name\":\""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":14,"predicted_ms":182.903,"predicted_per_token_ms":14.069461538461537,"predicted_per_second":71.07592549056056}} + + +222 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"echo"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":15,"predicted_ms":196.873,"predicted_per_token_ms":14.062357142857142,"predicted_per_second":71.1118335170389}} + + +220 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":20,"predicted_ms":266.06,"predicted_per_token_ms":14.003157894736843,"predicted_per_second":71.41246335413065}} + + +223 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":",\"arguments\":"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":26,"predicted_ms":349.491,"predicted_per_token_ms":13.97964,"predicted_per_second":71.5326002672458}} + + +222 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":27,"predicted_ms":363.312,"predicted_per_token_ms":13.973538461538462,"predicted_per_second":71.56383494076717}} + + +223 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"text"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":28,"predicted_ms":377.136,"predicted_per_token_ms":13.968000000000002,"predicted_per_second":71.59221076746849}} + + +217 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":29,"predicted_ms":391.062,"predicted_per_token_ms":13.9665,"predicted_per_second":71.59989976014033}} + + +221 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":30,"predicted_ms":404.922,"predicted_per_token_ms":13.962827586206897,"predicted_per_second":71.6187315087844}} + + +222 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"box"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":31,"predicted_ms":418.766,"predicted_per_token_ms":13.958866666666667,"predicted_per_second":71.63905379137752}} + + +222 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":32,"predicted_ms":432.558,"predicted_per_token_ms":13.953483870967741,"predicted_per_second":71.66668978495369}} + + +220 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":41,"predicted_ms":556.994,"predicted_per_token_ms":13.924850000000001,"predicted_per_second":71.81405903833793}} + + +1f1 +data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":43,"predicted_ms":584.671,"predicted_per_token_ms":13.920738095238097,"predicted_per_second":71.83527146035975}} + + +e +data: [DONE] + + +0 + diff --git a/crates/loopd/tests/fixtures/http/context_full.http b/crates/loopd/tests/fixtures/http/context_full.http new file mode 100644 index 0000000..1ae8c95 --- /dev/null +++ b/crates/loopd/tests/fixtures/http/context_full.http @@ -0,0 +1,13 @@ +HTTP/1.1 400 Bad Request +Server: llama.cpp +Access-Control-Allow-Origin: +Connection: close +X-Accel-Buffering: no +Content-Type: application/json; charset=utf-8 +Transfer-Encoding: chunked +Connection: close + +cb +{"error":{"code":400,"message":"request (140114 tokens) exceeds the available context size (131072 tokens), try increasing it","type":"exceed_context_size_error","n_prompt_tokens":140114,"n_ctx":131072}} +0 + diff --git a/crates/loopd/tests/fixtures/http/find_tool.http b/crates/loopd/tests/fixtures/http/find_tool.http new file mode 100644 index 0000000..4e57463 --- /dev/null +++ b/crates/loopd/tests/fixtures/http/find_tool.http @@ -0,0 +1,141 @@ +HTTP/1.1 200 OK +Server: llama.cpp +Access-Control-Allow-Origin: +Connection: close +X-Accel-Buffering: no +Content-Type: text/event-stream +Transfer-Encoding: chunked +Connection: close + +212 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775467,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":0,"time_ms":9}} + + +234 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775467,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":22,"prompt_ms":74.959,"prompt_per_token_ms":3.407227272727273,"prompt_per_second":293.4937765978735,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":22,"time_ms":74}} + + +239 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775468,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":504,"prompt_ms":783.411,"prompt_per_token_ms":1.5543869047619046,"prompt_per_second":643.3404687960726,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":504,"time_ms":783}} + + +23b +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":534,"prompt_ms":1202.764,"prompt_per_token_ms":2.2523670411985015,"prompt_per_second":443.9773721195514,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":534,"time_ms":1202}} + + +23b +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.231,"prompt_per_token_ms":3.0357453531598515,"prompt_per_second":329.4083935462895,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":538,"time_ms":1633}} + + +2ea +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"} + +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"I"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}} + + +1f4 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" don"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":2,"predicted_ms":16.961,"predicted_per_token_ms":16.961,"predicted_per_second":58.958787807322686}} + + +1f1 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"'t"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":3,"predicted_ms":31.372,"predicted_per_token_ms":15.686,"predicted_per_second":63.75111564452378}} + + +1ff +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" have"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":4,"predicted_ms":45.286,"predicted_per_token_ms":15.095333333333334,"predicted_per_second":66.2456388287771}} + + +1f2 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" an"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":5,"predicted_ms":59.168,"predicted_per_token_ms":14.792,"predicted_per_second":67.60411032990805}} + + +1f5 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" echo"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":6,"predicted_ms":73.053,"predicted_per_token_ms":14.6106,"predicted_per_second":68.44345885863687}} + + +200 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" tool"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":7,"predicted_ms":86.936,"predicted_per_token_ms":14.489333333333335,"predicted_per_second":69.01628784393117}} + + +206 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" available"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":8,"predicted_ms":100.789,"predicted_per_token_ms":14.398428571428571,"predicted_per_second":69.45202353431426}} + + +1f1 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":9,"predicted_ms":114.632,"predicted_per_token_ms":14.329,"predicted_per_second":69.78854072161351}} + + +1f5 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" Let"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":10,"predicted_ms":128.448,"predicted_per_token_ms":14.272,"predicted_per_second":70.06726457399103}} + + +1f5 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" me"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":11,"predicted_ms":142.257,"predicted_per_token_ms":14.2257,"predicted_per_second":70.29531059982988}} + + +1f5 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" check"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":12,"predicted_ms":156.068,"predicted_per_token_ms":14.188,"predicted_per_second":70.482097547223}} + + +201 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" what"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":13,"predicted_ms":169.859,"predicted_per_token_ms":14.154916666666667,"predicted_per_second":70.6468306065619}} + + +202 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" tools"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":14,"predicted_ms":183.663,"predicted_per_token_ms":14.127923076923079,"predicted_per_second":70.7818123410812}} + + +1ff +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" I"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":15,"predicted_ms":197.492,"predicted_per_token_ms":14.106571428571428,"predicted_per_second":70.88894740040102}} + + +1f5 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" have"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":16,"predicted_ms":211.38,"predicted_per_token_ms":14.092,"predicted_per_second":70.96224808401931}} + + +1fc +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" access"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":17,"predicted_ms":225.181,"predicted_per_token_ms":14.0738125,"predicted_per_second":71.05395215404496}} + + +200 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" to"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":18,"predicted_ms":239.005,"predicted_per_token_ms":14.059117647058823,"predicted_per_second":71.12821907491475}} + + +1fd +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":19,"predicted_ms":253.028,"predicted_per_token_ms":14.05711111111111,"predicted_per_second":71.13837203787723}} + + +200 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"\n\n"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":20,"predicted_ms":266.87,"predicted_per_token_ms":14.045789473684211,"predicted_per_second":71.19571326863266}} + + +275 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"id":"URrmW3zkoa237tPoV3HFK8MnDSwEYlaV","type":"function","function":{"name":"find_tool","arguments":"{"}}]}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":28,"predicted_ms":377.364,"predicted_per_token_ms":13.976444444444443,"predicted_per_second":71.54895538525139}} + + +22a +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"query\":\""}}]}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":33,"predicted_ms":446.63,"predicted_per_token_ms":13.9571875,"predicted_per_second":71.64767257013635}} + + +22a +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"echo"}}]}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":34,"predicted_ms":460.47,"predicted_per_token_ms":13.953636363636365,"predicted_per_second":71.66590657371815}} + + +228 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1789775470,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":39,"predicted_ms":529.999,"predicted_per_token_ms":13.947342105263159,"predicted_per_second":71.6982484872613}} + + +227 +data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}],"created":1789775470,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":43,"predicted_ms":585.28,"predicted_per_token_ms":13.935238095238095,"predicted_per_second":71.76052487698196}} + + +1f9 +data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{}}],"created":1789775470,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":45,"predicted_ms":612.912,"predicted_per_token_ms":13.929818181818183,"predicted_per_second":71.78844597593128}} + + +e +data: [DONE] + + +0 + diff --git a/crates/loopd/tests/limits.rs b/crates/loopd/tests/limits.rs new file mode 100644 index 0000000..053fc22 --- /dev/null +++ b/crates/loopd/tests/limits.rs @@ -0,0 +1,303 @@ +//! Tests for the turn limits and for the append-only property over generated conversations. +//! Do not edit. + +mod support; +#[path = "support/turn.rs"] +mod turn_support; + +use loopd::session::Session; +use loopd::turn::TurnError; +use proto::{LogRecord, SessionId, TurnEvent}; +use support::{Reply, ScriptedPort, ok_result}; +use turn_support::{setup, types, without_cache_loss}; + +const CHAT: &str = "/v1/chat/completions"; + +#[test] +fn too_many_tool_iterations_end_the_turn() { + let mut s = setup(vec![]); + s.cfg.r#loop.tool_iterations = 2; + s.cfg.r#loop.repeat_detection = false; + s.server.route(CHAT, vec![Reply::fixture("tool_call")]); + let mut session = s.session("a"); + let (result, _) = s.turn(&mut session, "x"); + assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}"); + // Two iterations ran their tool; the third completion was recorded and then stopped. + assert_eq!( + types(session.records()), + [ + "start", + "user", + "assistant", + "usage", + "tool_result", + "assistant", + "usage", + "tool_result", + "assistant", + "usage" + ] + ); + assert_eq!(s.port.calls().len(), 2); + assert_eq!(s.server.requests_to(CHAT).len(), 3); + // The session is still usable: the log ends at a record boundary and a new turn works. + s.server.route(CHAT, vec![Reply::fixture("plain")]); + assert!(s.turn(&mut session, "again").0.is_ok()); +} + +#[test] +fn a_repeated_identical_call_is_not_run_and_a_second_repeat_ends_the_turn() { + let s = setup(vec![]); + s.server.route(CHAT, vec![Reply::fixture("tool_call")]); + let mut session = s.session("a"); + let (result, _) = s.turn(&mut session, "x"); + assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}"); + assert_eq!( + s.port.calls().len(), + 1, + "the first call ran; the repeat did not" + ); + match &without_cache_loss(session.records())[7] { + LogRecord::ToolResult { content, .. } => { + assert!(content.contains("already called"), "{content}") + } + other => panic!("{other:?}"), + } + assert_eq!( + s.server.requests_to(CHAT).len(), + 3, + "one more completion after the first repeat, then stop" + ); +} + +#[test] +fn repeat_detection_can_be_turned_off() { + let mut s = setup(vec![]); + s.cfg.r#loop.repeat_detection = false; + s.server.route( + CHAT, + vec![ + Reply::fixture("tool_call"), + Reply::fixture("tool_call"), + Reply::fixture("plain"), + ], + ); + let mut session = s.session("a"); + assert!(s.turn(&mut session, "x").0.is_ok()); + assert_eq!(s.port.calls().len(), 2); +} + +#[test] +fn a_full_context_ends_the_turn_with_nothing_appended_for_the_request() { + let s = setup(vec![]); + s.server.route(CHAT, vec![Reply::fixture("context_full")]); + let mut session = s.session("a"); + let (result, _) = s.turn(&mut session, "x"); + assert!(matches!(result, Err(TurnError::SessionFull)), "{result:?}"); + assert_eq!(types(session.records()), ["start", "user"]); + let e: Box = Box::new(result.unwrap_err()); + assert!(e.to_string().contains("full"), "{e}"); +} + +#[test] +fn other_server_errors_are_inference_errors() { + let s = setup(vec![]); + s.server.route(CHAT, vec![Reply::fixture("bad_request")]); + let mut session = s.session("a"); + let (result, _) = s.turn(&mut session, "x"); + assert!( + matches!( + result, + Err(TurnError::Infer(loopd::llama::InferError::Http { + status: 400, + .. + })) + ), + "{result:?}" + ); +} + +#[test] +fn a_cache_loss_is_recorded_and_reported() { + let s = setup(vec![]); + // progress leaves 15 + 7029 + 8 = 7052 tokens in the slot; plain reports 0 reused. + s.server.route( + CHAT, + vec![Reply::fixture("progress"), Reply::fixture("plain")], + ); + let mut session = s.session("a"); + assert!(s.turn(&mut session, "one").0.is_ok()); + let (result, events) = s.turn(&mut session, "two"); + assert!(result.is_ok()); + assert_eq!(session.records().len(), 8); + assert!( + matches!( + session.records()[7], + LogRecord::CacheLoss { + expected: 7052, + got: 0, + .. + } + ), + "{:?}", + session.records()[7] + ); + assert!(events.contains(&TurnEvent::CacheLoss { + expected: 7052, + got: 0 + })); + + // turn1 then turn2 are a real consecutive pair: 15 + 29 + 2 = 46 left, 45 reused. No loss. + let s = setup(vec![]); + s.server + .route(CHAT, vec![Reply::fixture("turn1"), Reply::fixture("turn2")]); + let mut session = s.session("b"); + assert!(s.turn(&mut session, "one").0.is_ok()); + let (result, events) = s.turn(&mut session, "two"); + assert!(result.is_ok()); + assert!( + !session + .records() + .iter() + .any(|r| matches!(r, LogRecord::CacheLoss { .. })) + ); + assert!( + !events + .iter() + .any(|e| matches!(e, TurnEvent::CacheLoss { .. })) + ); +} + +#[test] +fn retries_are_reported_and_leave_no_trace_in_the_log() { + let s = setup(vec![]); + s.server.route( + CHAT, + vec![ + Reply::fixture("plain").cut_after(400), + Reply::fixture("plain"), + ], + ); + let mut session = s.session("a"); + let (result, events) = s.turn(&mut session, "x"); + assert!(result.is_ok(), "{result:?}"); + assert!( + events + .iter() + .any(|e| matches!(e, TurnEvent::Retrying { attempt: 2, .. })) + ); + assert_eq!( + types(session.records()), + ["start", "user", "assistant", "usage"] + ); +} + +#[test] +fn resumed_sessions_continue_the_same_conversation() { + let s = setup(vec![]); + s.server + .route(CHAT, vec![Reply::fixture("plain"), Reply::fixture("turn2")]); + let mut session = s.session("a"); + assert!(s.turn(&mut session, "one").0.is_ok()); + drop(session); + let mut session = Session::open(&s.home.dir, SessionId::new("a").unwrap()).unwrap(); + assert!(s.turn(&mut session, "two").0.is_ok()); + let sent = s.server.requests_to(CHAT); + let m1 = sent[0].json()["messages"].as_array().unwrap().clone(); + let m2 = sent[1].json()["messages"].as_array().unwrap().clone(); + assert_eq!( + m2[..m1.len()], + m1[..], + "the second request extends the first" + ); + assert_eq!(m2.len(), m1.len() + 2); +} + +/// A small xorshift, so that the test needs no crate and a failure can be replayed by seed. +struct Rng(u64); + +impl Rng { + fn next(&mut self) -> u64 { + self.0 ^= self.0 << 13; + self.0 ^= self.0 >> 7; + self.0 ^= self.0 << 17; + self.0 + } + + fn below(&mut self, n: u64) -> u64 { + self.next() % n + } +} + +/// Every request of a generated conversation is a strict extension of the one before, and the +/// baseline (first message and tools) never changes. +#[test] +fn every_request_extends_the_previous_one() { + for seed in [1u64, 2, 3, 0xdead_beef, 0x9e37_79b9_7f4a_7c15] { + let mut rng = Rng(seed); + let mut s = setup(vec![]); + s.cfg.r#loop.repeat_detection = false; + s.cfg.r#loop.tool_result_cap = 64; + let mut replies = Vec::new(); + let mut port_replies = Vec::new(); + let turns = 2 + rng.below(4); + for _ in 0..turns { + let iterations = rng.below(3); + for _ in 0..iterations { + if rng.below(2) == 0 { + replies.push(Reply::fixture("tool_call")); + } else { + replies.push(Reply::fixture("find_tool")); + replies.push(Reply::fixture("call_tool")); + } + let size = rng.below(120) as usize; + port_replies.push(ok_result(&"r".repeat(size))); + port_replies.push(ok_result(&"r".repeat(size))); + } + replies.push(Reply::fixture( + ["plain", "turn1", "turn2"][rng.below(3) as usize], + )); + } + s.port = ScriptedPort::new(port_replies); + s.server.route(CHAT, replies); + let mut session = s.session("a"); + for t in 0..turns { + let text = format!("turn {t} {}", "u".repeat(rng.below(30) as usize)); + let (result, _) = s.turn(&mut session, &text); + assert!(result.is_ok(), "seed {seed}, turn {t}: {result:?}"); + } + let sent = s.server.requests_to(CHAT); + assert!(sent.len() >= turns as usize, "seed {seed}"); + let first = sent[0].json(); + for (i, pair) in sent.windows(2).enumerate() { + let a = pair[0].json(); + let b = pair[1].json(); + let ma = a["messages"].as_array().unwrap(); + let mb = b["messages"].as_array().unwrap(); + assert!( + mb.len() > ma.len(), + "seed {seed}, request {}: not longer", + i + 1 + ); + assert_eq!( + mb[..ma.len()], + ma[..], + "seed {seed}, request {}: not an extension", + i + 1 + ); + assert_eq!(b["tools"], first["tools"], "seed {seed}: tools changed"); + assert_eq!( + b["messages"][0], first["messages"][0], + "seed {seed}: system message changed" + ); + } + // And the log replays to exactly the last request. + let replayed = loopd::baseline::messages(session.baseline(), session.records()); + let last = sent.last().unwrap().json(); + assert_eq!( + replayed.len(), + last["messages"].as_array().unwrap().len() + 1, + "seed {seed}: the last request plus the final answer" + ); + } +} diff --git a/crates/loopd/tests/support/turn.rs b/crates/loopd/tests/support/turn.rs new file mode 100644 index 0000000..1458833 --- /dev/null +++ b/crates/loopd/tests/support/turn.rs @@ -0,0 +1,97 @@ +//! A turn-loop setup shared by the `turn` and `limits` tests: a home, a fake server, a scripted +//! port and a registry. Included with `#[path]` because it needs `loopd::turn`, which the earlier +//! tasks' tests must not depend on. Do not edit. + +#![allow(dead_code)] + +use crate::support::{FakeServer, Home, ScriptedPort}; +use loopd::baseline::Baseline; +use loopd::llama::Client; +use loopd::session::Session; +use loopd::tools::Registry; +use loopd::turn::{Runtime, TurnError, run_turn}; +use proto::{LogRecord, SessionId, ToolResponse, TurnEvent}; + +pub struct Setup { + pub home: Home, + pub server: FakeServer, + pub cfg: loopd::config::Config, + pub client: Client, + pub port: ScriptedPort, + pub registry: Registry, +} + +pub fn setup(replies: Vec) -> Setup { + let home = Home::new(); + let server = FakeServer::start(); + let cfg = home.config(&server.socket); + let client = Client::new(cfg.clone()); + Setup { + home, + server, + cfg, + client, + port: ScriptedPort::new(replies), + registry: Registry::m2b(), + } +} + +impl Setup { + pub fn session(&self, id: &str) -> Session { + let baseline = Baseline::assemble(&self.cfg, &self.registry).unwrap(); + Session::create( + &self.home.dir, + SessionId::new(id).unwrap(), + baseline, + self.cfg.slots.main, + ) + .unwrap() + } + + pub fn runtime(&self) -> Runtime<'_> { + Runtime { + cfg: &self.cfg, + client: &self.client, + port: &self.port, + registry: &self.registry, + } + } + + pub fn turn( + &self, + session: &mut Session, + text: &str, + ) -> (Result, Vec) { + let mut events = Vec::new(); + let result = run_turn(session, &self.runtime(), text, &mut |e| { + events.push(e.clone()) + }); + (result, events) + } +} + +/// The recordings were made in separate conversations, so their timings do not line up and the +/// loop rightly reports cache losses between them. Most tests are not about that, so `types` +/// and `records` leave `CacheLoss` out; one test checks it on purpose. +pub fn without_cache_loss(records: &[LogRecord]) -> Vec { + records + .iter() + .filter(|r| !matches!(r, LogRecord::CacheLoss { .. })) + .cloned() + .collect() +} + +pub fn types(records: &[LogRecord]) -> Vec<&'static str> { + without_cache_loss(records) + .iter() + .map(|r| match r { + LogRecord::SessionStart { .. } => "start", + LogRecord::User { .. } => "user", + LogRecord::Assistant { .. } => "assistant", + LogRecord::Usage { .. } => "usage", + LogRecord::ToolResult { .. } => "tool_result", + LogRecord::CacheLoss { .. } => "cache_loss", + LogRecord::EpochEnd { .. } => "epoch_end", + }) + .collect() +} diff --git a/crates/loopd/tests/turn.rs b/crates/loopd/tests/turn.rs new file mode 100644 index 0000000..110577e --- /dev/null +++ b/crates/loopd/tests/turn.rs @@ -0,0 +1,280 @@ +//! Tests for one turn: record sequences and tool dispatch. Do not edit. +//! The limits and the append-only property are in `limits.rs`. + +mod support; +#[path = "support/turn.rs"] +mod turn_support; + +use loopd::tools::Registry; +use proto::{DataClass, LogRecord, SessionId, ToolResponse, TurnEvent}; +use support::{Reply, ok_result}; +use turn_support::{setup, types}; + +const CHAT: &str = "/v1/chat/completions"; + +#[test] +fn a_plain_turn() { + let s = setup(vec![]); + s.server.route(CHAT, vec![Reply::fixture("plain")]); + let mut session = s.session("a"); + let (result, events) = s.turn(&mut session, "hello"); + let outcome = result.unwrap(); + assert_eq!( + outcome.content, + support::expected("plain")["content"].as_str().unwrap() + ); + assert_eq!( + (outcome.usage.prompt_n, outcome.usage.predicted_n), + (46, 16) + ); + assert_eq!( + types(session.records()), + ["start", "user", "assistant", "usage"] + ); + assert_eq!( + types(&s.home.records("a")), + ["start", "user", "assistant", "usage"], + "on disk too" + ); + assert!( + events + .iter() + .any(|e| matches!(e, TurnEvent::Content { .. })) + ); + assert!( + !events + .iter() + .any(|e| matches!(e, TurnEvent::ToolCallStarted { .. })) + ); + assert!(s.port.calls().is_empty()); + + let sent = s.server.requests_to(CHAT); + assert_eq!(sent.len(), 1); + let body = sent[0].json(); + assert_eq!(body["messages"][0]["role"], "system"); + assert_eq!( + body["messages"][0]["content"], + "You are Boxmaker, a test agent." + ); + assert_eq!( + body["messages"][1], + serde_json::json!({"role": "user", "content": "hello"}) + ); + let names: Vec<&str> = body["tools"] + .as_array() + .unwrap() + .iter() + .map(|t| t["function"]["name"].as_str().unwrap()) + .collect(); + assert_eq!(names, ["clock", "find_tool", "call_tool"]); + assert_eq!(body["id_slot"], 0); +} + +#[test] +fn a_tool_turn_goes_through_the_port_and_records_everything() { + let s = setup(vec![ok_result("straylight\n")]); + s.server.route( + CHAT, + vec![Reply::fixture("tool_call"), Reply::fixture("plain")], + ); + let mut session = s.session("a"); + let (result, events) = s.turn(&mut session, "read the hostname"); + assert!(result.is_ok(), "{result:?}"); + assert_eq!( + types(session.records()), + [ + "start", + "user", + "assistant", + "usage", + "tool_result", + "assistant", + "usage" + ] + ); + + let calls = s.port.calls(); + assert_eq!(calls.len(), 1); + assert_eq!(calls[0].tool, "read_file"); + assert_eq!( + calls[0].arguments, r#"{"path":"/etc/hostname"}"#, + "arguments are passed on unparsed" + ); + assert_eq!(calls[0].session, SessionId::new("a").unwrap()); + assert_eq!(calls[0].call, proto::CallId(1)); + + match &session.records()[4] { + LogRecord::ToolResult { + call, + tool_call_id, + content, + class, + untrusted, + truncated, + .. + } => { + assert_eq!(*call, proto::CallId(1)); + assert_eq!( + tool_call_id, "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F", + "the server's id, so the template can pair it" + ); + assert_eq!(content, "straylight\n"); + assert_eq!( + (*class, *untrusted, *truncated), + (DataClass::Private, true, false) + ); + } + other => panic!("{other:?}"), + } + let kinds: Vec<&str> = events + .iter() + .filter_map(|e| match e { + TurnEvent::ToolCallStarted { name } => Some(name.as_str()), + TurnEvent::ToolResult { name, .. } => Some(name.as_str()), + _ => None, + }) + .collect(); + assert_eq!(kinds, ["read_file", "read_file"]); + + // The second request extends the first: the tool result sits after the assistant turn. + let sent = s.server.requests_to(CHAT); + let m2 = sent[1].json()["messages"].as_array().unwrap().clone(); + assert_eq!(m2[2]["role"], "assistant"); + assert_eq!( + m2[2]["tool_calls"][0]["id"], + "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F" + ); + assert_eq!( + m2[3], + serde_json::json!({"role": "tool", "tool_call_id": "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F", "content": "straylight\n"}) + ); +} + +#[test] +fn find_tool_and_call_tool_reach_the_port_only_for_the_target() { + let s = setup(vec![ok_result("box")]); + s.server.route( + CHAT, + vec![ + Reply::fixture("find_tool"), + Reply::fixture("call_tool"), + Reply::fixture("plain"), + ], + ); + let mut session = s.session("a"); + let (result, _) = s.turn(&mut session, "echo box"); + assert!(result.is_ok(), "{result:?}"); + assert_eq!( + types(session.records()), + [ + "start", + "user", + "assistant", + "usage", + "tool_result", + "assistant", + "usage", + "tool_result", + "assistant", + "usage" + ] + ); + let calls = s.port.calls(); + assert_eq!( + calls.len(), + 1, + "find_tool is answered by loopd; only echo reaches the port" + ); + assert_eq!(calls[0].tool, "echo"); + assert_eq!( + serde_json::from_str::(&calls[0].arguments).unwrap(), + serde_json::json!({"text": "box"}) + ); + match &session.records()[4] { + LogRecord::ToolResult { + content, + class, + untrusted, + .. + } => { + assert!( + content.contains("\"name\":\"echo\"") + && content.ends_with("Call it with call_tool."), + "{content}" + ); + assert_eq!((*class, *untrusted), (DataClass::Public, false)); + } + other => panic!("{other:?}"), + } +} + +#[test] +fn a_call_tool_for_an_unknown_tool_never_reaches_the_port() { + // The recorded call_tool asks for "echo"; with echo removed from the registry it is unknown. + let mut s = setup(vec![]); + s.registry = Registry::new(vec![loopd::tools::Entry { + schema: loopd::tools::clock_schema(), + core: true, + }]); + s.server.route( + CHAT, + vec![Reply::fixture("call_tool"), Reply::fixture("plain")], + ); + let mut session = s.session("a"); + assert!(s.turn(&mut session, "x").0.is_ok()); + assert!(s.port.calls().is_empty()); + assert!( + matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content.contains("No tool named \"echo\"")) + ); +} + +#[test] +fn the_result_cap_applies_when_appended() { + let mut s = setup(vec![ok_result(&"x".repeat(100))]); + s.cfg.r#loop.tool_result_cap = 20; + s.server.route( + CHAT, + vec![Reply::fixture("tool_call"), Reply::fixture("plain")], + ); + let mut session = s.session("a"); + let (result, events) = s.turn(&mut session, "x"); + assert!(result.is_ok()); + match &session.records()[4] { + LogRecord::ToolResult { + content, truncated, .. + } => { + assert_eq!(content, &format!("{}\n[truncated]", "x".repeat(20))); + assert!(truncated); + } + other => panic!("{other:?}"), + } + assert!(events.iter().any(|e| matches!( + e, + TurnEvent::ToolResult { + truncated: true, + .. + } + ))); + let m2 = s.server.requests_to(CHAT)[1].json(); + assert_eq!( + m2["messages"][3]["content"].as_str().unwrap().len(), + 20 + "\n[truncated]".len(), + "the model sees the capped text" + ); +} + +#[test] +fn tool_failures_and_denials_become_results_the_model_can_read() { + let s = setup(vec![ToolResponse::Failed { + message: "disk on fire".to_string(), + }]); + s.server.route( + CHAT, + vec![Reply::fixture("tool_call"), Reply::fixture("plain")], + ); + let mut session = s.session("a"); + assert!(s.turn(&mut session, "x").0.is_ok()); + assert!( + matches!(&session.records()[4], LogRecord::ToolResult { content, class: DataClass::Public, .. } if content.contains("disk on fire")) + ); +} diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 5483d03..e142b81 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -35,6 +35,7 @@ reviewer adds findings under "Reviews" once per milestone. | M2b/03-loopd-tools | 2026-09-18 | done | 1 | pass | none | Added `pub mod tools;` to lib.rs and `serde::Serialize`/`serde::Deserialize`/`deny_unknown_fields` to ToolSchema; wrote crates/loopd/src/tools.rs with ToolPort, Entry, Registry (m2b/config), core_schemas/get/find, FIND_TOOL/CALL_TOOL constants, clock_schema/echo_schema, Dispatch with dispatch (find_tool/call_tool/local dispatch rows), cap_result via floor_char_boundary, and FakeTools recording calls and answering clock/echo/denying others with `unwrap_or_else(|p| p.into_inner())` on Mutex::lock. 7 tools tests pass, `make gate` prints `gate: ok`, no `unwrap()` in tools.rs. | ? | | M2b/04-loopd-baseline | 2026-09-18 | done | 2 | fail | none | Wrote crates/loopd/src/baseline.rs: `Baseline` (system prompt + core tool schemas, `deny_unknown_fields`), `BaselineError` (Read/Parse name the file, plus Hash) with Display/std::error::Error, `assemble` (system prompt trimmed of trailing whitespace, core memory appended with a blank line when its trimmed content is non-empty), `to_json`/`from_json`, `load`, and `hash` (sha256 of the canonical JSON). `messages` prepends the system message and replays every `LogRecord` variant explicitly named, so a new one is a compile error. The `\n\n` separator between system prompt and core memory had to be two newlines (a blank line), not one. All 6 baseline tests pass and all loopd tests pass with the new support module; first gate run failed on a rustfmt import-order diff, fixed with `cargo fmt`. | ? | | M2b/05-loopd-session | 2026-09-18 | done | 2 | fail | none | Copied the given test byte-identical and wrote crates/loopd/src/session.rs: `Session` (id, dir, baseline, records, appended log file, next_call) and `SessionError` (Exists/NotFound/Io/Torn/Baseline/Encode) with derived Debug, Display and std::error::Error::source. `create` refuses an existing dir, writes `0.baseline.json`, opens `0.jsonl` with `create_new`+`append`, and appends a `SessionStart` (`Timestamp::now()`, epoch 0, the slot, `baseline.hash()`). `open` reads the baseline from the file (not `system.md`), requires every log line to end in `\n` and parse as a `LogRecord` else `Torn` with the 1-based line and reason, and sets `next_call` to one past the highest `ToolResult` call. `append` encodes, writes, `sync_data()`, then pushes to memory. All 7 session tests pass. First gate failed on clippy: split the `source()` arm that bound three different error types into three arms, removed the redundant `.write(true)` (implied by `append`), and used `path.display()` for the `Torn` path. | ? | +| M2b/06-loopd-turn | 2026-09-18 | done | 1 | pass | none | Wrote crates/loopd/src/turn.rs (299 lines) and registered `pub mod turn;` in lib.rs; copied the two given tests and three fixtures byte-identical. `TurnError` (SessionFull/TurnLimit/Infer/Session, Display + std::error::Error + From), `TurnOutcome`, `Runtime`, `is_context_full` (the one 400 whose JSON `error.type` is `exceed_context_size_error`), and `run_turn`: append User, build the ChatRequest (slot, messages, tools, thinking), capture `last_usage`, `chat_with_retry` mapping ChatEvent->TurnEvent (dropping ToolCallDelta), append Assistant then Usage, report cache loss between the two conversations, and on no tool calls return `TurnOutcome { content: completion.content.unwrap_or_default(), usage }`; otherwise iterate tool calls under the iteration cap with a repeated-call detector (first repeat returns "already called", a second repeat is TurnLimit), `cap_result`, and `dispatch` (find_tool/call_tool local, every other tool — including read_file — to the port). `run_call` maps Dispatch::Local and every ToolResponse variant to (text, Public, untrusted). Two compile fixes before the gate: `u64::try_from(*ahead).unwrap_or(u64::MAX)` (usize has no From) and `let Ok(value) = from_str(body) else { return false }` (a temporary borrow); `session.baseline()` returns a reference so it is bound inside the loop. All 6 turn and 9 limits tests pass; `make gate` prints `gate: ok`. | ? | ## Reviews