kyle/boxmaker
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add the turn loop with its limits

Browse Source 
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
kyle
2026-09-18 19:14:52 -07:00
parent 9704a64507
commit 9063cf958e
9 changed files with 1218 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crates/loopd/src/lib.rs
+1
View File
@@ -8,3 +8,4 @@ pub mod selftest;
pub mod session;
pub mod sse;
pub mod tools;
pub mod turn;
crates/loopd/src/turn.rs
+299
View File
@@ -0,0 +1,299 @@
//! One turn: ask, run any tool calls under their limits, and return.
//! The loop is bounded every way a runaway can go: a cap on tool iterations, a repeated-call
//! detector, the two M2a limits (thinking tokens, a full context), and a tool-result cap.
use crate::baseline::messages;
use crate::config::Config;
use crate::llama::info::{CacheOutcome, cache_outcome};
use crate::llama::{ChatEvent, ChatRequest, Client, InferError};
use crate::session::{Session, SessionError};
use crate::tools::{Registry, ToolPort, cap_result, dispatch};
use proto::{CallId, DataClass, LogRecord, Timestamp, ToolRequest, ToolResponse, TurnEvent, Usage};
/// What one turn can fail with, beyond the inference server.
#[derive(Debug)]
pub enum TurnError {
SessionFull,
TurnLimit,
Infer(InferError),
Session(SessionError),
}
impl std::fmt::Display for TurnError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
TurnError::SessionFull => write!(f, "the context is full"),
TurnError::TurnLimit => write!(f, "the turn limit was reached"),
TurnError::Infer(e) => write!(f, "{e}"),
TurnError::Session(e) => write!(f, "{e}"),
}
}
}
impl std::error::Error for TurnError {}
impl From<SessionError> for TurnError {
fn from(e: SessionError) -> Self {
TurnError::Session(e)
}
}
/// The text to show and what the turn cost.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct TurnOutcome {
pub content: String,
pub usage: Usage,
}
/// What a turn needs besides the session.
pub struct Runtime<'a> {
pub cfg: &'a Config,
pub client: &'a Client,
pub port: &'a dyn ToolPort,
pub registry: &'a Registry,
}
/// True for the one 400 the server sends when the prompt does not fit.
pub fn is_context_full(error: &InferError) -> bool {
match error {
InferError::Http { status: 400, body } => {
let Ok(value) = serde_json::from_str::<serde_json::Value>(body) else {
return false;
};
matches!(
value
.get("error")
.and_then(|e| e.get("type"))
.and_then(|t| t.as_str()),
Some("exceed_context_size_error")
)
}
_ => false,
}
}
fn map_event(event: &ChatEvent, on_event: &mut dyn FnMut(&TurnEvent)) {
match event {
ChatEvent::Queued { ahead } => on_event(&TurnEvent::Queued {
ahead: u64::try_from(*ahead).unwrap_or(u64::MAX),
}),
ChatEvent::Waiting { slot_busy } => on_event(&TurnEvent::Waiting {
slot_busy: *slot_busy,
}),
ChatEvent::Progress {
total,
cache,
processed,
} => on_event(&TurnEvent::Progress {
total: *total,
cache: *cache,
processed: *processed,
}),
ChatEvent::Reasoning(text) => on_event(&TurnEvent::Reasoning { text: text.clone() }),
ChatEvent::Content(text) => on_event(&TurnEvent::Content { text: text.clone() }),
ChatEvent::ThinkingCapped { tokens } => {
on_event(&TurnEvent::ThinkingCapped { tokens: *tokens })
}
ChatEvent::Retrying {
attempt,
after_ms,
error,
} => on_event(&TurnEvent::Retrying {
attempt: *attempt,
after_ms: *after_ms,
error: error.clone(),
}),
ChatEvent::ToolCallDelta { .. } => {}
}
}
pub fn run_turn(
session: &mut Session,
rt: &Runtime<'_>,
content: &str,
on_event: &mut dyn FnMut(&TurnEvent),
) -> Result<TurnOutcome, TurnError> {
session
.append(LogRecord::User {
time: Timestamp::now(),
content: content.to_string(),
})
.map_err(TurnError::Session)?;
let mut seen: Vec<(String, String)> = Vec::new();
let mut iterations: u32 = 0;
loop {
let baseline = session.baseline();
let request = ChatRequest {
slot: rt.cfg.slots.main,
messages: messages(baseline, session.records()),
tools: baseline.tools.clone(),
thinking: true,
};
let previous = session.last_usage();
let completion = match rt
.client
.chat_with_retry(&request, &mut |event| map_event(event, on_event))
{
Ok(completion) => completion,
Err(e) => {
if is_context_full(&e) {
return Err(TurnError::SessionFull);
}
return Err(TurnError::Infer(e));
}
};
session
.append(LogRecord::Assistant {
time: Timestamp::now(),
content: completion.content.clone(),
reasoning_content: completion.reasoning_content.clone(),
tool_calls: completion.tool_calls.clone(),
})
.map_err(TurnError::Session)?;
let usage = Usage {
cache_n: completion.timings.cache_n,
prompt_n: completion.timings.prompt_n,
predicted_n: completion.timings.predicted_n,
reasoning_tokens: completion.reasoning_tokens,
thinking_capped: completion.thinking_capped,
};
session
.append(LogRecord::Usage {
time: Timestamp::now(),
cache_n: usage.cache_n,
prompt_n: usage.prompt_n,
predicted_n: usage.predicted_n,
reasoning_tokens: usage.reasoning_tokens,
thinking_capped: usage.thinking_capped,
})
.map_err(TurnError::Session)?;
// The recordings were made in separate conversations, so their cache numbers do not line up.
if let Some(previous) = previous {
let previous = crate::llama::Timings {
cache_n: previous.cache_n,
prompt_n: previous.prompt_n,
predicted_n: previous.predicted_n,
};
if let CacheOutcome::Loss { expected, got } =
cache_outcome(&previous, &completion.timings)
{
session
.append(LogRecord::CacheLoss {
time: Timestamp::now(),
expected,
got,
})
.map_err(TurnError::Session)?;
on_event(&TurnEvent::CacheLoss { expected, got });
}
}
if completion.tool_calls.is_empty() {
return Ok(TurnOutcome {
content: completion.content.clone().unwrap_or_default(),
usage,
});
}
iterations += 1;
if iterations > rt.cfg.r#loop.tool_iterations {
return Err(TurnError::TurnLimit);
}
for call in &completion.tool_calls {
on_event(&TurnEvent::ToolCallStarted {
name: call.name.clone(),
});
let call_id = session.next_call();
// A repeated identical (name, arguments) pair is not run the first time; a second repeat stops the turn.
let key = (call.name.clone(), call.arguments.clone());
let times = seen.iter().filter(|pair| **pair == key).count();
if rt.cfg.r#loop.repeat_detection && times >= 2 {
return Err(TurnError::TurnLimit);
}
seen.push(key);
let (text, class, untrusted) = if rt.cfg.r#loop.repeat_detection && times == 1 {
(
format!(
"The `{}` tool call was already called with these arguments this turn.",
call.name
),
DataClass::Public,
false,
)
} else {
run_call(rt, session, call, call_id)
};
let (text, truncated) = cap_result(&text, rt.cfg.r#loop.tool_result_cap);
session
.append(LogRecord::ToolResult {
time: Timestamp::now(),
call: call_id,
tool_call_id: call.id.clone(),
content: text,
class,
untrusted,
truncated,
})
.map_err(TurnError::Session)?;
on_event(&TurnEvent::ToolResult {
name: call.name.clone(),
class,
truncated,
});
}
}
}
/// Resolves one tool call to the text, class and trust flag of its result.
fn run_call(
rt: &Runtime<'_>,
session: &Session,
call: &proto::ToolCall,
call_id: CallId,
) -> (String, DataClass, bool) {
match dispatch(rt.registry, &call.name, &call.arguments) {
crate::tools::Dispatch::Local(text) => (text, DataClass::Public, false),
crate::tools::Dispatch::Port { tool, arguments } => {
let request = ToolRequest {
session: session.id().clone(),
call: call_id,
tool,
arguments,
};
match rt.port.call(&request) {
ToolResponse::Result {
content,
class,
untrusted,
..
} => (content, class, untrusted),
ToolResponse::Failed { message } => (
format!("The tool failed: {message}"),
DataClass::Public,
false,
),
ToolResponse::Denied { reason } => (
format!("The call was denied: {reason:?}"),
DataClass::Public,
false,
),
ToolResponse::PendingApproval { .. } => (
"This version of loopd cannot wait for approval; the call was rejected."
.to_string(),
DataClass::Public,
false,
),
}
}
}
}
crates/loopd/tests/fixtures/http/call_tool.http
Vendored
+83
View File
@@ -0,0 +1,83 @@
HTTP/1.1 200 OK
Server: llama.cpp
Access-Control-Allow-Origin:
Connection: close
X-Accel-Buffering: no
Content-Type: text/event-stream
Transfer-Encoding: chunked
Connection: close
218
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":657,"cache":582,"processed":582,"time_ms":0}}
23c
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":71,"prompt_ms":527.742,"prompt_per_token_ms":7.432985915492957,"prompt_per_second":134.53543587586358,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":657,"cache":582,"processed":653,"time_ms":527}}
23b
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.13,"prompt_per_token_ms":11.655066666666666,"prompt_per_second":85.79959502591149,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":657,"cache":582,"processed":657,"time_ms":874}}
1ea
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775470,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
25e
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"id":"3E643m2jnS3ahaAM6cU6YyRjVa4kOtBV","type":"function","function":{"name":"call_tool","arguments":"{"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":9,"predicted_ms":113.464,"predicted_per_token_ms":14.183,"predicted_per_second":70.506944934076}}
22a
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"name\":\""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":14,"predicted_ms":182.903,"predicted_per_token_ms":14.069461538461537,"predicted_per_second":71.07592549056056}}
222
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"echo"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":15,"predicted_ms":196.873,"predicted_per_token_ms":14.062357142857142,"predicted_per_second":71.1118335170389}}
220
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":20,"predicted_ms":266.06,"predicted_per_token_ms":14.003157894736843,"predicted_per_second":71.41246335413065}}
223
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":",\"arguments\":"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":26,"predicted_ms":349.491,"predicted_per_token_ms":13.97964,"predicted_per_second":71.5326002672458}}
222
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"{\""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":27,"predicted_ms":363.312,"predicted_per_token_ms":13.973538461538462,"predicted_per_second":71.56383494076717}}
223
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"text"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":28,"predicted_ms":377.136,"predicted_per_token_ms":13.968000000000002,"predicted_per_second":71.59221076746849}}
217
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\":"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":29,"predicted_ms":391.062,"predicted_per_token_ms":13.9665,"predicted_per_second":71.59989976014033}}
221
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":" \""}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":30,"predicted_ms":404.922,"predicted_per_token_ms":13.962827586206897,"predicted_per_second":71.6187315087844}}
222
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"box"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":31,"predicted_ms":418.766,"predicted_per_token_ms":13.958866666666667,"predicted_per_second":71.63905379137752}}
222
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"}"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":32,"predicted_ms":432.558,"predicted_per_token_ms":13.953483870967741,"predicted_per_second":71.66668978495369}}
220
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":41,"predicted_ms":556.994,"predicted_per_token_ms":13.924850000000001,"predicted_per_second":71.81405903833793}}
1f1
data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{}}],"created":1789775471,"id":"chatcmpl-1ObuAuYrmPlsddbKa77AAfAcX0fdbAeh","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":582,"prompt_n":75,"prompt_ms":874.518,"prompt_per_token_ms":11.66024,"prompt_per_second":85.76152806460244,"predicted_n":43,"predicted_ms":584.671,"predicted_per_token_ms":13.920738095238097,"predicted_per_second":71.83527146035975}}
e
data: [DONE]
0
crates/loopd/tests/fixtures/http/context_full.http
Vendored
+13
View File
@@ -0,0 +1,13 @@
HTTP/1.1 400 Bad Request
Server: llama.cpp
Access-Control-Allow-Origin:
Connection: close
X-Accel-Buffering: no
Content-Type: application/json; charset=utf-8
Transfer-Encoding: chunked
Connection: close
cb
{"error":{"code":400,"message":"request (140114 tokens) exceeds the available context size (131072 tokens), try increasing it","type":"exceed_context_size_error","n_prompt_tokens":140114,"n_ctx":131072}}
0
crates/loopd/tests/fixtures/http/find_tool.http
Vendored
+141
View File
@@ -0,0 +1,141 @@
HTTP/1.1 200 OK
Server: llama.cpp
Access-Control-Allow-Origin:
Connection: close
X-Accel-Buffering: no
Content-Type: text/event-stream
Transfer-Encoding: chunked
Connection: close
212
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775467,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":0,"time_ms":9}}
234
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775467,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":22,"prompt_ms":74.959,"prompt_per_token_ms":3.407227272727273,"prompt_per_second":293.4937765978735,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":22,"time_ms":74}}
239
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775468,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":504,"prompt_ms":783.411,"prompt_per_token_ms":1.5543869047619046,"prompt_per_second":643.3404687960726,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":504,"time_ms":783}}
23b
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":534,"prompt_ms":1202.764,"prompt_per_token_ms":2.2523670411985015,"prompt_per_second":443.9773721195514,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":534,"time_ms":1202}}
23b
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.231,"prompt_per_token_ms":3.0357453531598515,"prompt_per_second":329.4083935462895,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":538,"cache":0,"processed":538,"time_ms":1633}}
2ea
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"I"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
1f4
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" don"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":2,"predicted_ms":16.961,"predicted_per_token_ms":16.961,"predicted_per_second":58.958787807322686}}
1f1
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"'t"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":3,"predicted_ms":31.372,"predicted_per_token_ms":15.686,"predicted_per_second":63.75111564452378}}
1ff
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" have"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":4,"predicted_ms":45.286,"predicted_per_token_ms":15.095333333333334,"predicted_per_second":66.2456388287771}}
1f2
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" an"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":5,"predicted_ms":59.168,"predicted_per_token_ms":14.792,"predicted_per_second":67.60411032990805}}
1f5
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" echo"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":6,"predicted_ms":73.053,"predicted_per_token_ms":14.6106,"predicted_per_second":68.44345885863687}}
200
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" tool"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":7,"predicted_ms":86.936,"predicted_per_token_ms":14.489333333333335,"predicted_per_second":69.01628784393117}}
206
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" available"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":8,"predicted_ms":100.789,"predicted_per_token_ms":14.398428571428571,"predicted_per_second":69.45202353431426}}
1f1
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":9,"predicted_ms":114.632,"predicted_per_token_ms":14.329,"predicted_per_second":69.78854072161351}}
1f5
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" Let"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":10,"predicted_ms":128.448,"predicted_per_token_ms":14.272,"predicted_per_second":70.06726457399103}}
1f5
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" me"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":11,"predicted_ms":142.257,"predicted_per_token_ms":14.2257,"predicted_per_second":70.29531059982988}}
1f5
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" check"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":12,"predicted_ms":156.068,"predicted_per_token_ms":14.188,"predicted_per_second":70.482097547223}}
201
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" what"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":13,"predicted_ms":169.859,"predicted_per_token_ms":14.154916666666667,"predicted_per_second":70.6468306065619}}
202
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" tools"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":14,"predicted_ms":183.663,"predicted_per_token_ms":14.127923076923079,"predicted_per_second":70.7818123410812}}
1ff
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" I"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":15,"predicted_ms":197.492,"predicted_per_token_ms":14.106571428571428,"predicted_per_second":70.88894740040102}}
1f5
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" have"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":16,"predicted_ms":211.38,"predicted_per_token_ms":14.092,"predicted_per_second":70.96224808401931}}
1fc
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" access"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":17,"predicted_ms":225.181,"predicted_per_token_ms":14.0738125,"predicted_per_second":71.05395215404496}}
200
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" to"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":18,"predicted_ms":239.005,"predicted_per_token_ms":14.059117647058823,"predicted_per_second":71.12821907491475}}
1fd
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":19,"predicted_ms":253.028,"predicted_per_token_ms":14.05711111111111,"predicted_per_second":71.13837203787723}}
200
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"\n\n"}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":20,"predicted_ms":266.87,"predicted_per_token_ms":14.045789473684211,"predicted_per_second":71.19571326863266}}
275
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"id":"URrmW3zkoa237tPoV3HFK8MnDSwEYlaV","type":"function","function":{"name":"find_tool","arguments":"{"}}]}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":28,"predicted_ms":377.364,"predicted_per_token_ms":13.976444444444443,"predicted_per_second":71.54895538525139}}
22a
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"query\":\""}}]}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":33,"predicted_ms":446.63,"predicted_per_token_ms":13.9571875,"predicted_per_second":71.64767257013635}}
22a
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"echo"}}]}}],"created":1789775469,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":34,"predicted_ms":460.47,"predicted_per_token_ms":13.953636363636365,"predicted_per_second":71.66590657371815}}
228
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1789775470,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":39,"predicted_ms":529.999,"predicted_per_token_ms":13.947342105263159,"predicted_per_second":71.6982484872613}}
227
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}],"created":1789775470,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":43,"predicted_ms":585.28,"predicted_per_token_ms":13.935238095238095,"predicted_per_second":71.76052487698196}}
1f9
data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{}}],"created":1789775470,"id":"chatcmpl-svUY8ZpZlc5XKIsmNziTBtrSWz5KiYEW","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":538,"prompt_ms":1633.55,"prompt_per_token_ms":3.036338289962825,"prompt_per_second":329.3440666034098,"predicted_n":45,"predicted_ms":612.912,"predicted_per_token_ms":13.929818181818183,"predicted_per_second":71.78844597593128}}
e
data: [DONE]
0
crates/loopd/tests/limits.rs
+303
View File
@@ -0,0 +1,303 @@
//! Tests for the turn limits and for the append-only property over generated conversations.
//! Do not edit.
mod support;
#[path = "support/turn.rs"]
mod turn_support;
use loopd::session::Session;
use loopd::turn::TurnError;
use proto::{LogRecord, SessionId, TurnEvent};
use support::{Reply, ScriptedPort, ok_result};
use turn_support::{setup, types, without_cache_loss};
const CHAT: &str = "/v1/chat/completions";
#[test]
fn too_many_tool_iterations_end_the_turn() {
let mut s = setup(vec![]);
s.cfg.r#loop.tool_iterations = 2;
s.cfg.r#loop.repeat_detection = false;
s.server.route(CHAT, vec![Reply::fixture("tool_call")]);
let mut session = s.session("a");
let (result, _) = s.turn(&mut session, "x");
assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}");
// Two iterations ran their tool; the third completion was recorded and then stopped.
assert_eq!(
types(session.records()),
[
"start",
"user",
"assistant",
"usage",
"tool_result",
"assistant",
"usage",
"tool_result",
"assistant",
"usage"
]
);
assert_eq!(s.port.calls().len(), 2);
assert_eq!(s.server.requests_to(CHAT).len(), 3);
// The session is still usable: the log ends at a record boundary and a new turn works.
s.server.route(CHAT, vec![Reply::fixture("plain")]);
assert!(s.turn(&mut session, "again").0.is_ok());
}
#[test]
fn a_repeated_identical_call_is_not_run_and_a_second_repeat_ends_the_turn() {
let s = setup(vec![]);
s.server.route(CHAT, vec![Reply::fixture("tool_call")]);
let mut session = s.session("a");
let (result, _) = s.turn(&mut session, "x");
assert!(matches!(result, Err(TurnError::TurnLimit)), "{result:?}");
assert_eq!(
s.port.calls().len(),
1,
"the first call ran; the repeat did not"
);
match &without_cache_loss(session.records())[7] {
LogRecord::ToolResult { content, .. } => {
assert!(content.contains("already called"), "{content}")
}
other => panic!("{other:?}"),
}
assert_eq!(
s.server.requests_to(CHAT).len(),
3,
"one more completion after the first repeat, then stop"
);
}
#[test]
fn repeat_detection_can_be_turned_off() {
let mut s = setup(vec![]);
s.cfg.r#loop.repeat_detection = false;
s.server.route(
CHAT,
vec![
Reply::fixture("tool_call"),
Reply::fixture("tool_call"),
Reply::fixture("plain"),
],
);
let mut session = s.session("a");
assert!(s.turn(&mut session, "x").0.is_ok());
assert_eq!(s.port.calls().len(), 2);
}
#[test]
fn a_full_context_ends_the_turn_with_nothing_appended_for_the_request() {
let s = setup(vec![]);
s.server.route(CHAT, vec![Reply::fixture("context_full")]);
let mut session = s.session("a");
let (result, _) = s.turn(&mut session, "x");
assert!(matches!(result, Err(TurnError::SessionFull)), "{result:?}");
assert_eq!(types(session.records()), ["start", "user"]);
let e: Box<dyn std::error::Error> = Box::new(result.unwrap_err());
assert!(e.to_string().contains("full"), "{e}");
}
#[test]
fn other_server_errors_are_inference_errors() {
let s = setup(vec![]);
s.server.route(CHAT, vec![Reply::fixture("bad_request")]);
let mut session = s.session("a");
let (result, _) = s.turn(&mut session, "x");
assert!(
matches!(
result,
Err(TurnError::Infer(loopd::llama::InferError::Http {
status: 400,
..
}))
),
"{result:?}"
);
}
#[test]
fn a_cache_loss_is_recorded_and_reported() {
let s = setup(vec![]);
// progress leaves 15 + 7029 + 8 = 7052 tokens in the slot; plain reports 0 reused.
s.server.route(
CHAT,
vec![Reply::fixture("progress"), Reply::fixture("plain")],
);
let mut session = s.session("a");
assert!(s.turn(&mut session, "one").0.is_ok());
let (result, events) = s.turn(&mut session, "two");
assert!(result.is_ok());
assert_eq!(session.records().len(), 8);
assert!(
matches!(
session.records()[7],
LogRecord::CacheLoss {
expected: 7052,
got: 0,
..
}
),
"{:?}",
session.records()[7]
);
assert!(events.contains(&TurnEvent::CacheLoss {
expected: 7052,
got: 0
}));
// turn1 then turn2 are a real consecutive pair: 15 + 29 + 2 = 46 left, 45 reused. No loss.
let s = setup(vec![]);
s.server
.route(CHAT, vec![Reply::fixture("turn1"), Reply::fixture("turn2")]);
let mut session = s.session("b");
assert!(s.turn(&mut session, "one").0.is_ok());
let (result, events) = s.turn(&mut session, "two");
assert!(result.is_ok());
assert!(
!session
.records()
.iter()
.any(|r| matches!(r, LogRecord::CacheLoss { .. }))
);
assert!(
!events
.iter()
.any(|e| matches!(e, TurnEvent::CacheLoss { .. }))
);
}
#[test]
fn retries_are_reported_and_leave_no_trace_in_the_log() {
let s = setup(vec![]);
s.server.route(
CHAT,
vec![
Reply::fixture("plain").cut_after(400),
Reply::fixture("plain"),
],
);
let mut session = s.session("a");
let (result, events) = s.turn(&mut session, "x");
assert!(result.is_ok(), "{result:?}");
assert!(
events
.iter()
.any(|e| matches!(e, TurnEvent::Retrying { attempt: 2, .. }))
);
assert_eq!(
types(session.records()),
["start", "user", "assistant", "usage"]
);
}
#[test]
fn resumed_sessions_continue_the_same_conversation() {
let s = setup(vec![]);
s.server
.route(CHAT, vec![Reply::fixture("plain"), Reply::fixture("turn2")]);
let mut session = s.session("a");
assert!(s.turn(&mut session, "one").0.is_ok());
drop(session);
let mut session = Session::open(&s.home.dir, SessionId::new("a").unwrap()).unwrap();
assert!(s.turn(&mut session, "two").0.is_ok());
let sent = s.server.requests_to(CHAT);
let m1 = sent[0].json()["messages"].as_array().unwrap().clone();
let m2 = sent[1].json()["messages"].as_array().unwrap().clone();
assert_eq!(
m2[..m1.len()],
m1[..],
"the second request extends the first"
);
assert_eq!(m2.len(), m1.len() + 2);
}
/// A small xorshift, so that the test needs no crate and a failure can be replayed by seed.
struct Rng(u64);
impl Rng {
fn next(&mut self) -> u64 {
self.0 ^= self.0 << 13;
self.0 ^= self.0 >> 7;
self.0 ^= self.0 << 17;
self.0
}
fn below(&mut self, n: u64) -> u64 {
self.next() % n
}
}
/// Every request of a generated conversation is a strict extension of the one before, and the
/// baseline (first message and tools) never changes.
#[test]
fn every_request_extends_the_previous_one() {
for seed in [1u64, 2, 3, 0xdead_beef, 0x9e37_79b9_7f4a_7c15] {
let mut rng = Rng(seed);
let mut s = setup(vec![]);
s.cfg.r#loop.repeat_detection = false;
s.cfg.r#loop.tool_result_cap = 64;
let mut replies = Vec::new();
let mut port_replies = Vec::new();
let turns = 2 + rng.below(4);
for _ in 0..turns {
let iterations = rng.below(3);
for _ in 0..iterations {
if rng.below(2) == 0 {
replies.push(Reply::fixture("tool_call"));
} else {
replies.push(Reply::fixture("find_tool"));
replies.push(Reply::fixture("call_tool"));
}
let size = rng.below(120) as usize;
port_replies.push(ok_result(&"r".repeat(size)));
port_replies.push(ok_result(&"r".repeat(size)));
}
replies.push(Reply::fixture(
["plain", "turn1", "turn2"][rng.below(3) as usize],
));
}
s.port = ScriptedPort::new(port_replies);
s.server.route(CHAT, replies);
let mut session = s.session("a");
for t in 0..turns {
let text = format!("turn {t} {}", "u".repeat(rng.below(30) as usize));
let (result, _) = s.turn(&mut session, &text);
assert!(result.is_ok(), "seed {seed}, turn {t}: {result:?}");
}
let sent = s.server.requests_to(CHAT);
assert!(sent.len() >= turns as usize, "seed {seed}");
let first = sent[0].json();
for (i, pair) in sent.windows(2).enumerate() {
let a = pair[0].json();
let b = pair[1].json();
let ma = a["messages"].as_array().unwrap();
let mb = b["messages"].as_array().unwrap();
assert!(
mb.len() > ma.len(),
"seed {seed}, request {}: not longer",
i + 1
);
assert_eq!(
mb[..ma.len()],
ma[..],
"seed {seed}, request {}: not an extension",
i + 1
);
assert_eq!(b["tools"], first["tools"], "seed {seed}: tools changed");
assert_eq!(
b["messages"][0], first["messages"][0],
"seed {seed}: system message changed"
);
}
// And the log replays to exactly the last request.
let replayed = loopd::baseline::messages(session.baseline(), session.records());
let last = sent.last().unwrap().json();
assert_eq!(
replayed.len(),
last["messages"].as_array().unwrap().len() + 1,
"seed {seed}: the last request plus the final answer"
);
}
}
crates/loopd/tests/support/turn.rs
+97
View File
@@ -0,0 +1,97 @@
//! A turn-loop setup shared by the `turn` and `limits` tests: a home, a fake server, a scripted
//! port and a registry. Included with `#[path]` because it needs `loopd::turn`, which the earlier
//! tasks' tests must not depend on. Do not edit.
#![allow(dead_code)]
use crate::support::{FakeServer, Home, ScriptedPort};
use loopd::baseline::Baseline;
use loopd::llama::Client;
use loopd::session::Session;
use loopd::tools::Registry;
use loopd::turn::{Runtime, TurnError, run_turn};
use proto::{LogRecord, SessionId, ToolResponse, TurnEvent};
pub struct Setup {
pub home: Home,
pub server: FakeServer,
pub cfg: loopd::config::Config,
pub client: Client,
pub port: ScriptedPort,
pub registry: Registry,
}
pub fn setup(replies: Vec<ToolResponse>) -> Setup {
let home = Home::new();
let server = FakeServer::start();
let cfg = home.config(&server.socket);
let client = Client::new(cfg.clone());
Setup {
home,
server,
cfg,
client,
port: ScriptedPort::new(replies),
registry: Registry::m2b(),
}
}
impl Setup {
pub fn session(&self, id: &str) -> Session {
let baseline = Baseline::assemble(&self.cfg, &self.registry).unwrap();
Session::create(
&self.home.dir,
SessionId::new(id).unwrap(),
baseline,
self.cfg.slots.main,
)
.unwrap()
}
pub fn runtime(&self) -> Runtime<'_> {
Runtime {
cfg: &self.cfg,
client: &self.client,
port: &self.port,
registry: &self.registry,
}
}
pub fn turn(
&self,
session: &mut Session,
text: &str,
) -> (Result<loopd::turn::TurnOutcome, TurnError>, Vec<TurnEvent>) {
let mut events = Vec::new();
let result = run_turn(session, &self.runtime(), text, &mut |e| {
events.push(e.clone())
});
(result, events)
}
}
/// The recordings were made in separate conversations, so their timings do not line up and the
/// loop rightly reports cache losses between them. Most tests are not about that, so `types`
/// and `records` leave `CacheLoss` out; one test checks it on purpose.
pub fn without_cache_loss(records: &[LogRecord]) -> Vec<LogRecord> {
records
.iter()
.filter(|r| !matches!(r, LogRecord::CacheLoss { .. }))
.cloned()
.collect()
}
pub fn types(records: &[LogRecord]) -> Vec<&'static str> {
without_cache_loss(records)
.iter()
.map(|r| match r {
LogRecord::SessionStart { .. } => "start",
LogRecord::User { .. } => "user",
LogRecord::Assistant { .. } => "assistant",
LogRecord::Usage { .. } => "usage",
LogRecord::ToolResult { .. } => "tool_result",
LogRecord::CacheLoss { .. } => "cache_loss",
LogRecord::EpochEnd { .. } => "epoch_end",
})
.collect()
}
crates/loopd/tests/turn.rs
+280
View File
@@ -0,0 +1,280 @@
//! Tests for one turn: record sequences and tool dispatch. Do not edit.
//! The limits and the append-only property are in `limits.rs`.
mod support;
#[path = "support/turn.rs"]
mod turn_support;
use loopd::tools::Registry;
use proto::{DataClass, LogRecord, SessionId, ToolResponse, TurnEvent};
use support::{Reply, ok_result};
use turn_support::{setup, types};
const CHAT: &str = "/v1/chat/completions";
#[test]
fn a_plain_turn() {
let s = setup(vec![]);
s.server.route(CHAT, vec![Reply::fixture("plain")]);
let mut session = s.session("a");
let (result, events) = s.turn(&mut session, "hello");
let outcome = result.unwrap();
assert_eq!(
outcome.content,
support::expected("plain")["content"].as_str().unwrap()
);
assert_eq!(
(outcome.usage.prompt_n, outcome.usage.predicted_n),
(46, 16)
);
assert_eq!(
types(session.records()),
["start", "user", "assistant", "usage"]
);
assert_eq!(
types(&s.home.records("a")),
["start", "user", "assistant", "usage"],
"on disk too"
);
assert!(
events
.iter()
.any(|e| matches!(e, TurnEvent::Content { .. }))
);
assert!(
!events
.iter()
.any(|e| matches!(e, TurnEvent::ToolCallStarted { .. }))
);
assert!(s.port.calls().is_empty());
let sent = s.server.requests_to(CHAT);
assert_eq!(sent.len(), 1);
let body = sent[0].json();
assert_eq!(body["messages"][0]["role"], "system");
assert_eq!(
body["messages"][0]["content"],
"You are Boxmaker, a test agent."
);
assert_eq!(
body["messages"][1],
serde_json::json!({"role": "user", "content": "hello"})
);
let names: Vec<&str> = body["tools"]
.as_array()
.unwrap()
.iter()
.map(|t| t["function"]["name"].as_str().unwrap())
.collect();
assert_eq!(names, ["clock", "find_tool", "call_tool"]);
assert_eq!(body["id_slot"], 0);
}
#[test]
fn a_tool_turn_goes_through_the_port_and_records_everything() {
let s = setup(vec![ok_result("straylight\n")]);
s.server.route(
CHAT,
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
);
let mut session = s.session("a");
let (result, events) = s.turn(&mut session, "read the hostname");
assert!(result.is_ok(), "{result:?}");
assert_eq!(
types(session.records()),
[
"start",
"user",
"assistant",
"usage",
"tool_result",
"assistant",
"usage"
]
);
let calls = s.port.calls();
assert_eq!(calls.len(), 1);
assert_eq!(calls[0].tool, "read_file");
assert_eq!(
calls[0].arguments, r#"{"path":"/etc/hostname"}"#,
"arguments are passed on unparsed"
);
assert_eq!(calls[0].session, SessionId::new("a").unwrap());
assert_eq!(calls[0].call, proto::CallId(1));
match &session.records()[4] {
LogRecord::ToolResult {
call,
tool_call_id,
content,
class,
untrusted,
truncated,
..
} => {
assert_eq!(*call, proto::CallId(1));
assert_eq!(
tool_call_id, "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F",
"the server's id, so the template can pair it"
);
assert_eq!(content, "straylight\n");
assert_eq!(
(*class, *untrusted, *truncated),
(DataClass::Private, true, false)
);
}
other => panic!("{other:?}"),
}
let kinds: Vec<&str> = events
.iter()
.filter_map(|e| match e {
TurnEvent::ToolCallStarted { name } => Some(name.as_str()),
TurnEvent::ToolResult { name, .. } => Some(name.as_str()),
_ => None,
})
.collect();
assert_eq!(kinds, ["read_file", "read_file"]);
// The second request extends the first: the tool result sits after the assistant turn.
let sent = s.server.requests_to(CHAT);
let m2 = sent[1].json()["messages"].as_array().unwrap().clone();
assert_eq!(m2[2]["role"], "assistant");
assert_eq!(
m2[2]["tool_calls"][0]["id"],
"wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F"
);
assert_eq!(
m2[3],
serde_json::json!({"role": "tool", "tool_call_id": "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F", "content": "straylight\n"})
);
}
#[test]
fn find_tool_and_call_tool_reach_the_port_only_for_the_target() {
let s = setup(vec![ok_result("box")]);
s.server.route(
CHAT,
vec![
Reply::fixture("find_tool"),
Reply::fixture("call_tool"),
Reply::fixture("plain"),
],
);
let mut session = s.session("a");
let (result, _) = s.turn(&mut session, "echo box");
assert!(result.is_ok(), "{result:?}");
assert_eq!(
types(session.records()),
[
"start",
"user",
"assistant",
"usage",
"tool_result",
"assistant",
"usage",
"tool_result",
"assistant",
"usage"
]
);
let calls = s.port.calls();
assert_eq!(
calls.len(),
1,
"find_tool is answered by loopd; only echo reaches the port"
);
assert_eq!(calls[0].tool, "echo");
assert_eq!(
serde_json::from_str::<serde_json::Value>(&calls[0].arguments).unwrap(),
serde_json::json!({"text": "box"})
);
match &session.records()[4] {
LogRecord::ToolResult {
content,
class,
untrusted,
..
} => {
assert!(
content.contains("\"name\":\"echo\"")
&& content.ends_with("Call it with call_tool."),
"{content}"
);
assert_eq!((*class, *untrusted), (DataClass::Public, false));
}
other => panic!("{other:?}"),
}
}
#[test]
fn a_call_tool_for_an_unknown_tool_never_reaches_the_port() {
// The recorded call_tool asks for "echo"; with echo removed from the registry it is unknown.
let mut s = setup(vec![]);
s.registry = Registry::new(vec![loopd::tools::Entry {
schema: loopd::tools::clock_schema(),
core: true,
}]);
s.server.route(
CHAT,
vec![Reply::fixture("call_tool"), Reply::fixture("plain")],
);
let mut session = s.session("a");
assert!(s.turn(&mut session, "x").0.is_ok());
assert!(s.port.calls().is_empty());
assert!(
matches!(&session.records()[4], LogRecord::ToolResult { content, .. } if content.contains("No tool named \"echo\""))
);
}
#[test]
fn the_result_cap_applies_when_appended() {
let mut s = setup(vec![ok_result(&"x".repeat(100))]);
s.cfg.r#loop.tool_result_cap = 20;
s.server.route(
CHAT,
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
);
let mut session = s.session("a");
let (result, events) = s.turn(&mut session, "x");
assert!(result.is_ok());
match &session.records()[4] {
LogRecord::ToolResult {
content, truncated, ..
} => {
assert_eq!(content, &format!("{}\n[truncated]", "x".repeat(20)));
assert!(truncated);
}
other => panic!("{other:?}"),
}
assert!(events.iter().any(|e| matches!(
e,
TurnEvent::ToolResult {
truncated: true,
..
}
)));
let m2 = s.server.requests_to(CHAT)[1].json();
assert_eq!(
m2["messages"][3]["content"].as_str().unwrap().len(),
20 + "\n[truncated]".len(),
"the model sees the capped text"
);
}
#[test]
fn tool_failures_and_denials_become_results_the_model_can_read() {
let s = setup(vec![ToolResponse::Failed {
message: "disk on fire".to_string(),
}]);
s.server.route(
CHAT,
vec![Reply::fixture("tool_call"), Reply::fixture("plain")],
);
let mut session = s.session("a");
assert!(s.turn(&mut session, "x").0.is_ok());
assert!(
matches!(&session.records()[4], LogRecord::ToolResult { content, class: DataClass::Public, .. } if content.contains("disk on fire"))
);
}