Add M2a plan: thirteen tasks, tests, fake server and recordings
The tasks build the inference path: emsha-backed SHA-256, inferproxy, config, a hand-written HTTP and SSE client, request building, delta assembly, the chat state machine, the thinking cap, the slot gate with retry, the startup self-test and on-device verification. Everything the tasks copy in was checked against a private reference implementation: the gate passes after each task in order, the timing tests pass repeatedly under CPU load, and the reference passes the self-test and all four device checks on straylight. Expected results for the recorded streams were derived by a separate script. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
Claude Fable 5.1
@@ -0,0 +1,281 @@
|
||||
//! Tests for folding streamed chunks into a completion. Do not edit.
|
||||
//!
|
||||
//! The streams are recorded from the real server. The expected results in
|
||||
//! `tests/fixtures/expected/` were derived from the same recordings by a separate script, so
|
||||
//! this checks the Rust code against an independent reading of the same bytes.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::assemble::Assembler;
|
||||
use loopd::llama::{ChatEvent, Completion, FinishReason, InferError};
|
||||
|
||||
const ALL: [&str; 7] = [
|
||||
"plain",
|
||||
"thinking",
|
||||
"tool_call",
|
||||
"progress",
|
||||
"capped",
|
||||
"turn1",
|
||||
"turn2",
|
||||
];
|
||||
|
||||
/// The `data:` payloads of a recorded stream, without `[DONE]`.
|
||||
fn payloads(name: &str) -> Vec<String> {
|
||||
let raw = String::from_utf8(support::fixture_bytes("http", &format!("{name}.http"))).unwrap();
|
||||
raw.lines()
|
||||
.filter_map(|l| l.strip_prefix("data: "))
|
||||
.filter(|p| p.starts_with('{'))
|
||||
.map(str::to_string)
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn assemble(name: &str) -> (Completion, Vec<ChatEvent>) {
|
||||
let mut a = Assembler::new();
|
||||
let mut events = Vec::new();
|
||||
for p in payloads(name) {
|
||||
events.extend(a.push(&p).unwrap());
|
||||
}
|
||||
assert!(a.is_finished(), "{name}");
|
||||
(a.finish(false).unwrap(), events)
|
||||
}
|
||||
|
||||
fn opt(v: &serde_json::Value) -> Option<String> {
|
||||
v.as_str().map(str::to_string)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn every_recording_assembles_to_what_the_independent_script_found() {
|
||||
for name in ALL {
|
||||
let want = support::expected(name);
|
||||
let (got, events) = assemble(name);
|
||||
assert_eq!(Some(got.id.clone()), opt(&want["id"]), "{name}: id");
|
||||
assert_eq!(got.content, opt(&want["content"]), "{name}: content");
|
||||
assert_eq!(
|
||||
got.reasoning_content,
|
||||
opt(&want["reasoning_content"]),
|
||||
"{name}: reasoning"
|
||||
);
|
||||
let finish = match got.finish_reason {
|
||||
FinishReason::Stop => "stop",
|
||||
FinishReason::ToolCalls => "tool_calls",
|
||||
FinishReason::Length => "length",
|
||||
};
|
||||
assert_eq!(finish, want["finish_reason"], "{name}: finish reason");
|
||||
assert_eq!(got.timings.cache_n, want["cache_n"], "{name}");
|
||||
assert_eq!(got.timings.prompt_n, want["prompt_n"], "{name}");
|
||||
assert_eq!(got.timings.predicted_n, want["predicted_n"], "{name}");
|
||||
assert_eq!(
|
||||
got.reasoning_tokens, want["reasoning_tokens"],
|
||||
"{name}: reasoning tokens"
|
||||
);
|
||||
assert!(!got.thinking_capped, "{name}");
|
||||
|
||||
let calls: Vec<serde_json::Value> = got
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|c| serde_json::json!({"id": c.id, "name": c.name, "arguments": c.arguments}))
|
||||
.collect();
|
||||
assert_eq!(
|
||||
serde_json::Value::Array(calls),
|
||||
want["tool_calls"],
|
||||
"{name}: tool calls"
|
||||
);
|
||||
|
||||
let count = |f: fn(&ChatEvent) -> bool| events.iter().filter(|e| f(e)).count();
|
||||
assert_eq!(
|
||||
count(|e| matches!(e, ChatEvent::Reasoning(_))),
|
||||
want["reasoning_events"],
|
||||
"{name}"
|
||||
);
|
||||
assert_eq!(
|
||||
count(|e| matches!(e, ChatEvent::Content(_))),
|
||||
want["content_events"],
|
||||
"{name}"
|
||||
);
|
||||
assert_eq!(
|
||||
count(|e| matches!(e, ChatEvent::ToolCallDelta { .. })),
|
||||
want["tool_events"],
|
||||
"{name}"
|
||||
);
|
||||
let progress: Vec<serde_json::Value> = events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
ChatEvent::Progress {
|
||||
total,
|
||||
cache,
|
||||
processed,
|
||||
} => Some(serde_json::json!([total, cache, processed])),
|
||||
_ => None,
|
||||
})
|
||||
.collect();
|
||||
assert_eq!(
|
||||
serde_json::Value::Array(progress),
|
||||
want["progress"],
|
||||
"{name}: progress"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_events_carry_the_same_text_as_the_completion() {
|
||||
for name in ALL {
|
||||
let (got, events) = assemble(name);
|
||||
let mut content = String::new();
|
||||
let mut reasoning = String::new();
|
||||
let mut arguments = String::new();
|
||||
for e in &events {
|
||||
match e {
|
||||
ChatEvent::Content(t) => content.push_str(t),
|
||||
ChatEvent::Reasoning(t) => reasoning.push_str(t),
|
||||
ChatEvent::ToolCallDelta { arguments: a, .. } => arguments.push_str(a),
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
assert_eq!(got.content.unwrap_or_default(), content, "{name}");
|
||||
assert_eq!(
|
||||
got.reasoning_content.unwrap_or_default(),
|
||||
reasoning,
|
||||
"{name}"
|
||||
);
|
||||
let all_arguments: String = got
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|c| c.arguments.as_str())
|
||||
.collect();
|
||||
assert_eq!(all_arguments, arguments, "{name}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_tool_call_arguments_are_valid_json() {
|
||||
let (got, events) = assemble("tool_call");
|
||||
let args: serde_json::Value = serde_json::from_str(&got.tool_calls[0].arguments).unwrap();
|
||||
assert_eq!(args, serde_json::json!({"path": "/etc/hostname"}));
|
||||
// Only the first delta of a call carries its id and name.
|
||||
let named = events
|
||||
.iter()
|
||||
.filter(|e| matches!(e, ChatEvent::ToolCallDelta { name: Some(_), .. }))
|
||||
.count();
|
||||
assert_eq!(named, 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reasoning_state_follows_the_stream() {
|
||||
let mut a = Assembler::new();
|
||||
assert!(!a.in_reasoning());
|
||||
assert_eq!(a.reasoning_tokens(), 0);
|
||||
let mut seen_reasoning = false;
|
||||
let mut last = 0;
|
||||
for p in payloads("thinking") {
|
||||
let events = a.push(&p).unwrap();
|
||||
if events.iter().any(|e| matches!(e, ChatEvent::Reasoning(_))) {
|
||||
seen_reasoning = true;
|
||||
assert!(a.in_reasoning());
|
||||
assert!(
|
||||
a.reasoning_tokens() > last,
|
||||
"the count must grow with each reasoning chunk"
|
||||
);
|
||||
last = a.reasoning_tokens();
|
||||
}
|
||||
if events.iter().any(|e| matches!(e, ChatEvent::Content(_))) {
|
||||
assert!(!a.in_reasoning(), "content ends the reasoning block");
|
||||
assert_eq!(a.reasoning_tokens(), last, "and the count stops");
|
||||
}
|
||||
}
|
||||
assert!(seen_reasoning);
|
||||
assert_eq!(last, 49);
|
||||
assert!(a.finish(true).unwrap().thinking_capped);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn two_tool_calls_in_one_completion() {
|
||||
let chunk = |delta: &str, finish: &str| {
|
||||
format!(
|
||||
r#"{{"id":"c1","choices":[{{"index":0,"finish_reason":{finish},"delta":{delta}}}]}}"#
|
||||
)
|
||||
};
|
||||
let mut a = Assembler::new();
|
||||
for delta in [
|
||||
r#"{"tool_calls":[{"index":0,"id":"a","type":"function","function":{"name":"one","arguments":"{\"x\""}}]}"#,
|
||||
r#"{"tool_calls":[{"index":0,"function":{"arguments":":1}"}}]}"#,
|
||||
r#"{"tool_calls":[{"index":1,"id":"b","type":"function","function":{"name":"two","arguments":"{}"}}]}"#,
|
||||
] {
|
||||
a.push(&chunk(delta, "null")).unwrap();
|
||||
}
|
||||
a.push(&chunk("{}", "\"tool_calls\"")).unwrap();
|
||||
let got = a.finish(false).unwrap();
|
||||
let calls: Vec<(&str, &str, &str)> = got
|
||||
.tool_calls
|
||||
.iter()
|
||||
.map(|c| (c.id.as_str(), c.name.as_str(), c.arguments.as_str()))
|
||||
.collect();
|
||||
assert_eq!(calls, vec![("a", "one", "{\"x\":1}"), ("b", "two", "{}")]);
|
||||
}
|
||||
|
||||
/// The server sends fields this code does not use, and newer builds may send more. Unknown
|
||||
/// fields in the server's chunks must be ignored. (Our own formats reject unknown fields; the
|
||||
/// server's format is not ours.)
|
||||
#[test]
|
||||
fn unknown_fields_from_the_server_are_ignored() {
|
||||
let mut a = Assembler::new();
|
||||
let chunk = r#"{"id":"c1","zz_new":1,"choices":[{"index":0,"zz":2,"finish_reason":"stop","delta":{"content":"hi","zz":3}}],"timings":{"cache_n":1,"prompt_n":2,"predicted_n":3,"zz_ms":0.5}}"#;
|
||||
a.push(chunk).unwrap();
|
||||
let got = a.finish(false).unwrap();
|
||||
assert_eq!(got.content.as_deref(), Some("hi"));
|
||||
assert_eq!(
|
||||
(
|
||||
got.timings.cache_n,
|
||||
got.timings.prompt_n,
|
||||
got.timings.predicted_n
|
||||
),
|
||||
(1, 2, 3)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bad_streams_are_errors_not_panics() {
|
||||
let mut a = Assembler::new();
|
||||
assert!(matches!(a.push("not json"), Err(InferError::Protocol(_))));
|
||||
assert!(matches!(a.push("[1,2]"), Err(InferError::Protocol(_))));
|
||||
|
||||
let mut a = Assembler::new();
|
||||
let odd = r#"{"id":"c1","choices":[{"index":0,"finish_reason":"content_filter","delta":{}}]}"#;
|
||||
assert!(
|
||||
matches!(a.push(odd), Err(InferError::Protocol(_))),
|
||||
"an unknown finish reason"
|
||||
);
|
||||
|
||||
let mut a = Assembler::new();
|
||||
let skip = r#"{"id":"c1","choices":[{"index":0,"finish_reason":null,"delta":{"tool_calls":[{"index":5,"id":"a","function":{"name":"n","arguments":""}}]}}]}"#;
|
||||
assert!(
|
||||
matches!(a.push(skip), Err(InferError::Protocol(_))),
|
||||
"a tool call index that skips ahead"
|
||||
);
|
||||
|
||||
let mut a = Assembler::new();
|
||||
let nameless = r#"{"id":"c1","choices":[{"index":0,"finish_reason":"tool_calls","delta":{"tool_calls":[{"index":0,"function":{"arguments":"{}"}}]}}]}"#;
|
||||
a.push(nameless).unwrap();
|
||||
assert!(
|
||||
matches!(a.finish(false), Err(InferError::Protocol(_))),
|
||||
"a tool call without id or name"
|
||||
);
|
||||
|
||||
// A stream that stops before any finish_reason.
|
||||
let mut a = Assembler::new();
|
||||
for p in payloads("plain").iter().take(5) {
|
||||
a.push(p).unwrap();
|
||||
}
|
||||
assert!(!a.is_finished());
|
||||
assert!(matches!(
|
||||
a.finish(false),
|
||||
Err(InferError::StreamClosedEarly)
|
||||
));
|
||||
|
||||
let mut a = Assembler::new();
|
||||
a.push(r#"{"choices":[{"index":0,"finish_reason":"stop","delta":{}}]}"#)
|
||||
.unwrap();
|
||||
assert!(
|
||||
matches!(a.finish(false), Err(InferError::Protocol(_))),
|
||||
"no completion id"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,168 @@
|
||||
//! Tests for the thinking cap. Do not edit.
|
||||
//!
|
||||
//! The "capped" recording has 63 reasoning chunks (63 reasoning tokens) and then an answer. The
|
||||
//! fake server plays it regardless of the control call, so the cap and the overrun allowance
|
||||
//! decide what the client makes of it.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client, Completion, InferError};
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
const CONTROL: &str = "/v1/chat/completions/control";
|
||||
|
||||
fn run(
|
||||
server: &FakeServer,
|
||||
cap: u64,
|
||||
overrun: u64,
|
||||
) -> (Result<Completion, InferError>, Vec<ChatEvent>) {
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.limits.thinking_cap = cap;
|
||||
cfg.limits.thinking_overrun = overrun;
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![ChatMessage::User {
|
||||
content: "think".to_string(),
|
||||
}],
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let mut events = Vec::new();
|
||||
let result = Client::new(cfg).chat(&req, &mut |e| events.push(e.clone()));
|
||||
(result, events)
|
||||
}
|
||||
|
||||
fn capped_events(events: &[ChatEvent]) -> Vec<u64> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
ChatEvent::ThinkingCapped { tokens } => Some(*tokens),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn under_the_cap_nothing_happens() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("capped")]);
|
||||
let (result, events) = run(&server, 64, 10);
|
||||
let done = result.unwrap();
|
||||
assert!(!done.thinking_capped);
|
||||
assert_eq!(done.reasoning_tokens, 63);
|
||||
assert!(capped_events(&events).is_empty());
|
||||
assert!(
|
||||
server.requests_to(CONTROL).is_empty(),
|
||||
"63 tokens is under a cap of 64"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn at_the_cap_the_control_call_is_sent_once() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("capped")]);
|
||||
server.route(CONTROL, vec![Reply::fixture("control")]);
|
||||
let (result, events) = run(&server, 20, 100);
|
||||
let done = result.unwrap();
|
||||
assert!(done.thinking_capped);
|
||||
assert_eq!(
|
||||
done.reasoning_tokens, 63,
|
||||
"the count keeps running after the cap"
|
||||
);
|
||||
assert_eq!(
|
||||
done.content.as_deref(),
|
||||
support::expected("capped")["content"].as_str()
|
||||
);
|
||||
assert_eq!(
|
||||
capped_events(&events),
|
||||
vec![20],
|
||||
"one event, at the token count that hit the cap"
|
||||
);
|
||||
|
||||
let control = server.requests_to(CONTROL);
|
||||
assert_eq!(
|
||||
control.len(),
|
||||
1,
|
||||
"the control call is made once, not on every later chunk"
|
||||
);
|
||||
let id = support::expected("capped")["id"].clone();
|
||||
assert_eq!(
|
||||
control[0].json(),
|
||||
serde_json::json!({"id": id, "action": "reasoning_end", "model": "test-model"})
|
||||
);
|
||||
|
||||
// The event sits between the reasoning that hit the cap and what came after.
|
||||
let at = events
|
||||
.iter()
|
||||
.position(|e| matches!(e, ChatEvent::ThinkingCapped { .. }))
|
||||
.unwrap();
|
||||
let before = events[..at]
|
||||
.iter()
|
||||
.filter(|e| matches!(e, ChatEvent::Reasoning(_)))
|
||||
.count();
|
||||
assert_eq!(before, 20);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thinking_on_past_the_allowance_is_an_overrun() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("capped")]);
|
||||
server.route(CONTROL, vec![Reply::fixture("control")]);
|
||||
// Capped at 20; 63 reasoning tokens is more than 20 + 30.
|
||||
let (result, events) = run(&server, 20, 30);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::ThinkingOverrun)),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(capped_events(&events), vec![20]);
|
||||
let reasoning = events
|
||||
.iter()
|
||||
.filter(|e| matches!(e, ChatEvent::Reasoning(_)))
|
||||
.count();
|
||||
assert_eq!(reasoning, 50, "the client stops reading at 20 + 30 tokens");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_allowance_is_exact() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("capped")]);
|
||||
server.route(CONTROL, vec![Reply::fixture("control")]);
|
||||
// 63 is not 20 + 44, so this is allowed; with 43 it would be an overrun at token 63.
|
||||
assert!(run(&server, 20, 44).0.is_ok());
|
||||
assert!(matches!(
|
||||
run(&server, 20, 43).0,
|
||||
Err(InferError::ThinkingOverrun)
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_control_call_that_fails_is_an_overrun() {
|
||||
for reply in [
|
||||
Reply::json(200, r#"{"success":false,"message":"no such completion"}"#),
|
||||
Reply::json(500, r#"{"error":"boom"}"#),
|
||||
Reply::json(200, "not json"),
|
||||
] {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("capped")]);
|
||||
server.route(CONTROL, vec![reply]);
|
||||
let (result, events) = run(&server, 20, 100);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::ThinkingOverrun)),
|
||||
"{result:?}"
|
||||
);
|
||||
assert!(
|
||||
capped_events(&events).is_empty(),
|
||||
"the cap did not take effect, so no event"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_completion_without_reasoning_is_never_capped() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("plain")]);
|
||||
let (result, _) = run(&server, 1, 0);
|
||||
assert!(!result.unwrap().thinking_capped);
|
||||
assert!(server.requests_to(CONTROL).is_empty());
|
||||
}
|
||||
@@ -0,0 +1,294 @@
|
||||
//! Tests for one request: events, waits, liveness and errors. Do not edit.
|
||||
//!
|
||||
//! Limits come from `support::test_config`: poll 40 ms, busy wait 400 ms, load wait 300 ms,
|
||||
//! idle grace 200 ms, liveness 150 ms.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client, FinishReason, InferError};
|
||||
use std::time::{Duration, Instant};
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
const IDLE: &str = r#"[{"id":0,"is_processing":false},{"id":1,"is_processing":false}]"#;
|
||||
const SLOT0_BUSY: &str = r#"[{"id":0,"is_processing":true},{"id":1,"is_processing":false}]"#;
|
||||
const SLOT1_BUSY: &str = r#"[{"id":0,"is_processing":false},{"id":1,"is_processing":true}]"#;
|
||||
|
||||
fn request() -> ChatRequest {
|
||||
ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![ChatMessage::User {
|
||||
content: "hi".to_string(),
|
||||
}],
|
||||
tools: vec![],
|
||||
thinking: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn run(
|
||||
server: &FakeServer,
|
||||
) -> (
|
||||
Result<loopd::llama::Completion, InferError>,
|
||||
Vec<ChatEvent>,
|
||||
Duration,
|
||||
) {
|
||||
let client = Client::new(support::test_config(&server.socket));
|
||||
let mut events = Vec::new();
|
||||
let started = Instant::now();
|
||||
let result = client.chat(&request(), &mut |e| events.push(e.clone()));
|
||||
(result, events, started.elapsed())
|
||||
}
|
||||
|
||||
fn waiting(events: &[ChatEvent]) -> Vec<bool> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
ChatEvent::Waiting { slot_busy } => Some(*slot_busy),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_recorded_completion_comes_back_whole() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("tool_call")]);
|
||||
let (result, events, _) = run(&server);
|
||||
let done = result.unwrap();
|
||||
let want = support::expected("tool_call");
|
||||
assert_eq!(done.finish_reason, FinishReason::ToolCalls);
|
||||
assert_eq!(done.content.as_deref(), want["content"].as_str());
|
||||
assert_eq!(done.tool_calls[0].arguments, r#"{"path":"/etc/hostname"}"#);
|
||||
assert_eq!(done.timings.prompt_n, 312);
|
||||
assert!(!done.thinking_capped);
|
||||
assert!(
|
||||
waiting(&events).is_empty(),
|
||||
"the answer came at once, so nothing was polled"
|
||||
);
|
||||
assert!(matches!(
|
||||
events.first(),
|
||||
Some(ChatEvent::Progress { total: 312, .. })
|
||||
));
|
||||
|
||||
let sent = &server.requests_to(CHAT)[0];
|
||||
assert_eq!(sent.method, "POST");
|
||||
assert_eq!(sent.json()["messages"][0]["content"], "hi");
|
||||
assert_eq!(sent.json()["id_slot"], 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_result_does_not_depend_on_how_the_bytes_arrive() {
|
||||
for piece in [1, 17, 4096] {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("thinking").trickle(piece, 0)]);
|
||||
let done = run(&server).0.unwrap();
|
||||
let want = support::expected("thinking");
|
||||
assert_eq!(
|
||||
done.content.as_deref(),
|
||||
want["content"].as_str(),
|
||||
"pieces of {piece}"
|
||||
);
|
||||
assert_eq!(
|
||||
done.reasoning_content.as_deref(),
|
||||
want["reasoning_content"].as_str()
|
||||
);
|
||||
assert_eq!(done.reasoning_tokens, 49);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_busy_slot_is_waited_out() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("plain").head_delay(250)]);
|
||||
server.route("/slots", vec![Reply::json(200, SLOT0_BUSY)]);
|
||||
let (result, events, took) = run(&server);
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
let polls = waiting(&events);
|
||||
assert!(polls.len() >= 4, "250 ms at 40 ms per poll: {polls:?}");
|
||||
assert!(polls.iter().all(|busy| *busy));
|
||||
assert!(took >= Duration::from_millis(250));
|
||||
assert!(server.requests_to("/slots").len() >= 4);
|
||||
assert_eq!(
|
||||
server.requests_to("/slots")[0].target,
|
||||
"/slots?model=test-model"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slot_that_stays_busy_is_a_wait_timeout() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("plain").head_delay(5_000)]);
|
||||
server.route("/slots", vec![Reply::json(200, SLOT0_BUSY)]);
|
||||
let (result, events, took) = run(&server);
|
||||
assert!(matches!(result, Err(InferError::WaitTimeout)), "{result:?}");
|
||||
assert!(
|
||||
took >= Duration::from_millis(400) && took < Duration::from_millis(1500),
|
||||
"{took:?}"
|
||||
);
|
||||
assert!(waiting(&events).iter().all(|busy| *busy));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_the_requests_own_slot_counts_as_busy() {
|
||||
// Slot 1 is busy, slot 0 (ours) is idle and silent: that is a stall, not a queue.
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("plain").head_delay(5_000)]);
|
||||
server.route("/slots", vec![Reply::json(200, SLOT1_BUSY)]);
|
||||
let (result, events, took) = run(&server);
|
||||
assert!(matches!(result, Err(InferError::Stalled)), "{result:?}");
|
||||
assert!(
|
||||
took >= Duration::from_millis(200) && took < Duration::from_millis(1000),
|
||||
"{took:?}"
|
||||
);
|
||||
assert!(waiting(&events).iter().all(|busy| !*busy));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_unavailable_server_is_a_load_timeout() {
|
||||
for slots_reply in [
|
||||
Reply::json(503, r#"{"error":"loading"}"#),
|
||||
Reply::raw("").cut_after(0),
|
||||
] {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("plain").head_delay(5_000)]);
|
||||
server.route("/slots", vec![slots_reply]);
|
||||
let (result, _, took) = run(&server);
|
||||
assert!(matches!(result, Err(InferError::LoadTimeout)), "{result:?}");
|
||||
assert!(
|
||||
took >= Duration::from_millis(300) && took < Duration::from_millis(1500),
|
||||
"{took:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_wait_clocks_restart_when_the_state_changes() {
|
||||
// Busy for about 5 polls, then idle: the idle grace starts counting from there, so the
|
||||
// request outlives 200 ms of busy plus most of the 200 ms grace and then completes.
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("plain").head_delay(330)]);
|
||||
let busy = Reply::json(200, SLOT0_BUSY);
|
||||
server.route(
|
||||
"/slots",
|
||||
vec![
|
||||
busy.clone(),
|
||||
busy.clone(),
|
||||
busy.clone(),
|
||||
busy.clone(),
|
||||
busy,
|
||||
Reply::json(200, IDLE),
|
||||
],
|
||||
);
|
||||
let (result, events, _) = run(&server);
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
let polls = waiting(&events);
|
||||
assert!(
|
||||
polls.starts_with(&[true, true, true, true, true]),
|
||||
"{polls:?}"
|
||||
);
|
||||
assert_eq!(polls.last(), Some(&false), "{polls:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn silence_in_mid_stream_is_a_stall() {
|
||||
let server = FakeServer::start();
|
||||
let reply = Reply::fixture("thinking");
|
||||
let cut = reply.offset_after_events(10);
|
||||
server.route(CHAT, vec![reply.hang_after(cut, 5_000)]);
|
||||
let (result, events, took) = run(&server);
|
||||
assert!(matches!(result, Err(InferError::Stalled)), "{result:?}");
|
||||
assert!(
|
||||
took >= Duration::from_millis(150) && took < Duration::from_millis(1000),
|
||||
"{took:?}"
|
||||
);
|
||||
assert!(
|
||||
events.iter().any(|e| matches!(e, ChatEvent::Reasoning(_))),
|
||||
"events before the stall were delivered"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_slow_but_steady_stream_is_not_a_stall() {
|
||||
// 60 ms between pieces is well inside the 150 ms liveness limit, however long the whole takes.
|
||||
let server = FakeServer::start();
|
||||
let size = support::fixture_bytes("http", "turn1.http").len();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").trickle(size / 8, 60)]);
|
||||
let (result, _, took) = run(&server);
|
||||
assert_eq!(result.unwrap().content.as_deref(), Some("Blue"));
|
||||
assert!(
|
||||
took >= Duration::from_millis(400),
|
||||
"the stream took longer than the liveness limit: {took:?}"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stream_that_is_cut_is_closed_early_wherever_the_cut_falls() {
|
||||
let reply = Reply::fixture("plain");
|
||||
let whole = support::fixture_bytes("http", "plain.http").len();
|
||||
let after_five = reply.offset_after_events(5);
|
||||
// After the head only; in the middle of an event; between events; just before [DONE].
|
||||
for cut in [300, after_five + 20, after_five, whole - 30] {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![reply.clone().cut_after(cut)]);
|
||||
let (result, _, _) = run(&server);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::StreamClosedEarly)),
|
||||
"cut at {cut}: {result:?}"
|
||||
);
|
||||
}
|
||||
// Cut before the head is complete.
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![reply.cut_after(40)]);
|
||||
assert!(matches!(run(&server).0, Err(InferError::StreamClosedEarly)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn error_statuses_keep_their_body() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("bad_request")]);
|
||||
match run(&server).0 {
|
||||
Err(InferError::Http { status: 400, body }) => {
|
||||
assert!(body.contains("Expected 'messages'"), "{body}")
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
let server = FakeServer::start();
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![Reply::json(503, r#"{"error":{"message":"Loading model"}}"#)],
|
||||
);
|
||||
assert!(matches!(
|
||||
run(&server).0,
|
||||
Err(InferError::Http { status: 503, .. })
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn garbage_in_the_stream_is_a_protocol_error() {
|
||||
let server = FakeServer::start();
|
||||
let raw = "HTTP/1.1 200 OK\r\nContent-Type: text/event-stream\r\n\r\ndata: {not json}\n\n";
|
||||
server.route(CHAT, vec![Reply::raw(raw)]);
|
||||
assert!(matches!(run(&server).0, Err(InferError::Protocol(_))));
|
||||
|
||||
let server = FakeServer::start();
|
||||
let huge = format!(
|
||||
"HTTP/1.1 200 OK\r\n\r\ndata: {}\n\n",
|
||||
"x".repeat(2 * 1024 * 1024)
|
||||
);
|
||||
server.route(CHAT, vec![Reply::raw(huge)]);
|
||||
assert!(
|
||||
matches!(run(&server).0, Err(InferError::Protocol(_))),
|
||||
"a line over 1 MiB"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn no_server_is_a_connect_error() {
|
||||
let server = FakeServer::start();
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.infer.socket = std::env::temp_dir().join("loopd-no-such-socket.sock");
|
||||
let result = Client::new(cfg).chat(&request(), &mut |_| {});
|
||||
assert!(matches!(result, Err(InferError::Connect(_))), "{result:?}");
|
||||
let e: Box<dyn std::error::Error> = Box::new(InferError::Stalled);
|
||||
assert!(!e.to_string().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
//! Tests for `loopd`'s configuration. Do not edit: these define the required behaviour.
|
||||
|
||||
use loopd::config::{Config, ConfigError, Limits, Sampling};
|
||||
use std::path::{Path, PathBuf};
|
||||
|
||||
fn fixture(name: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures/config")
|
||||
.join(name)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn minimal_file_gets_the_documented_defaults() {
|
||||
let c = Config::load(&fixture("minimal.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.infer.socket,
|
||||
PathBuf::from("/run/boxmaker/infer/infer.sock")
|
||||
);
|
||||
assert_eq!(c.infer.model, "ornith-1.5-35b-a3b");
|
||||
assert_eq!((c.slots.main, c.slots.background), (0, 1));
|
||||
assert_eq!(c.expect.n_ctx, 131_072);
|
||||
assert_eq!(c.expect.slots, 2);
|
||||
assert_eq!(
|
||||
c.expect.template_sha256.to_hex(),
|
||||
"f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
);
|
||||
assert_eq!(
|
||||
c.sampling,
|
||||
Sampling {
|
||||
temperature: 0.6,
|
||||
top_p: 0.95,
|
||||
top_k: 20
|
||||
}
|
||||
);
|
||||
let want = Limits {
|
||||
poll_ms: 5_000,
|
||||
busy_wait_ms: 600_000,
|
||||
load_wait_ms: 180_000,
|
||||
idle_grace_ms: 30_000,
|
||||
liveness_ms: 30_000,
|
||||
thinking_cap: 4_096,
|
||||
thinking_overrun: 256,
|
||||
max_tokens: 8_192,
|
||||
queue_len: 8,
|
||||
retry_attempts: 4,
|
||||
retry_backoff_ms: vec![2_000, 8_000, 30_000],
|
||||
retry_window_ms: 300_000,
|
||||
};
|
||||
assert_eq!(c.limits, want);
|
||||
assert_eq!(Limits::default(), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_file_overrides_every_default() {
|
||||
let c = Config::load(&fixture("full.toml")).unwrap();
|
||||
assert_eq!(
|
||||
c.sampling,
|
||||
Sampling {
|
||||
temperature: 0.2,
|
||||
top_p: 0.9,
|
||||
top_k: 40
|
||||
}
|
||||
);
|
||||
let want = Limits {
|
||||
poll_ms: 50,
|
||||
busy_wait_ms: 200,
|
||||
load_wait_ms: 300,
|
||||
idle_grace_ms: 150,
|
||||
liveness_ms: 100,
|
||||
thinking_cap: 20,
|
||||
thinking_overrun: 10,
|
||||
max_tokens: 512,
|
||||
queue_len: 1,
|
||||
retry_attempts: 2,
|
||||
retry_backoff_ms: vec![10],
|
||||
retry_window_ms: 1_000,
|
||||
};
|
||||
assert_eq!(c.limits, want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_partial_limits_table_keeps_the_other_defaults() {
|
||||
let text = std::fs::read_to_string(fixture("minimal.toml")).unwrap();
|
||||
let c = Config::parse(&format!("{text}\n[limits]\nliveness_ms = 1234\n")).unwrap();
|
||||
assert_eq!(c.limits.liveness_ms, 1234);
|
||||
assert_eq!(c.limits.poll_ms, 5_000);
|
||||
}
|
||||
|
||||
/// Every table in the file must reject a key it does not know: a misspelt limit that silently
|
||||
/// fell back to its default would be a limit the owner believes is set and is not.
|
||||
#[test]
|
||||
fn unknown_keys_are_errors_in_every_table() {
|
||||
let text = std::fs::read_to_string(fixture("full.toml")).unwrap();
|
||||
assert!(Config::parse(&text).is_ok());
|
||||
for table in ["infer", "slots", "expect", "sampling", "limits"] {
|
||||
let header = format!("[{table}]\n");
|
||||
assert!(text.contains(&header), "fixture has no [{table}] table");
|
||||
let bad = text.replacen(&header, &format!("{header}zz_unknown = 1\n"), 1);
|
||||
assert!(
|
||||
Config::parse(&bad).is_err(),
|
||||
"[{table}] accepted an unknown key"
|
||||
);
|
||||
}
|
||||
assert!(
|
||||
Config::parse(&format!("zz_unknown = 1\n{text}")).is_err(),
|
||||
"top level"
|
||||
);
|
||||
assert!(
|
||||
Config::parse(&format!("{text}\n[zz_unknown]\nx = 1\n")).is_err(),
|
||||
"unknown table"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn required_tables_and_values_are_checked() {
|
||||
let text = std::fs::read_to_string(fixture("minimal.toml")).unwrap();
|
||||
for table in ["infer", "slots", "expect"] {
|
||||
let without: String = text
|
||||
.split("\n\n")
|
||||
.filter(|block| !block.trim_start().starts_with(&format!("[{table}]")))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n\n");
|
||||
assert!(Config::parse(&without).is_err(), "[{table}] is required");
|
||||
}
|
||||
let bad_hash = text.replace("f55f5293", "F55F5293");
|
||||
assert!(
|
||||
Config::parse(&bad_hash).is_err(),
|
||||
"the hash must be lowercase hex"
|
||||
);
|
||||
let negative = text.replace("main = 0", "main = -1");
|
||||
assert!(Config::parse(&negative).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn load_reports_which_file_failed() {
|
||||
let missing = fixture("does-not-exist.toml");
|
||||
match Config::load(&missing) {
|
||||
Err(ConfigError::Read(path, _)) => assert_eq!(path, missing),
|
||||
other => panic!("expected a read error, got {other:?}"),
|
||||
}
|
||||
let e: Box<dyn std::error::Error> = Box::new(Config::load(&missing).unwrap_err());
|
||||
assert!(e.to_string().contains("does-not-exist.toml"));
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
//! Checks against the real server. Ignored unless run by `make verify-device`. Do not edit.
|
||||
//!
|
||||
//! They need two environment variables:
|
||||
//! BOXMAKER_INFERPROXY path to the built `inferproxy` binary
|
||||
//! BOXMAKER_UPSTREAM host:port of the llama-server router, for example straylight:11434
|
||||
//! and, optionally, BOXMAKER_MODEL (default ornith-1.5-35b-a3b). They use slot 0.
|
||||
//!
|
||||
//! They never touch the real `llama-server` process. The "server dies" test kills and restarts
|
||||
//! its own `inferproxy`.
|
||||
|
||||
use loopd::config::Config;
|
||||
use loopd::llama::info::{CacheOutcome, cache_outcome};
|
||||
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::process::{Child, Command};
|
||||
use std::sync::mpsc;
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
struct Proxy {
|
||||
child: Child,
|
||||
socket: PathBuf,
|
||||
}
|
||||
|
||||
impl Proxy {
|
||||
fn start(socket: &Path) -> Proxy {
|
||||
let binary = std::env::var("BOXMAKER_INFERPROXY").expect("BOXMAKER_INFERPROXY is not set");
|
||||
let upstream = std::env::var("BOXMAKER_UPSTREAM").expect("BOXMAKER_UPSTREAM is not set");
|
||||
let child = Command::new(binary)
|
||||
.arg("--listen")
|
||||
.arg(socket)
|
||||
.arg("--upstream")
|
||||
.arg(upstream)
|
||||
.spawn()
|
||||
.expect("cannot start inferproxy");
|
||||
let mut proxy = Proxy {
|
||||
child,
|
||||
socket: socket.to_path_buf(),
|
||||
};
|
||||
for _ in 0..100 {
|
||||
if socket.exists() {
|
||||
return proxy;
|
||||
}
|
||||
thread::sleep(Duration::from_millis(20));
|
||||
}
|
||||
proxy.kill();
|
||||
panic!("inferproxy did not create {}", socket.display());
|
||||
}
|
||||
|
||||
fn kill(&mut self) {
|
||||
let _ = self.child.kill();
|
||||
let _ = self.child.wait();
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for Proxy {
|
||||
fn drop(&mut self) {
|
||||
self.kill();
|
||||
}
|
||||
}
|
||||
|
||||
fn socket_path(name: &str) -> PathBuf {
|
||||
let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
dir.join("infer.sock")
|
||||
}
|
||||
|
||||
fn config(socket: &Path) -> Config {
|
||||
let model =
|
||||
std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "{model}"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
"#,
|
||||
socket.display()
|
||||
);
|
||||
Config::parse(&text).unwrap()
|
||||
}
|
||||
|
||||
fn user(text: &str) -> ChatMessage {
|
||||
// A different prompt each run, so that an earlier run's cache cannot make a check pass.
|
||||
let nonce = std::time::SystemTime::now()
|
||||
.duration_since(std::time::UNIX_EPOCH)
|
||||
.unwrap()
|
||||
.as_nanos();
|
||||
ChatMessage::User {
|
||||
content: format!("{text} (run {nonce})"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn the_startup_self_test_passes() {
|
||||
let socket = socket_path("selftest");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let client = Client::new(config(&socket));
|
||||
loopd::selftest::run(&client, &mut |step| eprintln!("selftest: {step}")).unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_capped_thinking_block_ends_and_the_answer_arrives() {
|
||||
let socket = socket_path("cap");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let mut cfg = config(&socket);
|
||||
cfg.limits.thinking_cap = 60;
|
||||
let client = Client::new(cfg);
|
||||
let ask = "Think at great length about how many ways 8 rooks fit on a chessboard without attacking \
|
||||
each other and with none on the main diagonal. Then answer in one sentence.";
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![user(ask)],
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let mut capped = Vec::new();
|
||||
let done = client
|
||||
.chat_with_retry(&req, &mut |e| {
|
||||
if let ChatEvent::ThinkingCapped { tokens } = e {
|
||||
capped.push(*tokens);
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
assert!(done.thinking_capped);
|
||||
assert_eq!(capped.len(), 1);
|
||||
assert!(capped[0] >= 60 && capped[0] < 70, "capped at {capped:?}");
|
||||
assert!(
|
||||
done.reasoning_tokens < 60 + 256,
|
||||
"thinking went on to {}",
|
||||
done.reasoning_tokens
|
||||
);
|
||||
assert!(
|
||||
done.content.is_some_and(|c| !c.trim().is_empty()),
|
||||
"an answer followed the forced end"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_request_survives_its_proxy_being_killed_and_restarted() {
|
||||
let socket = socket_path("restart");
|
||||
let mut proxy = Proxy::start(&socket);
|
||||
let mut cfg = config(&socket);
|
||||
cfg.limits.retry_backoff_ms = vec![1_500];
|
||||
let client = Client::new(cfg);
|
||||
let ask = "Write about 300 words on the history of cork.";
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![user(ask)],
|
||||
tools: vec![],
|
||||
thinking: false,
|
||||
};
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let worker = thread::spawn(move || {
|
||||
let mut retries = 0;
|
||||
let mut told = false;
|
||||
let result = client.chat_with_retry(&req, &mut |e| match e {
|
||||
ChatEvent::Content(_) if !told => {
|
||||
told = true;
|
||||
let _ = tx.send(());
|
||||
}
|
||||
ChatEvent::Retrying { .. } => retries += 1,
|
||||
_ => {}
|
||||
});
|
||||
(result, retries)
|
||||
});
|
||||
|
||||
rx.recv_timeout(Duration::from_secs(120))
|
||||
.expect("no content arrived");
|
||||
proxy.kill(); // the stream dies in the middle of the answer
|
||||
thread::sleep(Duration::from_millis(300));
|
||||
let _proxy = Proxy::start(&proxy.socket);
|
||||
|
||||
let (result, retries) = worker.join().unwrap();
|
||||
let done = result.expect("the retry should have succeeded");
|
||||
assert!(retries >= 1, "the request was retried");
|
||||
assert!(
|
||||
done.content
|
||||
.is_some_and(|c| c.split_whitespace().count() > 100)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[ignore = "needs the real server; run with make verify-device"]
|
||||
fn a_second_turn_reuses_the_first_turns_cache() {
|
||||
let socket = socket_path("cache");
|
||||
let _proxy = Proxy::start(&socket);
|
||||
let client = Client::new(config(&socket));
|
||||
let mut messages = vec![user(
|
||||
"What is 17 * 23? Think briefly, then answer in one short sentence.",
|
||||
)];
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: messages.clone(),
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let turn1 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
|
||||
assert!(
|
||||
turn1.reasoning_content.is_some(),
|
||||
"this check is about replaying a thinking block"
|
||||
);
|
||||
messages.push(ChatMessage::Assistant {
|
||||
content: turn1.content.clone(),
|
||||
reasoning_content: turn1.reasoning_content.clone(),
|
||||
tool_calls: turn1.tool_calls.clone(),
|
||||
});
|
||||
messages.push(user("And 17 * 24?"));
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages,
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let turn2 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
|
||||
assert_eq!(
|
||||
cache_outcome(&turn1.timings, &turn2.timings),
|
||||
CacheOutcome::Hit,
|
||||
"{:?} then {:?}",
|
||||
turn1.timings,
|
||||
turn2.timings
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
[infer]
|
||||
socket = "/tmp/infer.sock"
|
||||
model = "some-model"
|
||||
|
||||
[slots]
|
||||
main = 2
|
||||
background = 3
|
||||
|
||||
[expect]
|
||||
template_sha256 = "000102030405060708090a0b0c0d0e0f101112131415161718191a1b1c1d1e1f"
|
||||
n_ctx = 4096
|
||||
slots = 4
|
||||
|
||||
[sampling]
|
||||
temperature = 0.2
|
||||
top_p = 0.9
|
||||
top_k = 40
|
||||
|
||||
[limits]
|
||||
poll_ms = 50
|
||||
busy_wait_ms = 200
|
||||
load_wait_ms = 300
|
||||
idle_grace_ms = 150
|
||||
liveness_ms = 100
|
||||
thinking_cap = 20
|
||||
thinking_overrun = 10
|
||||
max_tokens = 512
|
||||
queue_len = 1
|
||||
retry_attempts = 2
|
||||
retry_backoff_ms = [10]
|
||||
retry_window_ms = 1000
|
||||
@@ -0,0 +1,12 @@
|
||||
[infer]
|
||||
socket = "/run/boxmaker/infer/infer.sock"
|
||||
model = "ornith-1.5-35b-a3b"
|
||||
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"id": "chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0",
|
||||
"content": "The number of ways is the number of derangements of 8 elements, which equals **40320**. This counts permutations of 8 rooks (one per row and column) where none lands on the main diagonal (a fixed point would place a rook on the diagonal).",
|
||||
"reasoning_content": "The question asks about 8 rooks on a chessboard that don't attack each other, with none on the main diagonal.\n\nRooks that don't attack each other means exactly one rook per row and one rook per column. This is equivalent to a permutation matrix. So the number of ways to",
|
||||
"tool_calls": [],
|
||||
"finish_reason": "stop",
|
||||
"cache_n": 15,
|
||||
"prompt_n": 55,
|
||||
"predicted_n": 126,
|
||||
"reasoning_tokens": 63,
|
||||
"progress": [
|
||||
[
|
||||
70,
|
||||
15,
|
||||
15
|
||||
],
|
||||
[
|
||||
70,
|
||||
15,
|
||||
66
|
||||
],
|
||||
[
|
||||
70,
|
||||
15,
|
||||
70
|
||||
]
|
||||
],
|
||||
"reasoning_events": 63,
|
||||
"content_events": 60,
|
||||
"tool_events": 0
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"id": "chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2",
|
||||
"content": "The box is made. (run 6aac4728)",
|
||||
"reasoning_content": null,
|
||||
"tool_calls": [],
|
||||
"finish_reason": "stop",
|
||||
"cache_n": 0,
|
||||
"prompt_n": 46,
|
||||
"predicted_n": 16,
|
||||
"reasoning_tokens": 0,
|
||||
"progress": [
|
||||
[
|
||||
46,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
46,
|
||||
0,
|
||||
15
|
||||
],
|
||||
[
|
||||
46,
|
||||
0,
|
||||
42
|
||||
],
|
||||
[
|
||||
46,
|
||||
0,
|
||||
46
|
||||
]
|
||||
],
|
||||
"reasoning_events": 0,
|
||||
"content_events": 15,
|
||||
"tool_events": 0
|
||||
}
|
||||
@@ -0,0 +1,51 @@
|
||||
{
|
||||
"id": "chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u",
|
||||
"content": "ok (run 6aac47",
|
||||
"reasoning_content": null,
|
||||
"tool_calls": [],
|
||||
"finish_reason": "length",
|
||||
"cache_n": 15,
|
||||
"prompt_n": 7029,
|
||||
"predicted_n": 8,
|
||||
"reasoning_tokens": 0,
|
||||
"progress": [
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
15
|
||||
],
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
2063
|
||||
],
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
4111
|
||||
],
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
6159
|
||||
],
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
6528
|
||||
],
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
7040
|
||||
],
|
||||
[
|
||||
7044,
|
||||
15,
|
||||
7044
|
||||
]
|
||||
],
|
||||
"reasoning_events": 0,
|
||||
"content_events": 8,
|
||||
"tool_events": 0
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"id": "chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f",
|
||||
"content": "17 * 23 equals 391.",
|
||||
"reasoning_content": "17 * 23. Let me compute: 17 * 23 = 17 * 20 + 17 * 3 = 340 + 51 = 391.\n",
|
||||
"tool_calls": [],
|
||||
"finish_reason": "stop",
|
||||
"cache_n": 15,
|
||||
"prompt_n": 40,
|
||||
"predicted_n": 64,
|
||||
"reasoning_tokens": 49,
|
||||
"progress": [
|
||||
[
|
||||
55,
|
||||
15,
|
||||
15
|
||||
],
|
||||
[
|
||||
55,
|
||||
15,
|
||||
51
|
||||
],
|
||||
[
|
||||
55,
|
||||
15,
|
||||
55
|
||||
]
|
||||
],
|
||||
"reasoning_events": 49,
|
||||
"content_events": 12,
|
||||
"tool_events": 0
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
{
|
||||
"id": "chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP",
|
||||
"content": "I'll read the `/etc/hostname` file for you.\n\n",
|
||||
"reasoning_content": null,
|
||||
"tool_calls": [
|
||||
{
|
||||
"id": "wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F",
|
||||
"name": "read_file",
|
||||
"arguments": "{\"path\":\"/etc/hostname\"}"
|
||||
}
|
||||
],
|
||||
"finish_reason": "tool_calls",
|
||||
"cache_n": 0,
|
||||
"prompt_n": 312,
|
||||
"predicted_n": 41,
|
||||
"reasoning_tokens": 0,
|
||||
"progress": [
|
||||
[
|
||||
312,
|
||||
0,
|
||||
0
|
||||
],
|
||||
[
|
||||
312,
|
||||
0,
|
||||
278
|
||||
],
|
||||
[
|
||||
312,
|
||||
0,
|
||||
308
|
||||
],
|
||||
[
|
||||
312,
|
||||
0,
|
||||
312
|
||||
]
|
||||
],
|
||||
"reasoning_events": 0,
|
||||
"content_events": 14,
|
||||
"tool_events": 7
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
{
|
||||
"id": "chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN",
|
||||
"content": "Blue",
|
||||
"reasoning_content": null,
|
||||
"tool_calls": [],
|
||||
"finish_reason": "stop",
|
||||
"cache_n": 15,
|
||||
"prompt_n": 29,
|
||||
"predicted_n": 2,
|
||||
"reasoning_tokens": 0,
|
||||
"progress": [
|
||||
[
|
||||
44,
|
||||
15,
|
||||
15
|
||||
],
|
||||
[
|
||||
44,
|
||||
15,
|
||||
40
|
||||
],
|
||||
[
|
||||
44,
|
||||
15,
|
||||
44
|
||||
]
|
||||
],
|
||||
"reasoning_events": 0,
|
||||
"content_events": 1,
|
||||
"tool_events": 0
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
{
|
||||
"id": "chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE",
|
||||
"content": "Green",
|
||||
"reasoning_content": null,
|
||||
"tool_calls": [],
|
||||
"finish_reason": "stop",
|
||||
"cache_n": 45,
|
||||
"prompt_n": 30,
|
||||
"predicted_n": 2,
|
||||
"reasoning_tokens": 0,
|
||||
"progress": [
|
||||
[
|
||||
75,
|
||||
45,
|
||||
45
|
||||
],
|
||||
[
|
||||
75,
|
||||
45,
|
||||
47
|
||||
],
|
||||
[
|
||||
75,
|
||||
45,
|
||||
71
|
||||
],
|
||||
[
|
||||
75,
|
||||
45,
|
||||
75
|
||||
]
|
||||
],
|
||||
"reasoning_events": 0,
|
||||
"content_events": 1,
|
||||
"tool_events": 0
|
||||
}
|
||||
@@ -0,0 +1,13 @@
|
||||
HTTP/1.1 400 Bad Request
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: application/json; charset=utf-8
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
64
|
||||
{"error":{"code":400,"message":"Expected 'messages' to be an array","type":"invalid_request_error"}}
|
||||
0
|
||||
|
||||
@@ -0,0 +1,525 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
214
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":70,"cache":15,"processed":15,"time_ms":6}}
|
||||
|
||||
|
||||
238
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":51,"prompt_ms":173.924,"prompt_per_token_ms":3.4102745098039215,"prompt_per_second":293.2315264138359,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":70,"cache":15,"processed":66,"time_ms":173}}
|
||||
|
||||
|
||||
238
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.471,"prompt_per_token_ms":4.154018181818182,"prompt_per_second":240.73077108254438,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":70,"cache":15,"processed":70,"time_ms":228}}
|
||||
|
||||
|
||||
2f6
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"The"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" question"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":2,"predicted_ms":17.018,"predicted_per_token_ms":17.018,"predicted_per_second":58.76131155247385}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" asks"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":3,"predicted_ms":31.055,"predicted_per_token_ms":15.5275,"predicted_per_second":64.40186765416198}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" about"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":4,"predicted_ms":45.083,"predicted_per_token_ms":15.027666666666667,"predicted_per_second":66.5439300845108}}
|
||||
|
||||
|
||||
1fb
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":5,"predicted_ms":59.122,"predicted_per_token_ms":14.7805,"predicted_per_second":67.65670985419979}}
|
||||
|
||||
|
||||
1fb
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"8"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":6,"predicted_ms":73.101,"predicted_per_token_ms":14.6202,"predicted_per_second":68.39851712014884}}
|
||||
|
||||
|
||||
208
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ro"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":7,"predicted_ms":87.106,"predicted_per_token_ms":14.517666666666665,"predicted_per_second":68.88159254241958}}
|
||||
|
||||
|
||||
209
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"oks"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":8,"predicted_ms":101.658,"predicted_per_token_ms":14.522571428571428,"predicted_per_second":68.85832890672647}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" on"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":9,"predicted_ms":115.753,"predicted_per_token_ms":14.469125,"predicted_per_second":69.11267958497835}}
|
||||
|
||||
|
||||
209
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" a"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":10,"predicted_ms":129.719,"predicted_per_token_ms":14.413222222222222,"predicted_per_second":69.38073836523563}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" chess"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":11,"predicted_ms":143.691,"predicted_per_token_ms":14.3691,"predicted_per_second":69.59378109972093}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"board"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":12,"predicted_ms":157.663,"predicted_per_token_ms":14.333,"predicted_per_second":69.76906439684643}}
|
||||
|
||||
|
||||
20c
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" that"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":13,"predicted_ms":171.689,"predicted_per_token_ms":14.307416666666667,"predicted_per_second":69.89381963899842}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" don"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":14,"predicted_ms":185.705,"predicted_per_token_ms":14.285,"predicted_per_second":70.00350017500875}}
|
||||
|
||||
|
||||
209
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"'t"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":15,"predicted_ms":200.259,"predicted_per_token_ms":14.304214285714284,"predicted_per_second":69.90946723992431}}
|
||||
|
||||
|
||||
20e
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" attack"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":16,"predicted_ms":214.525,"predicted_per_token_ms":14.301666666666668,"predicted_per_second":69.92192052208367}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" each"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":17,"predicted_ms":228.576,"predicted_per_token_ms":14.286,"predicted_per_second":69.99860002799944}}
|
||||
|
||||
|
||||
20c
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" other"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":18,"predicted_ms":242.603,"predicted_per_token_ms":14.270764705882353,"predicted_per_second":70.0733296785283}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":","}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":19,"predicted_ms":256.628,"predicted_per_token_ms":14.25711111111111,"predicted_per_second":70.14043674111944}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" with"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":20,"predicted_ms":270.638,"predicted_per_token_ms":14.244105263157893,"predicted_per_second":70.2044797848048}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" none"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":21,"predicted_ms":285.363,"predicted_per_token_ms":14.26815,"predicted_per_second":70.08617094717955}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" on"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":22,"predicted_ms":299.519,"predicted_per_token_ms":14.262809523809524,"predicted_per_second":70.11241356975684}}
|
||||
|
||||
|
||||
209
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" the"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":23,"predicted_ms":313.597,"predicted_per_token_ms":14.25440909090909,"predicted_per_second":70.1537323380007}}
|
||||
|
||||
|
||||
20c
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" main"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":24,"predicted_ms":327.584,"predicted_per_token_ms":14.242782608695652,"predicted_per_second":70.21099931620593}}
|
||||
|
||||
|
||||
210
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" diagonal"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":25,"predicted_ms":341.533,"predicted_per_token_ms":14.230541666666667,"predicted_per_second":70.27139397949831}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"."}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":26,"predicted_ms":355.541,"predicted_per_token_ms":14.22164,"predicted_per_second":70.31537853580882}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"\n\n"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":27,"predicted_ms":369.531,"predicted_per_token_ms":14.21273076923077,"predicted_per_second":70.35945563430403}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"R"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":28,"predicted_ms":383.51,"predicted_per_token_ms":14.204074074074073,"predicted_per_second":70.40233631456807}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"ooks"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":29,"predicted_ms":397.548,"predicted_per_token_ms":14.198142857142857,"predicted_per_second":70.43174660669906}}
|
||||
|
||||
|
||||
20c
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" that"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":30,"predicted_ms":412.093,"predicted_per_token_ms":14.210103448275863,"predicted_per_second":70.37246446797204}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" don"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":31,"predicted_ms":426.207,"predicted_per_token_ms":14.2069,"predicted_per_second":70.38833243001639}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"'t"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":32,"predicted_ms":440.396,"predicted_per_token_ms":14.206322580645162,"predicted_per_second":70.391193380503}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" attack"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":33,"predicted_ms":454.559,"predicted_per_token_ms":14.20496875,"predicted_per_second":70.39790214251614}}
|
||||
|
||||
|
||||
20c
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" each"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":34,"predicted_ms":468.592,"predicted_per_token_ms":14.199757575757575,"predicted_per_second":70.42373749445147}}
|
||||
|
||||
|
||||
20d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" other"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":35,"predicted_ms":482.576,"predicted_per_token_ms":14.193411764705884,"predicted_per_second":70.45522363316866}}
|
||||
|
||||
|
||||
20d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" means"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":36,"predicted_ms":496.525,"predicted_per_token_ms":14.186428571428571,"predicted_per_second":70.48990483862848}}
|
||||
|
||||
|
||||
20f
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" exactly"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":37,"predicted_ms":510.648,"predicted_per_token_ms":14.184666666666667,"predicted_per_second":70.49866052545002}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" one"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":38,"predicted_ms":524.784,"predicted_per_token_ms":14.183351351351352,"predicted_per_second":70.50519832921735}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ro"}}],"created":1789675313,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":39,"predicted_ms":538.9,"predicted_per_token_ms":14.18157894736842,"predicted_per_second":70.51401002041196}}
|
||||
|
||||
|
||||
208
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"ok"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":40,"predicted_ms":553.038,"predicted_per_token_ms":14.18046153846154,"predicted_per_second":70.51956646740369}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" per"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":41,"predicted_ms":567.155,"predicted_per_token_ms":14.178875,"predicted_per_second":70.52745722068923}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" row"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":42,"predicted_ms":581.119,"predicted_per_token_ms":14.173634146341463,"predicted_per_second":70.55353550649694}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" and"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":43,"predicted_ms":595.281,"predicted_per_token_ms":14.173357142857142,"predicted_per_second":70.55491440176992}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" one"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":44,"predicted_ms":609.304,"predicted_per_token_ms":14.169860465116278,"predicted_per_second":70.57232514475533}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ro"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":45,"predicted_ms":623.255,"predicted_per_token_ms":14.164886363636363,"predicted_per_second":70.59710712308765}}
|
||||
|
||||
|
||||
209
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"ok"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":46,"predicted_ms":637.287,"predicted_per_token_ms":14.161933333333334,"predicted_per_second":70.61182795192747}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" per"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":47,"predicted_ms":651.242,"predicted_per_token_ms":14.157434782608695,"predicted_per_second":70.6342649890517}}
|
||||
|
||||
|
||||
20e
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" column"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":48,"predicted_ms":665.265,"predicted_per_token_ms":14.154574468085107,"predicted_per_second":70.64853855230622}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"."}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":49,"predicted_ms":679.28,"predicted_per_token_ms":14.151666666666666,"predicted_per_second":70.66305499941114}}
|
||||
|
||||
|
||||
20c
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" This"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":50,"predicted_ms":693.396,"predicted_per_token_ms":14.150938775510204,"predicted_per_second":70.66668974150413}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" is"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":51,"predicted_ms":707.413,"predicted_per_token_ms":14.14826,"predicted_per_second":70.68006949264432}}
|
||||
|
||||
|
||||
210
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" equivalent"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":52,"predicted_ms":721.45,"predicted_per_token_ms":14.14607843137255,"predicted_per_second":70.69096957516112}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" to"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":53,"predicted_ms":735.58,"predicted_per_token_ms":14.14576923076923,"predicted_per_second":70.6925147502651}}
|
||||
|
||||
|
||||
208
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" a"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":54,"predicted_ms":749.673,"predicted_per_token_ms":14.14477358490566,"predicted_per_second":70.69749077264355}}
|
||||
|
||||
|
||||
212
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" permutation"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":55,"predicted_ms":763.702,"predicted_per_token_ms":14.14262962962963,"predicted_per_second":70.70820817544016}}
|
||||
|
||||
|
||||
20d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" matrix"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":56,"predicted_ms":777.68,"predicted_per_token_ms":14.139636363636363,"predicted_per_second":70.72317662791895}}
|
||||
|
||||
|
||||
206
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"."}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":57,"predicted_ms":791.78,"predicted_per_token_ms":14.13892857142857,"predicted_per_second":70.72671701735331}}
|
||||
|
||||
|
||||
209
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" So"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":58,"predicted_ms":805.76,"predicted_per_token_ms":14.136140350877193,"predicted_per_second":70.74066719618746}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" the"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":59,"predicted_ms":819.948,"predicted_per_token_ms":14.13703448275862,"predicted_per_second":70.73619302687487}}
|
||||
|
||||
|
||||
20d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" number"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":60,"predicted_ms":834.052,"predicted_per_token_ms":14.136474576271187,"predicted_per_second":70.7389946909785}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" of"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":61,"predicted_ms":848.101,"predicted_per_token_ms":14.135016666666667,"predicted_per_second":70.74629083092698}}
|
||||
|
||||
|
||||
20b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ways"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":62,"predicted_ms":862.155,"predicted_per_token_ms":14.133688524590163,"predicted_per_second":70.7529388567021}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" to"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":63,"predicted_ms":876.321,"predicted_per_token_ms":14.134209677419355,"predicted_per_second":70.75033007311248}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"The"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":66,"predicted_ms":918.492,"predicted_per_token_ms":14.130646153846154,"predicted_per_second":70.76817217787416}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" number"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":67,"predicted_ms":932.589,"predicted_per_token_ms":14.130136363636364,"predicted_per_second":70.7707253677665}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" of"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":68,"predicted_ms":946.625,"predicted_per_token_ms":14.128731343283581,"predicted_per_second":70.7777631057705}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ways"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":69,"predicted_ms":960.683,"predicted_per_token_ms":14.127691176470588,"predicted_per_second":70.78297419648314}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" is"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":70,"predicted_ms":974.675,"predicted_per_token_ms":14.12572463768116,"predicted_per_second":70.79282837869033}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" the"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":71,"predicted_ms":988.81,"predicted_per_token_ms":14.125857142857143,"predicted_per_second":70.79216431872655}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" number"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":72,"predicted_ms":1002.867,"predicted_per_token_ms":14.124887323943662,"predicted_per_second":70.79702492952704}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" of"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":73,"predicted_ms":1016.819,"predicted_per_token_ms":14.12248611111111,"predicted_per_second":70.80906237983358}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" der"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":74,"predicted_ms":1030.918,"predicted_per_token_ms":14.122164383561643,"predicted_per_second":70.8106755338446}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"ang"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":75,"predicted_ms":1045.021,"predicted_per_token_ms":14.121905405405405,"predicted_per_second":70.81197411343888}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"ements"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":76,"predicted_ms":1059.04,"predicted_per_token_ms":14.120533333333332,"predicted_per_second":70.81885481190513}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" of"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":77,"predicted_ms":1073.065,"predicted_per_token_ms":14.119276315789474,"predicted_per_second":70.82515970607558}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" "}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":78,"predicted_ms":1087.054,"predicted_per_token_ms":14.117584415584417,"predicted_per_second":70.83364763847977}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"8"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":79,"predicted_ms":1101.088,"predicted_per_token_ms":14.11651282051282,"predicted_per_second":70.83902467377722}}
|
||||
|
||||
|
||||
204
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" elements"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":80,"predicted_ms":1115.11,"predicted_per_token_ms":14.1153164556962,"predicted_per_second":70.84502874155913}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":","}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":81,"predicted_ms":1129.124,"predicted_per_token_ms":14.11405,"predicted_per_second":70.85138567597537}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" which"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":82,"predicted_ms":1143.084,"predicted_per_token_ms":14.11214814814815,"predicted_per_second":70.86093410458024}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" equals"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":83,"predicted_ms":1157.083,"predicted_per_token_ms":14.110768292682927,"predicted_per_second":70.86786341169994}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" **"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":84,"predicted_ms":1171.065,"predicted_per_token_ms":14.10921686746988,"predicted_per_second":70.8756559200386}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"4"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":85,"predicted_ms":1185.174,"predicted_per_token_ms":14.109214285714286,"predicted_per_second":70.87566888912515}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"0"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":86,"predicted_ms":1199.147,"predicted_per_token_ms":14.107611764705881,"predicted_per_second":70.88371984418924}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"3"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":87,"predicted_ms":1213.27,"predicted_per_token_ms":14.10779069767442,"predicted_per_second":70.88282080658057}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"2"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":88,"predicted_ms":1227.245,"predicted_per_token_ms":14.10626436781609,"predicted_per_second":70.8904904888592}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"0"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":89,"predicted_ms":1241.444,"predicted_per_token_ms":14.107318181818181,"predicted_per_second":70.88519498261702}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"**."}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":90,"predicted_ms":1255.669,"predicted_per_token_ms":14.108640449438203,"predicted_per_second":70.87855159281625}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" This"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":91,"predicted_ms":1269.736,"predicted_per_token_ms":14.10817777777778,"predicted_per_second":70.88087602462244}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" counts"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":92,"predicted_ms":1283.792,"predicted_per_token_ms":14.107604395604394,"predicted_per_second":70.88375687027182}}
|
||||
|
||||
|
||||
20a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" permutations"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":93,"predicted_ms":1297.873,"predicted_per_token_ms":14.107315217391305,"predicted_per_second":70.8852098780081}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" of"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":94,"predicted_ms":1311.856,"predicted_per_token_ms":14.105978494623656,"predicted_per_second":70.89192716273737}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" "}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":95,"predicted_ms":1325.946,"predicted_per_token_ms":14.105808510638298,"predicted_per_second":70.89278145565507}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"8"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":96,"predicted_ms":1339.983,"predicted_per_token_ms":14.105084210526316,"predicted_per_second":70.89642182027683}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ro"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":97,"predicted_ms":1353.963,"predicted_per_token_ms":14.103781249999999,"predicted_per_second":70.90297149922118}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"oks"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":98,"predicted_ms":1368.053,"predicted_per_token_ms":14.103639175257733,"predicted_per_second":70.90368574901703}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ("}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":99,"predicted_ms":1382.062,"predicted_per_token_ms":14.102673469387755,"predicted_per_second":70.90854100611985}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"one"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":100,"predicted_ms":1396.044,"predicted_per_token_ms":14.101454545454546,"predicted_per_second":70.91467031125093}}
|
||||
|
||||
|
||||
1f7
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" per"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":101,"predicted_ms":1410.024,"predicted_per_token_ms":14.10024,"predicted_per_second":70.920778653413}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" row"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":102,"predicted_ms":1424.204,"predicted_per_token_ms":14.101029702970296,"predicted_per_second":70.91680686193833}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" and"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":103,"predicted_ms":1438.367,"predicted_per_token_ms":14.101637254901961,"predicted_per_second":70.91375149735777}}
|
||||
|
||||
|
||||
206
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" column"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":104,"predicted_ms":1452.392,"predicted_per_token_ms":14.100893203883496,"predicted_per_second":70.91749334890305}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":")"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":105,"predicted_ms":1466.597,"predicted_per_token_ms":14.101894230769231,"predicted_per_second":70.91245925090533}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" where"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":106,"predicted_ms":1480.643,"predicted_per_token_ms":14.101361904761905,"predicted_per_second":70.91513619420752}}
|
||||
|
||||
|
||||
204
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" none"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":107,"predicted_ms":1494.784,"predicted_per_token_ms":14.101735849056604,"predicted_per_second":70.91325569446823}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" lands"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":108,"predicted_ms":1508.863,"predicted_per_token_ms":14.101523364485981,"predicted_per_second":70.91432422956888}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" on"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":109,"predicted_ms":1522.882,"predicted_per_token_ms":14.10075925925926,"predicted_per_second":70.9181670017769}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" the"}}],"created":1789675314,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":110,"predicted_ms":1537.028,"predicted_per_token_ms":14.101174311926606,"predicted_per_second":70.91607960297405}}
|
||||
|
||||
|
||||
204
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" main"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":111,"predicted_ms":1551.029,"predicted_per_token_ms":14.100263636363636,"predicted_per_second":70.92065976845049}}
|
||||
|
||||
|
||||
207
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" diagonal"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":112,"predicted_ms":1565.05,"predicted_per_token_ms":14.099549549549549,"predicted_per_second":70.92425162135396}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ("}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":113,"predicted_ms":1579.222,"predicted_per_token_ms":14.100196428571428,"predicted_per_second":70.9209978077813}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"a"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":114,"predicted_ms":1593.406,"predicted_per_token_ms":14.100938053097344,"predicted_per_second":70.91726778987905}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" fixed"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":115,"predicted_ms":1607.404,"predicted_per_token_ms":14.100035087719299,"predicted_per_second":70.92180932733775}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" point"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":116,"predicted_ms":1621.798,"predicted_per_token_ms":14.102591304347825,"predicted_per_second":70.90895413608847}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" would"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":117,"predicted_ms":1636.345,"predicted_per_token_ms":14.106422413793103,"predicted_per_second":70.88969624376278}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" place"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":118,"predicted_ms":1650.394,"predicted_per_token_ms":14.105931623931625,"predicted_per_second":70.89216271993233}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" a"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":119,"predicted_ms":1664.436,"predicted_per_token_ms":14.105389830508473,"predicted_per_second":70.89488571504101}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ro"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":120,"predicted_ms":1678.598,"predicted_per_token_ms":14.105865546218487,"predicted_per_second":70.8924948081673}}
|
||||
|
||||
|
||||
1f8
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"ok"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":121,"predicted_ms":1692.759,"predicted_per_token_ms":14.106325,"predicted_per_second":70.89018578545439}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" on"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":122,"predicted_ms":1706.916,"predicted_per_token_ms":14.106743801652891,"predicted_per_second":70.88808119438801}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" the"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":123,"predicted_ms":1720.999,"predicted_per_token_ms":14.10654918032787,"predicted_per_second":70.88905920340453}}
|
||||
|
||||
|
||||
208
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" diagonal"}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":124,"predicted_ms":1735.081,"predicted_per_token_ms":14.106349593495935,"predicted_per_second":70.89006219306188}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":")."}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":125,"predicted_ms":1749.154,"predicted_per_token_ms":14.10608064516129,"predicted_per_second":70.8914137920389}}
|
||||
|
||||
|
||||
1ec
|
||||
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{}}],"created":1789675315,"id":"chatcmpl-1x092OMeEwmeoSzn7cE1GLkJlskrslR0","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":55,"prompt_ms":228.721,"prompt_per_token_ms":4.1585636363636365,"prompt_per_second":240.467643985467,"predicted_n":126,"predicted_ms":1763.228,"predicted_per_token_ms":14.105824,"predicted_per_second":70.89270360951618}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: application/json; charset=utf-8
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
10
|
||||
{"success":true}
|
||||
0
|
||||
|
||||
@@ -0,0 +1,97 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
212
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":46,"cache":0,"processed":0,"time_ms":11}}
|
||||
|
||||
|
||||
227
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":15,"prompt_ms":62.46,"prompt_per_token_ms":4.164,"prompt_per_second":240.15369836695484,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":46,"cache":0,"processed":15,"time_ms":62}}
|
||||
|
||||
|
||||
235
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":42,"prompt_ms":110.781,"prompt_per_token_ms":2.637642857142857,"prompt_per_second":379.1263844883148,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":46,"cache":0,"processed":42,"time_ms":110}}
|
||||
|
||||
|
||||
236
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.137,"prompt_per_token_ms":5.350804347826087,"prompt_per_second":186.88779013313723,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":46,"cache":0,"processed":46,"time_ms":246}}
|
||||
|
||||
|
||||
2ec
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"The"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" box"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":2,"predicted_ms":17.839,"predicted_per_token_ms":17.839,"predicted_per_second":56.05695386512697}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" is"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":3,"predicted_ms":32.208,"predicted_per_token_ms":16.104,"predicted_per_second":62.09637357178341}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" made"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":4,"predicted_ms":46.636,"predicted_per_token_ms":15.545333333333334,"predicted_per_second":64.3279869628613}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":5,"predicted_ms":61.039,"predicted_per_token_ms":15.25975,"predicted_per_second":65.53187306476187}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ("}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":6,"predicted_ms":75.194,"predicted_per_token_ms":15.0388,"predicted_per_second":66.49466712769636}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"run"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":7,"predicted_ms":89.347,"predicted_per_token_ms":14.891166666666665,"predicted_per_second":67.15390555922416}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" "}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":8,"predicted_ms":103.564,"predicted_per_token_ms":14.794857142857142,"predicted_per_second":67.5910548066896}}
|
||||
|
||||
|
||||
1f1
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"6"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":9,"predicted_ms":117.664,"predicted_per_token_ms":14.708,"predicted_per_second":67.99020940984498}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"aac"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":10,"predicted_ms":131.848,"predicted_per_token_ms":14.649777777777778,"predicted_per_second":68.26042109095322}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"4"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":11,"predicted_ms":145.997,"predicted_per_token_ms":14.599700000000002,"predicted_per_second":68.49455810735836}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"7"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":12,"predicted_ms":160.522,"predicted_per_token_ms":14.59290909090909,"predicted_per_second":68.52643251392333}}
|
||||
|
||||
|
||||
1f4
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"2"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":13,"predicted_ms":174.729,"predicted_per_token_ms":14.56075,"predicted_per_second":68.67778102089521}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"8"}}],"created":1789675304,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":14,"predicted_ms":189.019,"predicted_per_token_ms":14.539923076923078,"predicted_per_second":68.77615477809108}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":")"}}],"created":1789675305,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":15,"predicted_ms":203.221,"predicted_per_token_ms":14.515785714285714,"predicted_per_second":68.89051820431943}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{}}],"created":1789675305,"id":"chatcmpl-LMYge0IefEaKdgrXAjafeR0nrNUicPF2","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":46,"prompt_ms":246.429,"prompt_per_token_ms":5.357152173913043,"prompt_per_second":186.66634202955012,"predicted_n":16,"predicted_ms":217.376,"predicted_per_token_ms":14.491733333333334,"predicted_per_second":69.00485794199912}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
216
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675307,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":15,"time_ms":6}}
|
||||
|
||||
|
||||
23d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675308,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":2048,"prompt_ms":1243.164,"prompt_per_token_ms":0.607013671875,"prompt_per_second":1647.4093522656706,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":2063,"time_ms":1243}}
|
||||
|
||||
|
||||
23d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675310,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":4096,"prompt_ms":2760.804,"prompt_per_token_ms":0.6740244140625,"prompt_per_second":1483.625784372958,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":4111,"time_ms":2760}}
|
||||
|
||||
|
||||
240
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675311,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":6144,"prompt_ms":4317.25,"prompt_per_token_ms":0.7026774088541666,"prompt_per_second":1423.1281487057734,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":6159,"time_ms":4317}}
|
||||
|
||||
|
||||
240
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675312,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":6513,"prompt_ms":4710.221,"prompt_per_token_ms":0.7232029786580684,"prompt_per_second":1382.737667723022,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":6528,"time_ms":4710}}
|
||||
|
||||
|
||||
241
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675312,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7025,"prompt_ms":5045.515,"prompt_per_token_ms":0.7182227758007118,"prompt_per_second":1392.3256595213768,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":7040,"time_ms":5045}}
|
||||
|
||||
|
||||
241
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.547,"prompt_per_token_ms":0.7849689856309574,"prompt_per_second":1273.9356819253194,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":7044,"cache":15,"processed":7044,"time_ms":5517}}
|
||||
|
||||
|
||||
2ef
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"ok"}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" ("}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":2,"predicted_ms":17.844,"predicted_per_token_ms":17.844,"predicted_per_second":56.04124635731898}}
|
||||
|
||||
|
||||
1f4
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"run"}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":3,"predicted_ms":32.64,"predicted_per_token_ms":16.32,"predicted_per_second":61.27450980392157}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" "}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":4,"predicted_ms":47.315,"predicted_per_token_ms":15.771666666666667,"predicted_per_second":63.404839902779244}}
|
||||
|
||||
|
||||
1f6
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"6"}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":5,"predicted_ms":61.947,"predicted_per_token_ms":15.48675,"predicted_per_second":64.57132710220027}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"aac"}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":6,"predicted_ms":77.065,"predicted_per_token_ms":15.413,"predicted_per_second":64.8802958541491}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"4"}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":7,"predicted_ms":91.683,"predicted_per_token_ms":15.280500000000002,"predicted_per_second":65.44288472235856}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"7"}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":8,"predicted_ms":106.25,"predicted_per_token_ms":15.178571428571429,"predicted_per_second":65.88235294117648}}
|
||||
|
||||
|
||||
1f7
|
||||
data: {"choices":[{"finish_reason":"length","index":0,"delta":{}}],"created":1789675313,"id":"chatcmpl-l28a5WTzw2ZnKeMg92QlUvh4jZ1Tys3u","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":7029,"prompt_ms":5517.837,"prompt_per_token_ms":0.785010243277849,"prompt_per_second":1273.8687279091425,"predicted_n":8,"predicted_ms":106.25,"predicted_per_token_ms":15.178571428571429,"predicted_per_second":65.88235294117648}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -0,0 +1,13 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: application/json; charset=utf-8
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
a01
|
||||
[{"id":0,"n_ctx":131072,"speculative":false,"is_processing":false,"id_task":4323,"n_prompt_tokens":76,"n_prompt_tokens_processed":0,"n_prompt_tokens_cache":0,"params":{"seed":4294967295,"temperature":0.6000000238418579,"dynatemp_range":0.0,"dynatemp_exponent":1.0,"top_k":20,"top_p":0.949999988079071,"min_p":0.05000000074505806,"top_n_sigma":-1.0,"xtc_probability":0.0,"xtc_threshold":0.10000000149011612,"typical_p":1.0,"repeat_last_n":64,"repeat_penalty":1.0,"presence_penalty":0.0,"frequency_penalty":0.0,"dry_multiplier":0.0,"dry_base":1.75,"dry_allowed_length":2,"dry_penalty_last_n":64,"mirostat":0,"mirostat_tau":5.0,"mirostat_eta":0.10000000149011612,"adaptive_target":-1.0,"adaptive_decay":0.8999999761581421,"max_tokens":16,"n_predict":16,"n_keep":0,"n_discard":0,"ignore_eos":false,"stream":true,"n_probs":0,"min_keep":0,"chat_format":"peg-native","reasoning_format":"deepseek","reasoning_in_content":false,"generation_prompt":"<|im_start|>assistant\n<think>\n\n</think>\n\n","samplers":["penalties","dry","top_n_sigma","top_k","typ_p","top_p","min_p","xtc","temperature"],"speculative.types":"none","timings_per_token":true,"post_sampling_probs":false,"backend_sampling":false,"lora":[]},"next_token":[{"has_next_token":false,"has_new_line":false,"n_remain":-1,"n_decoded":0}]},{"id":1,"n_ctx":131072,"speculative":false,"is_processing":false,"id_task":1939,"n_prompt_tokens":1292,"n_prompt_tokens_processed":0,"n_prompt_tokens_cache":0,"params":{"seed":4294967295,"temperature":1.0,"dynatemp_range":0.0,"dynatemp_exponent":1.0,"top_k":20,"top_p":0.949999988079071,"min_p":0.05000000074505806,"top_n_sigma":-1.0,"xtc_probability":0.0,"xtc_threshold":0.10000000149011612,"typical_p":1.0,"repeat_last_n":64,"repeat_penalty":1.0,"presence_penalty":0.0,"frequency_penalty":0.0,"dry_multiplier":0.0,"dry_base":1.75,"dry_allowed_length":2,"dry_penalty_last_n":64,"mirostat":0,"mirostat_tau":5.0,"mirostat_eta":0.10000000149011612,"adaptive_target":-1.0,"adaptive_decay":0.8999999761581421,"max_tokens":16384,"n_predict":16384,"n_keep":0,"n_discard":0,"ignore_eos":false,"stream":true,"n_probs":0,"min_keep":0,"chat_format":"peg-native","reasoning_format":"deepseek","reasoning_in_content":false,"generation_prompt":"<|im_start|>assistant\n<think>\n","samplers":["penalties","dry","top_n_sigma","top_k","typ_p","top_p","min_p","xtc","temperature"],"speculative.types":"none","timings_per_token":false,"post_sampling_probs":false,"backend_sampling":false,"lora":[]},"next_token":[{"has_next_token":false,"has_new_line":false,"n_remain":-1,"n_decoded":0}]}]
|
||||
0
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
214
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":55,"cache":15,"processed":15,"time_ms":6}}
|
||||
|
||||
|
||||
238
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":36,"prompt_ms":151.463,"prompt_per_token_ms":4.207305555555555,"prompt_per_second":237.68181007902922,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":55,"cache":15,"processed":51,"time_ms":151}}
|
||||
|
||||
|
||||
22f
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.379,"prompt_per_token_ms":5.109475,"prompt_per_second":195.71482393005152,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":55,"cache":15,"processed":55,"time_ms":204}}
|
||||
|
||||
|
||||
2e9
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"1"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
1ef
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"7"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":2,"predicted_ms":17.581,"predicted_per_token_ms":17.581,"predicted_per_second":56.87958591661453}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" *"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":3,"predicted_ms":31.917,"predicted_per_token_ms":15.9585,"predicted_per_second":62.662530939624645}}
|
||||
|
||||
|
||||
1fb
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":4,"predicted_ms":46.043,"predicted_per_token_ms":15.347666666666667,"predicted_per_second":65.15648415611494}}
|
||||
|
||||
|
||||
1ef
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"2"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":5,"predicted_ms":60.132,"predicted_per_token_ms":15.033,"predicted_per_second":66.52032195835828}}
|
||||
|
||||
|
||||
1f0
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"3"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":6,"predicted_ms":74.214,"predicted_per_token_ms":14.8428,"predicted_per_second":67.37273290753765}}
|
||||
|
||||
|
||||
1fa
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"."}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":7,"predicted_ms":88.451,"predicted_per_token_ms":14.741833333333332,"predicted_per_second":67.8341680704571}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" Let"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":8,"predicted_ms":102.733,"predicted_per_token_ms":14.676142857142858,"predicted_per_second":68.13779408758626}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" me"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":9,"predicted_ms":116.897,"predicted_per_token_ms":14.612125,"predicted_per_second":68.43631573094262}}
|
||||
|
||||
|
||||
204
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" compute"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":10,"predicted_ms":130.975,"predicted_per_token_ms":14.552777777777777,"predicted_per_second":68.71540370299675}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":":"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":11,"predicted_ms":145.117,"predicted_per_token_ms":14.5117,"predicted_per_second":68.90991406933716}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":12,"predicted_ms":159.265,"predicted_per_token_ms":14.478636363636362,"predicted_per_second":69.06727780742787}}
|
||||
|
||||
|
||||
1f1
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"1"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":13,"predicted_ms":173.31,"predicted_per_token_ms":14.4425,"predicted_per_second":69.24009001211702}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"7"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":14,"predicted_ms":187.424,"predicted_per_token_ms":14.41723076923077,"predicted_per_second":69.36144784019123}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" *"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":15,"predicted_ms":201.657,"predicted_per_token_ms":14.404071428571429,"predicted_per_second":69.42481540437475}}
|
||||
|
||||
|
||||
1f1
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":16,"predicted_ms":216.105,"predicted_per_token_ms":14.407,"predicted_per_second":69.41070313042272}}
|
||||
|
||||
|
||||
1f4
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"2"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":17,"predicted_ms":230.278,"predicted_per_token_ms":14.392375,"predicted_per_second":69.48123572377735}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"3"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":18,"predicted_ms":244.396,"predicted_per_token_ms":14.376235294117647,"predicted_per_second":69.55923992209367}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ="}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":19,"predicted_ms":258.485,"predicted_per_token_ms":14.360277777777778,"predicted_per_second":69.63653596920517}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":20,"predicted_ms":272.641,"predicted_per_token_ms":14.349526315789475,"predicted_per_second":69.68871152908036}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"1"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":21,"predicted_ms":286.727,"predicted_per_token_ms":14.33635,"predicted_per_second":69.75276133743945}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"7"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":22,"predicted_ms":300.787,"predicted_per_token_ms":14.323190476190476,"predicted_per_second":69.8168471376755}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" *"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":23,"predicted_ms":314.87,"predicted_per_token_ms":14.312272727272727,"predicted_per_second":69.87010512274907}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":24,"predicted_ms":328.974,"predicted_per_token_ms":14.303217391304347,"predicted_per_second":69.91433973505505}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"2"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":25,"predicted_ms":343.02,"predicted_per_token_ms":14.292499999999999,"predicted_per_second":69.96676578625154}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"0"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":26,"predicted_ms":357.078,"predicted_per_token_ms":14.283119999999998,"predicted_per_second":70.0127143089185}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" +"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":27,"predicted_ms":371.151,"predicted_per_token_ms":14.275038461538461,"predicted_per_second":70.05235066051283}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":28,"predicted_ms":385.265,"predicted_per_token_ms":14.269074074074073,"predicted_per_second":70.08163212334367}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"1"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":29,"predicted_ms":399.303,"predicted_per_token_ms":14.260821428571429,"predicted_per_second":70.12218791243743}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"7"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":30,"predicted_ms":413.34,"predicted_per_token_ms":14.253103448275862,"predicted_per_second":70.16015870711763}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" *"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":31,"predicted_ms":427.537,"predicted_per_token_ms":14.251233333333333,"predicted_per_second":70.16936545842816}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":32,"predicted_ms":441.571,"predicted_per_token_ms":14.244225806451613,"predicted_per_second":70.20388567183986}}
|
||||
|
||||
|
||||
1f6
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"3"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":33,"predicted_ms":455.541,"predicted_per_token_ms":14.23565625,"predicted_per_second":70.24614688908353}}
|
||||
|
||||
|
||||
1fe
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ="}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":34,"predicted_ms":469.487,"predicted_per_token_ms":14.226878787878789,"predicted_per_second":70.28948618385598}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":35,"predicted_ms":483.621,"predicted_per_token_ms":14.22414705882353,"predicted_per_second":70.30298518881521}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"3"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":36,"predicted_ms":497.817,"predicted_per_token_ms":14.223342857142857,"predicted_per_second":70.30696018818159}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"4"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":37,"predicted_ms":511.843,"predicted_per_token_ms":14.217861111111112,"predicted_per_second":70.33406728235025}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"0"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":38,"predicted_ms":525.825,"predicted_per_token_ms":14.211486486486487,"predicted_per_second":70.36561593686112}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" +"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":39,"predicted_ms":540.0,"predicted_per_token_ms":14.210526315789474,"predicted_per_second":70.37037037037037}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":40,"predicted_ms":554.023,"predicted_per_token_ms":14.205717948717949,"predicted_per_second":70.39418941090894}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"5"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":41,"predicted_ms":568.083,"predicted_per_token_ms":14.202074999999999,"predicted_per_second":70.41224609784135}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"1"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":42,"predicted_ms":582.132,"predicted_per_token_ms":14.198341463414632,"predicted_per_second":70.43076140806554}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" ="}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":43,"predicted_ms":596.24,"predicted_per_token_ms":14.196190476190477,"predicted_per_second":70.44143298000805}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":" "}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":44,"predicted_ms":610.366,"predicted_per_token_ms":14.194558139534884,"predicted_per_second":70.4495335585534}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"3"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":45,"predicted_ms":624.37,"predicted_per_token_ms":14.190227272727272,"predicted_per_second":70.47103480308151}}
|
||||
|
||||
|
||||
1fc
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"9"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":46,"predicted_ms":638.436,"predicted_per_token_ms":14.187466666666667,"predicted_per_second":70.4847471007274}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"1"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":47,"predicted_ms":652.715,"predicted_per_token_ms":14.189456521739132,"predicted_per_second":70.47486268892243}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"."}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":48,"predicted_ms":666.873,"predicted_per_token_ms":14.188787234042554,"predicted_per_second":70.47818700112315}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"reasoning_content":"\n"}}],"created":1789675305,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":49,"predicted_ms":680.936,"predicted_per_token_ms":14.186166666666667,"predicted_per_second":70.4912062220238}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"1"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":52,"predicted_ms":723.201,"predicted_per_token_ms":14.180411764705882,"predicted_per_second":70.51981399362003}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"7"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":53,"predicted_ms":737.249,"predicted_per_token_ms":14.177865384615385,"predicted_per_second":70.53247952862601}}
|
||||
|
||||
|
||||
1f4
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" *"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":54,"predicted_ms":751.363,"predicted_per_token_ms":14.176660377358491,"predicted_per_second":70.53847474522966}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" "}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":55,"predicted_ms":765.475,"predicted_per_token_ms":14.175462962962964,"predicted_per_second":70.54443319507496}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"2"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":56,"predicted_ms":779.545,"predicted_per_token_ms":14.173545454545454,"predicted_per_second":70.5539769994035}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"3"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":57,"predicted_ms":793.62,"predicted_per_token_ms":14.171785714285715,"predicted_per_second":70.56273783422796}}
|
||||
|
||||
|
||||
1f9
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" equals"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":58,"predicted_ms":807.654,"predicted_per_token_ms":14.169368421052631,"predicted_per_second":70.57477583222519}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" "}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":59,"predicted_ms":821.65,"predicted_per_token_ms":14.166379310344826,"predicted_per_second":70.58966713320757}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"3"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":60,"predicted_ms":835.866,"predicted_per_token_ms":14.16722033898305,"predicted_per_second":70.58547661945813}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"9"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":61,"predicted_ms":850.108,"predicted_per_token_ms":14.168466666666665,"predicted_per_second":70.57926757541395}}
|
||||
|
||||
|
||||
1f2
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"1"}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":62,"predicted_ms":864.085,"predicted_per_token_ms":14.16532786885246,"predicted_per_second":70.59490675107195}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":63,"predicted_ms":878.051,"predicted_per_token_ms":14.162112903225808,"predicted_per_second":70.61093262236476}}
|
||||
|
||||
|
||||
1e8
|
||||
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{}}],"created":1789675306,"id":"chatcmpl-oSb4bEgcGEBGNn7Bn6LXqGrh20qMbo4f","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":40,"prompt_ms":204.752,"prompt_per_token_ms":5.1188,"prompt_per_second":195.3582870985387,"predicted_n":64,"predicted_ms":892.097,"predicted_per_token_ms":14.160269841269841,"predicted_per_second":70.62012314804332}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: application/json; charset=utf-8
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
21
|
||||
{"tokens":[760,3618,369,1801,13]}
|
||||
0
|
||||
|
||||
@@ -0,0 +1,121 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
212
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":312,"cache":0,"processed":0,"time_ms":7}}
|
||||
|
||||
|
||||
238
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":278,"prompt_ms":370.003,"prompt_per_token_ms":1.3309460431654676,"prompt_per_second":751.345259362762,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":312,"cache":0,"processed":278,"time_ms":370}}
|
||||
|
||||
|
||||
237
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":308,"prompt_ms":413.826,"prompt_per_token_ms":1.343590909090909,"prompt_per_second":744.274163537332,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":312,"cache":0,"processed":308,"time_ms":413}}
|
||||
|
||||
|
||||
239
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":552.907,"prompt_per_token_ms":1.7721378205128206,"prompt_per_second":564.2901970855858,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":312,"cache":0,"processed":312,"time_ms":552}}
|
||||
|
||||
|
||||
2eb
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"I"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
1f3
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"'ll"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":2,"predicted_ms":16.527,"predicted_per_token_ms":16.527,"predicted_per_second":60.50704907121679}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" read"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":3,"predicted_ms":30.874,"predicted_per_token_ms":15.437,"predicted_per_second":64.77942605428517}}
|
||||
|
||||
|
||||
200
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" the"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":4,"predicted_ms":45.026,"predicted_per_token_ms":15.008666666666668,"predicted_per_second":66.62817039044107}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" `/"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":5,"predicted_ms":59.061,"predicted_per_token_ms":14.76525,"predicted_per_second":67.72658776519192}}
|
||||
|
||||
|
||||
1f4
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"etc"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":6,"predicted_ms":73.188,"predicted_per_token_ms":14.6376,"predicted_per_second":68.31721047166201}}
|
||||
|
||||
|
||||
1fd
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"/"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":7,"predicted_ms":87.607,"predicted_per_token_ms":14.601166666666666,"predicted_per_second":68.48767792527994}}
|
||||
|
||||
|
||||
205
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"hostname"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":8,"predicted_ms":101.729,"predicted_per_token_ms":14.532714285714286,"predicted_per_second":68.81027042436277}}
|
||||
|
||||
|
||||
1f5
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"`"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":9,"predicted_ms":115.893,"predicted_per_token_ms":14.486625,"predicted_per_second":69.02919071902531}}
|
||||
|
||||
|
||||
203
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" file"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":10,"predicted_ms":130.046,"predicted_per_token_ms":14.449555555555555,"predicted_per_second":69.20628085446688}}
|
||||
|
||||
|
||||
201
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" for"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":11,"predicted_ms":144.156,"predicted_per_token_ms":14.415600000000001,"predicted_per_second":69.3692943755376}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":" you"}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":12,"predicted_ms":158.604,"predicted_per_token_ms":14.418545454545455,"predicted_per_second":69.35512345211974}}
|
||||
|
||||
|
||||
1ff
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"."}}],"created":1789675306,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":13,"predicted_ms":172.624,"predicted_per_token_ms":14.385333333333334,"predicted_per_second":69.51524701084438}}
|
||||
|
||||
|
||||
202
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"\n\n"}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":14,"predicted_ms":186.606,"predicted_per_token_ms":14.354307692307692,"predicted_per_second":69.66549842984683}}
|
||||
|
||||
|
||||
274
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"id":"wgE8iFI58Zni4WCTiCMNp4TzCcM8ou7F","type":"function","function":{"name":"read_file","arguments":"{"}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":22,"predicted_ms":299.638,"predicted_per_token_ms":14.26847619047619,"predicted_per_second":70.0845687129136}}
|
||||
|
||||
|
||||
232
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\"path\":\""}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":27,"predicted_ms":370.16,"predicted_per_token_ms":14.236923076923079,"predicted_per_second":70.23989626107628}}
|
||||
|
||||
|
||||
22b
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"/etc"}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":28,"predicted_ms":384.27,"predicted_per_token_ms":14.232222222222221,"predicted_per_second":70.26309626044188}}
|
||||
|
||||
|
||||
229
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"/"}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":29,"predicted_ms":398.284,"predicted_per_token_ms":14.224428571428572,"predicted_per_second":70.30159383756315}}
|
||||
|
||||
|
||||
230
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"hostname"}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":30,"predicted_ms":412.404,"predicted_per_token_ms":14.220827586206896,"predicted_per_second":70.31939554417514}}
|
||||
|
||||
|
||||
229
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"\""}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":35,"predicted_ms":482.812,"predicted_per_token_ms":14.20035294117647,"predicted_per_second":70.42078490178372}}
|
||||
|
||||
|
||||
228
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"tool_calls":[{"index":0,"function":{"arguments":"}"}}]}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":39,"predicted_ms":538.943,"predicted_per_token_ms":14.18271052631579,"predicted_per_second":70.50838400350315}}
|
||||
|
||||
|
||||
1f1
|
||||
data: {"choices":[{"finish_reason":"tool_calls","index":0,"delta":{}}],"created":1789675307,"id":"chatcmpl-e9aFhGwvvU0YGs2GTOy83IQWDpVtNvcP","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":0,"prompt_n":312,"prompt_ms":553.156,"prompt_per_token_ms":1.7729358974358973,"prompt_per_second":564.0361850906436,"predicted_n":41,"predicted_ms":566.991,"predicted_per_token_ms":14.174775,"predicted_per_second":70.54785702065817}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
214
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":44,"cache":15,"processed":15,"time_ms":6}}
|
||||
|
||||
|
||||
22e
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":25,"prompt_ms":106.593,"prompt_per_token_ms":4.26372,"prompt_per_second":234.53697709980955,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":44,"cache":15,"processed":40,"time_ms":106}}
|
||||
|
||||
|
||||
236
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":29,"prompt_ms":161.482,"prompt_per_token_ms":5.568344827586207,"prompt_per_second":179.586579309149,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":44,"cache":15,"processed":44,"time_ms":161}}
|
||||
|
||||
|
||||
2ed
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"Blue"}}],"created":1789675316,"id":"chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":29,"prompt_ms":161.813,"prompt_per_token_ms":5.579758620689654,"prompt_per_second":179.2192221885757,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
1e5
|
||||
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{}}],"created":1789675316,"id":"chatcmpl-1OEWBOVSZ1NthqfpaLOboUWbY9OJixgN","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":15,"prompt_n":29,"prompt_ms":161.813,"prompt_per_token_ms":5.579758620689654,"prompt_per_second":179.2192221885757,"predicted_n":2,"predicted_ms":17.197,"predicted_per_token_ms":17.197,"predicted_per_second":58.14967726929116}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
HTTP/1.1 200 OK
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Connection: close
|
||||
X-Accel-Buffering: no
|
||||
Content-Type: text/event-stream
|
||||
Transfer-Encoding: chunked
|
||||
Connection: close
|
||||
|
||||
214
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":45,"prompt_n":0,"prompt_ms":0.0,"prompt_per_token_ms":0.0,"prompt_per_second":0.0,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":75,"cache":45,"processed":45,"time_ms":0}}
|
||||
|
||||
|
||||
22a
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":45,"prompt_n":2,"prompt_ms":32.846,"prompt_per_token_ms":16.423,"prompt_per_second":60.890214942458755,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":75,"cache":45,"processed":47,"time_ms":32}}
|
||||
|
||||
|
||||
239
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":45,"prompt_n":26,"prompt_ms":114.273,"prompt_per_token_ms":4.3951153846153845,"prompt_per_second":227.52531219098125,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":75,"cache":45,"processed":71,"time_ms":114}}
|
||||
|
||||
|
||||
22d
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":45,"prompt_n":30,"prompt_ms":194.325,"prompt_per_token_ms":6.4775,"prompt_per_second":154.38054805094558,"predicted_n":0,"predicted_ms":0.0,"predicted_per_token_ms":0.0,"predicted_per_second":0.0},"prompt_progress":{"total":75,"cache":45,"processed":75,"time_ms":194}}
|
||||
|
||||
|
||||
2ef
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"role":"assistant","content":null}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk"}
|
||||
|
||||
data: {"choices":[{"finish_reason":null,"index":0,"delta":{"content":"Green"}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":45,"prompt_n":30,"prompt_ms":194.635,"prompt_per_token_ms":6.487833333333333,"prompt_per_second":154.13466231664398,"predicted_n":1,"predicted_ms":0.001,"predicted_per_token_ms":0.0,"predicted_per_second":0.0}}
|
||||
|
||||
|
||||
1e6
|
||||
data: {"choices":[{"finish_reason":"stop","index":0,"delta":{}}],"created":1789675316,"id":"chatcmpl-Po3MDTUPhb5xQOmzad8o6J8hupMG8HcE","model":"ornith-1.5-35b-a3b","system_fingerprint":"b10809-5266f24","object":"chat.completion.chunk","timings":{"cache_n":45,"prompt_n":30,"prompt_ms":194.635,"prompt_per_token_ms":6.487833333333333,"prompt_per_second":154.13466231664398,"predicted_n":2,"predicted_ms":17.301,"predicted_per_token_ms":17.301,"predicted_per_second":57.80012716027976}}
|
||||
|
||||
|
||||
e
|
||||
data: [DONE]
|
||||
|
||||
|
||||
0
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
HTTP/1.1 400 Bad Request
|
||||
Server: llama.cpp
|
||||
Access-Control-Allow-Origin:
|
||||
Content-Type: application/json; charset=utf-8
|
||||
Content-Length: 97
|
||||
Connection: close
|
||||
|
||||
{"error":{"code":400,"message":"model 'no-such-model' not found","type":"invalid_request_error"}}
|
||||
@@ -0,0 +1,310 @@
|
||||
//! Tests for the HTTP client, against the fake server. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::http::{Connection, Head, HttpError, MAX_HEAD, Request, read_capped};
|
||||
use std::time::Duration;
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
const GET_PROPS: Request<'static> = Request {
|
||||
method: "GET",
|
||||
path: "/props?model=m",
|
||||
body: None,
|
||||
};
|
||||
|
||||
fn get(server: &FakeServer) -> (Connection, Result<Head, HttpError>) {
|
||||
let mut conn = Connection::open(&server.socket).unwrap();
|
||||
conn.set_read_timeout(Duration::from_secs(5)).unwrap();
|
||||
conn.send(&GET_PROPS).unwrap();
|
||||
let head = conn.read_head();
|
||||
(conn, head)
|
||||
}
|
||||
|
||||
fn get_body(server: &FakeServer) -> Result<Vec<u8>, HttpError> {
|
||||
let (mut conn, head) = get(server);
|
||||
let head = head?;
|
||||
let mut body = conn.body(&head)?;
|
||||
read_capped(&mut body, 4 * 1024 * 1024)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_recorded_chunked_response() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
let (mut conn, head) = get(&server);
|
||||
let head = head.unwrap();
|
||||
assert_eq!(head.status, 200);
|
||||
assert_eq!(head.header("transfer-encoding"), Some("chunked"));
|
||||
assert_eq!(
|
||||
head.header("content-type"),
|
||||
Some("application/json; charset=utf-8")
|
||||
);
|
||||
// The real server sends this header with an empty value, and `Connection` twice.
|
||||
assert_eq!(head.header("access-control-allow-origin"), Some(""));
|
||||
assert_eq!(
|
||||
head.headers
|
||||
.iter()
|
||||
.filter(|(n, _)| n == "connection")
|
||||
.count(),
|
||||
2
|
||||
);
|
||||
assert_eq!(head.header("x-missing"), None);
|
||||
|
||||
let bytes = read_capped(&mut conn.body(&head).unwrap(), 4 * 1024 * 1024).unwrap();
|
||||
let props: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
let template = props["chat_template"].as_str().unwrap();
|
||||
// The hash recorded in docs/inference-contract.md. If the de-chunking were off by a byte,
|
||||
// this would not match.
|
||||
assert_eq!(
|
||||
proto::sha256(template.as_bytes()).unwrap().to_hex(),
|
||||
"f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_result_does_not_depend_on_how_the_bytes_arrive() {
|
||||
let whole = {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
get_body(&server).unwrap()
|
||||
};
|
||||
for piece in [1, 2, 3, 7, 64, 1000] {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props").trickle(piece, 0)]);
|
||||
assert_eq!(get_body(&server).unwrap(), whole, "in pieces of {piece}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_body_with_a_content_length() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("unknown_model")]);
|
||||
let (mut conn, head) = get(&server);
|
||||
let head = head.unwrap();
|
||||
assert_eq!(head.status, 400);
|
||||
assert_eq!(head.header("content-length"), Some("97"));
|
||||
let bytes = read_capped(&mut conn.body(&head).unwrap(), 4096).unwrap();
|
||||
let body: serde_json::Value = serde_json::from_slice(&bytes).unwrap();
|
||||
assert_eq!(body["error"]["message"], "model 'no-such-model' not found");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reads_a_body_that_runs_to_the_close() {
|
||||
let server = FakeServer::start();
|
||||
server.route(
|
||||
"/props",
|
||||
vec![Reply::raw("HTTP/1.0 200 OK\r\nX-A: b\r\n\r\nto the end")],
|
||||
);
|
||||
assert_eq!(get_body(&server).unwrap(), b"to the end");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_empty_body_is_empty_in_every_framing() {
|
||||
for raw in [
|
||||
"HTTP/1.1 204 No Content\r\nContent-Length: 0\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n0\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\n\r\n",
|
||||
] {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::raw(raw)]);
|
||||
assert_eq!(get_body(&server).unwrap(), b"", "{raw:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn chunk_extensions_and_trailers_are_skipped() {
|
||||
let raw = "HTTP/1.1 200 OK\r\nTransfer-Encoding: Chunked\r\n\r\n\
|
||||
5;ext=1\r\nhello\r\n1\r\n,\r\n6\r\n world\r\n0\r\nX-Trailer: t\r\n\r\n";
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::raw(raw)]);
|
||||
assert_eq!(get_body(&server).unwrap(), b"hello, world");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn sends_a_well_formed_request() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/tokenize", vec![Reply::fixture("tokenize")]);
|
||||
let mut conn = Connection::open(&server.socket).unwrap();
|
||||
let body = br#"{"content":"x"}"#;
|
||||
conn.send(&Request {
|
||||
method: "POST",
|
||||
path: "/tokenize",
|
||||
body: Some(body),
|
||||
})
|
||||
.unwrap();
|
||||
conn.read_head().unwrap();
|
||||
let sent = &server.requests()[0];
|
||||
assert_eq!(
|
||||
(sent.method.as_str(), sent.target.as_str()),
|
||||
("POST", "/tokenize")
|
||||
);
|
||||
assert_eq!(sent.body, body);
|
||||
let has = |line: &str| sent.headers.iter().any(|h| h.eq_ignore_ascii_case(line));
|
||||
assert!(has("Connection: close"), "{:?}", sent.headers);
|
||||
assert!(has("Content-Type: application/json"), "{:?}", sent.headers);
|
||||
assert!(has("Content-Length: 15"), "{:?}", sent.headers);
|
||||
assert!(
|
||||
sent.headers
|
||||
.iter()
|
||||
.any(|h| h.to_ascii_lowercase().starts_with("host:"))
|
||||
);
|
||||
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
let _ = get(&server);
|
||||
let sent = &server.requests()[0];
|
||||
assert_eq!(sent.target, "/props?model=m");
|
||||
assert!(
|
||||
!sent
|
||||
.headers
|
||||
.iter()
|
||||
.any(|h| h.to_ascii_lowercase().starts_with("content-length"))
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_timeout_before_the_first_byte_can_be_waited_out() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props").head_delay(300)]);
|
||||
let mut conn = Connection::open(&server.socket).unwrap();
|
||||
conn.set_read_timeout(Duration::from_millis(50)).unwrap();
|
||||
conn.send(&GET_PROPS).unwrap();
|
||||
let mut timeouts = 0;
|
||||
let head = loop {
|
||||
match conn.read_head() {
|
||||
Ok(head) => break head,
|
||||
Err(HttpError::Timeout) => {
|
||||
assert!(!conn.received_any(), "nothing has arrived yet");
|
||||
timeouts += 1;
|
||||
assert!(timeouts < 100);
|
||||
}
|
||||
Err(e) => panic!("{e}"),
|
||||
}
|
||||
};
|
||||
assert!(
|
||||
timeouts >= 3,
|
||||
"only {timeouts} timeouts in 300 ms at 50 ms each"
|
||||
);
|
||||
assert!(conn.received_any());
|
||||
assert_eq!(head.status, 200);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_timeout_in_the_middle_of_the_head_keeps_what_arrived() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props").trickle(10, 120)]);
|
||||
let mut conn = Connection::open(&server.socket).unwrap();
|
||||
conn.set_read_timeout(Duration::from_millis(40)).unwrap();
|
||||
conn.send(&GET_PROPS).unwrap();
|
||||
let mut saw_partial = false;
|
||||
let head = loop {
|
||||
match conn.read_head() {
|
||||
Ok(head) => break head,
|
||||
Err(HttpError::Timeout) => saw_partial |= conn.received_any(),
|
||||
Err(e) => panic!("{e}"),
|
||||
}
|
||||
};
|
||||
assert!(
|
||||
saw_partial,
|
||||
"at least one timeout must have happened after some bytes arrived"
|
||||
);
|
||||
assert_eq!(head.status, 200);
|
||||
assert_eq!(head.header("server"), Some("llama.cpp"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_connection_closed_before_the_head_is_complete_is_closed() {
|
||||
for cut in [0, 5, 40] {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props").cut_after(cut)]);
|
||||
let (_, head) = get(&server);
|
||||
assert!(
|
||||
matches!(head, Err(HttpError::Closed)),
|
||||
"cut after {cut} bytes: {head:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_body_cut_short_is_closed_in_both_framings() {
|
||||
for name in ["props", "unknown_model"] {
|
||||
let full = support::fixture_bytes("http", &format!("{name}.http")).len();
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture(name).cut_after(full - 10)]);
|
||||
let got = get_body(&server);
|
||||
assert!(matches!(got, Err(HttpError::Closed)), "{name}: {got:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_heads_are_rejected() {
|
||||
for raw in [
|
||||
"HTTP/2 200 OK\r\n\r\n",
|
||||
"HELLO\r\n\r\n",
|
||||
"HTTP/1.1 abc OK\r\n\r\n",
|
||||
"HTTP/1.1 99 Low\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nno colon here\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\n: no name\r\n\r\n",
|
||||
] {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::raw(raw)]);
|
||||
let (_, head) = get(&server);
|
||||
assert!(
|
||||
matches!(head, Err(HttpError::Malformed(_))),
|
||||
"{raw:?}: {head:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn malformed_framing_is_an_error_not_a_panic() {
|
||||
for raw in [
|
||||
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nzz\r\nhello\r\n0\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n5\r\nhelloXX0\r\n\r\n",
|
||||
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\nffffffffffffffffffffffff\r\nx\r\n",
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: many\r\n\r\nhello",
|
||||
"HTTP/1.1 200 OK\r\nContent-Length: -5\r\n\r\nhello",
|
||||
] {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::raw(raw)]);
|
||||
assert!(get_body(&server).is_err(), "{raw:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn everything_read_from_the_peer_is_bounded() {
|
||||
// A head that never ends.
|
||||
let server = FakeServer::start();
|
||||
let endless = format!("HTTP/1.1 200 OK\r\nX-Pad: {}\r\n", "a".repeat(2 * MAX_HEAD));
|
||||
server.route("/props", vec![Reply::raw(endless)]);
|
||||
let (_, head) = get(&server);
|
||||
assert!(matches!(head, Err(HttpError::TooLarge(_))), "{head:?}");
|
||||
|
||||
// A chunk-size line that never ends.
|
||||
let server = FakeServer::start();
|
||||
let raw = format!(
|
||||
"HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n{}",
|
||||
"1".repeat(10_000)
|
||||
);
|
||||
server.route("/props", vec![Reply::raw(raw)]);
|
||||
assert!(get_body(&server).is_err());
|
||||
|
||||
// A body larger than the caller allows.
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
let (mut conn, head) = get(&server);
|
||||
let head = head.unwrap();
|
||||
let got = read_capped(&mut conn.body(&head).unwrap(), 1000);
|
||||
assert!(matches!(got, Err(HttpError::TooLarge(_))), "{got:?}");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_missing_socket_is_a_connect_error() {
|
||||
let missing = std::env::temp_dir().join("loopd-no-such-socket.sock");
|
||||
assert!(matches!(
|
||||
Connection::open(&missing),
|
||||
Err(HttpError::Connect(_))
|
||||
));
|
||||
let e: Box<dyn std::error::Error> = Box::new(HttpError::Timeout);
|
||||
assert!(!e.to_string().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
//! Tests for the small server calls and the cache-loss check. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::info::{CacheOutcome, SlotInfo, cache_outcome};
|
||||
use loopd::llama::{Client, InferError, Timings};
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
fn client(server: &FakeServer) -> Client {
|
||||
Client::new(support::test_config(&server.socket))
|
||||
}
|
||||
|
||||
fn t(cache_n: u64, prompt_n: u64, predicted_n: u64) -> Timings {
|
||||
Timings {
|
||||
cache_n,
|
||||
prompt_n,
|
||||
predicted_n,
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn props_slots_and_tokenize_read_the_recorded_responses() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route("/slots", vec![Reply::fixture("slots")]);
|
||||
server.route("/tokenize", vec![Reply::fixture("tokenize")]);
|
||||
let c = client(&server);
|
||||
|
||||
let props = c.props().unwrap();
|
||||
assert_eq!(props.n_ctx, 131_072);
|
||||
assert_eq!(props.total_slots, 2);
|
||||
assert_eq!(
|
||||
proto::sha256(props.chat_template.as_bytes())
|
||||
.unwrap()
|
||||
.to_hex(),
|
||||
"f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
);
|
||||
|
||||
let want = vec![
|
||||
SlotInfo {
|
||||
id: 0,
|
||||
is_processing: false,
|
||||
},
|
||||
SlotInfo {
|
||||
id: 1,
|
||||
is_processing: false,
|
||||
},
|
||||
];
|
||||
assert_eq!(c.slots().unwrap(), want);
|
||||
|
||||
assert_eq!(c.tokenize("The box is made.").unwrap(), 5);
|
||||
|
||||
// In router mode the model goes in the query for GET and in the body for POST.
|
||||
assert_eq!(
|
||||
server.requests_to("/props")[0].target,
|
||||
"/props?model=test-model"
|
||||
);
|
||||
assert_eq!(
|
||||
server.requests_to("/slots")[0].target,
|
||||
"/slots?model=test-model"
|
||||
);
|
||||
let sent = server.requests_to("/tokenize")[0].json();
|
||||
assert_eq!(
|
||||
sent,
|
||||
serde_json::json!({"model": "test-model", "content": "The box is made."})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn an_error_status_keeps_its_body() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("unknown_model")]);
|
||||
match client(&server).props() {
|
||||
Err(InferError::Http { status: 400, body }) => {
|
||||
assert!(body.contains("no-such-model"), "{body}")
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_long_error_body_is_cut_at_4_kib_on_a_character_boundary() {
|
||||
let server = FakeServer::start();
|
||||
// 3 bytes of padding, then 2-byte characters, so that byte 4096 falls inside a character.
|
||||
let long = format!("abc{}", "\u{e9}".repeat(5000));
|
||||
server.route("/props", vec![Reply::json(500, &long)]);
|
||||
match client(&server).props() {
|
||||
Err(InferError::Http { status: 500, body }) => {
|
||||
assert!(
|
||||
body.len() <= 4096 && body.len() >= 4094,
|
||||
"{} bytes",
|
||||
body.len()
|
||||
);
|
||||
assert!(body.starts_with("abc\u{e9}"));
|
||||
}
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn unexpected_bodies_are_protocol_errors() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::json(200, r#"{"chat_template": 5}"#)]);
|
||||
server.route("/slots", vec![Reply::json(200, "not json")]);
|
||||
server.route("/tokenize", vec![Reply::json(200, r#"{"tokens": "many"}"#)]);
|
||||
let c = client(&server);
|
||||
assert!(matches!(c.props(), Err(InferError::Protocol(_))));
|
||||
assert!(matches!(c.slots(), Err(InferError::Protocol(_))));
|
||||
assert!(matches!(c.tokenize("x"), Err(InferError::Protocol(_))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_dead_server_is_a_connect_error() {
|
||||
let server = FakeServer::start();
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.infer.socket = std::env::temp_dir().join("loopd-no-such-socket.sock");
|
||||
assert!(matches!(
|
||||
Client::new(cfg).props(),
|
||||
Err(InferError::Connect(_))
|
||||
));
|
||||
}
|
||||
|
||||
/// Numbers from docs/inference-contract.md.
|
||||
#[test]
|
||||
fn cache_outcome_matches_what_was_measured() {
|
||||
// (b) a normal turn: 539 processed + 53 generated, then 591 reused.
|
||||
assert_eq!(
|
||||
cache_outcome(&t(0, 539, 53), &t(591, 29, 61)),
|
||||
CacheOutcome::Hit
|
||||
);
|
||||
// (k) a natural end to thinking: 64 + 104 = 168 left, 167 reused.
|
||||
assert_eq!(
|
||||
cache_outcome(&t(0, 64, 104), &t(167, 22, 2)),
|
||||
CacheOutcome::Hit
|
||||
);
|
||||
// (k) a forced end to thinking: 89 + 717 = 806 left, only 85 reused.
|
||||
assert_eq!(
|
||||
cache_outcome(&t(0, 89, 717), &t(85, 743, 2)),
|
||||
CacheOutcome::Loss {
|
||||
expected: 806,
|
||||
got: 85
|
||||
}
|
||||
);
|
||||
// (h) the tool list changed: everything re-read.
|
||||
assert_eq!(
|
||||
cache_outcome(&t(685, 27, 55), &t(23, 920, 126)),
|
||||
CacheOutcome::Loss {
|
||||
expected: 767,
|
||||
got: 23
|
||||
}
|
||||
);
|
||||
// The recorded two-turn exchange used by the self-test.
|
||||
assert_eq!(
|
||||
cache_outcome(&t(15, 29, 2), &t(45, 30, 2)),
|
||||
CacheOutcome::Hit
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cache_outcome_tolerates_64_tokens_and_no_more() {
|
||||
let previous = t(1000, 100, 50); // 1150 left in the slot
|
||||
assert_eq!(cache_outcome(&previous, &t(1150, 5, 5)), CacheOutcome::Hit);
|
||||
assert_eq!(
|
||||
cache_outcome(&previous, &t(1086, 5, 5)),
|
||||
CacheOutcome::Hit,
|
||||
"64 short"
|
||||
);
|
||||
assert_eq!(
|
||||
cache_outcome(&previous, &t(1085, 5, 5)),
|
||||
CacheOutcome::Loss {
|
||||
expected: 1150,
|
||||
got: 1085
|
||||
},
|
||||
"65 short"
|
||||
);
|
||||
assert_eq!(
|
||||
cache_outcome(&t(0, 0, 0), &t(0, 10, 1)),
|
||||
CacheOutcome::Hit,
|
||||
"a first request"
|
||||
);
|
||||
assert_eq!(
|
||||
cache_outcome(&previous, &t(5000, 5, 5)),
|
||||
CacheOutcome::Hit,
|
||||
"more than expected"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,179 @@
|
||||
//! Tests for the request body. Do not edit.
|
||||
//!
|
||||
//! Bodies are compared as JSON values, so key order is free. What is not free: which keys
|
||||
//! appear, and that the same input always gives the same bytes.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::request::build_body;
|
||||
use loopd::llama::{ChatMessage, ChatRequest, ToolSchema};
|
||||
use serde_json::{Value, json};
|
||||
use std::path::Path;
|
||||
|
||||
fn body(req: &ChatRequest) -> Value {
|
||||
let cfg = support::test_config(Path::new("/tmp/unused.sock"));
|
||||
serde_json::from_str(&build_body(&cfg, req).unwrap()).unwrap()
|
||||
}
|
||||
|
||||
fn user(text: &str) -> ChatMessage {
|
||||
ChatMessage::User {
|
||||
content: text.to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
fn read_file_tool() -> ToolSchema {
|
||||
ToolSchema {
|
||||
name: "read_file".to_string(),
|
||||
description: "Read a text file.".to_string(),
|
||||
parameters: json!({"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]}),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_plain_request_carries_every_setting() {
|
||||
let req = ChatRequest {
|
||||
slot: 1,
|
||||
messages: vec![user("hi")],
|
||||
tools: vec![],
|
||||
thinking: true,
|
||||
};
|
||||
let want = json!({
|
||||
"model": "test-model",
|
||||
"messages": [{"role": "user", "content": "hi"}],
|
||||
"id_slot": 1,
|
||||
"cache_prompt": true,
|
||||
"stream": true,
|
||||
"return_progress": true,
|
||||
"timings_per_token": true,
|
||||
"reasoning_control": true,
|
||||
"max_tokens": 8192,
|
||||
"temperature": 0.6,
|
||||
"top_p": 0.95,
|
||||
"top_k": 20,
|
||||
"chat_template_kwargs": {"enable_thinking": true},
|
||||
});
|
||||
assert_eq!(body(&req), want, "no `tools` key when there are no tools");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn thinking_off_and_tools() {
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![user("hi")],
|
||||
tools: vec![read_file_tool()],
|
||||
thinking: false,
|
||||
};
|
||||
let got = body(&req);
|
||||
assert_eq!(
|
||||
got["chat_template_kwargs"],
|
||||
json!({"enable_thinking": false})
|
||||
);
|
||||
let want_tools = json!([{
|
||||
"type": "function",
|
||||
"function": {
|
||||
"name": "read_file",
|
||||
"description": "Read a text file.",
|
||||
"parameters": {"type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"]},
|
||||
},
|
||||
}]);
|
||||
assert_eq!(got["tools"], want_tools);
|
||||
}
|
||||
|
||||
/// One message of every kind, and both shapes of assistant message. An example of each is
|
||||
/// here on purpose: they are rendered differently.
|
||||
#[test]
|
||||
fn every_kind_of_message() {
|
||||
let call = proto::ToolCall {
|
||||
id: "call_1".to_string(),
|
||||
name: "read_file".to_string(),
|
||||
arguments: r#"{"path":"/etc/hostname"}"#.to_string(),
|
||||
};
|
||||
let messages = vec![
|
||||
ChatMessage::System {
|
||||
content: "be careful".to_string(),
|
||||
},
|
||||
user("read it"),
|
||||
ChatMessage::Assistant {
|
||||
content: None,
|
||||
reasoning_content: Some("I should read the file.".to_string()),
|
||||
tool_calls: vec![call],
|
||||
},
|
||||
ChatMessage::Tool {
|
||||
tool_call_id: "call_1".to_string(),
|
||||
content: "straylight\n".to_string(),
|
||||
},
|
||||
ChatMessage::Assistant {
|
||||
content: Some("It says straylight.".to_string()),
|
||||
reasoning_content: None,
|
||||
tool_calls: vec![],
|
||||
},
|
||||
];
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages,
|
||||
tools: vec![read_file_tool()],
|
||||
thinking: true,
|
||||
};
|
||||
let want = json!([
|
||||
{"role": "system", "content": "be careful"},
|
||||
{"role": "user", "content": "read it"},
|
||||
{
|
||||
"role": "assistant",
|
||||
"content": "",
|
||||
"reasoning_content": "I should read the file.",
|
||||
"tool_calls": [{
|
||||
"id": "call_1",
|
||||
"type": "function",
|
||||
"function": {"name": "read_file", "arguments": "{\"path\":\"/etc/hostname\"}"},
|
||||
}],
|
||||
},
|
||||
{"role": "tool", "tool_call_id": "call_1", "content": "straylight\n"},
|
||||
{"role": "assistant", "content": "It says straylight."},
|
||||
]);
|
||||
assert_eq!(body(&req)["messages"], want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn values_come_from_the_config() {
|
||||
let mut cfg = support::test_config(Path::new("/tmp/unused.sock"));
|
||||
cfg.infer.model = "another-model".to_string();
|
||||
cfg.limits.max_tokens = 99;
|
||||
cfg.sampling.temperature = 0.25;
|
||||
cfg.sampling.top_p = 0.5;
|
||||
cfg.sampling.top_k = 7;
|
||||
let req = ChatRequest {
|
||||
slot: 3,
|
||||
messages: vec![user("hi")],
|
||||
tools: vec![],
|
||||
thinking: false,
|
||||
};
|
||||
let got: Value = serde_json::from_str(&build_body(&cfg, &req).unwrap()).unwrap();
|
||||
assert_eq!(got["model"], "another-model");
|
||||
assert_eq!(got["max_tokens"], 99);
|
||||
assert_eq!(got["temperature"], 0.25);
|
||||
assert_eq!(got["top_p"], 0.5);
|
||||
assert_eq!(got["top_k"], 7);
|
||||
assert_eq!(got["id_slot"], 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_same_input_gives_the_same_bytes() {
|
||||
let cfg = support::test_config(Path::new("/tmp/unused.sock"));
|
||||
let req = ChatRequest {
|
||||
slot: 0,
|
||||
messages: vec![user(
|
||||
"text with \"quotes\", a \\ backslash, a\nnewline and caf\u{e9} \u{1f4e6}",
|
||||
)],
|
||||
tools: vec![read_file_tool()],
|
||||
thinking: true,
|
||||
};
|
||||
let first = build_body(&cfg, &req).unwrap();
|
||||
for _ in 0..20 {
|
||||
assert_eq!(build_body(&cfg, &req).unwrap(), first);
|
||||
}
|
||||
let back: Value = serde_json::from_str(&first).unwrap();
|
||||
assert_eq!(
|
||||
back["messages"][0]["content"],
|
||||
"text with \"quotes\", a \\ backslash, a\nnewline and caf\u{e9} \u{1f4e6}"
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
//! Tests for the slot gate and for retrying. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::gate::{GateFull, SlotGate};
|
||||
use loopd::llama::retry::{backoff_ms, is_retryable};
|
||||
use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client, InferError};
|
||||
use std::sync::atomic::{AtomicUsize, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::{Duration, Instant};
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
fn request(slot: u32) -> ChatRequest {
|
||||
ChatRequest {
|
||||
slot,
|
||||
messages: vec![ChatMessage::User {
|
||||
content: "hi".to_string(),
|
||||
}],
|
||||
tools: vec![],
|
||||
thinking: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn io(kind: std::io::ErrorKind) -> std::io::Error {
|
||||
std::io::Error::from(kind)
|
||||
}
|
||||
|
||||
// ---- the gate on its own ----
|
||||
|
||||
#[test]
|
||||
fn a_free_slot_is_taken_at_once_and_slots_are_independent() {
|
||||
let gate = SlotGate::new();
|
||||
let mut queued = Vec::new();
|
||||
let a = gate.acquire(0, 8, &mut |n| queued.push(n)).unwrap();
|
||||
let b = gate.acquire(1, 8, &mut |n| queued.push(n)).unwrap();
|
||||
assert!(queued.is_empty(), "nobody had to wait");
|
||||
drop(a);
|
||||
drop(b);
|
||||
let _again = gate.acquire(0, 8, &mut |n| queued.push(n)).unwrap();
|
||||
assert!(queued.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn waiters_are_served_in_order_one_at_a_time() {
|
||||
let gate = Arc::new(SlotGate::new());
|
||||
let order = Arc::new(Mutex::new(Vec::new()));
|
||||
let inside = Arc::new(AtomicUsize::new(0));
|
||||
let first = gate.acquire(0, 8, &mut |_| {}).unwrap();
|
||||
let mut threads = Vec::new();
|
||||
for i in 0..4usize {
|
||||
let (gate, order, inside) = (Arc::clone(&gate), Arc::clone(&order), Arc::clone(&inside));
|
||||
threads.push(thread::spawn(move || {
|
||||
let mut ahead = None;
|
||||
let permit = gate.acquire(0, 8, &mut |n| ahead = Some(n)).unwrap();
|
||||
assert_eq!(
|
||||
inside.fetch_add(1, Ordering::SeqCst),
|
||||
0,
|
||||
"two permits for one slot at once"
|
||||
);
|
||||
order.lock().unwrap().push((i, ahead));
|
||||
thread::sleep(Duration::from_millis(20));
|
||||
inside.fetch_sub(1, Ordering::SeqCst);
|
||||
drop(permit);
|
||||
}));
|
||||
thread::sleep(Duration::from_millis(30)); // so that the arrival order is known
|
||||
}
|
||||
drop(first);
|
||||
for t in threads {
|
||||
t.join().unwrap();
|
||||
}
|
||||
// Each was told how many were ahead of it: the holder plus the earlier waiters.
|
||||
let want: Vec<(usize, Option<usize>)> = (0..4).map(|i| (i, Some(i + 1))).collect();
|
||||
assert_eq!(*order.lock().unwrap(), want);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_queue_refuses_at_once() {
|
||||
let gate = Arc::new(SlotGate::new());
|
||||
let _holder = gate.acquire(0, 1, &mut |_| {}).unwrap();
|
||||
let waiter = {
|
||||
let gate = Arc::clone(&gate);
|
||||
thread::spawn(move || drop(gate.acquire(0, 1, &mut |_| {}).unwrap()))
|
||||
};
|
||||
thread::sleep(Duration::from_millis(50));
|
||||
let started = Instant::now();
|
||||
assert!(
|
||||
matches!(gate.acquire(0, 1, &mut |_| {}), Err(GateFull)),
|
||||
"one holder and one waiter is the limit"
|
||||
);
|
||||
assert!(
|
||||
started.elapsed() < Duration::from_millis(50),
|
||||
"refusal must not wait"
|
||||
);
|
||||
assert!(
|
||||
gate.acquire(1, 1, &mut |_| {}).is_ok(),
|
||||
"another slot is unaffected"
|
||||
);
|
||||
drop(_holder);
|
||||
waiter.join().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_queue_of_zero_means_no_waiting_at_all() {
|
||||
let gate = SlotGate::new();
|
||||
let _holder = gate.acquire(0, 0, &mut |_| {}).unwrap();
|
||||
assert!(matches!(gate.acquire(0, 0, &mut |_| {}), Err(GateFull)));
|
||||
}
|
||||
|
||||
// ---- the gate inside the client ----
|
||||
|
||||
#[test]
|
||||
fn two_requests_for_one_slot_never_overlap_at_the_server() {
|
||||
let server = FakeServer::start();
|
||||
let size = support::fixture_bytes("http", "turn1.http").len();
|
||||
// Each reply takes about 200 ms to arrive.
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").trickle(size / 4, 50)]);
|
||||
let client = Arc::new(Client::new(support::test_config(&server.socket)));
|
||||
let started = Instant::now();
|
||||
let second = {
|
||||
let client = Arc::clone(&client);
|
||||
thread::spawn(move || {
|
||||
thread::sleep(Duration::from_millis(30));
|
||||
let mut events = Vec::new();
|
||||
let done = client.chat(&request(0), &mut |e| events.push(e.clone()));
|
||||
(done.map(|d| d.content), events, Instant::now())
|
||||
})
|
||||
};
|
||||
let first = client.chat(&request(0), &mut |_| {}).unwrap();
|
||||
let first_done = Instant::now();
|
||||
let (second_result, second_events, second_done) = second.join().unwrap();
|
||||
assert_eq!(first.content.as_deref(), Some("Blue"));
|
||||
assert_eq!(second_result.unwrap().as_deref(), Some("Blue"));
|
||||
assert_eq!(second_events.first(), Some(&ChatEvent::Queued { ahead: 1 }));
|
||||
assert!(second_done > first_done);
|
||||
assert!(
|
||||
started.elapsed() >= Duration::from_millis(280),
|
||||
"the two ran one after the other"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_full_queue_is_busy() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").head_delay(300)]);
|
||||
// Both slots report busy, so that the 300 ms of silence reads as a queue and not a stall.
|
||||
let busy = r#"[{"id":0,"is_processing":true},{"id":1,"is_processing":true}]"#;
|
||||
server.route("/slots", vec![Reply::json(200, busy)]);
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.limits.queue_len = 0;
|
||||
let client = Arc::new(Client::new(cfg));
|
||||
let holder = {
|
||||
let client = Arc::clone(&client);
|
||||
thread::spawn(move || client.chat(&request(0), &mut |_| {}).map(|_| ()))
|
||||
};
|
||||
thread::sleep(Duration::from_millis(80));
|
||||
assert!(matches!(
|
||||
client.chat(&request(0), &mut |_| {}),
|
||||
Err(InferError::Busy)
|
||||
));
|
||||
assert!(
|
||||
client.chat(&request(1), &mut |_| {}).is_ok(),
|
||||
"another slot is free"
|
||||
);
|
||||
holder.join().unwrap().unwrap();
|
||||
assert_eq!(
|
||||
server.requests_to(CHAT).len(),
|
||||
2,
|
||||
"the refused request was never sent"
|
||||
);
|
||||
}
|
||||
|
||||
// ---- retry ----
|
||||
|
||||
#[test]
|
||||
fn which_errors_are_retried() {
|
||||
use std::io::ErrorKind::ConnectionRefused;
|
||||
let http = |status| InferError::Http {
|
||||
status,
|
||||
body: String::new(),
|
||||
};
|
||||
let retried = [
|
||||
InferError::Connect(io(ConnectionRefused)),
|
||||
InferError::StreamClosedEarly,
|
||||
InferError::Stalled,
|
||||
InferError::LoadTimeout,
|
||||
http(503),
|
||||
];
|
||||
for e in &retried {
|
||||
assert!(
|
||||
is_retryable(e),
|
||||
"{e:?} means the server went away and may be back"
|
||||
);
|
||||
}
|
||||
let not_retried = [
|
||||
InferError::Busy,
|
||||
InferError::WaitTimeout,
|
||||
InferError::ThinkingOverrun,
|
||||
InferError::Protocol("x".to_string()),
|
||||
http(400),
|
||||
http(404),
|
||||
http(500),
|
||||
];
|
||||
for e in ¬_retried {
|
||||
assert!(
|
||||
!is_retryable(e),
|
||||
"{e:?} would fail again, or has already waited"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn backoff_follows_the_schedule_with_a_quarter_of_jitter() {
|
||||
let schedule = [2_000, 8_000, 30_000];
|
||||
assert_eq!(backoff_ms(&schedule, 1, 0), 2_000);
|
||||
assert_eq!(backoff_ms(&schedule, 2, 0), 8_000);
|
||||
assert_eq!(backoff_ms(&schedule, 3, 0), 30_000);
|
||||
assert_eq!(
|
||||
backoff_ms(&schedule, 4, 0),
|
||||
30_000,
|
||||
"past the end, the last entry"
|
||||
);
|
||||
assert_eq!(
|
||||
backoff_ms(&schedule, 0, 0),
|
||||
2_000,
|
||||
"retry numbers start at 1; 0 is treated as 1"
|
||||
);
|
||||
assert_eq!(backoff_ms(&schedule, 1, 1000), 2_500);
|
||||
assert_eq!(backoff_ms(&schedule, 1, -1000), 1_500);
|
||||
assert_eq!(backoff_ms(&schedule, 2, 500), 9_000);
|
||||
assert_eq!(
|
||||
backoff_ms(&schedule, 1, 99_999),
|
||||
2_500,
|
||||
"jitter is clamped to -1000..=1000"
|
||||
);
|
||||
assert_eq!(backoff_ms(&schedule, 1, i32::MIN), 1_500);
|
||||
assert_eq!(backoff_ms(&[], 1, 1000), 0, "no schedule, no wait");
|
||||
// Huge values must not overflow or panic.
|
||||
let _ = backoff_ms(&[u64::MAX], 1, 1000);
|
||||
let _ = backoff_ms(&[u64::MAX], u32::MAX, -1000);
|
||||
}
|
||||
|
||||
fn run_retry(
|
||||
server: &FakeServer,
|
||||
attempts: u32,
|
||||
window_ms: u64,
|
||||
) -> (Result<Option<String>, InferError>, Vec<ChatEvent>) {
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
cfg.limits.retry_attempts = attempts;
|
||||
cfg.limits.retry_window_ms = window_ms;
|
||||
let mut events = Vec::new();
|
||||
let result = Client::new(cfg).chat_with_retry(&request(0), &mut |e| events.push(e.clone()));
|
||||
(result.map(|d| d.content), events)
|
||||
}
|
||||
|
||||
fn retrying(events: &[ChatEvent]) -> Vec<(u32, u64)> {
|
||||
events
|
||||
.iter()
|
||||
.filter_map(|e| match e {
|
||||
ChatEvent::Retrying {
|
||||
attempt, after_ms, ..
|
||||
} => Some((*attempt, *after_ms)),
|
||||
_ => None,
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_that_comes_back_is_survived() {
|
||||
let server = FakeServer::start();
|
||||
let dead = Reply::fixture("turn1").cut_after(400);
|
||||
server.route(CHAT, vec![dead.clone(), dead, Reply::fixture("turn1")]);
|
||||
let (result, events) = run_retry(&server, 4, 5_000);
|
||||
assert_eq!(result.unwrap().as_deref(), Some("Blue"));
|
||||
let retries = retrying(&events);
|
||||
assert_eq!(
|
||||
retries
|
||||
.iter()
|
||||
.map(|(attempt, _)| *attempt)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![2, 3]
|
||||
);
|
||||
// The test schedule is 10, 20, 30 ms, each moved by at most a quarter.
|
||||
assert!((8..=12).contains(&retries[0].1), "{retries:?}");
|
||||
assert!((15..=25).contains(&retries[1].1), "{retries:?}");
|
||||
let sent = server.requests_to(CHAT);
|
||||
assert_eq!(sent.len(), 3);
|
||||
assert!(
|
||||
sent.iter().all(|r| r.body == sent[0].body),
|
||||
"a retry sends the same bytes again"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retries_stop_at_the_attempt_limit() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").cut_after(400)]);
|
||||
let (result, events) = run_retry(&server, 3, 5_000);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::StreamClosedEarly)),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(server.requests_to(CHAT).len(), 3, "three attempts in all");
|
||||
assert_eq!(retrying(&events).len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn retries_stop_at_the_time_window() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("turn1").cut_after(400)]);
|
||||
// The first retry would wait about 10 ms, which does not fit in a 5 ms window.
|
||||
let (result, events) = run_retry(&server, 10, 5);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::StreamClosedEarly)),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(server.requests_to(CHAT).len(), 1);
|
||||
assert!(retrying(&events).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn errors_that_would_fail_again_are_not_retried() {
|
||||
let server = FakeServer::start();
|
||||
server.route(CHAT, vec![Reply::fixture("bad_request")]);
|
||||
let (result, events) = run_retry(&server, 4, 5_000);
|
||||
assert!(
|
||||
matches!(result, Err(InferError::Http { status: 400, .. })),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(server.requests_to(CHAT).len(), 1);
|
||||
assert!(retrying(&events).is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_loading_server_is_retried() {
|
||||
let server = FakeServer::start();
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::json(503, r#"{"error":{"message":"Loading model"}}"#),
|
||||
Reply::fixture("turn1"),
|
||||
],
|
||||
);
|
||||
let (result, events) = run_retry(&server, 4, 5_000);
|
||||
assert_eq!(result.unwrap().as_deref(), Some("Blue"));
|
||||
let ChatEvent::Retrying { error, .. } = events
|
||||
.iter()
|
||||
.find(|e| matches!(e, ChatEvent::Retrying { .. }))
|
||||
.unwrap()
|
||||
else {
|
||||
unreachable!()
|
||||
};
|
||||
assert!(error.contains("503"), "the event says why: {error}");
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
//! Tests for the startup self-test, against the fake server. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::llama::Client;
|
||||
use loopd::selftest::{SelfTestError, run};
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
const CHAT: &str = "/v1/chat/completions";
|
||||
|
||||
/// A change to the config before the self-test runs.
|
||||
type Edit = fn(&mut loopd::config::Config);
|
||||
|
||||
/// A server that passes: the recorded props, then the three recorded completions in order.
|
||||
fn healthy() -> FakeServer {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("turn1"),
|
||||
Reply::fixture("turn2"),
|
||||
],
|
||||
);
|
||||
server
|
||||
}
|
||||
|
||||
fn run_with(
|
||||
server: &FakeServer,
|
||||
edit: impl FnOnce(&mut loopd::config::Config),
|
||||
) -> (Result<(), SelfTestError>, Vec<String>) {
|
||||
let mut cfg = support::test_config(&server.socket);
|
||||
edit(&mut cfg);
|
||||
let mut steps = Vec::new();
|
||||
let result = run(&Client::new(cfg), &mut |s| steps.push(s.to_string()));
|
||||
(result, steps)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_healthy_server_passes_all_three_checks_in_order() {
|
||||
let server = healthy();
|
||||
let (result, steps) = run_with(&server, |_| {});
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert_eq!(
|
||||
steps,
|
||||
[
|
||||
"server matches config",
|
||||
"tool call round trip",
|
||||
"turn 2 cache hit"
|
||||
]
|
||||
);
|
||||
|
||||
let chats = server.requests_to(CHAT);
|
||||
assert_eq!(chats.len(), 3);
|
||||
// The tool-call check offers exactly one tool; the cache check offers none.
|
||||
assert_eq!(chats[0].json()["tools"].as_array().map(Vec::len), Some(1));
|
||||
assert!(chats[1].json().get("tools").is_none());
|
||||
// All three run on the main slot.
|
||||
assert!(chats.iter().all(|c| c.json()["id_slot"] == 0));
|
||||
// Turn 2 is turn 1 plus the assistant's answer plus a new question: a strict extension.
|
||||
let turn1 = chats[1].json()["messages"].as_array().unwrap().clone();
|
||||
let turn2 = chats[2].json()["messages"].as_array().unwrap().clone();
|
||||
assert_eq!(turn2.len(), turn1.len() + 2);
|
||||
assert_eq!(turn2[..turn1.len()], turn1[..]);
|
||||
assert_eq!(
|
||||
turn2[turn1.len()],
|
||||
serde_json::json!({"role": "assistant", "content": "Blue"})
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_main_slot_comes_from_the_config() {
|
||||
let server = healthy();
|
||||
let (result, _) = run_with(&server, |c| c.slots.main = 1);
|
||||
assert!(result.is_ok(), "{result:?}");
|
||||
assert!(
|
||||
server
|
||||
.requests_to(CHAT)
|
||||
.iter()
|
||||
.all(|c| c.json()["id_slot"] == 1)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn each_expected_value_is_checked() {
|
||||
let zero = proto::Hash32::ZERO;
|
||||
let cases: [(&str, Edit); 3] = [
|
||||
("chat template sha256", |c| {
|
||||
c.expect.template_sha256 = proto::Hash32::ZERO
|
||||
}),
|
||||
("context per slot", |c| c.expect.n_ctx = 4096),
|
||||
("slot count", |c| c.expect.slots = 3),
|
||||
];
|
||||
for (want_what, edit) in cases {
|
||||
let server = healthy();
|
||||
let (result, steps) = run_with(&server, edit);
|
||||
match result {
|
||||
Err(SelfTestError::Mismatch {
|
||||
what,
|
||||
expected,
|
||||
got,
|
||||
}) => {
|
||||
assert_eq!(what, want_what);
|
||||
assert_ne!(expected, got);
|
||||
if what == "chat template sha256" {
|
||||
assert_eq!(expected, zero.to_hex());
|
||||
}
|
||||
}
|
||||
other => panic!("{want_what}: {other:?}"),
|
||||
}
|
||||
assert_eq!(
|
||||
steps.len(),
|
||||
1,
|
||||
"a server that is not the expected one is not sent any prompt"
|
||||
);
|
||||
assert!(server.requests_to(CHAT).is_empty());
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_completion_without_the_tool_call_fails_the_second_check() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(CHAT, vec![Reply::fixture("plain")]);
|
||||
let (result, steps) = run_with(&server, |_| {});
|
||||
assert!(
|
||||
matches!(result, Err(SelfTestError::ToolCall(_))),
|
||||
"{result:?}"
|
||||
);
|
||||
assert_eq!(steps.len(), 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_cold_second_turn_fails_the_third_check() {
|
||||
// turn1 twice: the "second turn" reuses 15 tokens where 46 were left.
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(
|
||||
CHAT,
|
||||
vec![
|
||||
Reply::fixture("tool_call"),
|
||||
Reply::fixture("progress"),
|
||||
Reply::fixture("turn1"),
|
||||
],
|
||||
);
|
||||
let (result, steps) = run_with(&server, |_| {});
|
||||
match result {
|
||||
Err(SelfTestError::CacheMiss { expected, got }) => assert_eq!((expected, got), (7052, 15)),
|
||||
other => panic!("{other:?}"),
|
||||
}
|
||||
assert_eq!(steps.len(), 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_server_that_is_down_is_an_inference_error_after_retries() {
|
||||
let server = FakeServer::start();
|
||||
server.route("/props", vec![Reply::fixture("props")]);
|
||||
server.route(CHAT, vec![Reply::fixture("tool_call").cut_after(500)]);
|
||||
let (result, _) = run_with(&server, |c| c.limits.retry_attempts = 2);
|
||||
assert!(matches!(result, Err(SelfTestError::Infer(_))), "{result:?}");
|
||||
assert_eq!(
|
||||
server.requests_to(CHAT).len(),
|
||||
2,
|
||||
"the self-test retries like everything else"
|
||||
);
|
||||
let e: Box<dyn std::error::Error> = Box::new(result.unwrap_err());
|
||||
assert!(!e.to_string().is_empty());
|
||||
}
|
||||
@@ -0,0 +1,165 @@
|
||||
//! Tests for the SSE reader. Do not edit.
|
||||
|
||||
mod support;
|
||||
|
||||
use loopd::http::{Connection, Request};
|
||||
use loopd::sse::{Events, SseError, SseItem};
|
||||
use std::io::{Cursor, Read};
|
||||
use std::time::Duration;
|
||||
use support::{FakeServer, Reply};
|
||||
|
||||
fn data(s: &str) -> Option<SseItem> {
|
||||
Some(SseItem::Data(s.to_string()))
|
||||
}
|
||||
|
||||
/// Hands out one byte per call.
|
||||
struct OneByte(Cursor<Vec<u8>>);
|
||||
|
||||
impl Read for OneByte {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
let end = buf.len().min(1);
|
||||
self.0.read(&mut buf[..end])
|
||||
}
|
||||
}
|
||||
|
||||
/// Serves `bytes`, then fails every read with `kind`.
|
||||
struct ThenError(Cursor<Vec<u8>>, std::io::ErrorKind);
|
||||
|
||||
impl Read for ThenError {
|
||||
fn read(&mut self, buf: &mut [u8]) -> std::io::Result<usize> {
|
||||
match self.0.read(buf)? {
|
||||
0 => Err(std::io::Error::from(self.1)),
|
||||
n => Ok(n),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const STREAM: &str =
|
||||
"data: {\"a\":1}\n\n: a comment\ndata:{\"b\":2}\r\n\r\nevent: x\nid: 7\n\ndata: [DONE]\n\n";
|
||||
|
||||
#[test]
|
||||
fn yields_data_lines_and_done() {
|
||||
let mut ev = Events::new(Cursor::new(STREAM), 1024);
|
||||
assert_eq!(ev.next_item().unwrap(), data("{\"a\":1}"));
|
||||
assert_eq!(
|
||||
ev.next_item().unwrap(),
|
||||
data("{\"b\":2}"),
|
||||
"no space after the colon, and CRLF"
|
||||
);
|
||||
assert_eq!(ev.next_item().unwrap(), Some(SseItem::Done));
|
||||
assert_eq!(ev.next_item().unwrap(), None);
|
||||
assert_eq!(
|
||||
ev.next_item().unwrap(),
|
||||
None,
|
||||
"asking again after the end is harmless"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn the_result_does_not_depend_on_how_the_bytes_arrive() {
|
||||
let mut ev = Events::new(OneByte(Cursor::new(STREAM.as_bytes().to_vec())), 1024);
|
||||
assert_eq!(ev.next_item().unwrap(), data("{\"a\":1}"));
|
||||
assert_eq!(ev.next_item().unwrap(), data("{\"b\":2}"));
|
||||
assert_eq!(ev.next_item().unwrap(), Some(SseItem::Done));
|
||||
assert_eq!(ev.next_item().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn only_a_leading_space_is_stripped_and_multibyte_text_survives() {
|
||||
let mut ev = Events::new(
|
||||
OneByte(Cursor::new(
|
||||
"data: two spaces\ndata: caf\u{e9} \u{1f4e6}\n".into(),
|
||||
)),
|
||||
1024,
|
||||
);
|
||||
assert_eq!(ev.next_item().unwrap(), data(" two spaces"));
|
||||
assert_eq!(ev.next_item().unwrap(), data("caf\u{e9} \u{1f4e6}"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn a_stream_that_ends_mid_line_is_truncated() {
|
||||
let mut ev = Events::new(Cursor::new("data: {\"a\":1}\n\ndata: {\"b\""), 1024);
|
||||
assert_eq!(ev.next_item().unwrap(), data("{\"a\":1}"));
|
||||
assert!(matches!(ev.next_item(), Err(SseError::Truncated)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reader_errors_are_classified() {
|
||||
use std::io::ErrorKind::{ConnectionReset, TimedOut, UnexpectedEof, WouldBlock};
|
||||
let first = "data: 1\n";
|
||||
for (kind, want) in [
|
||||
(WouldBlock, "timeout"),
|
||||
(TimedOut, "timeout"),
|
||||
(UnexpectedEof, "truncated"),
|
||||
(ConnectionReset, "io"),
|
||||
] {
|
||||
let mut ev = Events::new(ThenError(Cursor::new(first.into()), kind), 1024);
|
||||
assert_eq!(ev.next_item().unwrap(), data("1"));
|
||||
let got = match ev.next_item() {
|
||||
Err(SseError::Timeout) => "timeout",
|
||||
Err(SseError::Truncated) => "truncated",
|
||||
Err(SseError::Io(_)) => "io",
|
||||
other => panic!("{kind:?}: {other:?}"),
|
||||
};
|
||||
assert_eq!(got, want, "{kind:?}");
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn long_lines_and_bad_text_are_errors() {
|
||||
let long = format!("data: {}\n", "x".repeat(5000));
|
||||
let mut ev = Events::new(Cursor::new(long.clone()), 100);
|
||||
assert!(matches!(ev.next_item(), Err(SseError::LineTooLong)));
|
||||
let mut ev = Events::new(OneByte(Cursor::new("x".repeat(5000).into())), 100);
|
||||
assert!(
|
||||
matches!(ev.next_item(), Err(SseError::LineTooLong)),
|
||||
"with no newline at all"
|
||||
);
|
||||
let mut ev = Events::new(Cursor::new(long), 10_000);
|
||||
assert!(
|
||||
matches!(ev.next_item(), Ok(Some(SseItem::Data(_)))),
|
||||
"under the limit it is fine"
|
||||
);
|
||||
|
||||
let mut ev = Events::new(Cursor::new(b"data: \xff\xfe\n".to_vec()), 1024);
|
||||
assert!(matches!(ev.next_item(), Err(SseError::NotUtf8)));
|
||||
let e: Box<dyn std::error::Error> = Box::new(SseError::Truncated);
|
||||
assert!(!e.to_string().is_empty());
|
||||
}
|
||||
|
||||
/// The recorded streams, read through the HTTP client as `loopd` will read them.
|
||||
#[test]
|
||||
fn recorded_streams_have_the_expected_shape() {
|
||||
for (name, want_data) in [("plain", 21), ("tool_call", 27), ("turn1", 6), ("turn2", 7)] {
|
||||
for piece in [usize::MAX, 1, 13] {
|
||||
let server = FakeServer::start();
|
||||
server.route(
|
||||
"/v1/chat/completions",
|
||||
vec![Reply::fixture(name).trickle(piece, 0)],
|
||||
);
|
||||
let mut conn = Connection::open(&server.socket).unwrap();
|
||||
conn.set_read_timeout(Duration::from_secs(5)).unwrap();
|
||||
conn.send(&Request {
|
||||
method: "POST",
|
||||
path: "/v1/chat/completions",
|
||||
body: Some(b"{}"),
|
||||
})
|
||||
.unwrap();
|
||||
let head = conn.read_head().unwrap();
|
||||
let mut ev = Events::new(conn.body(&head).unwrap(), 1024 * 1024);
|
||||
let mut items = Vec::new();
|
||||
while let Some(item) = ev.next_item().unwrap() {
|
||||
items.push(item);
|
||||
}
|
||||
assert_eq!(items.last(), Some(&SseItem::Done), "{name}");
|
||||
assert_eq!(items.len() - 1, want_data, "{name}: data items");
|
||||
for item in &items[..items.len() - 1] {
|
||||
let SseItem::Data(text) = item else {
|
||||
panic!("{name}: Done in the middle")
|
||||
};
|
||||
let json: serde_json::Value = serde_json::from_str(text).unwrap();
|
||||
assert!(json["choices"].is_array(), "{name}");
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,290 @@
|
||||
//! A scripted stand-in for `llama-server`, for tests. Do not edit.
|
||||
//!
|
||||
//! It listens on a Unix socket in a temporary directory. Each path has a list of replies that are
|
||||
//! served in order; the last one repeats. A reply is raw bytes, normally a response recorded from
|
||||
//! the real server (`tests/fixtures/http/*.http`), and can be delayed, sent in small pieces, cut
|
||||
//! short, or left hanging. Every request is recorded.
|
||||
|
||||
#![allow(dead_code)] // each test file uses a different part of this module
|
||||
|
||||
use std::collections::VecDeque;
|
||||
use std::io::{Read, Write};
|
||||
use std::os::unix::net::{UnixListener, UnixStream};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::atomic::{AtomicU32, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::Duration;
|
||||
|
||||
static NEXT: AtomicU32 = AtomicU32::new(0);
|
||||
|
||||
pub fn fixture_path(kind: &str, name: &str) -> PathBuf {
|
||||
Path::new(env!("CARGO_MANIFEST_DIR"))
|
||||
.join("tests/fixtures")
|
||||
.join(kind)
|
||||
.join(name)
|
||||
}
|
||||
|
||||
pub fn fixture_bytes(kind: &str, name: &str) -> Vec<u8> {
|
||||
let path = fixture_path(kind, name);
|
||||
std::fs::read(&path).unwrap_or_else(|e| panic!("{}: {e}", path.display()))
|
||||
}
|
||||
|
||||
/// A config that points at `socket`, with every limit short enough for a test.
|
||||
pub fn test_config(socket: &Path) -> loopd::config::Config {
|
||||
let text = format!(
|
||||
r#"
|
||||
[infer]
|
||||
socket = "{}"
|
||||
model = "test-model"
|
||||
[slots]
|
||||
main = 0
|
||||
background = 1
|
||||
[expect]
|
||||
template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
|
||||
n_ctx = 131072
|
||||
slots = 2
|
||||
[limits]
|
||||
poll_ms = 40
|
||||
busy_wait_ms = 400
|
||||
load_wait_ms = 300
|
||||
idle_grace_ms = 200
|
||||
liveness_ms = 150
|
||||
retry_backoff_ms = [10, 20, 30]
|
||||
retry_window_ms = 5000
|
||||
"#,
|
||||
socket.display()
|
||||
);
|
||||
loopd::config::Config::parse(&text).unwrap()
|
||||
}
|
||||
|
||||
/// What an independent script derived from a recorded stream: `tests/fixtures/expected/`.
|
||||
pub fn expected(name: &str) -> serde_json::Value {
|
||||
serde_json::from_slice(&fixture_bytes("expected", &format!("{name}.json"))).unwrap()
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct Reply {
|
||||
bytes: Vec<u8>,
|
||||
head_delay_ms: u64,
|
||||
piece: usize,
|
||||
piece_delay_ms: u64,
|
||||
stop_after: Option<usize>,
|
||||
hang_ms: u64,
|
||||
}
|
||||
|
||||
impl Reply {
|
||||
/// Exactly these bytes, then close.
|
||||
pub fn raw(bytes: impl Into<Vec<u8>>) -> Reply {
|
||||
Reply {
|
||||
bytes: bytes.into(),
|
||||
head_delay_ms: 0,
|
||||
piece: usize::MAX,
|
||||
piece_delay_ms: 0,
|
||||
stop_after: None,
|
||||
hang_ms: 0,
|
||||
}
|
||||
}
|
||||
|
||||
/// A response recorded from the real server: `tests/fixtures/http/<name>.http`.
|
||||
pub fn fixture(name: &str) -> Reply {
|
||||
Reply::raw(fixture_bytes("http", &format!("{name}.http")))
|
||||
}
|
||||
|
||||
/// A small JSON response with a content length.
|
||||
pub fn json(status: u16, body: &str) -> Reply {
|
||||
Reply::raw(format!(
|
||||
"HTTP/1.1 {status} X\r\nContent-Type: application/json\r\nContent-Length: {}\r\n\r\n{body}",
|
||||
body.len()
|
||||
))
|
||||
}
|
||||
|
||||
/// Wait this long before sending the first byte, as a queued request does.
|
||||
pub fn head_delay(mut self, ms: u64) -> Reply {
|
||||
self.head_delay_ms = ms;
|
||||
self
|
||||
}
|
||||
|
||||
/// Send `piece` bytes at a time, waiting `delay_ms` before each piece after the first.
|
||||
pub fn trickle(mut self, piece: usize, delay_ms: u64) -> Reply {
|
||||
self.piece = piece.max(1);
|
||||
self.piece_delay_ms = delay_ms;
|
||||
self
|
||||
}
|
||||
|
||||
/// Send only the first `bytes` bytes, then close, as a server that dies does.
|
||||
pub fn cut_after(mut self, bytes: usize) -> Reply {
|
||||
self.stop_after = Some(bytes);
|
||||
self
|
||||
}
|
||||
|
||||
/// Send only the first `bytes` bytes, then stay silent for `ms` before closing.
|
||||
pub fn hang_after(mut self, bytes: usize, ms: u64) -> Reply {
|
||||
self.stop_after = Some(bytes);
|
||||
self.hang_ms = ms;
|
||||
self
|
||||
}
|
||||
|
||||
/// The offset just after the `n`th `data:` line of the body, for use with `cut_after`.
|
||||
pub fn offset_after_events(&self, n: usize) -> usize {
|
||||
let mut seen = 0;
|
||||
let mut at = 0;
|
||||
while let Some(found) = find(&self.bytes[at..], b"\n\n") {
|
||||
at += found + 2;
|
||||
seen += 1;
|
||||
if seen == n {
|
||||
return at;
|
||||
}
|
||||
}
|
||||
panic!("the reply has only {seen} events");
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct Recorded {
|
||||
pub method: String,
|
||||
/// Path and query as sent.
|
||||
pub target: String,
|
||||
/// Header lines as sent, without the request line.
|
||||
pub headers: Vec<String>,
|
||||
pub body: Vec<u8>,
|
||||
}
|
||||
|
||||
impl Recorded {
|
||||
pub fn path(&self) -> &str {
|
||||
self.target.split('?').next().unwrap_or("")
|
||||
}
|
||||
|
||||
pub fn json(&self) -> serde_json::Value {
|
||||
serde_json::from_slice(&self.body).expect("request body is JSON")
|
||||
}
|
||||
}
|
||||
|
||||
struct State {
|
||||
routes: Mutex<Vec<(String, VecDeque<Reply>)>>,
|
||||
requests: Mutex<Vec<Recorded>>,
|
||||
}
|
||||
|
||||
pub struct FakeServer {
|
||||
pub socket: PathBuf,
|
||||
state: Arc<State>,
|
||||
}
|
||||
|
||||
impl FakeServer {
|
||||
pub fn start() -> FakeServer {
|
||||
let n = NEXT.fetch_add(1, Ordering::SeqCst);
|
||||
let dir = std::env::temp_dir().join(format!("loopd-fake-{}-{n}", std::process::id()));
|
||||
std::fs::create_dir_all(&dir).unwrap();
|
||||
let socket = dir.join("infer.sock");
|
||||
let _ = std::fs::remove_file(&socket);
|
||||
let listener = UnixListener::bind(&socket).unwrap();
|
||||
let state = Arc::new(State {
|
||||
routes: Mutex::new(Vec::new()),
|
||||
requests: Mutex::new(Vec::new()),
|
||||
});
|
||||
let accept_state = Arc::clone(&state);
|
||||
thread::spawn(move || {
|
||||
for stream in listener.incoming() {
|
||||
let Ok(stream) = stream else { break };
|
||||
let state = Arc::clone(&accept_state);
|
||||
thread::spawn(move || serve(stream, &state));
|
||||
}
|
||||
});
|
||||
FakeServer { socket, state }
|
||||
}
|
||||
|
||||
/// Replies for `path` (the query is ignored), served in order. The last one repeats.
|
||||
pub fn route(&self, path: &str, replies: Vec<Reply>) {
|
||||
assert!(!replies.is_empty());
|
||||
let mut routes = self.state.routes.lock().unwrap();
|
||||
routes.retain(|(p, _)| p != path);
|
||||
routes.push((path.to_string(), replies.into()));
|
||||
}
|
||||
|
||||
pub fn requests(&self) -> Vec<Recorded> {
|
||||
self.state.requests.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
pub fn requests_to(&self, path: &str) -> Vec<Recorded> {
|
||||
self.requests()
|
||||
.into_iter()
|
||||
.filter(|r| r.path() == path)
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
fn find(haystack: &[u8], needle: &[u8]) -> Option<usize> {
|
||||
haystack.windows(needle.len()).position(|w| w == needle)
|
||||
}
|
||||
|
||||
fn read_request(stream: &mut UnixStream) -> Option<Recorded> {
|
||||
let mut buf = Vec::new();
|
||||
let mut chunk = [0u8; 4096];
|
||||
let head_end = loop {
|
||||
if let Some(end) = find(&buf, b"\r\n\r\n") {
|
||||
break end;
|
||||
}
|
||||
match stream.read(&mut chunk) {
|
||||
Ok(0) | Err(_) => return None,
|
||||
Ok(n) => buf.extend_from_slice(&chunk[..n]),
|
||||
}
|
||||
};
|
||||
let head = String::from_utf8_lossy(&buf[..head_end]).to_string();
|
||||
let mut lines = head.split("\r\n");
|
||||
let mut request_line = lines.next()?.split(' ');
|
||||
let method = request_line.next()?.to_string();
|
||||
let target = request_line.next()?.to_string();
|
||||
let headers: Vec<String> = lines.map(str::to_string).collect();
|
||||
let length = headers
|
||||
.iter()
|
||||
.filter_map(|h| h.split_once(':'))
|
||||
.find(|(name, _)| name.eq_ignore_ascii_case("content-length"))
|
||||
.and_then(|(_, v)| v.trim().parse::<usize>().ok())
|
||||
.unwrap_or(0);
|
||||
let mut body = buf[head_end + 4..].to_vec();
|
||||
while body.len() < length {
|
||||
match stream.read(&mut chunk) {
|
||||
Ok(0) | Err(_) => return None,
|
||||
Ok(n) => body.extend_from_slice(&chunk[..n]),
|
||||
}
|
||||
}
|
||||
Some(Recorded {
|
||||
method,
|
||||
target,
|
||||
headers,
|
||||
body,
|
||||
})
|
||||
}
|
||||
|
||||
fn serve(mut stream: UnixStream, state: &State) {
|
||||
let Some(request) = read_request(&mut stream) else {
|
||||
return;
|
||||
};
|
||||
let path = request.path().to_string();
|
||||
state.requests.lock().unwrap().push(request);
|
||||
let reply = {
|
||||
let mut routes = state.routes.lock().unwrap();
|
||||
match routes.iter_mut().find(|(p, _)| *p == path) {
|
||||
Some((_, replies)) if replies.len() > 1 => replies.pop_front(),
|
||||
Some((_, replies)) => replies.front().cloned(),
|
||||
None => None,
|
||||
}
|
||||
};
|
||||
let reply =
|
||||
reply.unwrap_or_else(|| Reply::json(404, r#"{"error":"no route in the fake server"}"#));
|
||||
thread::sleep(Duration::from_millis(reply.head_delay_ms));
|
||||
let end = reply
|
||||
.stop_after
|
||||
.unwrap_or(reply.bytes.len())
|
||||
.min(reply.bytes.len());
|
||||
for (i, piece) in reply.bytes[..end].chunks(reply.piece).enumerate() {
|
||||
if i > 0 {
|
||||
thread::sleep(Duration::from_millis(reply.piece_delay_ms));
|
||||
}
|
||||
if stream.write_all(piece).is_err() {
|
||||
return; // the client went away, which some tests do on purpose
|
||||
}
|
||||
let _ = stream.flush();
|
||||
}
|
||||
thread::sleep(Duration::from_millis(reply.hang_ms));
|
||||
}
|
||||
Reference in New Issue
Block a user