Add the thinking cap to Client::chat
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -65,12 +65,39 @@ impl Client {
|
||||
let mut body = conn.body(&head).map_err(map_http)?;
|
||||
let mut events = Events::new(&mut body, MAX_SSE_LINE);
|
||||
let mut assembler = Assembler::new();
|
||||
// Where the cap fired, if it has: the overrun allowance is measured from here.
|
||||
let mut cap_at: Option<u64> = None;
|
||||
loop {
|
||||
match events.next_item() {
|
||||
Ok(Some(SseItem::Data(text))) => {
|
||||
for event in assembler.push(&text)? {
|
||||
on_event(&event);
|
||||
}
|
||||
if assembler.in_reasoning() {
|
||||
let tokens = assembler.reasoning_tokens();
|
||||
match cap_at {
|
||||
None if tokens >= self.cfg.limits.thinking_cap => {
|
||||
let Some(id) = assembler.id() else {
|
||||
return Err(InferError::Protocol(
|
||||
"a reasoning chunk carried no completion id".to_string(),
|
||||
));
|
||||
};
|
||||
match self.end_reasoning(id) {
|
||||
Ok(true) => {
|
||||
cap_at = Some(tokens);
|
||||
on_event(&ChatEvent::ThinkingCapped { tokens });
|
||||
}
|
||||
Ok(false) | Err(_) => {
|
||||
return Err(InferError::ThinkingOverrun);
|
||||
}
|
||||
}
|
||||
}
|
||||
Some(at) if tokens >= at + self.cfg.limits.thinking_overrun => {
|
||||
return Err(InferError::ThinkingOverrun);
|
||||
}
|
||||
_ => {}
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(Some(SseItem::Done)) | Ok(None) => break,
|
||||
Err(SseError::Timeout) => return Err(InferError::Stalled),
|
||||
@@ -81,7 +108,7 @@ impl Client {
|
||||
|
||||
// 5. Finish. A stream that ended without a finish_reason is a server that died between two
|
||||
// events.
|
||||
assembler.finish(false)
|
||||
assembler.finish(cap_at.is_some())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -142,4 +142,24 @@ impl Client {
|
||||
.map(Vec::len)
|
||||
.ok_or_else(|| InferError::Protocol("tokenize response has no tokens".to_string()))
|
||||
}
|
||||
|
||||
pub(crate) fn end_reasoning(&self, completion_id: &str) -> Result<bool, InferError> {
|
||||
let body = serde_json::to_string(&serde_json::json!({
|
||||
"id": completion_id,
|
||||
"action": "reasoning_end",
|
||||
"model": self.cfg.infer.model,
|
||||
}))
|
||||
.map_err(|_| InferError::Protocol("could not encode reasoning_end request".to_string()))?;
|
||||
let bytes = self.call(
|
||||
"POST",
|
||||
"/v1/chat/completions/control",
|
||||
Some(body.as_bytes()),
|
||||
)?;
|
||||
let value: serde_json::Value = serde_json::from_slice(&bytes)
|
||||
.map_err(|_| InferError::Protocol("control response is not JSON".to_string()))?;
|
||||
value
|
||||
.get("success")
|
||||
.and_then(|v| v.as_bool())
|
||||
.ok_or_else(|| InferError::Protocol("control response has no success".to_string()))
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user