loopd: the self-test checks the model's tags against the tokenizer

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/loopd/src/llama/info.rs

@@ -25,6 +25,12 @@ pub struct SlotInfo {
     pub is_processing: bool,
 }
 
+/// `/tokenize`'s answer without `with_pieces`: `{"tokens": [ids]}`. Other fields are ignored.
+#[derive(serde::Deserialize)]
+struct TokenIds {
+    tokens: Vec<u32>,
+}
+
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum CacheOutcome {
     Hit,
@@ -143,6 +149,23 @@ impl Client {
             .ok_or_else(|| InferError::Protocol("tokenize response has no tokens".to_string()))
     }
 
+    /// The token ids the server gives `text`, with special tokens parsed as a chat prompt's are
+    /// and no BOS added. The self-test checks the model's tags with it (M5b spec, section 8).
+    pub fn token_ids(&self, text: &str) -> Result<Vec<u32>, InferError> {
+        let body = serde_json::to_string(&serde_json::json!({
+            "model": self.cfg.infer.model,
+            "content": text,
+            "add_special": false,
+            "parse_special": true,
+        }))
+        .map_err(|_| InferError::Protocol("could not encode tokenize request".to_string()))?;
+        let bytes = self.call("POST", "/tokenize", Some(body.as_bytes()))?;
+        let ids: TokenIds = serde_json::from_slice(&bytes).map_err(|_| {
+            InferError::Protocol("tokenize response has no list of token ids".to_string())
+        })?;
+        Ok(ids.tokens)
+    }
+
     pub(crate) fn end_reasoning(&self, completion_id: &str) -> Result<bool, InferError> {
         let body = serde_json::to_string(&serde_json::json!({
             "id": completion_id,