loopd: the self-test checks the model's tags against the tokenizer

Co-Authored-By: ornith-1.5-35b-a3b <agent@wntrmute.dev>

crates/loopd/tests/support/mod.rs

@@ -3,7 +3,8 @@
 //! It listens on a Unix socket in a temporary directory. Each path has a list of replies that are
 //! served in order; the last one repeats. A reply is raw bytes, normally a response recorded from
 //! the real server (`tests/fixtures/http/*.http`), and can be delayed, sent in small pieces, cut
-//! short, or left hanging. Every request is recorded.
+//! short, or left hanging. A path can instead compute each reply from its request, as the fake
+//! tokenizer does. Every request is recorded.
 
 #![allow(dead_code)] // each test file uses a different part of this module
 
@@ -171,6 +172,26 @@ impl loopd::tools::ToolPort for ReturnsPendingPort {
     }
 }
 
+/// The fake `/tokenize`, for `FakeServer::respond`: the text of the request's `content` is one
+/// token when it is one of the model's tags (`bxcore::sanitize::TAGS`), the tag's id, and
+/// otherwise one token per byte, the byte's value. A body without a string `content` gets a 400.
+pub fn tokenizer(request: &Recorded) -> Reply {
+    let body: serde_json::Value = serde_json::from_slice(&request.body).unwrap_or_default();
+    let Some(content) = body.get("content").and_then(|c| c.as_str()) else {
+        return Reply::json(400, r#"{"error":"no content"}"#);
+    };
+    let ids: Vec<u32> = match bxcore::sanitize::TAGS.iter().find(|t| t.text == content) {
+        Some(tag) => vec![tag.id],
+        None => content.bytes().map(u32::from).collect(),
+    };
+    tokens(&ids)
+}
+
+/// A `/tokenize` answer as llama-server gives one without `with_pieces`: `{"tokens": [ids]}`.
+pub fn tokens(ids: &[u32]) -> Reply {
+    Reply::json(200, &serde_json::json!({ "tokens": ids }).to_string())
+}
+
 /// What an independent script derived from a recorded stream: `tests/fixtures/expected/`.
 pub fn expected(name: &str) -> serde_json::Value {
     serde_json::from_slice(&fixture_bytes("expected", &format!("{name}.json"))).unwrap()
@@ -273,8 +294,12 @@ impl Recorded {
     }
 }
 
+/// A reply computed from its request.
+type Responder = Arc<dyn Fn(&Recorded) -> Reply + Send + Sync>;
+
 struct State {
     routes: Mutex<Vec<(String, VecDeque<Reply>)>>,
+    responders: Mutex<Vec<(String, Responder)>>,
     requests: Mutex<Vec<Recorded>>,
 }
 
@@ -295,6 +320,7 @@ impl FakeServer {
         let listener = UnixListener::bind(&socket).unwrap();
         let state = Arc::new(State {
             routes: Mutex::new(Vec::new()),
+            responders: Mutex::new(Vec::new()),
             requests: Mutex::new(Vec::new()),
         });
         let accept_state = Arc::clone(&state);
@@ -308,14 +334,29 @@ impl FakeServer {
         FakeServer { socket, state, dir }
     }
 
-    /// Replies for `path` (the query is ignored), served in order. The last one repeats.
+    /// Replies for `path` (the query is ignored), served in order. The last one repeats. They
+    /// replace a responder for `path`.
     pub fn route(&self, path: &str, replies: Vec<Reply>) {
         assert!(!replies.is_empty());
+        self.state
+            .responders
+            .lock()
+            .unwrap()
+            .retain(|(p, _)| p != path);
         let mut routes = self.state.routes.lock().unwrap();
         routes.retain(|(p, _)| p != path);
         routes.push((path.to_string(), replies.into()));
     }
 
+    /// For each request to `path` (the query is ignored), the reply `reply` computes from it. It
+    /// replaces the replies `route` gave `path`.
+    pub fn respond(&self, path: &str, reply: impl Fn(&Recorded) -> Reply + Send + Sync + 'static) {
+        self.state.routes.lock().unwrap().retain(|(p, _)| p != path);
+        let mut responders = self.state.responders.lock().unwrap();
+        responders.retain(|(p, _)| p != path);
+        responders.push((path.to_string(), Arc::new(reply)));
+    }
+
     pub fn requests(&self) -> Vec<Recorded> {
         self.state.requests.lock().unwrap().clone()
     }
@@ -376,15 +417,23 @@ fn serve(mut stream: UnixStream, state: &State) {
         return;
     };
     let path = request.path().to_string();
+    let responder = state
+        .responders
+        .lock()
+        .unwrap()
+        .iter()
+        .find(|(p, _)| *p == path)
+        .map(|(_, reply)| Arc::clone(reply));
+    let computed = responder.map(|reply| reply(&request));
     state.requests.lock().unwrap().push(request);
-    let reply = {
+    let reply = computed.or_else(|| {
         let mut routes = state.routes.lock().unwrap();
         match routes.iter_mut().find(|(p, _)| *p == path) {
             Some((_, replies)) if replies.len() > 1 => replies.pop_front(),
             Some((_, replies)) => replies.front().cloned(),
             None => None,
         }
-    };
+    });
     let reply =
         reply.unwrap_or_else(|| Reply::json(404, r#"{"error":"no route in the fake server"}"#));
     thread::sleep(Duration::from_millis(reply.head_delay_ms));