Add on-device verification for the inference path

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)

Makefile

@@ -17,6 +17,12 @@ gate:
 audit:
 	cargo deny check advisories
 
-# Checks that need straylight. Filled in from M2.
+# Checks that need straylight. They go through a private inferproxy to the real server.
+# Override the server with: make verify-device UPSTREAM=host:port
+UPSTREAM ?= straylight:11434
+
 verify-device:
-	@echo "verify-device: nothing to check until M2"
+	cargo build --workspace --locked
+	BOXMAKER_INFERPROXY=$(CURDIR)/target/debug/inferproxy BOXMAKER_UPSTREAM=$(UPSTREAM) \
+		cargo test -p loopd --test device --locked -- --ignored --test-threads=1
+	@echo "verify-device: ok"

crates/loopd/tests/device.rs

@@ -0,0 +1,233 @@
+//! Checks against the real server. Ignored unless run by `make verify-device`. Do not edit.
+//!
+//! They need two environment variables:
+//!   BOXMAKER_INFERPROXY  path to the built `inferproxy` binary
+//!   BOXMAKER_UPSTREAM    host:port of the llama-server router, for example straylight:11434
+//! and, optionally, BOXMAKER_MODEL (default ornith-1.5-35b-a3b). They use slot 0.
+//!
+//! They never touch the real `llama-server` process. The "server dies" test kills and restarts
+//! its own `inferproxy`.
+
+use loopd::config::Config;
+use loopd::llama::info::{CacheOutcome, cache_outcome};
+use loopd::llama::{ChatEvent, ChatMessage, ChatRequest, Client};
+use std::path::{Path, PathBuf};
+use std::process::{Child, Command};
+use std::sync::mpsc;
+use std::thread;
+use std::time::Duration;
+
+struct Proxy {
+    child: Child,
+    socket: PathBuf,
+}
+
+impl Proxy {
+    fn start(socket: &Path) -> Proxy {
+        let binary = std::env::var("BOXMAKER_INFERPROXY").expect("BOXMAKER_INFERPROXY is not set");
+        let upstream = std::env::var("BOXMAKER_UPSTREAM").expect("BOXMAKER_UPSTREAM is not set");
+        let child = Command::new(binary)
+            .arg("--listen")
+            .arg(socket)
+            .arg("--upstream")
+            .arg(upstream)
+            .spawn()
+            .expect("cannot start inferproxy");
+        let mut proxy = Proxy {
+            child,
+            socket: socket.to_path_buf(),
+        };
+        for _ in 0..100 {
+            if socket.exists() {
+                return proxy;
+            }
+            thread::sleep(Duration::from_millis(20));
+        }
+        proxy.kill();
+        panic!("inferproxy did not create {}", socket.display());
+    }
+
+    fn kill(&mut self) {
+        let _ = self.child.kill();
+        let _ = self.child.wait();
+    }
+}
+
+impl Drop for Proxy {
+    fn drop(&mut self) {
+        self.kill();
+    }
+}
+
+fn socket_path(name: &str) -> PathBuf {
+    let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id()));
+    std::fs::create_dir_all(&dir).unwrap();
+    dir.join("infer.sock")
+}
+
+fn config(socket: &Path) -> Config {
+    let model =
+        std::env::var("BOXMAKER_MODEL").unwrap_or_else(|_| "ornith-1.5-35b-a3b".to_string());
+    let text = format!(
+        r#"
+[infer]
+socket = "{}"
+model = "{model}"
+[slots]
+main = 0
+background = 1
+[expect]
+template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b"
+n_ctx = 131072
+slots = 2
+"#,
+        socket.display()
+    );
+    Config::parse(&text).unwrap()
+}
+
+fn user(text: &str) -> ChatMessage {
+    // A different prompt each run, so that an earlier run's cache cannot make a check pass.
+    let nonce = std::time::SystemTime::now()
+        .duration_since(std::time::UNIX_EPOCH)
+        .unwrap()
+        .as_nanos();
+    ChatMessage::User {
+        content: format!("{text} (run {nonce})"),
+    }
+}
+
+#[test]
+#[ignore = "needs the real server; run with make verify-device"]
+fn the_startup_self_test_passes() {
+    let socket = socket_path("selftest");
+    let _proxy = Proxy::start(&socket);
+    let client = Client::new(config(&socket));
+    loopd::selftest::run(&client, &mut |step| eprintln!("selftest: {step}")).unwrap();
+}
+
+#[test]
+#[ignore = "needs the real server; run with make verify-device"]
+fn a_capped_thinking_block_ends_and_the_answer_arrives() {
+    let socket = socket_path("cap");
+    let _proxy = Proxy::start(&socket);
+    let mut cfg = config(&socket);
+    cfg.limits.thinking_cap = 60;
+    let client = Client::new(cfg);
+    let ask = "Think at great length about how many ways 8 rooks fit on a chessboard without attacking \
+               each other and with none on the main diagonal. Then answer in one sentence.";
+    let req = ChatRequest {
+        slot: 0,
+        messages: vec![user(ask)],
+        tools: vec![],
+        thinking: true,
+    };
+    let mut capped = Vec::new();
+    let done = client
+        .chat_with_retry(&req, &mut |e| {
+            if let ChatEvent::ThinkingCapped { tokens } = e {
+                capped.push(*tokens);
+            }
+        })
+        .unwrap();
+    assert!(done.thinking_capped);
+    assert_eq!(capped.len(), 1);
+    assert!(capped[0] >= 60 && capped[0] < 70, "capped at {capped:?}");
+    assert!(
+        done.reasoning_tokens < 60 + 256,
+        "thinking went on to {}",
+        done.reasoning_tokens
+    );
+    assert!(
+        done.content.is_some_and(|c| !c.trim().is_empty()),
+        "an answer followed the forced end"
+    );
+}
+
+#[test]
+#[ignore = "needs the real server; run with make verify-device"]
+fn a_request_survives_its_proxy_being_killed_and_restarted() {
+    let socket = socket_path("restart");
+    let mut proxy = Proxy::start(&socket);
+    let mut cfg = config(&socket);
+    cfg.limits.retry_backoff_ms = vec![1_500];
+    let client = Client::new(cfg);
+    let ask = "Write about 300 words on the history of cork.";
+    let req = ChatRequest {
+        slot: 0,
+        messages: vec![user(ask)],
+        tools: vec![],
+        thinking: false,
+    };
+
+    let (tx, rx) = mpsc::channel();
+    let worker = thread::spawn(move || {
+        let mut retries = 0;
+        let mut told = false;
+        let result = client.chat_with_retry(&req, &mut |e| match e {
+            ChatEvent::Content(_) if !told => {
+                told = true;
+                let _ = tx.send(());
+            }
+            ChatEvent::Retrying { .. } => retries += 1,
+            _ => {}
+        });
+        (result, retries)
+    });
+
+    rx.recv_timeout(Duration::from_secs(120))
+        .expect("no content arrived");
+    proxy.kill(); // the stream dies in the middle of the answer
+    thread::sleep(Duration::from_millis(300));
+    let _proxy = Proxy::start(&proxy.socket);
+
+    let (result, retries) = worker.join().unwrap();
+    let done = result.expect("the retry should have succeeded");
+    assert!(retries >= 1, "the request was retried");
+    assert!(
+        done.content
+            .is_some_and(|c| c.split_whitespace().count() > 100)
+    );
+}
+
+#[test]
+#[ignore = "needs the real server; run with make verify-device"]
+fn a_second_turn_reuses_the_first_turns_cache() {
+    let socket = socket_path("cache");
+    let _proxy = Proxy::start(&socket);
+    let client = Client::new(config(&socket));
+    let mut messages = vec![user(
+        "What is 17 * 23? Think briefly, then answer in one short sentence.",
+    )];
+    let req = ChatRequest {
+        slot: 0,
+        messages: messages.clone(),
+        tools: vec![],
+        thinking: true,
+    };
+    let turn1 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
+    assert!(
+        turn1.reasoning_content.is_some(),
+        "this check is about replaying a thinking block"
+    );
+    messages.push(ChatMessage::Assistant {
+        content: turn1.content.clone(),
+        reasoning_content: turn1.reasoning_content.clone(),
+        tool_calls: turn1.tool_calls.clone(),
+    });
+    messages.push(user("And 17 * 24?"));
+    let req = ChatRequest {
+        slot: 0,
+        messages,
+        tools: vec![],
+        thinking: true,
+    };
+    let turn2 = client.chat_with_retry(&req, &mut |_| {}).unwrap();
+    assert_eq!(
+        cache_outcome(&turn1.timings, &turn2.timings),
+        CacheOutcome::Hit,
+        "{:?} then {:?}",
+        turn1.timings,
+        turn2.timings
+    );
+}

docs/implementer-log.md

@@ -26,7 +26,8 @@ reviewer adds findings under "Reviews" once per milestone.
 | M2a/11-llama-gate-retry | 2026-09-18 | done | 3 | fail | none | Prerequisite `chat` (M2a/09) now exists, so the task was possible. Implemented `SlotGate` in gate.rs: per-slot holder + a `VecDeque` of arrival tickets, `notify_all`, a woken waiter takes the slot only if free and its ticket is at the front (and claims it by setting holder), `Drop` frees and wakes; mutex/condvar poison recovered via `unwrap_or_else(...into_inner)`, no `unwrap`. Implemented `chat_with_retry` + `is_retryable` (all nine variants, a new one is a compile error) + `backoff_ms` in retry.rs: base is `schedule[retry-1]` or last or 0, jitter clamped and computed in `i128` so `u64::MAX` never overflows, jitter from sub-second nanos. `chat` acquires the gate for `req.slot` and maps `GateFull`->`InferError::Busy`; `Client` gained a `gate` field. First gate failed on three clippy lints (derivable `Default`, `or_insert_with`->`or_default`), fixed. One logic bug caught by `waiters_are_served_in_order`: `take_if_front` claimed the ticket but not `holder`, letting two permits overlap — set holder on claim. All 79 loopd tests pass; retry 13/13 over ten runs; `make gate` prints `gate: ok`. | ? |
 | M2a/09-llama-chat | 2026-09-18 | done | 1 | pass | none | Wrote crates/loopd/src/llama/chat.rs (`chat`, with the head wait in a separate `wait_for_head`) and registered `pub mod chat;`. `chat` builds the body (build error -> Protocol), opens and POSTs, then `wait_for_head` loops `read_head` at `poll_ms`: a Timeout is classified Idle/Busy/Unavailable by `received_any` then a `slots()` poll, emits `Waiting { slot_busy }` on every poll, keeps a per-state `since` that resets on state change, and returns `WaitTimeout`/`LoadTimeout`/`Stalled` at the right limits; 200 streams via `Events`+`Assembler` mapping `Timeout->Stalled`, `Truncated->StreamClosedEarly`, else `Protocol`, then `finish(false)`; non-200 returns `Http { status, error_text }`. All 13 chat tests pass five runs in a row. Table-to-test map: state (table 1) Busy -> a_busy_slot_is_waited_out / a_slot_that_stays_busy_is_a_wait_timeout, Idle-not-processing -> only_the_requests_own_slot_counts_as_busy, Unavailable -> an_unavailable_server_is_a_load_timeout, Idle-received_any -> a_slow_but_steady_stream_is_not_a_stall (turn1 head is 738 B, pieces are 3140/8=392 B, so the head-wait times out with a partial head); thresholds (table 2) -> a_slot_that_stays_busy_is_a_wait_timeout / an_unavailable_server_is_a_load_timeout / only_the_requests_own_slot_counts_as_busy; stream (table 3) Data/Done/None -> the recorded completion and trickle tests, Timeout -> silence_in_mid_stream_is_a_stall, Truncated -> a_stream_that_is_cut_is_closed_early_wherever_the_cut_falls, other -> garbage_in_the_stream_is_a_protocol_error; clock-restart -> the_wait_clocks_restart_when_the_state_changes. One path fix: info and request live under crate::llama, not crate::, so the imports use crate::llama::{info::..., request::...}. | ? |
 | M2a/10-llama-cap | 2026-09-18 | done | 1 | pass | none | Added `Client::end_reasoning` to info.rs: POSTs `{"id","action":"reasoning_end","model"}` to `/v1/chat/completions/control` via `call`, reads `success` as a bool from the server's JSON (ignoring `message`), non-200 stays an Err through `call`, a missing/non-bool `success` is Protocol. Threaded the cap into `chat` step 4: after passing a chunk's events on, when `assembler.in_reasoning()`, a local `cap_at: Option<u64>` holds where the cap fired (None while it has not fired); on `tokens >= thinking_cap` it calls `end_reasoning(assembler.id())` once, remembers `tokens` and emits `ThinkingCapped` on `Ok(true)`, returns `ThinkingOverrun` on `Ok(false)`/`Err`, and after firing returns `ThinkingOverrun` once `tokens >= at + thinking_overrun`; `finish(cap_at.is_some())`. The `the_allowance_is_exact` test passes with `>=` in both rows (63 is not `20+44`, and is `>= 20+43`). One guard: a reasoning chunk with no id at cap time is Protocol rather than a panic. 6 cap tests + 13 chat tests pass; `make gate` prints `gate: ok`. | ? |
-| M2a/12-selftest | 2026-09-18 | done | 1 | pass | none | Wrote crates/loopd/src/selftest.rs (SelfTestError with Display/std::error::Error/From<InferError>, and `run` doing the three checks in order) and registered `pub mod selftest;` in lib.rs; rewrote main.rs into `loopd selftest --config <path>`. Check 1 calls props() and compares chat-template sha256, then n_ctx, then total_slots, a sha256 error being Hash so a wrong server is sent no prompt; check 2 posts one read_file tool with chat_with_retry and requires finish_reason ToolCalls, a first call named read_file whose arguments parse as JSON carrying a string path, wrapping an InferError as Infer; check 3 runs turn 1 then an extension of it and maps a cache Loss to CacheMiss. The copied test's cache_outcome/CacheOutcome live at crate::llama::info, so the import follows that. main.rs parses args as &[&str] via a two-step String->&str collect; unknown/missing args are exit 2 and a config load failure is `loopd: <error>` exit 1. Real server via inferproxy against straylight: minimal.toml gave three step lines and `selftest: ok` exit 0; setting slots=3 gave `selftest: FAILED: slot count: expected 3, got 2` exit 1. | ? |
+| M2a/12-selftest | 2026-09-18 | done | 1 | pass | none |
+| M2a/13-verify-device | 2026-09-18 | done | 1 | pass | none | Wrote no library code. Put the two given files in place (`crates/loopd/tests/device.rs`, and a `Makefile` whose only difference from the old one is the new `verify-device` target), confirmed `make gate` prints `gate: ok` with `device` at `0 passed; 0 failed; 4 ignored`, and `curl http://straylight:11434/health` returned `{"status":"ok"}`. `make verify-device` ran the four `#[ignore]` checks one at a time against the real server on slot 0: self-test, capped thinking block, a request surviving its own proxy being killed and restarted, and a second turn reusing the cache of a first turn that contained thinking. All 4 passed in ~21s (well under two minutes). | ? | Wrote crates/loopd/src/selftest.rs (SelfTestError with Display/std::error::Error/From<InferError>, and `run` doing the three checks in order) and registered `pub mod selftest;` in lib.rs; rewrote main.rs into `loopd selftest --config <path>`. Check 1 calls props() and compares chat-template sha256, then n_ctx, then total_slots, a sha256 error being Hash so a wrong server is sent no prompt; check 2 posts one read_file tool with chat_with_retry and requires finish_reason ToolCalls, a first call named read_file whose arguments parse as JSON carrying a string path, wrapping an InferError as Infer; check 3 runs turn 1 then an extension of it and maps a cache Loss to CacheMiss. The copied test's cache_outcome/CacheOutcome live at crate::llama::info, so the import follows that. main.rs parses args as &[&str] via a two-step String->&str collect; unknown/missing args are exit 2 and a config load failure is `loopd: <error>` exit 1. Real server via inferproxy against straylight: minimal.toml gave three step lines and `selftest: ok` exit 0; setting slots=3 gave `selftest: FAILED: slot count: expected 3, got 2` exit 1. | ? |
 
 
 ## Reviews