diff --git a/crates/loopd/tests/device.rs b/crates/loopd/tests/device.rs index 61793c6..58d167b 100644 --- a/crates/loopd/tests/device.rs +++ b/crates/loopd/tests/device.rs @@ -59,6 +59,13 @@ impl Drop for Proxy { } } +/// The server as deployed on straylight, recorded in one place. When the deployment changes on +/// purpose, change this and `tools/check-m3a-device.sh` together, and note it in +/// `docs/inference-contract.md`. Since 2026-09-22: four slots over one 262,144-token pool. +const EXPECT: &str = r#"template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" +n_ctx = 262144 +slots = 4"#; + fn socket_path(name: &str) -> PathBuf { let dir = std::env::temp_dir().join(format!("loopd-device-{}-{name}", std::process::id())); std::fs::create_dir_all(&dir).unwrap(); @@ -77,9 +84,7 @@ model = "{model}" main = 0 background = 1 [expect] -template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" -n_ctx = 131072 -slots = 2 +{EXPECT} "#, socket.display() ); @@ -253,9 +258,7 @@ model = "{model}" main = 0 background = 1 [expect] -template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" -n_ctx = 131072 -slots = 2 +{EXPECT} [paths] home = "{}" "#, diff --git a/docs/inference-contract.md b/docs/inference-contract.md index bad95ca..6ef6871 100644 --- a/docs/inference-contract.md +++ b/docs/inference-contract.md @@ -10,6 +10,25 @@ the server README at tag `b10809`, the build that is running. not affected by that. Throughput is, so section (a) reports a separate run taken after the GPU went idle. Section (d) was also run with the GPU idle. +## Deployment change, 2026-09-20 + +Found in the M3a review on 2026-09-22, when `loopd`'s self-test refused the server +(`context per slot: expected 131072, got 262144`). The owner changed Ornith's preset in +`~/src/nixos/hw/straylight/default.nix` after measuring prompt-cache thrash between OpenCode, Hermes +and its subagents on two slots: + +- `parallel = 4` with `kv-unified = true`: four slots share one 262,144-token KV pool. Any one + session may use the whole pool while the others are idle. +- A 16 GiB host prompt cache (was 8 GiB), `--models-max 3`, and a server-side + `reasoning-budget = 8192`. + +Not re-measured yet. Two findings above rest on the old layout and need checking before M3b or M5 +leans on them: (d), that a second session on another slot leaves the first slot's cache intact, +now that the slots share one pool; and P9's picture of eviction, where another client's long +prompt can now crowd a harness session out of the pool without touching its slot. The harness's +own expectations (`crates/loopd/tests/device.rs`, `tools/check-m3a-device.sh`) record the new +layout; `make verify-device` passes against it (6 of 6, 2026-09-22). + ## What is running | Item | Value | @@ -19,7 +38,7 @@ idle. Section (d) was also run with the GPU idle. | Public listener | `0.0.0.0:11434`, firewalled to the tailnet; Tailscale Serve adds HTTPS on `:10000` | | Other clients | Open WebUI and OpenCode use the same endpoint and the same Ornith instance | | Ornith flags | `--jinja --no-mmap --ctx-size 262144 --parallel 2 --cache-type-k q8_0 --cache-type-v q8_0 --flash-attn on --n-gpu-layers 999 --sleep-idle-seconds 21600 --hf-repo ornith-ai/Ornith-1.5-35B-A3B-GGUF:Q4_K_M` | -| Slots | 2, each `n_ctx` 131072 (the 262144 is split, not shared) | +| Slots | Until 2026-09-20: 2, each `n_ctx` 131072 (the 262144 split, not shared). Since then: 4 over one unified 262144-token pool (`kv-unified`); `/props` reports `n_ctx` 262144 per slot. See "Deployment change, 2026-09-20" | | Server default sampling | temperature 1.0, top_k 20, top_p 0.95, min_p 0.05. The harness must send its own. | | Chat template | 7,828 bytes, sha256 `f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b` | | Source of truth for flags | `~/src/nixos/hw/straylight/default.nix` on straylight, not this repo | diff --git a/tools/check-m3a-device.sh b/tools/check-m3a-device.sh index e5cd69c..d5e960a 100755 --- a/tools/check-m3a-device.sh +++ b/tools/check-m3a-device.sh @@ -84,8 +84,8 @@ main = 0 background = 1 [expect] template_sha256 = "f55f52930aa8bf44ab5cb85f99370fcc3c56e9a85640b812086d5330bce5d86b" -n_ctx = 131072 -slots = 2 +n_ctx = 262144 +slots = 4 [paths] home = "$HOME_DIR" [broker]