hw/straylight/default.nix

@@ -25,6 +25,12 @@ let
     parallel = 2
     ; Unload a model's weights and KV cache after six idle hours; the next request reloads it.
     sleep-idle-seconds = 21600
+    ; Host-RAM prompt cache. This limit is PER RESIDENT MODEL, not global: each model's
+    ; child llama-server gets its own. The llama.cpp default is 8192 MiB, so with
+    ; --models-max 3 the worst case was 3 x 8 = 24 GiB. Helper, VL and comparison models
+    ; rarely reuse long prefixes, so they get 2 GiB; the model that does gets more in its
+    ; own section. Worst case is now 16 + 2 + 2 = 20 GiB.
+    cache-ram = 2048
 
     ; Qwen3.6-35B-A3B, refusal-ablated (HauhauCS "Aggressive"). Benchmark candidate A.
     [qwen3.6-35b-a3b-abliterated]
@@ -85,6 +91,23 @@ let
     top-k = 20
     reasoning-budget = 8192
     dedup-cache-models = true
+    ; Prompt-cache thrash (measured 2026-09-20, 30 h of logs): opencode, Hermes and its
+    ; delegation subagents shared 2 slots and an 8 GiB host cache whose entries are
+    ; 1-2 GiB each. 235 evictions; 138 prompt re-evaluations over 20k tokens costing
+    ; ~196 minutes; one 102,922-token prompt re-processed from scratch (~3.4 min).
+    ;
+    ; Four slots over ONE unified 262144-token KV pool: same GPU memory as before, but
+    ; four conversations keep their prefix live instead of two, and any one of them may
+    ; still use the whole pool when the others are idle. Cost: when all four decode at
+    ; once, each gets a smaller share of ~30 t/s.
+    parallel = 4
+    kv-unified = true
+    ; 16 GiB holds about ten evicted conversations instead of five.
+    ; IF LAGUNA IS RE-ENABLED this must drop to 4096 (Laguna 8192, helpers 0) and
+    ; --models-max to 2: Laguna ~73 GiB + this model ~26 + a 9B helper ~9 + desktop and
+    ; services ~20 + the honcho CPU lanes ~9 is ~137 GiB on a 125 GiB machine before
+    ; any cache at all. That is the 2026-09-19 OOM.
+    cache-ram = 16384
 
     ; Qwen 3.8 27B dense, refusal-ablated. Preferred uncensored coding agent.
     ; orcarouter/Qwen3.8-27B-Uncensored-GGUF is gated; huihui Q4_K_L keeps