straylight: llama-server router mode on llama.cpp 0.4.0

Drop the b9828 source pin and take llama.cpp from a refreshed
nixpkgs-unstable (0.4.0), which supports the Laguna and Gemma 4
architectures. Run llama-server in router mode with a preset file:
abliterated Qwen3.6-35B-A3B and Gemma 4 26B-A4B for benchmarking,
Poolside Laguna S 2.1 as the primary coding agent, and Ornith kept for
comparison. At most two models stay resident; idle models unload after
six hours. Add llama-models and llama-unload helper commands.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

hw/straylight/default.nix

@@ -1,19 +1,79 @@
 { inputs, pkgs, lib, ... }:
 let
   pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.hostPlatform.system};
-  # Pin to b9828; nixpkgs-unstable lags too far behind for the features we need.
-  # To update: nix-prefetch-url --unpack https://github.com/ggml-org/llama.cpp/archive/refs/tags/bNNNN.tar.gz
-  # then: nix run nixpkgs#prefetch-npm-deps -- <unpacked>/tools/ui/package-lock.json
-  llama-cpp-b9828 = (pkgsUnstable.llama-cpp.override { vulkanSupport = true; }).overrideAttrs (_: {
-    version = "9828";
-    src = pkgsUnstable.fetchFromGitHub {
-      owner = "ggml-org";
-      repo = "llama.cpp";
-      tag = "b9828";
-      hash = "sha256-CRj+smOs5T80fT5OqHZGbko7/PwChhYsc8MGZZkE/dQ=";
-    };
-    npmDepsHash = "sha256-X1DZgmhS/zHTqDT5zq0kywwntthcJ9vRXeqyO3zz6UU=";
-  });
+  # llama.cpp from nixpkgs-unstable (0.4.x as of Sept 2026) with the Vulkan backend for
+  # gfx1151 (Strix Halo). Laguna S 2.1 and Gemma 4 need a build newer than mid-July 2026,
+  # so keep the unstable input reasonably fresh (`nix flake update nixpkgs-unstable`).
+  llama-cpp = pkgsUnstable.llama-cpp.override { vulkanSupport = true; };
+  llamaModelsDir = "/var/lib/llama-server/models";
+  # llama-server runs in router mode: one endpoint, models load on demand, at most two
+  # resident at a time (see --models-max below). Section names are the model ids that
+  # clients pass in the "model" field. Weights are fetched by
+  # /var/lib/llama-server/models/download.sh, not by the service.
+  llamaModelsIni = pkgs.writeText "llama-models.ini" ''
+    version = 1
+
+    [*]
+    jinja = true
+    flash-attn = on
+    cache-type-k = q8_0
+    cache-type-v = q8_0
+    n-gpu-layers = 999
+    no-mmap = true
+    ctx-size = 131072
+    parallel = 2
+    ; Unload a model's weights and KV cache after six idle hours; the next request reloads it.
+    sleep-idle-seconds = 21600
+
+    ; Qwen3.6-35B-A3B, refusal-ablated (HauhauCS "Aggressive"). Benchmark candidate A.
+    [qwen3.6-35b-a3b-abliterated]
+    model = ${llamaModelsDir}/qwen3.6-35b-a3b-abliterated/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
+
+    ; Gemma 4 26B-A4B QAT, refusal-ablated (HauhauCS "Balanced"), with its MTP draft head.
+    ; Benchmark candidate B (the Western model of the same MoE class).
+    [gemma4-26b-a4b-abliterated]
+    model = ${llamaModelsDir}/gemma4-26b-a4b-abliterated/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf
+    model-draft = ${llamaModelsDir}/gemma4-26b-a4b-abliterated/mtp-gemma-4-26B-A4B-it.gguf
+    spec-type = draft-mtp
+
+    ; Poolside Laguna S 2.1 (118B total, 8B active): primary coding agent.
+    ; ~69 GB of weights, so it only loads once the raised TTM/GTT limit is active (reboot).
+    [laguna-s-2.1]
+    model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf
+    parallel = 1
+
+    ; Previous default model (a Qwen 3.5 derivative), kept for comparison runs.
+    [ornith-1.0-35b]
+    model = /var/lib/llama-server/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c/ornith-1.0-35b-Q4_K_M.gguf
+  '';
+  # `llama-models` shows what the router has resident; `llama-unload` frees every loaded
+  # model (e.g. before a gaming session). Both talk to the router on localhost.
+  llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" ''
+    set -euo pipefail
+    URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434}
+    ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c '
+    import json, sys
+    d = json.load(sys.stdin)
+    for m in d.get("data", d if isinstance(d, list) else []):
+        st = (m.get("status") or {}).get("value", "?")
+        print(f"{st:11} {m.get(\"id\") or m.get(\"model\") or m.get(\"name\")}")
+    '
+  '';
+  llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" ''
+    set -euo pipefail
+    URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434}
+    ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c '
+    import json, sys
+    d = json.load(sys.stdin)
+    for m in d.get("data", d if isinstance(d, list) else []):
+        if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping"):
+            print(m.get("id") or m.get("model") or m.get("name"))
+    ' | while read -r id; do
+      printf "unloading %s: " "$id"
+      ${pkgs.curl}/bin/curl -fsS -X POST "$URL/models/unload" -H "Content-Type: application/json" -d "{\"model\":\"$id\"}"
+      echo
+    done
+  '';
 in
 {
   imports = [
@@ -30,8 +90,8 @@ in
     # amdgpu.gttsize is deprecated on recent kernels; ttm.* is the supported knob.
     boot.kernelParams = [ "ttm.pages_limit=27262976" "ttm.page_pool_size=27262976" ];
 
-    # llama.cpp server with Vulkan backend for gfx1151 (Strix Halo).
-    # Model is downloaded from HuggingFace on first start and cached in StateDirectory.
+    # llama.cpp server (router mode) with the Vulkan backend for gfx1151 (Strix Halo).
+    # Models and per-model settings live in llamaModelsIni above.
     # NOTE: BIOS "UMA Frame Buffer Size" must be ≥32GB for a 23GB model to fit on-GPU.
     users.users.llama-server = {
       isSystemUser = true;
@@ -51,17 +111,11 @@ in
       };
       serviceConfig = {
         ExecStart = ''
-          ${llama-cpp-b9828}/bin/llama-server \
-            --jinja \
-            -hf deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M \
+          ${llama-cpp}/bin/llama-server \
             --host 127.0.0.1 \
             --port 11434 \
-            -c 262144 \
-            --no-mmap \
-            -fa on \
-            --cache-type-k q8_0 \
-            --cache-type-v q8_0 \
-            -ngl all
+            --models-preset ${llamaModelsIni} \
+            --models-max 2
         '';
         User = "llama-server";
         Group = "llama-server";
@@ -125,6 +179,8 @@ in
     # Pinned to the upstream flake (v0.9.0); nixpkgs-unstable only has 0.7.1.
     environment.systemPackages = [
       inputs.herdr.packages.x86_64-linux.default
+      llamaModelsCmd
+      llamaUnloadCmd
     ];
 
     services.open-webui = {