models update

hw/straylight/default.nix

@@ -20,7 +20,8 @@ let
     cache-type-v = q8_0
     n-gpu-layers = 999
     no-mmap = true
-    ctx-size = 131072
+    ; llama.cpp splits ctx-size across parallel slots: 262144 / 2 = 131072 per slot.
+    ctx-size = 262144
     parallel = 2
     ; Unload a model's weights and KV cache after six idle hours; the next request reloads it.
     sleep-idle-seconds = 21600
@@ -41,39 +42,49 @@ let
     [laguna-s-2.1]
     model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf
     parallel = 1
+    ctx-size = 131072
 
-    ; Previous default model (a Qwen 3.5 derivative), kept for comparison runs.
+    ; Previous default model (a Qwen 3.5 derivative), kept for comparison runs. It was
+    ; fetched by the old -hf flag into the HF cache; dedup hides the cache's own entry.
     [ornith-1.0-35b]
-    model = /var/lib/llama-server/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c/ornith-1.0-35b-Q4_K_M.gguf
+    hf-repo = deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M
+    dedup-cache-models = true
   '';
   # `llama-models` shows what the router has resident; `llama-unload` frees every loaded
-  # model (e.g. before a gaming session). Both talk to the router on localhost.
+  # model (e.g. before a gaming session), or just the ones named on the command line.
-  llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" ''
+  llamaCtl = pkgs.writeText "llama-ctl.py" ''
-    set -euo pipefail
+    import json, sys, urllib.request
-    URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434}
+
-    ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c '
+    URL = "http://127.0.0.1:11434"
-    import json, sys
+
-    d = json.load(sys.stdin)
+    def models():
-    for m in d.get("data", d if isinstance(d, list) else []):
+        with urllib.request.urlopen(URL + "/models", timeout=10) as r:
-        st = (m.get("status") or {}).get("value", "?")
+            d = json.load(r)
-        print(f"{st:11} {m.get(\"id\") or m.get(\"model\") or m.get(\"name\")}")
+        return d.get("data", d if isinstance(d, list) else [])
-    '
+
-  '';
+    def name(m):
-  llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" ''
+        return m.get("id") or m.get("model") or m.get("name")
-    set -euo pipefail
+
-    URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434}
+    cmd = sys.argv[1] if len(sys.argv) > 1 else "list"
-    ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c '
+    if cmd == "list":
-    import json, sys
+        for m in models():
-    d = json.load(sys.stdin)
+            print("%-10s %s" % ((m.get("status") or {}).get("value", "?"), name(m)))
-    for m in d.get("data", d if isinstance(d, list) else []):
+    elif cmd == "unload":
-        if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping"):
+        loaded = [name(m) for m in models()
-            print(m.get("id") or m.get("model") or m.get("name"))
+                  if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping")]
-    ' | while read -r id; do
+        targets = sys.argv[2:] or loaded
-      printf "unloading %s: " "$id"
+        if not targets:
-      ${pkgs.curl}/bin/curl -fsS -X POST "$URL/models/unload" -H "Content-Type: application/json" -d "{\"model\":\"$id\"}"
+            print("nothing loaded")
-      echo
+        for t in targets:
-    done
+            req = urllib.request.Request(URL + "/models/unload", data=json.dumps({"model": t}).encode(),
+                                         headers={"Content-Type": "application/json"}, method="POST")
+            with urllib.request.urlopen(req, timeout=30) as r:
+                print("unload %s: %s" % (t, r.read().decode().strip()))
+    else:
+        sys.exit("usage: llama-ctl.py [list|unload [model ...]]")
   '';
+  llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} list \"$@\"";
+  llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} unload \"$@\"";
 in
 {
   imports = [