diff --git a/hw/straylight/default.nix b/hw/straylight/default.nix index 0c2f614..a41a291 100644 --- a/hw/straylight/default.nix +++ b/hw/straylight/default.nix @@ -20,7 +20,8 @@ let cache-type-v = q8_0 n-gpu-layers = 999 no-mmap = true - ctx-size = 131072 + ; llama.cpp splits ctx-size across parallel slots: 262144 / 2 = 131072 per slot. + ctx-size = 262144 parallel = 2 ; Unload a model's weights and KV cache after six idle hours; the next request reloads it. sleep-idle-seconds = 21600 @@ -41,39 +42,49 @@ let [laguna-s-2.1] model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf parallel = 1 + ctx-size = 131072 - ; Previous default model (a Qwen 3.5 derivative), kept for comparison runs. + ; Previous default model (a Qwen 3.5 derivative), kept for comparison runs. It was + ; fetched by the old -hf flag into the HF cache; dedup hides the cache's own entry. [ornith-1.0-35b] - model = /var/lib/llama-server/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c/ornith-1.0-35b-Q4_K_M.gguf + hf-repo = deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M + dedup-cache-models = true ''; # `llama-models` shows what the router has resident; `llama-unload` frees every loaded - # model (e.g. before a gaming session). Both talk to the router on localhost. - llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" '' - set -euo pipefail - URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434} - ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c ' - import json, sys - d = json.load(sys.stdin) - for m in d.get("data", d if isinstance(d, list) else []): - st = (m.get("status") or {}).get("value", "?") - print(f"{st:11} {m.get(\"id\") or m.get(\"model\") or m.get(\"name\")}") - ' - ''; - llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" '' - set -euo pipefail - URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434} - ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c ' - import json, sys - d = json.load(sys.stdin) - for m in d.get("data", d if isinstance(d, list) else []): - if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping"): - print(m.get("id") or m.get("model") or m.get("name")) - ' | while read -r id; do - printf "unloading %s: " "$id" - ${pkgs.curl}/bin/curl -fsS -X POST "$URL/models/unload" -H "Content-Type: application/json" -d "{\"model\":\"$id\"}" - echo - done + # model (e.g. before a gaming session), or just the ones named on the command line. + llamaCtl = pkgs.writeText "llama-ctl.py" '' + import json, sys, urllib.request + + URL = "http://127.0.0.1:11434" + + def models(): + with urllib.request.urlopen(URL + "/models", timeout=10) as r: + d = json.load(r) + return d.get("data", d if isinstance(d, list) else []) + + def name(m): + return m.get("id") or m.get("model") or m.get("name") + + cmd = sys.argv[1] if len(sys.argv) > 1 else "list" + if cmd == "list": + for m in models(): + print("%-10s %s" % ((m.get("status") or {}).get("value", "?"), name(m))) + elif cmd == "unload": + loaded = [name(m) for m in models() + if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping")] + targets = sys.argv[2:] or loaded + if not targets: + print("nothing loaded") + for t in targets: + req = urllib.request.Request(URL + "/models/unload", data=json.dumps({"model": t}).encode(), + headers={"Content-Type": "application/json"}, method="POST") + with urllib.request.urlopen(req, timeout=30) as r: + print("unload %s: %s" % (t, r.read().decode().strip())) + else: + sys.exit("usage: llama-ctl.py [list|unload [model ...]]") ''; + llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} list \"$@\""; + llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} unload \"$@\""; in { imports = [