kyle/imladris
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

models update

Browse Source
This commit is contained in:
kyle
2026-09-15 21:03:01 -07:00
parent 1714600aab
commit 35f8d81ab6
1 changed files with 40 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hw/straylight/default.nix
+40 -29
View File
@@ -20,7 +20,8 @@ let
cache-type-v = q8_0
n-gpu-layers = 999
no-mmap = true
ctx-size = 131072
; llama.cpp splits ctx-size across parallel slots: 262144 / 2 = 131072 per slot.
ctx-size = 262144
parallel = 2
; Unload a model's weights and KV cache after six idle hours; the next request reloads it.
sleep-idle-seconds = 21600
@@ -41,39 +42,49 @@ let
[laguna-s-2.1]
model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf
parallel = 1
ctx-size = 131072
; Previous default model (a Qwen 3.5 derivative), kept for comparison runs.
; Previous default model (a Qwen 3.5 derivative), kept for comparison runs. It was
; fetched by the old -hf flag into the HF cache; dedup hides the cache's own entry.
[ornith-1.0-35b]
model = /var/lib/llama-server/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c/ornith-1.0-35b-Q4_K_M.gguf
hf-repo = deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M
dedup-cache-models = true
'';
# `llama-models` shows what the router has resident; `llama-unload` frees every loaded
# model (e.g. before a gaming session). Both talk to the router on localhost.
llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" ''
set -euo pipefail
URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434}
${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c '
import json, sys
d = json.load(sys.stdin)
for m in d.get("data", d if isinstance(d, list) else []):
st = (m.get("status") or {}).get("value", "?")
print(f"{st:11} {m.get(\"id\") or m.get(\"model\") or m.get(\"name\")}")
'
'';
llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" ''
set -euo pipefail
URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434}
${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c '
import json, sys
d = json.load(sys.stdin)
for m in d.get("data", d if isinstance(d, list) else []):
if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping"):
print(m.get("id") or m.get("model") or m.get("name"))
' | while read -r id; do
printf "unloading %s: " "$id"
${pkgs.curl}/bin/curl -fsS -X POST "$URL/models/unload" -H "Content-Type: application/json" -d "{\"model\":\"$id\"}"
echo
done
# model (e.g. before a gaming session), or just the ones named on the command line.
llamaCtl = pkgs.writeText "llama-ctl.py" ''
import json, sys, urllib.request
URL = "http://127.0.0.1:11434"
def models():
with urllib.request.urlopen(URL + "/models", timeout=10) as r:
d = json.load(r)
return d.get("data", d if isinstance(d, list) else [])
def name(m):
return m.get("id") or m.get("model") or m.get("name")
cmd = sys.argv[1] if len(sys.argv) > 1 else "list"
if cmd == "list":
for m in models():
print("%-10s %s" % ((m.get("status") or {}).get("value", "?"), name(m)))
elif cmd == "unload":
loaded = [name(m) for m in models()
if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping")]
targets = sys.argv[2:] or loaded
if not targets:
print("nothing loaded")
for t in targets:
req = urllib.request.Request(URL + "/models/unload", data=json.dumps({"model": t}).encode(),
headers={"Content-Type": "application/json"}, method="POST")
with urllib.request.urlopen(req, timeout=30) as r:
print("unload %s: %s" % (t, r.read().decode().strip()))
else:
sys.exit("usage: llama-ctl.py [list|unload [model ...]]")
'';
llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} list \"$@\"";
llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} unload \"$@\"";
in
{
imports = [