diff --git a/flake.lock b/flake.lock index ce70351..ec6e3f7 100644 --- a/flake.lock +++ b/flake.lock @@ -334,11 +334,11 @@ }, "nixpkgs-unstable": { "locked": { - "lastModified": 1783224372, - "narHash": "sha256-FGQ8TfSm9WviNE/fQAWsKs+y4GkX8fU4EScewpw54Y8=", - "rev": "d407951447dcd00442e97087bf374aad70c04cea", + "lastModified": 1789286504, + "narHash": "sha256-3i80JnZ9pZ7tWReb/Sqpc32cfzffCu8TWLnbnH0hN9c=", + "rev": "ef34387ddd751e1ab8857adf4676492d32eb24ec", "type": "tarball", - "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1027867.d407951447dc/nixexprs.tar.xz" + "url": "https://releases.nixos.org/nixos/unstable/nixos-26.11pre1073009.ef34387ddd75/nixexprs.tar.zst" }, "original": { "id": "nixpkgs", diff --git a/hw/straylight/default.nix b/hw/straylight/default.nix index 90b351b..0c2f614 100644 --- a/hw/straylight/default.nix +++ b/hw/straylight/default.nix @@ -1,19 +1,79 @@ { inputs, pkgs, lib, ... }: let pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.hostPlatform.system}; - # Pin to b9828; nixpkgs-unstable lags too far behind for the features we need. - # To update: nix-prefetch-url --unpack https://github.com/ggml-org/llama.cpp/archive/refs/tags/bNNNN.tar.gz - # then: nix run nixpkgs#prefetch-npm-deps -- /tools/ui/package-lock.json - llama-cpp-b9828 = (pkgsUnstable.llama-cpp.override { vulkanSupport = true; }).overrideAttrs (_: { - version = "9828"; - src = pkgsUnstable.fetchFromGitHub { - owner = "ggml-org"; - repo = "llama.cpp"; - tag = "b9828"; - hash = "sha256-CRj+smOs5T80fT5OqHZGbko7/PwChhYsc8MGZZkE/dQ="; - }; - npmDepsHash = "sha256-X1DZgmhS/zHTqDT5zq0kywwntthcJ9vRXeqyO3zz6UU="; - }); + # llama.cpp from nixpkgs-unstable (0.4.x as of Sept 2026) with the Vulkan backend for + # gfx1151 (Strix Halo). Laguna S 2.1 and Gemma 4 need a build newer than mid-July 2026, + # so keep the unstable input reasonably fresh (`nix flake update nixpkgs-unstable`). + llama-cpp = pkgsUnstable.llama-cpp.override { vulkanSupport = true; }; + llamaModelsDir = "/var/lib/llama-server/models"; + # llama-server runs in router mode: one endpoint, models load on demand, at most two + # resident at a time (see --models-max below). Section names are the model ids that + # clients pass in the "model" field. Weights are fetched by + # /var/lib/llama-server/models/download.sh, not by the service. + llamaModelsIni = pkgs.writeText "llama-models.ini" '' + version = 1 + + [*] + jinja = true + flash-attn = on + cache-type-k = q8_0 + cache-type-v = q8_0 + n-gpu-layers = 999 + no-mmap = true + ctx-size = 131072 + parallel = 2 + ; Unload a model's weights and KV cache after six idle hours; the next request reloads it. + sleep-idle-seconds = 21600 + + ; Qwen3.6-35B-A3B, refusal-ablated (HauhauCS "Aggressive"). Benchmark candidate A. + [qwen3.6-35b-a3b-abliterated] + model = ${llamaModelsDir}/qwen3.6-35b-a3b-abliterated/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf + + ; Gemma 4 26B-A4B QAT, refusal-ablated (HauhauCS "Balanced"), with its MTP draft head. + ; Benchmark candidate B (the Western model of the same MoE class). + [gemma4-26b-a4b-abliterated] + model = ${llamaModelsDir}/gemma4-26b-a4b-abliterated/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf + model-draft = ${llamaModelsDir}/gemma4-26b-a4b-abliterated/mtp-gemma-4-26B-A4B-it.gguf + spec-type = draft-mtp + + ; Poolside Laguna S 2.1 (118B total, 8B active): primary coding agent. + ; ~69 GB of weights, so it only loads once the raised TTM/GTT limit is active (reboot). + [laguna-s-2.1] + model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf + parallel = 1 + + ; Previous default model (a Qwen 3.5 derivative), kept for comparison runs. + [ornith-1.0-35b] + model = /var/lib/llama-server/huggingface/hub/models--deepreinforce-ai--Ornith-1.0-35B-GGUF/snapshots/c2e1703039380de4ce6820e97afd185682d3c16c/ornith-1.0-35b-Q4_K_M.gguf + ''; + # `llama-models` shows what the router has resident; `llama-unload` frees every loaded + # model (e.g. before a gaming session). Both talk to the router on localhost. + llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" '' + set -euo pipefail + URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434} + ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c ' + import json, sys + d = json.load(sys.stdin) + for m in d.get("data", d if isinstance(d, list) else []): + st = (m.get("status") or {}).get("value", "?") + print(f"{st:11} {m.get(\"id\") or m.get(\"model\") or m.get(\"name\")}") + ' + ''; + llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" '' + set -euo pipefail + URL=''${LLAMA_SERVER_URL:-http://127.0.0.1:11434} + ${pkgs.curl}/bin/curl -fsS "$URL/models" | ${pkgs.python3}/bin/python3 -c ' + import json, sys + d = json.load(sys.stdin) + for m in d.get("data", d if isinstance(d, list) else []): + if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping"): + print(m.get("id") or m.get("model") or m.get("name")) + ' | while read -r id; do + printf "unloading %s: " "$id" + ${pkgs.curl}/bin/curl -fsS -X POST "$URL/models/unload" -H "Content-Type: application/json" -d "{\"model\":\"$id\"}" + echo + done + ''; in { imports = [ @@ -30,8 +90,8 @@ in # amdgpu.gttsize is deprecated on recent kernels; ttm.* is the supported knob. boot.kernelParams = [ "ttm.pages_limit=27262976" "ttm.page_pool_size=27262976" ]; - # llama.cpp server with Vulkan backend for gfx1151 (Strix Halo). - # Model is downloaded from HuggingFace on first start and cached in StateDirectory. + # llama.cpp server (router mode) with the Vulkan backend for gfx1151 (Strix Halo). + # Models and per-model settings live in llamaModelsIni above. # NOTE: BIOS "UMA Frame Buffer Size" must be ≥32GB for a 23GB model to fit on-GPU. users.users.llama-server = { isSystemUser = true; @@ -51,17 +111,11 @@ in }; serviceConfig = { ExecStart = '' - ${llama-cpp-b9828}/bin/llama-server \ - --jinja \ - -hf deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M \ + ${llama-cpp}/bin/llama-server \ --host 127.0.0.1 \ --port 11434 \ - -c 262144 \ - --no-mmap \ - -fa on \ - --cache-type-k q8_0 \ - --cache-type-v q8_0 \ - -ngl all + --models-preset ${llamaModelsIni} \ + --models-max 2 ''; User = "llama-server"; Group = "llama-server"; @@ -125,6 +179,8 @@ in # Pinned to the upstream flake (v0.9.0); nixpkgs-unstable only has 0.7.1. environment.systemPackages = [ inputs.herdr.packages.x86_64-linux.default + llamaModelsCmd + llamaUnloadCmd ]; services.open-webui = {