diff --git a/hw/straylight/default.nix b/hw/straylight/default.nix index 39739fd..42ee247 100644 --- a/hw/straylight/default.nix +++ b/hw/straylight/default.nix @@ -52,6 +52,29 @@ let [trinity-mini] model = ${llamaModelsDir}/trinity-mini/Trinity-Mini-Q4_K_M.gguf + ; Gemma 4 31B (dense, 31B params, 256k ctx, multimodal, Apache 2.0). unsloth + ; Q4_K_M (~18.3 GB) + the bundled MTP draft head (spec-type draft-mtp) for + ; speculative decode speedup. mmproj is explicit (router-mode --hf-repo does not + ; attach it); reports image input once loaded. ~20 GB on disk. + [gemma4-31b-it] + model = ${llamaModelsDir}/gemma4-31b-it/gemma-4-31B-it-Q4_K_M.gguf + model-draft = ${llamaModelsDir}/gemma4-31b-it/mtp-gemma-4-31B-it.gguf + spec-type = draft-mtp + mmproj-url = https://huggingface.co/unsloth/gemma-4-31B-it-GGUF/resolve/main/mmproj-BF16.gguf + + ; Muse Glimmer 30B (Meta dense ~29.6B + 1.8B vision encoder + DFlash drafter, + ; agentic, multimodal, Apache 2.0). Needs llama.cpp >= b10353 (muse-glimmer arch, + ; merged PR #26841) — the running 0.4.1 build satisfies this. Q4_K_M (~15.6 GB) + + ; mmproj + the DFlash drafter (spec-type draft-dflash). 131k ctx / 1 slot so the + ; weights + KV + drafter fit the 104 GiB TTM window; mmproj explicit. ~18 GB on disk. + [muse-glimmer-30b] + model = ${llamaModelsDir}/muse-glimmer-30b/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf + model-draft = ${llamaModelsDir}/muse-glimmer-30b/dflash-Muse-Glimmer-30B-Q4_K_M.gguf + spec-type = draft-dflash + mmproj-url = https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/resolve/main/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf + parallel = 1 + dedup-cache-models = true + ; Poolside Laguna S 2.1 (118B total, 8B active): primary coding agent. ; ~69 GB of weights, so it only loads once the raised TTM/GTT limit is active (reboot). ; DFlash GGUF is on disk (laguna-s-2.1-DFlash-BF16.gguf). nixpkgs llama.cpp 0.4.0