kyle/imladris
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

llama-server: add Gemma4 31B and Muse-Glimmer-30B models

Browse Source 
Both fetched to /var/lib/llama-server/models by download.sh. Gemma4 31B
uses MTP draft (spec-type draft-mtp); Muse-Glimmer uses DFlash drafter
(spec-type draft-dflash, parallel=1 for the 104 GiB TTM window).
This commit is contained in:
kyle
2026-10-05 18:14:15 -07:00
parent d7c1092d52
commit b5e189bc5b
1 changed files with 23 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hw/straylight/default.nix
+23
View File
@@ -52,6 +52,29 @@ let
[trinity-mini]
model = ${llamaModelsDir}/trinity-mini/Trinity-Mini-Q4_K_M.gguf
; Gemma 4 31B (dense, 31B params, 256k ctx, multimodal, Apache 2.0). unsloth
; Q4_K_M (~18.3 GB) + the bundled MTP draft head (spec-type draft-mtp) for
; speculative decode speedup. mmproj is explicit (router-mode --hf-repo does not
; attach it); reports image input once loaded. ~20 GB on disk.
[gemma4-31b-it]
model = ${llamaModelsDir}/gemma4-31b-it/gemma-4-31B-it-Q4_K_M.gguf
model-draft = ${llamaModelsDir}/gemma4-31b-it/mtp-gemma-4-31B-it.gguf
spec-type = draft-mtp
mmproj-url = https://huggingface.co/unsloth/gemma-4-31B-it-GGUF/resolve/main/mmproj-BF16.gguf
; Muse Glimmer 30B (Meta dense ~29.6B + 1.8B vision encoder + DFlash drafter,
; agentic, multimodal, Apache 2.0). Needs llama.cpp >= b10353 (muse-glimmer arch,
; merged PR #26841) — the running 0.4.1 build satisfies this. Q4_K_M (~15.6 GB) +
; mmproj + the DFlash drafter (spec-type draft-dflash). 131k ctx / 1 slot so the
; weights + KV + drafter fit the 104 GiB TTM window; mmproj explicit. ~18 GB on disk.
[muse-glimmer-30b]
model = ${llamaModelsDir}/muse-glimmer-30b/Muse-Glimmer-30B-KQuant-17GB-Q4_K_M.gguf
model-draft = ${llamaModelsDir}/muse-glimmer-30b/dflash-Muse-Glimmer-30B-Q4_K_M.gguf
spec-type = draft-dflash
mmproj-url = https://huggingface.co/meta-models/Muse-Glimmer-30B-GGUF/resolve/main/mmproj-Muse-Glimmer-30B-Q4_K_M.gguf
parallel = 1
dedup-cache-models = true
; Poolside Laguna S 2.1 (118B total, 8B active): primary coding agent.
; ~69 GB of weights, so it only loads once the raised TTM/GTT limit is active (reboot).
; DFlash GGUF is on disk (laguna-s-2.1-DFlash-BF16.gguf). nixpkgs llama.cpp 0.4.0