dixie: unified KV per model; Qwen thinking sampling for the 9B
kv-unified lets one request use a model's whole 16k (split slots capped each at 8k and every Honcho dialectic prompt of 11-17k tokens failed). 9B gets temp 0.6 / top-p 0.95 / top-k 20 / min-p 0: in a 20-batch replay of Honcho's deriver the 0.8 default dropped 7 batches (empty or wrong-shape output); with these, 3, and 1 with a format instruction on rift. VRAM unchanged at 11.5 GB. Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
@@ -31,6 +31,9 @@ let
|
|||||||
; three resident models share 12 GB: keep contexts modest
|
; three resident models share 12 GB: keep contexts modest
|
||||||
ctx-size = 16384
|
ctx-size = 16384
|
||||||
parallel = 2
|
parallel = 2
|
||||||
|
; one KV pool per model shared by its slots, so a single long request can use all 16k
|
||||||
|
; (split slots capped each at 8k: Honcho dialectic prompts of 11-17k all failed)
|
||||||
|
kv-unified = true
|
||||||
cache-ram = 512
|
cache-ram = 512
|
||||||
|
|
||||||
; Ornith 1.5 9B, refusal-ablated. Hermes auxiliary model (approval guardian, title,
|
; Ornith 1.5 9B, refusal-ablated. Hermes auxiliary model (approval guardian, title,
|
||||||
@@ -40,6 +43,12 @@ let
|
|||||||
; Runaway guard carried over from straylight: helper tasks need short answers.
|
; Runaway guard carried over from straylight: helper tasks need short answers.
|
||||||
reasoning-budget = 2048
|
reasoning-budget = 2048
|
||||||
n-predict = 4096
|
n-predict = 4096
|
||||||
|
; Qwen thinking-mode sampling (llama-server's 0.8 default made the Honcho deriver drop
|
||||||
|
; ~1 in 3 batches, measured 2026-09-22). Clients that send their own temperature win.
|
||||||
|
temp = 0.6
|
||||||
|
top-p = 0.95
|
||||||
|
top-k = 20
|
||||||
|
min-p = 0
|
||||||
|
|
||||||
; Honcho dialectic: non-thinking instruct model so answers are short and fast.
|
; Honcho dialectic: non-thinking instruct model so answers are short and fast.
|
||||||
[honcho-dialectic]
|
[honcho-dialectic]
|
||||||
|
|||||||
Reference in New Issue
Block a user