diff --git a/hw/dixie/default.nix b/hw/dixie/default.nix index 38e1409..0b6649c 100644 --- a/hw/dixie/default.nix +++ b/hw/dixie/default.nix @@ -49,6 +49,11 @@ let ; at 96k incl. 5.5 GB weights), so 128k costs ~0.65 GB of the ~2.5 GB free. ctx-size = 131072 parallel = 8 + ; One scheduler step = up to batch-size prompt tokens, and every decoding slot gets one + ; token per step. At the 2048 default a step was ~2.4 s while Honcho's 10-28k prompts + ; prefilled, so 16-token guardian calls hit p90 10 s. The GPU already computes in + ; 512-token ubatches, so 512 keeps prefill throughput and makes steps ~0.6 s. + batch-size = 512 ; Runaway guard carried over from straylight: helper tasks need short answers. reasoning-budget = 2048 n-predict = 4096