From 91b2e6b33e1f87fd62d0808f8bea8100e7c5234e Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Tue, 22 Sep 2026 11:24:35 -0700 Subject: [PATCH] dixie: honcho-embed back to 2048/slot, with explicit batch-size 8192/slot (previous commit) cost 6.6 GB of compute buffers and the 9B then failed to load; 4096/slot did not fit beside the 9B and dialectic either. 2048/slot with batch-size set explicitly; long inputs are handled on rift instead (EMBEDDING_MAX_INPUT_TOKENS=1536 makes Honcho chunk). All three models resident at 11.5 GB. Co-Authored-By: Claude Opus 5.5 (1M context) --- hw/dixie/default.nix | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/hw/dixie/default.nix b/hw/dixie/default.nix index 2fbdb1e..7a21a63 100644 --- a/hw/dixie/default.nix +++ b/hw/dixie/default.nix @@ -51,13 +51,14 @@ let model = ${modelsDir}/qwen3-embedding-0.6b/Qwen3-Embedding-0.6B-Q8_0.gguf embedding = true pooling = cls - ; A whole input must fit in one physical batch AND one slot. batch-size defaults to 2048 - ; and silently caps ubatch, and ctx is split across slots, so both were 2048 in practice: - ; Honcho's longer messages (up to its 8192-token EMBEDDING.MAX_INPUT_TOKENS) failed with - ; "input is too large to process". Size all three to 8192 per slot. - batch-size = 8192 - ubatch-size = 8192 - ctx-size = 16384 + ; A whole input must fit in one physical batch AND one slot (batch-size silently caps + ; ubatch; ctx is split across slots). Bigger batches cost compute buffers fast on a + ; shared 12 GB card: 8192/slot took 6.6 GB and 4096/slot did not fit beside the 9B and + ; dialectic. So 2048/slot here, and rift's EMBEDDING_MAX_INPUT_TOKENS=1536 makes Honcho + ; chunk below that (its token estimate is not Qwen's tokenizer, hence the margin). + batch-size = 2048 + ubatch-size = 2048 + ctx-size = 4096 parallel = 2 '';