diff --git a/hw/dixie/default.nix b/hw/dixie/default.nix index 2fbdb1e..7a21a63 100644 --- a/hw/dixie/default.nix +++ b/hw/dixie/default.nix @@ -51,13 +51,14 @@ let model = ${modelsDir}/qwen3-embedding-0.6b/Qwen3-Embedding-0.6B-Q8_0.gguf embedding = true pooling = cls - ; A whole input must fit in one physical batch AND one slot. batch-size defaults to 2048 - ; and silently caps ubatch, and ctx is split across slots, so both were 2048 in practice: - ; Honcho's longer messages (up to its 8192-token EMBEDDING.MAX_INPUT_TOKENS) failed with - ; "input is too large to process". Size all three to 8192 per slot. - batch-size = 8192 - ubatch-size = 8192 - ctx-size = 16384 + ; A whole input must fit in one physical batch AND one slot (batch-size silently caps + ; ubatch; ctx is split across slots). Bigger batches cost compute buffers fast on a + ; shared 12 GB card: 8192/slot took 6.6 GB and 4096/slot did not fit beside the 9B and + ; dialectic. So 2048/slot here, and rift's EMBEDDING_MAX_INPUT_TOKENS=1536 makes Honcho + ; chunk below that (its token estimate is not Qwen's tokenizer, hence the margin). + batch-size = 2048 + ubatch-size = 2048 + ctx-size = 4096 parallel = 2 '';