From 1e94e6638e2e12ce2895750c6ba6350eb0283e49 Mon Sep 17 00:00:00 2001
From: "K. Isom" <kyle@imap.cc>
Date: Wed, 16 Sep 2026 07:18:28 -0700
Subject: [PATCH] straylight: local Whisper ASR and Qwen3.8-Flash-Next
 uncensored

whisper.cpp large-v3-turbo on :11435 (CPU, OpenAI transcriptions path)
so ASR does not take GTT from llama-server. Flash-Next is the cygnal
IQ4_XS-NGQ4 GGUF (~98 GB, gfx1151), qwen4exp, mmproj pinned; unload
Laguna before loading.
---
 configs/opencode.nix      |  1 +
 hw/straylight/default.nix | 75 ++++++++++++++++++++++++++++++++++++++-
 2 files changed, 75 insertions(+), 1 deletion(-)

diff --git a/configs/opencode.nix b/configs/opencode.nix
index 346008e..64ffadb 100644
--- a/configs/opencode.nix
+++ b/configs/opencode.nix
@@ -35,6 +35,7 @@ let
           "ornith-1.5-9b-uncensored" = vision "Ornith 1.5 9B";
           "ornith-1.5-35b-a3b" = text "Ornith 1.5 35B MoE";
           "qwen3.8-27b-uncensored" = text "Qwen3.8 27B";
+          "qwen3.8-flash-next-uncensored" = vision "Qwen3.8 Flash Next";
           "qwen3-vl-8b-abliterated" = vision "Qwen3-VL 8B";
         };
       };
diff --git a/hw/straylight/default.nix b/hw/straylight/default.nix
index 8b2c4ea..8330a3b 100644
--- a/hw/straylight/default.nix
+++ b/hw/straylight/default.nix
@@ -76,6 +76,18 @@ let
     [qwen3.8-27b-uncensored]
     hf-repo = huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF:Q4_K_L
     dedup-cache-models = true
+
+    ; Qwen3.8-Flash-Next (~177B / 6B active, qwen4exp), orcarouter-abliterated,
+    ; cygnal IQ4_XS-NGQ4 GGUF (~98 GB) measured on gfx1151. Needs ~99 GB GTT
+    ; plus KV: unload Laguna first. 32k ctx / 1 slot so it fits beside the
+    ; 104 GiB TTM cap. mmproj is explicit; hf-repo alone misses it in router mode.
+    [qwen3.8-flash-next-uncensored]
+    hf-repo = cygnal/Qwen3.8-Flash-Next-Uncensored-IQ4XS-NGQ4-GGUF
+    hf-file = Qwen3.8-Flash-Next-Uncensored-IQ4XS-NGQ4.gguf
+    mmproj-url = https://huggingface.co/cygnal/Qwen3.8-Flash-Next-Uncensored-IQ4XS-NGQ4-GGUF/resolve/main/mmproj-Qwen3.8-Flash-Next-Uncensored-BF16.gguf
+    parallel = 1
+    ctx-size = 32768
+    dedup-cache-models = true
   '';
   # `llama-models` shows what the router has resident; `llama-unload` frees every loaded
   # model (e.g. before a gaming session), or just the ones named on the command line.
@@ -112,6 +124,25 @@ let
   '';
   llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} list \"$@\"";
   llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} unload \"$@\"";
+  # CPU whisper.cpp: keep GTT for llama-server. large-v3-turbo is ~1.5 GB.
+  whisperCpp = pkgsUnstable.whisper-cpp.override { withFFmpegSupport = true; };
+  whisperModel = "/var/lib/whisper-server/models/ggml-large-v3-turbo.bin";
+  whisperFetch = pkgs.writeShellScript "whisper-fetch-model" ''
+    set -eu
+    D=/var/lib/whisper-server/models
+    mkdir -p "$D"
+    F=${whisperModel}
+    if [ ! -s "$F" ]; then
+      echo "fetching whisper large-v3-turbo"
+      ${pkgs.curl}/bin/curl -L --retry 20 --retry-delay 10 --retry-all-errors \
+        -o "$F.part" \
+        https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-large-v3-turbo.bin
+      mv "$F.part" "$F"
+    fi
+  '';
+  whisperTranscribe = pkgs.writeShellScriptBin "whisper-transcribe" ''
+    exec ${whisperCpp}/bin/whisper-cli -m ${whisperModel} --language auto -f "$@"
+  '';
 in
 {
   imports = [
@@ -168,6 +199,43 @@ in
         TimeoutStartSec = "600";
       };
     };
+
+    # Local ASR: OpenAI-compatible /v1/audio/transcriptions on 11435.
+    # CPU-only so it does not contend with llama-server for GTT.
+    users.users.whisper-server = {
+      isSystemUser = true;
+      group = "whisper-server";
+      home = "/var/lib/whisper-server";
+    };
+    users.groups.whisper-server = {};
+    systemd.services.whisper-server = {
+      description = "whisper.cpp transcription server";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      serviceConfig = {
+        ExecStartPre = whisperFetch;
+        ExecStart = ''
+          ${whisperCpp}/bin/whisper-server \
+            --model ${whisperModel} \
+            --host 0.0.0.0 \
+            --port 11435 \
+            --inference-path /v1/audio/transcriptions \
+            --language auto \
+            --convert \
+            --threads 8 \
+            --no-gpu
+        '';
+        User = "whisper-server";
+        Group = "whisper-server";
+        StateDirectory = "whisper-server";
+        WorkingDirectory = "/var/lib/whisper-server";
+        Restart = "on-failure";
+        RestartSec = "10s";
+        TimeoutStartSec = "600";
+      };
+    };
+
     # straylight is the unikernel host. The shared mcp.nix locks the agent
     # down with PrivateDevices=true, which hides /dev/kvm and /dev/net/tun.
     # Relax that here (only on straylight) so the agent can boot Nanos
@@ -222,6 +290,8 @@ in
       inputs.herdr.packages.x86_64-linux.default
       llamaModelsCmd
       llamaUnloadCmd
+      whisperTranscribe
+      whisperCpp
     ];
 
     services.open-webui = {
@@ -236,6 +306,9 @@ in
         OPENAI_API_BASE_URLS = "http://127.0.0.1:11434/v1";
         OPENAI_API_KEYS = "none";
         ENABLE_OLLAMA_API = "False";
+        AUDIO_STT_ENGINE = "openai";
+        AUDIO_STT_OPENAI_API_BASE_URL = "http://127.0.0.1:11435/v1";
+        AUDIO_STT_OPENAI_API_KEY = "none";
       };
     };
 
@@ -243,7 +316,7 @@ in
     networking.firewall.allowedTCPPorts = [ 53 443 8080 8443 9443 9444 9555 ];
     networking.firewall.allowedUDPPorts = [ 53 ];
     # llama.cpp OpenAI-compatible API: tailnet only (localhost is always allowed).
-    networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11434 ];
+    networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11434 11435 ];
 
     # DNS: MCNS for internal zones, public resolvers as fallback.
     networking.nameservers = [