imladris/hw/dixie/default.nix

{ inputs, pkgs, lib, config, ... }:
let
  # Unstable instantiated with unfree allowed: CUDA is unfree, and legacyPackages does not
  # inherit the host's nixpkgs.config (straylight's Vulkan build never hit this).
  pkgsUnstable = import inputs.nixpkgs-unstable {
    system = pkgs.stdenv.hostPlatform.system;
    config = { allowUnfree = true; cudaSupport = true; };
  };
  # llama.cpp with the CUDA backend, from unstable to match straylight's build (same model
  # files, same router semantics). Built once on straylight and shipped with dixie-push
  # --closure, or pulled from the cache when Dixie's link allows.
  llama-cpp = pkgsUnstable.llama-cpp.override { cudaSupport = true; };
  modelsDir = "/var/lib/llama-server/models";

  # Dixie is the fleet's helper tier: everything that is latency-sensitive, small, and was
  # starving on straylight's CPU (2026-09-21). One RTX 3060 (12 GB) today; card 2 joins
  # when its fans are replaced.
  #
  # Router: the three models that must never be swapped out — approval guardian / Honcho
  # deriver (9B), Honcho dialectic (4B, non-thinking), Honcho embeddings — resident together:
  # 5.6 + 2.5 + 0.7 GB weights, ~3 GB left for KV. Vision does NOT go here: 5 GB + 1.2 GB
  # mmproj does not fit beside them, so it gets its own on-demand process below.
  llamaModelsIni = pkgs.writeText "llama-models.ini" ''
    version = 1
    [*]
    jinja = true
    flash-attn = on
    cache-type-k = q8_0
    cache-type-v = q8_0
    n-gpu-layers = 999
    ; three resident models share 12 GB: keep contexts modest
    ctx-size = 16384
    parallel = 2
    ; one KV pool per model shared by its slots, so a single long request can use all 16k
    ; (split slots capped each at 8k: Honcho dialectic prompts of 11-17k all failed)
    kv-unified = true
    cache-ram = 512

    ; Ornith 1.5 9B, refusal-ablated. Hermes auxiliary model (approval guardian, title,
    ; web_extract) for all agents and Honcho's deriver/summary/dream model.
    [ornith-1.5-9b-uncensored]
    model = ${modelsDir}/ornith-1.5-9b-uncensored/Ornith-1.5-9B-uncensored.Q4_K_M.gguf
    ; serves the deriver, all dialectic levels and three agents' approval guardians at once:
    ; 8 slots over one 96k pool so tiny guardian/title calls still find a slot while Honcho
    ; catch-up (deriver + dreams + dialectic) holds 6 (at 2 x 16k, concurrent calls hit "failed to find
    ; free space in the KV cache" and stalled). 96k overflowed ~every 2-3 min under
    ; load on 2026-09-23 (39 events 00:50-04:28; each fails ALL in-flight slots with
    ; "Context size has been exceeded"). KV is ~20 KB/token here (9B process was 7.5 GB
    ; at 96k incl. 5.5 GB weights), so 128k costs ~0.65 GB of the ~2.5 GB free.
    ctx-size = 131072
    parallel = 8
    ; One scheduler step = up to batch-size prompt tokens, and every decoding slot gets one
    ; token per step. At the 2048 default a step was ~2.4 s while Honcho's 10-28k prompts
    ; prefilled, so 16-token guardian calls hit p90 10 s. The GPU already computes in
    ; 512-token ubatches, so 512 keeps prefill throughput and makes steps ~0.6 s.
    batch-size = 512
    ; Runaway guard carried over from straylight: helper tasks need short answers.
    reasoning-budget = 2048
    n-predict = 4096
    ; stock template raises on tool-loop requests with no plain user turn (Honcho dialectic)
    chat-template-file = ${./ornith-9b-chat-template.jinja}
    ; Qwen thinking-mode sampling (llama-server's 0.8 default made the Honcho deriver drop
    ; ~1 in 3 batches, measured 2026-09-22). Clients that send their own temperature win.
    temp = 0.6
    top-p = 0.95
    top-k = 20
    min-p = 0

    ; honcho-dialectic (Qwen3-4B-Instruct-2507) retired 2026-09-22: fast but often wrong in a
    ; dialectic benchmark, where the 9B answered correctly. All dialectic levels use the 9B now;
    ; the 4B's VRAM went to the 9B's KV pool. File stays on the stick under qwen3-4b-instruct-2507/.

    ; Honcho embeddings (1024-dim; EMBEDDING_VECTOR_DIMENSIONS=1024 on rift).
    [honcho-embed]
    model = ${modelsDir}/qwen3-embedding-0.6b/Qwen3-Embedding-0.6B-Q8_0.gguf
    embedding = true
    pooling = cls
    ; A whole input must fit in one physical batch AND one slot (batch-size silently caps
    ; ubatch; ctx is split across slots). Bigger batches cost compute buffers fast on a
    ; shared 12 GB card: 8192/slot took 6.6 GB and 4096/slot did not fit beside the 9B and
    ; dialectic. So 2048/slot here, and rift's EMBEDDING_MAX_INPUT_TOKENS=1536 makes Honcho
    ; chunk below that (its token estimate is not Qwen's tokenizer, hence the margin).
    batch-size = 2048
    ubatch-size = 2048
    ctx-size = 4096
    parallel = 2
  '';

  # Vision (auxiliary.vision for all agents): its own server on :11433, loaded on first
  # request and dropped again after an idle hour so the router's three keep their VRAM.
  visionModelsIni = pkgs.writeText "llama-vision.ini" ''
    version = 1
    [*]
    jinja = true
    flash-attn = on
    n-gpu-layers = 999
    ctx-size = 16384
    parallel = 1
    sleep-idle-seconds = 3600

    [qwen3-vl-8b-abliterated]
    model = ${modelsDir}/qwen3-vl-8b-abliterated/Qwen3-VL-8B-Instruct-abliterated-v2.0.Q4_K_M.gguf
    mmproj = ${modelsDir}/qwen3-vl-8b-abliterated/Qwen3-VL-8B-Instruct-abliterated-v2.0.mmproj-f16.gguf
  '';

  llamaService = { description, port, preset, modelsMax }: {
    inherit description;
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "var-lib-llama\\x2dserver-models.mount" ];
    wants = [ "network-online.target" ];
    environment.HOME = "/var/lib/llama-server";
    serviceConfig = {
      ExecStart = ''
        ${llama-cpp}/bin/llama-server \
          --host 0.0.0.0 \
          --port ${toString port} \
          --models-preset ${preset} \
          --models-max ${toString modelsMax} \
          --metrics \
          --timeout 7200
      '';
      User = "llama-server";
      Group = "llama-server";
      StateDirectory = "llama-server";
      WorkingDirectory = "/var/lib/llama-server";
      SupplementaryGroups = [ "video" ];
      Restart = "on-failure";
      RestartSec = "10s";
      TimeoutStartSec = "600";
    };
  };

  # USB AX9L (mt7921u) is primary; internal CNVi (iwlwifi/wlo1) is fallback.
  # Connection profiles stay on-box (PSK); this only selects which radio is up.
  usbWifiMac = "74:19:f8:17:5e:53";
  usbWifiConn = "ATTjkThnDS";
  internalWifiConn = "ATTjkThnDS-iwlwifi";
  internalWifiIf = "wlo1";
  wifiUsbFailover = pkgs.writeShellScript "wifi-usb-failover" ''
    set -eu
    nmcli=${pkgs.networkmanager}/bin/nmcli
    logger=${pkgs.util-linux}/bin/logger
    systemd_run=${pkgs.systemd}/bin/systemd-run
    tr=${pkgs.coreutils}/bin/tr

    USB_MAC="${usbWifiMac}"
    USB_CONN="${usbWifiConn}"
    FALLBACK_CONN="${internalWifiConn}"
    INTERNAL_IF="${internalWifiIf}"

    log() { "$logger" -t wifi-usb-failover -- "$*"; }

    iface_mac() {
      "$tr" 'A-F' 'a-f' < "/sys/class/net/$1/address" 2>/dev/null || true
    }

    usb_iface() {
      for n in /sys/class/net/*; do
        n=''${n##*/}
        [ "$n" = "lo" ] && continue
        [ "$(iface_mac "$n")" = "$USB_MAC" ] && { echo "$n"; return 0; }
      done
      return 1
    }

    is_usb_iface() {
      [ "$(iface_mac "$1")" = "$USB_MAC" ]
    }

    demote_internal() {
      "$nmcli" device set "$INTERNAL_IF" autoconnect no >/dev/null 2>&1 || true
      "$nmcli" connection down "$FALLBACK_CONN" >/dev/null 2>&1 || true
      "$nmcli" device disconnect "$INTERNAL_IF" >/dev/null 2>&1 || true
    }

    promote_internal() {
      if usb_iface >/dev/null 2>&1; then
        log "USB still present; not failing over"
        return 0
      fi
      log "USB absent; bringing up $FALLBACK_CONN on $INTERNAL_IF"
      "$nmcli" device set "$INTERNAL_IF" autoconnect yes >/dev/null 2>&1 || true
      "$nmcli" connection up "$FALLBACK_CONN" ifname "$INTERNAL_IF" >/dev/null 2>&1 || true
    }

    settle() {
      i=0
      while [ "$i" -lt 20 ]; do
        if usb_iface >/dev/null 2>&1; then
          u=$(usb_iface)
          log "settle: USB $u present; demoting internal"
          demote_internal
          if ${pkgs.iproute2}/bin/ip -4 -o addr show dev "$u" | ${pkgs.gnugrep}/bin/grep -q ' inet '; then
            return 0
          fi
          "$nmcli" connection up "$USB_CONN" ifname "$u" >/dev/null 2>&1 || true
          return 0
        fi
        i=$((i + 1))
        sleep 1
      done
      log "settle: no USB after 20s; failing over"
      promote_internal
    }

    IFACE=''${1-}
    ACTION=''${2-}

    case "$IFACE" in
      boot|settle) settle; exit 0 ;;
      after-usb-down) sleep 3; promote_internal; exit 0 ;;
    esac

    case "$ACTION" in
      up)
        if is_usb_iface "$IFACE"; then
          log "USB $IFACE up; demoting $INTERNAL_IF"
          demote_internal
        fi
        ;;
      down)
        if is_usb_iface "$IFACE"; then
          log "USB $IFACE down; scheduling failover check"
          "$systemd_run" --quiet --collect --unit=wifi-usb-failover-down \
            --property=Type=oneshot "$0" after-usb-down >/dev/null 2>&1 || true
        fi
        ;;
    esac
    exit 0
  '';
in
{
  imports = [
    ./hardware-configuration.nix
    ./disk-config.nix
    inputs.tether.nixosModules.tether
    ../../configs/metrics/exporters.nix
  ];

  config = {
    # Headless. The base configuration.nix has no desktop; nothing GUI is added here.
    boot.loader.systemd-boot.enable = true;
    boot.loader.efi.canTouchEfiVariables = true;
    # 6.12 LTS: mt7921u (the AX9L USB Wi-Fi) is in-tree from 5.18; avoids the 6.17/6.18
    # mt7921 monitor-mode regression for good measure.
    boot.kernelPackages = pkgs.linuxPackages_6_12;

    # NVIDIA, compute only: no X, no modesetting needed for CUDA.
    hardware.graphics.enable = true;
    services.xserver.videoDrivers = [ "nvidia" ];
    hardware.nvidia = {
      open = false;
      modesetting.enable = false;
      nvidiaPersistenced = true;
      package = config.boot.kernelPackages.nvidiaPackages.production;
    };
    hardware.enableRedistributableFirmware = true;

    # Power cap for the 3060(s): thermal hour on 2026-09-21 held 75 °C at 140 W, case closed.
    systemd.services.nvidia-power-cap = {
      description = "Cap GPU power (thermals in the Aurora chassis)";
      wantedBy = [ "multi-user.target" ];
      after = [ "nvidia-persistenced.service" ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        ExecStart = "${config.hardware.nvidia.package.bin}/bin/nvidia-smi -pl 140";
      };
    };

    # Headless and Wi-Fi only: there is no console to add a key from after install, so the
    # keys go in with the system. Same set the live ISO used (~/src/dixie-iso).
    users.users.kyle.openssh.authorizedKeys.keyFiles = [ ./authorized_keys.pub ];
    users.users.root.openssh.authorizedKeys.keyFiles = [ ./authorized_keys.pub ];

    users.users.llama-server = { isSystemUser = true; group = "llama-server"; home = "/var/lib/llama-server"; };
    users.groups.llama-server = { };

    # Model weights live on the USB cache stick (ext4, label dixie-cache), populated from
    # straylight with dixie-push. Both servers wait for the mount.
    fileSystems."/mnt/cache" = {
      device = "/dev/disk/by-label/dixie-cache";
      fsType = "ext4";
      options = [ "noatime" "nofail" "x-systemd.device-timeout=30s" ];
    };
    fileSystems."/var/lib/llama-server/models" = {
      device = "/mnt/cache/models";
      fsType = "none";
      options = [ "bind" "nofail" "x-systemd.requires=/mnt/cache" ];
    };

    systemd.services.llama-server = llamaService {
      description = "llama.cpp router: helper tier (9B guardian/deriver/dialectic, embeddings)";
      port = 11434; preset = llamaModelsIni; modelsMax = 2;
    };
    systemd.services.llama-vision = llamaService {
      description = "llama.cpp vision server (qwen3-vl-8b, on demand)";
      port = 11433; preset = visionModelsIni; modelsMax = 1;
    };

    services.tether = {
      enable = true;
      listen = "0.0.0.0:11435";
      upstream = "http://127.0.0.1:11434";
    };

    # Wi-Fi only host. NetworkManager comes from the base config; the AX9L (mt7921u) is
    # primary and the internal CNVi radio is fallback. Connection profiles are copied
    # from the live-USB session at install time (see the install notes), not stored here.
    networking.networkmanager.wifi.backend = "wpa_supplicant";
    networking.networkmanager.dispatcherScripts = [
      { source = wifiUsbFailover; type = "basic"; }
    ];
    systemd.services.wifi-usb-failover-boot = {
      description = "Prefer USB Wi-Fi; fail over to internal CNVi if the dongle is absent";
      after = [ "NetworkManager.service" ];
      wants = [ "NetworkManager.service" ];
      wantedBy = [ "multi-user.target" ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        ExecStart = "${wifiUsbFailover} boot";
      };
    };

    # Reachable only over the tailnet: 11434 (router), 11433 (vision), 11435 (tether)
    # on tailscale0, and SSH from the LAN (base config opens 22) for bring-up.
    networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11433 11434 11435 9835 ];

    services.prometheus.exporters.nvidia-gpu = {
      enable = true;
      listenAddress = "0.0.0.0";
      port = 9835;
      openFirewall = false;
    };

    environment.systemPackages = with pkgs; [
      llama-cpp
      pciutils usbutils lm_sensors dmidecode nvme-cli smartmontools ethtool iw wpa_supplicant
      htop tmux git jq curl rsync nvtopPackages.nvidia
    ];

  };
}