imladris/hw/dixie/default.nix

{ inputs, pkgs, lib, config, ... }:
let
  # Unstable instantiated with unfree allowed: CUDA is unfree, and legacyPackages does not
  # inherit the host's nixpkgs.config (straylight's Vulkan build never hit this).
  pkgsUnstable = import inputs.nixpkgs-unstable {
    system = pkgs.stdenv.hostPlatform.system;
    config = { allowUnfree = true; cudaSupport = true; };
  };
  # llama.cpp with the CUDA backend, from unstable to match straylight's build (same model
  # files, same router semantics). Built once on straylight and shipped with dixie-push
  # --closure, or pulled from the cache when Dixie's link allows.
  llama-cpp = pkgsUnstable.llama-cpp.override { cudaSupport = true; };
  modelsDir = "/var/lib/llama-server/models";

  # Dixie is the fleet's helper tier: everything that is latency-sensitive, small, and was
  # starving on straylight's CPU (2026-09-21). One RTX 3060 (12 GB) today; card 2 joins
  # when its fans are replaced.
  #
  # Router: the three models that must never be swapped out — approval guardian / Honcho
  # deriver (9B), Honcho dialectic (4B, non-thinking), Honcho embeddings — resident together:
  # 5.6 + 2.5 + 0.7 GB weights, ~3 GB left for KV. Vision does NOT go here: 5 GB + 1.2 GB
  # mmproj does not fit beside them, so it gets its own on-demand process below.
  llamaModelsIni = pkgs.writeText "llama-models.ini" ''
    version = 1
    [*]
    jinja = true
    flash-attn = on
    cache-type-k = q8_0
    cache-type-v = q8_0
    n-gpu-layers = 999
    ; three resident models share 12 GB: keep contexts modest
    ctx-size = 16384
    parallel = 2
    ; one KV pool per model shared by its slots, so a single long request can use all 16k
    ; (split slots capped each at 8k: Honcho dialectic prompts of 11-17k all failed)
    kv-unified = true
    cache-ram = 512

    ; Ornith 1.5 9B, refusal-ablated. Hermes auxiliary model (approval guardian, title,
    ; web_extract) for all agents and Honcho's deriver/summary/dream model.
    [ornith-1.5-9b-uncensored]
    model = ${modelsDir}/ornith-1.5-9b-uncensored/Ornith-1.5-9B-uncensored.Q4_K_M.gguf
    ; serves the deriver, all dialectic levels and three agents' approval guardians at once:
    ; 4 slots over one 48k pool (at 2 x 16k, concurrent deriver + dialectic hit "failed to find
    ; free space in the KV cache" and stalled)
    ctx-size = 49152
    parallel = 4
    ; Runaway guard carried over from straylight: helper tasks need short answers.
    reasoning-budget = 2048
    n-predict = 4096
    ; stock template raises on tool-loop requests with no plain user turn (Honcho dialectic)
    chat-template-file = ${./ornith-9b-chat-template.jinja}
    ; Qwen thinking-mode sampling (llama-server's 0.8 default made the Honcho deriver drop
    ; ~1 in 3 batches, measured 2026-09-22). Clients that send their own temperature win.
    temp = 0.6
    top-p = 0.95
    top-k = 20
    min-p = 0

    ; honcho-dialectic (Qwen3-4B-Instruct-2507) retired 2026-09-22: fast but often wrong in a
    ; dialectic benchmark, where the 9B answered correctly. All dialectic levels use the 9B now;
    ; the 4B's VRAM went to the 9B's KV pool. File stays on the stick under qwen3-4b-instruct-2507/.

    ; Honcho embeddings (1024-dim; EMBEDDING_VECTOR_DIMENSIONS=1024 on rift).
    [honcho-embed]
    model = ${modelsDir}/qwen3-embedding-0.6b/Qwen3-Embedding-0.6B-Q8_0.gguf
    embedding = true
    pooling = cls
    ; A whole input must fit in one physical batch AND one slot (batch-size silently caps
    ; ubatch; ctx is split across slots). Bigger batches cost compute buffers fast on a
    ; shared 12 GB card: 8192/slot took 6.6 GB and 4096/slot did not fit beside the 9B and
    ; dialectic. So 2048/slot here, and rift's EMBEDDING_MAX_INPUT_TOKENS=1536 makes Honcho
    ; chunk below that (its token estimate is not Qwen's tokenizer, hence the margin).
    batch-size = 2048
    ubatch-size = 2048
    ctx-size = 4096
    parallel = 2
  '';

  # Vision (auxiliary.vision for all agents): its own server on :11433, loaded on first
  # request and dropped again after an idle hour so the router's three keep their VRAM.
  visionModelsIni = pkgs.writeText "llama-vision.ini" ''
    version = 1
    [*]
    jinja = true
    flash-attn = on
    n-gpu-layers = 999
    ctx-size = 16384
    parallel = 1
    sleep-idle-seconds = 3600

    [qwen3-vl-8b-abliterated]
    model = ${modelsDir}/qwen3-vl-8b-abliterated/Qwen3-VL-8B-Instruct-abliterated-v2.0.Q4_K_M.gguf
    mmproj = ${modelsDir}/qwen3-vl-8b-abliterated/Qwen3-VL-8B-Instruct-abliterated-v2.0.mmproj-f16.gguf
  '';

  llamaService = { description, port, preset, modelsMax }: {
    inherit description;
    wantedBy = [ "multi-user.target" ];
    after = [ "network-online.target" "var-lib-llama\\x2dserver-models.mount" ];
    wants = [ "network-online.target" ];
    environment.HOME = "/var/lib/llama-server";
    serviceConfig = {
      ExecStart = ''
        ${llama-cpp}/bin/llama-server \
          --host 0.0.0.0 \
          --port ${toString port} \
          --models-preset ${preset} \
          --models-max ${toString modelsMax} \
          --timeout 7200
      '';
      User = "llama-server";
      Group = "llama-server";
      StateDirectory = "llama-server";
      WorkingDirectory = "/var/lib/llama-server";
      SupplementaryGroups = [ "video" ];
      Restart = "on-failure";
      RestartSec = "10s";
      TimeoutStartSec = "600";
    };
  };
in
{
  imports = [
    ./hardware-configuration.nix
    ./disk-config.nix
  ];

  config = {
    # Headless. The base configuration.nix has no desktop; nothing GUI is added here.
    boot.loader.systemd-boot.enable = true;
    boot.loader.efi.canTouchEfiVariables = true;
    # 6.12 LTS: mt7921u (the AX9L USB Wi-Fi) is in-tree from 5.18; avoids the 6.17/6.18
    # mt7921 monitor-mode regression for good measure.
    boot.kernelPackages = pkgs.linuxPackages_6_12;

    # NVIDIA, compute only: no X, no modesetting needed for CUDA.
    hardware.graphics.enable = true;
    services.xserver.videoDrivers = [ "nvidia" ];
    hardware.nvidia = {
      open = false;
      modesetting.enable = false;
      nvidiaPersistenced = true;
      package = config.boot.kernelPackages.nvidiaPackages.production;
    };
    hardware.enableRedistributableFirmware = true;

    # Power cap for the 3060(s): thermal hour on 2026-09-21 held 75 °C at 140 W, case closed.
    systemd.services.nvidia-power-cap = {
      description = "Cap GPU power (thermals in the Aurora chassis)";
      wantedBy = [ "multi-user.target" ];
      after = [ "nvidia-persistenced.service" ];
      serviceConfig = {
        Type = "oneshot";
        RemainAfterExit = true;
        ExecStart = "${config.hardware.nvidia.package.bin}/bin/nvidia-smi -pl 140";
      };
    };

    # Headless and Wi-Fi only: there is no console to add a key from after install, so the
    # keys go in with the system. Same set the live ISO used (~/src/dixie-iso).
    users.users.kyle.openssh.authorizedKeys.keyFiles = [ ./authorized_keys.pub ];
    users.users.root.openssh.authorizedKeys.keyFiles = [ ./authorized_keys.pub ];

    users.users.llama-server = { isSystemUser = true; group = "llama-server"; home = "/var/lib/llama-server"; };
    users.groups.llama-server = { };

    # Model weights live on the USB cache stick (ext4, label dixie-cache), populated from
    # straylight with dixie-push. Both servers wait for the mount.
    fileSystems."/mnt/cache" = {
      device = "/dev/disk/by-label/dixie-cache";
      fsType = "ext4";
      options = [ "noatime" "nofail" "x-systemd.device-timeout=30s" ];
    };
    fileSystems."/var/lib/llama-server/models" = {
      device = "/mnt/cache/models";
      fsType = "none";
      options = [ "bind" "nofail" "x-systemd.requires=/mnt/cache" ];
    };

    systemd.services.llama-server = llamaService {
      description = "llama.cpp router: helper tier (9B guardian/deriver/dialectic, embeddings)";
      port = 11434; preset = llamaModelsIni; modelsMax = 2;
    };
    systemd.services.llama-vision = llamaService {
      description = "llama.cpp vision server (qwen3-vl-8b, on demand)";
      port = 11433; preset = visionModelsIni; modelsMax = 1;
    };

    # Wi-Fi only host. NetworkManager comes from the base config; the AX9L (mt7921u) becomes
    # the primary once plugged in and the internal CNVi radio stays as fallback. Connection
    # profiles are copied from the live-USB session at install time (see the install notes),
    # not stored in this repo.
    networking.networkmanager.wifi.backend = "wpa_supplicant";

    # Reachable only over the tailnet: 11434 (router) and 11433 (vision) on tailscale0, and
    # SSH from the LAN (base config opens 22) for bring-up.
    networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11433 11434 ];

    environment.systemPackages = with pkgs; [
      llama-cpp
      pciutils usbutils lm_sensors dmidecode nvme-cli smartmontools ethtool iw wpa_supplicant
      htop tmux git jq curl rsync nvtopPackages.nvidia
    ];

  };
}