kyle
5a734118b0
mradermacher Q4_K_M of junafinity/Ornith-1.5-9B-uncensored. Multimodal 9B; llama-server fetches weights and mmproj on first load.
246 lines
11 KiB
Nix
246 lines
11 KiB
Nix
{ inputs, pkgs, lib, ... }:
|
|
let
|
|
pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.stdenv.hostPlatform.system};
|
|
# llama.cpp from nixpkgs-unstable (0.4.x as of Sept 2026) with the Vulkan backend for
|
|
# gfx1151 (Strix Halo). Laguna S 2.1 and Gemma 4 need a build newer than mid-July 2026,
|
|
# so keep the unstable input reasonably fresh (`nix flake update nixpkgs-unstable`).
|
|
llama-cpp = pkgsUnstable.llama-cpp.override { vulkanSupport = true; };
|
|
llamaModelsDir = "/var/lib/llama-server/models";
|
|
# llama-server runs in router mode: one endpoint, models load on demand, at most two
|
|
# resident at a time (see --models-max below). Section names are the model ids that
|
|
# clients pass in the "model" field. Weights are fetched by
|
|
# /var/lib/llama-server/models/download.sh, not by the service.
|
|
llamaModelsIni = pkgs.writeText "llama-models.ini" ''
|
|
version = 1
|
|
|
|
[*]
|
|
jinja = true
|
|
flash-attn = on
|
|
cache-type-k = q8_0
|
|
cache-type-v = q8_0
|
|
n-gpu-layers = 999
|
|
no-mmap = true
|
|
; llama.cpp splits ctx-size across parallel slots: 262144 / 2 = 131072 per slot.
|
|
ctx-size = 262144
|
|
parallel = 2
|
|
; Unload a model's weights and KV cache after six idle hours; the next request reloads it.
|
|
sleep-idle-seconds = 21600
|
|
|
|
; Qwen3.6-35B-A3B, refusal-ablated (HauhauCS "Aggressive"). Benchmark candidate A.
|
|
[qwen3.6-35b-a3b-abliterated]
|
|
model = ${llamaModelsDir}/qwen3.6-35b-a3b-abliterated/Qwen3.6-35B-A3B-Uncensored-HauhauCS-Aggressive-Q4_K_M.gguf
|
|
|
|
; Gemma 4 26B-A4B QAT, refusal-ablated (HauhauCS "Balanced"), with its MTP draft head.
|
|
; Benchmark candidate B (the Western model of the same MoE class).
|
|
[gemma4-26b-a4b-abliterated]
|
|
model = ${llamaModelsDir}/gemma4-26b-a4b-abliterated/Gemma4-26B-A4B-QAT-Uncensored-HauhauCS-Balanced-Q4_K_M.gguf
|
|
model-draft = ${llamaModelsDir}/gemma4-26b-a4b-abliterated/mtp-gemma-4-26B-A4B-it.gguf
|
|
spec-type = draft-mtp
|
|
|
|
; Poolside Laguna S 2.1 (118B total, 8B active): primary coding agent.
|
|
; ~69 GB of weights, so it only loads once the raised TTM/GTT limit is active (reboot).
|
|
[laguna-s-2.1]
|
|
model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf
|
|
parallel = 1
|
|
ctx-size = 131072
|
|
|
|
; Previous default model (a Qwen 3.5 derivative), kept for comparison runs. It was
|
|
; fetched by the old -hf flag into the HF cache; dedup hides the cache's own entry.
|
|
[ornith-1.0-35b]
|
|
hf-repo = deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M
|
|
dedup-cache-models = true
|
|
|
|
; Ornith 1.5 9B, refusal-ablated (junafinity / mradermacher GGUF). Multimodal:
|
|
; llama-server pulls mmproj with --hf-repo. Q4_K_M is the recommended quant.
|
|
[ornith-1.5-9b-uncensored]
|
|
hf-repo = mradermacher/Ornith-1.5-9B-uncensored-GGUF:Q4_K_M
|
|
dedup-cache-models = true
|
|
|
|
; Qwen 3.8 27B dense, refusal-ablated. Preferred uncensored coding agent.
|
|
; orcarouter/Qwen3.8-27B-Uncensored-GGUF is gated; huihui Q4_K_L keeps
|
|
; ablation-target tensors at Q8_0. llama-server fetches on first load.
|
|
[qwen3.8-27b-uncensored]
|
|
hf-repo = huihui-ai/Huihui-Qwen3.8-27B-abliterated-GGUF:Q4_K_L
|
|
dedup-cache-models = true
|
|
'';
|
|
# `llama-models` shows what the router has resident; `llama-unload` frees every loaded
|
|
# model (e.g. before a gaming session), or just the ones named on the command line.
|
|
llamaCtl = pkgs.writeText "llama-ctl.py" ''
|
|
import json, sys, urllib.request
|
|
|
|
URL = "http://127.0.0.1:11434"
|
|
|
|
def models():
|
|
with urllib.request.urlopen(URL + "/models", timeout=10) as r:
|
|
d = json.load(r)
|
|
return d.get("data", d if isinstance(d, list) else [])
|
|
|
|
def name(m):
|
|
return m.get("id") or m.get("model") or m.get("name")
|
|
|
|
cmd = sys.argv[1] if len(sys.argv) > 1 else "list"
|
|
if cmd == "list":
|
|
for m in models():
|
|
print("%-10s %s" % ((m.get("status") or {}).get("value", "?"), name(m)))
|
|
elif cmd == "unload":
|
|
loaded = [name(m) for m in models()
|
|
if (m.get("status") or {}).get("value") in ("loaded", "loading", "sleeping")]
|
|
targets = sys.argv[2:] or loaded
|
|
if not targets:
|
|
print("nothing loaded")
|
|
for t in targets:
|
|
req = urllib.request.Request(URL + "/models/unload", data=json.dumps({"model": t}).encode(),
|
|
headers={"Content-Type": "application/json"}, method="POST")
|
|
with urllib.request.urlopen(req, timeout=30) as r:
|
|
print("unload %s: %s" % (t, r.read().decode().strip()))
|
|
else:
|
|
sys.exit("usage: llama-ctl.py [list|unload [model ...]]")
|
|
'';
|
|
llamaModelsCmd = pkgs.writeShellScriptBin "llama-models" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} list \"$@\"";
|
|
llamaUnloadCmd = pkgs.writeShellScriptBin "llama-unload" "exec ${pkgs.python3}/bin/python3 ${llamaCtl} unload \"$@\"";
|
|
in
|
|
{
|
|
imports = [
|
|
./hardware-configuration.nix
|
|
../../configs/desktop.nix
|
|
../../configs/qemu.nix
|
|
../../configs/mcpkg.nix
|
|
../../configs/mcp.nix # MCP agent + mcp user (straylight is becoming the core host)
|
|
];
|
|
|
|
config = {
|
|
# Let the GPU address up to 104 GiB of the 128 GiB of unified memory (the TTM default
|
|
# caps GTT at 50% of RAM, ~62 GiB). Units are 4 KiB pages; takes effect after a reboot.
|
|
# amdgpu.gttsize is deprecated on recent kernels; ttm.* is the supported knob.
|
|
boot.kernelParams = [ "ttm.pages_limit=27262976" "ttm.page_pool_size=27262976" ];
|
|
|
|
# llama.cpp server (router mode) with the Vulkan backend for gfx1151 (Strix Halo).
|
|
# Models and per-model settings live in llamaModelsIni above.
|
|
# NOTE: BIOS "UMA Frame Buffer Size" must be ≥32GB for a 23GB model to fit on-GPU.
|
|
users.users.llama-server = {
|
|
isSystemUser = true;
|
|
group = "llama-server";
|
|
home = "/var/lib/llama-server";
|
|
};
|
|
users.groups.llama-server = {};
|
|
|
|
systemd.services.llama-server = {
|
|
description = "llama.cpp inference server";
|
|
wantedBy = [ "multi-user.target" ];
|
|
after = [ "network-online.target" ];
|
|
wants = [ "network-online.target" ];
|
|
environment = {
|
|
HOME = "/var/lib/llama-server";
|
|
HF_HOME = "/var/lib/llama-server/huggingface";
|
|
};
|
|
serviceConfig = {
|
|
# Bind all interfaces so localhost and the tailnet can both reach the
|
|
# OpenAI-compatible API. LAN access is still blocked: 11434 is opened
|
|
# only on tailscale0, not in the global allowedTCPPorts list.
|
|
ExecStart = ''
|
|
${llama-cpp}/bin/llama-server \
|
|
--host 0.0.0.0 \
|
|
--port 11434 \
|
|
--models-preset ${llamaModelsIni} \
|
|
--models-max 2
|
|
'';
|
|
User = "llama-server";
|
|
Group = "llama-server";
|
|
StateDirectory = "llama-server";
|
|
WorkingDirectory = "/var/lib/llama-server";
|
|
SupplementaryGroups = [ "render" "video" ];
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
TimeoutStartSec = "600";
|
|
};
|
|
};
|
|
# straylight is the unikernel host. The shared mcp.nix locks the agent
|
|
# down with PrivateDevices=true, which hides /dev/kvm and /dev/net/tun.
|
|
# Relax that here (only on straylight) so the agent can boot Nanos
|
|
# unikernel VMs under QEMU/KVM and (Phase 2) manage TAP devices.
|
|
systemd.services.mcp-agent.serviceConfig = {
|
|
PrivateDevices = lib.mkForce false;
|
|
DeviceAllow = [ "/dev/kvm rw" "/dev/net/tun rw" ];
|
|
SupplementaryGroups = [ "kvm" ];
|
|
AmbientCapabilities = [ "CAP_NET_ADMIN" ];
|
|
# The agent launches each unikernel as a daemonized QEMU process in its
|
|
# own cgroup. With the default KillMode=control-group, restarting the
|
|
# agent would SIGKILL every running VM. KillMode=process kills only the
|
|
# agent's main process on stop/restart, so VMs survive an agent upgrade.
|
|
# (If a VM does die, the agent's startup Recover restarts it.)
|
|
KillMode = lib.mkForce "process";
|
|
};
|
|
# Let the mcp user reach /dev/kvm directly as well.
|
|
users.users.mcp.extraGroups = [ "kvm" ];
|
|
|
|
# Isolated host-only bridge for unikernel VMs (Phase 2). Each unikernel
|
|
# gets a TAP on this bridge and a 10.99.0.0/24 static IP. The bridge has
|
|
# NO uplink and NO NAT, and the firewall drops any VM traffic leaving the
|
|
# bridge, so a VM can reach only the host gateway (10.99.0.1) -- mediation
|
|
# is enforced by network topology, not convention.
|
|
networking.bridges.mcp-br0.interfaces = [ ];
|
|
networking.interfaces.mcp-br0.ipv4.addresses = [
|
|
{ address = "10.99.0.1"; prefixLength = 24; }
|
|
];
|
|
# The host accepts traffic from VMs (so mc-proxy on the gateway can serve
|
|
# them); the FORWARD drop prevents VMs from routing anywhere off-bridge.
|
|
networking.firewall.trustedInterfaces = [ "mcp-br0" ];
|
|
networking.firewall.extraCommands = ''
|
|
iptables -D FORWARD -i mcp-br0 ! -o mcp-br0 -j DROP 2>/dev/null || true
|
|
iptables -A FORWARD -i mcp-br0 ! -o mcp-br0 -j DROP
|
|
'';
|
|
networking.firewall.extraStopCommands = ''
|
|
iptables -D FORWARD -i mcp-br0 ! -o mcp-br0 -j DROP 2>/dev/null || true
|
|
'';
|
|
|
|
# Allow rootless containers (podman) to bind low ports (53 for MCNS,
|
|
# 443/8443/9443 for mc-proxy) as straylight takes over the core role.
|
|
boot.kernel.sysctl."net.ipv4.ip_unprivileged_port_start" = 53;
|
|
|
|
hardware.bluetooth.enable = true; # enables BlueZ + bluetooth.service
|
|
|
|
programs.steam.enable = true;
|
|
programs.steam.protontricks.enable = true;
|
|
|
|
# herdr: terminal workspace manager for coding agents (https://herdr.dev).
|
|
# Pinned to the upstream flake (v0.9.0); nixpkgs-unstable only has 0.7.1.
|
|
environment.systemPackages = [
|
|
inputs.herdr.packages.x86_64-linux.default
|
|
llamaModelsCmd
|
|
llamaUnloadCmd
|
|
];
|
|
|
|
services.open-webui = {
|
|
enable = true;
|
|
host = "0.0.0.0";
|
|
port = 8080;
|
|
environment = {
|
|
SCARF_NO_ANALYTICS = "True";
|
|
DO_NOT_TRACK = "True";
|
|
ANONYMIZED_TELEMETRY = "False";
|
|
# Point at the local llama.cpp server (OpenAI-compatible API).
|
|
OPENAI_API_BASE_URLS = "http://127.0.0.1:11434/v1";
|
|
OPENAI_API_KEYS = "none";
|
|
ENABLE_OLLAMA_API = "False";
|
|
};
|
|
};
|
|
|
|
# Open ports: DNS (53), mc-proxy (443/8443/9443), agent (9444), master (9555), open-webui (8080).
|
|
networking.firewall.allowedTCPPorts = [ 53 443 8080 8443 9443 9444 9555 ];
|
|
networking.firewall.allowedUDPPorts = [ 53 ];
|
|
# llama.cpp OpenAI-compatible API: tailnet only (localhost is always allowed).
|
|
networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11434 ];
|
|
|
|
# DNS: MCNS for internal zones, public resolvers as fallback.
|
|
networking.nameservers = [
|
|
"192.168.88.181"
|
|
"100.95.252.120"
|
|
"1.1.1.1"
|
|
"8.8.8.8"
|
|
];
|
|
services.resolved.settings.Resolve.Domains = [
|
|
"~mcp.metacircular.net"
|
|
];
|
|
};
|
|
}
|