straylight: pin amdgpu DPM high and enable Laguna DFlash
Keep the iGPU out of powersave while llama-server runs, and pair Laguna S 2.1 with Poolside's DFlash draft (f16 draft KV, n-max 4).
This commit is contained in:
@@ -39,8 +39,16 @@ let
|
||||
|
||||
; Poolside Laguna S 2.1 (118B total, 8B active): primary coding agent.
|
||||
; ~69 GB of weights, so it only loads once the raised TTM/GTT limit is active (reboot).
|
||||
; DFlash draft (~2.2 GB BF16) from poolside/Laguna-S-2.1-GGUF. Keep draft KV in
|
||||
; f16; q8 draft KV collapses acceptance. spec-draft-n-max 4 is the gfx1151
|
||||
; sweet spot (n=15 is the trained cap and is slower here).
|
||||
[laguna-s-2.1]
|
||||
model = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/Laguna-S-2.1-UD-Q4_K_S-00001-of-00003.gguf
|
||||
model-draft = ${llamaModelsDir}/laguna-s-2.1-UD-Q4_K_S/laguna-s-2.1-DFlash-BF16.gguf
|
||||
spec-type = draft-dflash
|
||||
spec-draft-n-max = 4
|
||||
cache-type-k-draft = f16
|
||||
cache-type-v-draft = f16
|
||||
parallel = 1
|
||||
ctx-size = 131072
|
||||
|
||||
@@ -143,6 +151,15 @@ let
|
||||
whisperTranscribe = pkgs.writeShellScriptBin "whisper-transcribe" ''
|
||||
exec ${whisperCpp}/bin/whisper-cli -m ${whisperModel} --language auto -f "$@"
|
||||
'';
|
||||
amdgpuDpmHigh = pkgs.writeShellScript "amdgpu-dpm-high" ''
|
||||
set -eu
|
||||
for f in /sys/class/drm/card*/device/power_dpm_force_performance_level; do
|
||||
vendor="$(dirname "$f")/vendor"
|
||||
[ -r "$vendor" ] || continue
|
||||
grep -qx '0x1002' "$vendor" || continue
|
||||
echo high > "$f"
|
||||
done
|
||||
'';
|
||||
in
|
||||
{
|
||||
imports = [
|
||||
@@ -159,6 +176,19 @@ in
|
||||
# amdgpu.gttsize is deprecated on recent kernels; ttm.* is the supported knob.
|
||||
boot.kernelParams = [ "ttm.pages_limit=27262976" "ttm.page_pool_size=27262976" ];
|
||||
|
||||
# amdgpu defaults to powersave/auto and will clock the iGPU down under a
|
||||
# desktop compositor. llama.cpp on gfx1151 wants the high DPM profile.
|
||||
systemd.services.amdgpu-dpm-high = {
|
||||
description = "Set amdgpu DPM performance level to high";
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
after = [ "systemd-udev-settle.service" ];
|
||||
serviceConfig = {
|
||||
Type = "oneshot";
|
||||
RemainAfterExit = true;
|
||||
ExecStart = amdgpuDpmHigh;
|
||||
};
|
||||
};
|
||||
|
||||
# llama.cpp server (router mode) with the Vulkan backend for gfx1151 (Strix Halo).
|
||||
# Models and per-model settings live in llamaModelsIni above.
|
||||
# NOTE: BIOS "UMA Frame Buffer Size" must be ≥32GB for a 23GB model to fit on-GPU.
|
||||
@@ -182,6 +212,8 @@ in
|
||||
# Bind all interfaces so localhost and the tailnet can both reach the
|
||||
# OpenAI-compatible API. LAN access is still blocked: 11434 is opened
|
||||
# only on tailscale0, not in the global allowedTCPPorts list.
|
||||
# "+" runs as root so we can pin amdgpu DPM before the server starts.
|
||||
ExecStartPre = "+${amdgpuDpmHigh}";
|
||||
ExecStart = ''
|
||||
${llama-cpp}/bin/llama-server \
|
||||
--host 0.0.0.0 \
|
||||
|
||||
Reference in New Issue
Block a user