From 385529da39c56a17bd10608ee4cc8d385e73d078 Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Thu, 24 Sep 2026 00:42:06 -0700 Subject: [PATCH] metrics: phase 3 straylight/rift scrapes and hyperborea units Review, no switch. node-laptop for talos/titan is excluded from FleetTargetDown. llama-straylight scrape uses params.model for ornith-1.5-35b-a3b (router 400 without it; will load if cold). --- configs/metrics/hyperborea/override.conf | 7 +++ .../hyperborea/prometheus-node-exporter | 3 ++ configs/metrics/straylight.nix | 45 +++++++++++++++++++ hw/orion/metrics.nix | 36 ++++++++++++++- hw/rift/default.nix | 1 + hw/straylight/default.nix | 2 + 6 files changed, 93 insertions(+), 1 deletion(-) create mode 100644 configs/metrics/hyperborea/override.conf create mode 100644 configs/metrics/hyperborea/prometheus-node-exporter create mode 100644 configs/metrics/straylight.nix diff --git a/configs/metrics/hyperborea/override.conf b/configs/metrics/hyperborea/override.conf new file mode 100644 index 0000000..fcd380e --- /dev/null +++ b/configs/metrics/hyperborea/override.conf @@ -0,0 +1,7 @@ +# /etc/systemd/system/prometheus-node-exporter.service.d/override.conf +[Unit] +After=tailscaled.service +Wants=tailscaled.service + +[Service] +Restart=on-failure diff --git a/configs/metrics/hyperborea/prometheus-node-exporter b/configs/metrics/hyperborea/prometheus-node-exporter new file mode 100644 index 0000000..3be208e --- /dev/null +++ b/configs/metrics/hyperborea/prometheus-node-exporter @@ -0,0 +1,3 @@ +# /etc/default/prometheus-node-exporter on hyperborea (Debian). +# Bind the tailscale IP only. LAN 192.168.88.154:9100 must stay closed. +ARGS="--web.listen-address=100.112.40.10:9100" diff --git a/configs/metrics/straylight.nix b/configs/metrics/straylight.nix new file mode 100644 index 0000000..ab13ce3 --- /dev/null +++ b/configs/metrics/straylight.nix @@ -0,0 +1,45 @@ +# Straylight node_exporter extras: amdgpu hwmon/DRM plus GTT textfile gauges. +{ pkgs, ... }: +{ + imports = [ ./exporters.nix ]; + + services.prometheus.exporters.node = { + enabledCollectors = [ "hwmon" "drm" "textfile" ]; + extraFlags = [ + "--collector.textfile.directory=/var/lib/prometheus-node-exporter" + ]; + }; + + systemd.tmpfiles.rules = [ + "d /var/lib/prometheus-node-exporter 0755 root root -" + ]; + + systemd.services.node-exporter-gtt = { + description = "Write AMD GTT gauges for node_exporter textfile"; + serviceConfig.Type = "oneshot"; + script = '' + set -eu + d=/var/lib/prometheus-node-exporter + total=$(${pkgs.coreutils}/bin/cat /sys/class/drm/card1/device/mem_info_gtt_total) + used=$(${pkgs.coreutils}/bin/cat /sys/class/drm/card1/device/mem_info_gtt_used) + tmp="$d/gtt.prom.$$" + ${pkgs.coreutils}/bin/printf '%s\n' \ + '# HELP gtt_total_bytes AMD GTT size in bytes' \ + '# TYPE gtt_total_bytes gauge' \ + "gtt_total_bytes $total" \ + '# HELP gtt_used_bytes AMD GTT used in bytes' \ + '# TYPE gtt_used_bytes gauge' \ + "gtt_used_bytes $used" > "$tmp" + ${pkgs.coreutils}/bin/mv "$tmp" "$d/gtt.prom" + ''; + }; + + systemd.timers.node-exporter-gtt = { + wantedBy = [ "timers.target" ]; + timerConfig = { + OnBootSec = "15s"; + OnUnitActiveSec = "15s"; + AccuracySec = "1s"; + }; + }; +} diff --git a/hw/orion/metrics.nix b/hw/orion/metrics.nix index cdfc2d5..20aa8fc 100644 --- a/hw/orion/metrics.nix +++ b/hw/orion/metrics.nix @@ -32,6 +32,18 @@ in static_configs = [ { targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; } { targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; } + { targets = [ "straylight:9100" ]; labels = { host = "straylight"; }; } + { targets = [ "rift:9100" ]; labels = { host = "rift"; }; } + { targets = [ "hyperborea:9100" ]; labels = { host = "hyperborea"; }; } + ]; + } + { + # Laptops sleep; scrape them but do not page on down. + job_name = "node-laptop"; + scrape_interval = "30s"; + static_configs = [ + { targets = [ "talos:9100" ]; labels = { host = "talos"; }; } + { targets = [ "titan:9100" ]; labels = { host = "titan"; }; } ]; } { @@ -44,6 +56,14 @@ in params.model = [ "ornith-1.5-9b-uncensored" ]; static_configs = [{ targets = [ "dixie:11434" ]; }]; } + { + job_name = "llama-straylight"; + metrics_path = "/metrics"; + # Router mode 400 without a model. 35B is the usual resident; scrape + # will load it if cold — drop params after switch if that happens. + params.model = [ "ornith-1.5-35b-a3b" ]; + static_configs = [{ targets = [ "straylight:11434" ]; }]; + } { job_name = "tether"; metrics_path = "/tether/metrics"; @@ -191,6 +211,20 @@ in labels.severity = "page"; annotations.summary = "dixie GPU > 83C"; } + { + alert = "StraylightAmdgpuHot"; + expr = ''node_hwmon_temp_celsius{chip=~".*amdgpu.*"} > 90''; + for = "2m"; + labels.severity = "page"; + annotations.summary = "straylight amdgpu > 90C"; + } + { + alert = "StraylightGttHigh"; + expr = "(gtt_used_bytes / gtt_total_bytes) > 0.95"; + for = "5m"; + labels.severity = "page"; + annotations.summary = "straylight GTT > 95%"; + } ]; } { @@ -204,7 +238,7 @@ in } { alert = "FleetTargetDown"; - expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|tether|sift|honcho-api|honcho-deriver"} == 0''; + expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-straylight|tether|sift|honcho-api|honcho-deriver"} == 0''; for = "2m"; labels.severity = "page"; annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})"; diff --git a/hw/rift/default.nix b/hw/rift/default.nix index 43c451d..0889d99 100644 --- a/hw/rift/default.nix +++ b/hw/rift/default.nix @@ -4,6 +4,7 @@ imports = [ ./hardware-configuration.nix ./disk-config.nix + ../../configs/metrics/exporters.nix ]; config = { diff --git a/hw/straylight/default.nix b/hw/straylight/default.nix index e556858..81120f2 100644 --- a/hw/straylight/default.nix +++ b/hw/straylight/default.nix @@ -205,6 +205,7 @@ in ./hardware-configuration.nix ../../configs/desktop.nix ../../configs/qemu.nix + ../../configs/metrics/straylight.nix ]; config = { @@ -257,6 +258,7 @@ in --port 11434 \ --models-preset ${llamaModelsIni} \ --models-max 3 \ + --metrics \ --timeout 7200 ''; User = "llama-server";