metrics: phase 3 straylight/rift scrapes and hyperborea units

Review, no switch. node-laptop for talos/titan is excluded from
FleetTargetDown. llama-straylight scrape uses params.model for
ornith-1.5-35b-a3b (router 400 without it; will load if cold).

configs/metrics/hyperborea/override.conf

@@ -0,0 +1,7 @@
+# /etc/systemd/system/prometheus-node-exporter.service.d/override.conf
+[Unit]
+After=tailscaled.service
+Wants=tailscaled.service
+
+[Service]
+Restart=on-failure

configs/metrics/hyperborea/prometheus-node-exporter

@@ -0,0 +1,3 @@
+# /etc/default/prometheus-node-exporter on hyperborea (Debian).
+# Bind the tailscale IP only. LAN 192.168.88.154:9100 must stay closed.
+ARGS="--web.listen-address=100.112.40.10:9100"

configs/metrics/straylight.nix

@@ -0,0 +1,45 @@
+# Straylight node_exporter extras: amdgpu hwmon/DRM plus GTT textfile gauges.
+{ pkgs, ... }:
+{
+  imports = [ ./exporters.nix ];
+
+  services.prometheus.exporters.node = {
+    enabledCollectors = [ "hwmon" "drm" "textfile" ];
+    extraFlags = [
+      "--collector.textfile.directory=/var/lib/prometheus-node-exporter"
+    ];
+  };
+
+  systemd.tmpfiles.rules = [
+    "d /var/lib/prometheus-node-exporter 0755 root root -"
+  ];
+
+  systemd.services.node-exporter-gtt = {
+    description = "Write AMD GTT gauges for node_exporter textfile";
+    serviceConfig.Type = "oneshot";
+    script = ''
+      set -eu
+      d=/var/lib/prometheus-node-exporter
+      total=$(${pkgs.coreutils}/bin/cat /sys/class/drm/card1/device/mem_info_gtt_total)
+      used=$(${pkgs.coreutils}/bin/cat /sys/class/drm/card1/device/mem_info_gtt_used)
+      tmp="$d/gtt.prom.$$"
+      ${pkgs.coreutils}/bin/printf '%s\n' \
+        '# HELP gtt_total_bytes AMD GTT size in bytes' \
+        '# TYPE gtt_total_bytes gauge' \
+        "gtt_total_bytes $total" \
+        '# HELP gtt_used_bytes AMD GTT used in bytes' \
+        '# TYPE gtt_used_bytes gauge' \
+        "gtt_used_bytes $used" > "$tmp"
+      ${pkgs.coreutils}/bin/mv "$tmp" "$d/gtt.prom"
+    '';
+  };
+
+  systemd.timers.node-exporter-gtt = {
+    wantedBy = [ "timers.target" ];
+    timerConfig = {
+      OnBootSec = "15s";
+      OnUnitActiveSec = "15s";
+      AccuracySec = "1s";
+    };
+  };
+}

hw/orion/metrics.nix

@@ -32,6 +32,18 @@ in
           static_configs = [
             { targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; }
             { targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; }
+            { targets = [ "straylight:9100" ]; labels = { host = "straylight"; }; }
+            { targets = [ "rift:9100" ]; labels = { host = "rift"; }; }
+            { targets = [ "hyperborea:9100" ]; labels = { host = "hyperborea"; }; }
+          ];
+        }
+        {
+          # Laptops sleep; scrape them but do not page on down.
+          job_name = "node-laptop";
+          scrape_interval = "30s";
+          static_configs = [
+            { targets = [ "talos:9100" ]; labels = { host = "talos"; }; }
+            { targets = [ "titan:9100" ]; labels = { host = "titan"; }; }
           ];
         }
         {
@@ -44,6 +56,14 @@ in
           params.model = [ "ornith-1.5-9b-uncensored" ];
           static_configs = [{ targets = [ "dixie:11434" ]; }];
         }
+        {
+          job_name = "llama-straylight";
+          metrics_path = "/metrics";
+          # Router mode 400 without a model. 35B is the usual resident; scrape
+          # will load it if cold — drop params after switch if that happens.
+          params.model = [ "ornith-1.5-35b-a3b" ];
+          static_configs = [{ targets = [ "straylight:11434" ]; }];
+        }
         {
           job_name = "tether";
           metrics_path = "/tether/metrics";
@@ -191,6 +211,20 @@ in
               labels.severity = "page";
               annotations.summary = "dixie GPU > 83C";
             }
+            {
+              alert = "StraylightAmdgpuHot";
+              expr = ''node_hwmon_temp_celsius{chip=~".*amdgpu.*"} > 90'';
+              for = "2m";
+              labels.severity = "page";
+              annotations.summary = "straylight amdgpu > 90C";
+            }
+            {
+              alert = "StraylightGttHigh";
+              expr = "(gtt_used_bytes / gtt_total_bytes) > 0.95";
+              for = "5m";
+              labels.severity = "page";
+              annotations.summary = "straylight GTT > 95%";
+            }
           ];
         }
         {
@@ -204,7 +238,7 @@ in
             }
             {
               alert = "FleetTargetDown";
-              expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|tether|sift|honcho-api|honcho-deriver"} == 0'';
+              expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-straylight|tether|sift|honcho-api|honcho-deriver"} == 0'';
               for = "2m";
               labels.severity = "page";
               annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";

hw/rift/default.nix

@@ -4,6 +4,7 @@
   imports = [
     ./hardware-configuration.nix
     ./disk-config.nix
+    ../../configs/metrics/exporters.nix
   ];
 
   config = {

hw/straylight/default.nix

@@ -205,6 +205,7 @@ in
     ./hardware-configuration.nix
     ../../configs/desktop.nix
     ../../configs/qemu.nix
+    ../../configs/metrics/straylight.nix
   ];
 
   config = {
@@ -257,6 +258,7 @@ in
             --port 11434 \
             --models-preset ${llamaModelsIni} \
             --models-max 3 \
+            --metrics \
             --timeout 7200
         '';
         User = "llama-server";