metrics: phase 1 follow-up (review, no switch)

Honcho workspace_name keep, node_exporter restart metrics,
drop min_over_time on rule 1, Watchdog heartbeat, pin tether
to b267e2f so labelled series exist and rule 2 can fire.

configs/metrics/exporters.nix

@@ -6,6 +6,10 @@
     port = 9100;
     listenAddress = "0.0.0.0";
     enabledCollectors = [ "systemd" ];
+    extraFlags = [
+      "--collector.systemd.enable-restarts-metrics"
+      ''--collector.systemd.unit-include=(llama-server|llama-vision)\.service''
+    ];
     openFirewall = false;
   };
 

flake.lock

@@ -497,16 +497,16 @@
         ]
       },
       "locked": {
-        "lastModified": 1790149025,
-        "narHash": "sha256-3EUf6ke80hm8o+YyS3wsEfEl5P4a6z6XCNwgYRIhxos=",
+        "lastModified": 1790231260,
+        "narHash": "sha256-q/W1fOY/+LRL1NzIPEY0lQ/hIoSFu7lxnQlxJQVDaFQ=",
         "ref": "refs/heads/master",
-        "rev": "c89894b8dcac89aaea9ad72529524c1204884bc0",
-        "revCount": 3,
+        "rev": "b267e2fde780f6bd26fd4747a7a2df655788b7fc",
+        "revCount": 6,
         "type": "git",
         "url": "ssh://git@git.wntrmute.dev/kyle/tether"
       },
       "original": {
-        "rev": "c89894b8dcac89aaea9ad72529524c1204884bc0",
+        "rev": "b267e2fde780f6bd26fd4747a7a2df655788b7fc",
         "type": "git",
         "url": "ssh://git@git.wntrmute.dev/kyle/tether"
       }

flake.nix

@@ -26,7 +26,7 @@
     sift.url = "git+ssh://git@git.wntrmute.dev/kyle/sift?rev=4372673184f2e353c9b3e12d964951556c0ec2ff";
     sift.inputs.nixpkgs.follows = "nixpkgs";
 
-    tether.url = "git+ssh://git@git.wntrmute.dev/kyle/tether?rev=c89894b8dcac89aaea9ad72529524c1204884bc0";
+    tether.url = "git+ssh://git@git.wntrmute.dev/kyle/tether?rev=b267e2fde780f6bd26fd4747a7a2df655788b7fc";
     tether.inputs.nixpkgs.follows = "nixpkgs";
 
     lector.url = "github:kisom/lector";

hw/orion/metrics.nix

@@ -3,6 +3,11 @@
 { ... }:
 let
   honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight";
+  honchoRelabel = [
+    { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
+    # dialectic_calls is workspace-labelled; empty or hermes-fleet only.
+    { source_labels = [ "workspace_name" ]; regex = "|hermes-fleet"; action = "keep"; }
+  ];
 in
 {
   imports = [ ../../configs/metrics/exporters.nix ];
@@ -62,17 +67,13 @@ in
           job_name = "honcho-api";
           metrics_path = "/metrics";
           static_configs = [{ targets = [ "rift:8000" ]; }];
-          metric_relabel_configs = [
-            { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
-          ];
+          metric_relabel_configs = honchoRelabel;
         }
         {
           job_name = "honcho-deriver";
           metrics_path = "/metrics";
           static_configs = [{ targets = [ "rift:9090" ]; }];
-          metric_relabel_configs = [
-            { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
-          ];
+          metric_relabel_configs = honchoRelabel;
         }
       ];
     };
@@ -94,6 +95,15 @@ in
         group_wait = "30s";
         group_interval = "5m";
         repeat_interval = "4h";
+        routes = [
+          {
+            matchers = [ ''alertname="Watchdog"'' ];
+            receiver = "mattermost";
+            repeat_interval = "24h";
+            group_interval = "24h";
+            group_wait = "0s";
+          }
+        ];
       };
       receivers = [
         {
@@ -101,6 +111,7 @@ in
           slack_configs = [
             {
               api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook";
+              channel = "~alerts";
               send_resolved = true;
               title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}";
               text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}";
@@ -130,15 +141,15 @@ in
           rules = [
             {
               alert = "Dixie9BSlotsSaturated";
-              expr = ''min_over_time(llamacpp:requests_processing{job="llama-dixie-helper"}[10m]) >= 8'';
+              expr = ''llamacpp:requests_processing{job="llama-dixie-helper"} >= 8'';
               for = "10m";
               labels.severity = "page";
               annotations.summary = "dixie 9B slots busy for 10m";
             }
             {
               alert = "TetherCancelStorm";
-              # Needs titan's labelled tether series. Unlabelled sum is a
-              # double-count — never sum() a tether series without a selector.
+              # Tether b267e2f: labelled series exist. Never sum() without a selector
+              # (unlabelled totals sit next to them).
               expr = ''
                 sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m]))
                 /
@@ -183,6 +194,12 @@ in
         {
           name = "fleet-meta";
           rules = [
+            {
+              alert = "Watchdog";
+              expr = "vector(1)";
+              labels.severity = "heartbeat";
+              annotations.summary = "alerting pipeline is alive";
+            }
             {
               alert = "FleetTargetDown";
               expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0'';