metrics: phase 1 follow-up (review, no switch)

Honcho workspace_name keep, node_exporter restart metrics,
drop min_over_time on rule 1, Watchdog heartbeat, pin tether
to b267e2f so labelled series exist and rule 2 can fire.

hw/orion/metrics.nix

@@ -3,6 +3,11 @@
 { ... }:
 let
   honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight";
+  honchoRelabel = [
+    { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
+    # dialectic_calls is workspace-labelled; empty or hermes-fleet only.
+    { source_labels = [ "workspace_name" ]; regex = "|hermes-fleet"; action = "keep"; }
+  ];
 in
 {
   imports = [ ../../configs/metrics/exporters.nix ];
@@ -62,17 +67,13 @@ in
           job_name = "honcho-api";
           metrics_path = "/metrics";
           static_configs = [{ targets = [ "rift:8000" ]; }];
-          metric_relabel_configs = [
-            { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
-          ];
+          metric_relabel_configs = honchoRelabel;
         }
         {
           job_name = "honcho-deriver";
           metrics_path = "/metrics";
           static_configs = [{ targets = [ "rift:9090" ]; }];
-          metric_relabel_configs = [
-            { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
-          ];
+          metric_relabel_configs = honchoRelabel;
         }
       ];
     };
@@ -94,6 +95,15 @@ in
         group_wait = "30s";
         group_interval = "5m";
         repeat_interval = "4h";
+        routes = [
+          {
+            matchers = [ ''alertname="Watchdog"'' ];
+            receiver = "mattermost";
+            repeat_interval = "24h";
+            group_interval = "24h";
+            group_wait = "0s";
+          }
+        ];
       };
       receivers = [
         {
@@ -101,6 +111,7 @@ in
           slack_configs = [
             {
               api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook";
+              channel = "~alerts";
               send_resolved = true;
               title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}";
               text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}";
@@ -130,15 +141,15 @@ in
           rules = [
             {
               alert = "Dixie9BSlotsSaturated";
-              expr = ''min_over_time(llamacpp:requests_processing{job="llama-dixie-helper"}[10m]) >= 8'';
+              expr = ''llamacpp:requests_processing{job="llama-dixie-helper"} >= 8'';
               for = "10m";
               labels.severity = "page";
               annotations.summary = "dixie 9B slots busy for 10m";
             }
             {
               alert = "TetherCancelStorm";
-              # Needs titan's labelled tether series. Unlabelled sum is a
-              # double-count — never sum() a tether series without a selector.
+              # Tether b267e2f: labelled series exist. Never sum() without a selector
+              # (unlabelled totals sit next to them).
               expr = ''
                 sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m]))
                 /
@@ -183,6 +194,12 @@ in
         {
           name = "fleet-meta";
           rules = [
+            {
+              alert = "Watchdog";
+              expr = "vector(1)";
+              labels.severity = "heartbeat";
+              annotations.summary = "alerting pipeline is alive";
+            }
             {
               alert = "FleetTargetDown";
               expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0'';