From ed5e1a1e0e3cd9144d5bbfbfa841fa0d2c8dd50f Mon Sep 17 00:00:00 2001
From: "K. Isom" <kyle@imap.cc>
Date: Thu, 24 Sep 2026 00:21:36 -0700
Subject: [PATCH] metrics: HonchoDeriverBacklog and include honcho in
 target-down

---
 hw/orion/metrics.nix | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/hw/orion/metrics.nix b/hw/orion/metrics.nix
index 389ee31..b8a59a2 100644
--- a/hw/orion/metrics.nix
+++ b/hw/orion/metrics.nix
@@ -162,6 +162,13 @@ in
               labels.severity = "page";
               annotations.summary = "tether client-disconnect / converted > 50% over 15m";
             }
+            {
+              alert = "HonchoDeriverBacklog";
+              expr = ''max(deriver_queue_items_pending) > 100 and max(deriv(deriver_queue_oldest_pending_age_seconds[30m])) > 0'';
+              for = "10m";
+              labels.severity = "page";
+              annotations.summary = "Honcho pending queue > 100 and oldest age rising";
+            }
             {
               alert = "LlamaServerRestarting";
               expr = ''increase(node_systemd_service_restart_total{name=~"llama-server.service|llama-vision.service"}[15m]) > 0'';
@@ -202,7 +209,7 @@ in
             }
             {
               alert = "FleetTargetDown";
-              expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0'';
+              expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift|honcho-api|honcho-deriver"} == 0'';
               for = "2m";
               labels.severity = "page";
               annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";