kyle/imladris
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

metrics: HonchoDeriverBacklog and include honcho in target-down

Browse Source
This commit is contained in:
kyle
2026-09-24 00:21:36 -07:00
parent 5d8e8c4335
commit ed5e1a1e0e
1 changed files with 8 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hw/orion/metrics.nix
+8 -1
View File
@@ -162,6 +162,13 @@ in
labels.severity = "page";
annotations.summary = "tether client-disconnect / converted > 50% over 15m";
}
{
alert = "HonchoDeriverBacklog";
expr = ''max(deriver_queue_items_pending) > 100 and max(deriv(deriver_queue_oldest_pending_age_seconds[30m])) > 0'';
for = "10m";
labels.severity = "page";
annotations.summary = "Honcho pending queue > 100 and oldest age rising";
}
{
alert = "LlamaServerRestarting";
expr = ''increase(node_systemd_service_restart_total{name=~"llama-server.service|llama-vision.service"}[15m]) > 0'';
@@ -202,7 +209,7 @@ in
}
{
alert = "FleetTargetDown";
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0'';
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift|honcho-api|honcho-deriver"} == 0'';
for = "2m";
labels.severity = "page";
annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";