kyle/imladris
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

metrics: phase 3 straylight/rift scrapes and hyperborea units

Browse Source 
Review, no switch. node-laptop for talos/titan is excluded from
FleetTargetDown. llama-straylight scrape uses params.model for
ornith-1.5-35b-a3b (router 400 without it; will load if cold).
This commit is contained in:
kyle
2026-09-24 00:42:06 -07:00
parent b861c6ee9f
commit 385529da39
6 changed files with 93 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hw/orion/metrics.nix
+35 -1
View File
@@ -32,6 +32,18 @@ in
static_configs = [
{ targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; }
{ targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; }
{ targets = [ "straylight:9100" ]; labels = { host = "straylight"; }; }
{ targets = [ "rift:9100" ]; labels = { host = "rift"; }; }
{ targets = [ "hyperborea:9100" ]; labels = { host = "hyperborea"; }; }
];
}
{
# Laptops sleep; scrape them but do not page on down.
job_name = "node-laptop";
scrape_interval = "30s";
static_configs = [
{ targets = [ "talos:9100" ]; labels = { host = "talos"; }; }
{ targets = [ "titan:9100" ]; labels = { host = "titan"; }; }
];
}
{
@@ -44,6 +56,14 @@ in
params.model = [ "ornith-1.5-9b-uncensored" ];
static_configs = [{ targets = [ "dixie:11434" ]; }];
}
{
job_name = "llama-straylight";
metrics_path = "/metrics";
# Router mode 400 without a model. 35B is the usual resident; scrape
# will load it if cold — drop params after switch if that happens.
params.model = [ "ornith-1.5-35b-a3b" ];
static_configs = [{ targets = [ "straylight:11434" ]; }];
}
{
job_name = "tether";
metrics_path = "/tether/metrics";
@@ -191,6 +211,20 @@ in
labels.severity = "page";
annotations.summary = "dixie GPU > 83C";
}
{
alert = "StraylightAmdgpuHot";
expr = ''node_hwmon_temp_celsius{chip=~".*amdgpu.*"} > 90'';
for = "2m";
labels.severity = "page";
annotations.summary = "straylight amdgpu > 90C";
}
{
alert = "StraylightGttHigh";
expr = "(gtt_used_bytes / gtt_total_bytes) > 0.95";
for = "5m";
labels.severity = "page";
annotations.summary = "straylight GTT > 95%";
}
];
}
{
@@ -204,7 +238,7 @@ in
}
{
alert = "FleetTargetDown";
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|tether|sift|honcho-api|honcho-deriver"} == 0'';
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-straylight|tether|sift|honcho-api|honcho-deriver"} == 0'';
for = "2m";
labels.severity = "page";
annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";