metrics: phase 3 straylight/rift scrapes and hyperborea units
Review, no switch. node-laptop for talos/titan is excluded from FleetTargetDown. llama-straylight scrape uses params.model for ornith-1.5-35b-a3b (router 400 without it; will load if cold).
This commit is contained in:
+35
-1
@@ -32,6 +32,18 @@ in
|
||||
static_configs = [
|
||||
{ targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; }
|
||||
{ targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; }
|
||||
{ targets = [ "straylight:9100" ]; labels = { host = "straylight"; }; }
|
||||
{ targets = [ "rift:9100" ]; labels = { host = "rift"; }; }
|
||||
{ targets = [ "hyperborea:9100" ]; labels = { host = "hyperborea"; }; }
|
||||
];
|
||||
}
|
||||
{
|
||||
# Laptops sleep; scrape them but do not page on down.
|
||||
job_name = "node-laptop";
|
||||
scrape_interval = "30s";
|
||||
static_configs = [
|
||||
{ targets = [ "talos:9100" ]; labels = { host = "talos"; }; }
|
||||
{ targets = [ "titan:9100" ]; labels = { host = "titan"; }; }
|
||||
];
|
||||
}
|
||||
{
|
||||
@@ -44,6 +56,14 @@ in
|
||||
params.model = [ "ornith-1.5-9b-uncensored" ];
|
||||
static_configs = [{ targets = [ "dixie:11434" ]; }];
|
||||
}
|
||||
{
|
||||
job_name = "llama-straylight";
|
||||
metrics_path = "/metrics";
|
||||
# Router mode 400 without a model. 35B is the usual resident; scrape
|
||||
# will load it if cold — drop params after switch if that happens.
|
||||
params.model = [ "ornith-1.5-35b-a3b" ];
|
||||
static_configs = [{ targets = [ "straylight:11434" ]; }];
|
||||
}
|
||||
{
|
||||
job_name = "tether";
|
||||
metrics_path = "/tether/metrics";
|
||||
@@ -191,6 +211,20 @@ in
|
||||
labels.severity = "page";
|
||||
annotations.summary = "dixie GPU > 83C";
|
||||
}
|
||||
{
|
||||
alert = "StraylightAmdgpuHot";
|
||||
expr = ''node_hwmon_temp_celsius{chip=~".*amdgpu.*"} > 90'';
|
||||
for = "2m";
|
||||
labels.severity = "page";
|
||||
annotations.summary = "straylight amdgpu > 90C";
|
||||
}
|
||||
{
|
||||
alert = "StraylightGttHigh";
|
||||
expr = "(gtt_used_bytes / gtt_total_bytes) > 0.95";
|
||||
for = "5m";
|
||||
labels.severity = "page";
|
||||
annotations.summary = "straylight GTT > 95%";
|
||||
}
|
||||
];
|
||||
}
|
||||
{
|
||||
@@ -204,7 +238,7 @@ in
|
||||
}
|
||||
{
|
||||
alert = "FleetTargetDown";
|
||||
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|tether|sift|honcho-api|honcho-deriver"} == 0'';
|
||||
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-straylight|tether|sift|honcho-api|honcho-deriver"} == 0'';
|
||||
for = "2m";
|
||||
labels.severity = "page";
|
||||
annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
imports = [
|
||||
./hardware-configuration.nix
|
||||
./disk-config.nix
|
||||
../../configs/metrics/exporters.nix
|
||||
];
|
||||
|
||||
config = {
|
||||
|
||||
@@ -205,6 +205,7 @@ in
|
||||
./hardware-configuration.nix
|
||||
../../configs/desktop.nix
|
||||
../../configs/qemu.nix
|
||||
../../configs/metrics/straylight.nix
|
||||
];
|
||||
|
||||
config = {
|
||||
@@ -257,6 +258,7 @@ in
|
||||
--port 11434 \
|
||||
--models-preset ${llamaModelsIni} \
|
||||
--models-max 3 \
|
||||
--metrics \
|
||||
--timeout 7200
|
||||
'';
|
||||
User = "llama-server";
|
||||
|
||||
Reference in New Issue
Block a user