266 lines
9.6 KiB
Nix
266 lines
9.6 KiB
Nix
# Phase 1 metrics store + alerting. VictoriaMetrics on loopback; scrape
|
|
# orion locally and dixie over the tailnet. Do not switch until reviewed.
|
|
{ ... }:
|
|
let
|
|
honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight";
|
|
honchoRelabel = [
|
|
{ source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
|
|
# dialectic_calls is workspace-labelled; empty or hermes-fleet only.
|
|
{ source_labels = [ "workspace_name" ]; regex = "|hermes-fleet"; action = "keep"; }
|
|
];
|
|
in
|
|
{
|
|
imports = [ ../../configs/metrics/exporters.nix ];
|
|
|
|
services.victoriametrics = {
|
|
enable = true;
|
|
listenAddress = "127.0.0.1:8428";
|
|
retentionPeriod = "90d";
|
|
# bearer_token_file is a runtime credential; dry-run cannot see it.
|
|
checkConfig = false;
|
|
extraOptions = [ "-promscrape.maxScrapeSize=16MB" ];
|
|
prometheusConfig = {
|
|
global.scrape_interval = "15s";
|
|
scrape_configs = [
|
|
{
|
|
job_name = "vm";
|
|
static_configs = [{ targets = [ "127.0.0.1:8428" ]; }];
|
|
}
|
|
{
|
|
job_name = "node";
|
|
scrape_interval = "30s";
|
|
static_configs = [
|
|
{ targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; }
|
|
{ targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; }
|
|
{ targets = [ "straylight:9100" ]; labels = { host = "straylight"; }; }
|
|
{ targets = [ "rift:9100" ]; labels = { host = "rift"; }; }
|
|
{ targets = [ "hyperborea:9100" ]; labels = { host = "hyperborea"; }; }
|
|
];
|
|
}
|
|
{
|
|
# Laptops sleep; scrape them but do not page on down.
|
|
job_name = "node-laptop";
|
|
scrape_interval = "30s";
|
|
static_configs = [
|
|
{ targets = [ "talos:9100" ]; labels = { host = "talos"; }; }
|
|
{ targets = [ "titan:9100" ]; labels = { host = "titan"; }; }
|
|
];
|
|
}
|
|
{
|
|
job_name = "nvidia-dixie";
|
|
static_configs = [{ targets = [ "dixie:9835" ]; }];
|
|
}
|
|
{
|
|
job_name = "llama-dixie-helper";
|
|
metrics_path = "/metrics";
|
|
params.model = [ "ornith-1.5-9b-uncensored" ];
|
|
static_configs = [{ targets = [ "dixie:11434" ]; }];
|
|
}
|
|
{
|
|
job_name = "llama-straylight";
|
|
metrics_path = "/metrics";
|
|
# Router mode 400 without a model. 35B is the usual resident; scrape
|
|
# will load it if cold — drop params after switch if that happens.
|
|
params.model = [ "ornith-1.5-35b-a3b" ];
|
|
static_configs = [{ targets = [ "straylight:11434" ]; }];
|
|
}
|
|
{
|
|
job_name = "tether";
|
|
metrics_path = "/tether/metrics";
|
|
static_configs = [{ targets = [ "dixie:11435" ]; }];
|
|
}
|
|
{
|
|
job_name = "sift";
|
|
scheme = "https";
|
|
metrics_path = "/metrics";
|
|
bearer_token_file = "/run/credentials/victoriametrics.service/sift-metrics";
|
|
static_configs = [{ targets = [ "orion.scylla-hammerhead.ts.net:8443" ]; }];
|
|
}
|
|
# Phase 2 enablement is METRICS_ENABLED + publish :9090. Relabel is
|
|
# already here so a content-bearing label cannot land even if someone
|
|
# turns the scrape on early. up==0 until then; not in FleetTargetDown.
|
|
{
|
|
job_name = "honcho-api";
|
|
metrics_path = "/metrics";
|
|
static_configs = [{ targets = [ "rift:8000" ]; }];
|
|
metric_relabel_configs = honchoRelabel;
|
|
}
|
|
{
|
|
job_name = "honcho-deriver";
|
|
metrics_path = "/metrics";
|
|
static_configs = [{ targets = [ "rift:9090" ]; }];
|
|
metric_relabel_configs = honchoRelabel;
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
systemd.services.victoriametrics.serviceConfig.LoadCredential = [
|
|
"sift-metrics:/root/sift-metrics.token"
|
|
];
|
|
|
|
services.prometheus.alertmanager = {
|
|
enable = true;
|
|
listenAddress = "127.0.0.1";
|
|
port = 9093;
|
|
checkConfig = false;
|
|
configuration = {
|
|
route = {
|
|
receiver = "mattermost";
|
|
group_by = [ "alertname" ];
|
|
group_wait = "30s";
|
|
group_interval = "5m";
|
|
repeat_interval = "4h";
|
|
routes = [
|
|
{
|
|
matchers = [ ''alertname="Watchdog"'' ];
|
|
receiver = "mattermost";
|
|
repeat_interval = "24h";
|
|
group_interval = "24h";
|
|
group_wait = "0s";
|
|
}
|
|
];
|
|
};
|
|
receivers = [
|
|
{
|
|
name = "mattermost";
|
|
slack_configs = [
|
|
{
|
|
api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook";
|
|
send_resolved = true;
|
|
title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}";
|
|
text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}";
|
|
}
|
|
];
|
|
}
|
|
];
|
|
};
|
|
};
|
|
|
|
systemd.services.alertmanager.serviceConfig.LoadCredential = [
|
|
"mattermost-webhook:/root/mattermost-webhook"
|
|
];
|
|
|
|
services.vmalert.instances."" = {
|
|
enable = true;
|
|
settings = {
|
|
"datasource.url" = "http://127.0.0.1:8428";
|
|
"notifier.url" = [ "http://127.0.0.1:9093" ];
|
|
"httpListenAddr" = "127.0.0.1:8880";
|
|
"evaluationInterval" = "15s";
|
|
};
|
|
rules = {
|
|
groups = [
|
|
{
|
|
name = "fleet-phase1";
|
|
rules = [
|
|
{
|
|
alert = "Dixie9BSlotsSaturated";
|
|
expr = ''llamacpp:requests_processing{job="llama-dixie-helper"} >= 8'';
|
|
for = "10m";
|
|
labels.severity = "page";
|
|
annotations.summary = "dixie 9B slots busy for 10m";
|
|
}
|
|
{
|
|
alert = "TetherCancelStorm";
|
|
# Tether b267e2f: labelled series exist. Never sum() without a selector
|
|
# (unlabelled totals sit next to them).
|
|
expr = ''
|
|
sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m]))
|
|
/
|
|
clamp_min(sum(rate(tether_requests_total{kind="converted"}[15m])), 1e-6)
|
|
> 0.5
|
|
and
|
|
sum(rate(tether_requests_total{kind="converted"}[15m])) > 0.02
|
|
'';
|
|
for = "15m";
|
|
labels.severity = "page";
|
|
annotations.summary = "tether client-disconnect / converted > 50% over 15m";
|
|
}
|
|
{
|
|
alert = "HonchoDeriverBacklog";
|
|
expr = ''max(deriver_queue_items_pending) > 100 and max(deriv(deriver_queue_oldest_pending_age_seconds[30m])) > 0'';
|
|
for = "10m";
|
|
labels.severity = "page";
|
|
annotations.summary = "Honcho pending queue > 100 and oldest age rising";
|
|
}
|
|
{
|
|
alert = "LlamaServerRestarting";
|
|
expr = ''increase(node_systemd_service_restart_total{name=~"llama-server.service|llama-vision.service"}[15m]) > 0'';
|
|
labels.severity = "page";
|
|
annotations.summary = "llama-server or llama-vision restarted";
|
|
}
|
|
{
|
|
alert = "LlamaKVSaturated";
|
|
expr = ''llamacpp:kv_cache_usage_ratio{job="llama-dixie-helper"} > 0.99'';
|
|
for = "5m";
|
|
labels.severity = "page";
|
|
annotations.summary = "dixie helper KV cache > 99%";
|
|
}
|
|
{
|
|
alert = "GpuMemoryHigh";
|
|
expr = "(nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes) > 0.95";
|
|
for = "5m";
|
|
labels.severity = "page";
|
|
annotations.summary = "dixie GPU memory > 95%";
|
|
}
|
|
{
|
|
alert = "GpuHot";
|
|
expr = "nvidia_smi_temperature_gpu > 83";
|
|
for = "2m";
|
|
labels.severity = "page";
|
|
annotations.summary = "dixie GPU > 83C";
|
|
}
|
|
{
|
|
alert = "StraylightAmdgpuHot";
|
|
expr = ''node_hwmon_temp_celsius{chip=~".*amdgpu.*"} > 90'';
|
|
for = "2m";
|
|
labels.severity = "page";
|
|
annotations.summary = "straylight amdgpu > 90C";
|
|
}
|
|
{
|
|
alert = "StraylightGttHigh";
|
|
expr = "(gtt_used_bytes / gtt_total_bytes) > 0.95";
|
|
for = "5m";
|
|
labels.severity = "page";
|
|
annotations.summary = "straylight GTT > 95%";
|
|
}
|
|
];
|
|
}
|
|
{
|
|
name = "fleet-meta";
|
|
rules = [
|
|
{
|
|
alert = "Watchdog";
|
|
expr = "vector(1)";
|
|
labels.severity = "heartbeat";
|
|
annotations.summary = "alerting pipeline is alive";
|
|
}
|
|
{
|
|
alert = "FleetTargetDown";
|
|
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|tether|sift|honcho-api|honcho-deriver"} == 0'';
|
|
for = "2m";
|
|
labels.severity = "page";
|
|
annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";
|
|
}
|
|
{
|
|
alert = "VictoriaMetricsDataTooLarge";
|
|
expr = "sum(vm_data_size_bytes) > 21474836480";
|
|
for = "15m";
|
|
labels.severity = "page";
|
|
annotations.summary = "VictoriaMetrics data > 20GiB";
|
|
}
|
|
{
|
|
alert = "OrionRootDiskLow";
|
|
expr = ''node_filesystem_avail_bytes{host="orion",mountpoint="/"} < 107374182400'';
|
|
for = "15m";
|
|
labels.severity = "page";
|
|
annotations.summary = "orion root < 100GiB free";
|
|
}
|
|
];
|
|
}
|
|
];
|
|
};
|
|
};
|
|
}
|