kyle/imladris
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

metrics: phase 1 orion+dixie (review, no switch)

Browse Source 
VictoriaMetrics on 127.0.0.1:8428. LoadCredential for the Mattermost
webhook and the sift scrape token. Honcho scrapes keep a closed metric
name set. Meta-alerts for target-down, VM size, orion disk. Dixie gets
node_exporter, nvidia_gpu_exporter, and llama-server --metrics.

Tether labelled series still wait on titan's fix. Rule 2 is written
against reason=client_disconnect so it stays silent until then.
This commit is contained in:
kyle
2026-09-23 23:22:38 -07:00
parent 888e8e24fb
commit 4958493485
4 changed files with 236 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files
hw/orion/default.nix
+1
View File
@@ -6,6 +6,7 @@
# the disk is already provisioned.
# ./disk-config.nix
inputs.sift.nixosModules.sift
./metrics.nix
];
config = {
hw/orion/metrics.nix
+212
View File
@@ -0,0 +1,212 @@
# Phase 1 metrics store + alerting. VictoriaMetrics on loopback; scrape
# orion locally and dixie over the tailnet. Do not switch until reviewed.
{ ... }:
let
honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight";
in
{
imports = [ ../../configs/metrics/exporters.nix ];
services.victoriametrics = {
enable = true;
listenAddress = "127.0.0.1:8428";
retentionPeriod = "90d";
# bearer_token_file is a runtime credential; dry-run cannot see it.
checkConfig = false;
extraOptions = [ "-promscrape.maxScrapeSize=16MB" ];
prometheusConfig = {
global.scrape_interval = "15s";
scrape_configs = [
{
job_name = "vm";
static_configs = [{ targets = [ "127.0.0.1:8428" ]; }];
}
{
job_name = "node";
scrape_interval = "30s";
static_configs = [
{ targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; }
{ targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; }
];
}
{
job_name = "nvidia-dixie";
static_configs = [{ targets = [ "dixie:9835" ]; }];
}
{
job_name = "llama-dixie-helper";
metrics_path = "/metrics";
static_configs = [{ targets = [ "dixie:11434" ]; }];
}
{
job_name = "llama-dixie-vision";
metrics_path = "/metrics";
static_configs = [{ targets = [ "dixie:11433" ]; }];
}
{
job_name = "tether";
metrics_path = "/tether/metrics";
static_configs = [{ targets = [ "dixie:11435" ]; }];
}
{
job_name = "sift";
scheme = "https";
metrics_path = "/metrics";
bearer_token_file = "/run/credentials/victoriametrics.service/sift-metrics";
static_configs = [{ targets = [ "orion.scylla-hammerhead.ts.net:8443" ]; }];
}
# Phase 2 enablement is METRICS_ENABLED + publish :9090. Relabel is
# already here so a content-bearing label cannot land even if someone
# turns the scrape on early. up==0 until then; not in FleetTargetDown.
{
job_name = "honcho-api";
metrics_path = "/metrics";
static_configs = [{ targets = [ "rift:8000" ]; }];
metric_relabel_configs = [
{ source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
];
}
{
job_name = "honcho-deriver";
metrics_path = "/metrics";
static_configs = [{ targets = [ "rift:9090" ]; }];
metric_relabel_configs = [
{ source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
];
}
];
};
};
systemd.services.victoriametrics.serviceConfig.LoadCredential = [
"sift-metrics:/root/sift-metrics.token"
];
services.prometheus.alertmanager = {
enable = true;
listenAddress = "127.0.0.1";
port = 9093;
checkConfig = false;
configuration = {
route = {
receiver = "mattermost";
group_by = [ "alertname" ];
group_wait = "30s";
group_interval = "5m";
repeat_interval = "4h";
};
receivers = [
{
name = "mattermost";
slack_configs = [
{
api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook";
send_resolved = true;
title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}";
text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}";
}
];
}
];
};
};
systemd.services.alertmanager.serviceConfig.LoadCredential = [
"mattermost-webhook:/root/mattermost-webhook"
];
services.vmalert.instances."" = {
enable = true;
settings = {
"datasource.url" = "http://127.0.0.1:8428";
"notifier.url" = [ "http://127.0.0.1:9093" ];
"httpListenAddr" = "127.0.0.1:8880";
"evaluationInterval" = "15s";
};
rules = {
groups = [
{
name = "fleet-phase1";
rules = [
{
alert = "Dixie9BSlotsSaturated";
expr = ''min_over_time(llamacpp:requests_processing{job="llama-dixie-helper"}[10m]) >= 8'';
for = "10m";
labels.severity = "page";
annotations.summary = "dixie 9B slots busy for 10m";
}
{
alert = "TetherCancelStorm";
# Needs titan's labelled tether series. Unlabelled sum is a
# double-count — never sum() a tether series without a selector.
expr = ''
sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m]))
/
clamp_min(sum(rate(tether_requests_total{kind="converted"}[15m])), 1e-6)
> 0.5
and
sum(rate(tether_requests_total{kind="converted"}[15m])) > 0.02
'';
for = "15m";
labels.severity = "page";
annotations.summary = "tether client-disconnect / converted > 50% over 15m";
}
{
alert = "LlamaServerRestarting";
expr = ''increase(node_systemd_service_restart_total{name=~"llama-server.service|llama-vision.service"}[15m]) > 0'';
labels.severity = "page";
annotations.summary = "llama-server or llama-vision restarted";
}
{
alert = "LlamaKVSaturated";
expr = ''llamacpp:kv_cache_usage_ratio{job="llama-dixie-helper"} > 0.99'';
for = "5m";
labels.severity = "page";
annotations.summary = "dixie helper KV cache > 99%";
}
{
alert = "GpuMemoryHigh";
expr = "(nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes) > 0.95";
for = "5m";
labels.severity = "page";
annotations.summary = "dixie GPU memory > 95%";
}
{
alert = "GpuHot";
expr = "nvidia_smi_temperature_gpu > 83";
for = "2m";
labels.severity = "page";
annotations.summary = "dixie GPU > 83C";
}
];
}
{
name = "fleet-meta";
rules = [
{
alert = "FleetTargetDown";
expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0'';
for = "2m";
labels.severity = "page";
annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";
}
{
alert = "VictoriaMetricsDataTooLarge";
expr = "sum(vm_data_size_bytes) > 21474836480";
for = "15m";
labels.severity = "page";
annotations.summary = "VictoriaMetrics data > 20GiB";
}
{
alert = "OrionRootDiskLow";
expr = ''node_filesystem_avail_bytes{host="orion",mountpoint="/"} < 107374182400'';
for = "15m";
labels.severity = "page";
annotations.summary = "orion root < 100GiB free";
}
];
}
];
};
};
}