From 495849348511c5f968ce1a6542e123a3c9e3db60 Mon Sep 17 00:00:00 2001 From: "K. Isom" Date: Wed, 23 Sep 2026 23:22:38 -0700 Subject: [PATCH] metrics: phase 1 orion+dixie (review, no switch) VictoriaMetrics on 127.0.0.1:8428. LoadCredential for the Mattermost webhook and the sift scrape token. Honcho scrapes keep a closed metric name set. Meta-alerts for target-down, VM size, orion disk. Dixie gets node_exporter, nvidia_gpu_exporter, and llama-server --metrics. Tether labelled series still wait on titan's fix. Rule 2 is written against reason=client_disconnect so it stays silent until then. --- configs/metrics/exporters.nix | 13 +++ hw/dixie/default.nix | 11 +- hw/orion/default.nix | 1 + hw/orion/metrics.nix | 212 ++++++++++++++++++++++++++++++++++ 4 files changed, 236 insertions(+), 1 deletion(-) create mode 100644 configs/metrics/exporters.nix create mode 100644 hw/orion/metrics.nix diff --git a/configs/metrics/exporters.nix b/configs/metrics/exporters.nix new file mode 100644 index 0000000..bb1de06 --- /dev/null +++ b/configs/metrics/exporters.nix @@ -0,0 +1,13 @@ +# Node exporter on tailscale0 only. openFirewall stays false (that hole is LAN-wide). +{ ... }: +{ + services.prometheus.exporters.node = { + enable = true; + port = 9100; + listenAddress = "0.0.0.0"; + enabledCollectors = [ "systemd" ]; + openFirewall = false; + }; + + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 9100 ]; +} diff --git a/hw/dixie/default.nix b/hw/dixie/default.nix index db7c711..4a71a00 100644 --- a/hw/dixie/default.nix +++ b/hw/dixie/default.nix @@ -116,6 +116,7 @@ let --port ${toString port} \ --models-preset ${preset} \ --models-max ${toString modelsMax} \ + --metrics \ --timeout 7200 ''; User = "llama-server"; @@ -233,6 +234,7 @@ in ./hardware-configuration.nix ./disk-config.nix inputs.tether.nixosModules.tether + ../../configs/metrics/exporters.nix ]; config = { @@ -323,7 +325,14 @@ in # Reachable only over the tailnet: 11434 (router), 11433 (vision), 11435 (tether) # on tailscale0, and SSH from the LAN (base config opens 22) for bring-up. - networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11433 11434 11435 ]; + networking.firewall.interfaces.tailscale0.allowedTCPPorts = [ 11433 11434 11435 9835 ]; + + services.prometheus.exporters.nvidia-gpu = { + enable = true; + listenAddress = "0.0.0.0"; + port = 9835; + openFirewall = false; + }; environment.systemPackages = with pkgs; [ llama-cpp diff --git a/hw/orion/default.nix b/hw/orion/default.nix index 4d8174e..b55aebd 100644 --- a/hw/orion/default.nix +++ b/hw/orion/default.nix @@ -6,6 +6,7 @@ # the disk is already provisioned. # ./disk-config.nix inputs.sift.nixosModules.sift + ./metrics.nix ]; config = { diff --git a/hw/orion/metrics.nix b/hw/orion/metrics.nix new file mode 100644 index 0000000..9805355 --- /dev/null +++ b/hw/orion/metrics.nix @@ -0,0 +1,212 @@ +# Phase 1 metrics store + alerting. VictoriaMetrics on loopback; scrape +# orion locally and dixie over the tailnet. Do not switch until reviewed. +{ ... }: +let + honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight"; +in +{ + imports = [ ../../configs/metrics/exporters.nix ]; + + services.victoriametrics = { + enable = true; + listenAddress = "127.0.0.1:8428"; + retentionPeriod = "90d"; + # bearer_token_file is a runtime credential; dry-run cannot see it. + checkConfig = false; + extraOptions = [ "-promscrape.maxScrapeSize=16MB" ]; + prometheusConfig = { + global.scrape_interval = "15s"; + scrape_configs = [ + { + job_name = "vm"; + static_configs = [{ targets = [ "127.0.0.1:8428" ]; }]; + } + { + job_name = "node"; + scrape_interval = "30s"; + static_configs = [ + { targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; } + { targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; } + ]; + } + { + job_name = "nvidia-dixie"; + static_configs = [{ targets = [ "dixie:9835" ]; }]; + } + { + job_name = "llama-dixie-helper"; + metrics_path = "/metrics"; + static_configs = [{ targets = [ "dixie:11434" ]; }]; + } + { + job_name = "llama-dixie-vision"; + metrics_path = "/metrics"; + static_configs = [{ targets = [ "dixie:11433" ]; }]; + } + { + job_name = "tether"; + metrics_path = "/tether/metrics"; + static_configs = [{ targets = [ "dixie:11435" ]; }]; + } + { + job_name = "sift"; + scheme = "https"; + metrics_path = "/metrics"; + bearer_token_file = "/run/credentials/victoriametrics.service/sift-metrics"; + static_configs = [{ targets = [ "orion.scylla-hammerhead.ts.net:8443" ]; }]; + } + # Phase 2 enablement is METRICS_ENABLED + publish :9090. Relabel is + # already here so a content-bearing label cannot land even if someone + # turns the scrape on early. up==0 until then; not in FleetTargetDown. + { + job_name = "honcho-api"; + metrics_path = "/metrics"; + static_configs = [{ targets = [ "rift:8000" ]; }]; + metric_relabel_configs = [ + { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; } + ]; + } + { + job_name = "honcho-deriver"; + metrics_path = "/metrics"; + static_configs = [{ targets = [ "rift:9090" ]; }]; + metric_relabel_configs = [ + { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; } + ]; + } + ]; + }; + }; + + systemd.services.victoriametrics.serviceConfig.LoadCredential = [ + "sift-metrics:/root/sift-metrics.token" + ]; + + services.prometheus.alertmanager = { + enable = true; + listenAddress = "127.0.0.1"; + port = 9093; + checkConfig = false; + configuration = { + route = { + receiver = "mattermost"; + group_by = [ "alertname" ]; + group_wait = "30s"; + group_interval = "5m"; + repeat_interval = "4h"; + }; + receivers = [ + { + name = "mattermost"; + slack_configs = [ + { + api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook"; + send_resolved = true; + title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}"; + text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}"; + } + ]; + } + ]; + }; + }; + + systemd.services.alertmanager.serviceConfig.LoadCredential = [ + "mattermost-webhook:/root/mattermost-webhook" + ]; + + services.vmalert.instances."" = { + enable = true; + settings = { + "datasource.url" = "http://127.0.0.1:8428"; + "notifier.url" = [ "http://127.0.0.1:9093" ]; + "httpListenAddr" = "127.0.0.1:8880"; + "evaluationInterval" = "15s"; + }; + rules = { + groups = [ + { + name = "fleet-phase1"; + rules = [ + { + alert = "Dixie9BSlotsSaturated"; + expr = ''min_over_time(llamacpp:requests_processing{job="llama-dixie-helper"}[10m]) >= 8''; + for = "10m"; + labels.severity = "page"; + annotations.summary = "dixie 9B slots busy for 10m"; + } + { + alert = "TetherCancelStorm"; + # Needs titan's labelled tether series. Unlabelled sum is a + # double-count — never sum() a tether series without a selector. + expr = '' + sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m])) + / + clamp_min(sum(rate(tether_requests_total{kind="converted"}[15m])), 1e-6) + > 0.5 + and + sum(rate(tether_requests_total{kind="converted"}[15m])) > 0.02 + ''; + for = "15m"; + labels.severity = "page"; + annotations.summary = "tether client-disconnect / converted > 50% over 15m"; + } + { + alert = "LlamaServerRestarting"; + expr = ''increase(node_systemd_service_restart_total{name=~"llama-server.service|llama-vision.service"}[15m]) > 0''; + labels.severity = "page"; + annotations.summary = "llama-server or llama-vision restarted"; + } + { + alert = "LlamaKVSaturated"; + expr = ''llamacpp:kv_cache_usage_ratio{job="llama-dixie-helper"} > 0.99''; + for = "5m"; + labels.severity = "page"; + annotations.summary = "dixie helper KV cache > 99%"; + } + { + alert = "GpuMemoryHigh"; + expr = "(nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes) > 0.95"; + for = "5m"; + labels.severity = "page"; + annotations.summary = "dixie GPU memory > 95%"; + } + { + alert = "GpuHot"; + expr = "nvidia_smi_temperature_gpu > 83"; + for = "2m"; + labels.severity = "page"; + annotations.summary = "dixie GPU > 83C"; + } + ]; + } + { + name = "fleet-meta"; + rules = [ + { + alert = "FleetTargetDown"; + expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0''; + for = "2m"; + labels.severity = "page"; + annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})"; + } + { + alert = "VictoriaMetricsDataTooLarge"; + expr = "sum(vm_data_size_bytes) > 21474836480"; + for = "15m"; + labels.severity = "page"; + annotations.summary = "VictoriaMetrics data > 20GiB"; + } + { + alert = "OrionRootDiskLow"; + expr = ''node_filesystem_avail_bytes{host="orion",mountpoint="/"} < 107374182400''; + for = "15m"; + labels.severity = "page"; + annotations.summary = "orion root < 100GiB free"; + } + ]; + } + ]; + }; + }; +}