imladris/hw/orion/metrics.nix

# Phase 1 metrics store + alerting. VictoriaMetrics on loopback; scrape
# orion locally and dixie over the tailnet. Do not switch until reviewed.
{ ... }:
let
  honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight";
  honchoRelabel = [
    { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; }
    # dialectic_calls is workspace-labelled; empty or hermes-fleet only.
    { source_labels = [ "workspace_name" ]; regex = "|hermes-fleet"; action = "keep"; }
  ];
in
{
  imports = [ ../../configs/metrics/exporters.nix ];

  services.victoriametrics = {
    enable = true;
    listenAddress = "127.0.0.1:8428";
    retentionPeriod = "90d";
    # bearer_token_file is a runtime credential; dry-run cannot see it.
    checkConfig = false;
    extraOptions = [ "-promscrape.maxScrapeSize=16MB" ];
    prometheusConfig = {
      global.scrape_interval = "15s";
      scrape_configs = [
        {
          job_name = "vm";
          static_configs = [{ targets = [ "127.0.0.1:8428" ]; }];
        }
        {
          job_name = "node";
          scrape_interval = "30s";
          static_configs = [
            { targets = [ "127.0.0.1:9100" ]; labels = { host = "orion"; }; }
            { targets = [ "dixie:9100" ]; labels = { host = "dixie"; }; }
          ];
        }
        {
          job_name = "nvidia-dixie";
          static_configs = [{ targets = [ "dixie:9835" ]; }];
        }
        {
          job_name = "llama-dixie-helper";
          metrics_path = "/metrics";
          params.model = [ "ornith-1.5-9b-uncensored" ];
          static_configs = [{ targets = [ "dixie:11434" ]; }];
        }
        {
          job_name = "llama-dixie-vision";
          metrics_path = "/metrics";
          static_configs = [{ targets = [ "dixie:11433" ]; }];
        }
        {
          job_name = "tether";
          metrics_path = "/tether/metrics";
          static_configs = [{ targets = [ "dixie:11435" ]; }];
        }
        {
          job_name = "sift";
          scheme = "https";
          metrics_path = "/metrics";
          bearer_token_file = "/run/credentials/victoriametrics.service/sift-metrics";
          static_configs = [{ targets = [ "orion.scylla-hammerhead.ts.net:8443" ]; }];
        }
        # Phase 2 enablement is METRICS_ENABLED + publish :9090. Relabel is
        # already here so a content-bearing label cannot land even if someone
        # turns the scrape on early. up==0 until then; not in FleetTargetDown.
        {
          job_name = "honcho-api";
          metrics_path = "/metrics";
          static_configs = [{ targets = [ "rift:8000" ]; }];
          metric_relabel_configs = honchoRelabel;
        }
        {
          job_name = "honcho-deriver";
          metrics_path = "/metrics";
          static_configs = [{ targets = [ "rift:9090" ]; }];
          metric_relabel_configs = honchoRelabel;
        }
      ];
    };
  };

  systemd.services.victoriametrics.serviceConfig.LoadCredential = [
    "sift-metrics:/root/sift-metrics.token"
  ];

  services.prometheus.alertmanager = {
    enable = true;
    listenAddress = "127.0.0.1";
    port = 9093;
    checkConfig = false;
    configuration = {
      route = {
        receiver = "mattermost";
        group_by = [ "alertname" ];
        group_wait = "30s";
        group_interval = "5m";
        repeat_interval = "4h";
        routes = [
          {
            matchers = [ ''alertname="Watchdog"'' ];
            receiver = "mattermost";
            repeat_interval = "24h";
            group_interval = "24h";
            group_wait = "0s";
          }
        ];
      };
      receivers = [
        {
          name = "mattermost";
          slack_configs = [
            {
              api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook";
              send_resolved = true;
              title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}";
              text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}";
            }
          ];
        }
      ];
    };
  };

  systemd.services.alertmanager.serviceConfig.LoadCredential = [
    "mattermost-webhook:/root/mattermost-webhook"
  ];

  services.vmalert.instances."" = {
    enable = true;
    settings = {
      "datasource.url" = "http://127.0.0.1:8428";
      "notifier.url" = [ "http://127.0.0.1:9093" ];
      "httpListenAddr" = "127.0.0.1:8880";
      "evaluationInterval" = "15s";
    };
    rules = {
      groups = [
        {
          name = "fleet-phase1";
          rules = [
            {
              alert = "Dixie9BSlotsSaturated";
              expr = ''llamacpp:requests_processing{job="llama-dixie-helper"} >= 8'';
              for = "10m";
              labels.severity = "page";
              annotations.summary = "dixie 9B slots busy for 10m";
            }
            {
              alert = "TetherCancelStorm";
              # Tether b267e2f: labelled series exist. Never sum() without a selector
              # (unlabelled totals sit next to them).
              expr = ''
                sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m]))
                /
                clamp_min(sum(rate(tether_requests_total{kind="converted"}[15m])), 1e-6)
                > 0.5
                and
                sum(rate(tether_requests_total{kind="converted"}[15m])) > 0.02
              '';
              for = "15m";
              labels.severity = "page";
              annotations.summary = "tether client-disconnect / converted > 50% over 15m";
            }
            {
              alert = "HonchoDeriverBacklog";
              expr = ''max(deriver_queue_items_pending) > 100 and max(deriv(deriver_queue_oldest_pending_age_seconds[30m])) > 0'';
              for = "10m";
              labels.severity = "page";
              annotations.summary = "Honcho pending queue > 100 and oldest age rising";
            }
            {
              alert = "LlamaServerRestarting";
              expr = ''increase(node_systemd_service_restart_total{name=~"llama-server.service|llama-vision.service"}[15m]) > 0'';
              labels.severity = "page";
              annotations.summary = "llama-server or llama-vision restarted";
            }
            {
              alert = "LlamaKVSaturated";
              expr = ''llamacpp:kv_cache_usage_ratio{job="llama-dixie-helper"} > 0.99'';
              for = "5m";
              labels.severity = "page";
              annotations.summary = "dixie helper KV cache > 99%";
            }
            {
              alert = "GpuMemoryHigh";
              expr = "(nvidia_smi_memory_used_bytes / nvidia_smi_memory_total_bytes) > 0.95";
              for = "5m";
              labels.severity = "page";
              annotations.summary = "dixie GPU memory > 95%";
            }
            {
              alert = "GpuHot";
              expr = "nvidia_smi_temperature_gpu > 83";
              for = "2m";
              labels.severity = "page";
              annotations.summary = "dixie GPU > 83C";
            }
          ];
        }
        {
          name = "fleet-meta";
          rules = [
            {
              alert = "Watchdog";
              expr = "vector(1)";
              labels.severity = "heartbeat";
              annotations.summary = "alerting pipeline is alive";
            }
            {
              alert = "FleetTargetDown";
              expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift|honcho-api|honcho-deriver"} == 0'';
              for = "2m";
              labels.severity = "page";
              annotations.summary = "scrape target down ({{ $labels.job }} {{ $labels.instance }})";
            }
            {
              alert = "VictoriaMetricsDataTooLarge";
              expr = "sum(vm_data_size_bytes) > 21474836480";
              for = "15m";
              labels.severity = "page";
              annotations.summary = "VictoriaMetrics data > 20GiB";
            }
            {
              alert = "OrionRootDiskLow";
              expr = ''node_filesystem_avail_bytes{host="orion",mountpoint="/"} < 107374182400'';
              for = "15m";
              labels.severity = "page";
              annotations.summary = "orion root < 100GiB free";
            }
          ];
        }
      ];
    };
  };
}