diff --git a/configs/metrics/exporters.nix b/configs/metrics/exporters.nix index bb1de06..38bc406 100644 --- a/configs/metrics/exporters.nix +++ b/configs/metrics/exporters.nix @@ -6,6 +6,10 @@ port = 9100; listenAddress = "0.0.0.0"; enabledCollectors = [ "systemd" ]; + extraFlags = [ + "--collector.systemd.enable-restarts-metrics" + ''--collector.systemd.unit-include=(llama-server|llama-vision)\.service'' + ]; openFirewall = false; }; diff --git a/flake.lock b/flake.lock index 50de557..153f67d 100644 --- a/flake.lock +++ b/flake.lock @@ -497,16 +497,16 @@ ] }, "locked": { - "lastModified": 1790149025, - "narHash": "sha256-3EUf6ke80hm8o+YyS3wsEfEl5P4a6z6XCNwgYRIhxos=", + "lastModified": 1790231260, + "narHash": "sha256-q/W1fOY/+LRL1NzIPEY0lQ/hIoSFu7lxnQlxJQVDaFQ=", "ref": "refs/heads/master", - "rev": "c89894b8dcac89aaea9ad72529524c1204884bc0", - "revCount": 3, + "rev": "b267e2fde780f6bd26fd4747a7a2df655788b7fc", + "revCount": 6, "type": "git", "url": "ssh://git@git.wntrmute.dev/kyle/tether" }, "original": { - "rev": "c89894b8dcac89aaea9ad72529524c1204884bc0", + "rev": "b267e2fde780f6bd26fd4747a7a2df655788b7fc", "type": "git", "url": "ssh://git@git.wntrmute.dev/kyle/tether" } diff --git a/flake.nix b/flake.nix index 2ea702d..9391b2e 100644 --- a/flake.nix +++ b/flake.nix @@ -26,7 +26,7 @@ sift.url = "git+ssh://git@git.wntrmute.dev/kyle/sift?rev=4372673184f2e353c9b3e12d964951556c0ec2ff"; sift.inputs.nixpkgs.follows = "nixpkgs"; - tether.url = "git+ssh://git@git.wntrmute.dev/kyle/tether?rev=c89894b8dcac89aaea9ad72529524c1204884bc0"; + tether.url = "git+ssh://git@git.wntrmute.dev/kyle/tether?rev=b267e2fde780f6bd26fd4747a7a2df655788b7fc"; tether.inputs.nixpkgs.follows = "nixpkgs"; lector.url = "github:kisom/lector"; diff --git a/hw/orion/metrics.nix b/hw/orion/metrics.nix index 9805355..105d5bf 100644 --- a/hw/orion/metrics.nix +++ b/hw/orion/metrics.nix @@ -3,6 +3,11 @@ { ... }: let honchoKeep = "up|deriver_queue_items_pending|deriver_queue_oldest_pending_age_seconds|deriver_queue_work_units_claimed|deriver_queue_work_units_eligible|dialectic_calls.*|db_connections_open|db_queries_in_flight"; + honchoRelabel = [ + { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; } + # dialectic_calls is workspace-labelled; empty or hermes-fleet only. + { source_labels = [ "workspace_name" ]; regex = "|hermes-fleet"; action = "keep"; } + ]; in { imports = [ ../../configs/metrics/exporters.nix ]; @@ -62,17 +67,13 @@ in job_name = "honcho-api"; metrics_path = "/metrics"; static_configs = [{ targets = [ "rift:8000" ]; }]; - metric_relabel_configs = [ - { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; } - ]; + metric_relabel_configs = honchoRelabel; } { job_name = "honcho-deriver"; metrics_path = "/metrics"; static_configs = [{ targets = [ "rift:9090" ]; }]; - metric_relabel_configs = [ - { source_labels = [ "__name__" ]; regex = honchoKeep; action = "keep"; } - ]; + metric_relabel_configs = honchoRelabel; } ]; }; @@ -94,6 +95,15 @@ in group_wait = "30s"; group_interval = "5m"; repeat_interval = "4h"; + routes = [ + { + matchers = [ ''alertname="Watchdog"'' ]; + receiver = "mattermost"; + repeat_interval = "24h"; + group_interval = "24h"; + group_wait = "0s"; + } + ]; }; receivers = [ { @@ -101,6 +111,7 @@ in slack_configs = [ { api_url_file = "/run/credentials/alertmanager.service/mattermost-webhook"; + channel = "~alerts"; send_resolved = true; title = "{{ .Status | toUpper }} {{ .CommonLabels.alertname }}"; text = "{{ range .Alerts }}{{ .Annotations.summary }}{{ end }}"; @@ -130,15 +141,15 @@ in rules = [ { alert = "Dixie9BSlotsSaturated"; - expr = ''min_over_time(llamacpp:requests_processing{job="llama-dixie-helper"}[10m]) >= 8''; + expr = ''llamacpp:requests_processing{job="llama-dixie-helper"} >= 8''; for = "10m"; labels.severity = "page"; annotations.summary = "dixie 9B slots busy for 10m"; } { alert = "TetherCancelStorm"; - # Needs titan's labelled tether series. Unlabelled sum is a - # double-count — never sum() a tether series without a selector. + # Tether b267e2f: labelled series exist. Never sum() without a selector + # (unlabelled totals sit next to them). expr = '' sum(rate(tether_cancelled_total{reason="client_disconnect"}[15m])) / @@ -183,6 +194,12 @@ in { name = "fleet-meta"; rules = [ + { + alert = "Watchdog"; + expr = "vector(1)"; + labels.severity = "heartbeat"; + annotations.summary = "alerting pipeline is alive"; + } { alert = "FleetTargetDown"; expr = ''up{job=~"vm|node|nvidia-dixie|llama-dixie-helper|llama-dixie-vision|tether|sift"} == 0'';