metrics: phase 3 straylight/rift scrapes and hyperborea units
Review, no switch. node-laptop for talos/titan is excluded from FleetTargetDown. llama-straylight scrape uses params.model for ornith-1.5-35b-a3b (router 400 without it; will load if cold).
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
# /etc/systemd/system/prometheus-node-exporter.service.d/override.conf
|
||||
[Unit]
|
||||
After=tailscaled.service
|
||||
Wants=tailscaled.service
|
||||
|
||||
[Service]
|
||||
Restart=on-failure
|
||||
@@ -0,0 +1,3 @@
|
||||
# /etc/default/prometheus-node-exporter on hyperborea (Debian).
|
||||
# Bind the tailscale IP only. LAN 192.168.88.154:9100 must stay closed.
|
||||
ARGS="--web.listen-address=100.112.40.10:9100"
|
||||
@@ -0,0 +1,45 @@
|
||||
# Straylight node_exporter extras: amdgpu hwmon/DRM plus GTT textfile gauges.
|
||||
{ pkgs, ... }:
|
||||
{
|
||||
imports = [ ./exporters.nix ];
|
||||
|
||||
services.prometheus.exporters.node = {
|
||||
enabledCollectors = [ "hwmon" "drm" "textfile" ];
|
||||
extraFlags = [
|
||||
"--collector.textfile.directory=/var/lib/prometheus-node-exporter"
|
||||
];
|
||||
};
|
||||
|
||||
systemd.tmpfiles.rules = [
|
||||
"d /var/lib/prometheus-node-exporter 0755 root root -"
|
||||
];
|
||||
|
||||
systemd.services.node-exporter-gtt = {
|
||||
description = "Write AMD GTT gauges for node_exporter textfile";
|
||||
serviceConfig.Type = "oneshot";
|
||||
script = ''
|
||||
set -eu
|
||||
d=/var/lib/prometheus-node-exporter
|
||||
total=$(${pkgs.coreutils}/bin/cat /sys/class/drm/card1/device/mem_info_gtt_total)
|
||||
used=$(${pkgs.coreutils}/bin/cat /sys/class/drm/card1/device/mem_info_gtt_used)
|
||||
tmp="$d/gtt.prom.$$"
|
||||
${pkgs.coreutils}/bin/printf '%s\n' \
|
||||
'# HELP gtt_total_bytes AMD GTT size in bytes' \
|
||||
'# TYPE gtt_total_bytes gauge' \
|
||||
"gtt_total_bytes $total" \
|
||||
'# HELP gtt_used_bytes AMD GTT used in bytes' \
|
||||
'# TYPE gtt_used_bytes gauge' \
|
||||
"gtt_used_bytes $used" > "$tmp"
|
||||
${pkgs.coreutils}/bin/mv "$tmp" "$d/gtt.prom"
|
||||
'';
|
||||
};
|
||||
|
||||
systemd.timers.node-exporter-gtt = {
|
||||
wantedBy = [ "timers.target" ];
|
||||
timerConfig = {
|
||||
OnBootSec = "15s";
|
||||
OnUnitActiveSec = "15s";
|
||||
AccuracySec = "1s";
|
||||
};
|
||||
};
|
||||
}
|
||||
Reference in New Issue
Block a user