222 lines
9.3 KiB
Nix
222 lines
9.3 KiB
Nix
# dropsonde2 analysis machine (dixie)
|
|
#
|
|
# dixie is the fleet's analysis tier: Postgres/Timescale for the ADS-B/AIS buckets
|
|
# + labels + the eventual signal feed, DuckDB-over-Parquet for the 100-300 GB
|
|
# historical scans, and a sentiment encoder. The collectors stay on hyperborea
|
|
# (Pi 5, /srv/dropsonde2); this box just does the heavy lifting.
|
|
#
|
|
# The 5.5 TB SATA HDD (sda, WDC WD6003FZBX, serial V7G47PKH) is mounted by label
|
|
# as /srv/cold — cold Parquet + a local restore copy. /srv/analysis is a bind
|
|
# mount off the NVMe root (hot Parquet). Both are plain fileSystems, not disko:
|
|
# the disko root layout on the PM9A1 is untouched.
|
|
#
|
|
# The HDD wipe is the only destructive step and is authorized for sda only. It is
|
|
# NOT declarative — it runs once as an imperative step during deploy (see below).
|
|
#
|
|
# HDD wipe (authorized for sda = WDC WD6003FZBX-00K5WB0, serial V7G47PKH only):
|
|
# wipefs -a /dev/sda2 /dev/sda1 /dev/sda # partitions first, then the disk
|
|
# printf 'label: gpt\n,,L\n' | sfdisk /dev/sda # GPT + one Linux span
|
|
# udevadm settle
|
|
# mkfs.xfs -L ds2-cold /dev/sda1
|
|
# One GPT partition, XFS, label ds2-cold. sdb (stick) and nvme0n1 (root) untouched.
|
|
#
|
|
# Password: generated on dixie as root, never committed:
|
|
# umask 077; openssl rand -base64 32 > /root/dropsonde-pg
|
|
# The dropsonde-pg-setup service reads it via a systemd LoadCredential. Local
|
|
# analysis uses the unix socket as the dropsonde user (peer auth, no password).
|
|
# The only remote client is the future ADS-B collector on hyperborea; when that
|
|
# is built, copy /root/dropsonde-pg over ssh to
|
|
# /srv/dropsonde2/.env.pg (mode 600). Nobody needs to know the value.
|
|
|
|
{ inputs, pkgs, lib, config, ... }:
|
|
|
|
let
|
|
tailnet = "100.114.166.41"; # dixie's tailscale0
|
|
dropsondeRange = "100.64.0.0/10"; # tailnet CGNAT range
|
|
in
|
|
{
|
|
config = {
|
|
# --- dropsonde system user ------------------------------------------------
|
|
users.users.dropsonde = {
|
|
isSystemUser = true;
|
|
group = "dropsonde";
|
|
home = "/home/dropsonde";
|
|
createHome = true;
|
|
shell = pkgs.bash;
|
|
# Hyperborea's nightly sync key (collector -> dixie). The forced
|
|
# command runs rsync's own --server --daemon mode against the
|
|
# restricted config below — write-only (read only = false) modules
|
|
# rooted at the dropsonde2 dirs, no shell. pkgs.rsync only (no
|
|
# rrsync/python3 pulled onto dixie).
|
|
openssh.authorizedKeys.keys = [
|
|
''command="${pkgs.rsync}/bin/rsync --server --daemon --config=/etc/rsyncd-dropsonde.conf .",no-agent-forwarding,no-port-forwarding,no-pty,no-X11-forwarding ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKOGU2ZPL8d5iDWo/PIGRtulLbFqOL8XapgZPp1i8oii dropsonde-sync''
|
|
];
|
|
};
|
|
users.groups.dropsonde = {};
|
|
|
|
# --- rsync daemon-over-ssh for the sync key --------------------------------
|
|
# The dropsonde sync key's forced command runs `rsync --server --daemon`
|
|
# against this config: two write-only modules rooted at the dropsonde2
|
|
# subdirs. use chroot = false so /srv paths resolve directly; the daemon
|
|
# runs as dropsonde (the SSH user), so no uid/gid — setting them would
|
|
# make the non-root daemon fail at setgroups(). lock file is a global
|
|
# rsyncd parameter, so it sits at file scope (not per-module).
|
|
environment.etc."rsyncd-dropsonde.conf".text = ''
|
|
lock file = /home/dropsonde/rsyncd.lock
|
|
|
|
[cold]
|
|
path = /srv/cold/dropsonde2
|
|
read only = false
|
|
use chroot = false
|
|
max connections = 2
|
|
|
|
[analysis]
|
|
path = /srv/analysis/dropsonde2
|
|
read only = false
|
|
use chroot = false
|
|
max connections = 2
|
|
'';
|
|
|
|
# --- /srv/cold (5.5 TB HDD, by label) -------------------------------------
|
|
# The wipe+format of sda happens once during deploy; this entry just mounts
|
|
# by label with nofail so a missing disk never blocks boot.
|
|
# Label is ds2-cold (8 chars; XFS caps labels at 12). Not "dropsonde-cold":
|
|
# hyperborea's NVMe is already labelled "dropsonde", and duplicate labels
|
|
# across hosts is a footgun.
|
|
fileSystems."/srv/cold" = {
|
|
device = "/dev/disk/by-label/ds2-cold";
|
|
fsType = "xfs";
|
|
options = [ "noatime" "nofail" "x-systemd.device-timeout=30s" ];
|
|
};
|
|
|
|
# --- /srv/analysis (hot Parquet on the NVMe root) -------------------------
|
|
# Bind mount off the NVMe so hot Parquet lives on the fast disk, not the
|
|
# spinning HDD. tmpfiles provisions the bind source directory.
|
|
fileSystems."/srv/analysis" = {
|
|
device = "/var/lib/dropsonde-analysis";
|
|
fsType = "none";
|
|
options = [ "bind" "nofail" ];
|
|
};
|
|
systemd.tmpfiles.rules = [
|
|
"d /var/lib/dropsonde-analysis 0755 dropsonde dropsonde"
|
|
"d /srv/cold/dropsonde2 0755 dropsonde dropsonde"
|
|
"d /srv/analysis/dropsonde2 0755 dropsonde dropsonde"
|
|
# rsync created these trees 0700 (dropsonde-only) before --chmod was
|
|
# added; Z relabels mode+ownership recursively at activation. ~0755
|
|
# masks the mode against existing bits: dirs become 0755, files stay
|
|
# 0644 (exec dropped since none were set).
|
|
"Z /srv/cold/dropsonde2 ~0755 dropsonde dropsonde - -"
|
|
"Z /srv/analysis/dropsonde2 ~0755 dropsonde dropsonde - -"
|
|
];
|
|
|
|
# --- analysis.slice: bounded, best-effort, below llama-server -------------
|
|
# CPUQuota 600% = 6 of 16 threads; MemoryMax 40G leaves ~24G for llama.
|
|
# Launch dropsonde analysis under this slice with:
|
|
# systemd-run --slice=analysis.slice --property=Slice=analysis.slice <cmd>
|
|
systemd.slices.analysis = {
|
|
sliceConfig = {
|
|
CPUQuota = "600%";
|
|
MemoryMax = "40G";
|
|
Nice = 10;
|
|
IOWeight = 50;
|
|
};
|
|
};
|
|
|
|
# --- llama.slice: keep the helper tier above default CPUWeight ------------
|
|
# The router's three resident models (5.6 + 2.5 + 0.7 GB) share the 12 GB
|
|
# card; analysis must not starve them. Default CPUWeight is 1000; 2000 gives
|
|
# it roughly double the fair-share. Wired into the llama-server service below.
|
|
systemd.slices.llama = {
|
|
sliceConfig = {
|
|
CPUWeight = 2000;
|
|
};
|
|
};
|
|
|
|
# --- analysis packages ----------------------------------------------------
|
|
# DuckDB CLI only. The analysis layer is DuckDB + Parquet; Python (pyarrow,
|
|
# polars) drags in arrow-cpp + llvm-dev (~1 GB of downloads) and is avoided
|
|
# unless a specific job forces it. Add it back per-job when needed.
|
|
environment.systemPackages = with pkgs; [
|
|
duckdb
|
|
];
|
|
|
|
# --- Postgres 17 + Timescale + PostGIS + h3-pg ----------------------------
|
|
services.postgresql = {
|
|
enable = true;
|
|
package = pkgs.postgresql_17;
|
|
|
|
# Listen on all interfaces; the firewall opens 5432 on tailscale0 only, so
|
|
# nothing else is exposed. Binding to the tailnet IP directly would race
|
|
# tailscale0 coming up after postgres at boot and leave the service down
|
|
# until a manual restart.
|
|
enableTCPIP = true;
|
|
|
|
settings = {
|
|
listen_addresses = "*";
|
|
shared_preload_libraries = "timescaledb";
|
|
shared_buffers = "8GB";
|
|
work_mem = "256MB";
|
|
maintenance_work_mem = "2GB";
|
|
max_wal_size = "4GB";
|
|
};
|
|
|
|
# Extensions installed into the pg17 package. All three exist for 17.11 in
|
|
# the pinned nixpkgs (verified: timescaledb 2.27.1, postgis 3.6.4, h3-pg
|
|
# 4.2.3).
|
|
extensions = [
|
|
pkgs.postgresql17Packages.timescaledb
|
|
pkgs.postgresql17Packages.postgis
|
|
pkgs.postgresql17Packages."h3-pg"
|
|
];
|
|
|
|
ensureDatabases = [ "dropsonde" ];
|
|
ensureUsers = [
|
|
{
|
|
name = "dropsonde";
|
|
ensureDBOwnership = true; # owns the dropsonde db, so it can CREATE EXTENSION
|
|
}
|
|
];
|
|
|
|
# Allow the dropsonde role to connect over TCP from the tailnet with scram.
|
|
# The default localhost md5 lines stay for local admin access.
|
|
authentication = lib.mkAfter ''
|
|
host dropsonde dropsonde ${dropsondeRange} scram-sha-256
|
|
'';
|
|
};
|
|
|
|
# Create the extensions in the dropsonde db once postgres is up and the db +
|
|
# role exist. Idempotent (IF NOT EXISTS) so redeploys are safe. The password
|
|
# is generated on dixie (see note) and read out of git via a systemd
|
|
# LoadCredential; psql interpolates it as a bound variable so it never shows
|
|
# in the process list or journal.
|
|
systemd.services.dropsonde-pg-setup = {
|
|
description = "dropsonde2: set dropsonde role password + create extensions";
|
|
wants = [ "postgresql-setup.service" ];
|
|
after = [ "postgresql-setup.service" "postgresql.service" ];
|
|
path = [ config.services.postgresql.package ];
|
|
serviceConfig = {
|
|
Type = "oneshot";
|
|
RemainAfterExit = true;
|
|
User = "postgres";
|
|
Group = "postgres";
|
|
Restart = "on-failure";
|
|
RestartSec = "10s";
|
|
LoadCredential = [ "dropsonde-pg-password:/root/dropsonde-pg" ];
|
|
};
|
|
script = ''
|
|
pw=$(cat "$CREDENTIALS_DIRECTORY/dropsonde-pg-password")
|
|
psql -v ON_ERROR_STOP=1 -v pw="$pw" -d postgres <<'SQL'
|
|
ALTER ROLE dropsonde WITH LOGIN PASSWORD :'pw';
|
|
SQL
|
|
psql -v ON_ERROR_STOP=1 -d dropsonde <<'SQL'
|
|
CREATE EXTENSION IF NOT EXISTS timescaledb;
|
|
CREATE EXTENSION IF NOT EXISTS postgis;
|
|
CREATE EXTENSION IF NOT EXISTS h3;
|
|
CREATE EXTENSION IF NOT EXISTS h3_postgis CASCADE;
|
|
SQL
|
|
'';
|
|
wantedBy = [ "multi-user.target" ];
|
|
};
|
|
};
|
|
}
|