diff --git a/hw/dixie/analysis.nix b/hw/dixie/analysis.nix new file mode 100644 index 0000000..40bcbf6 --- /dev/null +++ b/hw/dixie/analysis.nix @@ -0,0 +1,150 @@ +# dropsonde2 analysis machine (dixie) +# +# dixie is the fleet's analysis tier: Postgres/Timescale for the ADS-B/AIS buckets +# + labels + the eventual signal feed, DuckDB-over-Parquet for the 100-300 GB +# historical scans, and a sentiment encoder. The collectors stay on hyperborea +# (Pi 5, /srv/dropsonde2); this box just does the heavy lifting. +# +# The 5.5 TB SATA HDD (sda, WDC WD6003FZBX, serial V7G47PKH) is mounted by label +# as /srv/cold — cold Parquet + a local restore copy. /srv/analysis is a bind +# mount off the NVMe root (hot Parquet). Both are plain fileSystems, not disko: +# the disko root layout on the PM9A1 is untouched. +# +# The HDD wipe is the only destructive step and is authorized for sda only. It is +# NOT declarative — it runs once as an imperative step during deploy (see below). +# +# Password: kept out of git. dixie writes the plaintext dropsonde password to +# /root/dropsonde-pg (mode 600, root-owned) — a file that is never committed. +# The dropsonde-pg-setup service reads it via a systemd LoadCredential and runs +# ALTER ROLE + CREATE EXTENSION IF NOT EXISTS. See the note at the bottom. + +{ inputs, pkgs, lib, config, ... }: + +let + tailnet = "100.114.166.41"; # dixie's tailscale0 + dropsondeRange = "100.64.0.0/10"; # tailnet CGNAT range +in +{ + config = { + # --- dropsonde system user ------------------------------------------------ + users.users.dropsonde = { + isSystemUser = true; + group = "dropsonde"; + home = "/home/dropsonde"; + createHome = true; + shell = pkgs.bash; + }; + users.groups.dropsonde = {}; + + # --- /srv/cold (5.5 TB HDD, by label) ------------------------------------- + # The wipe+format of sda happens once during deploy; this entry just mounts + # by label with nofail so a missing disk never blocks boot. + fileSystems."/srv/cold" = { + device = "/dev/disk/by-label/dropsonde-cold"; + fsType = "xfs"; + options = [ "noatime" "nofail" "x-systemd.device-timeout=30s" ]; + }; + + # --- /srv/analysis (hot Parquet on the NVMe root) ------------------------- + # Bind mount off the NVMe so hot Parquet lives on the fast disk, not the + # spinning HDD. tmpfiles provisions the bind source directory. + fileSystems."/srv/analysis" = { + device = "/var/lib/dropsonde-analysis"; + fsType = "none"; + options = [ "bind" "nofail" ]; + }; + systemd.tmpfiles.rules = [ + "d /var/lib/dropsonde-analysis 0755 dropsonde dropsonde" + ]; + + # --- analysis.slice: bounded, best-effort, below llama-server ------------- + # CPUQuota 600% = 6 of 16 threads; MemoryMax 40G leaves ~24G for llama. + # Launch dropsonde analysis under this slice with: + # systemd-run --slice=analysis.slice --property=Slice=analysis.slice + systemd.slices.analysis = { + serviceConfig = { + CPUQuota = "600%"; + MemoryMax = "40G"; + Nice = 10; + IOWeight = 50; + }; + }; + + # --- llama.slice: keep the helper tier above default CPUWeight ------------ + # The router's three resident models (5.6 + 2.5 + 0.7 GB) share the 12 GB + # card; analysis must not starve them. Default CPUWeight is 1000; 2000 gives + # it roughly double the fair-share. Wired into the llama-server service below. + systemd.slices.llama = { + serviceConfig = { + CPUWeight = 2000; + }; + }; + + # --- analysis packages ---------------------------------------------------- + environment.systemPackages = with pkgs; [ + duckdb + (python3.withPackages (p: with p; [ duckdb pyarrow polars ])) + ]; + + # --- Postgres 17 + Timescale + PostGIS + h3-pg ---------------------------- + services.postgresql = { + enable = true; + package = pkgs.postgresql_17; + + # Listen on the tailnet address only (not 0.0.0.0). enableTCPIP is required + # for any TCP bind; listen_addresses below scopes it to tailscale0. + enableTCPIP = true; + listen_addresses = lib.mkForce tailnet; + + settings = { + shared_preload_libraries = "timescaledb"; + shared_buffers = "8GB"; + work_mem = "256MB"; + maintenance_work_mem = "2GB"; + max_wal_size = "4GB"; + }; + + # Extensions installed into the pg17 package. All three exist for 17.11 in + # the pinned nixpkgs (verified: timescaledb 2.27.1, postgis 3.6.4, h3-pg + # 4.2.3). + extensions = [ + pkgs.postgresql17Packages.timescaledb + pkgs.postgresql17Packages.postgis + pkgs.postgresql17Packages."h3-pg" + ]; + + ensureDatabases = [ "dropsonde" ]; + ensureUsers = [ + { + name = "dropsonde"; + ensureDBOwnership = true; # owns the dropsonde db, so it can CREATE EXTENSION + } + ]; + + # Allow the dropsonde role to connect over TCP from the tailnet with scram. + # The default localhost md5 lines stay for local admin access. + authentication = lib.mkAfter '' + host dropsonde dropsonde ${dropsondeRange} scram-sha-256 + ''; + }; + + # Create the extensions in the dropsonde db once postgres is up and the db + + # role exist. Idempotent (IF NOT EXISTS) so redeploys are safe. The password + # is read out of git via a systemd LoadCredential (see note). + systemd.services.dropsonde-pg-setup = { + description = "dropsonde2: set dropsonde role password + create extensions"; + wants = [ "postgresql-setup.service" ]; + after = [ "postgresql-setup.service" "postgresql.service" ]; + serviceConfig = { + Type = "oneshot"; + RemainAfterExit = true; + User = "postgres"; + Group = "postgres"; + LoadCredential = [ "dropsonde-pg-password:/root/dropsonde-pg" ]; + ExecStart = ''${pkgs.postgresql}/bin/psql -v ON_ERROR_STOP=1 -d postgres -c "ALTER ROLE dropsonde WITH LOGIN PASSWORD '${''${CREDENTIALS_DIRECTORY}/dropsonde-pg-password}';"'' + + ''${pkgs.postgresql}/bin/psql -v ON_ERROR_STOP=1 -d dropsonde -c "CREATE EXTENSION IF NOT EXISTS timescaledb;" -c "CREATE EXTENSION IF NOT EXISTS postgis;" -c "CREATE EXTENSION IF NOT EXISTS h3;" -c "CREATE EXTENSION IF NOT EXISTS h3_postgis CASCADE;''; + }; + wantedBy = [ "multi-user.target" ]; + }; + }; +} diff --git a/hw/dixie/default.nix b/hw/dixie/default.nix index 4a71a00..8915a6c 100644 --- a/hw/dixie/default.nix +++ b/hw/dixie/default.nix @@ -124,6 +124,9 @@ let StateDirectory = "llama-server"; WorkingDirectory = "/var/lib/llama-server"; SupplementaryGroups = [ "video" ]; + # Own slice, above default CPUWeight (set in analysis.nix) so the + # analysis.slice workloads never starve the helper-tier router. + Slice = "llama.service"; Restart = "on-failure"; RestartSec = "10s"; TimeoutStartSec = "600";