dixie: dropsonde2 analysis tier (HDD /srv/cold, pg17+timescale+postgis+h3, analysis.slice, duckdb/polars)

2026-09-24 22:03:44 -07:00
parent c12b2adbfd
commit 3c2a272722
2 changed files with 153 additions and 0 deletions
@@ -0,0 +1,150 @@
+# dropsonde2 analysis machine (dixie)
+#
+# dixie is the fleet's analysis tier: Postgres/Timescale for the ADS-B/AIS buckets
+# + labels + the eventual signal feed, DuckDB-over-Parquet for the 100-300 GB
+# historical scans, and a sentiment encoder. The collectors stay on hyperborea
+# (Pi 5, /srv/dropsonde2); this box just does the heavy lifting.
+#
+# The 5.5 TB SATA HDD (sda, WDC WD6003FZBX, serial V7G47PKH) is mounted by label
+# as /srv/cold — cold Parquet + a local restore copy. /srv/analysis is a bind
+# mount off the NVMe root (hot Parquet). Both are plain fileSystems, not disko:
+# the disko root layout on the PM9A1 is untouched.
+#
+# The HDD wipe is the only destructive step and is authorized for sda only. It is
+# NOT declarative — it runs once as an imperative step during deploy (see below).
+#
+# Password: kept out of git. dixie writes the plaintext dropsonde password to
+# /root/dropsonde-pg (mode 600, root-owned) — a file that is never committed.
+# The dropsonde-pg-setup service reads it via a systemd LoadCredential and runs
+# ALTER ROLE + CREATE EXTENSION IF NOT EXISTS. See the note at the bottom.
+
+{ inputs, pkgs, lib, config, ... }:
+
+let
+  tailnet = "100.114.166.41";  # dixie's tailscale0
+  dropsondeRange = "100.64.0.0/10";  # tailnet CGNAT range
+in
+{
+  config = {
+    # --- dropsonde system user ------------------------------------------------
+    users.users.dropsonde = {
+      isSystemUser = true;
+      group = "dropsonde";
+      home = "/home/dropsonde";
+      createHome = true;
+      shell = pkgs.bash;
+    };
+    users.groups.dropsonde = {};
+
+    # --- /srv/cold (5.5 TB HDD, by label) -------------------------------------
+    # The wipe+format of sda happens once during deploy; this entry just mounts
+    # by label with nofail so a missing disk never blocks boot.
+    fileSystems."/srv/cold" = {
+      device = "/dev/disk/by-label/dropsonde-cold";
+      fsType = "xfs";
+      options = [ "noatime" "nofail" "x-systemd.device-timeout=30s" ];
+    };
+
+    # --- /srv/analysis (hot Parquet on the NVMe root) -------------------------
+    # Bind mount off the NVMe so hot Parquet lives on the fast disk, not the
+    # spinning HDD. tmpfiles provisions the bind source directory.
+    fileSystems."/srv/analysis" = {
+      device = "/var/lib/dropsonde-analysis";
+      fsType = "none";
+      options = [ "bind" "nofail" ];
+    };
+    systemd.tmpfiles.rules = [
+      "d /var/lib/dropsonde-analysis 0755 dropsonde dropsonde"
+    ];
+
+    # --- analysis.slice: bounded, best-effort, below llama-server -------------
+    # CPUQuota 600% = 6 of 16 threads; MemoryMax 40G leaves ~24G for llama.
+    # Launch dropsonde analysis under this slice with:
+    #   systemd-run --slice=analysis.slice --property=Slice=analysis.slice <cmd>
+    systemd.slices.analysis = {
+      serviceConfig = {
+        CPUQuota = "600%";
+        MemoryMax = "40G";
+        Nice = 10;
+        IOWeight = 50;
+      };
+    };
+
+    # --- llama.slice: keep the helper tier above default CPUWeight ------------
+    # The router's three resident models (5.6 + 2.5 + 0.7 GB) share the 12 GB
+    # card; analysis must not starve them. Default CPUWeight is 1000; 2000 gives
+    # it roughly double the fair-share. Wired into the llama-server service below.
+    systemd.slices.llama = {
+      serviceConfig = {
+        CPUWeight = 2000;
+      };
+    };
+
+    # --- analysis packages ----------------------------------------------------
+    environment.systemPackages = with pkgs; [
+      duckdb
+      (python3.withPackages (p: with p; [ duckdb pyarrow polars ]))
+    ];
+
+    # --- Postgres 17 + Timescale + PostGIS + h3-pg ----------------------------
+    services.postgresql = {
+      enable = true;
+      package = pkgs.postgresql_17;
+
+      # Listen on the tailnet address only (not 0.0.0.0). enableTCPIP is required
+      # for any TCP bind; listen_addresses below scopes it to tailscale0.
+      enableTCPIP = true;
+      listen_addresses = lib.mkForce tailnet;
+
+      settings = {
+        shared_preload_libraries = "timescaledb";
+        shared_buffers = "8GB";
+        work_mem = "256MB";
+        maintenance_work_mem = "2GB";
+        max_wal_size = "4GB";
+      };
+
+      # Extensions installed into the pg17 package. All three exist for 17.11 in
+      # the pinned nixpkgs (verified: timescaledb 2.27.1, postgis 3.6.4, h3-pg
+      # 4.2.3).
+      extensions = [
+        pkgs.postgresql17Packages.timescaledb
+        pkgs.postgresql17Packages.postgis
+        pkgs.postgresql17Packages."h3-pg"
+      ];
+
+      ensureDatabases = [ "dropsonde" ];
+      ensureUsers = [
+        {
+          name = "dropsonde";
+          ensureDBOwnership = true;  # owns the dropsonde db, so it can CREATE EXTENSION
+        }
+      ];
+
+      # Allow the dropsonde role to connect over TCP from the tailnet with scram.
+      # The default localhost md5 lines stay for local admin access.
+      authentication = lib.mkAfter ''
+        host  dropsonde dropsonde ${dropsondeRange} scram-sha-256
+      '';
+    };
+
+    # Create the extensions in the dropsonde db once postgres is up and the db +
+    # role exist. Idempotent (IF NOT EXISTS) so redeploys are safe. The password
+    # is read out of git via a systemd LoadCredential (see note).
+    systemd.services.dropsonde-pg-setup = {
+      description = "dropsonde2: set dropsonde role password + create extensions";
+      wants = [ "postgresql-setup.service" ];
+      after = [ "postgresql-setup.service" "postgresql.service" ];
+      serviceConfig = {
+        Type = "oneshot";
+        RemainAfterExit = true;
+        User = "postgres";
+        Group = "postgres";
+        LoadCredential = [ "dropsonde-pg-password:/root/dropsonde-pg" ];
+        ExecStart = ''${pkgs.postgresql}/bin/psql -v ON_ERROR_STOP=1 -d postgres -c "ALTER ROLE dropsonde WITH LOGIN PASSWORD '${''${CREDENTIALS_DIRECTORY}/dropsonde-pg-password}';"''
+        + ''${pkgs.postgresql}/bin/psql -v ON_ERROR_STOP=1 -d dropsonde -c "CREATE EXTENSION IF NOT EXISTS timescaledb;" -c "CREATE EXTENSION IF NOT EXISTS postgis;" -c "CREATE EXTENSION IF NOT EXISTS h3;" -c "CREATE EXTENSION IF NOT EXISTS h3_postgis CASCADE;'';
+      };
+      wantedBy = [ "multi-user.target" ];
+    };
+  };
+}
@@ -124,6 +124,9 @@ let
      StateDirectory = "llama-server";
      WorkingDirectory = "/var/lib/llama-server";
      SupplementaryGroups = [ "video" ];
+      # Own slice, above default CPUWeight (set in analysis.nix) so the
+      # analysis.slice workloads never starve the helper-tier router.
+      Slice = "llama.service";
      Restart = "on-failure";
      RestartSec = "10s";
      TimeoutStartSec = "600";