try switching from ollama to llama.cpp

hw/straylight/default.nix

@@ -1,4 +1,20 @@
 { inputs, pkgs, lib, ... }:
+let
+  pkgsUnstable = inputs.nixpkgs-unstable.legacyPackages.${pkgs.system};
+  # Pin to b9828; nixpkgs-unstable lags too far behind for the features we need.
+  # To update: nix-prefetch-url --unpack https://github.com/ggml-org/llama.cpp/archive/refs/tags/bNNNN.tar.gz
+  # then: nix run nixpkgs#prefetch-npm-deps -- <unpacked>/tools/ui/package-lock.json
+  llama-cpp-b9828 = (pkgsUnstable.llama-cpp.override { vulkanSupport = true; }).overrideAttrs (_: {
+    version = "9828";
+    src = pkgsUnstable.fetchFromGitHub {
+      owner = "ggml-org";
+      repo = "llama.cpp";
+      tag = "b9828";
+      hash = "sha256-CRj+smOs5T80fT5OqHZGbko7/PwChhYsc8MGZZkE/dQ=";
+    };
+    npmDepsHash = "sha256-X1DZgmhS/zHTqDT5zq0kywwntthcJ9vRXeqyO3zz6UU=";
+  });
+in
 {
   imports = [
     ./hardware-configuration.nix
@@ -9,16 +25,48 @@
   ];
 
   config = {
-    # Vulkan backend (Mesa RADV) sidesteps ROCm packaging for gfx1151 (Strix Halo),
+    # llama.cpp server with Vulkan backend for gfx1151 (Strix Halo).
-    # which is still Preview-tier in ROCm. To switch to ROCm (~2x throughput):
+    # Model is downloaded from HuggingFace on first start and cached in StateDirectory.
-    #   acceleration = "rocm";
-    #   rocmOverrideGfx = "11.5.1";  # sets HSA_OVERRIDE_GFX_VERSION
     # NOTE: BIOS "UMA Frame Buffer Size" must be ≥32GB for a 23GB model to fit on-GPU.
-    services.ollama = {
+    users.users.llama-server = {
-      enable = true;
+      isSystemUser = true;
-      acceleration = "vulkan";
+      group = "llama-server";
-      user = "ollama";
+      home = "/var/lib/llama-server";
-      group = "ollama";
+    };
+    users.groups.llama-server = {};
+
+    systemd.services.llama-server = {
+      description = "llama.cpp inference server";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      environment = {
+        HOME = "/var/lib/llama-server";
+        HF_HOME = "/var/lib/llama-server/huggingface";
+      };
+      serviceConfig = {
+        ExecStart = ''
+          ${llama-cpp-b9828}/bin/llama-server \
+            --jinja \
+            -hf deepreinforce-ai/Ornith-1.0-35B-GGUF:Q4_K_M \
+            --host 127.0.0.1 \
+            --port 11434 \
+            -c 262144 \
+            --no-mmap \
+            -fa on \
+            --cache-type-k q8_0 \
+            --cache-type-v q8_0 \
+            -ngl all
+        '';
+        User = "llama-server";
+        Group = "llama-server";
+        StateDirectory = "llama-server";
+        WorkingDirectory = "/var/lib/llama-server";
+        SupplementaryGroups = [ "render" "video" ];
+        Restart = "on-failure";
+        RestartSec = "10s";
+        TimeoutStartSec = "600";
+      };
     };
     # straylight is the unikernel host. The shared mcp.nix locks the agent
     # down with PrivateDevices=true, which hides /dev/kvm and /dev/net/tun.
@@ -71,12 +119,10 @@
         SCARF_NO_ANALYTICS = "True";
         DO_NOT_TRACK = "True";
         ANONYMIZED_TELEMETRY = "False";
-        # Point at the local Ollama instance.
+        # Point at the local llama.cpp server (OpenAI-compatible API).
-        OLLAMA_BASE_URL = "http://127.0.0.1:11434";
+        OPENAI_API_BASE_URLS = "http://127.0.0.1:11434/v1";
-        # Use Ollama for RAG embeddings (pull nomic-embed-text first).
+        OPENAI_API_KEYS = "none";
-        RAG_EMBEDDING_ENGINE = "ollama";
+        ENABLE_OLLAMA_API = "False";
-        RAG_OLLAMA_BASE_URL = "http://127.0.0.1:11434";
-        RAG_EMBEDDING_MODEL = "nomic-embed-text:latest";
       };
     };