v2.3 plan: clients that manage their own slots (control calls, route affinity, queue = false, route listeners, llama-server ctx error); given tests

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 17:36:59 -07:00
parent 4c6415819e
commit 3518e84dd7
16 changed files with 1655 additions and 0 deletions
@@ -0,0 +1,43 @@
+# crossbar example configuration (v1). Replace <tailnet> and the addresses with your own.
+listen        = "127.0.0.1:17777"   # never 0.0.0.0 — bind the tailnet address in production
+db            = "crossbar.db"       # SQLite: leases + accounting (WAL). /var/lib/crossbar/crossbar.db under systemd
+poll_interval = "1s"                # 60s in production; 1s makes the smoke run quick
+lease_idle    = "30m"               # a conversation idle this long loses its host
+retention     = "180d"              # per-request rows older than this are rolled up daily
+queue_max     = 1                   # waiting places per (host, model) beyond `parallel`; 503 past that
+identity      = "off"               # "tailscale" gates routes with `peers` by `tailscale whois`; "header" trusts X-Crossbar-Peer (TEST ONLY)
+
+[hosts.alpha]
+base_url = "http://127.0.0.1:18081"    # e.g. http://straylight.<tailnet>:11434
+weight   = 1.0
+models   = { "ornith-1.5-35b-a3b" = { parallel = 1 }, "small-9b" = { parallel = 6 } }
+
+[hosts.beta]
+base_url = "http://127.0.0.1:18082"    # e.g. http://titan.<tailnet>:8081
+weight   = 2.0
+models   = { "ornith-1.5-35b-a3b" = { parallel = 2 } }
+[hosts.beta.wake]                      # v2: wake a sleeping host when nothing else can take a new lease
+mac       = "aa:bb:cc:dd:ee:02"
+broadcast = "127.0.0.1:19082"          # the LAN broadcast address, port 9, in production
+# broadcasts = ["127.0.0.1:19082", "127.0.0.1:19083"]  # v2.2: a roaming host on several networks; this or broadcast, not both, and at least one
+wait      = "20s"
+
+# v1: a route is a set of candidate hosts; each conversation gets a sticky lease on the host with
+# the most free slots × weight at the time it starts. Pins and drains come from the admin API.
+[routes.opencode-a]
+hosts         = ["alpha", "beta"]
+default_model = "ornith-1.5-35b-a3b"
+
+[routes.hermes-x]
+hosts = ["beta", "alpha"]
+# peers = ["talos"]                    # v2: with identity = "tailscale", only these tailnet nodes may use the route
+
+# v2.3: a client that manages its own llama-server slot (it pins id_slot, polls /slots, steers a
+# running completion through /v1/chat/completions/control) and cannot put a route in the path.
+# The route gets its own port; every request there is this route and the path goes upstream as is.
+[routes.boxmaker-a]
+hosts         = ["beta", "alpha"]
+default_model = "ornith-1.5-35b-a3b"
+listen        = "127.0.0.1:17801"      # a tailnet address in production; never the main listen address
+affinity      = "route"                # one lease for the whole route, not one per conversation
+queue         = false                  # counted as load but never held or refused: the server's own slot queue does that