package admin_test // v1 admin: read the tables, pin/release a route, drain a host, usage rollups, metrics. import ( "encoding/json" "net/http" "net/http/httptest" "path/filepath" "strings" "testing" "time" "git.wntrmute.dev/kyle/crossbar/internal/admin" "git.wntrmute.dev/kyle/crossbar/internal/config" "git.wntrmute.dev/kyle/crossbar/internal/health" "git.wntrmute.dev/kyle/crossbar/internal/lease" "git.wntrmute.dev/kyle/crossbar/internal/limiter" "git.wntrmute.dev/kyle/crossbar/internal/store" ) type fakeHosts struct { st map[string]health.Status draining map[string]bool } func (f *fakeHosts) All() map[string]health.Status { return f.st } func (f *fakeHosts) Healthy(n string) bool { return f.st[n].Healthy } func (f *fakeHosts) Draining(n string) bool { return f.draining[n] } func (f *fakeHosts) SetDraining(n string, on bool) { f.draining[n] = on } func (f *fakeHosts) Choose(c []string, model string) (string, bool) { for _, h := range c { if f.st[h].Healthy && !f.draining[h] { return h, true } } return "", false } type rig struct { h http.Handler store *store.Store leases *lease.Table hosts *fakeHosts } func newRig(t *testing.T) *rig { cfg, err := config.Parse(strings.NewReader(` listen = "127.0.0.1:1" [hosts.alpha] base_url = "http://alpha:1" models = { "m" = { parallel = 2 } } [hosts.beta] base_url = "http://beta:1" models = { "m" = { parallel = 4 } } [routes.r] hosts = ["alpha", "beta"] default_model = "m" `)) if err != nil { t.Fatal(err) } st, err := store.Open(filepath.Join(t.TempDir(), "x.db")) if err != nil { t.Fatal(err) } t.Cleanup(func() { _ = st.Close() }) hosts := &fakeHosts{ st: map[string]health.Status{ "alpha": {Healthy: true, Loaded: []string{"m"}, LastOK: time.Date(2026, 9, 25, 8, 0, 0, 0, time.UTC)}, "beta": {Healthy: false, LastErr: "HTTP 503"}, }, draining: map[string]bool{}, } lt, err := lease.New(st, hosts, hosts, 30*time.Minute) if err != nil { t.Fatal(err) } lim := limiter.New() lim.Configure("alpha", "m", 2, 8) lim.Configure("beta", "m", 4, 8) return &rig{h: admin.Handler(cfg, hosts, lt, lim, st, hosts), store: st, leases: lt, hosts: hosts} } func (r *rig) do(t *testing.T, method, path, body string, hdr ...string) *httptest.ResponseRecorder { req := httptest.NewRequest(method, path, strings.NewReader(body)) if body != "" { req.Header.Set("Content-Type", "application/json") } for i := 0; i+1 < len(hdr); i += 2 { req.Header.Set(hdr[i], hdr[i+1]) } rec := httptest.NewRecorder() r.h.ServeHTTP(rec, req) return rec } func TestHostsShowsSlotsAndDrain(t *testing.T) { r := newRig(t) rec := r.do(t, "GET", "/_crossbar/hosts", "") if rec.Code != 200 { t.Fatalf("%d %s", rec.Code, rec.Body.String()) } var out map[string]admin.HostView if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { t.Fatal(err) } a := out["alpha"] if !a.Healthy || a.FreeSlots != 2 || a.InFlight != 0 || a.Queued != 0 || a.Draining || a.LastOK != "2026-09-25T08:00:00Z" { t.Errorf("alpha = %+v", a) } if b := out["beta"]; b.Healthy || b.LastErr != "HTTP 503" || b.FreeSlots != 4 || b.Loaded == nil { t.Errorf("beta = %+v (loaded must be [] not null)", b) } } func TestRoutesShowsLeases(t *testing.T) { r := newRig(t) now := time.Date(2026, 9, 25, 9, 0, 0, 0, time.UTC) if _, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "abc", Model: "m"}, []string{"alpha", "beta"}, now); err != nil { t.Fatal(err) } rec := r.do(t, "GET", "/_crossbar/routes", "") var out map[string]admin.RouteView if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil { t.Fatalf("%v: %s", err, rec.Body.String()) } rv := out["r"] if len(rv.Hosts) != 2 || rv.DefaultModel != "m" || rv.Pinned != "" { t.Errorf("route view = %+v", rv) } if len(rv.Leases) != 1 || rv.Leases[0].FP != "abc" || rv.Leases[0].Host != "alpha" || rv.Leases[0].State != "active" || rv.Leases[0].LastUsed != "2026-09-25T09:00:00Z" { t.Errorf("leases = %+v", rv.Leases) } } func TestPinReleaseDrain(t *testing.T) { r := newRig(t) rec := r.do(t, "POST", "/_crossbar/routes/r", `{"host":"beta","pin":true}`) if rec.Code != 200 { t.Fatalf("pin: %d %s", rec.Code, rec.Body.String()) } if h, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "x", Model: "m"}, []string{"alpha", "beta"}, time.Now()); err == nil || h != "" { // beta is unhealthy in the rig: a pin to a down host is honoured, not silently moved t.Errorf("acquire on a route pinned to a down host: %q %v, want ErrPinnedDown", h, err) } rec = r.do(t, "GET", "/_crossbar/routes", "") var out map[string]admin.RouteView _ = json.Unmarshal(rec.Body.Bytes(), &out) if out["r"].Pinned != "beta" { t.Errorf("Pinned = %q after pin", out["r"].Pinned) } rec = r.do(t, "POST", "/_crossbar/routes/r", `{"release":true}`) if rec.Code != 200 { t.Fatalf("release: %d %s", rec.Code, rec.Body.String()) } if h, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "x", Model: "m"}, []string{"alpha", "beta"}, time.Now()); err != nil || h != "alpha" { t.Errorf("after release: %q %v, want alpha (the only healthy host)", h, err) } for _, tc := range []struct { body string want int }{ {`{"host":"nobody","pin":true}`, 404}, {`{"pin":true}`, 400}, {`not json`, 400}, {`{"release":true,"pin":true,"host":"alpha"}`, 400}, } { if rec := r.do(t, "POST", "/_crossbar/routes/r", tc.body); rec.Code != tc.want { t.Errorf("POST %s: %d, want %d (%s)", tc.body, rec.Code, tc.want, rec.Body.String()) } } if rec := r.do(t, "POST", "/_crossbar/routes/nope", `{"release":true}`); rec.Code != 404 { t.Errorf("unknown route: %d", rec.Code) } rec = r.do(t, "POST", "/_crossbar/hosts/alpha", `{"drain":true}`) if rec.Code != 200 || !r.hosts.Draining("alpha") { t.Fatalf("drain: %d %s draining=%v", rec.Code, rec.Body.String(), r.hosts.Draining("alpha")) } rec = r.do(t, "GET", "/_crossbar/hosts", "") var hv map[string]admin.HostView _ = json.Unmarshal(rec.Body.Bytes(), &hv) if !hv["alpha"].Draining { t.Errorf("hosts view must show draining") } if rec := r.do(t, "POST", "/_crossbar/hosts/alpha", `{"drain":false}`); rec.Code != 200 || r.hosts.Draining("alpha") { t.Errorf("undrain: %d draining=%v", rec.Code, r.hosts.Draining("alpha")) } if rec := r.do(t, "POST", "/_crossbar/hosts/nobody", `{"drain":true}`); rec.Code != 404 { t.Errorf("unknown host: %d", rec.Code) } } func seedUsage(t *testing.T, st *store.Store) { t0 := time.Now().UTC().Add(-time.Hour) for i, r := range []store.Request{ {Route: "r", FP: "a", Model: "m", Host: "alpha", Status: 200, TotalMs: 1000, PromptTokens: 100, CachedTokens: 80, CompletionTokens: 10}, {Route: "r", FP: "a", Model: "m", Host: "alpha", Status: 200, TotalMs: 500, QueuedMs: 30, PromptTokens: 100, CachedTokens: 100, CompletionTokens: 5}, {Route: "r2", FP: "b", Model: "m", Host: "beta", Status: 503, TotalMs: 1, Err: "queue full"}, } { r.Started = t0.Add(time.Duration(i) * time.Minute) if err := st.RecordRequest(r); err != nil { t.Fatal(err) } } } func TestUsageJSONAndText(t *testing.T) { r := newRig(t) seedUsage(t, r.store) rec := r.do(t, "GET", "/_crossbar/usage?by=route", "") if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") { t.Fatalf("%d %q", rec.Code, rec.Header().Get("Content-Type")) } var rows []store.UsageRow if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil { t.Fatalf("%v: %s", err, rec.Body.String()) } if len(rows) != 2 { t.Fatalf("rows = %+v", rows) } for _, row := range rows { if row.Key == "r" && (row.Requests != 2 || row.CachedTokens != 180 || row.QueuedMs != 30) { t.Errorf("r = %+v", row) } if row.Key == "r2" && (row.Requests != 1 || row.Errors != 1) { t.Errorf("r2 = %+v", row) } } rec = r.do(t, "GET", "/_crossbar/usage?by=host&since=24h", "", "Accept", "text/plain") if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") { t.Fatalf("text: %d %q", rec.Code, rec.Header().Get("Content-Type")) } body := rec.Body.String() if !strings.Contains(body, "alpha") || !strings.Contains(body, "beta") || !strings.Contains(strings.ToLower(body), "cache") { t.Errorf("text table = %q", body) } if rec := r.do(t, "GET", "/_crossbar/usage?by=colour", ""); rec.Code != 400 { t.Errorf("bad by: %d", rec.Code) } if rec := r.do(t, "GET", "/_crossbar/usage?since=yesterday", ""); rec.Code != 400 { t.Errorf("bad since: %d", rec.Code) } rec = r.do(t, "GET", "/_crossbar/usage?since=2026-09-25T00:00:00Z&by=model", "") if rec.Code != 200 { t.Errorf("RFC3339 since: %d %s", rec.Code, rec.Body.String()) } } func TestMetrics(t *testing.T) { r := newRig(t) seedUsage(t, r.store) rec := r.do(t, "GET", "/_crossbar/metrics", "") if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") { t.Fatalf("%d %q", rec.Code, rec.Header().Get("Content-Type")) } body := rec.Body.String() for _, want := range []string{ `# TYPE crossbar_requests_total counter`, `crossbar_requests_total{route="r",host="alpha",status="200"} 2`, `crossbar_requests_total{route="r2",host="beta",status="503"} 1`, `crossbar_host_healthy{host="alpha"} 1`, `crossbar_host_healthy{host="beta"} 0`, `crossbar_host_free_slots{host="alpha"} 2`, `crossbar_prompt_tokens_total{route="r"} 200`, `crossbar_cached_tokens_total{route="r"} 180`, `crossbar_queue_wait_ms_total{route="r"} 30`, } { if !strings.Contains(body, want) { t.Errorf("metrics missing %q\n%s", want, body) } } } func TestMethodsAndUnknown(t *testing.T) { r := newRig(t) for _, tc := range []struct { method, path string want int }{ {http.MethodPost, "/_crossbar/hosts", 405}, {http.MethodDelete, "/_crossbar/routes", 405}, {http.MethodGet, "/_crossbar/routes/r", 405}, {http.MethodGet, "/_crossbar/nope", 404}, {http.MethodPut, "/_crossbar/usage", 405}, } { rec := r.do(t, tc.method, tc.path, "") if rec.Code != tc.want || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") { t.Errorf("%s %s = %d %q, want %d JSON", tc.method, tc.path, rec.Code, rec.Header().Get("Content-Type"), tc.want) } } }