crossbar/docs/plans/v1/_files/internal/admin/admin_test.go

package admin_test

// v1 admin: read the tables, pin/release a route, drain a host, usage rollups, metrics.

import (
	"encoding/json"
	"net/http"
	"net/http/httptest"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"git.wntrmute.dev/kyle/crossbar/internal/admin"
	"git.wntrmute.dev/kyle/crossbar/internal/config"
	"git.wntrmute.dev/kyle/crossbar/internal/health"
	"git.wntrmute.dev/kyle/crossbar/internal/lease"
	"git.wntrmute.dev/kyle/crossbar/internal/limiter"
	"git.wntrmute.dev/kyle/crossbar/internal/store"
)

type fakeHosts struct {
	st       map[string]health.Status
	draining map[string]bool
}

func (f *fakeHosts) All() map[string]health.Status { return f.st }
func (f *fakeHosts) Healthy(n string) bool           { return f.st[n].Healthy }
func (f *fakeHosts) Draining(n string) bool          { return f.draining[n] }
func (f *fakeHosts) SetDraining(n string, on bool)   { f.draining[n] = on }
func (f *fakeHosts) Choose(c []string, model string) (string, bool) {
	for _, h := range c {
		if f.st[h].Healthy && !f.draining[h] {
			return h, true
		}
	}
	return "", false
}

type rig struct {
	h      http.Handler
	store  *store.Store
	leases *lease.Table
	hosts  *fakeHosts
}

func newRig(t *testing.T) *rig {
	cfg, err := config.Parse(strings.NewReader(`
listen = "127.0.0.1:1"
[hosts.alpha]
base_url = "http://alpha:1"
models = { "m" = { parallel = 2 } }
[hosts.beta]
base_url = "http://beta:1"
models = { "m" = { parallel = 4 } }
[routes.r]
hosts = ["alpha", "beta"]
default_model = "m"
`))
	if err != nil {
		t.Fatal(err)
	}
	st, err := store.Open(filepath.Join(t.TempDir(), "x.db"))
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(func() { _ = st.Close() })
	hosts := &fakeHosts{
		st: map[string]health.Status{
			"alpha": {Healthy: true, Loaded: []string{"m"}, LastOK: time.Date(2026, 9, 25, 8, 0, 0, 0, time.UTC)},
			"beta":  {Healthy: false, LastErr: "HTTP 503"},
		},
		draining: map[string]bool{},
	}
	lt, err := lease.New(st, hosts, hosts, 30*time.Minute)
	if err != nil {
		t.Fatal(err)
	}
	lim := limiter.New()
	lim.Configure("alpha", "m", 2, 8)
	lim.Configure("beta", "m", 4, 8)
	return &rig{h: admin.Handler(cfg, hosts, lt, lim, st, hosts), store: st, leases: lt, hosts: hosts}
}

func (r *rig) do(t *testing.T, method, path, body string, hdr ...string) *httptest.ResponseRecorder {
	req := httptest.NewRequest(method, path, strings.NewReader(body))
	if body != "" {
		req.Header.Set("Content-Type", "application/json")
	}
	for i := 0; i+1 < len(hdr); i += 2 {
		req.Header.Set(hdr[i], hdr[i+1])
	}
	rec := httptest.NewRecorder()
	r.h.ServeHTTP(rec, req)
	return rec
}

func TestHostsShowsSlotsAndDrain(t *testing.T) {
	r := newRig(t)
	rec := r.do(t, "GET", "/_crossbar/hosts", "")
	if rec.Code != 200 {
		t.Fatalf("%d %s", rec.Code, rec.Body.String())
	}
	var out map[string]admin.HostView
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatal(err)
	}
	a := out["alpha"]
	if !a.Healthy || a.FreeSlots != 2 || a.InFlight != 0 || a.Queued != 0 || a.Draining || a.LastOK != "2026-09-25T08:00:00Z" {
		t.Errorf("alpha = %+v", a)
	}
	if b := out["beta"]; b.Healthy || b.LastErr != "HTTP 503" || b.FreeSlots != 4 || b.Loaded == nil {
		t.Errorf("beta = %+v (loaded must be [] not null)", b)
	}
}

func TestRoutesShowsLeases(t *testing.T) {
	r := newRig(t)
	now := time.Date(2026, 9, 25, 9, 0, 0, 0, time.UTC)
	if _, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "abc", Model: "m"}, []string{"alpha", "beta"}, now); err != nil {
		t.Fatal(err)
	}
	rec := r.do(t, "GET", "/_crossbar/routes", "")
	var out map[string]admin.RouteView
	if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
		t.Fatalf("%v: %s", err, rec.Body.String())
	}
	rv := out["r"]
	if len(rv.Hosts) != 2 || rv.DefaultModel != "m" || rv.Pinned != "" {
		t.Errorf("route view = %+v", rv)
	}
	if len(rv.Leases) != 1 || rv.Leases[0].FP != "abc" || rv.Leases[0].Host != "alpha" || rv.Leases[0].State != "active" || rv.Leases[0].LastUsed != "2026-09-25T09:00:00Z" {
		t.Errorf("leases = %+v", rv.Leases)
	}
}

func TestPinReleaseDrain(t *testing.T) {
	r := newRig(t)
	rec := r.do(t, "POST", "/_crossbar/routes/r", `{"host":"beta","pin":true}`)
	if rec.Code != 200 {
		t.Fatalf("pin: %d %s", rec.Code, rec.Body.String())
	}
	if h, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "x", Model: "m"}, []string{"alpha", "beta"}, time.Now()); err == nil || h != "" {
		// beta is unhealthy in the rig: a pin to a down host is honoured, not silently moved
		t.Errorf("acquire on a route pinned to a down host: %q %v, want ErrPinnedDown", h, err)
	}
	rec = r.do(t, "GET", "/_crossbar/routes", "")
	var out map[string]admin.RouteView
	_ = json.Unmarshal(rec.Body.Bytes(), &out)
	if out["r"].Pinned != "beta" {
		t.Errorf("Pinned = %q after pin", out["r"].Pinned)
	}
	rec = r.do(t, "POST", "/_crossbar/routes/r", `{"release":true}`)
	if rec.Code != 200 {
		t.Fatalf("release: %d %s", rec.Code, rec.Body.String())
	}
	if h, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "x", Model: "m"}, []string{"alpha", "beta"}, time.Now()); err != nil || h != "alpha" {
		t.Errorf("after release: %q %v, want alpha (the only healthy host)", h, err)
	}
	for _, tc := range []struct {
		body string
		want int
	}{
		{`{"host":"nobody","pin":true}`, 404},
		{`{"pin":true}`, 400},
		{`not json`, 400},
		{`{"release":true,"pin":true,"host":"alpha"}`, 400},
	} {
		if rec := r.do(t, "POST", "/_crossbar/routes/r", tc.body); rec.Code != tc.want {
			t.Errorf("POST %s: %d, want %d (%s)", tc.body, rec.Code, tc.want, rec.Body.String())
		}
	}
	if rec := r.do(t, "POST", "/_crossbar/routes/nope", `{"release":true}`); rec.Code != 404 {
		t.Errorf("unknown route: %d", rec.Code)
	}

	rec = r.do(t, "POST", "/_crossbar/hosts/alpha", `{"drain":true}`)
	if rec.Code != 200 || !r.hosts.Draining("alpha") {
		t.Fatalf("drain: %d %s draining=%v", rec.Code, rec.Body.String(), r.hosts.Draining("alpha"))
	}
	rec = r.do(t, "GET", "/_crossbar/hosts", "")
	var hv map[string]admin.HostView
	_ = json.Unmarshal(rec.Body.Bytes(), &hv)
	if !hv["alpha"].Draining {
		t.Errorf("hosts view must show draining")
	}
	if rec := r.do(t, "POST", "/_crossbar/hosts/alpha", `{"drain":false}`); rec.Code != 200 || r.hosts.Draining("alpha") {
		t.Errorf("undrain: %d draining=%v", rec.Code, r.hosts.Draining("alpha"))
	}
	if rec := r.do(t, "POST", "/_crossbar/hosts/nobody", `{"drain":true}`); rec.Code != 404 {
		t.Errorf("unknown host: %d", rec.Code)
	}
}

func seedUsage(t *testing.T, st *store.Store) {
	t0 := time.Now().UTC().Add(-time.Hour)
	for i, r := range []store.Request{
		{Route: "r", FP: "a", Model: "m", Host: "alpha", Status: 200, TotalMs: 1000, PromptTokens: 100, CachedTokens: 80, CompletionTokens: 10},
		{Route: "r", FP: "a", Model: "m", Host: "alpha", Status: 200, TotalMs: 500, QueuedMs: 30, PromptTokens: 100, CachedTokens: 100, CompletionTokens: 5},
		{Route: "r2", FP: "b", Model: "m", Host: "beta", Status: 503, TotalMs: 1, Err: "queue full"},
	} {
		r.Started = t0.Add(time.Duration(i) * time.Minute)
		if err := st.RecordRequest(r); err != nil {
			t.Fatal(err)
		}
	}
}

func TestUsageJSONAndText(t *testing.T) {
	r := newRig(t)
	seedUsage(t, r.store)
	rec := r.do(t, "GET", "/_crossbar/usage?by=route", "")
	if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
		t.Fatalf("%d %q", rec.Code, rec.Header().Get("Content-Type"))
	}
	var rows []store.UsageRow
	if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
		t.Fatalf("%v: %s", err, rec.Body.String())
	}
	if len(rows) != 2 {
		t.Fatalf("rows = %+v", rows)
	}
	for _, row := range rows {
		if row.Key == "r" && (row.Requests != 2 || row.CachedTokens != 180 || row.QueuedMs != 30) {
			t.Errorf("r = %+v", row)
		}
		if row.Key == "r2" && (row.Requests != 1 || row.Errors != 1) {
			t.Errorf("r2 = %+v", row)
		}
	}
	rec = r.do(t, "GET", "/_crossbar/usage?by=host&since=24h", "", "Accept", "text/plain")
	if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
		t.Fatalf("text: %d %q", rec.Code, rec.Header().Get("Content-Type"))
	}
	body := rec.Body.String()
	if !strings.Contains(body, "alpha") || !strings.Contains(body, "beta") || !strings.Contains(strings.ToLower(body), "cache") {
		t.Errorf("text table = %q", body)
	}
	if rec := r.do(t, "GET", "/_crossbar/usage?by=colour", ""); rec.Code != 400 {
		t.Errorf("bad by: %d", rec.Code)
	}
	if rec := r.do(t, "GET", "/_crossbar/usage?since=yesterday", ""); rec.Code != 400 {
		t.Errorf("bad since: %d", rec.Code)
	}
	rec = r.do(t, "GET", "/_crossbar/usage?since=2026-09-25T00:00:00Z&by=model", "")
	if rec.Code != 200 {
		t.Errorf("RFC3339 since: %d %s", rec.Code, rec.Body.String())
	}
}

func TestMetrics(t *testing.T) {
	r := newRig(t)
	seedUsage(t, r.store)
	rec := r.do(t, "GET", "/_crossbar/metrics", "")
	if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
		t.Fatalf("%d %q", rec.Code, rec.Header().Get("Content-Type"))
	}
	body := rec.Body.String()
	for _, want := range []string{
		`# TYPE crossbar_requests_total counter`,
		`crossbar_requests_total{route="r",host="alpha",status="200"} 2`,
		`crossbar_requests_total{route="r2",host="beta",status="503"} 1`,
		`crossbar_host_healthy{host="alpha"} 1`,
		`crossbar_host_healthy{host="beta"} 0`,
		`crossbar_host_free_slots{host="alpha"} 2`,
		`crossbar_prompt_tokens_total{route="r"} 200`,
		`crossbar_cached_tokens_total{route="r"} 180`,
		`crossbar_queue_wait_ms_total{route="r"} 30`,
	} {
		if !strings.Contains(body, want) {
			t.Errorf("metrics missing %q\n%s", want, body)
		}
	}
}

func TestMethodsAndUnknown(t *testing.T) {
	r := newRig(t)
	for _, tc := range []struct {
		method, path string
		want         int
	}{
		{http.MethodPost, "/_crossbar/hosts", 405},
		{http.MethodDelete, "/_crossbar/routes", 405},
		{http.MethodGet, "/_crossbar/routes/r", 405},
		{http.MethodGet, "/_crossbar/nope", 404},
		{http.MethodPut, "/_crossbar/usage", 405},
	} {
		rec := r.do(t, tc.method, tc.path, "")
		if rec.Code != tc.want || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
			t.Errorf("%s %s = %d %q, want %d JSON", tc.method, tc.path, rec.Code, rec.Header().Get("Content-Type"), tc.want)
		}
	}
}