kyle/crossbar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Admin: leases, pin, release, drain, usage, metrics

Browse Source 
cmd/crossbar/main.go calls admin.Handler with the new 6-arg signature, passing nil for the not-yet-wired leases/limiter/store/drainer (task 07 wires them) so go vet and go test ./... pass on cmd/crossbar. This is a compile fix, not the task-07 wiring; noted in the implementer-log.

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
kyle
2026-09-25 06:33:29 -07:00
parent c2a8b88a3f
commit 32ac7f549a
9 changed files with 808 additions and 97 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/admin/admin_test.go
+231 -37
View File
@@ -1,9 +1,12 @@
package admin_test
// v1 admin: read the tables, pin/release a route, drain a host, usage rollups, metrics.
import (
"encoding/json"
"net/http"
"net/http/httptest"
"path/filepath"
"strings"
"testing"
"time"
@@ -11,21 +14,45 @@ import (
"git.wntrmute.dev/kyle/crossbar/internal/admin"
"git.wntrmute.dev/kyle/crossbar/internal/config"
"git.wntrmute.dev/kyle/crossbar/internal/health"
"git.wntrmute.dev/kyle/crossbar/internal/lease"
"git.wntrmute.dev/kyle/crossbar/internal/limiter"
"git.wntrmute.dev/kyle/crossbar/internal/store"
)
type fakeHosts map[string]health.Status
type fakeHosts struct {
st map[string]health.Status
draining map[string]bool
}
func (f fakeHosts) All() map[string]health.Status { return f }
func (f *fakeHosts) All() map[string]health.Status { return f.st }
func (f *fakeHosts) Healthy(n string) bool { return f.st[n].Healthy }
func (f *fakeHosts) Draining(n string) bool { return f.draining[n] }
func (f *fakeHosts) SetDraining(n string, on bool) { f.draining[n] = on }
func (f *fakeHosts) Choose(c []string, model string) (string, bool) {
for _, h := range c {
if f.st[h].Healthy && !f.draining[h] {
return h, true
}
}
return "", false
}
func testConfig(t *testing.T) *config.Config {
c, err := config.Parse(strings.NewReader(`
type rig struct {
h http.Handler
store *store.Store
leases *lease.Table
hosts *fakeHosts
}
func newRig(t *testing.T) *rig {
cfg, err := config.Parse(strings.NewReader(`
listen = "127.0.0.1:1"
[hosts.alpha]
base_url = "http://alpha:1"
models = { "m" = { } }
models = { "m" = { parallel = 2 } }
[hosts.beta]
base_url = "http://beta:1"
models = { "m" = { } }
models = { "m" = { parallel = 4 } }
[routes.r]
hosts = ["alpha", "beta"]
default_model = "m"
@@ -33,67 +60,234 @@ default_model = "m"
if err != nil {
t.Fatal(err)
}
return c
st, err := store.Open(filepath.Join(t.TempDir(), "x.db"))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = st.Close() })
hosts := &fakeHosts{
st: map[string]health.Status{
"alpha": {Healthy: true, Loaded: []string{"m"}, LastOK: time.Date(2026, 9, 25, 8, 0, 0, 0, time.UTC)},
"beta": {Healthy: false, LastErr: "HTTP 503"},
},
draining: map[string]bool{},
}
lt, err := lease.New(st, hosts, hosts, 30*time.Minute)
if err != nil {
t.Fatal(err)
}
lim := limiter.New()
lim.Configure("alpha", "m", 2, 8)
lim.Configure("beta", "m", 4, 8)
return &rig{h: admin.Handler(cfg, hosts, lt, lim, st, hosts), store: st, leases: lt, hosts: hosts}
}
func TestHosts(t *testing.T) {
when := time.Date(2026, 9, 25, 8, 0, 0, 0, time.UTC)
h := admin.Handler(testConfig(t), fakeHosts{
"alpha": {Healthy: true, Loaded: []string{"m"}, LastOK: when},
"beta": {Healthy: false, LastErr: "HTTP 503"},
})
func (r *rig) do(t *testing.T, method, path, body string, hdr ...string) *httptest.ResponseRecorder {
req := httptest.NewRequest(method, path, strings.NewReader(body))
if body != "" {
req.Header.Set("Content-Type", "application/json")
}
for i := 0; i+1 < len(hdr); i += 2 {
req.Header.Set(hdr[i], hdr[i+1])
}
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/_crossbar/hosts", nil))
if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
t.Fatalf("status %d, content-type %q", rec.Code, rec.Header().Get("Content-Type"))
r.h.ServeHTTP(rec, req)
return rec
}
func TestHostsShowsSlotsAndDrain(t *testing.T) {
r := newRig(t)
rec := r.do(t, "GET", "/_crossbar/hosts", "")
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
var out map[string]admin.HostView
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatal(err)
}
if a := out["alpha"]; !a.Healthy || len(a.Loaded) != 1 || a.LastOK != "2026-09-25T08:00:00Z" || a.LastErr != "" {
a := out["alpha"]
if !a.Healthy || a.FreeSlots != 2 || a.InFlight != 0 || a.Queued != 0 || a.Draining || a.LastOK != "2026-09-25T08:00:00Z" {
t.Errorf("alpha = %+v", a)
}
if b := out["beta"]; b.Healthy || b.LastOK != "" || b.LastErr != "HTTP 503" || b.Loaded == nil {
if b := out["beta"]; b.Healthy || b.LastErr != "HTTP 503" || b.FreeSlots != 4 || b.Loaded == nil {
t.Errorf("beta = %+v (loaded must be [] not null)", b)
}
if !strings.Contains(rec.Body.String(), `"loaded":[]`) {
t.Errorf("beta.loaded must encode as []: %s", rec.Body.String())
}
}
func TestRoutes(t *testing.T) {
h := admin.Handler(testConfig(t), fakeHosts{})
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/_crossbar/routes", nil))
func TestRoutesShowsLeases(t *testing.T) {
r := newRig(t)
now := time.Date(2026, 9, 25, 9, 0, 0, 0, time.UTC)
if _, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "abc", Model: "m"}, []string{"alpha", "beta"}, now); err != nil {
t.Fatal(err)
}
rec := r.do(t, "GET", "/_crossbar/routes", "")
var out map[string]admin.RouteView
if err := json.Unmarshal(rec.Body.Bytes(), &out); err != nil {
t.Fatalf("%v: %s", err, rec.Body.String())
}
r := out["r"]
if len(r.Hosts) != 2 || r.Hosts[0] != "alpha" || r.DefaultModel != "m" {
t.Errorf("routes = %+v", out)
rv := out["r"]
if len(rv.Hosts) != 2 || rv.DefaultModel != "m" || rv.Pinned != "" {
t.Errorf("route view = %+v", rv)
}
if len(rv.Leases) != 1 || rv.Leases[0].FP != "abc" || rv.Leases[0].Host != "alpha" || rv.Leases[0].State != "active" || rv.Leases[0].LastUsed != "2026-09-25T09:00:00Z" {
t.Errorf("leases = %+v", rv.Leases)
}
}
func TestPinReleaseDrain(t *testing.T) {
r := newRig(t)
rec := r.do(t, "POST", "/_crossbar/routes/r", `{"host":"beta","pin":true}`)
if rec.Code != 200 {
t.Fatalf("pin: %d %s", rec.Code, rec.Body.String())
}
if h, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "x", Model: "m"}, []string{"alpha", "beta"}, time.Now()); err == nil || h != "" {
// beta is unhealthy in the rig: a pin to a down host is honoured, not silently moved
t.Errorf("acquire on a route pinned to a down host: %q %v, want ErrPinnedDown", h, err)
}
rec = r.do(t, "GET", "/_crossbar/routes", "")
var out map[string]admin.RouteView
_ = json.Unmarshal(rec.Body.Bytes(), &out)
if out["r"].Pinned != "beta" {
t.Errorf("Pinned = %q after pin", out["r"].Pinned)
}
rec = r.do(t, "POST", "/_crossbar/routes/r", `{"release":true}`)
if rec.Code != 200 {
t.Fatalf("release: %d %s", rec.Code, rec.Body.String())
}
if h, _, err := r.leases.Acquire(lease.Key{Route: "r", FP: "x", Model: "m"}, []string{"alpha", "beta"}, time.Now()); err != nil || h != "alpha" {
t.Errorf("after release: %q %v, want alpha (the only healthy host)", h, err)
}
for _, tc := range []struct {
body string
want int
}{
{`{"host":"nobody","pin":true}`, 404},
{`{"pin":true}`, 400},
{`not json`, 400},
{`{"release":true,"pin":true,"host":"alpha"}`, 400},
} {
if rec := r.do(t, "POST", "/_crossbar/routes/r", tc.body); rec.Code != tc.want {
t.Errorf("POST %s: %d, want %d (%s)", tc.body, rec.Code, tc.want, rec.Body.String())
}
}
if rec := r.do(t, "POST", "/_crossbar/routes/nope", `{"release":true}`); rec.Code != 404 {
t.Errorf("unknown route: %d", rec.Code)
}
rec = r.do(t, "POST", "/_crossbar/hosts/alpha", `{"drain":true}`)
if rec.Code != 200 || !r.hosts.Draining("alpha") {
t.Fatalf("drain: %d %s draining=%v", rec.Code, rec.Body.String(), r.hosts.Draining("alpha"))
}
rec = r.do(t, "GET", "/_crossbar/hosts", "")
var hv map[string]admin.HostView
_ = json.Unmarshal(rec.Body.Bytes(), &hv)
if !hv["alpha"].Draining {
t.Errorf("hosts view must show draining")
}
if rec := r.do(t, "POST", "/_crossbar/hosts/alpha", `{"drain":false}`); rec.Code != 200 || r.hosts.Draining("alpha") {
t.Errorf("undrain: %d draining=%v", rec.Code, r.hosts.Draining("alpha"))
}
if rec := r.do(t, "POST", "/_crossbar/hosts/nobody", `{"drain":true}`); rec.Code != 404 {
t.Errorf("unknown host: %d", rec.Code)
}
}
func seedUsage(t *testing.T, st *store.Store) {
t0 := time.Now().UTC().Add(-time.Hour)
for i, r := range []store.Request{
{Route: "r", FP: "a", Model: "m", Host: "alpha", Status: 200, TotalMs: 1000, PromptTokens: 100, CachedTokens: 80, CompletionTokens: 10},
{Route: "r", FP: "a", Model: "m", Host: "alpha", Status: 200, TotalMs: 500, QueuedMs: 30, PromptTokens: 100, CachedTokens: 100, CompletionTokens: 5},
{Route: "r2", FP: "b", Model: "m", Host: "beta", Status: 503, TotalMs: 1, Err: "queue full"},
} {
r.Started = t0.Add(time.Duration(i) * time.Minute)
if err := st.RecordRequest(r); err != nil {
t.Fatal(err)
}
}
}
func TestUsageJSONAndText(t *testing.T) {
r := newRig(t)
seedUsage(t, r.store)
rec := r.do(t, "GET", "/_crossbar/usage?by=route", "")
if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
t.Fatalf("%d %q", rec.Code, rec.Header().Get("Content-Type"))
}
var rows []store.UsageRow
if err := json.Unmarshal(rec.Body.Bytes(), &rows); err != nil {
t.Fatalf("%v: %s", err, rec.Body.String())
}
if len(rows) != 2 {
t.Fatalf("rows = %+v", rows)
}
for _, row := range rows {
if row.Key == "r" && (row.Requests != 2 || row.CachedTokens != 180 || row.QueuedMs != 30) {
t.Errorf("r = %+v", row)
}
if row.Key == "r2" && (row.Requests != 1 || row.Errors != 1) {
t.Errorf("r2 = %+v", row)
}
}
rec = r.do(t, "GET", "/_crossbar/usage?by=host&since=24h", "", "Accept", "text/plain")
if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
t.Fatalf("text: %d %q", rec.Code, rec.Header().Get("Content-Type"))
}
body := rec.Body.String()
if !strings.Contains(body, "alpha") || !strings.Contains(body, "beta") || !strings.Contains(strings.ToLower(body), "cache") {
t.Errorf("text table = %q", body)
}
if rec := r.do(t, "GET", "/_crossbar/usage?by=colour", ""); rec.Code != 400 {
t.Errorf("bad by: %d", rec.Code)
}
if rec := r.do(t, "GET", "/_crossbar/usage?since=yesterday", ""); rec.Code != 400 {
t.Errorf("bad since: %d", rec.Code)
}
rec = r.do(t, "GET", "/_crossbar/usage?since=2026-09-25T00:00:00Z&by=model", "")
if rec.Code != 200 {
t.Errorf("RFC3339 since: %d %s", rec.Code, rec.Body.String())
}
}
func TestMetrics(t *testing.T) {
r := newRig(t)
seedUsage(t, r.store)
rec := r.do(t, "GET", "/_crossbar/metrics", "")
if rec.Code != 200 || !strings.HasPrefix(rec.Header().Get("Content-Type"), "text/plain") {
t.Fatalf("%d %q", rec.Code, rec.Header().Get("Content-Type"))
}
body := rec.Body.String()
for _, want := range []string{
`# TYPE crossbar_requests_total counter`,
`crossbar_requests_total{route="r",host="alpha",status="200"} 2`,
`crossbar_requests_total{route="r2",host="beta",status="503"} 1`,
`crossbar_host_healthy{host="alpha"} 1`,
`crossbar_host_healthy{host="beta"} 0`,
`crossbar_host_free_slots{host="alpha"} 2`,
`crossbar_prompt_tokens_total{route="r"} 200`,
`crossbar_cached_tokens_total{route="r"} 180`,
`crossbar_queue_wait_ms_total{route="r"} 30`,
} {
if !strings.Contains(body, want) {
t.Errorf("metrics missing %q\n%s", want, body)
}
}
}
func TestMethodsAndUnknown(t *testing.T) {
h := admin.Handler(testConfig(t), fakeHosts{})
r := newRig(t)
for _, tc := range []struct {
method, path string
want int
}{
{http.MethodPost, "/_crossbar/hosts", 405},
{http.MethodDelete, "/_crossbar/routes", 405},
{http.MethodGet, "/_crossbar/routes/r", 405},
{http.MethodGet, "/_crossbar/nope", 404},
{http.MethodGet, "/_crossbar/", 404},
{http.MethodPut, "/_crossbar/usage", 405},
} {
rec := httptest.NewRecorder()
h.ServeHTTP(rec, httptest.NewRequest(tc.method, tc.path, nil))
if rec.Code != tc.want {
t.Errorf("%s %s = %d, want %d", tc.method, tc.path, rec.Code, tc.want)
}
if !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
t.Errorf("%s %s: errors are JSON too", tc.method, tc.path)
rec := r.do(t, tc.method, tc.path, "")
if rec.Code != tc.want || !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
t.Errorf("%s %s = %d %q, want %d JSON", tc.method, tc.path, rec.Code, rec.Header().Get("Content-Type"), tc.want)
}
}
}