diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 5a716ad..6068805 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -5,6 +5,7 @@ owner fills in the Model column. The reviewer adds findings under "Reviews" once | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| v2/01-props | 2026-09-25 | done | 1 | pass | none | Implemented /props learning in `internal/health/health.go`: added `Status.NCtx`/`Status.Slots`, `PerSlotCtx()`, and a best-effort `GET /props` appended to the poll after `/v1/models`, setting NCtx/Slots to 0 (negative → 0) on any failure without counting the poll as failed; exposed them in `internal/admin/admin.go` `HostView`. Copied `internal/health/props_test.go` and the replacement `internal/proxy/helpers_test.go` byte-identical to `docs/plans/v2/_files/`. `go test -race ./...` and `make gate` pass on the first run. | ? | | v2/01-props | 2026-09-25 | stopped | 1 | fail | none | Implemented /props learning in `internal/health/health.go` (added `Status.NCtx`/`Status.Slots`, `PerSlotCtx`, and a best-effort `GET /props` appended to the poll; 0/unknown on any failure without failing the poll) and exposed them in `internal/admin/admin.go` `HostView`; copied `internal/health/props_test.go` byte-identical to `docs/plans/v2/_files/`. `go test -race ./internal/health/ ./internal/admin/` ok. `make gate` fails on two GIVEN v1 proxy tests — `TestConversationIsStickyAndLeaseHeaderTellsWhy` (alpha 1/beta 7, want 0/6) and `TestDifferentConversationsSpreadByFreeSlots` (beta 3/alpha 2, want 2/1) — which assert exact upstream hit counts; the task-required `/props` poll now lands on that scaffold's `/` catch-all and bumps the counter by exactly 1 per host (deterministic, confirmed over 3 repeated runs, not a flake). `internal/proxy/helpers_test.go` is byte-identical to `docs/plans/v1/_files/` (protected) and cannot be updated here; the `/props` request is unavoidable per the task, so the owner must hand over a scaffold that registers `/props` without counting it as a hit. Code left uncommitted for review. | ? | | v1.1/01-review-fixes | 2026-09-25 | done | 1 | pass | none | Copied `cancel_test.go` and `usage_empty_test.go` byte-identical from `docs/plans/v1.1/_files/`; the earlier session's fixes in `internal/proxy/proxy.go`, `internal/proxy/forward.go` and `internal/admin/admin_ops.go` were already in the working tree. `make gate` printed `gate: ok` on the first run. | llama.cpp/ornith-1.5-35b-a3b | | v1/08-smoke-readme | 2026-09-25 | done | 1 | pass | owner-directed fix to `Free` in `proxy.Chooser` | Changed `Free` from `c.lim.FreeSlots(host)` (sum over every model) to per-model free slots, `freeForModel(cfg.Hosts[host], model, c.lim.InFlight(host, model))`, floored at 0 and 0 when the host does not list the model (new helper in hosts.go); the one code change the task directs. `go test -race ./internal/proxy/` and `make gate` pass on the first run; `make smoke` → `smoke: ok (stream spread 1006 ms)`. README intro, `## Configure` (added db/lease_idle/retention, rewrote queue_max and hosts..hosts) and `## Inspect`→`## Operate` (all six endpoints, examples taken from the smoke run) updated. | llama.cpp/ornith-1.5-35b-a3b | diff --git a/internal/admin/admin.go b/internal/admin/admin.go index e45817c..d3dffab 100644 --- a/internal/admin/admin.go +++ b/internal/admin/admin.go @@ -38,6 +38,8 @@ type HostView struct { InFlight int `json:"in_flight"` // sum over the host's configured models Queued int `json:"queued"` // same Draining bool `json:"draining"` + NCtx int `json:"n_ctx"` // from /props; 0 = unknown + Slots int `json:"slots"` // from /props; 0 = unknown } // LeaseView is one lease's row in a route's leases. @@ -125,6 +127,8 @@ func (hx *handler) hostView(name string, s health.Status) HostView { InFlight: inflight, Queued: queued, Draining: hx.d.Draining(name), + NCtx: s.NCtx, + Slots: s.Slots, } } diff --git a/internal/health/health.go b/internal/health/health.go index d483353..171a627 100644 --- a/internal/health/health.go +++ b/internal/health/health.go @@ -28,6 +28,17 @@ type Status struct { LastOK time.Time `json:"last_ok"` // zero if never LastErr string `json:"last_err"` // "" after a good poll Consecutive int `json:"consecutive"` // good polls in a row + NCtx int `json:"n_ctx"` // total context from /props; 0 = unknown + Slots int `json:"slots"` // total_slots from /props; 0 = unknown +} + +// PerSlotCtx is the context one request may use: NCtx divided by Slots, or the +// whole NCtx when Slots is unknown (0). It is 0 when NCtx is unknown. +func (s Status) PerSlotCtx() int { + if s.NCtx == 0 || s.Slots == 0 { + return s.NCtx + } + return s.NCtx / s.Slots } type entry struct { @@ -40,6 +51,8 @@ type pollResult struct { cancelled bool reason string loaded []string + nctx int + slots int } // Table maps a host name to its health status. All methods are safe for concurrent use. @@ -156,6 +169,8 @@ func (t *Table) pollHost(ctx context.Context, name string) { e.status.LastOK = time.Now() e.status.LastErr = "" e.status.Loaded = r.loaded + e.status.NCtx = r.nctx + e.status.Slots = r.slots e.status.Healthy = !e.everFailed || e.status.Consecutive >= RecoveryPolls } else { e.everFailed = true @@ -174,13 +189,64 @@ func (t *Table) poll(ctx context.Context, base string) pollResult { return r } loaded, r = t.check(ctx, base+"/v1/models", "models") - if r.cancelled { + if r.cancelled || r.reason != "" { return r } - if r.reason != "" { + nctx, slots, r := t.props(ctx, base) + if r.cancelled || r.reason != "" { return r } - return pollResult{ok: true, loaded: loaded} + return pollResult{ok: true, loaded: loaded, nctx: nctx, slots: slots} +} + +// props reads /props best-effort. A request that fails because ctx is +// done yields a cancelled result so the caller records nothing; any other +// outcome (status, body, or missing fields) leaves context unknown without +// failing the poll. +func (t *Table) props(ctx context.Context, base string) (int, int, pollResult) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/props", nil) + if err != nil { + if ctx.Err() != nil { + return 0, 0, pollResult{cancelled: true} + } + return 0, 0, pollResult{} + } + resp, err := t.client.Do(req) + if err != nil { + if ctx.Err() != nil { + return 0, 0, pollResult{cancelled: true} + } + return 0, 0, pollResult{} + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return 0, 0, pollResult{} + } + var p struct { + Generation struct { + NCtx *int `json:"n_ctx"` + } `json:"default_generation_settings"` + TotalSlots *int `json:"total_slots"` + } + if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&p); err != nil { + return 0, 0, pollResult{} + } + + nctx, slots := 0, 0 + if p.Generation.NCtx != nil { + nctx = *p.Generation.NCtx + } + if p.TotalSlots != nil { + slots = *p.TotalSlots + } + if nctx < 0 { + nctx = 0 + } + if slots < 0 { + slots = 0 + } + return nctx, slots, pollResult{} } // check performs one GET and, on success, returns the decoded model ids. Health checks use the diff --git a/internal/health/props_test.go b/internal/health/props_test.go new file mode 100644 index 0000000..ed77b95 --- /dev/null +++ b/internal/health/props_test.go @@ -0,0 +1,75 @@ +package health_test + +import ( + "context" + "fmt" + "net/http" + "net/http/httptest" + "testing" + "time" + + "git.wntrmute.dev/kyle/crossbar/internal/health" +) + +// propsFake answers /health, /v1/models and a configurable /props. +func propsFake(t *testing.T, props string, status int) *httptest.Server { + mux := http.NewServeMux() + mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"ok"}`) }) + mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"data":[{"id":"m"}]}`) }) + mux.HandleFunc("/props", func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(status) + fmt.Fprint(w, props) + }) + srv := httptest.NewServer(mux) + t.Cleanup(srv.Close) + return srv +} + +func TestPropsLearned(t *testing.T) { + srv := propsFake(t, `{"default_generation_settings":{"n_ctx":131072,"params":{}},"total_slots":4,"model_path":"/x/m.gguf","chat_template":"..."}`, 200) + tbl := health.New(map[string]string{"a": srv.URL}, time.Hour, nil) + tbl.PollOnce(context.Background()) + s, _ := tbl.Get("a") + if !s.Healthy || s.NCtx != 131072 || s.Slots != 4 { + t.Fatalf("status = %+v, want healthy with NCtx 131072 and Slots 4", s) + } + if got := s.PerSlotCtx(); got != 32768 { + t.Errorf("PerSlotCtx = %d, want 131072/4", got) + } +} + +func TestPropsAbsentOrBrokenIsNotAFailure(t *testing.T) { + for name, tc := range map[string]struct { + props string + status int + }{ + "404": {`not found`, 404}, + "not json": {``, 200}, + "no fields": {`{"model_path":"/x"}`, 200}, + "zero ctx": {`{"default_generation_settings":{"n_ctx":0},"total_slots":0}`, 200}, + } { + t.Run(name, func(t *testing.T) { + srv := propsFake(t, tc.props, tc.status) + tbl := health.New(map[string]string{"a": srv.URL}, time.Hour, nil) + tbl.PollOnce(context.Background()) + s, _ := tbl.Get("a") + if !s.Healthy { + t.Fatalf("a bad /props must not make the host unhealthy: %+v", s) + } + if s.NCtx != 0 || s.Slots != 0 || s.PerSlotCtx() != 0 { + t.Errorf("unknown context must read as 0: %+v", s) + } + }) + } +} + +func TestPerSlotCtxWithUnknownSlots(t *testing.T) { + s := health.Status{NCtx: 8192, Slots: 0} + if s.PerSlotCtx() != 8192 { + t.Errorf("with Slots unknown the whole context is the per-slot value; got %d", s.PerSlotCtx()) + } + s = health.Status{NCtx: 8192, Slots: 3} + if s.PerSlotCtx() != 2730 { + t.Errorf("integer division: got %d, want 2730", s.PerSlotCtx()) + } +} diff --git a/internal/proxy/helpers_test.go b/internal/proxy/helpers_test.go index e1c8807..da689db 100644 --- a/internal/proxy/helpers_test.go +++ b/internal/proxy/helpers_test.go @@ -79,6 +79,11 @@ func newUpstream(t *testing.T, name string) *upstream { u.mu.Unlock() fmt.Fprint(w, `{"object":"list","data":[{"id":"shared"},{"id":"`+name+`-only"}]}`) }) + // The v2 poller also asks /props; it is a health request, not a hit, so it is not counted. + // No n_ctx here: "unknown context" is what the v1 tests and TestUnknownContextNeverBlocks want. + mux.HandleFunc("/props", func(w http.ResponseWriter, r *http.Request) { + fmt.Fprint(w, `{"model_path":"`+name+`"}`) + }) mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) { u.hits.Add(1) b, _ := io.ReadAll(r.Body)