kyle/crossbar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Learn per-model context from /props?model=; only status "loaded" is loaded

Browse Source 
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
kyle
2026-09-25 11:47:35 -07:00
parent f3dfdbfa50
commit 055ab079d4
9 changed files with 562 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/health/health.go
+64 -63
View File
@@ -21,15 +21,22 @@ const RecoveryPolls = 2
// MaxModelsBody bounds how many bytes we read from either /health or /v1/models.
const MaxModelsBody = 1 << 20
// ModelCtx is what /props?model=X taught us about one loaded model.
type ModelCtx struct {
NCtx int `json:"n_ctx"`
Slots int `json:"slots"`
}
// Status is a snapshot of one host's health, safe to copy.
type Status struct {
Healthy bool `json:"healthy"`
Loaded []string `json:"loaded"` // sorted, unique model ids from the last good poll
LastOK time.Time `json:"last_ok"` // zero if never
LastErr string `json:"last_err"` // "" after a good poll
Consecutive int `json:"consecutive"` // good polls in a row
NCtx int `json:"n_ctx"` // total context from /props; 0 = unknown
Slots int `json:"slots"` // total_slots from /props; 0 = unknown
Healthy bool `json:"healthy"`
Loaded []string `json:"loaded"` // sorted, unique model ids from the last good poll
LastOK time.Time `json:"last_ok"` // zero if never
LastErr string `json:"last_err"` // "" after a good poll
Consecutive int `json:"consecutive"` // good polls in a row
NCtx int `json:"n_ctx"` // total context from /props; 0 = unknown (a router's own /props carries none)
Slots int `json:"slots"` // total_slots from /props; 0 = unknown
Models map[string]ModelCtx `json:"models"` // per loaded model; never nil after a poll
}
// PerSlotCtx is the context one request may use: NCtx divided by Slots, or the
@@ -41,6 +48,22 @@ func (s Status) PerSlotCtx() int {
return s.NCtx / s.Slots
}
// PerSlotCtxFor is the per-slot context for one model on this host: Models[model]
// when present (NCtx/Slots, 0 when either is 0); else, when model is in Loaded,
// the host-level PerSlotCtx(); else 0 ("unknown" / not resident).
func (s Status) PerSlotCtxFor(model string) int {
if mc, ok := s.Models[model]; ok {
if mc.NCtx == 0 || mc.Slots == 0 {
return 0
}
return mc.NCtx / mc.Slots
}
if contains(s.Loaded, model) {
return s.PerSlotCtx()
}
return 0
}
type entry struct {
status Status
everFailed bool
@@ -53,6 +76,7 @@ type pollResult struct {
loaded []string
nctx int
slots int
models map[string]ModelCtx
}
// Table maps a host name to its health status. All methods are safe for concurrent use.
@@ -171,6 +195,7 @@ func (t *Table) pollHost(ctx context.Context, name string) {
e.status.Loaded = r.loaded
e.status.NCtx = r.nctx
e.status.Slots = r.slots
e.status.Models = r.models
e.status.Healthy = !e.everFailed || e.status.Consecutive >= RecoveryPolls
} else {
e.everFailed = true
@@ -196,57 +221,11 @@ func (t *Table) poll(ctx context.Context, base string) pollResult {
if r.cancelled || r.reason != "" {
return r
}
return pollResult{ok: true, loaded: loaded, nctx: nctx, slots: slots}
}
// props reads <base>/props best-effort. A request that fails because ctx is
// done yields a cancelled result so the caller records nothing; any other
// outcome (status, body, or missing fields) leaves context unknown without
// failing the poll.
func (t *Table) props(ctx context.Context, base string) (int, int, pollResult) {
req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/props", nil)
if err != nil {
if ctx.Err() != nil {
return 0, 0, pollResult{cancelled: true}
}
return 0, 0, pollResult{}
models, r := t.propsModels(ctx, base, loaded)
if r.cancelled || r.reason != "" {
return r
}
resp, err := t.client.Do(req)
if err != nil {
if ctx.Err() != nil {
return 0, 0, pollResult{cancelled: true}
}
return 0, 0, pollResult{}
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return 0, 0, pollResult{}
}
var p struct {
Generation struct {
NCtx *int `json:"n_ctx"`
} `json:"default_generation_settings"`
TotalSlots *int `json:"total_slots"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&p); err != nil {
return 0, 0, pollResult{}
}
nctx, slots := 0, 0
if p.Generation.NCtx != nil {
nctx = *p.Generation.NCtx
}
if p.TotalSlots != nil {
slots = *p.TotalSlots
}
if nctx < 0 {
nctx = 0
}
if slots < 0 {
slots = 0
}
return nctx, slots, pollResult{}
return pollResult{ok: true, loaded: loaded, nctx: nctx, slots: slots, models: models}
}
// check performs one GET and, on success, returns the decoded model ids. Health checks use the
@@ -280,7 +259,10 @@ func (t *Table) check(ctx context.Context, url, prefix string) ([]string, pollRe
}
var m struct {
Data []struct {
ID string `json:"id"`
ID string `json:"id"`
Status struct {
Value string `json:"value"`
} `json:"status"`
} `json:"data"`
}
if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&m); err != nil {
@@ -289,6 +271,8 @@ func (t *Table) check(ctx context.Context, url, prefix string) ([]string, pollRe
}
defer resp.Body.Close()
// A model is loaded when it has no status or status.value == "loaded"; any other
// value ("unloaded", "loading", …) is not loaded and must never be asked /props?model=.
loaded := make([]string, 0, len(m.Data))
seen := make(map[string]struct{}, len(m.Data))
for _, d := range m.Data {
@@ -298,6 +282,9 @@ func (t *Table) check(ctx context.Context, url, prefix string) ([]string, pollRe
if _, ok := seen[d.ID]; ok {
continue
}
if d.Status.Value != "" && d.Status.Value != "loaded" {
continue
}
seen[d.ID] = struct{}{}
loaded = append(loaded, d.ID)
}
@@ -314,11 +301,25 @@ func (t *Table) fail(ctx context.Context, prefix string, err error) pollResult {
}
func copyStatus(s Status) Status {
if s.Loaded == nil {
return s
}
out := s
out.Loaded = make([]string, len(s.Loaded))
copy(out.Loaded, s.Loaded)
if s.Loaded != nil {
out.Loaded = make([]string, len(s.Loaded))
copy(out.Loaded, s.Loaded)
}
if s.Models != nil {
out.Models = make(map[string]ModelCtx, len(s.Models))
for k, v := range s.Models {
out.Models[k] = v
}
}
return out
}
func contains(list []string, v string) bool {
for _, s := range list {
if s == v {
return true
}
}
return false
}