kyle/crossbar
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Learn per-model context from /props?model=; only status "loaded" is loaded

Browse Source 
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
kyle
2026-09-25 11:47:35 -07:00
parent f3dfdbfa50
commit 055ab079d4
9 changed files with 562 additions and 86 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/proxy/ctxguard.go
+12 -12
View File
@@ -11,9 +11,9 @@ import (
)
// movedHeader announces a context-driven move in the response header:
// "moved: old><new". The client learns which host served it.
// "moved: old>new". The client learns which host served it.
func movedHeader(from, to string) string {
return "moved:" + from + "><" + to
return "moved:" + from + ">" + to
}
// drainer is the drain flag the guard consults when rule 3 prefers a host not
@@ -54,7 +54,7 @@ func (p *Handler) guard(w http.ResponseWriter, r *http.Request, hosts []string,
// already fits its per-slot context.
psc := 0
if s, ok := p.health.Get(host); ok {
psc = s.PerSlotCtx()
psc = s.PerSlotCtxFor(model)
}
if psc == 0 || estimate <= psc {
return host, "", 0, false
@@ -84,7 +84,7 @@ func (p *Handler) guard(w http.ResponseWriter, r *http.Request, hosts []string,
}
// A host with a known per-slot context smaller than the estimate
// cannot serve it no matter how it wakes.
if psc := s.PerSlotCtx(); psc != 0 && psc < estimate {
if psc := s.PerSlotCtxFor(model); psc != 0 && psc < estimate {
continue
}
if p.cfg.Hosts[name].Wake == nil || !p.waker.Wake(r.Context(), name) {
@@ -100,7 +100,7 @@ func (p *Handler) guard(w http.ResponseWriter, r *http.Request, hosts []string,
// Rule 4: no host fits. Answer 400 with the estimate and the largest
// available per-slot context, and record the row.
p.refuseCtx(w, host, route, model, fp, started, estimate, largestSlotCtx(hosts, p.health))
p.refuseCtx(w, host, route, model, fp, started, estimate, largestSlotCtx(hosts, p.health, model))
return host, "", estimate, true
}
@@ -120,7 +120,7 @@ func ctxFitHost(hosts []string, model string, estimate int, h Health, cfg *confi
if !ok || !s.Healthy || isDraining(drn, name) {
continue
}
if s.PerSlotCtx() >= estimate && contains(s.Loaded, model) {
if s.PerSlotCtxFor(model) >= estimate && contains(s.Loaded, model) {
return name, true
}
}
@@ -132,23 +132,23 @@ func ctxFitHost(hosts []string, model string, estimate int, h Health, cfg *confi
if !ok || !s.Healthy || isDraining(drn, name) {
continue
}
if s.PerSlotCtx() >= estimate && cfg.Serves(name, model) {
if s.PerSlotCtxFor(model) >= estimate && cfg.Serves(name, model) {
return name, true
}
}
return "", false
}
// largestSlotCtx is the largest PerSlotCtx across the route's healthy hosts, or
// 0 when none is healthy or reports a context size.
func largestSlotCtx(hosts []string, h Health) int {
// largestSlotCtx is the largest per-slot context for model across the route's
// healthy hosts that have it loaded, or 0 when none is healthy or reports one.
func largestSlotCtx(hosts []string, h Health, model string) int {
best := 0
for _, name := range hosts {
s, ok := h.Get(name)
if !ok || !s.Healthy {
if !ok || !s.Healthy || !contains(s.Loaded, model) {
continue
}
if psc := s.PerSlotCtx(); psc > best {
if psc := s.PerSlotCtxFor(model); psc > best {
best = psc
}
}