Proxy: move or refuse prompts that do not fit the leased host's context
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
+10
-2
@@ -28,6 +28,7 @@ const (
|
||||
HostHeader = "X-Crossbar-Host"
|
||||
LeaseHeader = "X-Crossbar-Lease" // "new" or "reused"
|
||||
RouteHeader = "X-Crossbar-Route" // client may name the route here instead of the path
|
||||
CtxHeader = "X-Crossbar-Ctx" // "moved:<old>><new>" when the context was relocated
|
||||
)
|
||||
|
||||
// errBodyTooLarge is returned when a request body exceeds MaxBody during the model peek.
|
||||
@@ -213,7 +214,7 @@ func (p *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
p.writeError(w, http.StatusServiceUnavailable, "no healthy host")
|
||||
return
|
||||
}
|
||||
p.forward(w, r, route, name, "", rest, fp, model, started, 0)
|
||||
p.forward(w, r, route, name, "", rest, fp, model, started, 0, 0, "")
|
||||
return
|
||||
}
|
||||
|
||||
@@ -263,5 +264,12 @@ func (p *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
}
|
||||
defer release()
|
||||
|
||||
p.forward(w, r, route, host, leaseState(reused), rest, fp, model, started, waited)
|
||||
// Context guard: if the prompt does not fit the leased host's per-slot
|
||||
// context, move the conversation to a host where it fits, else answer 400.
|
||||
now := time.Now()
|
||||
host, header, _, done := p.guard(w, r, routeCfg.Hosts, host, route, model, fp, started)
|
||||
if done {
|
||||
return
|
||||
}
|
||||
p.forward(w, r, route, host, leaseState(reused), rest, fp, model, now, waited, 0, header)
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user