Proxy: move or refuse prompts that do not fit the leased host's context

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
2026-09-25 09:56:46 -07:00
parent 959aec92f1
commit 2fe3b9c865
7 changed files with 352 additions and 6 deletions
+10 -4
View File
@@ -14,8 +14,10 @@ import (
)
// forward builds the reverse proxy for one host, tees the response, records the accounting row, and
// logs. leaseState is "new" or "reused"; waited is the time spent in the queue.
func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, leaseState, rest, fp, model string, started time.Time, waited time.Duration) {
// logs. leaseState is "new" or "reused"; waited is the time spent in the queue. ctxEst is the
// prompt size the context guard estimated (0 when the guard did not run); ctxHeader is the
// "moved:…<host>" header to set when the guard relocated the conversation.
func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, leaseState, rest, fp, model string, started time.Time, waited time.Duration, ctxEst int, ctxHeader string) {
hostCfg, ok := p.cfg.Hosts[host]
if !ok {
p.writeError(w, http.StatusBadGateway, "upstream failed")
@@ -28,7 +30,7 @@ func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, l
}
rev := &forwardState{started: started}
rp := newReverseProxy(p.health, host, leaseState, target, rest, rev)
rp := newReverseProxy(p.health, host, leaseState, ctxHeader, target, rest, rev)
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
// ServeHTTP unwinds with http.ErrAbortHandler when a client leaves mid-stream; recover so the
@@ -71,6 +73,7 @@ func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, l
"lease", leaseState,
"queued_ms", waited.Milliseconds(),
"fp", fp8,
"ctx_est", ctxEst,
"ms", total.Milliseconds(),
)
}
@@ -163,7 +166,7 @@ func (p *Handler) writeRecord(req store.Request) {
// original query string. It flushes after every write so long server-sent-event streams are not
// buffered, tees the response for usage/timings, and marks the host down on any transport error
// other than a client disconnect.
func newReverseProxy(h Health, host, leaseState string, target *url.URL, rest string, rev *forwardState) *httputil.ReverseProxy {
func newReverseProxy(h Health, host, leaseState, ctxHeader string, target *url.URL, rest string, rev *forwardState) *httputil.ReverseProxy {
return &httputil.ReverseProxy{
Rewrite: func(pr *httputil.ProxyRequest) {
pr.SetURL(target)
@@ -176,6 +179,9 @@ func newReverseProxy(h Health, host, leaseState string, target *url.URL, rest st
ModifyResponse: func(resp *http.Response) error {
resp.Header.Set(HostHeader, host)
resp.Header.Set(LeaseHeader, leaseState)
if ctxHeader != "" {
resp.Header.Set(CtxHeader, ctxHeader)
}
rev.ttfb = time.Now()
rev.streamed = strings.HasPrefix(resp.Header.Get("Content-Type"), "text/event-stream")
t := newTee(resp.Body, rev.streamed)