Proxy: move or refuse prompts that do not fit the leased host's context
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
@@ -14,8 +14,10 @@ import (
|
||||
)
|
||||
|
||||
// forward builds the reverse proxy for one host, tees the response, records the accounting row, and
|
||||
// logs. leaseState is "new" or "reused"; waited is the time spent in the queue.
|
||||
func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, leaseState, rest, fp, model string, started time.Time, waited time.Duration) {
|
||||
// logs. leaseState is "new" or "reused"; waited is the time spent in the queue. ctxEst is the
|
||||
// prompt size the context guard estimated (0 when the guard did not run); ctxHeader is the
|
||||
// "moved:…<host>" header to set when the guard relocated the conversation.
|
||||
func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, leaseState, rest, fp, model string, started time.Time, waited time.Duration, ctxEst int, ctxHeader string) {
|
||||
hostCfg, ok := p.cfg.Hosts[host]
|
||||
if !ok {
|
||||
p.writeError(w, http.StatusBadGateway, "upstream failed")
|
||||
@@ -28,7 +30,7 @@ func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, l
|
||||
}
|
||||
|
||||
rev := &forwardState{started: started}
|
||||
rp := newReverseProxy(p.health, host, leaseState, target, rest, rev)
|
||||
rp := newReverseProxy(p.health, host, leaseState, ctxHeader, target, rest, rev)
|
||||
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
|
||||
|
||||
// ServeHTTP unwinds with http.ErrAbortHandler when a client leaves mid-stream; recover so the
|
||||
@@ -71,6 +73,7 @@ func (p *Handler) forward(w http.ResponseWriter, r *http.Request, route, host, l
|
||||
"lease", leaseState,
|
||||
"queued_ms", waited.Milliseconds(),
|
||||
"fp", fp8,
|
||||
"ctx_est", ctxEst,
|
||||
"ms", total.Milliseconds(),
|
||||
)
|
||||
}
|
||||
@@ -163,7 +166,7 @@ func (p *Handler) writeRecord(req store.Request) {
|
||||
// original query string. It flushes after every write so long server-sent-event streams are not
|
||||
// buffered, tees the response for usage/timings, and marks the host down on any transport error
|
||||
// other than a client disconnect.
|
||||
func newReverseProxy(h Health, host, leaseState string, target *url.URL, rest string, rev *forwardState) *httputil.ReverseProxy {
|
||||
func newReverseProxy(h Health, host, leaseState, ctxHeader string, target *url.URL, rest string, rev *forwardState) *httputil.ReverseProxy {
|
||||
return &httputil.ReverseProxy{
|
||||
Rewrite: func(pr *httputil.ProxyRequest) {
|
||||
pr.SetURL(target)
|
||||
@@ -176,6 +179,9 @@ func newReverseProxy(h Health, host, leaseState string, target *url.URL, rest st
|
||||
ModifyResponse: func(resp *http.Response) error {
|
||||
resp.Header.Set(HostHeader, host)
|
||||
resp.Header.Set(LeaseHeader, leaseState)
|
||||
if ctxHeader != "" {
|
||||
resp.Header.Set(CtxHeader, ctxHeader)
|
||||
}
|
||||
rev.ttfb = time.Now()
|
||||
rev.streamed = strings.HasPrefix(resp.Header.Get("Content-Type"), "text/event-stream")
|
||||
t := newTee(resp.Body, rev.streamed)
|
||||
|
||||
Reference in New Issue
Block a user