Route by lease, queue per host and model, record every request
Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
This commit is contained in:
+241
-288
@@ -1,318 +1,271 @@
|
||||
package proxy_test
|
||||
|
||||
// v1 acceptance tests for the proxy: leases, queueing, accounting, header route override.
|
||||
// They drive the whole handler over real HTTP against fake upstreams; only what a client or an
|
||||
// operator can observe is asserted (status codes, headers, the accounting rows, the health table).
|
||||
// The rig, the fake upstream and the request helpers live in helpers_test.go.
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/crossbar/internal/config"
|
||||
"git.wntrmute.dev/kyle/crossbar/internal/health"
|
||||
"git.wntrmute.dev/kyle/crossbar/internal/proxy"
|
||||
"git.wntrmute.dev/kyle/crossbar/internal/store"
|
||||
)
|
||||
|
||||
// fakeHealth is a hand-set health table that also records MarkDown calls.
|
||||
type fakeHealth struct {
|
||||
mu sync.Mutex
|
||||
st map[string]health.Status
|
||||
marked []string
|
||||
}
|
||||
|
||||
func (f *fakeHealth) Get(name string) (health.Status, bool) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
s, ok := f.st[name]
|
||||
return s, ok
|
||||
}
|
||||
|
||||
func (f *fakeHealth) MarkDown(name, reason string) {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
f.marked = append(f.marked, name)
|
||||
s := f.st[name]
|
||||
s.Healthy = false
|
||||
s.LastErr = reason
|
||||
f.st[name] = s
|
||||
}
|
||||
|
||||
func (f *fakeHealth) markedHosts() []string {
|
||||
f.mu.Lock()
|
||||
defer f.mu.Unlock()
|
||||
return append([]string{}, f.marked...)
|
||||
}
|
||||
|
||||
// upstream records what it received and answers with its name.
|
||||
type upstream struct {
|
||||
name string
|
||||
srv *httptest.Server
|
||||
mu sync.Mutex
|
||||
reqs []recorded
|
||||
}
|
||||
|
||||
type recorded struct {
|
||||
method, path, host, xff string
|
||||
body string
|
||||
}
|
||||
|
||||
func newUpstream(t *testing.T, name string) *upstream {
|
||||
u := &upstream{name: name}
|
||||
u.srv = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
b, _ := io.ReadAll(r.Body)
|
||||
u.mu.Lock()
|
||||
u.reqs = append(u.reqs, recorded{r.Method, r.URL.RequestURI(), r.Host, r.Header.Get("X-Forwarded-For"), string(b)})
|
||||
u.mu.Unlock()
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fmt.Fprintf(w, `{"from":%q}`, name)
|
||||
}))
|
||||
t.Cleanup(u.srv.Close)
|
||||
return u
|
||||
}
|
||||
|
||||
func (u *upstream) last(t *testing.T) recorded {
|
||||
u.mu.Lock()
|
||||
defer u.mu.Unlock()
|
||||
if len(u.reqs) == 0 {
|
||||
t.Fatalf("%s: no request received", u.name)
|
||||
func TestConversationIsStickyAndLeaseHeaderTellsWhy(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
r := newRig(t, twoHosts, alpha, beta)
|
||||
first := r.post("/r/v1/chat/completions", conversation(1, 1))
|
||||
drain(first)
|
||||
host := first.Header.Get(proxy.HostHeader)
|
||||
if first.StatusCode != 200 || host != "beta" { // beta: same free slots, double weight
|
||||
t.Fatalf("first turn: %d from %q, want 200 from beta", first.StatusCode, host)
|
||||
}
|
||||
if got := first.Header.Get(proxy.LeaseHeader); got != "new" {
|
||||
t.Errorf("%s = %q on the first turn, want new", proxy.LeaseHeader, got)
|
||||
}
|
||||
// Take alpha's slots away as a "better host" signal: it must not matter, the lease holds.
|
||||
for turn := 2; turn <= 6; turn++ {
|
||||
resp := r.post("/r/v1/chat/completions", conversation(1, turn))
|
||||
drain(resp)
|
||||
if resp.Header.Get(proxy.HostHeader) != host || resp.Header.Get(proxy.LeaseHeader) != "reused" {
|
||||
t.Fatalf("turn %d: host %q lease %q, want %q reused", turn, resp.Header.Get(proxy.HostHeader), resp.Header.Get(proxy.LeaseHeader), host)
|
||||
}
|
||||
}
|
||||
if alpha.hits.Load() != 0 || beta.hits.Load() != 6 {
|
||||
t.Errorf("hits alpha=%d beta=%d, want 0 and 6", alpha.hits.Load(), beta.hits.Load())
|
||||
}
|
||||
return u.reqs[len(u.reqs)-1]
|
||||
}
|
||||
|
||||
func cfgFor(t *testing.T, alpha, beta string) *config.Config {
|
||||
c, err := config.Parse(strings.NewReader(fmt.Sprintf(`
|
||||
// spreadHosts: beta is preferred (weight 10) until both of its "shared" slots are busy; then
|
||||
// alpha (2 free × 1) beats beta (0 free × 10), and a new conversation must start on alpha.
|
||||
const spreadHosts = `
|
||||
listen = "127.0.0.1:1"
|
||||
queue_max = 4
|
||||
lease_idle = "30m"
|
||||
[hosts.alpha]
|
||||
base_url = %q
|
||||
models = { "shared" = { }, "alpha-only" = { } }
|
||||
weight = 1.0
|
||||
models = { "shared" = { parallel = 2 } }
|
||||
[hosts.beta]
|
||||
base_url = %q
|
||||
models = { "shared" = { }, "beta-only" = { } }
|
||||
weight = 10.0
|
||||
models = { "shared" = { parallel = 2 } }
|
||||
[routes.r]
|
||||
hosts = ["alpha", "beta"]
|
||||
default_model = "shared"
|
||||
[routes.beta-first]
|
||||
hosts = ["beta", "alpha"]
|
||||
`, alpha, beta)))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return c
|
||||
}
|
||||
`
|
||||
|
||||
func healthy(loaded ...string) health.Status {
|
||||
return health.Status{Healthy: true, Loaded: loaded, Consecutive: 1}
|
||||
}
|
||||
|
||||
func TestSplitRoute(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
path, route, rest string
|
||||
ok bool
|
||||
}{
|
||||
{"/a/v1/x", "a", "/v1/x", true},
|
||||
{"/a/v1/x?q=1", "a", "/v1/x?q=1", true},
|
||||
{"/a", "a", "/", true},
|
||||
{"/a/", "a", "/", true},
|
||||
{"/opencode-a/v1/chat/completions", "opencode-a", "/v1/chat/completions", true},
|
||||
{"/", "", "", false},
|
||||
{"//x", "", "", false},
|
||||
{"", "", "", false},
|
||||
{"noslash/v1", "", "", false},
|
||||
} {
|
||||
route, rest, ok := proxy.SplitRoute(tc.path)
|
||||
if route != tc.route || rest != tc.rest || ok != tc.ok {
|
||||
t.Errorf("SplitRoute(%q) = %q %q %v, want %q %q %v", tc.path, route, rest, ok, tc.route, tc.rest, tc.ok)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestChoose(t *testing.T) {
|
||||
h := &fakeHealth{st: map[string]health.Status{
|
||||
"down": {Healthy: false, Loaded: []string{"m"}},
|
||||
"alpha": healthy("shared", "alpha-only"),
|
||||
"beta": healthy("shared", "beta-only"),
|
||||
}}
|
||||
hosts := []string{"down", "alpha", "beta"}
|
||||
if got, ok := proxy.Choose(hosts, "", h); !ok || got != "alpha" {
|
||||
t.Errorf("no model: %q %v, want alpha (first healthy)", got, ok)
|
||||
}
|
||||
if got, ok := proxy.Choose(hosts, "beta-only", h); !ok || got != "beta" {
|
||||
t.Errorf("beta-only: %q %v, want beta (has the model loaded)", got, ok)
|
||||
}
|
||||
if got, ok := proxy.Choose(hosts, "nobody-has-it", h); !ok || got != "alpha" {
|
||||
t.Errorf("unknown model falls back to the first healthy host: %q %v", got, ok)
|
||||
}
|
||||
if got, ok := proxy.Choose([]string{"down", "missing"}, "m", h); ok {
|
||||
t.Errorf("no healthy host must give ok=false, got %q", got)
|
||||
}
|
||||
if got, ok := proxy.Choose(nil, "m", h); ok {
|
||||
t.Errorf("empty hosts: %q %v", got, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoutesToFirstHealthyAndRewrites(t *testing.T) {
|
||||
func TestDifferentConversationsSpreadByFreeSlots(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
h := &fakeHealth{st: map[string]health.Status{"alpha": healthy("shared"), "beta": healthy("shared")}}
|
||||
p := proxy.New(cfgFor(t, alpha.srv.URL, beta.srv.URL), h, nil)
|
||||
rec := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodGet, "http://crossbar.local:7777/r/v1/models?x=1", nil)
|
||||
req.RemoteAddr = "10.9.8.7:5555"
|
||||
p.ServeHTTP(rec, req)
|
||||
if rec.Code != 200 || rec.Header().Get(proxy.HostHeader) != "alpha" {
|
||||
t.Fatalf("status %d host %q body %s", rec.Code, rec.Header().Get(proxy.HostHeader), rec.Body.String())
|
||||
}
|
||||
got := alpha.last(t)
|
||||
if got.path != "/v1/models?x=1" {
|
||||
t.Errorf("upstream path = %q, want route stripped and query kept", got.path)
|
||||
}
|
||||
if got.host != strings.TrimPrefix(alpha.srv.URL, "http://") {
|
||||
t.Errorf("Host header = %q, want the upstream's %q", got.host, strings.TrimPrefix(alpha.srv.URL, "http://"))
|
||||
}
|
||||
if got.xff != "10.9.8.7" {
|
||||
t.Errorf("X-Forwarded-For = %q, want the client address", got.xff)
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), `"from":"alpha"`) {
|
||||
t.Errorf("body = %s", rec.Body.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestModelPreferenceAndBodyPassThrough(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
h := &fakeHealth{st: map[string]health.Status{"alpha": healthy("shared", "alpha-only"), "beta": healthy("shared", "beta-only")}}
|
||||
p := proxy.New(cfgFor(t, alpha.srv.URL, beta.srv.URL), h, nil)
|
||||
body := `{"model":"beta-only","messages":[{"role":"user","content":"hi"}],"stream":false}`
|
||||
rec := httptest.NewRecorder()
|
||||
p.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/r/v1/chat/completions", strings.NewReader(body)))
|
||||
if rec.Code != 200 || rec.Header().Get(proxy.HostHeader) != "beta" {
|
||||
t.Fatalf("status %d host %q", rec.Code, rec.Header().Get(proxy.HostHeader))
|
||||
}
|
||||
if got := beta.last(t); got.body != body || got.method != http.MethodPost {
|
||||
t.Errorf("upstream got %+v; the body must arrive unchanged after the model peek", got)
|
||||
}
|
||||
// Not JSON: no model, the route default ("shared") applies, first healthy wins.
|
||||
rec = httptest.NewRecorder()
|
||||
p.ServeHTTP(rec, httptest.NewRequest(http.MethodPost, "/r/v1/embeddings", strings.NewReader("plain text")))
|
||||
if rec.Header().Get(proxy.HostHeader) != "alpha" {
|
||||
t.Errorf("non-JSON body: host %q, want alpha", rec.Header().Get(proxy.HostHeader))
|
||||
}
|
||||
if got := alpha.last(t); got.body != "plain text" {
|
||||
t.Errorf("non-JSON body must pass through unchanged, got %q", got.body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailoverOnUpstreamError(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
h := &fakeHealth{st: map[string]health.Status{"alpha": healthy("shared"), "beta": healthy("shared")}}
|
||||
p := proxy.New(cfgFor(t, alpha.srv.URL, beta.srv.URL), h, nil)
|
||||
alpha.srv.Close() // health still believes alpha is up
|
||||
rec := httptest.NewRecorder()
|
||||
p.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/r/v1/models", nil))
|
||||
if rec.Code != http.StatusBadGateway {
|
||||
t.Fatalf("first request after alpha died: %d, want 502", rec.Code)
|
||||
}
|
||||
var e map[string]string
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &e); err != nil || e["error"] != "upstream failed" || e["host"] != "alpha" {
|
||||
t.Errorf("502 body = %s", rec.Body.String())
|
||||
}
|
||||
if m := h.markedHosts(); len(m) != 1 || m[0] != "alpha" {
|
||||
t.Errorf("MarkDown calls = %v, want [alpha]", m)
|
||||
}
|
||||
rec = httptest.NewRecorder()
|
||||
p.ServeHTTP(rec, httptest.NewRequest(http.MethodGet, "/r/v1/models", nil))
|
||||
if rec.Code != 200 || rec.Header().Get(proxy.HostHeader) != "beta" {
|
||||
t.Errorf("second request: %d %q, want 200 from beta", rec.Code, rec.Header().Get(proxy.HostHeader))
|
||||
}
|
||||
}
|
||||
|
||||
func TestErrors(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
h := &fakeHealth{st: map[string]health.Status{"alpha": {Healthy: false}, "beta": {Healthy: false}}}
|
||||
p := proxy.New(cfgFor(t, alpha.srv.URL, beta.srv.URL), h, nil)
|
||||
for _, tc := range []struct {
|
||||
name, method, path string
|
||||
body io.Reader
|
||||
want int
|
||||
msg string
|
||||
}{
|
||||
{"bare slash", http.MethodGet, "/", nil, 400, "missing route"},
|
||||
{"double slash", http.MethodGet, "//v1/models", nil, 400, "missing route"},
|
||||
{"unknown route", http.MethodGet, "/nope/v1/models", nil, 404, "unknown route"},
|
||||
{"disallowed path", http.MethodGet, "/r/slots", nil, 404, "not found"},
|
||||
{"admin through proxy", http.MethodGet, "/r/_crossbar/hosts", nil, 404, "not found"},
|
||||
{"no healthy host", http.MethodGet, "/r/v1/models", nil, 503, "no healthy host"},
|
||||
{"body too large", http.MethodPost, "/r/v1/chat/completions", strings.NewReader(strings.Repeat("x", proxy.MaxBody+1)), 413, "body too large"},
|
||||
} {
|
||||
rec := httptest.NewRecorder()
|
||||
p.ServeHTTP(rec, httptest.NewRequest(tc.method, tc.path, tc.body))
|
||||
if rec.Code != tc.want {
|
||||
t.Errorf("%s: status %d, want %d", tc.name, rec.Code, tc.want)
|
||||
}
|
||||
var e map[string]string
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &e); err != nil || e["error"] != tc.msg {
|
||||
t.Errorf("%s: body %s, want error %q", tc.name, rec.Body.String(), tc.msg)
|
||||
}
|
||||
if !strings.HasPrefix(rec.Header().Get("Content-Type"), "application/json") {
|
||||
t.Errorf("%s: errors are JSON", tc.name)
|
||||
}
|
||||
}
|
||||
if len(h.markedHosts()) != 0 {
|
||||
t.Errorf("errors before choosing a host must not mark anything down: %v", h.markedHosts())
|
||||
}
|
||||
}
|
||||
|
||||
// TestStreamingIsNotBuffered: the upstream writes one chunk, flushes, and then waits until the
|
||||
// test has *read* that chunk. If the proxy buffered, the read would never complete.
|
||||
func TestStreamingIsNotBuffered(t *testing.T) {
|
||||
release := make(chan struct{})
|
||||
up := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.Header().Set("Content-Type", "text/event-stream")
|
||||
w.WriteHeader(200)
|
||||
fmt.Fprint(w, "data: first\n\n")
|
||||
w.(http.Flusher).Flush()
|
||||
select {
|
||||
case <-release:
|
||||
case <-time.After(5 * time.Second):
|
||||
}
|
||||
fmt.Fprint(w, "data: second\n\n")
|
||||
}))
|
||||
t.Cleanup(up.Close)
|
||||
beta := newUpstream(t, "beta")
|
||||
h := &fakeHealth{st: map[string]health.Status{"alpha": healthy("shared"), "beta": healthy("shared")}}
|
||||
front := httptest.NewServer(proxy.New(cfgFor(t, up.URL, beta.srv.URL), h, nil))
|
||||
t.Cleanup(front.Close)
|
||||
|
||||
resp, err := http.Post(front.URL+"/r/v1/chat/completions", "application/json", strings.NewReader(`{"model":"shared","stream":true}`))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
buf := make([]byte, 64)
|
||||
done := make(chan string, 1)
|
||||
go func() {
|
||||
n, err := resp.Body.Read(buf)
|
||||
if err != nil {
|
||||
done <- "read error: " + err.Error()
|
||||
return
|
||||
}
|
||||
done <- string(buf[:n])
|
||||
}()
|
||||
select {
|
||||
case got := <-done:
|
||||
if !strings.HasPrefix(got, "data: first") {
|
||||
t.Fatalf("first read = %q", got)
|
||||
}
|
||||
case <-time.After(2 * time.Second):
|
||||
t.Fatal("the first chunk did not arrive before the upstream finished: the proxy buffers")
|
||||
}
|
||||
close(release)
|
||||
rest, _ := io.ReadAll(resp.Body)
|
||||
if !strings.Contains(string(rest), "data: second") {
|
||||
t.Errorf("rest = %q", rest)
|
||||
beta.delay = 400 * time.Millisecond
|
||||
r := newRig(t, spreadHosts, alpha, beta)
|
||||
// Two slow conversations occupy beta's two "shared" slots…
|
||||
var wg sync.WaitGroup
|
||||
for i := 1; i <= 2; i++ {
|
||||
wg.Add(1)
|
||||
go func(i int) { defer wg.Done(); drain(r.post("/r/v1/chat/completions", conversation(i, 1))) }(i)
|
||||
time.Sleep(50 * time.Millisecond) // arrive one after the other so both pick beta (10 > 2)
|
||||
}
|
||||
time.Sleep(50 * time.Millisecond)
|
||||
// …so a third conversation starting now is sent to alpha (beta has 0 free slots, alpha 2).
|
||||
resp := r.post("/r/v1/chat/completions", conversation(3, 1))
|
||||
drain(resp)
|
||||
if resp.Header.Get(proxy.HostHeader) != "alpha" {
|
||||
t.Errorf("host header %q", resp.Header.Get(proxy.HostHeader))
|
||||
t.Errorf("third conversation went to %q, want alpha (free slots beat weight)", resp.Header.Get(proxy.HostHeader))
|
||||
}
|
||||
wg.Wait()
|
||||
if beta.hits.Load() != 2 || alpha.hits.Load() != 1 {
|
||||
t.Errorf("hits beta=%d alpha=%d, want 2 and 1", beta.hits.Load(), alpha.hits.Load())
|
||||
}
|
||||
}
|
||||
|
||||
func TestQueueFullIs503(t *testing.T) {
|
||||
alpha := newUpstream(t, "alpha")
|
||||
alpha.delay = 400 * time.Millisecond
|
||||
r := newRig(t, `
|
||||
listen = "127.0.0.1:1"
|
||||
queue_max = 1
|
||||
[hosts.alpha]
|
||||
base_url = %q
|
||||
models = { "shared" = { parallel = 1 } }
|
||||
[routes.r]
|
||||
hosts = ["alpha"]
|
||||
default_model = "shared"
|
||||
`, alpha)
|
||||
codes := make(chan int, 3)
|
||||
for i := 1; i <= 3; i++ {
|
||||
go func(i int) {
|
||||
resp := r.post("/r/v1/chat/completions", conversation(i, 1))
|
||||
drain(resp)
|
||||
codes <- resp.StatusCode
|
||||
}(i)
|
||||
time.Sleep(30 * time.Millisecond) // arrival order: 1 runs, 2 queues, 3 finds the queue full
|
||||
}
|
||||
got := map[int]int{}
|
||||
for i := 0; i < 3; i++ {
|
||||
got[<-codes]++
|
||||
}
|
||||
if got[200] != 2 || got[503] != 1 {
|
||||
t.Fatalf("status counts = %v, want two 200 and one 503", got)
|
||||
}
|
||||
// Rows are written after each response completes; allow the store a moment to catch up.
|
||||
var rows []store.UsageRow
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
for time.Now().Before(deadline) {
|
||||
rows, _ = r.store.Usage(time.Time{}, store.ByRoute)
|
||||
if len(rows) == 1 && rows[0].Requests == 3 {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Requests != 3 || rows[0].Errors != 1 {
|
||||
t.Fatalf("usage = %+v, want 3 requests, 1 error (the 503 is recorded too)", rows)
|
||||
}
|
||||
if rows[0].QueuedMs <= 0 {
|
||||
t.Errorf("the queued request must record its wait: %+v", rows[0])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUnhealthyHostReleasesAndMoves(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
r := newRig(t, twoHosts, alpha, beta)
|
||||
drain(r.post("/r/v1/chat/completions", conversation(1, 1))) // lands on beta
|
||||
beta.srv.Close()
|
||||
resp := r.post("/r/v1/chat/completions", conversation(1, 2))
|
||||
drain(resp)
|
||||
if resp.StatusCode != http.StatusBadGateway {
|
||||
t.Fatalf("first request after beta died: %d, want 502", resp.StatusCode)
|
||||
}
|
||||
if s, _ := r.health.Get("beta"); s.Healthy {
|
||||
t.Fatalf("beta must be marked down after the 502")
|
||||
}
|
||||
resp = r.post("/r/v1/chat/completions", conversation(1, 3))
|
||||
drain(resp)
|
||||
if resp.StatusCode != 200 || resp.Header.Get(proxy.HostHeader) != "alpha" || resp.Header.Get(proxy.LeaseHeader) != "new" {
|
||||
t.Errorf("after the move: %d from %q lease %q, want 200 alpha new", resp.StatusCode, resp.Header.Get(proxy.HostHeader), resp.Header.Get(proxy.LeaseHeader))
|
||||
}
|
||||
ev, _ := r.store.Events(time.Time{}, 10)
|
||||
var reasons []string
|
||||
for _, e := range ev {
|
||||
reasons = append(reasons, e.Reason)
|
||||
}
|
||||
if len(reasons) != 2 || reasons[0] != store.ReasonNew || reasons[1] != store.ReasonUnhealthy {
|
||||
t.Errorf("lease events = %v, want [new unhealthy]", reasons)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAccountingRowsFromUsageAndTimings(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
r := newRig(t, twoHosts, alpha, beta)
|
||||
drain(r.post("/r/v1/chat/completions", conversation(1, 1))) // non-streamed
|
||||
drain(r.post("/r/v1/chat/completions", strings.Replace(conversation(1, 2), `"stream":false`, `"stream":true`, 1))) // streamed
|
||||
deadline := time.Now().Add(2 * time.Second)
|
||||
var rows []store.UsageRow
|
||||
for time.Now().Before(deadline) {
|
||||
rows, _ = r.store.Usage(time.Time{}, store.ByHost)
|
||||
if len(rows) == 1 && rows[0].Requests == 2 {
|
||||
break
|
||||
}
|
||||
time.Sleep(20 * time.Millisecond)
|
||||
}
|
||||
if len(rows) != 1 || rows[0].Requests != 2 {
|
||||
t.Fatalf("usage by host = %+v, want one host with 2 requests (rows may be written after the response completes, within 2 s)", rows)
|
||||
}
|
||||
u := rows[0]
|
||||
if u.PromptTokens != 300 || u.CachedTokens != 240 || u.CompletionTokens != 30 {
|
||||
t.Errorf("tokens = prompt %d cached %d completion %d, want 300/240/30 (100+200, 90+150, 10+20)", u.PromptTokens, u.CachedTokens, u.CompletionTokens)
|
||||
}
|
||||
if u.BusyMs <= 0 || u.Errors != 0 {
|
||||
t.Errorf("busy %d errors %d", u.BusyMs, u.Errors)
|
||||
}
|
||||
if got := u.CacheHitRatio(); got < 0.79 || got > 0.81 {
|
||||
t.Errorf("cache hit ratio = %v, want 0.8", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStreamIsUnalteredWhileTeed(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
r := newRig(t, twoHosts, alpha, beta)
|
||||
resp := r.post("/r/v1/chat/completions", strings.Replace(conversation(9, 1), `"stream":false`, `"stream":true`, 1))
|
||||
body := drain(resp)
|
||||
want := 0
|
||||
for _, line := range strings.Split(body, "\n") {
|
||||
if strings.HasPrefix(line, "data: ") {
|
||||
want++
|
||||
}
|
||||
}
|
||||
if want != 5 || !strings.HasSuffix(strings.TrimSpace(body), "data: [DONE]") {
|
||||
t.Errorf("client must receive every SSE line untouched (3 deltas, usage, DONE); got %d data lines:\n%s", want, body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestHeaderRouteOverride(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
r := newRig(t, twoHosts, alpha, beta)
|
||||
// The header names the route; the path has none.
|
||||
resp := r.post("/v1/chat/completions", conversation(1, 1), proxy.RouteHeader, "other")
|
||||
drain(resp)
|
||||
if resp.StatusCode != 200 || resp.Header.Get(proxy.HostHeader) != "alpha" {
|
||||
t.Errorf("header route 'other' (alpha only): %d from %q", resp.StatusCode, resp.Header.Get(proxy.HostHeader))
|
||||
}
|
||||
if alpha.lastReq().path != "/v1/chat/completions" {
|
||||
t.Errorf("upstream path = %q", alpha.lastReq().path)
|
||||
}
|
||||
// A path route and a header route that disagree: the header is the operator's intent → 400.
|
||||
resp = r.post("/r/v1/chat/completions", conversation(1, 1), proxy.RouteHeader, "other")
|
||||
if drain(resp); resp.StatusCode != 400 {
|
||||
t.Errorf("conflicting route in path and header: %d, want 400", resp.StatusCode)
|
||||
}
|
||||
resp = r.post("/v1/chat/completions", conversation(1, 1), proxy.RouteHeader, "nope")
|
||||
if drain(resp); resp.StatusCode != 404 {
|
||||
t.Errorf("unknown header route: %d, want 404", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestV0BehaviourStillHolds(t *testing.T) {
|
||||
alpha, beta := newUpstream(t, "alpha"), newUpstream(t, "beta")
|
||||
r := newRig(t, twoHosts, alpha, beta)
|
||||
for _, tc := range []struct {
|
||||
method, path string
|
||||
want int
|
||||
msg string
|
||||
}{
|
||||
{http.MethodGet, "/", 400, "missing route"},
|
||||
{http.MethodGet, "/nope/v1/models", 404, "unknown route"},
|
||||
{http.MethodGet, "/r/slots", 404, "not found"},
|
||||
{http.MethodGet, "/r/_crossbar/hosts", 404, "not found"},
|
||||
} {
|
||||
req, _ := http.NewRequest(tc.method, r.front.URL+tc.path, nil)
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
body := drain(resp)
|
||||
var e map[string]string
|
||||
if resp.StatusCode != tc.want || json.Unmarshal([]byte(body), &e) != nil || e["error"] != tc.msg {
|
||||
t.Errorf("%s: %d %s, want %d %q", tc.path, resp.StatusCode, body, tc.want, tc.msg)
|
||||
}
|
||||
}
|
||||
big := strings.Repeat("x", proxy.MaxBody+1)
|
||||
resp := r.post("/r/v1/chat/completions", big)
|
||||
if drain(resp); resp.StatusCode != 413 {
|
||||
t.Errorf("oversize body: %d, want 413", resp.StatusCode)
|
||||
}
|
||||
// GET pass-through with query string, Host and X-Forwarded-For as in v0.
|
||||
resp, err := http.Get(r.front.URL + "/r/v1/models?x=1")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
drain(resp)
|
||||
host := resp.Header.Get(proxy.HostHeader)
|
||||
u := map[string]*upstream{"alpha": alpha, "beta": beta}[host]
|
||||
if u == nil || u.lastReq().path != "/v1/models?x=1" || u.lastReq().host != strings.TrimPrefix(u.srv.URL, "http://") || u.lastReq().xff == "" {
|
||||
t.Errorf("GET pass-through: host %q last %+v", host, u.lastReq())
|
||||
}
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user