Learn per-model context from /props?model=; only status "loaded" is loaded

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
2026-09-25 11:47:35 -07:00
parent f3dfdbfa50
commit 055ab079d4
9 changed files with 562 additions and 86 deletions
@@ -21,15 +21,22 @@ const RecoveryPolls = 2
 // MaxModelsBody bounds how many bytes we read from either /health or /v1/models.
 const MaxModelsBody = 1 << 20

+// ModelCtx is what /props?model=X taught us about one loaded model.
+type ModelCtx struct {
+	NCtx  int `json:"n_ctx"`
+	Slots int `json:"slots"`
+}
+
 // Status is a snapshot of one host's health, safe to copy.
 type Status struct {
-	Healthy     bool      `json:"healthy"`
-	Loaded      []string  `json:"loaded"`      // sorted, unique model ids from the last good poll
-	LastOK      time.Time `json:"last_ok"`     // zero if never
-	LastErr     string    `json:"last_err"`    // "" after a good poll
-	Consecutive int       `json:"consecutive"` // good polls in a row
-	NCtx        int       `json:"n_ctx"`       // total context from /props; 0 = unknown
-	Slots       int       `json:"slots"`       // total_slots from /props; 0 = unknown
+	Healthy     bool                `json:"healthy"`
+	Loaded      []string            `json:"loaded"`      // sorted, unique model ids from the last good poll
+	LastOK      time.Time           `json:"last_ok"`     // zero if never
+	LastErr     string              `json:"last_err"`    // "" after a good poll
+	Consecutive int                 `json:"consecutive"` // good polls in a row
+	NCtx        int                 `json:"n_ctx"`       // total context from /props; 0 = unknown (a router's own /props carries none)
+	Slots       int                 `json:"slots"`       // total_slots from /props; 0 = unknown
+	Models      map[string]ModelCtx `json:"models"`      // per loaded model; never nil after a poll
 }

 // PerSlotCtx is the context one request may use: NCtx divided by Slots, or the
@@ -41,6 +48,22 @@ func (s Status) PerSlotCtx() int {
 	return s.NCtx / s.Slots
 }

+// PerSlotCtxFor is the per-slot context for one model on this host: Models[model]
+// when present (NCtx/Slots, 0 when either is 0); else, when model is in Loaded,
+// the host-level PerSlotCtx(); else 0 ("unknown" / not resident).
+func (s Status) PerSlotCtxFor(model string) int {
+	if mc, ok := s.Models[model]; ok {
+		if mc.NCtx == 0 || mc.Slots == 0 {
+			return 0
+		}
+		return mc.NCtx / mc.Slots
+	}
+	if contains(s.Loaded, model) {
+		return s.PerSlotCtx()
+	}
+	return 0
+}
+
 type entry struct {
 	status     Status
 	everFailed bool
@@ -53,6 +76,7 @@ type pollResult struct {
 	loaded    []string
 	nctx      int
 	slots     int
+	models    map[string]ModelCtx
 }

 // Table maps a host name to its health status. All methods are safe for concurrent use.
@@ -171,6 +195,7 @@ func (t *Table) pollHost(ctx context.Context, name string) {
 		e.status.Loaded = r.loaded
 		e.status.NCtx = r.nctx
 		e.status.Slots = r.slots
+		e.status.Models = r.models
 		e.status.Healthy = !e.everFailed || e.status.Consecutive >= RecoveryPolls
 	} else {
 		e.everFailed = true
@@ -196,57 +221,11 @@ func (t *Table) poll(ctx context.Context, base string) pollResult {
 	if r.cancelled || r.reason != "" {
 		return r
 	}
-	return pollResult{ok: true, loaded: loaded, nctx: nctx, slots: slots}
-}
-
-// props reads <base>/props best-effort. A request that fails because ctx is
-// done yields a cancelled result so the caller records nothing; any other
-// outcome (status, body, or missing fields) leaves context unknown without
-// failing the poll.
-func (t *Table) props(ctx context.Context, base string) (int, int, pollResult) {
-	req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/props", nil)
-	if err != nil {
-		if ctx.Err() != nil {
-			return 0, 0, pollResult{cancelled: true}
-		}
-		return 0, 0, pollResult{}
+	models, r := t.propsModels(ctx, base, loaded)
+	if r.cancelled || r.reason != "" {
+		return r
 	}
-	resp, err := t.client.Do(req)
-	if err != nil {
-		if ctx.Err() != nil {
-			return 0, 0, pollResult{cancelled: true}
-		}
-		return 0, 0, pollResult{}
-	}
-	defer resp.Body.Close()
-
-	if resp.StatusCode != http.StatusOK {
-		return 0, 0, pollResult{}
-	}
-	var p struct {
-		Generation struct {
-			NCtx *int `json:"n_ctx"`
-		} `json:"default_generation_settings"`
-		TotalSlots *int `json:"total_slots"`
-	}
-	if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&p); err != nil {
-		return 0, 0, pollResult{}
-	}
-
-	nctx, slots := 0, 0
-	if p.Generation.NCtx != nil {
-		nctx = *p.Generation.NCtx
-	}
-	if p.TotalSlots != nil {
-		slots = *p.TotalSlots
-	}
-	if nctx < 0 {
-		nctx = 0
-	}
-	if slots < 0 {
-		slots = 0
-	}
-	return nctx, slots, pollResult{}
+	return pollResult{ok: true, loaded: loaded, nctx: nctx, slots: slots, models: models}
 }

 // check performs one GET and, on success, returns the decoded model ids. Health checks use the
@@ -280,7 +259,10 @@ func (t *Table) check(ctx context.Context, url, prefix string) ([]string, pollRe
 	}
 	var m struct {
 		Data []struct {
-			ID string `json:"id"`
+			ID     string `json:"id"`
+			Status struct {
+				Value string `json:"value"`
+			} `json:"status"`
 		} `json:"data"`
 	}
 	if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&m); err != nil {
@@ -289,6 +271,8 @@ func (t *Table) check(ctx context.Context, url, prefix string) ([]string, pollRe
 	}
 	defer resp.Body.Close()

+	// A model is loaded when it has no status or status.value == "loaded"; any other
+	// value ("unloaded", "loading", …) is not loaded and must never be asked /props?model=.
 	loaded := make([]string, 0, len(m.Data))
 	seen := make(map[string]struct{}, len(m.Data))
 	for _, d := range m.Data {
@@ -298,6 +282,9 @@ func (t *Table) check(ctx context.Context, url, prefix string) ([]string, pollRe
 		if _, ok := seen[d.ID]; ok {
 			continue
 		}
+		if d.Status.Value != "" && d.Status.Value != "loaded" {
+			continue
+		}
 		seen[d.ID] = struct{}{}
 		loaded = append(loaded, d.ID)
 	}
@@ -314,11 +301,25 @@ func (t *Table) fail(ctx context.Context, prefix string, err error) pollResult {
 }

 func copyStatus(s Status) Status {
-	if s.Loaded == nil {
-		return s
-	}
 	out := s
-	out.Loaded = make([]string, len(s.Loaded))
-	copy(out.Loaded, s.Loaded)
+	if s.Loaded != nil {
+		out.Loaded = make([]string, len(s.Loaded))
+		copy(out.Loaded, s.Loaded)
+	}
+	if s.Models != nil {
+		out.Models = make(map[string]ModelCtx, len(s.Models))
+		for k, v := range s.Models {
+			out.Models[k] = v
+		}
+	}
 	return out
 }
+
+func contains(list []string, v string) bool {
+	for _, s := range list {
+		if s == v {
+			return true
+		}
+	}
+	return false
+}
@@ -0,0 +1,140 @@
+package health
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/url"
+)
+
+// modelProps is the part of a /props body that carries a model's context size.
+type modelProps struct {
+	Generation struct {
+		NCtx *int `json:"n_ctx"`
+	} `json:"default_generation_settings"`
+	TotalSlots *int `json:"total_slots"`
+}
+
+// props reads <base>/props best-effort. A request that fails because ctx is
+// done yields a cancelled result so the caller records nothing; any other
+// outcome (status, body, or missing fields) leaves context unknown without
+// failing the poll. When the body says "role":"router" the host carries no
+// context of its own, so NCtx and Slots stay 0 whatever else it reports.
+func (t *Table) props(ctx context.Context, base string) (int, int, pollResult) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/props", nil)
+	if err != nil {
+		if ctx.Err() != nil {
+			return 0, 0, pollResult{cancelled: true}
+		}
+		return 0, 0, pollResult{}
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		if ctx.Err() != nil {
+			return 0, 0, pollResult{cancelled: true}
+		}
+		return 0, 0, pollResult{}
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return 0, 0, pollResult{}
+	}
+	var p struct {
+		Role       string `json:"role"`
+		Generation struct {
+			NCtx *int `json:"n_ctx"`
+		} `json:"default_generation_settings"`
+		TotalSlots *int `json:"total_slots"`
+	}
+	if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&p); err != nil {
+		return 0, 0, pollResult{}
+	}
+	if p.Role == "router" {
+		return 0, 0, pollResult{}
+	}
+	nctx, slots := 0, 0
+	if p.Generation.NCtx != nil {
+		nctx = *p.Generation.NCtx
+	}
+	if p.TotalSlots != nil {
+		slots = *p.TotalSlots
+	}
+	if nctx < 0 {
+		nctx = 0
+	}
+	if slots < 0 {
+		slots = 0
+	}
+	return nctx, slots, pollResult{}
+}
+
+// propsModels asks /props?model= for each loaded model and returns a fresh,
+// non-nil map of what each answered. A failed or malformed answer leaves that
+// id absent; a request that fails because ctx is done cancels the whole poll.
+// No model is ever asked that is not in loaded.
+func (t *Table) propsModels(ctx context.Context, base string, loaded []string) (map[string]ModelCtx, pollResult) {
+	models := make(map[string]ModelCtx, len(loaded))
+	for _, id := range loaded {
+		mc, ok, r := t.propsModel(ctx, base, id)
+		if r.cancelled || r.reason != "" {
+			return nil, r
+		}
+		if !ok {
+			continue
+		}
+		models[id] = mc
+	}
+	return models, pollResult{}
+}
+
+// propsModel reads <base>/props?model=<id> best-effort. ok is false when the
+// answer failed or was malformed (the model stays unknown); a request that
+// fails because ctx is done yields a cancelled result so the caller records
+// nothing.
+func (t *Table) propsModel(ctx context.Context, base, id string) (ModelCtx, bool, pollResult) {
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, base+"/props?model="+url.QueryEscape(id), nil)
+	if err != nil {
+		if ctx.Err() != nil {
+			return ModelCtx{}, false, pollResult{cancelled: true}
+		}
+		return ModelCtx{}, false, pollResult{}
+	}
+	resp, err := t.client.Do(req)
+	if err != nil {
+		if ctx.Err() != nil {
+			return ModelCtx{}, false, pollResult{cancelled: true}
+		}
+		return ModelCtx{}, false, pollResult{}
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode != http.StatusOK {
+		return ModelCtx{}, false, pollResult{}
+	}
+	var p modelProps
+	if err := json.NewDecoder(io.LimitReader(resp.Body, MaxModelsBody)).Decode(&p); err != nil {
+		return ModelCtx{}, false, pollResult{}
+	}
+	mc, r := clampCtx(p.Generation.NCtx, p.TotalSlots)
+	return mc, true, r
+}
+
+// clampCtx turns the two optional fields into a non-negative context and slots.
+func clampCtx(nctx, slots *int) (ModelCtx, pollResult) {
+	c := ModelCtx{}
+	if nctx != nil {
+		c.NCtx = *nctx
+	}
+	if slots != nil {
+		c.Slots = *slots
+	}
+	if c.NCtx < 0 {
+		c.NCtx = 0
+	}
+	if c.Slots < 0 {
+		c.Slots = 0
+	}
+	return c, pollResult{}
+}
@@ -0,0 +1,177 @@
+package health_test
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"sync"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/crossbar/internal/health"
+)
+
+// routerFake is shaped like llama-server's router mode: /v1/models lists every configured model
+// with a status, a plain /props answers as the router itself (no context), and /props?model=X
+// answers for one loaded child server. It counts the per-model /props queries it receives.
+type routerFake struct {
+	srv     *httptest.Server
+	mu      sync.Mutex
+	queries map[string]int
+}
+
+func newRouterFake(t *testing.T) *routerFake {
+	f := &routerFake{queries: map[string]int{}}
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"ok"}`) })
+	mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `{"object":"list","data":[
+			{"id":"big","object":"model","status":{"value":"loaded","args":["--ctx-size","262144"]}},
+			{"id":"small","object":"model","status":{"value":"loaded"}},
+			{"id":"cold","object":"model","status":{"value":"unloaded"}},
+			{"id":"warming","object":"model","status":{"value":"loading"}}]}`)
+	})
+	mux.HandleFunc("/props", func(w http.ResponseWriter, r *http.Request) {
+		model := r.URL.Query().Get("model")
+		if model == "" {
+			fmt.Fprint(w, `{"role":"router","model_alias":"llama-server","model_path":"none","default_generation_settings":{"params":null,"n_ctx":0}}`)
+			return
+		}
+		f.mu.Lock()
+		f.queries[model]++
+		f.mu.Unlock()
+		switch model {
+		case "big":
+			fmt.Fprint(w, `{"default_generation_settings":{"n_ctx":262144,"params":{}},"total_slots":4,"model_alias":"big"}`)
+		case "small":
+			fmt.Fprint(w, `{"default_generation_settings":{"n_ctx":32768,"params":{}},"total_slots":1,"model_alias":"small"}`)
+		default:
+			// Asking a router for an unloaded model would make it load the model. The fake
+			// answers 500 so a wrong query is visible in the counts and cannot look like success.
+			w.WriteHeader(http.StatusInternalServerError)
+			fmt.Fprint(w, `{"error":"the poller must not ask for a model that is not loaded"}`)
+		}
+	})
+	f.srv = httptest.NewServer(mux)
+	t.Cleanup(f.srv.Close)
+	return f
+}
+
+func (f *routerFake) count(model string) int {
+	f.mu.Lock()
+	defer f.mu.Unlock()
+	return f.queries[model]
+}
+
+func TestRouterLoadedMeansStatusLoaded(t *testing.T) {
+	f := newRouterFake(t)
+	tbl := health.New(map[string]string{"r": f.srv.URL}, time.Hour, nil)
+	tbl.PollOnce(context.Background())
+	s, ok := tbl.Get("r")
+	if !ok || !s.Healthy {
+		t.Fatalf("status = %+v, want a healthy host", s)
+	}
+	if len(s.Loaded) != 2 || s.Loaded[0] != "big" || s.Loaded[1] != "small" {
+		t.Errorf("Loaded = %v, want [big small]: unloaded and loading models are not loaded", s.Loaded)
+	}
+}
+
+func TestRouterContextIsLearnedPerModel(t *testing.T) {
+	f := newRouterFake(t)
+	tbl := health.New(map[string]string{"r": f.srv.URL}, time.Hour, nil)
+	tbl.PollOnce(context.Background())
+	s, _ := tbl.Get("r")
+	if s.NCtx != 0 || s.Slots != 0 || s.PerSlotCtx() != 0 {
+		t.Errorf("a router's own /props carries no context; host-level must stay unknown: %+v", s)
+	}
+	if got := s.Models["big"]; got != (health.ModelCtx{NCtx: 262144, Slots: 4}) {
+		t.Errorf("Models[big] = %+v, want {262144 4}", got)
+	}
+	if got := s.Models["small"]; got != (health.ModelCtx{NCtx: 32768, Slots: 1}) {
+		t.Errorf("Models[small] = %+v, want {32768 1}", got)
+	}
+	if got := s.PerSlotCtxFor("big"); got != 65536 {
+		t.Errorf("PerSlotCtxFor(big) = %d, want 262144/4", got)
+	}
+	if got := s.PerSlotCtxFor("small"); got != 32768 {
+		t.Errorf("PerSlotCtxFor(small) = %d, want 32768/1", got)
+	}
+	if got := s.PerSlotCtxFor("cold"); got != 0 {
+		t.Errorf("PerSlotCtxFor(cold) = %d, want 0: nothing is known about an unloaded model", got)
+	}
+	if _, present := s.Models["cold"]; present {
+		t.Errorf("Models must not carry an entry for an unloaded model: %+v", s.Models)
+	}
+}
+
+func TestRouterUnloadedModelsAreNeverQueried(t *testing.T) {
+	f := newRouterFake(t)
+	tbl := health.New(map[string]string{"r": f.srv.URL}, time.Hour, nil)
+	for i := 0; i < 3; i++ {
+		tbl.PollOnce(context.Background())
+	}
+	if f.count("cold") != 0 || f.count("warming") != 0 {
+		t.Fatalf("/props?model= was asked for a model that is not loaded (cold %d, warming %d): on a real router that loads the model", f.count("cold"), f.count("warming"))
+	}
+	if f.count("big") == 0 || f.count("small") == 0 {
+		t.Errorf("loaded models must be asked: big %d, small %d", f.count("big"), f.count("small"))
+	}
+}
+
+func TestPlainServerStillReadsHostLevelContext(t *testing.T) {
+	// A single llama-server (no status field, no router role) behaves as in v2: every listed model
+	// is loaded, the host-level context comes from the plain /props, and the per-model view falls
+	// back to it for any loaded model.
+	srv := propsFake(t, `{"default_generation_settings":{"n_ctx":131072,"params":{}},"total_slots":4,"model_path":"/x/m.gguf"}`, 200)
+	tbl := health.New(map[string]string{"a": srv.URL}, time.Hour, nil)
+	tbl.PollOnce(context.Background())
+	s, _ := tbl.Get("a")
+	if !s.Healthy || len(s.Loaded) != 1 || s.Loaded[0] != "m" || s.NCtx != 131072 || s.Slots != 4 {
+		t.Fatalf("status = %+v, want healthy, Loaded [m], NCtx 131072, Slots 4", s)
+	}
+	if got := s.PerSlotCtxFor("m"); got != 32768 {
+		t.Errorf("PerSlotCtxFor(m) = %d, want the host-level 131072/4", got)
+	}
+	if got := s.PerSlotCtxFor("other"); got != 0 {
+		t.Errorf("PerSlotCtxFor(other) = %d, want 0 for a model the host does not list", got)
+	}
+	if s.Models == nil {
+		t.Errorf("Models must be an empty map after a poll, never nil")
+	}
+}
+
+func TestPerModelPropsFailureLeavesTheModelUnknown(t *testing.T) {
+	mux := http.NewServeMux()
+	mux.HandleFunc("/health", func(w http.ResponseWriter, r *http.Request) { fmt.Fprint(w, `{"status":"ok"}`) })
+	mux.HandleFunc("/v1/models", func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `{"data":[{"id":"ok","status":{"value":"loaded"}},{"id":"broken","status":{"value":"loaded"}}]}`)
+	})
+	mux.HandleFunc("/props", func(w http.ResponseWriter, r *http.Request) {
+		switch r.URL.Query().Get("model") {
+		case "":
+			fmt.Fprint(w, `{"role":"router","default_generation_settings":{"n_ctx":0}}`)
+		case "ok":
+			fmt.Fprint(w, `{"default_generation_settings":{"n_ctx":8192},"total_slots":2}`)
+		default:
+			fmt.Fprint(w, `<html>not json</html>`)
+		}
+	})
+	srv := httptest.NewServer(mux)
+	t.Cleanup(srv.Close)
+	tbl := health.New(map[string]string{"r": srv.URL}, time.Hour, nil)
+	tbl.PollOnce(context.Background())
+	s, _ := tbl.Get("r")
+	if !s.Healthy {
+		t.Fatalf("a broken per-model /props must not make the host unhealthy: %+v", s)
+	}
+	if len(s.Loaded) != 2 {
+		t.Errorf("Loaded = %v, want both models: /props is advisory", s.Loaded)
+	}
+	if got := s.PerSlotCtxFor("ok"); got != 4096 {
+		t.Errorf("PerSlotCtxFor(ok) = %d, want 8192/2", got)
+	}
+	if _, present := s.Models["broken"]; present || s.PerSlotCtxFor("broken") != 0 {
+		t.Errorf("a model whose /props failed stays unknown: %+v", s.Models)
+	}
+}