Add the per-host-model limiter and the host chooser

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
2026-09-25 03:53:38 -07:00
parent 463cea18de
commit 7e0dbb4a6f
5 changed files with 505 additions and 0 deletions
@@ -0,0 +1,63 @@
+// Package choose picks the host for a new lease: among the healthy, non-draining, known hosts it
+// prefers those that have the model loaded over those that would only be able to serve it, then the
+// one with the most free slots times weight, breaking ties by shortest queue and finally list
+// order.
+package choose
+
+// Info is one candidate host's view of itself for a single model.
+type Info struct {
+	Healthy  bool // answered its last poll
+	Draining bool // operator is draining it; no new leases
+	Loaded   bool // the model is resident here
+	CanServe bool // config lists the model, so we may load it
+	Free     int
+	Queued   int
+	Weight   float64
+}
+
+// Best returns the best host for a new lease, or ok=false when nothing is eligible. It runs two
+// passes over the candidates in order: the first over those that have the model loaded, the second,
+// only if the first found nothing, over those that can serve it. A host must be healthy, not
+// draining, and known (info reported ok) to be eligible in either pass; a host with zero free slots
+// is still eligible, since it will queue. Among the eligible ones the highest Free*Weight wins, ties
+// go to the lowest queue, and a remaining tie keeps the earlier candidate.
+func Best(candidates []string, info func(host string) (Info, bool)) (string, bool) {
+	const (
+		passLoaded   = 0
+		passCanServe = 1
+	)
+	best := ""
+	var (
+		bestScore  float64
+		bestQueued int
+		found      bool
+	)
+	for pass := passLoaded; pass <= passCanServe; pass++ {
+		for _, host := range candidates {
+			v, ok := info(host)
+			if !ok || !v.Healthy || v.Draining {
+				continue
+			}
+			switch pass {
+			case passLoaded:
+				if !v.Loaded {
+					continue
+				}
+			case passCanServe:
+				if !v.CanServe {
+					continue
+				}
+			}
+			score := float64(v.Free) * v.Weight
+			// Replace only when strictly better on score, or equal score with a shorter queue; a
+			// further tie keeps the earlier candidate because we scan in order and use "<".
+			if !found || score > bestScore || (score == bestScore && v.Queued < bestQueued) {
+				best, bestScore, bestQueued, found = host, score, v.Queued, true
+			}
+		}
+		if found {
+			break
+		}
+	}
+	return best, found
+}
@@ -0,0 +1,83 @@
+package choose_test
+
+import (
+	"testing"
+
+	"git.wntrmute.dev/kyle/crossbar/internal/choose"
+)
+
+func infoFor(m map[string]choose.Info) func(string) (choose.Info, bool) {
+	return func(name string) (choose.Info, bool) { i, ok := m[name]; return i, ok }
+}
+
+func TestMostFreeSlotsTimesWeightWins(t *testing.T) {
+	info := infoFor(map[string]choose.Info{
+		"alpha": {Healthy: true, Loaded: true, CanServe: true, Free: 3, Weight: 1.0},
+		"beta":  {Healthy: true, Loaded: true, CanServe: true, Free: 2, Weight: 2.0}, // 4 > 3
+		"gamma": {Healthy: true, Loaded: true, CanServe: true, Free: 4, Weight: 0.5}, // 2
+	})
+	got, ok := choose.Best([]string{"alpha", "beta", "gamma"}, info)
+	if !ok || got != "beta" {
+		t.Errorf("got %q %v, want beta", got, ok)
+	}
+}
+
+func TestTieGoesToShortestQueueThenListOrder(t *testing.T) {
+	info := infoFor(map[string]choose.Info{
+		"alpha": {Healthy: true, Loaded: true, CanServe: true, Free: 2, Weight: 1, Queued: 3},
+		"beta":  {Healthy: true, Loaded: true, CanServe: true, Free: 2, Weight: 1, Queued: 1},
+		"gamma": {Healthy: true, Loaded: true, CanServe: true, Free: 2, Weight: 1, Queued: 1},
+	})
+	if got, _ := choose.Best([]string{"alpha", "beta", "gamma"}, info); got != "beta" {
+		t.Errorf("tie on score: shortest queue wins, then list order; got %q", got)
+	}
+	if got, _ := choose.Best([]string{"gamma", "beta"}, info); got != "gamma" {
+		t.Errorf("full tie: first in list order wins; got %q", got)
+	}
+}
+
+func TestLoadedBeatsMerelyCapable(t *testing.T) {
+	info := infoFor(map[string]choose.Info{
+		"alpha": {Healthy: true, Loaded: false, CanServe: true, Free: 8, Weight: 4},
+		"beta":  {Healthy: true, Loaded: true, CanServe: true, Free: 1, Weight: 1},
+	})
+	got, ok := choose.Best([]string{"alpha", "beta"}, info)
+	if !ok || got != "beta" {
+		t.Errorf("a host that has the model loaded wins over one that would have to load it; got %q", got)
+	}
+}
+
+func TestFallsBackToCapableHost(t *testing.T) {
+	info := infoFor(map[string]choose.Info{
+		"alpha": {Healthy: true, Loaded: false, CanServe: true, Free: 1, Weight: 1},
+		"beta":  {Healthy: true, Loaded: false, CanServe: false, Free: 9, Weight: 9},
+	})
+	got, ok := choose.Best([]string{"beta", "alpha"}, info)
+	if !ok || got != "alpha" {
+		t.Errorf("only a host configured to serve the model may load it; got %q %v", got, ok)
+	}
+}
+
+func TestSkipsUnhealthyDrainingUnknownAndFull(t *testing.T) {
+	info := infoFor(map[string]choose.Info{
+		"down":  {Healthy: false, Loaded: true, CanServe: true, Free: 9, Weight: 9},
+		"drain": {Healthy: true, Draining: true, Loaded: true, CanServe: true, Free: 9, Weight: 9},
+		"full":  {Healthy: true, Loaded: true, CanServe: true, Free: 0, Weight: 9, Queued: 0},
+		"ok":    {Healthy: true, Loaded: true, CanServe: true, Free: 1, Weight: 1},
+	})
+	got, ok := choose.Best([]string{"down", "drain", "missing", "full", "ok"}, info)
+	if !ok || got != "ok" {
+		t.Errorf("got %q %v, want ok", got, ok)
+	}
+	// A full host is still better than nothing: it gets the request (it will queue).
+	got, ok = choose.Best([]string{"down", "full"}, info)
+	if !ok || got != "full" {
+		t.Errorf("with only a full host left it must still be chosen; got %q %v", got, ok)
+	}
+	if _, ok := choose.Best([]string{"down", "drain", "missing"}, info); ok {
+		t.Errorf("nothing usable must give ok=false")
+	}
+	if _, ok := choose.Best(nil, info); ok {
+		t.Errorf("empty candidates must give ok=false")
+	}
+}