Add the sticky lease table

Implemented-By: OpenCode session (model recorded in docs/implementer-log.md)
2026-09-25 04:26:47 -07:00
parent e65826b369
commit 9133240cfb
3 changed files with 624 additions and 0 deletions
@@ -5,6 +5,7 @@ owner fills in the Model column. The reviewer adds findings under "Reviews" once

 | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model |
 |---|---|---|---|---|---|---|---|
+| v1/04-lease | 2026-09-25 | done | 1 | pass | The given `TestPinAndUnpin` was wrong and replaced by the owner mid-task; the corrected `internal/lease/lease_test.go` is byte-identical to `docs/plans/v1/_files/internal/lease/lease_test.go`. A `fmt.Printf("DEBUG …")` line the prior session left in `event` was removed before the gate. | `Acquire` order (pinned, existing, inherit, choose) with memory rolled back only after a successful save; `Pin` writes a pin event, then the pin row, then deletes other-host leases, so the pin event always precedes the unpin's release event in the log. | ? |
 | v1/02-fingerprint-config | 2026-09-25 | done | 1 | pass | Switched the existing `TestBadFiles` unknown-key example from `lease_idle` to `bogus_key`, and updated `testdata/bad-unknown-key.toml` to match: this task makes `lease_idle` a valid key, so the old example was stale. `config_test.go` and that testdata are not `_files`-protected, so the edit was permitted even though the task's file list named only `config.go` and `implementer-log.md`; the unknown-key rejection is still covered. | fingerprint.go truncates each input to its first 4096 bytes and uses a presence flag so an empty first system prompt is not overwritten by a later one; `Duration.UnmarshalText` matches `^[0-9]+d$` (regexp) before falling to `time.ParseDuration`. | ? |
 | v1/01-store | 2026-09-25 | done | 1 | pass | none | Gate passed on the first run once the owner gofmt'd the three previously-un-clean _files plan-tests under docs/plans/v1/_files/; the blocker in the stopped row no longer applies. | ? |
 | v1/01-store | 2026-09-25 | stopped | 2 | fail | none | Store implemented in `internal/store/store.go` + `schema.go`; `go test -race -count=1 ./internal/store/` is ok and `go vet`/`check-lines` pass. `make gate` cannot print `gate: ok` here: its `gofmt -l .` step flags three committed plan-tests under `docs/plans/v1/_files/` (admin, choose, proxy) that are not gofmt-clean under Go 1.26.7 (formatted by a gofmt that aligns one-line function bodies two columns wider; same diff on a pristine master). They live under `docs/plans/` (must not edit) and the gate covers them; the check cannot be scoped down without weakening it. Code left uncommitted for review. | llama.cpp/ornith-1.5-35b-a3b |
@@ -0,0 +1,315 @@
+// Package lease is crossbar's sticky placement table. It remembers which host each
+// conversation (and each route) is on and keeps it there unless the host is
+// unhealthy, the lease has been idle past lease_idle, or an operator releases or
+// pins the route. Every change is written through to a Persister and replayed back
+// at start so a restart does not reshuffle sessions.
+package lease
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"sync"
+	"time"
+
+	"git.wntrmute.dev/kyle/crossbar/internal/store"
+)
+
+var (
+	ErrNoHost      = errors.New("lease: no usable host")
+	ErrPinnedDown  = errors.New("lease: pinned host is not healthy")
+	ErrUnknownHost = errors.New("lease: unknown host")
+)
+
+// Key identifies one conversation: a route, the fingerprint of its first request
+// (empty for a request with no user message, which leases the route itself), and
+// the model it wants.
+type Key struct {
+	Route, FP, Model string
+}
+
+// Lease is one routed model on one host, in memory.
+type Lease struct {
+	Key
+	Host              string
+	State             store.State
+	Created, LastUsed time.Time
+}
+
+// Persister is the durable half of the table: leases, the pin row, and the event
+// log. *store.Store satisfies it.
+type Persister interface {
+	SaveLease(store.Lease) error
+	DeleteLease(route, fp, model string) error
+	ListLeases() ([]store.Lease, error)
+	RecordEvent(store.LeaseEvent) error
+}
+
+// Hosts is the live health view the table consults before placing a lease.
+type Hosts interface {
+	Healthy(name string) bool
+	Draining(name string) bool
+}
+
+// Chooser picks a host for a new lease among the eligible candidates.
+type Chooser interface {
+	Choose(candidates []string, model string) (string, bool)
+}
+
+type Table struct {
+	mu     sync.Mutex
+	leases map[Key]*Lease             // active leases, keyed by (route, fp, model)
+	pins   map[string]string          // route -> pinned host
+	seen   map[string]map[string]bool // route -> candidate hosts ever asked for or stored
+	p      Persister
+	hosts  Hosts
+	choose Chooser
+	idle   time.Duration
+}
+
+// New builds a table and loads its state. Rows with FP=="" && Model=="" &&
+// State==Pinned are pins; every other row becomes an active lease and its host
+// counts as a candidate already seen for the route.
+func New(p Persister, h Hosts, c Chooser, idle time.Duration) (*Table, error) {
+	loads, err := p.ListLeases()
+	if err != nil {
+		return nil, fmt.Errorf("lease: list leases: %w", err)
+	}
+	t := &Table{
+		leases: make(map[Key]*Lease),
+		pins:   make(map[string]string),
+		seen:   make(map[string]map[string]bool),
+		p:      p,
+		hosts:  h,
+		choose: c,
+		idle:   idle,
+	}
+	for _, l := range loads {
+		if l.FP == "" && l.Model == "" && l.State == store.Pinned {
+			t.pins[l.Route] = l.Host
+		} else {
+			t.leases[Key{l.Route, l.FP, l.Model}] = &Lease{Key{l.Route, l.FP, l.Model}, l.Host, l.State, l.Created, l.LastUsed}
+		}
+		if t.seen[l.Route] == nil {
+			t.seen[l.Route] = make(map[string]bool)
+		}
+		t.seen[l.Route][l.Host] = true
+	}
+	return t, nil
+}
+
+func (l *Lease) store() store.Lease {
+	return store.Lease{Route: l.Route, FP: l.FP, Model: l.Model, Host: l.Host, State: l.State, Created: l.Created, LastUsed: l.LastUsed}
+}
+
+// Acquire places k, keeping it sticky. See the task's Acquire ordering: pinned
+// route, existing lease, inherit the route's host, else choose. Only one lease is
+// created per call, and a failed save is rolled back in memory.
+func (t *Table) Acquire(k Key, candidates []string, now time.Time) (host string, reused bool, err error) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.seen[k.Route] == nil {
+		t.seen[k.Route] = make(map[string]bool)
+	}
+	for _, c := range candidates {
+		t.seen[k.Route][c] = true
+	}
+
+	// unhealthyFrom is set when an existing lease sat on a dead host; the next
+	// create records an unhealthy move instead of a fresh new.
+	var unhealthyFrom string
+	reason := store.ReasonNew
+
+	// 1. Pinned route.
+	if pin, ok := t.pins[k.Route]; ok {
+		if !t.hosts.Healthy(pin) {
+			return "", false, ErrPinnedDown
+		}
+		if _, exists := t.leases[k]; exists {
+			return pin, true, nil
+		}
+		l := &Lease{k, pin, store.Active, now, now}
+		t.leases[k] = l
+		if err := t.save(l); err != nil {
+			delete(t.leases, k)
+			return "", false, err
+		}
+		t.event(now, k, store.ReasonNew, "", pin)
+		return pin, false, nil
+	}
+
+	// 2. Existing lease for k.
+	if l, exists := t.leases[k]; exists {
+		if t.hosts.Healthy(l.Host) {
+			l.LastUsed = now
+			if err := t.save(l); err != nil {
+				return "", false, err
+			}
+			return l.Host, true, nil
+		}
+		unhealthyFrom = l.Host
+	}
+
+	// 3. Inherit the route's own host when a fingerprinted request can start
+	//    where the route already lives.
+	if k.FP != "" {
+		rk := Key{k.Route, "", k.Model}
+		if rl, exists := t.leases[rk]; exists && t.hosts.Healthy(rl.Host) {
+			l := &Lease{k, rl.Host, store.Active, now, now}
+			t.leases[k] = l
+			if err := t.save(l); err != nil {
+				delete(t.leases, k)
+				return "", false, err
+			}
+			if unhealthyFrom != "" {
+				reason = store.ReasonUnhealthy
+			}
+			t.event(now, k, reason, unhealthyFrom, rl.Host)
+			return rl.Host, true, nil
+		}
+	}
+
+	// 4. Choose among healthy, non-draining candidates.
+	filtered := make([]string, 0, len(candidates))
+	for _, c := range candidates {
+		if t.hosts.Healthy(c) && !t.hosts.Draining(c) {
+			filtered = append(filtered, c)
+		}
+	}
+	host, ok := t.choose.Choose(filtered, k.Model)
+	if !ok {
+		return "", false, ErrNoHost
+	}
+	l := &Lease{k, host, store.Active, now, now}
+	t.leases[k] = l
+	if err := t.save(l); err != nil {
+		delete(t.leases, k)
+		return "", false, err
+	}
+	if unhealthyFrom != "" {
+		reason = store.ReasonUnhealthy
+	}
+	t.event(now, k, reason, unhealthyFrom, host)
+	return host, false, nil
+}
+
+// save writes a lease through, failing the call on a persister error.
+func (t *Table) save(l *Lease) error {
+	if err := t.p.SaveLease(l.store()); err != nil {
+		return fmt.Errorf("lease: %w", err)
+	}
+	return nil
+}
+
+// event appends a lease event. from is empty for a fresh placement.
+func (t *Table) event(now time.Time, k Key, reason, from, to string) {
+	_ = t.p.RecordEvent(store.LeaseEvent{TS: now, Route: k.Route, Model: k.Model, FromHost: from, ToHost: to, Reason: reason})
+}
+
+// ExpireIdle removes leases idle longer than lease_idle (never pins, which are
+// not in the lease map) and records an idle event for each. It returns how many
+// it removed.
+func (t *Table) ExpireIdle(now time.Time) int {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	var idle []Key
+	for k, l := range t.leases {
+		if now.Sub(l.LastUsed) > t.idle {
+			idle = append(idle, k)
+		}
+	}
+	for _, k := range idle {
+		l := t.leases[k]
+		delete(t.leases, k)
+		_ = t.p.DeleteLease(k.Route, k.FP, k.Model)
+		t.event(now, k, store.ReasonIdle, l.Host, "")
+	}
+	return len(idle)
+}
+
+// Pin routes route to host. host must be a candidate the table has seen for the
+// route, else ErrUnknownHost. It records a pin event, stores the pin row, and
+// deletes the route's existing leases on other hosts so the next turn lands on
+// the pin.
+func (t *Table) Pin(route, host string, now time.Time) error {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	if t.seen[route] == nil || !t.seen[route][host] {
+		return ErrUnknownHost
+	}
+	t.event(now, Key{Route: route}, store.ReasonPin, "", host)
+	if err := t.p.SaveLease(store.Lease{Route: route, FP: "", Model: "", Host: host, State: store.Pinned, Created: now, LastUsed: now}); err != nil {
+		return fmt.Errorf("lease: %w", err)
+	}
+	t.pins[route] = host
+	for k, l := range t.leases {
+		if k.Route == route && l.Host != host {
+			delete(t.leases, k)
+			_ = t.p.DeleteLease(k.Route, k.FP, k.Model)
+			t.event(now, k, store.ReasonPin, l.Host, host)
+		}
+	}
+	return nil
+}
+
+// Unpin clears the pin for route and records a release. Existing leases stay put.
+func (t *Table) Unpin(route string) {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	delete(t.pins, route)
+	_ = t.p.DeleteLease(route, "", "")
+	t.event(time.Now(), Key{Route: route}, store.ReasonRelease, "", "")
+}
+
+// Pinned returns the pinned host for route, or "" if it is not pinned.
+func (t *Table) Pinned(route string) string {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+	return t.pins[route]
+}
+
+// Release drops every lease (not the pin) of route and records a release event
+// per dropped lease. It returns how many were removed.
+func (t *Table) Release(route string) int {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	var keys []Key
+	for k := range t.leases {
+		if k.Route == route {
+			keys = append(keys, k)
+		}
+	}
+	for _, k := range keys {
+		delete(t.leases, k)
+		_ = t.p.DeleteLease(k.Route, k.FP, k.Model)
+		t.event(time.Now(), k, store.ReasonRelease, "", "")
+	}
+	return len(keys)
+}
+
+// Snapshot returns copies of the active leases, sorted by route, fp, model. Pins
+// are excluded.
+func (t *Table) Snapshot() []Lease {
+	t.mu.Lock()
+	defer t.mu.Unlock()
+
+	out := make([]Lease, 0, len(t.leases))
+	for _, l := range t.leases {
+		out = append(out, *l)
+	}
+	sort.Slice(out, func(i, j int) bool {
+		if out[i].Route != out[j].Route {
+			return out[i].Route < out[j].Route
+		}
+		if out[i].FP != out[j].FP {
+			return out[i].FP < out[j].FP
+		}
+		return out[i].Model < out[j].Model
+	})
+	return out
+}
@@ -0,0 +1,308 @@
+package lease_test
+
+import (
+	"errors"
+	"sync"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/crossbar/internal/lease"
+	"git.wntrmute.dev/kyle/crossbar/internal/store"
+)
+
+// memPersister is an in-memory Persister that also counts writes.
+type memPersister struct {
+	mu     sync.Mutex
+	leases map[[3]string]store.Lease
+	events []store.LeaseEvent
+	saves  int
+}
+
+func newPersister() *memPersister { return &memPersister{leases: map[[3]string]store.Lease{}} }
+
+func (m *memPersister) SaveLease(l store.Lease) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.saves++
+	m.leases[[3]string{l.Route, l.FP, l.Model}] = l
+	return nil
+}
+func (m *memPersister) DeleteLease(route, fp, model string) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	delete(m.leases, [3]string{route, fp, model})
+	return nil
+}
+func (m *memPersister) ListLeases() ([]store.Lease, error) {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	out := []store.Lease{}
+	for _, l := range m.leases {
+		out = append(out, l)
+	}
+	return out, nil
+}
+func (m *memPersister) RecordEvent(e store.LeaseEvent) error {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	m.events = append(m.events, e)
+	return nil
+}
+func (m *memPersister) reasons() []string {
+	m.mu.Lock()
+	defer m.mu.Unlock()
+	var r []string
+	for _, e := range m.events {
+		r = append(r, e.Reason)
+	}
+	return r
+}
+
+// world is a hand-set view of hosts plus a chooser that returns a fixed answer.
+type world struct {
+	mu       sync.Mutex
+	healthy  map[string]bool
+	draining map[string]bool
+	pick     string
+	picks    []string // candidates seen by Choose, for assertions
+}
+
+func (w *world) Healthy(name string) bool  { w.mu.Lock(); defer w.mu.Unlock(); return w.healthy[name] }
+func (w *world) Draining(name string) bool { w.mu.Lock(); defer w.mu.Unlock(); return w.draining[name] }
+func (w *world) Choose(candidates []string, model string) (string, bool) {
+	w.mu.Lock()
+	defer w.mu.Unlock()
+	w.picks = append([]string{}, candidates...)
+	for _, c := range candidates {
+		if c == w.pick {
+			return c, true
+		}
+	}
+	if len(candidates) > 0 {
+		return candidates[0], true
+	}
+	return "", false
+}
+
+var t0 = time.Date(2026, 9, 25, 10, 0, 0, 0, time.UTC)
+
+func newTable(t *testing.T, p *memPersister, w *world) *lease.Table {
+	tbl, err := lease.New(p, w, w, 30*time.Minute)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return tbl
+}
+
+func TestNewLeaseThenSticky(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "beta"}
+	tbl := newTable(t, p, w)
+	k := lease.Key{Route: "r", FP: "conv1", Model: "m"}
+	host, reused, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0)
+	if err != nil || host != "beta" || reused {
+		t.Fatalf("first: %q %v %v", host, reused, err)
+	}
+	w.pick = "alpha" // the chooser would now prefer alpha; the lease must hold
+	for i := 1; i <= 5; i++ {
+		host, reused, err = tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(time.Duration(i)*time.Minute))
+		if err != nil || host != "beta" || !reused {
+			t.Fatalf("turn %d: %q reused=%v %v, want beta reused", i, host, reused, err)
+		}
+	}
+	if got := p.reasons(); len(got) != 1 || got[0] != store.ReasonNew {
+		t.Errorf("events = %v, want one 'new'", got)
+	}
+	snap := tbl.Snapshot()
+	if len(snap) != 1 || snap[0].Host != "beta" || !snap[0].LastUsed.Equal(t0.Add(5*time.Minute)) {
+		t.Errorf("snapshot = %+v", snap)
+	}
+	if p.saves < 2 {
+		t.Errorf("LastUsed must be written through (saves=%d)", p.saves)
+	}
+}
+
+func TestUnhealthyHostMovesTheLease(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"}
+	tbl := newTable(t, p, w)
+	k := lease.Key{Route: "r", FP: "c", Model: "m"}
+	if host, _, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0); host != "alpha" {
+		t.Fatalf("first: %q", host)
+	}
+	w.mu.Lock()
+	w.healthy["alpha"] = false
+	w.pick = "beta"
+	w.mu.Unlock()
+	host, reused, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(time.Minute))
+	if err != nil || host != "beta" || reused {
+		t.Fatalf("after alpha down: %q reused=%v %v", host, reused, err)
+	}
+	if got := p.reasons(); len(got) != 2 || got[1] != store.ReasonUnhealthy {
+		t.Errorf("events = %v, want [new unhealthy]", got)
+	}
+	if len(w.picks) != 1 || w.picks[0] != "beta" {
+		t.Errorf("Choose must not see the unhealthy host: %v", w.picks)
+	}
+}
+
+func TestIdleExpiry(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"}
+	tbl := newTable(t, p, w)
+	k := lease.Key{Route: "r", FP: "c", Model: "m"}
+	tbl.Acquire(k, []string{"alpha", "beta"}, t0)
+	if n := tbl.ExpireIdle(t0.Add(29 * time.Minute)); n != 0 {
+		t.Errorf("expired %d before lease_idle", n)
+	}
+	if n := tbl.ExpireIdle(t0.Add(31 * time.Minute)); n != 1 {
+		t.Errorf("expired %d after lease_idle, want 1", n)
+	}
+	if got := p.reasons(); got[len(got)-1] != store.ReasonIdle {
+		t.Errorf("events = %v, want idle last", got)
+	}
+	w.pick = "beta"
+	if host, reused, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(32*time.Minute)); host != "beta" || reused {
+		t.Errorf("after expiry a new lease is chosen: %q reused=%v", host, reused)
+	}
+	if l, _ := p.ListLeases(); len(l) != 1 {
+		t.Errorf("persister holds %d leases, want 1", len(l))
+	}
+}
+
+func TestFingerprintInheritsRouteLease(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "beta"}
+	tbl := newTable(t, p, w)
+	// A request without a fingerprint (no user message) leases the route itself…
+	if host, _, _ := tbl.Acquire(lease.Key{Route: "r", FP: "", Model: "m"}, []string{"alpha", "beta"}, t0); host != "beta" {
+		t.Fatalf("route lease: %q", host)
+	}
+	w.pick = "alpha"
+	// …and a new conversation on that route starts where the route already is.
+	host, reused, err := tbl.Acquire(lease.Key{Route: "r", FP: "conv", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Second))
+	if err != nil || host != "beta" || !reused {
+		t.Errorf("fingerprint lease must inherit the route's host: %q reused=%v %v", host, reused, err)
+	}
+	if len(tbl.Snapshot()) != 2 {
+		t.Errorf("both the route lease and the conversation lease exist: %+v", tbl.Snapshot())
+	}
+}
+
+func TestPinAndUnpin(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"}
+	tbl := newTable(t, p, w)
+	k := lease.Key{Route: "r", FP: "c", Model: "m"}
+	tbl.Acquire(k, []string{"alpha", "beta"}, t0)
+	if err := tbl.Pin("r", "beta", t0.Add(time.Minute)); err != nil {
+		t.Fatal(err)
+	}
+	host, _, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(2*time.Minute))
+	if err != nil || host != "beta" {
+		t.Fatalf("pinned route must go to beta: %q %v", host, err)
+	}
+	host, _, err = tbl.Acquire(lease.Key{Route: "r", FP: "other", Model: "m"}, []string{"alpha", "beta"}, t0.Add(2*time.Minute))
+	if err != nil || host != "beta" {
+		t.Fatalf("new conversations on a pinned route go to the pin too: %q %v", host, err)
+	}
+	w.mu.Lock()
+	w.healthy["beta"] = false
+	w.mu.Unlock()
+	if _, _, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(3*time.Minute)); !errors.Is(err, lease.ErrPinnedDown) {
+		t.Errorf("a pinned host that is down is ErrPinnedDown, never a silent move: %v", err)
+	}
+	if err := tbl.Pin("r", "nobody", t0); !errors.Is(err, lease.ErrUnknownHost) {
+		t.Errorf("pinning to a host not in the candidates of any lease: %v, want ErrUnknownHost", err)
+	}
+	tbl.Unpin("r")
+	w.mu.Lock()
+	w.healthy["beta"] = true
+	w.mu.Unlock()
+	if host, _, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(4*time.Minute)); host != "beta" {
+		t.Errorf("after unpin the existing lease (on beta) simply continues: %q", host)
+	}
+	// Events: a pin event naming beta must exist, and the unpin's release event must come after it.
+	// Acquires under the pin may record their own events in between; their number is not fixed here.
+	got := p.reasons()
+	pinAt, releaseAt := -1, -1
+	for i, r := range got {
+		if r == store.ReasonPin && pinAt < 0 {
+			pinAt = i
+		}
+		if r == store.ReasonRelease {
+			releaseAt = i
+		}
+	}
+	if pinAt < 0 || releaseAt < pinAt {
+		t.Errorf("events = %v, want a pin event followed later by a release event", got)
+	}
+	p.mu.Lock()
+	if pinAt >= 0 && p.events[pinAt].ToHost != "beta" {
+		t.Errorf("pin event = %+v, want ToHost beta", p.events[pinAt])
+	}
+	p.mu.Unlock()
+}
+
+func TestDrainKeepsExistingRefusesNew(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, draining: map[string]bool{}, pick: "alpha"}
+	tbl := newTable(t, p, w)
+	k := lease.Key{Route: "r", FP: "c", Model: "m"}
+	tbl.Acquire(k, []string{"alpha", "beta"}, t0)
+	w.mu.Lock()
+	w.draining["alpha"] = true
+	w.mu.Unlock()
+	if host, reused, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(time.Minute)); host != "alpha" || !reused {
+		t.Errorf("an existing lease on a draining host continues: %q reused=%v", host, reused)
+	}
+	host, _, err := tbl.Acquire(lease.Key{Route: "r2", FP: "x", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Minute))
+	if err != nil || host != "beta" {
+		t.Errorf("a new lease avoids the draining host: %q %v", host, err)
+	}
+	if len(w.picks) != 1 || w.picks[0] != "beta" {
+		t.Errorf("Choose must not see the draining host: %v", w.picks)
+	}
+	if _, _, err := tbl.Acquire(lease.Key{Route: "r3", FP: "y", Model: "m"}, []string{"alpha"}, t0); !errors.Is(err, lease.ErrNoHost) {
+		t.Errorf("only draining candidates: %v, want ErrNoHost", err)
+	}
+}
+
+func TestReleaseRoute(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"}
+	tbl := newTable(t, p, w)
+	tbl.Acquire(lease.Key{Route: "r", FP: "a", Model: "m"}, []string{"alpha", "beta"}, t0)
+	tbl.Acquire(lease.Key{Route: "r", FP: "b", Model: "m"}, []string{"alpha", "beta"}, t0)
+	tbl.Acquire(lease.Key{Route: "other", FP: "c", Model: "m"}, []string{"alpha", "beta"}, t0)
+	if n := tbl.Release("r"); n != 2 {
+		t.Errorf("Release removed %d, want 2", n)
+	}
+	if n := tbl.Release("r"); n != 0 {
+		t.Errorf("second Release removed %d", n)
+	}
+	if l, _ := p.ListLeases(); len(l) != 1 || l[0].Route != "other" {
+		t.Errorf("persister after release: %+v", l)
+	}
+	w.pick = "beta"
+	if host, reused, _ := tbl.Acquire(lease.Key{Route: "r", FP: "a", Model: "m"}, []string{"alpha", "beta"}, t0); host != "beta" || reused {
+		t.Errorf("after release the route is re-chosen: %q reused=%v", host, reused)
+	}
+}
+
+func TestLoadsFromPersister(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"}
+	_ = p.SaveLease(store.Lease{Route: "r", FP: "c", Model: "m", Host: "beta", State: store.Active, Created: t0, LastUsed: t0})
+	_ = p.SaveLease(store.Lease{Route: "pinned", FP: "", Model: "", Host: "beta", State: store.Pinned, Created: t0, LastUsed: t0})
+	tbl := newTable(t, p, w)
+	if host, reused, _ := tbl.Acquire(lease.Key{Route: "r", FP: "c", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Second)); host != "beta" || !reused {
+		t.Errorf("a restart must not reshuffle: %q reused=%v", host, reused)
+	}
+	if host, _, _ := tbl.Acquire(lease.Key{Route: "pinned", FP: "new", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Second)); host != "beta" {
+		t.Errorf("a pin survives a restart: %q", host)
+	}
+}
+
+func TestNoCandidates(t *testing.T) {
+	p, w := newPersister(), &world{healthy: map[string]bool{}, pick: ""}
+	tbl := newTable(t, p, w)
+	if _, _, err := tbl.Acquire(lease.Key{Route: "r", FP: "c", Model: "m"}, []string{"alpha"}, t0); !errors.Is(err, lease.ErrNoHost) {
+		t.Errorf("no healthy host: %v, want ErrNoHost", err)
+	}
+	if len(tbl.Snapshot()) != 0 {
+		t.Errorf("a failed acquire must not create a lease")
+	}
+}