diff --git a/docs/implementer-log.md b/docs/implementer-log.md index ad37976..8ffb4d1 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -5,6 +5,7 @@ owner fills in the Model column. The reviewer adds findings under "Reviews" once | Task | Date | Status | Gate runs | First gate | Deviations | Notes | Model | |---|---|---|---|---|---|---|---| +| v1/04-lease | 2026-09-25 | done | 1 | pass | The given `TestPinAndUnpin` was wrong and replaced by the owner mid-task; the corrected `internal/lease/lease_test.go` is byte-identical to `docs/plans/v1/_files/internal/lease/lease_test.go`. A `fmt.Printf("DEBUG …")` line the prior session left in `event` was removed before the gate. | `Acquire` order (pinned, existing, inherit, choose) with memory rolled back only after a successful save; `Pin` writes a pin event, then the pin row, then deletes other-host leases, so the pin event always precedes the unpin's release event in the log. | ? | | v1/02-fingerprint-config | 2026-09-25 | done | 1 | pass | Switched the existing `TestBadFiles` unknown-key example from `lease_idle` to `bogus_key`, and updated `testdata/bad-unknown-key.toml` to match: this task makes `lease_idle` a valid key, so the old example was stale. `config_test.go` and that testdata are not `_files`-protected, so the edit was permitted even though the task's file list named only `config.go` and `implementer-log.md`; the unknown-key rejection is still covered. | fingerprint.go truncates each input to its first 4096 bytes and uses a presence flag so an empty first system prompt is not overwritten by a later one; `Duration.UnmarshalText` matches `^[0-9]+d$` (regexp) before falling to `time.ParseDuration`. | ? | | v1/01-store | 2026-09-25 | done | 1 | pass | none | Gate passed on the first run once the owner gofmt'd the three previously-un-clean _files plan-tests under docs/plans/v1/_files/; the blocker in the stopped row no longer applies. | ? | | v1/01-store | 2026-09-25 | stopped | 2 | fail | none | Store implemented in `internal/store/store.go` + `schema.go`; `go test -race -count=1 ./internal/store/` is ok and `go vet`/`check-lines` pass. `make gate` cannot print `gate: ok` here: its `gofmt -l .` step flags three committed plan-tests under `docs/plans/v1/_files/` (admin, choose, proxy) that are not gofmt-clean under Go 1.26.7 (formatted by a gofmt that aligns one-line function bodies two columns wider; same diff on a pristine master). They live under `docs/plans/` (must not edit) and the gate covers them; the check cannot be scoped down without weakening it. Code left uncommitted for review. | llama.cpp/ornith-1.5-35b-a3b | diff --git a/internal/lease/lease.go b/internal/lease/lease.go new file mode 100644 index 0000000..1a76374 --- /dev/null +++ b/internal/lease/lease.go @@ -0,0 +1,315 @@ +// Package lease is crossbar's sticky placement table. It remembers which host each +// conversation (and each route) is on and keeps it there unless the host is +// unhealthy, the lease has been idle past lease_idle, or an operator releases or +// pins the route. Every change is written through to a Persister and replayed back +// at start so a restart does not reshuffle sessions. +package lease + +import ( + "errors" + "fmt" + "sort" + "sync" + "time" + + "git.wntrmute.dev/kyle/crossbar/internal/store" +) + +var ( + ErrNoHost = errors.New("lease: no usable host") + ErrPinnedDown = errors.New("lease: pinned host is not healthy") + ErrUnknownHost = errors.New("lease: unknown host") +) + +// Key identifies one conversation: a route, the fingerprint of its first request +// (empty for a request with no user message, which leases the route itself), and +// the model it wants. +type Key struct { + Route, FP, Model string +} + +// Lease is one routed model on one host, in memory. +type Lease struct { + Key + Host string + State store.State + Created, LastUsed time.Time +} + +// Persister is the durable half of the table: leases, the pin row, and the event +// log. *store.Store satisfies it. +type Persister interface { + SaveLease(store.Lease) error + DeleteLease(route, fp, model string) error + ListLeases() ([]store.Lease, error) + RecordEvent(store.LeaseEvent) error +} + +// Hosts is the live health view the table consults before placing a lease. +type Hosts interface { + Healthy(name string) bool + Draining(name string) bool +} + +// Chooser picks a host for a new lease among the eligible candidates. +type Chooser interface { + Choose(candidates []string, model string) (string, bool) +} + +type Table struct { + mu sync.Mutex + leases map[Key]*Lease // active leases, keyed by (route, fp, model) + pins map[string]string // route -> pinned host + seen map[string]map[string]bool // route -> candidate hosts ever asked for or stored + p Persister + hosts Hosts + choose Chooser + idle time.Duration +} + +// New builds a table and loads its state. Rows with FP=="" && Model=="" && +// State==Pinned are pins; every other row becomes an active lease and its host +// counts as a candidate already seen for the route. +func New(p Persister, h Hosts, c Chooser, idle time.Duration) (*Table, error) { + loads, err := p.ListLeases() + if err != nil { + return nil, fmt.Errorf("lease: list leases: %w", err) + } + t := &Table{ + leases: make(map[Key]*Lease), + pins: make(map[string]string), + seen: make(map[string]map[string]bool), + p: p, + hosts: h, + choose: c, + idle: idle, + } + for _, l := range loads { + if l.FP == "" && l.Model == "" && l.State == store.Pinned { + t.pins[l.Route] = l.Host + } else { + t.leases[Key{l.Route, l.FP, l.Model}] = &Lease{Key{l.Route, l.FP, l.Model}, l.Host, l.State, l.Created, l.LastUsed} + } + if t.seen[l.Route] == nil { + t.seen[l.Route] = make(map[string]bool) + } + t.seen[l.Route][l.Host] = true + } + return t, nil +} + +func (l *Lease) store() store.Lease { + return store.Lease{Route: l.Route, FP: l.FP, Model: l.Model, Host: l.Host, State: l.State, Created: l.Created, LastUsed: l.LastUsed} +} + +// Acquire places k, keeping it sticky. See the task's Acquire ordering: pinned +// route, existing lease, inherit the route's host, else choose. Only one lease is +// created per call, and a failed save is rolled back in memory. +func (t *Table) Acquire(k Key, candidates []string, now time.Time) (host string, reused bool, err error) { + t.mu.Lock() + defer t.mu.Unlock() + + if t.seen[k.Route] == nil { + t.seen[k.Route] = make(map[string]bool) + } + for _, c := range candidates { + t.seen[k.Route][c] = true + } + + // unhealthyFrom is set when an existing lease sat on a dead host; the next + // create records an unhealthy move instead of a fresh new. + var unhealthyFrom string + reason := store.ReasonNew + + // 1. Pinned route. + if pin, ok := t.pins[k.Route]; ok { + if !t.hosts.Healthy(pin) { + return "", false, ErrPinnedDown + } + if _, exists := t.leases[k]; exists { + return pin, true, nil + } + l := &Lease{k, pin, store.Active, now, now} + t.leases[k] = l + if err := t.save(l); err != nil { + delete(t.leases, k) + return "", false, err + } + t.event(now, k, store.ReasonNew, "", pin) + return pin, false, nil + } + + // 2. Existing lease for k. + if l, exists := t.leases[k]; exists { + if t.hosts.Healthy(l.Host) { + l.LastUsed = now + if err := t.save(l); err != nil { + return "", false, err + } + return l.Host, true, nil + } + unhealthyFrom = l.Host + } + + // 3. Inherit the route's own host when a fingerprinted request can start + // where the route already lives. + if k.FP != "" { + rk := Key{k.Route, "", k.Model} + if rl, exists := t.leases[rk]; exists && t.hosts.Healthy(rl.Host) { + l := &Lease{k, rl.Host, store.Active, now, now} + t.leases[k] = l + if err := t.save(l); err != nil { + delete(t.leases, k) + return "", false, err + } + if unhealthyFrom != "" { + reason = store.ReasonUnhealthy + } + t.event(now, k, reason, unhealthyFrom, rl.Host) + return rl.Host, true, nil + } + } + + // 4. Choose among healthy, non-draining candidates. + filtered := make([]string, 0, len(candidates)) + for _, c := range candidates { + if t.hosts.Healthy(c) && !t.hosts.Draining(c) { + filtered = append(filtered, c) + } + } + host, ok := t.choose.Choose(filtered, k.Model) + if !ok { + return "", false, ErrNoHost + } + l := &Lease{k, host, store.Active, now, now} + t.leases[k] = l + if err := t.save(l); err != nil { + delete(t.leases, k) + return "", false, err + } + if unhealthyFrom != "" { + reason = store.ReasonUnhealthy + } + t.event(now, k, reason, unhealthyFrom, host) + return host, false, nil +} + +// save writes a lease through, failing the call on a persister error. +func (t *Table) save(l *Lease) error { + if err := t.p.SaveLease(l.store()); err != nil { + return fmt.Errorf("lease: %w", err) + } + return nil +} + +// event appends a lease event. from is empty for a fresh placement. +func (t *Table) event(now time.Time, k Key, reason, from, to string) { + _ = t.p.RecordEvent(store.LeaseEvent{TS: now, Route: k.Route, Model: k.Model, FromHost: from, ToHost: to, Reason: reason}) +} + +// ExpireIdle removes leases idle longer than lease_idle (never pins, which are +// not in the lease map) and records an idle event for each. It returns how many +// it removed. +func (t *Table) ExpireIdle(now time.Time) int { + t.mu.Lock() + defer t.mu.Unlock() + + var idle []Key + for k, l := range t.leases { + if now.Sub(l.LastUsed) > t.idle { + idle = append(idle, k) + } + } + for _, k := range idle { + l := t.leases[k] + delete(t.leases, k) + _ = t.p.DeleteLease(k.Route, k.FP, k.Model) + t.event(now, k, store.ReasonIdle, l.Host, "") + } + return len(idle) +} + +// Pin routes route to host. host must be a candidate the table has seen for the +// route, else ErrUnknownHost. It records a pin event, stores the pin row, and +// deletes the route's existing leases on other hosts so the next turn lands on +// the pin. +func (t *Table) Pin(route, host string, now time.Time) error { + t.mu.Lock() + defer t.mu.Unlock() + + if t.seen[route] == nil || !t.seen[route][host] { + return ErrUnknownHost + } + t.event(now, Key{Route: route}, store.ReasonPin, "", host) + if err := t.p.SaveLease(store.Lease{Route: route, FP: "", Model: "", Host: host, State: store.Pinned, Created: now, LastUsed: now}); err != nil { + return fmt.Errorf("lease: %w", err) + } + t.pins[route] = host + for k, l := range t.leases { + if k.Route == route && l.Host != host { + delete(t.leases, k) + _ = t.p.DeleteLease(k.Route, k.FP, k.Model) + t.event(now, k, store.ReasonPin, l.Host, host) + } + } + return nil +} + +// Unpin clears the pin for route and records a release. Existing leases stay put. +func (t *Table) Unpin(route string) { + t.mu.Lock() + defer t.mu.Unlock() + + delete(t.pins, route) + _ = t.p.DeleteLease(route, "", "") + t.event(time.Now(), Key{Route: route}, store.ReasonRelease, "", "") +} + +// Pinned returns the pinned host for route, or "" if it is not pinned. +func (t *Table) Pinned(route string) string { + t.mu.Lock() + defer t.mu.Unlock() + return t.pins[route] +} + +// Release drops every lease (not the pin) of route and records a release event +// per dropped lease. It returns how many were removed. +func (t *Table) Release(route string) int { + t.mu.Lock() + defer t.mu.Unlock() + + var keys []Key + for k := range t.leases { + if k.Route == route { + keys = append(keys, k) + } + } + for _, k := range keys { + delete(t.leases, k) + _ = t.p.DeleteLease(k.Route, k.FP, k.Model) + t.event(time.Now(), k, store.ReasonRelease, "", "") + } + return len(keys) +} + +// Snapshot returns copies of the active leases, sorted by route, fp, model. Pins +// are excluded. +func (t *Table) Snapshot() []Lease { + t.mu.Lock() + defer t.mu.Unlock() + + out := make([]Lease, 0, len(t.leases)) + for _, l := range t.leases { + out = append(out, *l) + } + sort.Slice(out, func(i, j int) bool { + if out[i].Route != out[j].Route { + return out[i].Route < out[j].Route + } + if out[i].FP != out[j].FP { + return out[i].FP < out[j].FP + } + return out[i].Model < out[j].Model + }) + return out +} diff --git a/internal/lease/lease_test.go b/internal/lease/lease_test.go new file mode 100644 index 0000000..5fdf3d2 --- /dev/null +++ b/internal/lease/lease_test.go @@ -0,0 +1,308 @@ +package lease_test + +import ( + "errors" + "sync" + "testing" + "time" + + "git.wntrmute.dev/kyle/crossbar/internal/lease" + "git.wntrmute.dev/kyle/crossbar/internal/store" +) + +// memPersister is an in-memory Persister that also counts writes. +type memPersister struct { + mu sync.Mutex + leases map[[3]string]store.Lease + events []store.LeaseEvent + saves int +} + +func newPersister() *memPersister { return &memPersister{leases: map[[3]string]store.Lease{}} } + +func (m *memPersister) SaveLease(l store.Lease) error { + m.mu.Lock() + defer m.mu.Unlock() + m.saves++ + m.leases[[3]string{l.Route, l.FP, l.Model}] = l + return nil +} +func (m *memPersister) DeleteLease(route, fp, model string) error { + m.mu.Lock() + defer m.mu.Unlock() + delete(m.leases, [3]string{route, fp, model}) + return nil +} +func (m *memPersister) ListLeases() ([]store.Lease, error) { + m.mu.Lock() + defer m.mu.Unlock() + out := []store.Lease{} + for _, l := range m.leases { + out = append(out, l) + } + return out, nil +} +func (m *memPersister) RecordEvent(e store.LeaseEvent) error { + m.mu.Lock() + defer m.mu.Unlock() + m.events = append(m.events, e) + return nil +} +func (m *memPersister) reasons() []string { + m.mu.Lock() + defer m.mu.Unlock() + var r []string + for _, e := range m.events { + r = append(r, e.Reason) + } + return r +} + +// world is a hand-set view of hosts plus a chooser that returns a fixed answer. +type world struct { + mu sync.Mutex + healthy map[string]bool + draining map[string]bool + pick string + picks []string // candidates seen by Choose, for assertions +} + +func (w *world) Healthy(name string) bool { w.mu.Lock(); defer w.mu.Unlock(); return w.healthy[name] } +func (w *world) Draining(name string) bool { w.mu.Lock(); defer w.mu.Unlock(); return w.draining[name] } +func (w *world) Choose(candidates []string, model string) (string, bool) { + w.mu.Lock() + defer w.mu.Unlock() + w.picks = append([]string{}, candidates...) + for _, c := range candidates { + if c == w.pick { + return c, true + } + } + if len(candidates) > 0 { + return candidates[0], true + } + return "", false +} + +var t0 = time.Date(2026, 9, 25, 10, 0, 0, 0, time.UTC) + +func newTable(t *testing.T, p *memPersister, w *world) *lease.Table { + tbl, err := lease.New(p, w, w, 30*time.Minute) + if err != nil { + t.Fatal(err) + } + return tbl +} + +func TestNewLeaseThenSticky(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "beta"} + tbl := newTable(t, p, w) + k := lease.Key{Route: "r", FP: "conv1", Model: "m"} + host, reused, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0) + if err != nil || host != "beta" || reused { + t.Fatalf("first: %q %v %v", host, reused, err) + } + w.pick = "alpha" // the chooser would now prefer alpha; the lease must hold + for i := 1; i <= 5; i++ { + host, reused, err = tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(time.Duration(i)*time.Minute)) + if err != nil || host != "beta" || !reused { + t.Fatalf("turn %d: %q reused=%v %v, want beta reused", i, host, reused, err) + } + } + if got := p.reasons(); len(got) != 1 || got[0] != store.ReasonNew { + t.Errorf("events = %v, want one 'new'", got) + } + snap := tbl.Snapshot() + if len(snap) != 1 || snap[0].Host != "beta" || !snap[0].LastUsed.Equal(t0.Add(5*time.Minute)) { + t.Errorf("snapshot = %+v", snap) + } + if p.saves < 2 { + t.Errorf("LastUsed must be written through (saves=%d)", p.saves) + } +} + +func TestUnhealthyHostMovesTheLease(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"} + tbl := newTable(t, p, w) + k := lease.Key{Route: "r", FP: "c", Model: "m"} + if host, _, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0); host != "alpha" { + t.Fatalf("first: %q", host) + } + w.mu.Lock() + w.healthy["alpha"] = false + w.pick = "beta" + w.mu.Unlock() + host, reused, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(time.Minute)) + if err != nil || host != "beta" || reused { + t.Fatalf("after alpha down: %q reused=%v %v", host, reused, err) + } + if got := p.reasons(); len(got) != 2 || got[1] != store.ReasonUnhealthy { + t.Errorf("events = %v, want [new unhealthy]", got) + } + if len(w.picks) != 1 || w.picks[0] != "beta" { + t.Errorf("Choose must not see the unhealthy host: %v", w.picks) + } +} + +func TestIdleExpiry(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"} + tbl := newTable(t, p, w) + k := lease.Key{Route: "r", FP: "c", Model: "m"} + tbl.Acquire(k, []string{"alpha", "beta"}, t0) + if n := tbl.ExpireIdle(t0.Add(29 * time.Minute)); n != 0 { + t.Errorf("expired %d before lease_idle", n) + } + if n := tbl.ExpireIdle(t0.Add(31 * time.Minute)); n != 1 { + t.Errorf("expired %d after lease_idle, want 1", n) + } + if got := p.reasons(); got[len(got)-1] != store.ReasonIdle { + t.Errorf("events = %v, want idle last", got) + } + w.pick = "beta" + if host, reused, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(32*time.Minute)); host != "beta" || reused { + t.Errorf("after expiry a new lease is chosen: %q reused=%v", host, reused) + } + if l, _ := p.ListLeases(); len(l) != 1 { + t.Errorf("persister holds %d leases, want 1", len(l)) + } +} + +func TestFingerprintInheritsRouteLease(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "beta"} + tbl := newTable(t, p, w) + // A request without a fingerprint (no user message) leases the route itself… + if host, _, _ := tbl.Acquire(lease.Key{Route: "r", FP: "", Model: "m"}, []string{"alpha", "beta"}, t0); host != "beta" { + t.Fatalf("route lease: %q", host) + } + w.pick = "alpha" + // …and a new conversation on that route starts where the route already is. + host, reused, err := tbl.Acquire(lease.Key{Route: "r", FP: "conv", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Second)) + if err != nil || host != "beta" || !reused { + t.Errorf("fingerprint lease must inherit the route's host: %q reused=%v %v", host, reused, err) + } + if len(tbl.Snapshot()) != 2 { + t.Errorf("both the route lease and the conversation lease exist: %+v", tbl.Snapshot()) + } +} + +func TestPinAndUnpin(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"} + tbl := newTable(t, p, w) + k := lease.Key{Route: "r", FP: "c", Model: "m"} + tbl.Acquire(k, []string{"alpha", "beta"}, t0) + if err := tbl.Pin("r", "beta", t0.Add(time.Minute)); err != nil { + t.Fatal(err) + } + host, _, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(2*time.Minute)) + if err != nil || host != "beta" { + t.Fatalf("pinned route must go to beta: %q %v", host, err) + } + host, _, err = tbl.Acquire(lease.Key{Route: "r", FP: "other", Model: "m"}, []string{"alpha", "beta"}, t0.Add(2*time.Minute)) + if err != nil || host != "beta" { + t.Fatalf("new conversations on a pinned route go to the pin too: %q %v", host, err) + } + w.mu.Lock() + w.healthy["beta"] = false + w.mu.Unlock() + if _, _, err := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(3*time.Minute)); !errors.Is(err, lease.ErrPinnedDown) { + t.Errorf("a pinned host that is down is ErrPinnedDown, never a silent move: %v", err) + } + if err := tbl.Pin("r", "nobody", t0); !errors.Is(err, lease.ErrUnknownHost) { + t.Errorf("pinning to a host not in the candidates of any lease: %v, want ErrUnknownHost", err) + } + tbl.Unpin("r") + w.mu.Lock() + w.healthy["beta"] = true + w.mu.Unlock() + if host, _, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(4*time.Minute)); host != "beta" { + t.Errorf("after unpin the existing lease (on beta) simply continues: %q", host) + } + // Events: a pin event naming beta must exist, and the unpin's release event must come after it. + // Acquires under the pin may record their own events in between; their number is not fixed here. + got := p.reasons() + pinAt, releaseAt := -1, -1 + for i, r := range got { + if r == store.ReasonPin && pinAt < 0 { + pinAt = i + } + if r == store.ReasonRelease { + releaseAt = i + } + } + if pinAt < 0 || releaseAt < pinAt { + t.Errorf("events = %v, want a pin event followed later by a release event", got) + } + p.mu.Lock() + if pinAt >= 0 && p.events[pinAt].ToHost != "beta" { + t.Errorf("pin event = %+v, want ToHost beta", p.events[pinAt]) + } + p.mu.Unlock() +} + +func TestDrainKeepsExistingRefusesNew(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, draining: map[string]bool{}, pick: "alpha"} + tbl := newTable(t, p, w) + k := lease.Key{Route: "r", FP: "c", Model: "m"} + tbl.Acquire(k, []string{"alpha", "beta"}, t0) + w.mu.Lock() + w.draining["alpha"] = true + w.mu.Unlock() + if host, reused, _ := tbl.Acquire(k, []string{"alpha", "beta"}, t0.Add(time.Minute)); host != "alpha" || !reused { + t.Errorf("an existing lease on a draining host continues: %q reused=%v", host, reused) + } + host, _, err := tbl.Acquire(lease.Key{Route: "r2", FP: "x", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Minute)) + if err != nil || host != "beta" { + t.Errorf("a new lease avoids the draining host: %q %v", host, err) + } + if len(w.picks) != 1 || w.picks[0] != "beta" { + t.Errorf("Choose must not see the draining host: %v", w.picks) + } + if _, _, err := tbl.Acquire(lease.Key{Route: "r3", FP: "y", Model: "m"}, []string{"alpha"}, t0); !errors.Is(err, lease.ErrNoHost) { + t.Errorf("only draining candidates: %v, want ErrNoHost", err) + } +} + +func TestReleaseRoute(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"} + tbl := newTable(t, p, w) + tbl.Acquire(lease.Key{Route: "r", FP: "a", Model: "m"}, []string{"alpha", "beta"}, t0) + tbl.Acquire(lease.Key{Route: "r", FP: "b", Model: "m"}, []string{"alpha", "beta"}, t0) + tbl.Acquire(lease.Key{Route: "other", FP: "c", Model: "m"}, []string{"alpha", "beta"}, t0) + if n := tbl.Release("r"); n != 2 { + t.Errorf("Release removed %d, want 2", n) + } + if n := tbl.Release("r"); n != 0 { + t.Errorf("second Release removed %d", n) + } + if l, _ := p.ListLeases(); len(l) != 1 || l[0].Route != "other" { + t.Errorf("persister after release: %+v", l) + } + w.pick = "beta" + if host, reused, _ := tbl.Acquire(lease.Key{Route: "r", FP: "a", Model: "m"}, []string{"alpha", "beta"}, t0); host != "beta" || reused { + t.Errorf("after release the route is re-chosen: %q reused=%v", host, reused) + } +} + +func TestLoadsFromPersister(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{"alpha": true, "beta": true}, pick: "alpha"} + _ = p.SaveLease(store.Lease{Route: "r", FP: "c", Model: "m", Host: "beta", State: store.Active, Created: t0, LastUsed: t0}) + _ = p.SaveLease(store.Lease{Route: "pinned", FP: "", Model: "", Host: "beta", State: store.Pinned, Created: t0, LastUsed: t0}) + tbl := newTable(t, p, w) + if host, reused, _ := tbl.Acquire(lease.Key{Route: "r", FP: "c", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Second)); host != "beta" || !reused { + t.Errorf("a restart must not reshuffle: %q reused=%v", host, reused) + } + if host, _, _ := tbl.Acquire(lease.Key{Route: "pinned", FP: "new", Model: "m"}, []string{"alpha", "beta"}, t0.Add(time.Second)); host != "beta" { + t.Errorf("a pin survives a restart: %q", host) + } +} + +func TestNoCandidates(t *testing.T) { + p, w := newPersister(), &world{healthy: map[string]bool{}, pick: ""} + tbl := newTable(t, p, w) + if _, _, err := tbl.Acquire(lease.Key{Route: "r", FP: "c", Model: "m"}, []string{"alpha"}, t0); !errors.Is(err, lease.ErrNoHost) { + t.Errorf("no healthy host: %v, want ErrNoHost", err) + } + if len(tbl.Snapshot()) != 0 { + t.Errorf("a failed acquire must not create a lease") + } +}