P6 plan: asynchronous batch fetch endpoint — phase spec, task plan, given tests

Co-Authored-By: DeepSeek V4 Pro <noreply@deepseek.com>
2026-09-26 12:09:38 -07:00
parent b1da18d49e
commit 63e70e19fc
12 changed files with 1564 additions and 0 deletions
@@ -0,0 +1,234 @@
+package batch
+
+import (
+	"context"
+	"errors"
+	"io"
+	"log/slog"
+	"sync"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/dcache"
+)
+
+func newRunner(t *testing.T, fetch FetchFunc, cfg Config) *Runner {
+	t.Helper()
+	db, err := dcache.Open("", 1<<20, slog.New(slog.NewTextHandler(io.Discard, nil)))
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+	return New(db, fetch, cfg, slog.New(slog.NewTextHandler(io.Discard, nil)))
+}
+
+func waitStatus(t *testing.T, r *Runner, id, want string) {
+	t.Helper()
+	deadline := time.Now().Add(3 * time.Second)
+	for time.Now().Before(deadline) {
+		j, _, ok := r.Status(id)
+		if ok && j.Status == want {
+			return
+		}
+		time.Sleep(time.Millisecond)
+	}
+	j, _, _ := r.Status(id)
+	t.Fatalf("job %s never reached %q (now %q)", id, want, j.Status)
+}
+
+func TestSerialOrder(t *testing.T) {
+	var mu sync.Mutex
+	var calls []string
+	inflight, peak := 0, 0
+	fetch := func(_ context.Context, tgt Target) (Result, error) {
+		mu.Lock()
+		inflight++
+		if inflight > peak {
+			peak = inflight
+		}
+		calls = append(calls, tgt.URL)
+		mu.Unlock()
+		time.Sleep(time.Millisecond)
+		mu.Lock()
+		inflight--
+		mu.Unlock()
+		return Result{Content: "ok " + tgt.URL}, nil
+	}
+	r := newRunner(t, fetch, Config{})
+	id, reused, err := r.Submit(context.Background(), "alice", "fleet", []Target{
+		{URL: "https://a.example/"}, {URL: "https://b.example/"}, {URL: "https://c.example/"},
+	})
+	if err != nil || reused {
+		t.Fatalf("submit err=%v reused=%v", err, reused)
+	}
+	waitStatus(t, r, id, "done")
+	mu.Lock()
+	order := append([]string(nil), calls...)
+	p := peak
+	mu.Unlock()
+	if len(order) != 3 || order[0] != "https://a.example/" || order[1] != "https://b.example/" || order[2] != "https://c.example/" {
+		t.Fatalf("order %v", order)
+	}
+	if p != 1 {
+		t.Fatalf("peak inflight %d, want 1 (serial)", p)
+	}
+}
+
+func TestFailureIsolation(t *testing.T) {
+	fetch := func(_ context.Context, tgt Target) (Result, error) {
+		if tgt.URL == "https://b.example/" {
+			return Result{}, errors.New("upstream 500")
+		}
+		return Result{Content: "ok " + tgt.URL}, nil
+	}
+	r := newRunner(t, fetch, Config{})
+	id, _, err := r.Submit(context.Background(), "alice", "fleet", []Target{
+		{URL: "https://a.example/"}, {URL: "https://b.example/"}, {URL: "https://c.example/"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitStatus(t, r, id, "partial")
+	j, ts, ok := r.Status(id)
+	if !ok {
+		t.Fatal("no job")
+	}
+	if j.Status != "partial" || j.TargetsDone != 3 {
+		t.Fatalf("job %+v", j)
+	}
+	if ts[0].Status != "done" || ts[1].Status != "failed" || ts[2].Status != "done" {
+		t.Fatalf("statuses %+v", ts)
+	}
+	if ts[1].Error == "" {
+		t.Fatal("failed target missing error")
+	}
+	if ts[0].Result == nil || ts[0].Result.Content != "ok https://a.example/" {
+		t.Fatalf("result %+v", ts[0].Result)
+	}
+}
+
+func TestIdempotentResubmit(t *testing.T) {
+	var mu sync.Mutex
+	n := 0
+	fetch := func(_ context.Context, tgt Target) (Result, error) {
+		mu.Lock()
+		n++
+		mu.Unlock()
+		return Result{Content: "ok"}, nil
+	}
+	r := newRunner(t, fetch, Config{})
+	targets := []Target{{URL: "https://a.example/"}, {URL: "https://b.example/"}}
+	id1, reused1, err := r.Submit(context.Background(), "alice", "fleet", targets)
+	if err != nil || reused1 {
+		t.Fatalf("first submit err=%v reused=%v", err, reused1)
+	}
+	id2, reused2, err := r.Submit(context.Background(), "alice", "fleet", targets)
+	if err != nil || !reused2 || id2 != id1 {
+		t.Fatalf("resubmit id=%q reused=%v err=%v", id2, reused2, err)
+	}
+	waitStatus(t, r, id1, "done")
+	mu.Lock()
+	calls := n
+	mu.Unlock()
+	if calls != 2 {
+		t.Fatalf("fetched %d times, want 2 (idempotent)", calls)
+	}
+}
+
+func TestTooManyTargets(t *testing.T) {
+	fetch := func(_ context.Context, tgt Target) (Result, error) { return Result{Content: "ok"}, nil }
+	r := newRunner(t, fetch, Config{TargetsPerJob: 2})
+	three := []Target{{URL: "https://a.example/"}, {URL: "https://b.example/"}, {URL: "https://c.example/"}}
+	if _, _, err := r.Submit(context.Background(), "alice", "fleet", three); !errors.Is(err, ErrTooManyTargets) {
+		t.Fatalf("want ErrTooManyTargets, got %v", err)
+	}
+}
+
+func TestCallerBusy(t *testing.T) {
+	release := make(chan struct{})
+	fetch := func(_ context.Context, tgt Target) (Result, error) {
+		<-release // hold the job in the running state
+		return Result{Content: "ok"}, nil
+	}
+	r := newRunner(t, fetch, Config{})
+	id, _, err := r.Submit(context.Background(), "alice", "fleet", []Target{{URL: "https://a.example/"}})
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitStatus(t, r, id, "running")
+	if _, _, err := r.Submit(context.Background(), "alice", "fleet", []Target{{URL: "https://b.example/"}}); !errors.Is(err, ErrCallerBusy) {
+		t.Fatalf("want ErrCallerBusy, got %v", err)
+	}
+	close(release)
+}
+
+func TestCancel(t *testing.T) {
+	started := make(chan struct{}, 1)
+	release := make(chan struct{})
+	var mu sync.Mutex
+	var calls []string
+	fetch := func(_ context.Context, tgt Target) (Result, error) {
+		mu.Lock()
+		calls = append(calls, tgt.URL)
+		mu.Unlock()
+		select {
+		case started <- struct{}{}:
+		default:
+		}
+		<-release
+		return Result{Content: "ok"}, nil
+	}
+	r := newRunner(t, fetch, Config{})
+	id, _, err := r.Submit(context.Background(), "alice", "fleet", []Target{
+		{URL: "https://a.example/"}, {URL: "https://b.example/"}, {URL: "https://c.example/"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	<-started // first target is in flight
+	if !r.Cancel(id) {
+		t.Fatal("cancel returned false")
+	}
+	close(release) // let the in-flight target finish
+	waitStatus(t, r, id, "cancelled")
+	mu.Lock()
+	n := len(calls)
+	mu.Unlock()
+	if n != 1 {
+		t.Fatalf("cancelled after %d fetches, want 1", n)
+	}
+	_, ts, ok := r.Status(id)
+	if !ok {
+		t.Fatal("no job")
+	}
+	if ts[0].Status != "done" {
+		t.Fatalf("in-flight target should be stored done: %+v", ts[0])
+	}
+	if ts[1].Status != "pending" || ts[2].Status != "pending" {
+		t.Fatalf("later targets must stay pending: %+v", ts[1:])
+	}
+}
+
+func TestPanicRecovered(t *testing.T) {
+	fetch := func(_ context.Context, tgt Target) (Result, error) {
+		if tgt.URL == "https://b.example/" {
+			panic("boom")
+		}
+		return Result{Content: "ok"}, nil
+	}
+	r := newRunner(t, fetch, Config{})
+	id, _, err := r.Submit(context.Background(), "alice", "fleet", []Target{
+		{URL: "https://a.example/"}, {URL: "https://b.example/"}, {URL: "https://c.example/"},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	waitStatus(t, r, id, "partial")
+	_, ts, _ := r.Status(id)
+	if ts[0].Status != "done" || ts[1].Status != "failed" || ts[2].Status != "done" {
+		t.Fatalf("statuses %+v", ts)
+	}
+	if ts[1].Error == "" {
+		t.Fatal("panic target missing error")
+	}
+}