kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P6 during the run: split the runner task (02 core, 03 workers); guard.PreCheck as an owner commit; AGENTS.md: write code into files early

Browse Source 
Task 02 (the whole runner) ended at 21:08 with no file and no commit: Ornith wrote the
implementation out in its reply (~700 lines), OpenCode compacted the session, and the model
then emitted a malformed tool call. It had also looked for the given test at a wrong path
("_files/…" relative to the plan: my ambiguity).

- 02-runner → 02-runner-core (types, Submit, Status, Touch, Cancel; no goroutines) and
  03-runner-workers; the given test split to match (runner_core_test.go gains three tests:
  queued storage, cancel while queued, Status never moves the lease); later tasks renumbered
  04-09 and every cross-reference checked.
- Copy paths spelled from the repo root.
- guard.PreCheck (+ its test) is committed here by the owner: the old task 05 told the
  implementer to edit internal/guard/, which AGENTS.md forbids. My contradiction.
- AGENTS.md rule 6: write code into files early, in small steps.
Both runner test files compile against a panic-only skeleton (the core file also alone) and
fail there on panic: todo.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kyle
2026-09-26 21:14:08 -07:00
co-authored by Claude Opus 5.5
parent 583a35883d
commit 103ec3d05a
19 changed files with 423 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
docs/plans/p6/_files/api/batchrun_test.go
+2 -2
View File
@@ -1,7 +1,7 @@
package api
// P6 task 05: the runner wired to fetchTarget and the metrics, with no HTTP yet. The helpers
// here (newBatchServer, pageServer, fakeDNS) are shared with task 06's batch_test.go.
// P6 task 06: the runner wired to fetchTarget and the metrics, with no HTTP yet. The helpers
// here (newBatchServer, pageServer, fakeDNS) are shared with task 07's batch_test.go.
import (
"context"
docs/plans/p6/_files/api/fetchtarget_test.go
+1 -1
View File
@@ -1,6 +1,6 @@
package api
// P6 task 04: v1fetch's per-target pipeline is extracted into s.fetchTarget, which the batch
// P6 task 05: v1fetch's per-target pipeline is extracted into s.fetchTarget, which the batch
// runner also calls. These tests pin that the two agree; every existing v1fetch test must also
// pass unchanged.
docs/plans/p6/_files/batch/runner_core_test.go
+182
View File
@@ -0,0 +1,182 @@
package batch
// P6 task 02: the runner's core — types, Submit (idempotency, caps, storage), Status, Touch and
// Cancel — with no workers yet: every job stays queued. The helpers here are shared with task
// 03's runner_test.go.
import (
"context"
"errors"
"io"
"log/slog"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/dcache"
)
var (
alice = &auth.Caller{Name: "alice", Workspace: "hermes-fleet"}
bob = &auth.Caller{Name: "bob", Workspace: "hermes-fleet"}
carol = &auth.Caller{Name: "carol", Workspace: "hermes-fleet"}
)
func newRunner(t *testing.T, fetch FetchFunc, cfg Config) *Runner {
t.Helper()
db, err := dcache.Open("", 1<<20, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
r := New(db, fetch, cfg, slog.New(slog.NewTextHandler(io.Discard, nil)))
t.Cleanup(r.Close) // runs before db.Close (cleanups are LIFO)
return r
}
func waitStatus(t *testing.T, r *Runner, id, want string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
j, _, ok := r.Status(id)
if ok && j.Status == want {
return
}
time.Sleep(2 * time.Millisecond)
}
j, _, _ := r.Status(id)
t.Fatalf("job %s never reached %q (now %q)", id, want, j.Status)
}
func urls(ss ...string) []Target {
var out []Target
for _, s := range ss {
out = append(out, Target{URL: "https://" + s + ".example/"})
}
return out
}
// Submit stores the job queued with its targets pending, before it returns.
func TestSubmitStoresQueued(t *testing.T) {
r := newRunner(t, func(context.Context, *auth.Caller, Target) (Result, error) {
return Result{Content: "ok"}, nil
}, Config{Workers: 1})
id, reused, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a", "b"))
if err != nil || reused || len(id) != 32 {
t.Fatalf("submit id=%q reused=%v err=%v, want a 32-hex-char id", id, reused, err)
}
j, ts, ok := r.Status(id)
if !ok || j.ID != id || j.Caller != "alice" || j.TargetsTotal != 2 || j.CreatedAt == 0 {
t.Fatalf("job %+v ok=%v", j, ok)
}
if j.Status != "queued" && j.Status != "running" && j.Status != "done" {
t.Fatalf("status %q", j.Status)
}
if len(ts) != 2 || ts[0].Idx != 0 || ts[0].URL != "https://a.example/" || ts[1].URL != "https://b.example/" {
t.Fatalf("targets %+v", ts)
}
}
// Cancelling a job no worker has reached: it is cancelled at once, its targets stay pending,
// and the same list then starts a new job.
func TestCancelQueued(t *testing.T) {
release := make(chan struct{})
defer close(release)
fetch := func(context.Context, *auth.Caller, Target) (Result, error) {
<-release
return Result{Content: "ok"}, nil
}
r := newRunner(t, fetch, Config{Workers: 1})
busy, _, err := r.Submit(context.Background(), bob, dcache.ClassFleet, urls("x")) // occupies the only worker once there is one
if err != nil {
t.Fatal(err)
}
_ = busy
id, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a", "b"))
if err != nil {
t.Fatal(err)
}
if !r.Cancel(id) {
t.Fatal("cancel returned false")
}
j, ts, _ := r.Status(id)
if j.Status != "cancelled" || ts[0].Status != "pending" || ts[1].Status != "pending" {
t.Fatalf("after cancel: %+v %+v", j, ts)
}
if r.Cancel(id) || r.Cancel("no-such-job") {
t.Fatal("cancel of a finished or unknown job must return false")
}
id2, reused, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a", "b"))
if err != nil || reused || id2 == id {
t.Fatalf("resubmit after cancel id=%q reused=%v err=%v", id2, reused, err)
}
}
// Touch keeps a job alive; Status never does.
func TestTouchAndStatusLease(t *testing.T) {
db, err := dcache.Open("", 1<<20, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
release := make(chan struct{})
fetch := func(context.Context, *auth.Caller, Target) (Result, error) {
<-release
return Result{Content: "ok"}, nil
}
r := New(db, fetch, Config{Workers: 1}, slog.New(slog.NewTextHandler(io.Discard, nil)))
t.Cleanup(r.Close)
defer close(release) // runs before the cleanups
id, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a"))
if err != nil {
t.Fatal(err)
}
time.Sleep(50 * time.Millisecond) // let a worker (task 03) take the target; it then blocks
before, _ := db.BatchJob(id)
time.Sleep(1100 * time.Millisecond) // the lease is in whole seconds
r.Status(id)
if j, _ := db.BatchJob(id); j.Lease != before.Lease {
t.Fatalf("Status moved the lease %d → %d; a read must not write", before.Lease, j.Lease)
}
r.Touch(id)
if j, _ := db.BatchJob(id); j.Lease <= before.Lease {
t.Fatalf("Touch did not move the lease: %d → %d", before.Lease, j.Lease)
}
}
func TestCaps(t *testing.T) {
release := make(chan struct{})
fetch := func(context.Context, *auth.Caller, Target) (Result, error) {
<-release
return Result{Content: "ok"}, nil
}
r := newRunner(t, fetch, Config{TargetsPerJob: 2, JobsTotal: 2})
defer close(release)
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a", "b", "c")); !errors.Is(err, ErrTooManyTargets) {
t.Fatalf("3 targets over a cap of 2: want ErrTooManyTargets, got %v", err)
}
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, nil); !errors.Is(err, ErrNoTargets) {
t.Fatalf("no targets: want ErrNoTargets, got %v", err)
}
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a")); err != nil {
t.Fatal(err)
}
// alice's job is live (queued or running): a second, different list is refused.
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("z")); !errors.Is(err, ErrCallerBusy) {
t.Fatalf("want ErrCallerBusy, got %v", err)
}
if _, _, err := r.Submit(context.Background(), bob, dcache.ClassFleet, urls("b")); err != nil {
t.Fatal(err)
}
// Two live jobs in total: a third caller is refused.
if _, _, err := r.Submit(context.Background(), carol, dcache.ClassFleet, urls("c")); !errors.Is(err, ErrBusy) {
t.Fatalf("want ErrBusy, got %v", err)
}
}
func TestStatusUnknown(t *testing.T) {
r := newRunner(t, func(context.Context, *auth.Caller, Target) (Result, error) { return Result{}, nil }, Config{})
if _, _, ok := r.Status("no-such-job"); ok {
t.Fatal("unknown id must be ok=false")
}
}
docs/plans/p6/_files/batch/runner_test.go
-79
View File
@@ -4,8 +4,6 @@ import (
"context"
"errors"
"fmt"
"io"
"log/slog"
"strings"
"sync"
"testing"
@@ -15,46 +13,6 @@ import (
"git.wntrmute.dev/kyle/sift/internal/dcache"
)
var (
alice = &auth.Caller{Name: "alice", Workspace: "hermes-fleet"}
bob = &auth.Caller{Name: "bob", Workspace: "hermes-fleet"}
carol = &auth.Caller{Name: "carol", Workspace: "hermes-fleet"}
)
func newRunner(t *testing.T, fetch FetchFunc, cfg Config) *Runner {
t.Helper()
db, err := dcache.Open("", 1<<20, slog.New(slog.NewTextHandler(io.Discard, nil)))
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
r := New(db, fetch, cfg, slog.New(slog.NewTextHandler(io.Discard, nil)))
t.Cleanup(r.Close) // runs before db.Close (cleanups are LIFO)
return r
}
func waitStatus(t *testing.T, r *Runner, id, want string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for time.Now().Before(deadline) {
j, _, ok := r.Status(id)
if ok && j.Status == want {
return
}
time.Sleep(2 * time.Millisecond)
}
j, _, _ := r.Status(id)
t.Fatalf("job %s never reached %q (now %q)", id, want, j.Status)
}
func urls(ss ...string) []Target {
var out []Target
for _, s := range ss {
out = append(out, Target{URL: "https://" + s + ".example/"})
}
return out
}
// Targets run one at a time, in order, and each fetch is given the submitting caller.
func TestSerialOrderAndCaller(t *testing.T) {
var mu sync.Mutex
@@ -235,36 +193,6 @@ func TestResubmitAfterCancelIsNew(t *testing.T) {
}
}
func TestCaps(t *testing.T) {
release := make(chan struct{})
fetch := func(context.Context, *auth.Caller, Target) (Result, error) {
<-release
return Result{Content: "ok"}, nil
}
r := newRunner(t, fetch, Config{TargetsPerJob: 2, JobsTotal: 2})
defer close(release)
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a", "b", "c")); !errors.Is(err, ErrTooManyTargets) {
t.Fatalf("3 targets over a cap of 2: want ErrTooManyTargets, got %v", err)
}
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, nil); !errors.Is(err, ErrNoTargets) {
t.Fatalf("no targets: want ErrNoTargets, got %v", err)
}
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("a")); err != nil {
t.Fatal(err)
}
// alice's job is live (queued or running): a second, different list is refused.
if _, _, err := r.Submit(context.Background(), alice, dcache.ClassFleet, urls("z")); !errors.Is(err, ErrCallerBusy) {
t.Fatalf("want ErrCallerBusy, got %v", err)
}
if _, _, err := r.Submit(context.Background(), bob, dcache.ClassFleet, urls("b")); err != nil {
t.Fatal(err)
}
// Two live jobs in total: a third caller is refused.
if _, _, err := r.Submit(context.Background(), carol, dcache.ClassFleet, urls("c")); !errors.Is(err, ErrBusy) {
t.Fatalf("want ErrBusy, got %v", err)
}
}
// Submit returns with the job stored "queued"; it becomes "running" when a worker takes it.
func TestQueuedThenRunning(t *testing.T) {
release := make(chan struct{})
@@ -443,10 +371,3 @@ func TestHooks(t *testing.T) {
t.Fatalf("stored result bytes %d, want > 0 for the done target", bytes)
}
}
func TestStatusUnknown(t *testing.T) {
r := newRunner(t, func(context.Context, *auth.Caller, Target) (Result, error) { return Result{}, nil }, Config{})
if _, _, ok := r.Status("no-such-job"); ok {
t.Fatal("unknown id must be ok=false")
}
}
docs/plans/p6/_files/config/batch_test.go
+1 -1
View File
@@ -1,6 +1,6 @@
package config
// P6 task 07: the [batch] block and ParseSize.
// P6 task 08: the [batch] block and ParseSize.
import (
"os"
docs/plans/p6/_files/guard/precheck_test.go
-53
View File
@@ -1,53 +0,0 @@
package guard
// P6 task 05: PreCheck is CheckURL without name resolution, for refusing a batch target at
// submit time.
import (
"context"
"net/netip"
"strings"
"testing"
)
func TestPreCheck(t *testing.T) {
g := &Guard{
AllowPrefixes: []netip.Prefix{netip.MustParsePrefix("127.0.0.0/8")},
LookupIP: func(context.Context, string) ([]netip.Addr, error) {
t.Fatal("PreCheck resolved a name")
return nil, nil
},
}
for _, u := range []string{
"https://example.com/",
"http://example.com/path?q=1",
"https://internal.test/", // a name: not resolved here, so not refused here
"http://127.0.0.1:8080/", // an allowed literal is exempt from the port rule, as in CheckURL
"http://93.184.215.14/", // a public literal
"http://0x7f000001:8080/", // inet_aton form of 127.0.0.1: an allowed literal
} {
if err := g.PreCheck(u); err != nil {
t.Errorf("PreCheck(%q) = %v, want nil", u, err)
}
}
for _, u := range []string{
"",
"ftp://example.com/",
"http://user@example.com/",
"http://example.com:8080/",
"http://10.0.0.5/",
"http://169.254.169.254/latest/meta-data/",
"http://[::1]/",
"http://0x0a000005/", // inet_aton form of 10.0.0.5
} {
err := g.PreCheck(u)
if err == nil || !strings.HasPrefix(err.Error(), "blocked") {
t.Errorf("PreCheck(%q) = %v, want a blocked error", u, err)
}
}
// With no allow list, loopback is refused like any other private literal.
var zero Guard
if err := zero.PreCheck("http://127.0.0.1/"); err == nil {
t.Error("a zero Guard allowed loopback")
}
}