P6 review: runner shutdown, cancel-while-busy, concurrent submits, batch-off 503, transactional store, CLI 202

Owner review of p6 (Ornith tasks 01-09), with an independent reviewer pass whose findings I
confirmed against the code. Every fix has a regression test that fails on the old code.

internal/batch
- Close let the workers keep draining every live job under the cancelled context, failing
  every target; they now stop at once, store nothing on a failure during shutdown, and leave
  the job live for MarkInFlightFailed("restart").
- A job cancelled while its in-flight target came back busy stayed in the rotation forever
  (depth gauge stuck, job held in memory); it now leaves. The task 03 design had this gap: mine.
- Submit's idempotency lookup, caps and create ran unlocked: 20 concurrent submits by one
  caller made 8 jobs. Serialised with submitMu.
- A failed target insert left a queued job no worker would run (holding the caller's cap);
  it is marked failed.
- A second concurrent cancel returned true and counted twice.
- Config: zero OR negative values take the defaults (negative Workers panicked the
  WaitGroup; a negative TTL or byte budget would sweep or evict everything).
- Retry timing used the pre-fetch time (a 5 s busy answer made RetryWait moot).
- The rotation cursor is kept within [0, len(live)] (it was absolute and drifted on removal).
- The unused in-memory callers map is gone (Submit racing Close wrote to a nil map).
internal/dcache
- PutBatchTargets, SweepBatch and eviction are transactional; the sweep never removes a
  queued or running job.
internal/api
- Every /v1/batch endpoint answers 503 when batch is off; POST panicked on the nil runner.
- GET /{id}/{idx} bounds idx by the rows actually returned.
- batchFetch copies a *fetch.Error instead of mutating one the PDF pool may share.
internal/config
- ParseSize no longer overflows to a negative size.
cmd/sift
- sift batch treated 202 Accepted (submit) as an error and exited 1 with no message.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

internal/batch/batch.go

@@ -118,14 +118,13 @@ type Runner struct {
 	fetch     FetchFunc
 	log       *slog.Logger
 	cfg       Config
-	callers   map[string]*auth.Caller // job id → submitting caller, held in memory only
-	callMu    sync.Mutex
 	closeOnce sync.Once
+	submitMu  sync.Mutex // serialises Submit: idempotency lookup, caps and create are one step
 
 	mu       sync.Mutex // guards everything below
 	cond     *sync.Cond // on mu
 	live     []*liveJob // the rotation, in submit order
-	cursor   int        // absolute rotation position; wraps at access, so it survives jobs being added
+	cursor   int        // where the next scan starts, within [0, len(live)]; read as cursor % len(live)
 	closed   bool
 	ctx      context.Context // the FetchFunc context; Close cancels it
 	stop     context.CancelFunc
@@ -134,42 +133,43 @@ type Runner struct {
 }
 
 func New(db *dcache.DB, fetch FetchFunc, cfg Config, log *slog.Logger) *Runner {
-	if cfg.TTL == 0 {
+	// Zero or negative means the default: a negative TTL or byte budget would make the sweep or
+	// the eviction delete everything, and negative workers would panic the WaitGroup.
+	if cfg.TTL <= 0 {
 		cfg.TTL = 24 * time.Hour
 	}
-	if cfg.TargetsPerJob == 0 {
+	if cfg.TargetsPerJob <= 0 {
 		cfg.TargetsPerJob = 32
 	}
-	if cfg.JobsPerCaller == 0 {
+	if cfg.JobsPerCaller <= 0 {
 		cfg.JobsPerCaller = 1
 	}
-	if cfg.JobsTotal == 0 {
+	if cfg.JobsTotal <= 0 {
 		cfg.JobsTotal = 8
 	}
-	if cfg.Workers == 0 {
+	if cfg.Workers <= 0 {
 		cfg.Workers = 1
 	}
 	if cfg.Workers > 2 {
 		cfg.Workers = 2
 	}
-	if cfg.MaxBytes == 0 {
+	if cfg.MaxBytes <= 0 {
 		cfg.MaxBytes = 64 << 20
 	}
-	if cfg.RetryWait == 0 {
+	if cfg.RetryWait <= 0 {
 		cfg.RetryWait = 2 * time.Second
 	}
-	if cfg.RetryFor == 0 {
+	if cfg.RetryFor <= 0 {
 		cfg.RetryFor = 10 * time.Minute
 	}
 	ctx, stop := context.WithCancel(context.Background())
 	r := &Runner{
-		db:      db,
-		fetch:   fetch,
-		log:     log,
-		cfg:     cfg,
-		callers: make(map[string]*auth.Caller),
-		ctx:     ctx,
-		stop:    stop,
+		db:    db,
+		fetch: fetch,
+		log:   log,
+		cfg:   cfg,
+		ctx:   ctx,
+		stop:  stop,
 	}
 	r.cond = sync.NewCond(&r.mu)
 	r.wg.Add(cfg.Workers)
@@ -193,6 +193,11 @@ func (r *Runner) Submit(ctx context.Context, c *auth.Caller, callerClass string,
 	sum := sha256.Sum256([]byte(c.Name + "\x00" + string(raw)))
 	idemKey := hex.EncodeToString(sum[:])
 
+	// The lookup, the caps and the create must be one step, or concurrent submits all pass the
+	// checks before any of them is stored.
+	r.submitMu.Lock()
+	defer r.submitMu.Unlock()
+
 	now := time.Now().Unix()
 	if existing, ok := r.db.BatchJobByIDem(c.Name, idemKey, now, r.cfg.TTL); ok {
 		r.callJob("reused")
@@ -241,13 +246,11 @@ func (r *Runner) Submit(ctx context.Context, c *auth.Caller, callerClass string,
 	}
 	if err := r.db.PutBatchTargets(id, btargets); err != nil {
 		r.log.Warn("batch: store targets", "job", id, "err", err)
+		// Never leave a queued job that no worker will run: it would hold the caller's cap.
+		_ = r.db.SetBatchJobStatus(id, "failed", "store", now)
 		return "", false, fmt.Errorf("batch: store targets: %w", err)
 	}
 
-	r.callMu.Lock()
-	r.callers[id] = c
-	r.callMu.Unlock()
-
 	r.mu.Lock()
 	r.live = append(r.live, &liveJob{
 		id:      id,
@@ -278,6 +281,10 @@ func (r *Runner) Cancel(id string) bool {
 		}
 	}
 	if j != nil {
+		if j.cancelled { // a concurrent DELETE got here first
+			r.mu.Unlock()
+			return false
+		}
 		j.cancelled = true
 		if err := r.db.SetBatchJobStatus(id, "cancelled", "", time.Now().Unix()); err != nil {
 			r.log.Warn("batch: cancel job", "job", id, "err", err)
@@ -285,7 +292,6 @@ func (r *Runner) Cancel(id string) bool {
 		r.callJob("cancelled")
 		if !j.running {
 			r.removeLocked(j)
-			r.forgetCaller(id)
 			r.callDepth(len(r.live))
 		}
 		r.cond.Broadcast()
@@ -302,9 +308,6 @@ func (r *Runner) Cancel(id string) bool {
 		r.log.Warn("batch: cancel job", "job", id, "err", err)
 		return false
 	}
-	r.callMu.Lock()
-	delete(r.callers, id)
-	r.callMu.Unlock()
 	r.callJob("cancelled")
 	return true
 }
@@ -356,13 +359,10 @@ func (r *Runner) Touch(id string) {
 }
 
 // Close stops the workers, cancels their context, and waits for an in-flight fetch to return.
-// Safe to call more than once. The in-memory callers are dropped.
+// Safe to call more than once. Jobs still live stay queued or running in the store, for
+// MarkInFlightFailed("restart") on the next start.
 func (r *Runner) Close() {
 	r.closeOnce.Do(func() {
-		r.callMu.Lock()
-		r.callers = nil
-		r.callMu.Unlock()
-
 		r.mu.Lock()
 		r.closed = true
 		r.stop()