batch: workers — round-robin, serial per job, busy retry, cancel in flight, hooks

Start cfg.Workers goroutines in New that drain one rotation of live jobs in
submit order, each taking the next job with a pending target and none in flight,
so targets alternate across jobs and each job stays serial. One target per fetch
under the Close-cancelled context: panic recovers into ErrInternal (never
echoed), ErrRetryLater retries after RetryWait (bounded by RetryFor), otherwise
the target is stored done/failed and the hooks fire. Terminal status is set once
per job; cancelling a running job stores the in-flight target without overwriting
cancelled and leaves later targets pending.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/batch/batch.go

@@ -121,6 +121,16 @@ type Runner struct {
 	callers   map[string]*auth.Caller // job id → submitting caller, held in memory only
 	callMu    sync.Mutex
 	closeOnce sync.Once
+
+	mu       sync.Mutex // guards everything below
+	cond     *sync.Cond // on mu
+	live     []*liveJob // the rotation, in submit order
+	cursor   int        // absolute rotation position; wraps at access, so it survives jobs being added
+	closed   bool
+	ctx      context.Context // the FetchFunc context; Close cancels it
+	stop     context.CancelFunc
+	wg       sync.WaitGroup // the workers
+	inflight int
 }
 
 func New(db *dcache.DB, fetch FetchFunc, cfg Config, log *slog.Logger) *Runner {
@@ -151,13 +161,22 @@ func New(db *dcache.DB, fetch FetchFunc, cfg Config, log *slog.Logger) *Runner {
 	if cfg.RetryFor == 0 {
 		cfg.RetryFor = 10 * time.Minute
 	}
-	return &Runner{
+	ctx, stop := context.WithCancel(context.Background())
+	r := &Runner{
 		db:      db,
 		fetch:   fetch,
 		log:     log,
 		cfg:     cfg,
 		callers: make(map[string]*auth.Caller),
+		ctx:     ctx,
+		stop:    stop,
 	}
+	r.cond = sync.NewCond(&r.mu)
+	r.wg.Add(cfg.Workers)
+	for i := 0; i < cfg.Workers; i++ {
+		go r.worker()
+	}
+	return r
 }
 
 // Submit stores the job and its targets before it returns. The order matters: an empty
@@ -229,14 +248,52 @@ func (r *Runner) Submit(ctx context.Context, c *auth.Caller, callerClass string,
 	r.callers[id] = c
 	r.callMu.Unlock()
 
+	r.mu.Lock()
+	r.live = append(r.live, &liveJob{
+		id:      id,
+		caller:  c,
+		targets: targets,
+		created: time.Unix(now, 0),
+	})
+	r.callDepth(len(r.live))
+	r.cond.Broadcast()
+	r.mu.Unlock()
+
 	r.callJob("created")
 	r.db.SweepBatch(now, r.cfg.TTL)
 	return id, false, nil
 }
 
-// Cancel sets a queued job cancelled at once; its pending targets stay pending. It is
-// false for an unknown id or a job already terminal.
+// Cancel cancels a job. A job still in the rotation (queued or with a target in flight) is marked
+// cancelled at once: its in-flight target, if any, finishes and is stored without overwriting
+// cancelled, and no further target starts. A job not in the rotation (unknown or already terminal)
+// keeps task 02's behaviour.
 func (r *Runner) Cancel(id string) bool {
+	r.mu.Lock()
+	var j *liveJob
+	for _, lj := range r.live {
+		if lj.id == id {
+			j = lj
+			break
+		}
+	}
+	if j != nil {
+		j.cancelled = true
+		if err := r.db.SetBatchJobStatus(id, "cancelled", "", time.Now().Unix()); err != nil {
+			r.log.Warn("batch: cancel job", "job", id, "err", err)
+		}
+		r.callJob("cancelled")
+		if !j.running {
+			r.removeLocked(j)
+			r.forgetCaller(id)
+			r.callDepth(len(r.live))
+		}
+		r.cond.Broadcast()
+		r.mu.Unlock()
+		return true
+	}
+	r.mu.Unlock()
+
 	jb, ok := r.db.BatchJob(id)
 	if !ok || isJobTerminal(jb.Status) {
 		return false
@@ -298,13 +355,20 @@ func (r *Runner) Touch(id string) {
 	}
 }
 
-// Close stops the workers and waits for an in-flight fetch to return. Safe to call more than
-// once. With no workers (task 02) there is nothing to stop; the in-memory callers are dropped.
+// Close stops the workers, cancels their context, and waits for an in-flight fetch to return.
+// Safe to call more than once. The in-memory callers are dropped.
 func (r *Runner) Close() {
 	r.closeOnce.Do(func() {
 		r.callMu.Lock()
 		r.callers = nil
 		r.callMu.Unlock()
+
+		r.mu.Lock()
+		r.closed = true
+		r.stop()
+		r.cond.Broadcast()
+		r.mu.Unlock()
+		r.wg.Wait()
 	})
 }