serve: [batch] config; open the store, reap jobs killed by a restart, start the runner

Add the [batch] config block (Batch type + File.Batch) and ParseSize (binary
KiB/MiB/GiB byte counts) in internal/config. serve now opens the dcache store
when batch is on, marks the queued/running jobs a restart killed as failed
(never resumed) and observes them, builds the runner from the config with the
runner's defaults filling zeros, and closes the runner on both shutdown paths.
Batch is on by default; disabled = true leaves srv.Batch nil so the endpoints
answer 503.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

cmd/sift/serve.go

@@ -10,9 +10,11 @@ import (
 	"os/signal"
 	"strings"
 	"syscall"
+	"time"
 
 	"git.wntrmute.dev/kyle/sift/internal/api"
 	"git.wntrmute.dev/kyle/sift/internal/auth"
+	"git.wntrmute.dev/kyle/sift/internal/batch"
 	"git.wntrmute.dev/kyle/sift/internal/config"
 	"git.wntrmute.dev/kyle/sift/internal/dcache"
 	"git.wntrmute.dev/kyle/sift/internal/digest"
@@ -119,7 +121,7 @@ func serveMain() {
 		cachePath = "/var/lib/sift/cache.db"
 	}
 	var cache *dcache.DB
-	if cfg.Dixie.Chat != "" || cfg.Search.AnyEnabled() {
+	if cfg.Dixie.Chat != "" || cfg.Search.AnyEnabled() || !cfg.Batch.Disabled {
 		c, err := dcache.Open(cachePath, 500<<20, log)
 		if err != nil {
 			log.Warn("cache", "err", err)
@@ -127,6 +129,23 @@ func serveMain() {
 			cache = c
 		}
 	}
+	if !cfg.Batch.Disabled && cache != nil {
+		now := time.Now().Unix()
+		if n := cache.MarkInFlightFailed("restart", now); n > 0 {
+			for i := 0; i < n; i++ {
+				srv.Prom.ObserveBatchJob("restart")
+			}
+			log.Info("batch reaped", "jobs", n)
+		}
+		srv.Batch = srv.NewBatchRunner(cache, batch.Config{
+			TTL:           config.ParseDuration(cfg.Batch.TTL, 0),
+			TargetsPerJob: cfg.Batch.TargetsPerJob,
+			JobsPerCaller: cfg.Batch.JobsPerCaller,
+			JobsTotal:     cfg.Batch.JobsTotal,
+			Workers:       cfg.Batch.Workers,
+			MaxBytes:      config.ParseSize(cfg.Batch.MaxBytes, 0),
+		})
+	}
 	if cfg.Dixie.Chat != "" {
 		srv.Digest = &digest.Engine{
 			ChatURL:    cfg.Dixie.Chat,
@@ -213,8 +232,14 @@ func serveMain() {
 	case <-sig:
 		stop()
 		_ = hs.Close()
+		if srv.Batch != nil {
+			srv.Batch.Close()
+		}
 	case err := <-errCh:
 		stop()
+		if srv.Batch != nil {
+			srv.Batch.Close()
+		}
 		if err != nil {
 			log.Error("serve", "err", err)
 			os.Exit(1)