dcache: batch job and target tables with a sliding lease, sweep and byte budget
Add the batch job store to internal/dcache (storage only; no HTTP, no worker). New internal/dcache/batch.go holds the BatchJob/BatchTarget types and all accessors; Open gains the batch_job/batch_target tables and indices, and DB gains BatchH/BatchM counters. Sliding lease: writes (UpdateBatchTarget, SetBatchJobStatus, TouchBatchJob, MarkInFlightFailed) bump the job lease to now; reads never write. BatchJobByIDem returns the newest (caller, idem_key) job with lease+ttl>now and status queued/running/done/partial. UpdateBatchTarget counts targets_done exactly once per target entering done/failed and adds len(result) to nbytes. SweepBatch removes by lease; EvictBatchOldest evicts whole non-live jobs oldest-lease-first; MarkInFlightFailed reconciles in-flight jobs on restart. Batch nbytes are kept out of Size() and the content evict union. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
@@ -47,6 +47,8 @@ type DB struct {
|
||||
SearchM uint64
|
||||
PDFH uint64
|
||||
PDFM uint64
|
||||
BatchH uint64
|
||||
BatchM uint64
|
||||
Evictions uint64
|
||||
}
|
||||
|
||||
@@ -110,6 +112,32 @@ func Open(path string, maxBytes int64, log *slog.Logger) (*DB, error) {
|
||||
text TEXT,
|
||||
PRIMARY KEY (k, page)
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS batch_job (
|
||||
id TEXT PRIMARY KEY,
|
||||
caller TEXT NOT NULL,
|
||||
caller_class TEXT NOT NULL,
|
||||
idem_key TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
error TEXT NOT NULL DEFAULT '',
|
||||
targets_total INTEGER NOT NULL,
|
||||
targets_done INTEGER NOT NULL DEFAULT 0,
|
||||
created_at INTEGER NOT NULL,
|
||||
lease INTEGER NOT NULL,
|
||||
nbytes INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
CREATE TABLE IF NOT EXISTS batch_target (
|
||||
job_id TEXT NOT NULL,
|
||||
idx INTEGER NOT NULL,
|
||||
url TEXT NOT NULL,
|
||||
options TEXT NOT NULL,
|
||||
status TEXT NOT NULL,
|
||||
result TEXT,
|
||||
error TEXT,
|
||||
hint TEXT,
|
||||
PRIMARY KEY (job_id, idx)
|
||||
);
|
||||
CREATE INDEX IF NOT EXISTS batch_job_idem ON batch_job(caller, idem_key);
|
||||
CREATE INDEX IF NOT EXISTS batch_job_lease ON batch_job(lease);
|
||||
`); err != nil {
|
||||
_ = sq.Close()
|
||||
return nil, err
|
||||
|
||||
Reference in New Issue
Block a user