dcache: PDF document cache (per URL and caller class, whole-document eviction)

Add pdf_doc/pdf_page tables plus PDFEntry, PDFKey, PutPDF (one transaction,
old pages deleted first), GetPDF (page-ordered, updates last_hit) and TouchPDF
(304 revalidation moves only fetched_at). PDF documents count toward Size() and
are evicted whole by oldest use; PDFH/PDFM counters added. evict() now breaks
last_hit ties by rowid so same-second inserts evict in insertion order.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
2026-09-25 23:05:12 -07:00
co-authored by ornith-1.5-35b-a3b
parent f57aaef59e
commit b93b7d5816
4 changed files with 314 additions and 6 deletions
+128
View File
@@ -0,0 +1,128 @@
package dcache_test
// P5a task 03: the PDF document cache. One entry per (URL, caller class): how to revalidate it,
// its metadata (opaque JSON here) and the text of every extracted page.
import (
"path/filepath"
"strings"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/dcache"
)
func openDB(t *testing.T, maxBytes int64) *dcache.DB {
t.Helper()
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), maxBytes, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
func entry(pages ...string) dcache.PDFEntry {
return dcache.PDFEntry{
ContentHash: "sha256:abc",
ETag: `"v1"`,
LastModified: "Fri, 25 Sep 2026 12:00:00 GMT",
FetchedAt: time.Date(2026, 9, 25, 20, 0, 0, 0, time.UTC),
Meta: []byte(`{"pages":3,"title":"T","outline":[]}`),
Pages: pages,
}
}
func TestPDFRoundTrip(t *testing.T) {
db := openDB(t, 0)
k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet)
if _, ok := db.GetPDF(k); ok {
t.Fatal("hit on an empty cache")
}
want := entry("page one", "page two", "", "page four") // an empty page keeps its place
db.PutPDF(k, want)
got, ok := db.GetPDF(k)
if !ok {
t.Fatal("miss after put")
}
if got.ContentHash != want.ContentHash || got.ETag != want.ETag || got.LastModified != want.LastModified ||
!got.FetchedAt.Equal(want.FetchedAt) || string(got.Meta) != string(want.Meta) {
t.Errorf("got %+v, want %+v", got, want)
}
if len(got.Pages) != 4 || got.Pages[0] != "page one" || got.Pages[2] != "" || got.Pages[3] != "page four" {
t.Errorf("pages = %q", got.Pages)
}
// A second put replaces the whole entry, pages included.
db.PutPDF(k, entry("only"))
got, _ = db.GetPDF(k)
if len(got.Pages) != 1 || got.Pages[0] != "only" {
t.Errorf("after replace, pages = %q; want [only]", got.Pages)
}
}
func TestPDFKeyPartitions(t *testing.T) {
db := openDB(t, 0)
u := "https://example.org/a.pdf"
fleet, restricted := dcache.PDFKey(u, dcache.ClassFleet), dcache.PDFKey(u, dcache.ClassRestricted)
if fleet == restricted {
t.Fatal("the same key for both caller classes")
}
db.PutPDF(restricted, entry("secret page"))
if _, ok := db.GetPDF(fleet); ok {
t.Error("a restricted caller's document is visible to the fleet")
}
if dcache.PDFKey(u, dcache.ClassFleet) != fleet || dcache.PDFKey("https://example.org/b.pdf", dcache.ClassFleet) == fleet {
t.Error("PDFKey must be deterministic and differ per URL")
}
if strings.Contains(fleet, "example.org") {
t.Error("the key must not carry the URL in the clear (it is hashed like the other keys)")
}
}
func TestPDFTouch(t *testing.T) {
db := openDB(t, 0)
k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet)
db.PutPDF(k, entry("x"))
later := time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC)
db.TouchPDF(k, later) // after a 304: the text is unchanged, only its freshness moves
got, _ := db.GetPDF(k)
if !got.FetchedAt.Equal(later) || len(got.Pages) != 1 {
t.Errorf("after touch: %+v", got)
}
db.TouchPDF(dcache.PDFKey("https://nope", dcache.ClassFleet), later) // a miss is a no-op
}
// Documents count toward maxBytes and are evicted whole, oldest use first.
func TestPDFEviction(t *testing.T) {
db := openDB(t, 300<<10) // 300 KiB
big := strings.Repeat("z", 50<<10)
var keys []string
for i := 0; i < 10; i++ { // 10 × 2 pages × 50 KiB = 1 MiB of text
k := dcache.PDFKey("https://example.org/"+string(rune('a'+i))+".pdf", dcache.ClassFleet)
keys = append(keys, k)
db.PutPDF(k, entry(big, big))
}
if db.Size() > 300<<10 {
t.Errorf("size %d after puts, want ≤ maxBytes %d", db.Size(), 300<<10)
}
if _, ok := db.GetPDF(keys[9]); !ok {
t.Error("the newest document was evicted")
}
if _, ok := db.GetPDF(keys[0]); ok {
t.Error("the oldest document survived; eviction did not reach PDF entries")
}
for _, k := range keys {
if e, ok := db.GetPDF(k); ok && len(e.Pages) != 2 {
t.Errorf("a partly evicted document: %d pages", len(e.Pages))
}
}
}
func TestPDFNilDB(t *testing.T) {
var db *dcache.DB
db.PutPDF("k", entry("x"))
db.TouchPDF("k", time.Now())
if _, ok := db.GetPDF("k"); ok {
t.Error("nil DB returned a hit")
}
}