dcache: PDF document cache (per URL and caller class, whole-document eviction)

Add pdf_doc/pdf_page tables plus PDFEntry, PDFKey, PutPDF (one transaction, old pages deleted first), GetPDF (page-ordered, updates last_hit) and TouchPDF (304 revalidation moves only fetched_at). PDF documents count toward Size() and are evicted whole by oldest use; PDFH/PDFM counters added. evict() now breaks last_hit ties by rowid so same-second inserts evict in insertion order. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
2026-09-25 23:05:12 -07:00
parent f57aaef59e
commit b93b7d5816
4 changed files with 314 additions and 6 deletions
@@ -6,3 +6,4 @@ One row per task session that ends in a commit. Newest last.
 |---|---|---|---|---|---|---|
 | p5a/01-kind-fetch | 2026-09-25 | done | resumed | gate: ok | none | Implemented Kind/`application/pdf` recognition, PDF body cap (100 MB) and deadline (120 s), and FetchIf with If-None-Match/If-Modified-Since via shared doFetch. Previous session could not pass TestFetchIf only because the given test held its mutex across the handler (deadlock); owner fixed the test. Session resumed after that fix — did not rewrite fetch code, only verified rules and ran gate. |
 | p5a/02-pdftext | 2026-09-25 | done | 1 | gate: ok | none | Built the child-extraction package: `Extract` (reader.Open with ErrWrongPassword→ErrEncrypted, per-page recover, trailer /Info /Title, rule-3 UTF-16BE/Latin-1 decode), `outline.go` (/Dest and /GoTo /A dests, 10k-item and no-double-visit bounds, never-nil outline), `Runner` (re-exec'd child, env-isolated via empty env, 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code→error mapping, Doc parse plus ErrNoText), and the `sift pdftext` case in main.go. Removed the tools.go module pin; go.mod/go.sum/vendor/modules.txt confirmed unchanged. All 10 given tests pass under -race -count=3 including the deadline-kill test; make gate ok; given files byte-identical; `printf nope | bin/sift pdftext` exits 4. |
+| p5a/03-dcache | 2026-09-25 | done | 1 | gate: ok | evict() tie-break | Added the pdf_doc/pdf_page tables, `PDFEntry`/`PDFKey`/`PutPDF`/`GetPDF`/`TouchPDF` (one-tx write deleting old pages first, page-ordered read, 304 touch moves only fetched_at), PDFH/PDFM counters, `Size()` including pdf_doc nbytes. One deviation: extended the eviction union with `pdf_doc` and added `rowid ASC` to the ORDER BY so same-second puts evict in insertion order (deterministic whole-document eviction); without it TestPDFEviction is order-dependent. Left Stats() signature unchanged for task 07. Given test byte-identical. |
@@ -45,6 +45,8 @@ type DB struct {
 	EmbedM    uint64
 	SearchH   uint64
 	SearchM   uint64
+	PDFH      uint64
+	PDFM      uint64
 	Evictions uint64
 }

@@ -92,6 +94,22 @@ func Open(path string, maxBytes int64, log *slog.Logger) (*DB, error) {
 			credits INTEGER NOT NULL,
 			PRIMARY KEY (source, day)
 		);
+		CREATE TABLE IF NOT EXISTS pdf_doc (
+			k TEXT PRIMARY KEY,
+			content_hash,
+			etag,
+			last_modified,
+			fetched_at INTEGER,
+			meta BLOB,
+			last_hit INTEGER,
+			nbytes INTEGER
+		);
+		CREATE TABLE IF NOT EXISTS pdf_page (
+			k TEXT,
+			page INTEGER,
+			text TEXT,
+			PRIMARY KEY (k, page)
+		);
 	`); err != nil {
 		_ = sq.Close()
 		return nil, err
@@ -230,11 +248,12 @@ func (d *DB) Size() int64 {
 	if d == nil {
 		return 0
 	}
-	var n, m, s sql.NullInt64
+	var n, m, s, p sql.NullInt64
 	_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM digest`).Scan(&n)
 	_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM embed`).Scan(&m)
 	_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM search`).Scan(&s)
-	return n.Int64 + m.Int64 + s.Int64
+	_ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM pdf_doc`).Scan(&p)
+	return n.Int64 + m.Int64 + s.Int64 + p.Int64
 }

 func (d *DB) GetSearch(key string) (payload []byte, ok bool) {
@@ -313,18 +332,22 @@ func (d *DB) evict() {
 		var k string
 		err := d.sql.QueryRow(`
 			SELECT k FROM (
-				SELECT k, last_hit FROM digest
+				SELECT k, last_hit, rowid AS rid FROM digest
 				UNION ALL
-				SELECT k, last_hit FROM embed
+				SELECT k, last_hit, rowid AS rid FROM embed
 				UNION ALL
-				SELECT k, last_hit FROM search
-			) ORDER BY last_hit ASC LIMIT 1`).Scan(&k)
+				SELECT k, last_hit, rowid AS rid FROM search
+				UNION ALL
+				SELECT k, last_hit, rowid AS rid FROM pdf_doc
+			) ORDER BY last_hit ASC, rid ASC LIMIT 1`).Scan(&k)
 		if err != nil {
 			return
 		}
 		_, _ = d.sql.Exec(`DELETE FROM digest WHERE k = ?`, k)
 		_, _ = d.sql.Exec(`DELETE FROM embed WHERE k = ?`, k)
 		_, _ = d.sql.Exec(`DELETE FROM search WHERE k = ?`, k)
+		_, _ = d.sql.Exec(`DELETE FROM pdf_doc WHERE k = ?`, k)
+		_, _ = d.sql.Exec(`DELETE FROM pdf_page WHERE k = ?`, k)
 		d.mu.Lock()
 		d.Evictions++
 		d.mu.Unlock()
@@ -0,0 +1,156 @@
+package dcache
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"time"
+)
+
+// PDFEntry is one cached PDF: how to revalidate it, its metadata (the pdftext.Meta JSON, opaque
+// here) and the text of each extracted page (Pages[i] is page i+1).
+type PDFEntry struct {
+	ContentHash  string
+	ETag         string
+	LastModified string
+	FetchedAt    time.Time
+	Meta         []byte
+	Pages        []string
+}
+
+// PDFKey is the cache key for a document: a hash of the URL and the caller class, like the
+// other keys here (the URL never appears in the clear).
+func PDFKey(url, callerClass string) string {
+	if callerClass == "" {
+		callerClass = ClassFleet
+	}
+	sum := sha256.Sum256([]byte(url + "\x00" + callerClass))
+	return hex.EncodeToString(sum[:])
+}
+
+// PutPDF writes the document and its pages in one transaction, deleting the old pages first so a
+// failed replace never leaves a partial document behind, then calls evict.
+func (d *DB) PutPDF(key string, e PDFEntry) {
+	if d == nil {
+		return
+	}
+	nbytes := len(e.Meta)
+	for _, p := range e.Pages {
+		nbytes += len(p)
+	}
+	tx, err := d.sql.Begin()
+	if err != nil {
+		if d.log != nil {
+			d.log.Warn("pdf cache begin", "err", err)
+		}
+		return
+	}
+	if _, err := tx.Exec(`DELETE FROM pdf_page WHERE k = ?`, key); err != nil {
+		_ = tx.Rollback()
+		if d.log != nil {
+			d.log.Warn("pdf cache pages del", "err", err)
+		}
+		return
+	}
+	if _, err := tx.Exec(`INSERT OR REPLACE INTO pdf_doc(k, content_hash, etag, last_modified, fetched_at, meta, last_hit, nbytes) VALUES(?,?,?,?,?,?,?,?)`,
+		key, e.ContentHash, e.ETag, e.LastModified, e.FetchedAt.Unix(), e.Meta, time.Now().Unix(), nbytes); err != nil {
+		_ = tx.Rollback()
+		if d.log != nil {
+			d.log.Warn("pdf cache doc", "err", err)
+		}
+		return
+	}
+	for i, p := range e.Pages {
+		if _, err := tx.Exec(`INSERT INTO pdf_page(k, page, text) VALUES(?,?,?)`, key, i, p); err != nil {
+			_ = tx.Rollback()
+			if d.log != nil {
+				d.log.Warn("pdf cache page put", "err", err)
+			}
+			return
+		}
+	}
+	if err := tx.Commit(); err != nil {
+		if d.log != nil {
+			d.log.Warn("pdf cache commit", "err", err)
+		}
+		return
+	}
+	d.evict()
+}
+
+// GetPDF returns the pages in page order (an empty page keeps its place), updates last_hit, and
+// counts the hit; a missing document is a miss.
+func (d *DB) GetPDF(key string) (PDFEntry, bool) {
+	if d == nil {
+		return PDFEntry{}, false
+	}
+	var e PDFEntry
+	var fetchedAt int64
+	var meta []byte
+	err := d.sql.QueryRow(`
+		SELECT content_hash, etag, last_modified, fetched_at, meta
+		FROM pdf_doc
+		WHERE k = ?`, key).Scan(&e.ContentHash, &e.ETag, &e.LastModified, &fetchedAt, &meta)
+	if err != nil {
+		d.mu.Lock()
+		d.PDFM++
+		d.mu.Unlock()
+		return PDFEntry{}, false
+	}
+	e.FetchedAt = time.Unix(fetchedAt, 0).UTC()
+	e.Meta = meta
+
+	rows, err := d.sql.Query(`SELECT page, text FROM pdf_page WHERE k = ? ORDER BY page ASC`, key)
+	if err != nil {
+		if d.log != nil {
+			d.log.Warn("pdf cache pages", "err", err)
+		}
+		d.mu.Lock()
+		d.PDFM++
+		d.mu.Unlock()
+		return PDFEntry{}, false
+	}
+	defer rows.Close()
+	pages := make([]string, 0)
+	for rows.Next() {
+		var pg int
+		var text string
+		if err := rows.Scan(&pg, &text); err != nil {
+			if d.log != nil {
+				d.log.Warn("pdf cache page", "err", err)
+			}
+			d.mu.Lock()
+			d.PDFM++
+			d.mu.Unlock()
+			return PDFEntry{}, false
+		}
+		pages = append(pages, text)
+	}
+	if err := rows.Err(); err != nil {
+		if d.log != nil {
+			d.log.Warn("pdf cache pages", "err", err)
+		}
+		d.mu.Lock()
+		d.PDFM++
+		d.mu.Unlock()
+		return PDFEntry{}, false
+	}
+	e.Pages = pages
+
+	_, _ = d.sql.Exec(`UPDATE pdf_doc SET last_hit = ? WHERE k = ?`, time.Now().Unix(), key)
+	d.mu.Lock()
+	d.PDFH++
+	d.mu.Unlock()
+	return e, true
+}
+
+// TouchPDF moves only the freshness timestamp after a 304; a miss changes nothing.
+func (d *DB) TouchPDF(key string, fetchedAt time.Time) {
+	if d == nil {
+		return
+	}
+	if _, err := d.sql.Exec(`UPDATE pdf_doc SET fetched_at = ? WHERE k = ?`, fetchedAt.Unix(), key); err != nil {
+		if d.log != nil {
+			d.log.Warn("pdf cache touch", "err", err)
+		}
+	}
+}
@@ -0,0 +1,128 @@
+package dcache_test
+
+// P5a task 03: the PDF document cache. One entry per (URL, caller class): how to revalidate it,
+// its metadata (opaque JSON here) and the text of every extracted page.
+
+import (
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/dcache"
+)
+
+func openDB(t *testing.T, maxBytes int64) *dcache.DB {
+	t.Helper()
+	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), maxBytes, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+	return db
+}
+
+func entry(pages ...string) dcache.PDFEntry {
+	return dcache.PDFEntry{
+		ContentHash:  "sha256:abc",
+		ETag:         `"v1"`,
+		LastModified: "Fri, 25 Sep 2026 12:00:00 GMT",
+		FetchedAt:    time.Date(2026, 9, 25, 20, 0, 0, 0, time.UTC),
+		Meta:         []byte(`{"pages":3,"title":"T","outline":[]}`),
+		Pages:        pages,
+	}
+}
+
+func TestPDFRoundTrip(t *testing.T) {
+	db := openDB(t, 0)
+	k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet)
+	if _, ok := db.GetPDF(k); ok {
+		t.Fatal("hit on an empty cache")
+	}
+	want := entry("page one", "page two", "", "page four") // an empty page keeps its place
+	db.PutPDF(k, want)
+	got, ok := db.GetPDF(k)
+	if !ok {
+		t.Fatal("miss after put")
+	}
+	if got.ContentHash != want.ContentHash || got.ETag != want.ETag || got.LastModified != want.LastModified ||
+		!got.FetchedAt.Equal(want.FetchedAt) || string(got.Meta) != string(want.Meta) {
+		t.Errorf("got %+v, want %+v", got, want)
+	}
+	if len(got.Pages) != 4 || got.Pages[0] != "page one" || got.Pages[2] != "" || got.Pages[3] != "page four" {
+		t.Errorf("pages = %q", got.Pages)
+	}
+	// A second put replaces the whole entry, pages included.
+	db.PutPDF(k, entry("only"))
+	got, _ = db.GetPDF(k)
+	if len(got.Pages) != 1 || got.Pages[0] != "only" {
+		t.Errorf("after replace, pages = %q; want [only]", got.Pages)
+	}
+}
+
+func TestPDFKeyPartitions(t *testing.T) {
+	db := openDB(t, 0)
+	u := "https://example.org/a.pdf"
+	fleet, restricted := dcache.PDFKey(u, dcache.ClassFleet), dcache.PDFKey(u, dcache.ClassRestricted)
+	if fleet == restricted {
+		t.Fatal("the same key for both caller classes")
+	}
+	db.PutPDF(restricted, entry("secret page"))
+	if _, ok := db.GetPDF(fleet); ok {
+		t.Error("a restricted caller's document is visible to the fleet")
+	}
+	if dcache.PDFKey(u, dcache.ClassFleet) != fleet || dcache.PDFKey("https://example.org/b.pdf", dcache.ClassFleet) == fleet {
+		t.Error("PDFKey must be deterministic and differ per URL")
+	}
+	if strings.Contains(fleet, "example.org") {
+		t.Error("the key must not carry the URL in the clear (it is hashed like the other keys)")
+	}
+}
+
+func TestPDFTouch(t *testing.T) {
+	db := openDB(t, 0)
+	k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet)
+	db.PutPDF(k, entry("x"))
+	later := time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC)
+	db.TouchPDF(k, later) // after a 304: the text is unchanged, only its freshness moves
+	got, _ := db.GetPDF(k)
+	if !got.FetchedAt.Equal(later) || len(got.Pages) != 1 {
+		t.Errorf("after touch: %+v", got)
+	}
+	db.TouchPDF(dcache.PDFKey("https://nope", dcache.ClassFleet), later) // a miss is a no-op
+}
+
+// Documents count toward maxBytes and are evicted whole, oldest use first.
+func TestPDFEviction(t *testing.T) {
+	db := openDB(t, 300<<10) // 300 KiB
+	big := strings.Repeat("z", 50<<10)
+	var keys []string
+	for i := 0; i < 10; i++ { // 10 × 2 pages × 50 KiB = 1 MiB of text
+		k := dcache.PDFKey("https://example.org/"+string(rune('a'+i))+".pdf", dcache.ClassFleet)
+		keys = append(keys, k)
+		db.PutPDF(k, entry(big, big))
+	}
+	if db.Size() > 300<<10 {
+		t.Errorf("size %d after puts, want ≤ maxBytes %d", db.Size(), 300<<10)
+	}
+	if _, ok := db.GetPDF(keys[9]); !ok {
+		t.Error("the newest document was evicted")
+	}
+	if _, ok := db.GetPDF(keys[0]); ok {
+		t.Error("the oldest document survived; eviction did not reach PDF entries")
+	}
+	for _, k := range keys {
+		if e, ok := db.GetPDF(k); ok && len(e.Pages) != 2 {
+			t.Errorf("a partly evicted document: %d pages", len(e.Pages))
+		}
+	}
+}
+
+func TestPDFNilDB(t *testing.T) {
+	var db *dcache.DB
+	db.PutPDF("k", entry("x"))
+	db.TouchPDF("k", time.Now())
+	if _, ok := db.GetPDF("k"); ok {
+		t.Error("nil DB returned a hit")
+	}
+}