From b93b7d5816b7722cee9650bc47909572f5e894a7 Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Fri, 25 Sep 2026 23:05:12 -0700 Subject: [PATCH] dcache: PDF document cache (per URL and caller class, whole-document eviction) Add pdf_doc/pdf_page tables plus PDFEntry, PDFKey, PutPDF (one transaction, old pages deleted first), GetPDF (page-ordered, updates last_hit) and TouchPDF (304 revalidation moves only fetched_at). PDF documents count toward Size() and are evicted whole by oldest use; PDFH/PDFM counters added. evict() now breaks last_hit ties by rowid so same-second inserts evict in insertion order. Co-Authored-By: ornith-1.5-35b-a3b --- docs/implementer-log.md | 1 + internal/dcache/dcache.go | 35 ++++++-- internal/dcache/pdf.go | 156 ++++++++++++++++++++++++++++++++++++ internal/dcache/pdf_test.go | 128 +++++++++++++++++++++++++++++ 4 files changed, 314 insertions(+), 6 deletions(-) create mode 100644 internal/dcache/pdf.go create mode 100644 internal/dcache/pdf_test.go diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 88cbcc6..6e7eccd 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -6,3 +6,4 @@ One row per task session that ends in a commit. Newest last. |---|---|---|---|---|---|---| | p5a/01-kind-fetch | 2026-09-25 | done | resumed | gate: ok | none | Implemented Kind/`application/pdf` recognition, PDF body cap (100 MB) and deadline (120 s), and FetchIf with If-None-Match/If-Modified-Since via shared doFetch. Previous session could not pass TestFetchIf only because the given test held its mutex across the handler (deadlock); owner fixed the test. Session resumed after that fix — did not rewrite fetch code, only verified rules and ran gate. | | p5a/02-pdftext | 2026-09-25 | done | 1 | gate: ok | none | Built the child-extraction package: `Extract` (reader.Open with ErrWrongPassword→ErrEncrypted, per-page recover, trailer /Info /Title, rule-3 UTF-16BE/Latin-1 decode), `outline.go` (/Dest and /GoTo /A dests, 10k-item and no-double-visit bounds, never-nil outline), `Runner` (re-exec'd child, env-isolated via empty env, 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code→error mapping, Doc parse plus ErrNoText), and the `sift pdftext` case in main.go. Removed the tools.go module pin; go.mod/go.sum/vendor/modules.txt confirmed unchanged. All 10 given tests pass under -race -count=3 including the deadline-kill test; make gate ok; given files byte-identical; `printf nope | bin/sift pdftext` exits 4. | +| p5a/03-dcache | 2026-09-25 | done | 1 | gate: ok | evict() tie-break | Added the pdf_doc/pdf_page tables, `PDFEntry`/`PDFKey`/`PutPDF`/`GetPDF`/`TouchPDF` (one-tx write deleting old pages first, page-ordered read, 304 touch moves only fetched_at), PDFH/PDFM counters, `Size()` including pdf_doc nbytes. One deviation: extended the eviction union with `pdf_doc` and added `rowid ASC` to the ORDER BY so same-second puts evict in insertion order (deterministic whole-document eviction); without it TestPDFEviction is order-dependent. Left Stats() signature unchanged for task 07. Given test byte-identical. | diff --git a/internal/dcache/dcache.go b/internal/dcache/dcache.go index eb12995..ebff459 100644 --- a/internal/dcache/dcache.go +++ b/internal/dcache/dcache.go @@ -45,6 +45,8 @@ type DB struct { EmbedM uint64 SearchH uint64 SearchM uint64 + PDFH uint64 + PDFM uint64 Evictions uint64 } @@ -92,6 +94,22 @@ func Open(path string, maxBytes int64, log *slog.Logger) (*DB, error) { credits INTEGER NOT NULL, PRIMARY KEY (source, day) ); + CREATE TABLE IF NOT EXISTS pdf_doc ( + k TEXT PRIMARY KEY, + content_hash, + etag, + last_modified, + fetched_at INTEGER, + meta BLOB, + last_hit INTEGER, + nbytes INTEGER + ); + CREATE TABLE IF NOT EXISTS pdf_page ( + k TEXT, + page INTEGER, + text TEXT, + PRIMARY KEY (k, page) + ); `); err != nil { _ = sq.Close() return nil, err @@ -230,11 +248,12 @@ func (d *DB) Size() int64 { if d == nil { return 0 } - var n, m, s sql.NullInt64 + var n, m, s, p sql.NullInt64 _ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM digest`).Scan(&n) _ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM embed`).Scan(&m) _ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM search`).Scan(&s) - return n.Int64 + m.Int64 + s.Int64 + _ = d.sql.QueryRow(`SELECT IFNULL(SUM(nbytes),0) FROM pdf_doc`).Scan(&p) + return n.Int64 + m.Int64 + s.Int64 + p.Int64 } func (d *DB) GetSearch(key string) (payload []byte, ok bool) { @@ -313,18 +332,22 @@ func (d *DB) evict() { var k string err := d.sql.QueryRow(` SELECT k FROM ( - SELECT k, last_hit FROM digest + SELECT k, last_hit, rowid AS rid FROM digest UNION ALL - SELECT k, last_hit FROM embed + SELECT k, last_hit, rowid AS rid FROM embed UNION ALL - SELECT k, last_hit FROM search - ) ORDER BY last_hit ASC LIMIT 1`).Scan(&k) + SELECT k, last_hit, rowid AS rid FROM search + UNION ALL + SELECT k, last_hit, rowid AS rid FROM pdf_doc + ) ORDER BY last_hit ASC, rid ASC LIMIT 1`).Scan(&k) if err != nil { return } _, _ = d.sql.Exec(`DELETE FROM digest WHERE k = ?`, k) _, _ = d.sql.Exec(`DELETE FROM embed WHERE k = ?`, k) _, _ = d.sql.Exec(`DELETE FROM search WHERE k = ?`, k) + _, _ = d.sql.Exec(`DELETE FROM pdf_doc WHERE k = ?`, k) + _, _ = d.sql.Exec(`DELETE FROM pdf_page WHERE k = ?`, k) d.mu.Lock() d.Evictions++ d.mu.Unlock() diff --git a/internal/dcache/pdf.go b/internal/dcache/pdf.go new file mode 100644 index 0000000..6f4af6f --- /dev/null +++ b/internal/dcache/pdf.go @@ -0,0 +1,156 @@ +package dcache + +import ( + "crypto/sha256" + "encoding/hex" + "time" +) + +// PDFEntry is one cached PDF: how to revalidate it, its metadata (the pdftext.Meta JSON, opaque +// here) and the text of each extracted page (Pages[i] is page i+1). +type PDFEntry struct { + ContentHash string + ETag string + LastModified string + FetchedAt time.Time + Meta []byte + Pages []string +} + +// PDFKey is the cache key for a document: a hash of the URL and the caller class, like the +// other keys here (the URL never appears in the clear). +func PDFKey(url, callerClass string) string { + if callerClass == "" { + callerClass = ClassFleet + } + sum := sha256.Sum256([]byte(url + "\x00" + callerClass)) + return hex.EncodeToString(sum[:]) +} + +// PutPDF writes the document and its pages in one transaction, deleting the old pages first so a +// failed replace never leaves a partial document behind, then calls evict. +func (d *DB) PutPDF(key string, e PDFEntry) { + if d == nil { + return + } + nbytes := len(e.Meta) + for _, p := range e.Pages { + nbytes += len(p) + } + tx, err := d.sql.Begin() + if err != nil { + if d.log != nil { + d.log.Warn("pdf cache begin", "err", err) + } + return + } + if _, err := tx.Exec(`DELETE FROM pdf_page WHERE k = ?`, key); err != nil { + _ = tx.Rollback() + if d.log != nil { + d.log.Warn("pdf cache pages del", "err", err) + } + return + } + if _, err := tx.Exec(`INSERT OR REPLACE INTO pdf_doc(k, content_hash, etag, last_modified, fetched_at, meta, last_hit, nbytes) VALUES(?,?,?,?,?,?,?,?)`, + key, e.ContentHash, e.ETag, e.LastModified, e.FetchedAt.Unix(), e.Meta, time.Now().Unix(), nbytes); err != nil { + _ = tx.Rollback() + if d.log != nil { + d.log.Warn("pdf cache doc", "err", err) + } + return + } + for i, p := range e.Pages { + if _, err := tx.Exec(`INSERT INTO pdf_page(k, page, text) VALUES(?,?,?)`, key, i, p); err != nil { + _ = tx.Rollback() + if d.log != nil { + d.log.Warn("pdf cache page put", "err", err) + } + return + } + } + if err := tx.Commit(); err != nil { + if d.log != nil { + d.log.Warn("pdf cache commit", "err", err) + } + return + } + d.evict() +} + +// GetPDF returns the pages in page order (an empty page keeps its place), updates last_hit, and +// counts the hit; a missing document is a miss. +func (d *DB) GetPDF(key string) (PDFEntry, bool) { + if d == nil { + return PDFEntry{}, false + } + var e PDFEntry + var fetchedAt int64 + var meta []byte + err := d.sql.QueryRow(` + SELECT content_hash, etag, last_modified, fetched_at, meta + FROM pdf_doc + WHERE k = ?`, key).Scan(&e.ContentHash, &e.ETag, &e.LastModified, &fetchedAt, &meta) + if err != nil { + d.mu.Lock() + d.PDFM++ + d.mu.Unlock() + return PDFEntry{}, false + } + e.FetchedAt = time.Unix(fetchedAt, 0).UTC() + e.Meta = meta + + rows, err := d.sql.Query(`SELECT page, text FROM pdf_page WHERE k = ? ORDER BY page ASC`, key) + if err != nil { + if d.log != nil { + d.log.Warn("pdf cache pages", "err", err) + } + d.mu.Lock() + d.PDFM++ + d.mu.Unlock() + return PDFEntry{}, false + } + defer rows.Close() + pages := make([]string, 0) + for rows.Next() { + var pg int + var text string + if err := rows.Scan(&pg, &text); err != nil { + if d.log != nil { + d.log.Warn("pdf cache page", "err", err) + } + d.mu.Lock() + d.PDFM++ + d.mu.Unlock() + return PDFEntry{}, false + } + pages = append(pages, text) + } + if err := rows.Err(); err != nil { + if d.log != nil { + d.log.Warn("pdf cache pages", "err", err) + } + d.mu.Lock() + d.PDFM++ + d.mu.Unlock() + return PDFEntry{}, false + } + e.Pages = pages + + _, _ = d.sql.Exec(`UPDATE pdf_doc SET last_hit = ? WHERE k = ?`, time.Now().Unix(), key) + d.mu.Lock() + d.PDFH++ + d.mu.Unlock() + return e, true +} + +// TouchPDF moves only the freshness timestamp after a 304; a miss changes nothing. +func (d *DB) TouchPDF(key string, fetchedAt time.Time) { + if d == nil { + return + } + if _, err := d.sql.Exec(`UPDATE pdf_doc SET fetched_at = ? WHERE k = ?`, fetchedAt.Unix(), key); err != nil { + if d.log != nil { + d.log.Warn("pdf cache touch", "err", err) + } + } +} diff --git a/internal/dcache/pdf_test.go b/internal/dcache/pdf_test.go new file mode 100644 index 0000000..2b71e00 --- /dev/null +++ b/internal/dcache/pdf_test.go @@ -0,0 +1,128 @@ +package dcache_test + +// P5a task 03: the PDF document cache. One entry per (URL, caller class): how to revalidate it, +// its metadata (opaque JSON here) and the text of every extracted page. + +import ( + "path/filepath" + "strings" + "testing" + "time" + + "git.wntrmute.dev/kyle/sift/internal/dcache" +) + +func openDB(t *testing.T, maxBytes int64) *dcache.DB { + t.Helper() + db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), maxBytes, nil) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { _ = db.Close() }) + return db +} + +func entry(pages ...string) dcache.PDFEntry { + return dcache.PDFEntry{ + ContentHash: "sha256:abc", + ETag: `"v1"`, + LastModified: "Fri, 25 Sep 2026 12:00:00 GMT", + FetchedAt: time.Date(2026, 9, 25, 20, 0, 0, 0, time.UTC), + Meta: []byte(`{"pages":3,"title":"T","outline":[]}`), + Pages: pages, + } +} + +func TestPDFRoundTrip(t *testing.T) { + db := openDB(t, 0) + k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet) + if _, ok := db.GetPDF(k); ok { + t.Fatal("hit on an empty cache") + } + want := entry("page one", "page two", "", "page four") // an empty page keeps its place + db.PutPDF(k, want) + got, ok := db.GetPDF(k) + if !ok { + t.Fatal("miss after put") + } + if got.ContentHash != want.ContentHash || got.ETag != want.ETag || got.LastModified != want.LastModified || + !got.FetchedAt.Equal(want.FetchedAt) || string(got.Meta) != string(want.Meta) { + t.Errorf("got %+v, want %+v", got, want) + } + if len(got.Pages) != 4 || got.Pages[0] != "page one" || got.Pages[2] != "" || got.Pages[3] != "page four" { + t.Errorf("pages = %q", got.Pages) + } + // A second put replaces the whole entry, pages included. + db.PutPDF(k, entry("only")) + got, _ = db.GetPDF(k) + if len(got.Pages) != 1 || got.Pages[0] != "only" { + t.Errorf("after replace, pages = %q; want [only]", got.Pages) + } +} + +func TestPDFKeyPartitions(t *testing.T) { + db := openDB(t, 0) + u := "https://example.org/a.pdf" + fleet, restricted := dcache.PDFKey(u, dcache.ClassFleet), dcache.PDFKey(u, dcache.ClassRestricted) + if fleet == restricted { + t.Fatal("the same key for both caller classes") + } + db.PutPDF(restricted, entry("secret page")) + if _, ok := db.GetPDF(fleet); ok { + t.Error("a restricted caller's document is visible to the fleet") + } + if dcache.PDFKey(u, dcache.ClassFleet) != fleet || dcache.PDFKey("https://example.org/b.pdf", dcache.ClassFleet) == fleet { + t.Error("PDFKey must be deterministic and differ per URL") + } + if strings.Contains(fleet, "example.org") { + t.Error("the key must not carry the URL in the clear (it is hashed like the other keys)") + } +} + +func TestPDFTouch(t *testing.T) { + db := openDB(t, 0) + k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet) + db.PutPDF(k, entry("x")) + later := time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC) + db.TouchPDF(k, later) // after a 304: the text is unchanged, only its freshness moves + got, _ := db.GetPDF(k) + if !got.FetchedAt.Equal(later) || len(got.Pages) != 1 { + t.Errorf("after touch: %+v", got) + } + db.TouchPDF(dcache.PDFKey("https://nope", dcache.ClassFleet), later) // a miss is a no-op +} + +// Documents count toward maxBytes and are evicted whole, oldest use first. +func TestPDFEviction(t *testing.T) { + db := openDB(t, 300<<10) // 300 KiB + big := strings.Repeat("z", 50<<10) + var keys []string + for i := 0; i < 10; i++ { // 10 × 2 pages × 50 KiB = 1 MiB of text + k := dcache.PDFKey("https://example.org/"+string(rune('a'+i))+".pdf", dcache.ClassFleet) + keys = append(keys, k) + db.PutPDF(k, entry(big, big)) + } + if db.Size() > 300<<10 { + t.Errorf("size %d after puts, want ≤ maxBytes %d", db.Size(), 300<<10) + } + if _, ok := db.GetPDF(keys[9]); !ok { + t.Error("the newest document was evicted") + } + if _, ok := db.GetPDF(keys[0]); ok { + t.Error("the oldest document survived; eviction did not reach PDF entries") + } + for _, k := range keys { + if e, ok := db.GetPDF(k); ok && len(e.Pages) != 2 { + t.Errorf("a partly evicted document: %d pages", len(e.Pages)) + } + } +} + +func TestPDFNilDB(t *testing.T) { + var db *dcache.DB + db.PutPDF("k", entry("x")) + db.TouchPDF("k", time.Now()) + if _, ok := db.GetPDF("k"); ok { + t.Error("nil DB returned a hit") + } +}