P5a plan: PDF documents — seven Ornith tasks with acceptance tests first

docs/plans/p5a: 01 kind+fetch limits, 02 pdftext child + runner, 03 dcache PDF tables,
04 pdfpool, 05 pdfdoc render (map/pages/grep), 06 api, 07 digest citations, metrics, CLI,
skill, unit, README. Given tests compile against a panic-only skeleton and fail there for the
intended reasons; PDFs for tests are built by internal/pdftest (given), plus two qpdf-encrypted
fixtures.

Owner changes for the run: vendor go-pdfkit/extract v0.3.0 + reader v0.6.0 (pinned via
tools.go until task 02); AGENTS.md, tools/run-plan.sh (enforces a Co-Authored-By trailer for
the model), docs/implementer-log.md, `make gate`; gofmt (whitespace) on clean.go and guard;
TestUnsupportedType uses a zip now that PDFs are supported; requestDeadline 90→210 s and
writeTimeout 120→240 s for the PDF path's worst case.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-25 20:47:38 -07:00
co-authored by Claude Opus 5.5
parent 3f2b78d4aa
commit cfde35397a
115 changed files with 26422 additions and 43 deletions
@@ -0,0 +1,128 @@
package dcache_test
// P5a task 03: the PDF document cache. One entry per (URL, caller class): how to revalidate it,
// its metadata (opaque JSON here) and the text of every extracted page.
import (
"path/filepath"
"strings"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/dcache"
)
func openDB(t *testing.T, maxBytes int64) *dcache.DB {
t.Helper()
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), maxBytes, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
return db
}
func entry(pages ...string) dcache.PDFEntry {
return dcache.PDFEntry{
ContentHash: "sha256:abc",
ETag: `"v1"`,
LastModified: "Fri, 25 Sep 2026 12:00:00 GMT",
FetchedAt: time.Date(2026, 9, 25, 20, 0, 0, 0, time.UTC),
Meta: []byte(`{"pages":3,"title":"T","outline":[]}`),
Pages: pages,
}
}
func TestPDFRoundTrip(t *testing.T) {
db := openDB(t, 0)
k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet)
if _, ok := db.GetPDF(k); ok {
t.Fatal("hit on an empty cache")
}
want := entry("page one", "page two", "", "page four") // an empty page keeps its place
db.PutPDF(k, want)
got, ok := db.GetPDF(k)
if !ok {
t.Fatal("miss after put")
}
if got.ContentHash != want.ContentHash || got.ETag != want.ETag || got.LastModified != want.LastModified ||
!got.FetchedAt.Equal(want.FetchedAt) || string(got.Meta) != string(want.Meta) {
t.Errorf("got %+v, want %+v", got, want)
}
if len(got.Pages) != 4 || got.Pages[0] != "page one" || got.Pages[2] != "" || got.Pages[3] != "page four" {
t.Errorf("pages = %q", got.Pages)
}
// A second put replaces the whole entry, pages included.
db.PutPDF(k, entry("only"))
got, _ = db.GetPDF(k)
if len(got.Pages) != 1 || got.Pages[0] != "only" {
t.Errorf("after replace, pages = %q; want [only]", got.Pages)
}
}
func TestPDFKeyPartitions(t *testing.T) {
db := openDB(t, 0)
u := "https://example.org/a.pdf"
fleet, restricted := dcache.PDFKey(u, dcache.ClassFleet), dcache.PDFKey(u, dcache.ClassRestricted)
if fleet == restricted {
t.Fatal("the same key for both caller classes")
}
db.PutPDF(restricted, entry("secret page"))
if _, ok := db.GetPDF(fleet); ok {
t.Error("a restricted caller's document is visible to the fleet")
}
if dcache.PDFKey(u, dcache.ClassFleet) != fleet || dcache.PDFKey("https://example.org/b.pdf", dcache.ClassFleet) == fleet {
t.Error("PDFKey must be deterministic and differ per URL")
}
if strings.Contains(fleet, "example.org") {
t.Error("the key must not carry the URL in the clear (it is hashed like the other keys)")
}
}
func TestPDFTouch(t *testing.T) {
db := openDB(t, 0)
k := dcache.PDFKey("https://example.org/a.pdf", dcache.ClassFleet)
db.PutPDF(k, entry("x"))
later := time.Date(2026, 9, 26, 20, 0, 0, 0, time.UTC)
db.TouchPDF(k, later) // after a 304: the text is unchanged, only its freshness moves
got, _ := db.GetPDF(k)
if !got.FetchedAt.Equal(later) || len(got.Pages) != 1 {
t.Errorf("after touch: %+v", got)
}
db.TouchPDF(dcache.PDFKey("https://nope", dcache.ClassFleet), later) // a miss is a no-op
}
// Documents count toward maxBytes and are evicted whole, oldest use first.
func TestPDFEviction(t *testing.T) {
db := openDB(t, 300<<10) // 300 KiB
big := strings.Repeat("z", 50<<10)
var keys []string
for i := 0; i < 10; i++ { // 10 × 2 pages × 50 KiB = 1 MiB of text
k := dcache.PDFKey("https://example.org/"+string(rune('a'+i))+".pdf", dcache.ClassFleet)
keys = append(keys, k)
db.PutPDF(k, entry(big, big))
}
if db.Size() > 300<<10 {
t.Errorf("size %d after puts, want ≤ maxBytes %d", db.Size(), 300<<10)
}
if _, ok := db.GetPDF(keys[9]); !ok {
t.Error("the newest document was evicted")
}
if _, ok := db.GetPDF(keys[0]); ok {
t.Error("the oldest document survived; eviction did not reach PDF entries")
}
for _, k := range keys {
if e, ok := db.GetPDF(k); ok && len(e.Pages) != 2 {
t.Errorf("a partly evicted document: %d pages", len(e.Pages))
}
}
}
func TestPDFNilDB(t *testing.T) {
var db *dcache.DB
db.PutPDF("k", entry("x"))
db.TouchPDF("k", time.Now())
if _, ok := db.GetPDF("k"); ok {
t.Error("nil DB returned a hit")
}
}