P5a plan: PDF documents — seven Ornith tasks with acceptance tests first

docs/plans/p5a: 01 kind+fetch limits, 02 pdftext child + runner, 03 dcache PDF tables, 04 pdfpool, 05 pdfdoc render (map/pages/grep), 06 api, 07 digest citations, metrics, CLI, skill, unit, README. Given tests compile against a panic-only skeleton and fail there for the intended reasons; PDFs for tests are built by internal/pdftest (given), plus two qpdf-encrypted fixtures. Owner changes for the run: vendor go-pdfkit/extract v0.3.0 + reader v0.6.0 (pinned via tools.go until task 02); AGENTS.md, tools/run-plan.sh (enforces a Co-Authored-By trailer for the model), docs/implementer-log.md, `make gate`; gofmt (whitespace) on clean.go and guard; TestUnsupportedType uses a zip now that PDFs are supported; requestDeadline 90→210 s and writeTimeout 120→240 s for the PDF path's worst case. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-09-25 20:47:38 -07:00
parent 3f2b78d4aa
commit cfde35397a
115 changed files with 26422 additions and 43 deletions
@@ -0,0 +1,55 @@
+package clean_test
+
+// P5a task 01: Kind decides which cleaner a fetched body gets, now including "pdf".
+
+import (
+	"strings"
+	"testing"
+
+	"git.wntrmute.dev/kyle/sift/internal/clean"
+)
+
+func TestKind(t *testing.T) {
+	pdf := []byte("%PDF-1.7\n%\xe2\xe3\xcf\xd3\n1 0 obj << >> endobj\n")
+	junkThenPDF := append([]byte(strings.Repeat(" ", 900)), pdf...) // the spec allows leading junk
+	lateMagic := append([]byte(strings.Repeat("x", 2000)), pdf...)  // past the first 1024 bytes
+	binary := []byte{0x50, 0x4b, 0x03, 0x04, 0x00, 0x00}            // a zip
+	for _, tc := range []struct {
+		name, ct string
+		body     []byte
+		want     string
+	}{
+		{"pdf by type", "application/pdf", pdf, "pdf"},
+		{"pdf by type with params", "Application/PDF; qs=0.9", pdf, "pdf"},
+		{"x-pdf", "application/x-pdf", pdf, "pdf"},
+		{"pdf type, body not checked", "application/pdf", []byte("oops"), "pdf"},
+		{"octet-stream sniffed as pdf", "application/octet-stream", pdf, "pdf"},
+		{"binary/octet-stream sniffed as pdf", "binary/octet-stream", pdf, "pdf"},
+		{"no type sniffed as pdf", "", pdf, "pdf"},
+		{"leading junk within 1024 bytes", "application/octet-stream", junkThenPDF, "pdf"},
+		{"magic too late", "application/octet-stream", lateMagic, "other"},
+		{"octet-stream that is not a pdf", "application/octet-stream", binary, "other"},
+		{"html stays html even if it mentions %PDF-", "text/html; charset=utf-8", []byte("<html>%PDF-1.4</html>"), "html"},
+		{"json", "application/json", []byte(`{"a":1}`), "json"},
+		{"problem+json", "application/problem+json", []byte(`{}`), "json"},
+		{"text", "text/plain", []byte("hello"), "text"},
+		{"no type, html", "", []byte("<!doctype html><html></html>"), "html"},
+		{"no type, json", "", []byte(`[1,2]`), "json"},
+		{"no type, text", "", []byte("just words"), "text"},
+		{"zip", "application/zip", binary, "other"},
+		{"image", "image/png", []byte{0x89, 'P', 'N', 'G'}, "other"},
+	} {
+		if got := clean.Kind(tc.ct, tc.body); got != tc.want {
+			t.Errorf("%s: Kind(%q, …) = %q, want %q", tc.name, tc.ct, got, tc.want)
+		}
+	}
+}
+
+// MediaKind keeps answering for callers that have no body: the pdf types are "pdf" there too.
+func TestMediaKindKnowsPDF(t *testing.T) {
+	for ct, want := range map[string]string{"application/pdf": "pdf", "application/x-pdf": "pdf", "text/html": "html", "": "sniff"} {
+		if got := clean.MediaKind(ct); got != want {
+			t.Errorf("MediaKind(%q) = %q, want %q", ct, got, want)
+		}
+	}
+}