diff --git a/cmd/sift/main.go b/cmd/sift/main.go index 8845557..91d0b1f 100644 --- a/cmd/sift/main.go +++ b/cmd/sift/main.go @@ -3,6 +3,8 @@ package main import ( "fmt" "os" + + "git.wntrmute.dev/kyle/sift/internal/pdftext" ) // version is set by ldflags: -X main.version=… @@ -24,6 +26,8 @@ func main() { searchMain() case "keys": keysMain() + case "pdftext": + os.Exit(pdftext.ChildMain(os.Args[1:], os.Stdin, os.Stdout)) case "version", "-version", "--version": fmt.Println(version) case "-h", "--help", "help": diff --git a/docs/implementer-log.md b/docs/implementer-log.md index 0fb6c7c..88cbcc6 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -5,3 +5,4 @@ One row per task session that ends in a commit. Newest last. | Task | Date | Status | Sessions | Gate | Deviations | Summary | |---|---|---|---|---|---|---| | p5a/01-kind-fetch | 2026-09-25 | done | resumed | gate: ok | none | Implemented Kind/`application/pdf` recognition, PDF body cap (100 MB) and deadline (120 s), and FetchIf with If-None-Match/If-Modified-Since via shared doFetch. Previous session could not pass TestFetchIf only because the given test held its mutex across the handler (deadlock); owner fixed the test. Session resumed after that fix — did not rewrite fetch code, only verified rules and ran gate. | +| p5a/02-pdftext | 2026-09-25 | done | 1 | gate: ok | none | Built the child-extraction package: `Extract` (reader.Open with ErrWrongPassword→ErrEncrypted, per-page recover, trailer /Info /Title, rule-3 UTF-16BE/Latin-1 decode), `outline.go` (/Dest and /GoTo /A dests, 10k-item and no-double-visit bounds, never-nil outline), `Runner` (re-exec'd child, env-isolated via empty env, 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code→error mapping, Doc parse plus ErrNoText), and the `sift pdftext` case in main.go. Removed the tools.go module pin; go.mod/go.sum/vendor/modules.txt confirmed unchanged. All 10 given tests pass under -race -count=3 including the deadline-kill test; make gate ok; given files byte-identical; `printf nope | bin/sift pdftext` exits 4. | diff --git a/internal/pdftest/pdftest.go b/internal/pdftest/pdftest.go new file mode 100644 index 0000000..62508de --- /dev/null +++ b/internal/pdftest/pdftest.go @@ -0,0 +1,133 @@ +// Package pdftest builds small, deterministic PDFs for tests: text pages in Helvetica, an optional +// outline (bookmarks), and image-only ("scanned") pages. It exists so the P5a tests need no binary +// fixtures from elsewhere. It is test support: nothing outside _test.go files imports it. +package pdftest + +import ( + "bytes" + "fmt" + "strings" +) + +// Outline is one bookmark: a title that points at a 1-based page, nested at Level (1 = top). +// Build supports one level; Level is kept so tests can state what they expect back. +type Outline struct { + Title string + Page int + Level int +} + +// Doc describes a PDF to build. Pages holds each page's text; a page's lines are split on "\n" +// and set one per line (long lines are not wrapped: callers keep them under ~90 characters). +// Scanned pages (1-based) carry a small image and no text at all. +type Doc struct { + Title string + Pages []string + Outline []Outline + Scanned []int +} + +// Build returns the PDF bytes for d. It panics on a Doc it cannot express (an outline entry +// pointing past the last page), since that is a bug in the test that built it. +func Build(d Doc) []byte { + var b bytes.Buffer + var offsets []int + obj := func(body string) int { + offsets = append(offsets, b.Len()) + n := len(offsets) + fmt.Fprintf(&b, "%d 0 obj\n%s\nendobj\n", n, body) + return n + } + b.WriteString("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n") + + // Object numbers are fixed by position: 1 catalog, 2 pages, 3 font, 4 image, 5 info, then + // two per page (page, content), then the outline. + nPages := len(d.Pages) + firstPage := 6 + pageObj := func(i int) int { return firstPage + 2*i } + outlineRoot := firstPage + 2*nPages + scanned := map[int]bool{} + for _, p := range d.Scanned { + scanned[p] = true + } + + catalog := "<< /Type /Catalog /Pages 2 0 R" + if len(d.Outline) > 0 { + catalog += fmt.Sprintf(" /Outlines %d 0 R /PageMode /UseOutlines", outlineRoot) + } + obj(catalog + " >>") + var kids []string + for i := range d.Pages { + kids = append(kids, fmt.Sprintf("%d 0 R", pageObj(i))) + } + obj(fmt.Sprintf("<< /Type /Pages /Kids [%s] /Count %d >>", strings.Join(kids, " "), nPages)) + obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>") + img := []byte{0x00, 0x80, 0x80, 0x00} // 2x2 grey pixels + obj(fmt.Sprintf("<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length %d >>\nstream\n%s\nendstream", len(img), img)) + obj(fmt.Sprintf("<< /Title (%s) /Producer (sift pdftest) >>", esc(d.Title))) + + for i, text := range d.Pages { + var content string + if scanned[i+1] { + content = "q 400 0 0 500 100 150 cm /Im1 Do Q" + } else { + var c strings.Builder + c.WriteString("BT /F1 11 Tf 13 TL 60 760 Td\n") + for _, line := range strings.Split(text, "\n") { + fmt.Fprintf(&c, "(%s) Tj T*\n", esc(line)) + } + c.WriteString("ET") + content = c.String() + } + obj(fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents %d 0 R >>", pageObj(i)+1)) + obj(fmt.Sprintf("<< /Length %d >>\nstream\n%s\nendstream", len(content), content)) + } + + if len(d.Outline) > 0 { + first := outlineRoot + 1 + last := outlineRoot + len(d.Outline) + obj(fmt.Sprintf("<< /Type /Outlines /First %d 0 R /Last %d 0 R /Count %d >>", first, last, len(d.Outline))) + for i, o := range d.Outline { + if o.Page < 1 || o.Page > nPages { + panic(fmt.Sprintf("pdftest: outline %q points at page %d of %d", o.Title, o.Page, nPages)) + } + n := first + i + item := fmt.Sprintf("<< /Title (%s) /Parent %d 0 R /Dest [%d 0 R /XYZ null null null]", esc(o.Title), outlineRoot, pageObj(o.Page-1)) + if i > 0 { + item += fmt.Sprintf(" /Prev %d 0 R", n-1) + } + if n < last { + item += fmt.Sprintf(" /Next %d 0 R", n+1) + } + obj(item + " >>") + } + } + + xref := b.Len() + fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1) + for _, off := range offsets { + fmt.Fprintf(&b, "%010d 00000 n \n", off) + } + fmt.Fprintf(&b, "trailer\n<< /Size %d /Root 1 0 R /Info 5 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref) + return b.Bytes() +} + +// Lorem returns n lines of distinct, searchable filler: "p l …", so a test can tell +// which page a line came from. +func Lorem(page, n int) string { + words := []string{"alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"} + var lines []string + for i := 1; i <= n; i++ { + var w []string + for j := 0; j < 10; j++ { + w = append(w, words[(page*7+i*3+j)%len(words)]) + } + lines = append(lines, fmt.Sprintf("p%d l%d %s", page, i, strings.Join(w, " "))) + } + return strings.Join(lines, "\n") +} + +func esc(s string) string { + r := strings.NewReplacer(`\`, `\\`, `(`, `\(`, `)`, `\)`) + return r.Replace(s) +} diff --git a/internal/pdftext/outline.go b/internal/pdftext/outline.go new file mode 100644 index 0000000..fd08bcc --- /dev/null +++ b/internal/pdftext/outline.go @@ -0,0 +1,142 @@ +package pdftext + +import ( + "github.com/go-pdfkit/reader" +) + +// maxOutlineItems bounds a walk over a malformed outline that claims more +// entries than the file could hold. +const maxOutlineItems = 10000 + +// walkOutline returns the document's outline as a depth-first, pre-order list: +// each item before its own children. The result is never nil. +func walkOutline(doc *reader.Document) []OutlineItem { + out := []OutlineItem{} + trailer := doc.Trailer() + if trailer == nil { + return out + } + cat, err := doc.Resolve(trailer.Get("Root")) + if err != nil || cat == nil { + return out + } + catDict, ok := reader.ToDict(cat) + if !ok { + return out + } + outlines, err := doc.Resolve(catDict.Get("Outlines")) + if err != nil || outlines == nil { + return out + } + outlinesDict, ok := reader.ToDict(outlines) + if !ok { + return out + } + first, ok := toRef(outlinesDict.Get("First")) + if !ok { + return out + } + seen := map[reader.Ref]bool{} + walkChildren(doc, first, 1, seen, &out) + return out +} + +// walkChildren walks the /Next chain from ref at the given level, emitting each +// resolvable item and recursing into its /First children (level + 1). It stops +// after maxOutlineItems items and never visits an object twice. +func walkChildren(doc *reader.Document, ref reader.Ref, level int, seen map[reader.Ref]bool, out *[]OutlineItem) { + cur := ref + for cur != (reader.Ref{}) { + if seen[cur] { + break + } + seen[cur] = true + + obj, err := doc.Resolve(cur) + if err != nil || obj == nil { + break + } + d, ok := reader.ToDict(obj) + if !ok { + break + } + + if page, ok := destPage(doc, d); ok { + *out = append(*out, OutlineItem{Title: textString(d.Get("Title")), Page: page, Level: level}) + if len(*out) >= maxOutlineItems { + return + } + } + + if child, ok := toRef(d.Get("First")); ok { + walkChildren(doc, child, level+1, seen, out) + if len(*out) >= maxOutlineItems { + return + } + } + + next, ok := toRef(d.Get("Next")) + if !ok { + break + } + cur = next + } +} + +// destPage resolves an item's destination to a 1-based page number. It reports +// ok only when the destination is an array whose first element is a page +// reference that maps to a real page; a named or otherwise unresolvable +// destination is not ok, so the caller skips the item. +func destPage(doc *reader.Document, d reader.Dict) (int, bool) { + if ref, ok := destRef(doc, d); ok { + if page, ok := pageOf(doc, ref); ok { + return page, true + } + } + return 0, false +} + +// destRef finds the destination page reference, from /Dest or from a /GoTo +// action's /D. +func destRef(doc *reader.Document, d reader.Dict) (reader.Ref, bool) { + if o := d.Get("Dest"); o != nil { + if arr, ok := reader.ToArray(o); ok && len(arr) > 0 { + if ref, ok := arr[0].(reader.Ref); ok { + return ref, true + } + } + } + if o := d.Get("A"); o != nil { + if a, ok := reader.ToDict(o); ok { + if s, ok := reader.ToName(a.Get("S")); ok && s == "GoTo" { + if d2 := a.Get("D"); d2 != nil { + if arr, ok := reader.ToArray(d2); ok && len(arr) > 0 { + if ref, ok := arr[0].(reader.Ref); ok { + return ref, true + } + } + } + } + } + } + return reader.Ref{}, false +} + +// pageOf maps a reference to its 1-based page by comparing it with each page's +// reference. +func pageOf(doc *reader.Document, ref reader.Ref) (int, bool) { + for i := 1; i <= doc.PageCount(); i++ { + pr, ok := doc.PageRef(i) + if ok && pr == ref { + return i, true + } + } + return 0, false +} + +// toRef reports the reference a dictionary entry names, when it is a direct +// reference rather than an already-resolved object. +func toRef(o reader.Object) (reader.Ref, bool) { + ref, ok := o.(reader.Ref) + return ref, ok +} diff --git a/internal/pdftext/pdftext.go b/internal/pdftext/pdftext.go new file mode 100644 index 0000000..e109b1a --- /dev/null +++ b/internal/pdftext/pdftext.go @@ -0,0 +1,216 @@ +// Package pdftext extracts text, title and outline from a PDF. Extraction runs +// in a child process (see Runner) so a page that hangs or panics cannot take +// the parent down; Extract is the child's own work, usable directly in tests. +package pdftext + +import ( + "encoding/json" + "errors" + "flag" + "fmt" + "io" + "os" + "runtime/debug" + "strings" + "unicode/utf16" + + "github.com/go-pdfkit/extract" + "github.com/go-pdfkit/reader" +) + +// The errors Extract and Runner can return. +var ( + ErrEncrypted = errors.New("encrypted PDF (password required)") + ErrUnreadable = errors.New("unreadable PDF") + ErrNoText = errors.New("no text layer (scanned PDF?)") + ErrTimeout = errors.New("pdf extraction timed out") + ErrTooMuchText = errors.New("pdf text over the output limit") +) + +// OutlineItem is one bookmark: its title, the 1-based page it points at, and +// its nesting level (1 = top level). +type OutlineItem struct { + Title string `json:"title"` + Page int `json:"page"` // 1-based + Level int `json:"level"` // 1 = top level +} + +// Meta is the document-level summary written as the first output line. +type Meta struct { + Pages int `json:"pages"` // pages in the document (not only those extracted) + Title string `json:"title"` // /Info /Title, "" when absent + Outline []OutlineItem `json:"outline"` // never nil: [] when the document has none +} + +// Doc is an extracted document: Text[i] is page i+1's text, for the pages +// extracted (len(Text) ≤ Pages). Failed lists pages whose extraction failed +// (their Text is ""). +type Doc struct { + Meta + Text []string + Failed []int +} + +// metaLine is the first output line: the document summary. +type metaLine struct { + Meta Meta `json:"meta"` +} + +// pageOK and pageErr are the per-page lines. A page that extracts yields pageOK +// even when its text is empty (an image-only page); one that fails yields +// pageErr and extraction continues. +type pageOK struct { + Page int `json:"page"` + Text string `json:"text"` +} + +type pageErr struct { + Page int `json:"page"` + Error string `json:"error"` +} + +// Extract writes one {"meta":{…}} line, then one {"page":n,"text":"…"} line per +// page from 1 up to min(Pages, maxPages); a page that fails yields +// {"page":n,"error":"…"} and extraction goes on. It returns ErrEncrypted when +// the document needs a user password, ErrUnreadable when it cannot be opened or +// has no pages, and never panics. +func Extract(pdf []byte, maxPages int, w io.Writer) error { + doc, err := reader.Open(pdf) + if err != nil { + if errors.Is(err, reader.ErrWrongPassword) { + return ErrEncrypted + } + return ErrUnreadable + } + if doc.PageCount() == 0 { + return ErrUnreadable + } + + meta := Meta{ + Pages: doc.PageCount(), + Title: infoTitle(doc), + Outline: walkOutline(doc), + } + if meta.Outline == nil { + meta.Outline = []OutlineItem{} + } + if err := writeJSONLine(w, metaLine{Meta: meta}); err != nil { + return err + } + + n := doc.PageCount() + if maxPages > 0 && n > maxPages { + n = maxPages + } + for i := 1; i <= n; i++ { + func() { + defer func() { + if r := recover(); r != nil { + _ = writeJSONLine(w, pageErr{Page: i, Error: fmt.Sprintf("panic: %v", r)}) + } + }() + text, err := extract.Text(doc, i) + if err != nil { + _ = writeJSONLine(w, pageErr{Page: i, Error: err.Error()}) + return + } + _ = writeJSONLine(w, pageOK{Page: i, Text: text}) + }() + } + return nil +} + +// writeJSONLine marshals v and writes it as one JSON line. +func writeJSONLine(w io.Writer, v any) error { + b, err := json.Marshal(v) + if err != nil { + return err + } + if _, err := w.Write(b); err != nil { + return err + } + _, err = w.Write([]byte{'\n'}) + return err +} + +// infoTitle returns the document title from the trailer's /Info /Title, decoded +// as a PDF text string (rule 3), "" when absent. +func infoTitle(doc *reader.Document) string { + trailer := doc.Trailer() + if trailer == nil { + return "" + } + info, err := doc.Resolve(trailer.Get("Info")) + if err != nil || info == nil { + return "" + } + d, ok := reader.ToDict(info) + if !ok { + return "" + } + return textString(d.Get("Title")) +} + +// textString decodes a reader.String (raw bytes) as a PDF text string per rule 3. +func textString(o reader.Object) string { + s, ok := reader.ToString(o) + if !ok { + return "" + } + return decodePDFString(s) +} + +// decodePDFString follows rule 3: bytes starting FE FF are UTF-16BE (the rest is +// decoded as such); otherwise each byte is treated as a Latin-1 code point. +func decodePDFString(b []byte) string { + if len(b) >= 2 && b[0] == 0xFE && b[1] == 0xFF { + return strings.TrimSpace(string(utf16.Decode(utf16BE(b[2:])))) + } + return strings.TrimSpace(string(b)) +} + +// utf16BE packs big-endian byte pairs into uint16s for utf16.Decode. +func utf16BE(b []byte) []uint16 { + n := len(b) / 2 + u := make([]uint16, n) + for i := 0; i < 2*n; i += 2 { + u[i/2] = uint16(b[i])<<8 | uint16(b[i+1]) + } + return u +} + +// ChildMain is `sift pdftext`: flags (-max-pages N, default 2000), the PDF on +// stdin (read with a 200 MiB cap), Extract to stdout. Exit codes: 0 ok, 3 +// ErrEncrypted, 4 ErrUnreadable, 1 anything else. It runs with a soft memory +// limit, reads no environment variable, and opens no network connection. +func ChildMain(args []string, stdin io.Reader, stdout io.Writer) int { + debug.SetMemoryLimit(768 << 20) + + fs := flag.NewFlagSet("pdftext", flag.ContinueOnError) + maxPages := fs.Int("max-pages", 2000, "maximum number of pages to extract") + if err := fs.Parse(args); err != nil { + return 1 + } + + pdf, err := io.ReadAll(io.LimitReader(stdin, 200<<20)) + if err != nil { + fmt.Fprintln(os.Stderr, "pdftext: read input:", err) + return 4 + } + if len(pdf) == 0 { + fmt.Fprintln(os.Stderr, "pdftext: empty input") + return 4 + } + + if err := Extract(pdf, *maxPages, stdout); err != nil { + switch { + case errors.Is(err, ErrEncrypted): + return 3 + case errors.Is(err, ErrUnreadable): + return 4 + default: + return 1 + } + } + return 0 +} diff --git a/internal/pdftext/pdftext_test.go b/internal/pdftext/pdftext_test.go new file mode 100644 index 0000000..aaeb80f --- /dev/null +++ b/internal/pdftext/pdftext_test.go @@ -0,0 +1,252 @@ +package pdftext_test + +// P5a task 02: Extract (the child's work), ChildMain (the `sift pdftext` entry point) and Runner +// (the parent: exec, deadline, kill, parse). +// +// The Runner tests run the test binary itself as the child: TestMain turns into the child when +// SIFT_PDFTEXT_CHILD=1 is set, exactly as `sift pdftext` would. The two other variables are test +// hooks that live only here: SIFT_PDFTEXT_SLEEP delays the child (for the deadline test) and +// SIFT_PDFTEXT_PIDFILE records its pid (to prove it is gone afterwards). + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "testing" + "time" + + "git.wntrmute.dev/kyle/sift/internal/pdftest" + "git.wntrmute.dev/kyle/sift/internal/pdftext" +) + +func TestMain(m *testing.M) { + if os.Getenv("SIFT_PDFTEXT_CHILD") == "1" { + if p := os.Getenv("SIFT_PDFTEXT_PIDFILE"); p != "" { + _ = os.WriteFile(p, []byte(strconv.Itoa(os.Getpid())), 0o600) + } + if d, err := time.ParseDuration(os.Getenv("SIFT_PDFTEXT_SLEEP")); err == nil { + time.Sleep(d) + } + os.Exit(pdftext.ChildMain(os.Args[1:], os.Stdin, os.Stdout)) + } + os.Exit(m.Run()) +} + +func three() []byte { + return pdftest.Build(pdftest.Doc{ + Title: "Three Pages", + Pages: []string{ + "Chapter one begins here.\nSecond line (with parens).", + "The quick brown fox is on page two.", + pdftest.Lorem(3, 5), + }, + Outline: []pdftest.Outline{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "3 Filler", Page: 3, Level: 1}}, + }) +} + +type line struct { + Meta *pdftext.Meta `json:"meta"` + Page int `json:"page"` + Text *string `json:"text"` + Error string `json:"error"` +} + +func lines(t *testing.T, out []byte) []line { + t.Helper() + var ls []line + sc := bufio.NewScanner(bytes.NewReader(out)) + sc.Buffer(make([]byte, 1<<20), 64<<20) + for sc.Scan() { + var l line + if err := json.Unmarshal(sc.Bytes(), &l); err != nil { + t.Fatalf("output line is not JSON: %q (%v)", sc.Text(), err) + } + ls = append(ls, l) + } + return ls +} + +func TestExtractWritesMetaThenPages(t *testing.T) { + var out bytes.Buffer + if err := pdftext.Extract(three(), 2000, &out); err != nil { + t.Fatal(err) + } + ls := lines(t, out.Bytes()) + if len(ls) != 4 || ls[0].Meta == nil { + t.Fatalf("got %d lines, first meta=%v; want a meta line then 3 page lines:\n%s", len(ls), ls[0].Meta, out.String()) + } + m := ls[0].Meta + if m.Pages != 3 || m.Title != "Three Pages" { + t.Errorf("meta = %+v, want 3 pages titled Three Pages", *m) + } + want := []pdftext.OutlineItem{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "3 Filler", Page: 3, Level: 1}} + if len(m.Outline) != 3 { + t.Fatalf("outline = %+v, want %+v", m.Outline, want) + } + for i := range want { + if m.Outline[i] != want[i] { + t.Errorf("outline[%d] = %+v, want %+v", i, m.Outline[i], want[i]) + } + } + for i, l := range ls[1:] { + if l.Page != i+1 || l.Text == nil || l.Error != "" { + t.Errorf("line %d = %+v, want page %d with text", i+1, l, i+1) + } + } + if !strings.Contains(*ls[1].Text, "Chapter one begins here.") || !strings.Contains(*ls[2].Text, "quick brown fox") || !strings.Contains(*ls[3].Text, "p3 l5") { + t.Errorf("page texts wrong: %q / %q / %q", *ls[1].Text, *ls[2].Text, *ls[3].Text) + } +} + +func TestExtractPageCap(t *testing.T) { + var out bytes.Buffer + if err := pdftext.Extract(three(), 2, &out); err != nil { + t.Fatal(err) + } + ls := lines(t, out.Bytes()) + if len(ls) != 3 || ls[0].Meta.Pages != 3 || ls[2].Page != 2 { + t.Errorf("maxPages 2: %d lines, meta %+v; want meta (pages 3) and pages 1-2", len(ls), ls[0].Meta) + } +} + +func TestExtractNoOutline(t *testing.T) { + var out bytes.Buffer + if err := pdftext.Extract(pdftest.Build(pdftest.Doc{Pages: []string{"a", "b"}}), 10, &out); err != nil { + t.Fatal(err) + } + ls := lines(t, out.Bytes()) + if ls[0].Meta.Outline == nil || len(ls[0].Meta.Outline) != 0 { + t.Errorf("outline = %#v, want an empty non-nil slice (JSON [])", ls[0].Meta.Outline) + } + if !strings.Contains(out.String(), `"outline":[]`) { + t.Errorf("meta line must carry \"outline\":[]: %s", out.String()) + } +} + +func TestExtractErrors(t *testing.T) { + enc, err := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf")) + if err != nil { + t.Fatal(err) + } + if err := pdftext.Extract(enc, 10, &bytes.Buffer{}); !errors.Is(err, pdftext.ErrEncrypted) { + t.Errorf("user-password pdf: %v, want ErrEncrypted", err) + } + owner, err := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-owner.pdf")) + if err != nil { + t.Fatal(err) + } + var out bytes.Buffer + if err := pdftext.Extract(owner, 10, &out); err != nil || !strings.Contains(out.String(), "quick brown fox") { + t.Errorf("owner-password-only pdf: err %v; want its text", err) + } + if err := pdftext.Extract([]byte("not a pdf at all"), 10, &bytes.Buffer{}); !errors.Is(err, pdftext.ErrUnreadable) { + t.Errorf("garbage: %v, want ErrUnreadable", err) + } + // A truncated file either yields what it can or ErrUnreadable; it never panics. + b := three() + _ = pdftext.Extract(b[:len(b)*6/10], 10, &bytes.Buffer{}) +} + +func TestChildMainExitCodes(t *testing.T) { + enc, _ := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf")) + for _, tc := range []struct { + name string + in []byte + code int + }{ + {"ok", three(), 0}, + {"encrypted", enc, 3}, + {"unreadable", []byte("nope"), 4}, + } { + var out bytes.Buffer + if code := pdftext.ChildMain([]string{"-max-pages", "5"}, bytes.NewReader(tc.in), &out); code != tc.code { + t.Errorf("%s: exit %d, want %d", tc.name, code, tc.code) + } + } +} + +func runner(env ...string) *pdftext.Runner { + return &pdftext.Runner{Exe: os.Args[0], Env: append([]string{"SIFT_PDFTEXT_CHILD=1"}, env...), Deadline: 20 * time.Second} +} + +func TestRunnerRun(t *testing.T) { + doc, err := runner().Run(context.Background(), three()) + if err != nil { + t.Fatal(err) + } + if doc.Pages != 3 || doc.Title != "Three Pages" || len(doc.Outline) != 3 || len(doc.Text) != 3 { + t.Fatalf("doc = %+v", doc) + } + if !strings.Contains(doc.Text[1], "quick brown fox") { + t.Errorf("page 2 = %q", doc.Text[1]) + } +} + +func TestRunnerErrors(t *testing.T) { + enc, _ := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf")) + if _, err := runner().Run(context.Background(), enc); !errors.Is(err, pdftext.ErrEncrypted) { + t.Errorf("encrypted: %v, want ErrEncrypted", err) + } + if _, err := runner().Run(context.Background(), []byte("nope")); !errors.Is(err, pdftext.ErrUnreadable) { + t.Errorf("garbage: %v, want ErrUnreadable", err) + } + scanned := pdftest.Build(pdftest.Doc{Pages: []string{"", "", ""}, Scanned: []int{1, 2, 3}}) + if _, err := runner().Run(context.Background(), scanned); !errors.Is(err, pdftext.ErrNoText) { + t.Errorf("scanned: %v, want ErrNoText", err) + } + r := runner() + r.MaxText = 100 // bytes of child output + if _, err := r.Run(context.Background(), three()); !errors.Is(err, pdftext.ErrTooMuchText) { + t.Errorf("output over MaxText: %v, want ErrTooMuchText", err) + } +} + +// The child is killed at the deadline and reaped: no process is left behind. +func TestRunnerDeadlineKillsTheChild(t *testing.T) { + pidfile := filepath.Join(t.TempDir(), "pid") + r := runner("SIFT_PDFTEXT_SLEEP=30s", "SIFT_PDFTEXT_PIDFILE="+pidfile) + r.Deadline = 300 * time.Millisecond + start := time.Now() + _, err := r.Run(context.Background(), three()) + if !errors.Is(err, pdftext.ErrTimeout) { + t.Fatalf("err %v, want ErrTimeout", err) + } + if d := time.Since(start); d > 3*time.Second { + t.Errorf("Run returned after %v; the 300 ms deadline was not enforced", d) + } + b, err := os.ReadFile(pidfile) + if err != nil { + t.Fatalf("child never wrote its pid: %v", err) + } + pid, _ := strconv.Atoi(string(b)) + if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) { + t.Errorf("child %d still exists after Run returned (kill -0: %v)", pid, err) + } +} + +// The caller's context cancels the child too. +func TestRunnerContextCancel(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond) + defer cancel() + _, err := runner("SIFT_PDFTEXT_SLEEP=30s").Run(ctx, three()) + if err == nil { + t.Fatal("cancelled run succeeded") + } +} + +// Runner passes only Env to the child: nothing of the parent's environment leaks in. +func TestRunnerEnvIsExplicit(t *testing.T) { + t.Setenv("SIFT_PDFTEXT_SLEEP", "30s") // in the parent only + r := runner() + r.Deadline = 5 * time.Second + if _, err := r.Run(context.Background(), three()); err != nil { + t.Errorf("parent env leaked into the child (it slept): %v", err) + } +} diff --git a/internal/pdftext/runner.go b/internal/pdftext/runner.go new file mode 100644 index 0000000..1d981d5 --- /dev/null +++ b/internal/pdftext/runner.go @@ -0,0 +1,269 @@ +package pdftext + +import ( + "bufio" + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "os/exec" + "strconv" + "strings" + "time" +) + +const ( + // defaultDeadline bounds a child that never finishes. + defaultDeadline = 60 * time.Second + // defaultMaxText caps child output before it is rejected. + defaultMaxText = 64 << 20 // 64 MiB + // defaultMaxPages caps how many pages the child extracts. + defaultMaxPages = 2000 + // stderrCap keeps only the first part of the child's stderr, for messages. + stderrCap = 4096 +) + +// Runner executes the extraction child and parses its output into a Doc. +type Runner struct { + Exe string // "" = /proc/self/exe + Args []string // nil = []string{"pdftext"}; -max-pages is appended + Env []string // the child's whole environment; nil = none + Deadline time.Duration // 0 = 60 s + MaxText int64 // bytes of child output; 0 = 64 MiB + MaxPages int // 0 = 2000 +} + +// Run executes the child, writes pdf to its stdin, and parses its output into a +// Doc. The child is killed at the deadline (or when ctx is done, whichever is +// sooner) and always reaped before Run returns. +func (r *Runner) Run(ctx context.Context, pdf []byte) (*Doc, error) { + deadline := r.Deadline + if deadline == 0 { + deadline = defaultDeadline + } + maxText := r.MaxText + if maxText == 0 { + maxText = defaultMaxText + } + maxPages := r.MaxPages + if maxPages == 0 { + maxPages = defaultMaxPages + } + + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline)) + defer cancel() + + cmd := exec.CommandContext(ctx, r.exe(), r.args(maxPages)...) + if r.Env == nil { + cmd.Env = []string{} + } else { + cmd.Env = r.Env + } + + stdin, err := cmd.StdinPipe() + if err != nil { + return nil, wrapUnreadable(err) + } + stdout, err := cmd.StdoutPipe() + if err != nil { + return nil, wrapUnreadable(err) + } + stderr, err := cmd.StderrPipe() + if err != nil { + return nil, wrapUnreadable(err) + } + + var stdoutBuf, stderrBuf bytes.Buffer + done := make(chan error, 1) + go func() { + // One extra byte so output over the limit is distinguishable from + // output exactly at it. + _, e := io.Copy(&stdoutBuf, io.LimitReader(stdout, maxText+1)) + done <- e + }() + stderrDone := make(chan struct{}) + go func() { + io.Copy(&limitedWriter{n: stderrCap, w: &stderrBuf}, stderr) + close(stderrDone) + }() + go func() { + if _, err := stdin.Write(pdf); err != nil { + return + } + _ = stdin.Close() + }() + + if err := cmd.Start(); err != nil { + cancel() + _ = stdout.Close() + _ = stderr.Close() + <-done + <-stderrDone + return nil, wrapUnreadable(err) + } + + readErr := <-done + <-stderrDone + + // Output over the limit: kill the child and report. + if int64(stdoutBuf.Len()) > maxText { + cancel() + _ = cmd.Wait() + return nil, ErrTooMuchText + } + + waitErr := cmd.Wait() + + // A deadline reached while the child ran is a timeout, even if the kill + // surfaced as some other error. + if ctx.Err() == context.DeadlineExceeded { + return nil, ErrTimeout + } + if readErr != nil && !errors.Is(readErr, io.EOF) { + return nil, wrapUnreadable(fmt.Errorf("read output: %w", readErr)) + } + if waitErr != nil { + exitErr, ok := waitErr.(*exec.ExitError) + if !ok { + return nil, wrapUnreadable(fmt.Errorf("child: %v", waitErr)) + } + switch exitErr.ExitCode() { + case 3: + return nil, ErrEncrypted + case 4: + return nil, ErrUnreadable + default: + return nil, wrapUnreadable([]byte(firstLine(stderrBuf.Bytes()))) + } + } + + doc, err := parseDoc(stdoutBuf.Bytes(), maxPages) + if err != nil { + return nil, err + } + return doc, nil +} + +// exe resolves the executable to run, defaulting to this process's own path. +func (r *Runner) exe() string { + if r.Exe != "" { + return r.Exe + } + if e, err := os.Readlink("/proc/self/exe"); err == nil { + return e + } + return "sift" +} + +// args builds the child's argument list: the configured args, or "pdftext", +// with -max-pages appended. +func (r *Runner) args(maxPages int) []string { + args := []string{"pdftext"} + if r.Args != nil { + args = r.Args + } + return append(args, "-max-pages", strconv.Itoa(maxPages)) +} + +// parseDoc turns the child's JSON Lines into a Doc. The first line is the meta; +// each following line is a page's text or its error. It returns ErrNoText when +// no page was extracted or every extracted page's text is blank, and +// ErrUnreadable when the output is not what the child should have produced. +func parseDoc(data []byte, maxPages int) (*Doc, error) { + sc := bufio.NewScanner(bytes.NewReader(data)) + sc.Buffer(make([]byte, 1<<20), 64<<20) + + var doc Doc + for sc.Scan() { + var line struct { + Meta *Meta `json:"meta"` + Page int `json:"page"` + Text *string `json:"text"` + Error string `json:"error"` + } + if err := json.Unmarshal(sc.Bytes(), &line); err != nil { + return nil, wrapUnreadable(nil) + } + if line.Meta != nil { + doc.Meta = *line.Meta + continue + } + appendPage(&doc, line.Page, line.Text, line.Error) + } + if err := sc.Err(); err != nil { + return nil, wrapUnreadable(nil) + } + + if len(doc.Text) == 0 { + return nil, ErrNoText + } + for _, t := range doc.Text { + if strings.TrimSpace(t) != "" { + return &doc, nil + } + } + return nil, ErrNoText +} + +// appendPage records one page's result, growing Text to fit page n. +func appendPage(doc *Doc, n int, text *string, errMsg string) { + for len(doc.Text) < n { + doc.Text = append(doc.Text, "") + } + if text != nil { + doc.Text[n-1] = *text + return + } + doc.Text[n-1] = "" + if errMsg != "" { + doc.Failed = append(doc.Failed, n) + } +} + +// wrapUnreadable wraps a cause (an error or a chunk of stderr) as ErrUnreadable. +func wrapUnreadable(v any) error { + switch s := v.(type) { + case nil: + return ErrUnreadable + case error: + return fmt.Errorf("%w: %v", ErrUnreadable, s) + case []byte: + return fmt.Errorf("%w: %s", ErrUnreadable, firstLine(s)) + case string: + return fmt.Errorf("%w: %s", ErrUnreadable, s) + } + return ErrUnreadable +} + +// firstLine returns the first line of b, or b when it has no newline. +func firstLine(b []byte) string { + if i := bytes.IndexByte(b, '\n'); i >= 0 { + return string(b[:i]) + } + return string(b) +} + +// limitedWriter writes at most n bytes, dropping the rest so a child that +// overflows stderr cannot block the parent. +type limitedWriter struct { + n int64 + w io.Writer +} + +func (l *limitedWriter) Write(p []byte) (int, error) { + if l.n <= 0 { + return 0, nil + } + if int64(len(p)) > l.n { + p = p[:l.n] + } + n, err := l.w.Write(p) + l.n -= int64(n) + if l.n < 0 { + l.n = 0 + } + return n, err +} diff --git a/tests/testdata/pdf/README.md b/tests/testdata/pdf/README.md new file mode 100644 index 0000000..f4d46db --- /dev/null +++ b/tests/testdata/pdf/README.md @@ -0,0 +1,14 @@ +# PDF fixtures + +Only the two encrypted files are binary; every other PDF the tests use is built at test time by +`internal/pdftest`. + +- `encrypted-user.pdf`: the `pdftest` three-page document, AES-256, user password `userpw`, + owner password `ownerpw`. Made with + `qpdf --encrypt userpw ownerpw 256 -- three.pdf encrypted-user.pdf`. +- `encrypted-owner.pdf`: the same document, AES-256 with an empty user password + (opens without one), owner password `ownerpw`: + `qpdf --encrypt "" ownerpw 256 -- three.pdf encrypted-owner.pdf`. + +Both contain the phrases "Chapter one begins here." (page 1) and +"The quick brown fox is on page two." (page 2). diff --git a/tests/testdata/pdf/encrypted-owner.pdf b/tests/testdata/pdf/encrypted-owner.pdf new file mode 100644 index 0000000..2aaddbf Binary files /dev/null and b/tests/testdata/pdf/encrypted-owner.pdf differ diff --git a/tests/testdata/pdf/encrypted-user.pdf b/tests/testdata/pdf/encrypted-user.pdf new file mode 100644 index 0000000..6e57733 --- /dev/null +++ b/tests/testdata/pdf/encrypted-user.pdf @@ -0,0 +1,81 @@ +%PDF-1.7 +% +1 0 obj +<< /Extensions << /ADBE << /BaseVersion /1.7 /ExtensionLevel 8 >> >> /Outlines 3 0 R /PageMode /UseOutlines /Pages 4 0 R /Type /Catalog >> +endobj +2 0 obj +<< /Producer <43b5e97c0a7de15bc0f4dbcc052b5f80f21e0e3bede0eda33137092879ea433d> /Title <29f3ced7effab532ee871ab3ebc38a6869b6b7379f2fa12e139f3e2a1d1ee664> >> +endobj +3 0 obj +<< /Count 3 /First 5 0 R /Last 6 0 R /Type /Outlines >> +endobj +4 0 obj +<< /Count 3 /Kids [ 7 0 R 8 0 R 9 0 R ] /Type /Pages >> +endobj +5 0 obj +<< /Dest [ 7 0 R /XYZ null null null ] /Next 10 0 R /Parent 3 0 R /Title >> +endobj +6 0 obj +<< /Dest [ 9 0 R /XYZ null null null ] /Parent 3 0 R /Prev 10 0 R /Title <13b9a70c4f34e06e182eb823c2cb79a1f82c91f6c69ee89bf2093cd61d451407> >> +endobj +7 0 obj +<< /Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources << /Font << /F1 12 0 R >> /XObject << /Im1 13 0 R >> >> /Type /Page >> +endobj +8 0 obj +<< /Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources << /Font << /F1 12 0 R >> /XObject << /Im1 13 0 R >> >> /Type /Page >> +endobj +9 0 obj +<< /Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources << /Font << /F1 12 0 R >> /XObject << /Im1 13 0 R >> >> /Type /Page >> +endobj +10 0 obj +<< /Dest [ 8 0 R /XYZ null null null ] /Next 6 0 R /Parent 3 0 R /Prev 5 0 R /Title >> +endobj +11 0 obj +<< /Length 128 /Filter /FlateDecode >> +stream +jl<ħ|kj>K""-'agcYӓ ^h9u4DY>hSkvJx '/ #o=:1Ǣh<|HJxYz4endstream +endobj +12 0 obj +<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Subtype /Type1 /Type /Font >> +endobj +13 0 obj +<< /BitsPerComponent 8 /ColorSpace /DeviceGray /Height 2 /Subtype /Image /Type /XObject /Width 2 /Length 32 /Filter /FlateDecode >> +stream +˙"_57Ӕ60%1V\$mG&0FMendstream +endobj +14 0 obj +<< /Length 112 /Filter /FlateDecode >> +stream +,#c}jjw)揄d7wpigvH6oAvD}9G`1wƎX݉ rܛVf:I)x9ߓ>Wxćʑdendstream +endobj +15 0 obj +<< /Length 160 /Filter /FlateDecode >> +stream +yx%"(h]T~B]+$kMe.bPת0;%KzC!̘)(;/@yhqR|c(({cAeΧDڼ)(! #5uk#W<8pI1XQ"G endstream +endobj +16 0 obj +<< /CF << /StdCF << /AuthEvent /DocOpen /CFM /AESV3 /Length 32 >> >> /Filter /Standard /Length 256 /O <2a0820bbf93fff95a410525ae20d4abca153af1e3464706a4f83e49c8caa92e62e160b6871c75b9e47a6cd6dd15fe6ab> /OE <210b63a9a2305637af02dc6e89409285f61f82adaafa125e950042528164e2a3> /P -4 /Perms <23688a77e80aeb8b536511ce497c729e> /R 6 /StmF /StdCF /StrF /StdCF /U <98531c24f9d0bbabd16cc589be7618a59de91065126679a37b26b5d684cbcbc76d34bf68dc61eafe1681b4d5a21826d1> /UE <75ac70233a6a28bcd012febedaf3cccf6c25af218f3318a9fa9bc0f94a0e1b53> /V 5 >> +endobj +xref +0 17 +0000000000 65535 f +0000000015 00000 n +0000000169 00000 n +0000000341 00000 n +0000000412 00000 n +0000000483 00000 n +0000000641 00000 n +0000000799 00000 n +0000000956 00000 n +0000001113 00000 n +0000001270 00000 n +0000001440 00000 n +0000001640 00000 n +0000001738 00000 n +0000001935 00000 n +0000002119 00000 n +0000002351 00000 n +trailer << /Info 2 0 R /Root 1 0 R /Size 17 /ID [<39c444018179222865caf844f4fb7254><39c444018179222865caf844f4fb7254>] /Encrypt 16 0 R >> +startxref +2899 +%%EOF diff --git a/tools.go b/tools.go deleted file mode 100644 index 40107ce..0000000 --- a/tools.go +++ /dev/null @@ -1,10 +0,0 @@ -//go:build tools - -// Package sift pins the PDF engine chosen for P5a (docs/P5a.md §3) so go.mod and vendor/ carry it -// before the code that imports it exists. Remove this file once internal/pdftext imports both. -package main - -import ( - _ "github.com/go-pdfkit/extract" - _ "github.com/go-pdfkit/reader" -)