pdftext: extract PDF text in a re-exec'd child (sift pdftext) with a deadline

The parser is pure Go but a goroutine cannot be killed, so extraction runs in
a child process: the binary re-executed as `sift pdftext`, reading the PDF on
stdin and writing JSON Lines on stdout, killed at a deadline by the parent.

pdftext.go holds the types, the five errors, Extract (reader.Open with
ErrWrongPassword -> ErrEncrypted, per-page recover, trailer /Info /Title, and
rule-3 UTF-16BE/Latin-1 text decode) and ChildMain (soft 768 MiB memory limit,
-no env read, no network, exit codes 0/3/4/1). outline.go walks /Outlines ->
/First -> /Next with /First children at level+1, resolving /Dest and /GoTo /A
dests to 1-based pages, bounded at 10,000 items and never visiting an object
twice. runner.go is the Runner: env-isolated child (nil env is an empty
environment), 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr
capped at 4 KiB, exit-code to error mapping, and Doc parsing with ErrNoText.
cmd/sift/main.go adds the unlisted `pdftext` case. tools.go is removed; the
go-pdfkit modules stay vendored.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

cmd/sift/main.go

@@ -3,6 +3,8 @@ package main
 import (
 	"fmt"
 	"os"
+
+	"git.wntrmute.dev/kyle/sift/internal/pdftext"
 )
 
 // version is set by ldflags: -X main.version=…
@@ -24,6 +26,8 @@ func main() {
 		searchMain()
 	case "keys":
 		keysMain()
+	case "pdftext":
+		os.Exit(pdftext.ChildMain(os.Args[1:], os.Stdin, os.Stdout))
 	case "version", "-version", "--version":
 		fmt.Println(version)
 	case "-h", "--help", "help":

docs/implementer-log.md

@@ -5,3 +5,4 @@ One row per task session that ends in a commit. Newest last.
 | Task | Date | Status | Sessions | Gate | Deviations | Summary |
 |---|---|---|---|---|---|---|
 | p5a/01-kind-fetch | 2026-09-25 | done | resumed | gate: ok | none | Implemented Kind/`application/pdf` recognition, PDF body cap (100 MB) and deadline (120 s), and FetchIf with If-None-Match/If-Modified-Since via shared doFetch. Previous session could not pass TestFetchIf only because the given test held its mutex across the handler (deadlock); owner fixed the test. Session resumed after that fix — did not rewrite fetch code, only verified rules and ran gate. |
+| p5a/02-pdftext | 2026-09-25 | done | 1 | gate: ok | none | Built the child-extraction package: `Extract` (reader.Open with ErrWrongPassword→ErrEncrypted, per-page recover, trailer /Info /Title, rule-3 UTF-16BE/Latin-1 decode), `outline.go` (/Dest and /GoTo /A dests, 10k-item and no-double-visit bounds, never-nil outline), `Runner` (re-exec'd child, env-isolated via empty env, 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code→error mapping, Doc parse plus ErrNoText), and the `sift pdftext` case in main.go. Removed the tools.go module pin; go.mod/go.sum/vendor/modules.txt confirmed unchanged. All 10 given tests pass under -race -count=3 including the deadline-kill test; make gate ok; given files byte-identical; `printf nope | bin/sift pdftext` exits 4. |

internal/pdftest/pdftest.go

@@ -0,0 +1,133 @@
+// Package pdftest builds small, deterministic PDFs for tests: text pages in Helvetica, an optional
+// outline (bookmarks), and image-only ("scanned") pages. It exists so the P5a tests need no binary
+// fixtures from elsewhere. It is test support: nothing outside _test.go files imports it.
+package pdftest
+
+import (
+	"bytes"
+	"fmt"
+	"strings"
+)
+
+// Outline is one bookmark: a title that points at a 1-based page, nested at Level (1 = top).
+// Build supports one level; Level is kept so tests can state what they expect back.
+type Outline struct {
+	Title string
+	Page  int
+	Level int
+}
+
+// Doc describes a PDF to build. Pages holds each page's text; a page's lines are split on "\n"
+// and set one per line (long lines are not wrapped: callers keep them under ~90 characters).
+// Scanned pages (1-based) carry a small image and no text at all.
+type Doc struct {
+	Title   string
+	Pages   []string
+	Outline []Outline
+	Scanned []int
+}
+
+// Build returns the PDF bytes for d. It panics on a Doc it cannot express (an outline entry
+// pointing past the last page), since that is a bug in the test that built it.
+func Build(d Doc) []byte {
+	var b bytes.Buffer
+	var offsets []int
+	obj := func(body string) int {
+		offsets = append(offsets, b.Len())
+		n := len(offsets)
+		fmt.Fprintf(&b, "%d 0 obj\n%s\nendobj\n", n, body)
+		return n
+	}
+	b.WriteString("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
+
+	// Object numbers are fixed by position: 1 catalog, 2 pages, 3 font, 4 image, 5 info, then
+	// two per page (page, content), then the outline.
+	nPages := len(d.Pages)
+	firstPage := 6
+	pageObj := func(i int) int { return firstPage + 2*i }
+	outlineRoot := firstPage + 2*nPages
+	scanned := map[int]bool{}
+	for _, p := range d.Scanned {
+		scanned[p] = true
+	}
+
+	catalog := "<< /Type /Catalog /Pages 2 0 R"
+	if len(d.Outline) > 0 {
+		catalog += fmt.Sprintf(" /Outlines %d 0 R /PageMode /UseOutlines", outlineRoot)
+	}
+	obj(catalog + " >>")
+	var kids []string
+	for i := range d.Pages {
+		kids = append(kids, fmt.Sprintf("%d 0 R", pageObj(i)))
+	}
+	obj(fmt.Sprintf("<< /Type /Pages /Kids [%s] /Count %d >>", strings.Join(kids, " "), nPages))
+	obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>")
+	img := []byte{0x00, 0x80, 0x80, 0x00} // 2x2 grey pixels
+	obj(fmt.Sprintf("<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length %d >>\nstream\n%s\nendstream", len(img), img))
+	obj(fmt.Sprintf("<< /Title (%s) /Producer (sift pdftest) >>", esc(d.Title)))
+
+	for i, text := range d.Pages {
+		var content string
+		if scanned[i+1] {
+			content = "q 400 0 0 500 100 150 cm /Im1 Do Q"
+		} else {
+			var c strings.Builder
+			c.WriteString("BT /F1 11 Tf 13 TL 60 760 Td\n")
+			for _, line := range strings.Split(text, "\n") {
+				fmt.Fprintf(&c, "(%s) Tj T*\n", esc(line))
+			}
+			c.WriteString("ET")
+			content = c.String()
+		}
+		obj(fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents %d 0 R >>", pageObj(i)+1))
+		obj(fmt.Sprintf("<< /Length %d >>\nstream\n%s\nendstream", len(content), content))
+	}
+
+	if len(d.Outline) > 0 {
+		first := outlineRoot + 1
+		last := outlineRoot + len(d.Outline)
+		obj(fmt.Sprintf("<< /Type /Outlines /First %d 0 R /Last %d 0 R /Count %d >>", first, last, len(d.Outline)))
+		for i, o := range d.Outline {
+			if o.Page < 1 || o.Page > nPages {
+				panic(fmt.Sprintf("pdftest: outline %q points at page %d of %d", o.Title, o.Page, nPages))
+			}
+			n := first + i
+			item := fmt.Sprintf("<< /Title (%s) /Parent %d 0 R /Dest [%d 0 R /XYZ null null null]", esc(o.Title), outlineRoot, pageObj(o.Page-1))
+			if i > 0 {
+				item += fmt.Sprintf(" /Prev %d 0 R", n-1)
+			}
+			if n < last {
+				item += fmt.Sprintf(" /Next %d 0 R", n+1)
+			}
+			obj(item + " >>")
+		}
+	}
+
+	xref := b.Len()
+	fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1)
+	for _, off := range offsets {
+		fmt.Fprintf(&b, "%010d 00000 n \n", off)
+	}
+	fmt.Fprintf(&b, "trailer\n<< /Size %d /Root 1 0 R /Info 5 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref)
+	return b.Bytes()
+}
+
+// Lorem returns n lines of distinct, searchable filler: "p<page> l<line> …", so a test can tell
+// which page a line came from.
+func Lorem(page, n int) string {
+	words := []string{"alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"}
+	var lines []string
+	for i := 1; i <= n; i++ {
+		var w []string
+		for j := 0; j < 10; j++ {
+			w = append(w, words[(page*7+i*3+j)%len(words)])
+		}
+		lines = append(lines, fmt.Sprintf("p%d l%d %s", page, i, strings.Join(w, " ")))
+	}
+	return strings.Join(lines, "\n")
+}
+
+func esc(s string) string {
+	r := strings.NewReplacer(`\`, `\\`, `(`, `\(`, `)`, `\)`)
+	return r.Replace(s)
+}

internal/pdftext/outline.go

@@ -0,0 +1,142 @@
+package pdftext
+
+import (
+	"github.com/go-pdfkit/reader"
+)
+
+// maxOutlineItems bounds a walk over a malformed outline that claims more
+// entries than the file could hold.
+const maxOutlineItems = 10000
+
+// walkOutline returns the document's outline as a depth-first, pre-order list:
+// each item before its own children. The result is never nil.
+func walkOutline(doc *reader.Document) []OutlineItem {
+	out := []OutlineItem{}
+	trailer := doc.Trailer()
+	if trailer == nil {
+		return out
+	}
+	cat, err := doc.Resolve(trailer.Get("Root"))
+	if err != nil || cat == nil {
+		return out
+	}
+	catDict, ok := reader.ToDict(cat)
+	if !ok {
+		return out
+	}
+	outlines, err := doc.Resolve(catDict.Get("Outlines"))
+	if err != nil || outlines == nil {
+		return out
+	}
+	outlinesDict, ok := reader.ToDict(outlines)
+	if !ok {
+		return out
+	}
+	first, ok := toRef(outlinesDict.Get("First"))
+	if !ok {
+		return out
+	}
+	seen := map[reader.Ref]bool{}
+	walkChildren(doc, first, 1, seen, &out)
+	return out
+}
+
+// walkChildren walks the /Next chain from ref at the given level, emitting each
+// resolvable item and recursing into its /First children (level + 1). It stops
+// after maxOutlineItems items and never visits an object twice.
+func walkChildren(doc *reader.Document, ref reader.Ref, level int, seen map[reader.Ref]bool, out *[]OutlineItem) {
+	cur := ref
+	for cur != (reader.Ref{}) {
+		if seen[cur] {
+			break
+		}
+		seen[cur] = true
+
+		obj, err := doc.Resolve(cur)
+		if err != nil || obj == nil {
+			break
+		}
+		d, ok := reader.ToDict(obj)
+		if !ok {
+			break
+		}
+
+		if page, ok := destPage(doc, d); ok {
+			*out = append(*out, OutlineItem{Title: textString(d.Get("Title")), Page: page, Level: level})
+			if len(*out) >= maxOutlineItems {
+				return
+			}
+		}
+
+		if child, ok := toRef(d.Get("First")); ok {
+			walkChildren(doc, child, level+1, seen, out)
+			if len(*out) >= maxOutlineItems {
+				return
+			}
+		}
+
+		next, ok := toRef(d.Get("Next"))
+		if !ok {
+			break
+		}
+		cur = next
+	}
+}
+
+// destPage resolves an item's destination to a 1-based page number. It reports
+// ok only when the destination is an array whose first element is a page
+// reference that maps to a real page; a named or otherwise unresolvable
+// destination is not ok, so the caller skips the item.
+func destPage(doc *reader.Document, d reader.Dict) (int, bool) {
+	if ref, ok := destRef(doc, d); ok {
+		if page, ok := pageOf(doc, ref); ok {
+			return page, true
+		}
+	}
+	return 0, false
+}
+
+// destRef finds the destination page reference, from /Dest or from a /GoTo
+// action's /D.
+func destRef(doc *reader.Document, d reader.Dict) (reader.Ref, bool) {
+	if o := d.Get("Dest"); o != nil {
+		if arr, ok := reader.ToArray(o); ok && len(arr) > 0 {
+			if ref, ok := arr[0].(reader.Ref); ok {
+				return ref, true
+			}
+		}
+	}
+	if o := d.Get("A"); o != nil {
+		if a, ok := reader.ToDict(o); ok {
+			if s, ok := reader.ToName(a.Get("S")); ok && s == "GoTo" {
+				if d2 := a.Get("D"); d2 != nil {
+					if arr, ok := reader.ToArray(d2); ok && len(arr) > 0 {
+						if ref, ok := arr[0].(reader.Ref); ok {
+							return ref, true
+						}
+					}
+				}
+			}
+		}
+	}
+	return reader.Ref{}, false
+}
+
+// pageOf maps a reference to its 1-based page by comparing it with each page's
+// reference.
+func pageOf(doc *reader.Document, ref reader.Ref) (int, bool) {
+	for i := 1; i <= doc.PageCount(); i++ {
+		pr, ok := doc.PageRef(i)
+		if ok && pr == ref {
+			return i, true
+		}
+	}
+	return 0, false
+}
+
+// toRef reports the reference a dictionary entry names, when it is a direct
+// reference rather than an already-resolved object.
+func toRef(o reader.Object) (reader.Ref, bool) {
+	ref, ok := o.(reader.Ref)
+	return ref, ok
+}

internal/pdftext/pdftext.go

@@ -0,0 +1,216 @@
+// Package pdftext extracts text, title and outline from a PDF. Extraction runs
+// in a child process (see Runner) so a page that hangs or panics cannot take
+// the parent down; Extract is the child's own work, usable directly in tests.
+package pdftext
+
+import (
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"runtime/debug"
+	"strings"
+	"unicode/utf16"
+
+	"github.com/go-pdfkit/extract"
+	"github.com/go-pdfkit/reader"
+)
+
+// The errors Extract and Runner can return.
+var (
+	ErrEncrypted   = errors.New("encrypted PDF (password required)")
+	ErrUnreadable  = errors.New("unreadable PDF")
+	ErrNoText      = errors.New("no text layer (scanned PDF?)")
+	ErrTimeout     = errors.New("pdf extraction timed out")
+	ErrTooMuchText = errors.New("pdf text over the output limit")
+)
+
+// OutlineItem is one bookmark: its title, the 1-based page it points at, and
+// its nesting level (1 = top level).
+type OutlineItem struct {
+	Title string `json:"title"`
+	Page  int    `json:"page"`  // 1-based
+	Level int    `json:"level"` // 1 = top level
+}
+
+// Meta is the document-level summary written as the first output line.
+type Meta struct {
+	Pages   int           `json:"pages"`   // pages in the document (not only those extracted)
+	Title   string        `json:"title"`   // /Info /Title, "" when absent
+	Outline []OutlineItem `json:"outline"` // never nil: [] when the document has none
+}
+
+// Doc is an extracted document: Text[i] is page i+1's text, for the pages
+// extracted (len(Text) ≤ Pages). Failed lists pages whose extraction failed
+// (their Text is "").
+type Doc struct {
+	Meta
+	Text   []string
+	Failed []int
+}
+
+// metaLine is the first output line: the document summary.
+type metaLine struct {
+	Meta Meta `json:"meta"`
+}
+
+// pageOK and pageErr are the per-page lines. A page that extracts yields pageOK
+// even when its text is empty (an image-only page); one that fails yields
+// pageErr and extraction continues.
+type pageOK struct {
+	Page int    `json:"page"`
+	Text string `json:"text"`
+}
+
+type pageErr struct {
+	Page  int    `json:"page"`
+	Error string `json:"error"`
+}
+
+// Extract writes one {"meta":{…}} line, then one {"page":n,"text":"…"} line per
+// page from 1 up to min(Pages, maxPages); a page that fails yields
+// {"page":n,"error":"…"} and extraction goes on. It returns ErrEncrypted when
+// the document needs a user password, ErrUnreadable when it cannot be opened or
+// has no pages, and never panics.
+func Extract(pdf []byte, maxPages int, w io.Writer) error {
+	doc, err := reader.Open(pdf)
+	if err != nil {
+		if errors.Is(err, reader.ErrWrongPassword) {
+			return ErrEncrypted
+		}
+		return ErrUnreadable
+	}
+	if doc.PageCount() == 0 {
+		return ErrUnreadable
+	}
+
+	meta := Meta{
+		Pages:   doc.PageCount(),
+		Title:   infoTitle(doc),
+		Outline: walkOutline(doc),
+	}
+	if meta.Outline == nil {
+		meta.Outline = []OutlineItem{}
+	}
+	if err := writeJSONLine(w, metaLine{Meta: meta}); err != nil {
+		return err
+	}
+
+	n := doc.PageCount()
+	if maxPages > 0 && n > maxPages {
+		n = maxPages
+	}
+	for i := 1; i <= n; i++ {
+		func() {
+			defer func() {
+				if r := recover(); r != nil {
+					_ = writeJSONLine(w, pageErr{Page: i, Error: fmt.Sprintf("panic: %v", r)})
+				}
+			}()
+			text, err := extract.Text(doc, i)
+			if err != nil {
+				_ = writeJSONLine(w, pageErr{Page: i, Error: err.Error()})
+				return
+			}
+			_ = writeJSONLine(w, pageOK{Page: i, Text: text})
+		}()
+	}
+	return nil
+}
+
+// writeJSONLine marshals v and writes it as one JSON line.
+func writeJSONLine(w io.Writer, v any) error {
+	b, err := json.Marshal(v)
+	if err != nil {
+		return err
+	}
+	if _, err := w.Write(b); err != nil {
+		return err
+	}
+	_, err = w.Write([]byte{'\n'})
+	return err
+}
+
+// infoTitle returns the document title from the trailer's /Info /Title, decoded
+// as a PDF text string (rule 3), "" when absent.
+func infoTitle(doc *reader.Document) string {
+	trailer := doc.Trailer()
+	if trailer == nil {
+		return ""
+	}
+	info, err := doc.Resolve(trailer.Get("Info"))
+	if err != nil || info == nil {
+		return ""
+	}
+	d, ok := reader.ToDict(info)
+	if !ok {
+		return ""
+	}
+	return textString(d.Get("Title"))
+}
+
+// textString decodes a reader.String (raw bytes) as a PDF text string per rule 3.
+func textString(o reader.Object) string {
+	s, ok := reader.ToString(o)
+	if !ok {
+		return ""
+	}
+	return decodePDFString(s)
+}
+
+// decodePDFString follows rule 3: bytes starting FE FF are UTF-16BE (the rest is
+// decoded as such); otherwise each byte is treated as a Latin-1 code point.
+func decodePDFString(b []byte) string {
+	if len(b) >= 2 && b[0] == 0xFE && b[1] == 0xFF {
+		return strings.TrimSpace(string(utf16.Decode(utf16BE(b[2:]))))
+	}
+	return strings.TrimSpace(string(b))
+}
+
+// utf16BE packs big-endian byte pairs into uint16s for utf16.Decode.
+func utf16BE(b []byte) []uint16 {
+	n := len(b) / 2
+	u := make([]uint16, n)
+	for i := 0; i < 2*n; i += 2 {
+		u[i/2] = uint16(b[i])<<8 | uint16(b[i+1])
+	}
+	return u
+}
+
+// ChildMain is `sift pdftext`: flags (-max-pages N, default 2000), the PDF on
+// stdin (read with a 200 MiB cap), Extract to stdout. Exit codes: 0 ok, 3
+// ErrEncrypted, 4 ErrUnreadable, 1 anything else. It runs with a soft memory
+// limit, reads no environment variable, and opens no network connection.
+func ChildMain(args []string, stdin io.Reader, stdout io.Writer) int {
+	debug.SetMemoryLimit(768 << 20)
+
+	fs := flag.NewFlagSet("pdftext", flag.ContinueOnError)
+	maxPages := fs.Int("max-pages", 2000, "maximum number of pages to extract")
+	if err := fs.Parse(args); err != nil {
+		return 1
+	}
+
+	pdf, err := io.ReadAll(io.LimitReader(stdin, 200<<20))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, "pdftext: read input:", err)
+		return 4
+	}
+	if len(pdf) == 0 {
+		fmt.Fprintln(os.Stderr, "pdftext: empty input")
+		return 4
+	}
+
+	if err := Extract(pdf, *maxPages, stdout); err != nil {
+		switch {
+		case errors.Is(err, ErrEncrypted):
+			return 3
+		case errors.Is(err, ErrUnreadable):
+			return 4
+		default:
+			return 1
+		}
+	}
+	return 0
+}

internal/pdftext/pdftext_test.go

@@ -0,0 +1,252 @@
+package pdftext_test
+
+// P5a task 02: Extract (the child's work), ChildMain (the `sift pdftext` entry point) and Runner
+// (the parent: exec, deadline, kill, parse).
+//
+// The Runner tests run the test binary itself as the child: TestMain turns into the child when
+// SIFT_PDFTEXT_CHILD=1 is set, exactly as `sift pdftext` would. The two other variables are test
+// hooks that live only here: SIFT_PDFTEXT_SLEEP delays the child (for the deadline test) and
+// SIFT_PDFTEXT_PIDFILE records its pid (to prove it is gone afterwards).
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"os"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/pdftest"
+	"git.wntrmute.dev/kyle/sift/internal/pdftext"
+)
+
+func TestMain(m *testing.M) {
+	if os.Getenv("SIFT_PDFTEXT_CHILD") == "1" {
+		if p := os.Getenv("SIFT_PDFTEXT_PIDFILE"); p != "" {
+			_ = os.WriteFile(p, []byte(strconv.Itoa(os.Getpid())), 0o600)
+		}
+		if d, err := time.ParseDuration(os.Getenv("SIFT_PDFTEXT_SLEEP")); err == nil {
+			time.Sleep(d)
+		}
+		os.Exit(pdftext.ChildMain(os.Args[1:], os.Stdin, os.Stdout))
+	}
+	os.Exit(m.Run())
+}
+
+func three() []byte {
+	return pdftest.Build(pdftest.Doc{
+		Title: "Three Pages",
+		Pages: []string{
+			"Chapter one begins here.\nSecond line (with parens).",
+			"The quick brown fox is on page two.",
+			pdftest.Lorem(3, 5),
+		},
+		Outline: []pdftest.Outline{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "3 Filler", Page: 3, Level: 1}},
+	})
+}
+
+type line struct {
+	Meta  *pdftext.Meta `json:"meta"`
+	Page  int           `json:"page"`
+	Text  *string       `json:"text"`
+	Error string        `json:"error"`
+}
+
+func lines(t *testing.T, out []byte) []line {
+	t.Helper()
+	var ls []line
+	sc := bufio.NewScanner(bytes.NewReader(out))
+	sc.Buffer(make([]byte, 1<<20), 64<<20)
+	for sc.Scan() {
+		var l line
+		if err := json.Unmarshal(sc.Bytes(), &l); err != nil {
+			t.Fatalf("output line is not JSON: %q (%v)", sc.Text(), err)
+		}
+		ls = append(ls, l)
+	}
+	return ls
+}
+
+func TestExtractWritesMetaThenPages(t *testing.T) {
+	var out bytes.Buffer
+	if err := pdftext.Extract(three(), 2000, &out); err != nil {
+		t.Fatal(err)
+	}
+	ls := lines(t, out.Bytes())
+	if len(ls) != 4 || ls[0].Meta == nil {
+		t.Fatalf("got %d lines, first meta=%v; want a meta line then 3 page lines:\n%s", len(ls), ls[0].Meta, out.String())
+	}
+	m := ls[0].Meta
+	if m.Pages != 3 || m.Title != "Three Pages" {
+		t.Errorf("meta = %+v, want 3 pages titled Three Pages", *m)
+	}
+	want := []pdftext.OutlineItem{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "3 Filler", Page: 3, Level: 1}}
+	if len(m.Outline) != 3 {
+		t.Fatalf("outline = %+v, want %+v", m.Outline, want)
+	}
+	for i := range want {
+		if m.Outline[i] != want[i] {
+			t.Errorf("outline[%d] = %+v, want %+v", i, m.Outline[i], want[i])
+		}
+	}
+	for i, l := range ls[1:] {
+		if l.Page != i+1 || l.Text == nil || l.Error != "" {
+			t.Errorf("line %d = %+v, want page %d with text", i+1, l, i+1)
+		}
+	}
+	if !strings.Contains(*ls[1].Text, "Chapter one begins here.") || !strings.Contains(*ls[2].Text, "quick brown fox") || !strings.Contains(*ls[3].Text, "p3 l5") {
+		t.Errorf("page texts wrong: %q / %q / %q", *ls[1].Text, *ls[2].Text, *ls[3].Text)
+	}
+}
+
+func TestExtractPageCap(t *testing.T) {
+	var out bytes.Buffer
+	if err := pdftext.Extract(three(), 2, &out); err != nil {
+		t.Fatal(err)
+	}
+	ls := lines(t, out.Bytes())
+	if len(ls) != 3 || ls[0].Meta.Pages != 3 || ls[2].Page != 2 {
+		t.Errorf("maxPages 2: %d lines, meta %+v; want meta (pages 3) and pages 1-2", len(ls), ls[0].Meta)
+	}
+}
+
+func TestExtractNoOutline(t *testing.T) {
+	var out bytes.Buffer
+	if err := pdftext.Extract(pdftest.Build(pdftest.Doc{Pages: []string{"a", "b"}}), 10, &out); err != nil {
+		t.Fatal(err)
+	}
+	ls := lines(t, out.Bytes())
+	if ls[0].Meta.Outline == nil || len(ls[0].Meta.Outline) != 0 {
+		t.Errorf("outline = %#v, want an empty non-nil slice (JSON [])", ls[0].Meta.Outline)
+	}
+	if !strings.Contains(out.String(), `"outline":[]`) {
+		t.Errorf("meta line must carry \"outline\":[]: %s", out.String())
+	}
+}
+
+func TestExtractErrors(t *testing.T) {
+	enc, err := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := pdftext.Extract(enc, 10, &bytes.Buffer{}); !errors.Is(err, pdftext.ErrEncrypted) {
+		t.Errorf("user-password pdf: %v, want ErrEncrypted", err)
+	}
+	owner, err := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-owner.pdf"))
+	if err != nil {
+		t.Fatal(err)
+	}
+	var out bytes.Buffer
+	if err := pdftext.Extract(owner, 10, &out); err != nil || !strings.Contains(out.String(), "quick brown fox") {
+		t.Errorf("owner-password-only pdf: err %v; want its text", err)
+	}
+	if err := pdftext.Extract([]byte("not a pdf at all"), 10, &bytes.Buffer{}); !errors.Is(err, pdftext.ErrUnreadable) {
+		t.Errorf("garbage: %v, want ErrUnreadable", err)
+	}
+	// A truncated file either yields what it can or ErrUnreadable; it never panics.
+	b := three()
+	_ = pdftext.Extract(b[:len(b)*6/10], 10, &bytes.Buffer{})
+}
+
+func TestChildMainExitCodes(t *testing.T) {
+	enc, _ := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf"))
+	for _, tc := range []struct {
+		name string
+		in   []byte
+		code int
+	}{
+		{"ok", three(), 0},
+		{"encrypted", enc, 3},
+		{"unreadable", []byte("nope"), 4},
+	} {
+		var out bytes.Buffer
+		if code := pdftext.ChildMain([]string{"-max-pages", "5"}, bytes.NewReader(tc.in), &out); code != tc.code {
+			t.Errorf("%s: exit %d, want %d", tc.name, code, tc.code)
+		}
+	}
+}
+
+func runner(env ...string) *pdftext.Runner {
+	return &pdftext.Runner{Exe: os.Args[0], Env: append([]string{"SIFT_PDFTEXT_CHILD=1"}, env...), Deadline: 20 * time.Second}
+}
+
+func TestRunnerRun(t *testing.T) {
+	doc, err := runner().Run(context.Background(), three())
+	if err != nil {
+		t.Fatal(err)
+	}
+	if doc.Pages != 3 || doc.Title != "Three Pages" || len(doc.Outline) != 3 || len(doc.Text) != 3 {
+		t.Fatalf("doc = %+v", doc)
+	}
+	if !strings.Contains(doc.Text[1], "quick brown fox") {
+		t.Errorf("page 2 = %q", doc.Text[1])
+	}
+}
+
+func TestRunnerErrors(t *testing.T) {
+	enc, _ := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf"))
+	if _, err := runner().Run(context.Background(), enc); !errors.Is(err, pdftext.ErrEncrypted) {
+		t.Errorf("encrypted: %v, want ErrEncrypted", err)
+	}
+	if _, err := runner().Run(context.Background(), []byte("nope")); !errors.Is(err, pdftext.ErrUnreadable) {
+		t.Errorf("garbage: %v, want ErrUnreadable", err)
+	}
+	scanned := pdftest.Build(pdftest.Doc{Pages: []string{"", "", ""}, Scanned: []int{1, 2, 3}})
+	if _, err := runner().Run(context.Background(), scanned); !errors.Is(err, pdftext.ErrNoText) {
+		t.Errorf("scanned: %v, want ErrNoText", err)
+	}
+	r := runner()
+	r.MaxText = 100 // bytes of child output
+	if _, err := r.Run(context.Background(), three()); !errors.Is(err, pdftext.ErrTooMuchText) {
+		t.Errorf("output over MaxText: %v, want ErrTooMuchText", err)
+	}
+}
+
+// The child is killed at the deadline and reaped: no process is left behind.
+func TestRunnerDeadlineKillsTheChild(t *testing.T) {
+	pidfile := filepath.Join(t.TempDir(), "pid")
+	r := runner("SIFT_PDFTEXT_SLEEP=30s", "SIFT_PDFTEXT_PIDFILE="+pidfile)
+	r.Deadline = 300 * time.Millisecond
+	start := time.Now()
+	_, err := r.Run(context.Background(), three())
+	if !errors.Is(err, pdftext.ErrTimeout) {
+		t.Fatalf("err %v, want ErrTimeout", err)
+	}
+	if d := time.Since(start); d > 3*time.Second {
+		t.Errorf("Run returned after %v; the 300 ms deadline was not enforced", d)
+	}
+	b, err := os.ReadFile(pidfile)
+	if err != nil {
+		t.Fatalf("child never wrote its pid: %v", err)
+	}
+	pid, _ := strconv.Atoi(string(b))
+	if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) {
+		t.Errorf("child %d still exists after Run returned (kill -0: %v)", pid, err)
+	}
+}
+
+// The caller's context cancels the child too.
+func TestRunnerContextCancel(t *testing.T) {
+	ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
+	defer cancel()
+	_, err := runner("SIFT_PDFTEXT_SLEEP=30s").Run(ctx, three())
+	if err == nil {
+		t.Fatal("cancelled run succeeded")
+	}
+}
+
+// Runner passes only Env to the child: nothing of the parent's environment leaks in.
+func TestRunnerEnvIsExplicit(t *testing.T) {
+	t.Setenv("SIFT_PDFTEXT_SLEEP", "30s") // in the parent only
+	r := runner()
+	r.Deadline = 5 * time.Second
+	if _, err := r.Run(context.Background(), three()); err != nil {
+		t.Errorf("parent env leaked into the child (it slept): %v", err)
+	}
+}

internal/pdftext/runner.go

@@ -0,0 +1,269 @@
+package pdftext
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"encoding/json"
+	"errors"
+	"fmt"
+	"io"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"time"
+)
+
+const (
+	// defaultDeadline bounds a child that never finishes.
+	defaultDeadline = 60 * time.Second
+	// defaultMaxText caps child output before it is rejected.
+	defaultMaxText = 64 << 20 // 64 MiB
+	// defaultMaxPages caps how many pages the child extracts.
+	defaultMaxPages = 2000
+	// stderrCap keeps only the first part of the child's stderr, for messages.
+	stderrCap = 4096
+)
+
+// Runner executes the extraction child and parses its output into a Doc.
+type Runner struct {
+	Exe      string        // "" = /proc/self/exe
+	Args     []string      // nil = []string{"pdftext"}; -max-pages is appended
+	Env      []string      // the child's whole environment; nil = none
+	Deadline time.Duration // 0 = 60 s
+	MaxText  int64         // bytes of child output; 0 = 64 MiB
+	MaxPages int           // 0 = 2000
+}
+
+// Run executes the child, writes pdf to its stdin, and parses its output into a
+// Doc. The child is killed at the deadline (or when ctx is done, whichever is
+// sooner) and always reaped before Run returns.
+func (r *Runner) Run(ctx context.Context, pdf []byte) (*Doc, error) {
+	deadline := r.Deadline
+	if deadline == 0 {
+		deadline = defaultDeadline
+	}
+	maxText := r.MaxText
+	if maxText == 0 {
+		maxText = defaultMaxText
+	}
+	maxPages := r.MaxPages
+	if maxPages == 0 {
+		maxPages = defaultMaxPages
+	}
+
+	ctx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline))
+	defer cancel()
+
+	cmd := exec.CommandContext(ctx, r.exe(), r.args(maxPages)...)
+	if r.Env == nil {
+		cmd.Env = []string{}
+	} else {
+		cmd.Env = r.Env
+	}
+
+	stdin, err := cmd.StdinPipe()
+	if err != nil {
+		return nil, wrapUnreadable(err)
+	}
+	stdout, err := cmd.StdoutPipe()
+	if err != nil {
+		return nil, wrapUnreadable(err)
+	}
+	stderr, err := cmd.StderrPipe()
+	if err != nil {
+		return nil, wrapUnreadable(err)
+	}
+
+	var stdoutBuf, stderrBuf bytes.Buffer
+	done := make(chan error, 1)
+	go func() {
+		// One extra byte so output over the limit is distinguishable from
+		// output exactly at it.
+		_, e := io.Copy(&stdoutBuf, io.LimitReader(stdout, maxText+1))
+		done <- e
+	}()
+	stderrDone := make(chan struct{})
+	go func() {
+		io.Copy(&limitedWriter{n: stderrCap, w: &stderrBuf}, stderr)
+		close(stderrDone)
+	}()
+	go func() {
+		if _, err := stdin.Write(pdf); err != nil {
+			return
+		}
+		_ = stdin.Close()
+	}()
+
+	if err := cmd.Start(); err != nil {
+		cancel()
+		_ = stdout.Close()
+		_ = stderr.Close()
+		<-done
+		<-stderrDone
+		return nil, wrapUnreadable(err)
+	}
+
+	readErr := <-done
+	<-stderrDone
+
+	// Output over the limit: kill the child and report.
+	if int64(stdoutBuf.Len()) > maxText {
+		cancel()
+		_ = cmd.Wait()
+		return nil, ErrTooMuchText
+	}
+
+	waitErr := cmd.Wait()
+
+	// A deadline reached while the child ran is a timeout, even if the kill
+	// surfaced as some other error.
+	if ctx.Err() == context.DeadlineExceeded {
+		return nil, ErrTimeout
+	}
+	if readErr != nil && !errors.Is(readErr, io.EOF) {
+		return nil, wrapUnreadable(fmt.Errorf("read output: %w", readErr))
+	}
+	if waitErr != nil {
+		exitErr, ok := waitErr.(*exec.ExitError)
+		if !ok {
+			return nil, wrapUnreadable(fmt.Errorf("child: %v", waitErr))
+		}
+		switch exitErr.ExitCode() {
+		case 3:
+			return nil, ErrEncrypted
+		case 4:
+			return nil, ErrUnreadable
+		default:
+			return nil, wrapUnreadable([]byte(firstLine(stderrBuf.Bytes())))
+		}
+	}
+
+	doc, err := parseDoc(stdoutBuf.Bytes(), maxPages)
+	if err != nil {
+		return nil, err
+	}
+	return doc, nil
+}
+
+// exe resolves the executable to run, defaulting to this process's own path.
+func (r *Runner) exe() string {
+	if r.Exe != "" {
+		return r.Exe
+	}
+	if e, err := os.Readlink("/proc/self/exe"); err == nil {
+		return e
+	}
+	return "sift"
+}
+
+// args builds the child's argument list: the configured args, or "pdftext",
+// with -max-pages appended.
+func (r *Runner) args(maxPages int) []string {
+	args := []string{"pdftext"}
+	if r.Args != nil {
+		args = r.Args
+	}
+	return append(args, "-max-pages", strconv.Itoa(maxPages))
+}
+
+// parseDoc turns the child's JSON Lines into a Doc. The first line is the meta;
+// each following line is a page's text or its error. It returns ErrNoText when
+// no page was extracted or every extracted page's text is blank, and
+// ErrUnreadable when the output is not what the child should have produced.
+func parseDoc(data []byte, maxPages int) (*Doc, error) {
+	sc := bufio.NewScanner(bytes.NewReader(data))
+	sc.Buffer(make([]byte, 1<<20), 64<<20)
+
+	var doc Doc
+	for sc.Scan() {
+		var line struct {
+			Meta  *Meta   `json:"meta"`
+			Page  int     `json:"page"`
+			Text  *string `json:"text"`
+			Error string  `json:"error"`
+		}
+		if err := json.Unmarshal(sc.Bytes(), &line); err != nil {
+			return nil, wrapUnreadable(nil)
+		}
+		if line.Meta != nil {
+			doc.Meta = *line.Meta
+			continue
+		}
+		appendPage(&doc, line.Page, line.Text, line.Error)
+	}
+	if err := sc.Err(); err != nil {
+		return nil, wrapUnreadable(nil)
+	}
+
+	if len(doc.Text) == 0 {
+		return nil, ErrNoText
+	}
+	for _, t := range doc.Text {
+		if strings.TrimSpace(t) != "" {
+			return &doc, nil
+		}
+	}
+	return nil, ErrNoText
+}
+
+// appendPage records one page's result, growing Text to fit page n.
+func appendPage(doc *Doc, n int, text *string, errMsg string) {
+	for len(doc.Text) < n {
+		doc.Text = append(doc.Text, "")
+	}
+	if text != nil {
+		doc.Text[n-1] = *text
+		return
+	}
+	doc.Text[n-1] = ""
+	if errMsg != "" {
+		doc.Failed = append(doc.Failed, n)
+	}
+}
+
+// wrapUnreadable wraps a cause (an error or a chunk of stderr) as ErrUnreadable.
+func wrapUnreadable(v any) error {
+	switch s := v.(type) {
+	case nil:
+		return ErrUnreadable
+	case error:
+		return fmt.Errorf("%w: %v", ErrUnreadable, s)
+	case []byte:
+		return fmt.Errorf("%w: %s", ErrUnreadable, firstLine(s))
+	case string:
+		return fmt.Errorf("%w: %s", ErrUnreadable, s)
+	}
+	return ErrUnreadable
+}
+
+// firstLine returns the first line of b, or b when it has no newline.
+func firstLine(b []byte) string {
+	if i := bytes.IndexByte(b, '\n'); i >= 0 {
+		return string(b[:i])
+	}
+	return string(b)
+}
+
+// limitedWriter writes at most n bytes, dropping the rest so a child that
+// overflows stderr cannot block the parent.
+type limitedWriter struct {
+	n int64
+	w io.Writer
+}
+
+func (l *limitedWriter) Write(p []byte) (int, error) {
+	if l.n <= 0 {
+		return 0, nil
+	}
+	if int64(len(p)) > l.n {
+		p = p[:l.n]
+	}
+	n, err := l.w.Write(p)
+	l.n -= int64(n)
+	if l.n < 0 {
+		l.n = 0
+	}
+	return n, err
+}

tests/testdata/pdf/README.md

@@ -0,0 +1,14 @@
+# PDF fixtures
+
+Only the two encrypted files are binary; every other PDF the tests use is built at test time by
+`internal/pdftest`.
+
+- `encrypted-user.pdf`: the `pdftest` three-page document, AES-256, user password `userpw`,
+  owner password `ownerpw`. Made with
+  `qpdf --encrypt userpw ownerpw 256 -- three.pdf encrypted-user.pdf`.
+- `encrypted-owner.pdf`: the same document, AES-256 with an empty user password
+  (opens without one), owner password `ownerpw`:
+  `qpdf --encrypt "" ownerpw 256 -- three.pdf encrypted-owner.pdf`.
+
+Both contain the phrases "Chapter one begins here." (page 1) and
+"The quick brown fox is on page two." (page 2).

tests/testdata/pdf/encrypted-user.pdf

@@ -0,0 +1,81 @@
+%PDF-1.7
+%¿÷¢þ
+1 0 obj
+<< /Extensions << /ADBE << /BaseVersion /1.7 /ExtensionLevel 8 >> >> /Outlines 3 0 R /PageMode /UseOutlines /Pages 4 0 R /Type /Catalog >>
+endobj
+2 0 obj
+<< /Producer <43b5e97c0a7de15bc0f4dbcc052b5f80f21e0e3bede0eda33137092879ea433d> /Title <29f3ced7effab532ee871ab3ebc38a6869b6b7379f2fa12e139f3e2a1d1ee664> >>
+endobj
+3 0 obj
+<< /Count 3 /First 5 0 R /Last 6 0 R /Type /Outlines >>
+endobj
+4 0 obj
+<< /Count 3 /Kids [ 7 0 R 8 0 R 9 0 R ] /Type /Pages >>
+endobj
+5 0 obj
+<< /Dest [ 7 0 R /XYZ null null null ] /Next 10 0 R /Parent 3 0 R /Title <d2951c68344226cca87b8f66af448a05c4ba3c4baf7c7863293f2328ce407d09> >>
+endobj
+6 0 obj
+<< /Dest [ 9 0 R /XYZ null null null ] /Parent 3 0 R /Prev 10 0 R /Title <13b9a70c4f34e06e182eb823c2cb79a1f82c91f6c69ee89bf2093cd61d451407> >>
+endobj
+7 0 obj
+<< /Contents 11 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources << /Font << /F1 12 0 R >> /XObject << /Im1 13 0 R >> >> /Type /Page >>
+endobj
+8 0 obj
+<< /Contents 14 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources << /Font << /F1 12 0 R >> /XObject << /Im1 13 0 R >> >> /Type /Page >>
+endobj
+9 0 obj
+<< /Contents 15 0 R /MediaBox [ 0 0 612 792 ] /Parent 4 0 R /Resources << /Font << /F1 12 0 R >> /XObject << /Im1 13 0 R >> >> /Type /Page >>
+endobj
+10 0 obj
+<< /Dest [ 8 0 R /XYZ null null null ] /Next 6 0 R /Parent 3 0 R /Prev 5 0 R /Title <b734ddb0e84e981224c172818d645644e95ae16c94cf45d02ef870d181f97422> >>
+endobj
+11 0 obj
+<< /Length 128 /Filter /FlateDecode >>
+stream
+jl<ħ|Ýkj>Kþ"æ"-‚'agcYÓ“¡Áù ‹ä¾Îç„Â^›™h9Àu¢ò4DY­>€ä‡ÝhSkvJxŠý␍÷À'‡ÿ­ù/טò£Ð#¼Æo=ÖÛ:³1áÇ¢hÇôªÀè<Â|Hü“J‹xYz4�ÿÄendstream
+endobj
+12 0 obj
+<< /BaseFont /Helvetica /Encoding /WinAnsiEncoding /Subtype /Type1 /Type /Font >>
+endobj
+13 0 obj
+<< /BitsPerComponent 8 /ColorSpace /DeviceGray /Height 2 /Subtype /Image /Type /XObject /Width 2 /Length 32 /Filter /FlateDecode >>
+stream
+Ë™"€_œ¼Ë57Ó”Ì60Ë%œ1´V\Ó$mG&0FåMendstream
+endobj
+14 0 obj
+<< /Length 112 /Filter /FlateDecode >>
+stream
+,#c¾}jè¹Ájw�)æ�„›d7Éwp¨íi»gvÍñÎâH»6oA•ï´vD}–Ð9·G`1wÆŽ×XŒÝ‰žÄrÜ›Vf:I¡)x9ÅéÌß“>��WãxŽÄ‡¡ãŠ¤Ô«âÊ‘dendstream
+endobj
+15 0 obj
+<< /Length 160 /Filter /FlateDecode >>
+stream
+yx÷â°í%"(hß]�T~BùÞ]+$ík±Mû†e.ŒbÿPת®0ž;äòãæë%½Kû¾¤z‹C”¤²!̘£)(žœÛôŒáéÄ;/¬@yèºÚh¥Á qR|Ôcݹ²Ã((¼Õ{úc¿AÆeýÏΧDíÚ¼)‡(íÄ!	·#5u·ké#¹âW’<8pŽ÷I1XQ"¥Gendstream
+endobj
+16 0 obj
+<< /CF << /StdCF << /AuthEvent /DocOpen /CFM /AESV3 /Length 32 >> >> /Filter /Standard /Length 256 /O <2a0820bbf93fff95a410525ae20d4abca153af1e3464706a4f83e49c8caa92e62e160b6871c75b9e47a6cd6dd15fe6ab> /OE <210b63a9a2305637af02dc6e89409285f61f82adaafa125e950042528164e2a3> /P -4 /Perms <23688a77e80aeb8b536511ce497c729e> /R 6 /StmF /StdCF /StrF /StdCF /U <98531c24f9d0bbabd16cc589be7618a59de91065126679a37b26b5d684cbcbc76d34bf68dc61eafe1681b4d5a21826d1> /UE <75ac70233a6a28bcd012febedaf3cccf6c25af218f3318a9fa9bc0f94a0e1b53> /V 5 >>
+endobj
+xref
+0 17
+0000000000 65535 f 
+0000000015 00000 n 
+0000000169 00000 n 
+0000000341 00000 n 
+0000000412 00000 n 
+0000000483 00000 n 
+0000000641 00000 n 
+0000000799 00000 n 
+0000000956 00000 n 
+0000001113 00000 n 
+0000001270 00000 n 
+0000001440 00000 n 
+0000001640 00000 n 
+0000001738 00000 n 
+0000001935 00000 n 
+0000002119 00000 n 
+0000002351 00000 n 
+trailer << /Info 2 0 R /Root 1 0 R /Size 17 /ID [<39c444018179222865caf844f4fb7254><39c444018179222865caf844f4fb7254>] /Encrypt 16 0 R >>
+startxref
+2899
+%%EOF

tools.go

@@ -1,10 +0,0 @@
-//go:build tools
-
-// Package sift pins the PDF engine chosen for P5a (docs/P5a.md §3) so go.mod and vendor/ carry it
-// before the code that imports it exists. Remove this file once internal/pdftext imports both.
-package main
-
-import (
-	_ "github.com/go-pdfkit/extract"
-	_ "github.com/go-pdfkit/reader"
-)