pdftext: extract PDF text in a re-exec'd child (sift pdftext) with a deadline
The parser is pure Go but a goroutine cannot be killed, so extraction runs in a child process: the binary re-executed as `sift pdftext`, reading the PDF on stdin and writing JSON Lines on stdout, killed at a deadline by the parent. pdftext.go holds the types, the five errors, Extract (reader.Open with ErrWrongPassword -> ErrEncrypted, per-page recover, trailer /Info /Title, and rule-3 UTF-16BE/Latin-1 text decode) and ChildMain (soft 768 MiB memory limit, -no env read, no network, exit codes 0/3/4/1). outline.go walks /Outlines -> /First -> /Next with /First children at level+1, resolving /Dest and /GoTo /A dests to 1-based pages, bounded at 10,000 items and never visiting an object twice. runner.go is the Runner: env-isolated child (nil env is an empty environment), 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code to error mapping, and Doc parsing with ErrNoText. cmd/sift/main.go adds the unlisted `pdftext` case. tools.go is removed; the go-pdfkit modules stay vendored. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
ornith-1.5-35b-a3b
@@ -0,0 +1,133 @@
|
||||
// Package pdftest builds small, deterministic PDFs for tests: text pages in Helvetica, an optional
|
||||
// outline (bookmarks), and image-only ("scanned") pages. It exists so the P5a tests need no binary
|
||||
// fixtures from elsewhere. It is test support: nothing outside _test.go files imports it.
|
||||
package pdftest
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Outline is one bookmark: a title that points at a 1-based page, nested at Level (1 = top).
|
||||
// Build supports one level; Level is kept so tests can state what they expect back.
|
||||
type Outline struct {
|
||||
Title string
|
||||
Page int
|
||||
Level int
|
||||
}
|
||||
|
||||
// Doc describes a PDF to build. Pages holds each page's text; a page's lines are split on "\n"
|
||||
// and set one per line (long lines are not wrapped: callers keep them under ~90 characters).
|
||||
// Scanned pages (1-based) carry a small image and no text at all.
|
||||
type Doc struct {
|
||||
Title string
|
||||
Pages []string
|
||||
Outline []Outline
|
||||
Scanned []int
|
||||
}
|
||||
|
||||
// Build returns the PDF bytes for d. It panics on a Doc it cannot express (an outline entry
|
||||
// pointing past the last page), since that is a bug in the test that built it.
|
||||
func Build(d Doc) []byte {
|
||||
var b bytes.Buffer
|
||||
var offsets []int
|
||||
obj := func(body string) int {
|
||||
offsets = append(offsets, b.Len())
|
||||
n := len(offsets)
|
||||
fmt.Fprintf(&b, "%d 0 obj\n%s\nendobj\n", n, body)
|
||||
return n
|
||||
}
|
||||
b.WriteString("%PDF-1.4\n%\xe2\xe3\xcf\xd3\n")
|
||||
|
||||
// Object numbers are fixed by position: 1 catalog, 2 pages, 3 font, 4 image, 5 info, then
|
||||
// two per page (page, content), then the outline.
|
||||
nPages := len(d.Pages)
|
||||
firstPage := 6
|
||||
pageObj := func(i int) int { return firstPage + 2*i }
|
||||
outlineRoot := firstPage + 2*nPages
|
||||
scanned := map[int]bool{}
|
||||
for _, p := range d.Scanned {
|
||||
scanned[p] = true
|
||||
}
|
||||
|
||||
catalog := "<< /Type /Catalog /Pages 2 0 R"
|
||||
if len(d.Outline) > 0 {
|
||||
catalog += fmt.Sprintf(" /Outlines %d 0 R /PageMode /UseOutlines", outlineRoot)
|
||||
}
|
||||
obj(catalog + " >>")
|
||||
var kids []string
|
||||
for i := range d.Pages {
|
||||
kids = append(kids, fmt.Sprintf("%d 0 R", pageObj(i)))
|
||||
}
|
||||
obj(fmt.Sprintf("<< /Type /Pages /Kids [%s] /Count %d >>", strings.Join(kids, " "), nPages))
|
||||
obj("<< /Type /Font /Subtype /Type1 /BaseFont /Helvetica /Encoding /WinAnsiEncoding >>")
|
||||
img := []byte{0x00, 0x80, 0x80, 0x00} // 2x2 grey pixels
|
||||
obj(fmt.Sprintf("<< /Type /XObject /Subtype /Image /Width 2 /Height 2 /ColorSpace /DeviceGray /BitsPerComponent 8 /Length %d >>\nstream\n%s\nendstream", len(img), img))
|
||||
obj(fmt.Sprintf("<< /Title (%s) /Producer (sift pdftest) >>", esc(d.Title)))
|
||||
|
||||
for i, text := range d.Pages {
|
||||
var content string
|
||||
if scanned[i+1] {
|
||||
content = "q 400 0 0 500 100 150 cm /Im1 Do Q"
|
||||
} else {
|
||||
var c strings.Builder
|
||||
c.WriteString("BT /F1 11 Tf 13 TL 60 760 Td\n")
|
||||
for _, line := range strings.Split(text, "\n") {
|
||||
fmt.Fprintf(&c, "(%s) Tj T*\n", esc(line))
|
||||
}
|
||||
c.WriteString("ET")
|
||||
content = c.String()
|
||||
}
|
||||
obj(fmt.Sprintf("<< /Type /Page /Parent 2 0 R /MediaBox [0 0 612 792] /Resources << /Font << /F1 3 0 R >> /XObject << /Im1 4 0 R >> >> /Contents %d 0 R >>", pageObj(i)+1))
|
||||
obj(fmt.Sprintf("<< /Length %d >>\nstream\n%s\nendstream", len(content), content))
|
||||
}
|
||||
|
||||
if len(d.Outline) > 0 {
|
||||
first := outlineRoot + 1
|
||||
last := outlineRoot + len(d.Outline)
|
||||
obj(fmt.Sprintf("<< /Type /Outlines /First %d 0 R /Last %d 0 R /Count %d >>", first, last, len(d.Outline)))
|
||||
for i, o := range d.Outline {
|
||||
if o.Page < 1 || o.Page > nPages {
|
||||
panic(fmt.Sprintf("pdftest: outline %q points at page %d of %d", o.Title, o.Page, nPages))
|
||||
}
|
||||
n := first + i
|
||||
item := fmt.Sprintf("<< /Title (%s) /Parent %d 0 R /Dest [%d 0 R /XYZ null null null]", esc(o.Title), outlineRoot, pageObj(o.Page-1))
|
||||
if i > 0 {
|
||||
item += fmt.Sprintf(" /Prev %d 0 R", n-1)
|
||||
}
|
||||
if n < last {
|
||||
item += fmt.Sprintf(" /Next %d 0 R", n+1)
|
||||
}
|
||||
obj(item + " >>")
|
||||
}
|
||||
}
|
||||
|
||||
xref := b.Len()
|
||||
fmt.Fprintf(&b, "xref\n0 %d\n0000000000 65535 f \n", len(offsets)+1)
|
||||
for _, off := range offsets {
|
||||
fmt.Fprintf(&b, "%010d 00000 n \n", off)
|
||||
}
|
||||
fmt.Fprintf(&b, "trailer\n<< /Size %d /Root 1 0 R /Info 5 0 R >>\nstartxref\n%d\n%%%%EOF\n", len(offsets)+1, xref)
|
||||
return b.Bytes()
|
||||
}
|
||||
|
||||
// Lorem returns n lines of distinct, searchable filler: "p<page> l<line> …", so a test can tell
|
||||
// which page a line came from.
|
||||
func Lorem(page, n int) string {
|
||||
words := []string{"alpha", "bravo", "charlie", "delta", "echo", "foxtrot", "golf", "hotel", "india", "juliet", "kilo", "lima"}
|
||||
var lines []string
|
||||
for i := 1; i <= n; i++ {
|
||||
var w []string
|
||||
for j := 0; j < 10; j++ {
|
||||
w = append(w, words[(page*7+i*3+j)%len(words)])
|
||||
}
|
||||
lines = append(lines, fmt.Sprintf("p%d l%d %s", page, i, strings.Join(w, " ")))
|
||||
}
|
||||
return strings.Join(lines, "\n")
|
||||
}
|
||||
|
||||
func esc(s string) string {
|
||||
r := strings.NewReplacer(`\`, `\\`, `(`, `\(`, `)`, `\)`)
|
||||
return r.Replace(s)
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
package pdftext
|
||||
|
||||
import (
|
||||
"github.com/go-pdfkit/reader"
|
||||
)
|
||||
|
||||
// maxOutlineItems bounds a walk over a malformed outline that claims more
|
||||
// entries than the file could hold.
|
||||
const maxOutlineItems = 10000
|
||||
|
||||
// walkOutline returns the document's outline as a depth-first, pre-order list:
|
||||
// each item before its own children. The result is never nil.
|
||||
func walkOutline(doc *reader.Document) []OutlineItem {
|
||||
out := []OutlineItem{}
|
||||
trailer := doc.Trailer()
|
||||
if trailer == nil {
|
||||
return out
|
||||
}
|
||||
cat, err := doc.Resolve(trailer.Get("Root"))
|
||||
if err != nil || cat == nil {
|
||||
return out
|
||||
}
|
||||
catDict, ok := reader.ToDict(cat)
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
outlines, err := doc.Resolve(catDict.Get("Outlines"))
|
||||
if err != nil || outlines == nil {
|
||||
return out
|
||||
}
|
||||
outlinesDict, ok := reader.ToDict(outlines)
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
first, ok := toRef(outlinesDict.Get("First"))
|
||||
if !ok {
|
||||
return out
|
||||
}
|
||||
seen := map[reader.Ref]bool{}
|
||||
walkChildren(doc, first, 1, seen, &out)
|
||||
return out
|
||||
}
|
||||
|
||||
// walkChildren walks the /Next chain from ref at the given level, emitting each
|
||||
// resolvable item and recursing into its /First children (level + 1). It stops
|
||||
// after maxOutlineItems items and never visits an object twice.
|
||||
func walkChildren(doc *reader.Document, ref reader.Ref, level int, seen map[reader.Ref]bool, out *[]OutlineItem) {
|
||||
cur := ref
|
||||
for cur != (reader.Ref{}) {
|
||||
if seen[cur] {
|
||||
break
|
||||
}
|
||||
seen[cur] = true
|
||||
|
||||
obj, err := doc.Resolve(cur)
|
||||
if err != nil || obj == nil {
|
||||
break
|
||||
}
|
||||
d, ok := reader.ToDict(obj)
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
|
||||
if page, ok := destPage(doc, d); ok {
|
||||
*out = append(*out, OutlineItem{Title: textString(d.Get("Title")), Page: page, Level: level})
|
||||
if len(*out) >= maxOutlineItems {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
if child, ok := toRef(d.Get("First")); ok {
|
||||
walkChildren(doc, child, level+1, seen, out)
|
||||
if len(*out) >= maxOutlineItems {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
next, ok := toRef(d.Get("Next"))
|
||||
if !ok {
|
||||
break
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
}
|
||||
|
||||
// destPage resolves an item's destination to a 1-based page number. It reports
|
||||
// ok only when the destination is an array whose first element is a page
|
||||
// reference that maps to a real page; a named or otherwise unresolvable
|
||||
// destination is not ok, so the caller skips the item.
|
||||
func destPage(doc *reader.Document, d reader.Dict) (int, bool) {
|
||||
if ref, ok := destRef(doc, d); ok {
|
||||
if page, ok := pageOf(doc, ref); ok {
|
||||
return page, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// destRef finds the destination page reference, from /Dest or from a /GoTo
|
||||
// action's /D.
|
||||
func destRef(doc *reader.Document, d reader.Dict) (reader.Ref, bool) {
|
||||
if o := d.Get("Dest"); o != nil {
|
||||
if arr, ok := reader.ToArray(o); ok && len(arr) > 0 {
|
||||
if ref, ok := arr[0].(reader.Ref); ok {
|
||||
return ref, true
|
||||
}
|
||||
}
|
||||
}
|
||||
if o := d.Get("A"); o != nil {
|
||||
if a, ok := reader.ToDict(o); ok {
|
||||
if s, ok := reader.ToName(a.Get("S")); ok && s == "GoTo" {
|
||||
if d2 := a.Get("D"); d2 != nil {
|
||||
if arr, ok := reader.ToArray(d2); ok && len(arr) > 0 {
|
||||
if ref, ok := arr[0].(reader.Ref); ok {
|
||||
return ref, true
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return reader.Ref{}, false
|
||||
}
|
||||
|
||||
// pageOf maps a reference to its 1-based page by comparing it with each page's
|
||||
// reference.
|
||||
func pageOf(doc *reader.Document, ref reader.Ref) (int, bool) {
|
||||
for i := 1; i <= doc.PageCount(); i++ {
|
||||
pr, ok := doc.PageRef(i)
|
||||
if ok && pr == ref {
|
||||
return i, true
|
||||
}
|
||||
}
|
||||
return 0, false
|
||||
}
|
||||
|
||||
// toRef reports the reference a dictionary entry names, when it is a direct
|
||||
// reference rather than an already-resolved object.
|
||||
func toRef(o reader.Object) (reader.Ref, bool) {
|
||||
ref, ok := o.(reader.Ref)
|
||||
return ref, ok
|
||||
}
|
||||
@@ -0,0 +1,216 @@
|
||||
// Package pdftext extracts text, title and outline from a PDF. Extraction runs
|
||||
// in a child process (see Runner) so a page that hangs or panics cannot take
|
||||
// the parent down; Extract is the child's own work, usable directly in tests.
|
||||
package pdftext
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"unicode/utf16"
|
||||
|
||||
"github.com/go-pdfkit/extract"
|
||||
"github.com/go-pdfkit/reader"
|
||||
)
|
||||
|
||||
// The errors Extract and Runner can return.
|
||||
var (
|
||||
ErrEncrypted = errors.New("encrypted PDF (password required)")
|
||||
ErrUnreadable = errors.New("unreadable PDF")
|
||||
ErrNoText = errors.New("no text layer (scanned PDF?)")
|
||||
ErrTimeout = errors.New("pdf extraction timed out")
|
||||
ErrTooMuchText = errors.New("pdf text over the output limit")
|
||||
)
|
||||
|
||||
// OutlineItem is one bookmark: its title, the 1-based page it points at, and
|
||||
// its nesting level (1 = top level).
|
||||
type OutlineItem struct {
|
||||
Title string `json:"title"`
|
||||
Page int `json:"page"` // 1-based
|
||||
Level int `json:"level"` // 1 = top level
|
||||
}
|
||||
|
||||
// Meta is the document-level summary written as the first output line.
|
||||
type Meta struct {
|
||||
Pages int `json:"pages"` // pages in the document (not only those extracted)
|
||||
Title string `json:"title"` // /Info /Title, "" when absent
|
||||
Outline []OutlineItem `json:"outline"` // never nil: [] when the document has none
|
||||
}
|
||||
|
||||
// Doc is an extracted document: Text[i] is page i+1's text, for the pages
|
||||
// extracted (len(Text) ≤ Pages). Failed lists pages whose extraction failed
|
||||
// (their Text is "").
|
||||
type Doc struct {
|
||||
Meta
|
||||
Text []string
|
||||
Failed []int
|
||||
}
|
||||
|
||||
// metaLine is the first output line: the document summary.
|
||||
type metaLine struct {
|
||||
Meta Meta `json:"meta"`
|
||||
}
|
||||
|
||||
// pageOK and pageErr are the per-page lines. A page that extracts yields pageOK
|
||||
// even when its text is empty (an image-only page); one that fails yields
|
||||
// pageErr and extraction continues.
|
||||
type pageOK struct {
|
||||
Page int `json:"page"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type pageErr struct {
|
||||
Page int `json:"page"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// Extract writes one {"meta":{…}} line, then one {"page":n,"text":"…"} line per
|
||||
// page from 1 up to min(Pages, maxPages); a page that fails yields
|
||||
// {"page":n,"error":"…"} and extraction goes on. It returns ErrEncrypted when
|
||||
// the document needs a user password, ErrUnreadable when it cannot be opened or
|
||||
// has no pages, and never panics.
|
||||
func Extract(pdf []byte, maxPages int, w io.Writer) error {
|
||||
doc, err := reader.Open(pdf)
|
||||
if err != nil {
|
||||
if errors.Is(err, reader.ErrWrongPassword) {
|
||||
return ErrEncrypted
|
||||
}
|
||||
return ErrUnreadable
|
||||
}
|
||||
if doc.PageCount() == 0 {
|
||||
return ErrUnreadable
|
||||
}
|
||||
|
||||
meta := Meta{
|
||||
Pages: doc.PageCount(),
|
||||
Title: infoTitle(doc),
|
||||
Outline: walkOutline(doc),
|
||||
}
|
||||
if meta.Outline == nil {
|
||||
meta.Outline = []OutlineItem{}
|
||||
}
|
||||
if err := writeJSONLine(w, metaLine{Meta: meta}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n := doc.PageCount()
|
||||
if maxPages > 0 && n > maxPages {
|
||||
n = maxPages
|
||||
}
|
||||
for i := 1; i <= n; i++ {
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
_ = writeJSONLine(w, pageErr{Page: i, Error: fmt.Sprintf("panic: %v", r)})
|
||||
}
|
||||
}()
|
||||
text, err := extract.Text(doc, i)
|
||||
if err != nil {
|
||||
_ = writeJSONLine(w, pageErr{Page: i, Error: err.Error()})
|
||||
return
|
||||
}
|
||||
_ = writeJSONLine(w, pageOK{Page: i, Text: text})
|
||||
}()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeJSONLine marshals v and writes it as one JSON line.
|
||||
func writeJSONLine(w io.Writer, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write([]byte{'\n'})
|
||||
return err
|
||||
}
|
||||
|
||||
// infoTitle returns the document title from the trailer's /Info /Title, decoded
|
||||
// as a PDF text string (rule 3), "" when absent.
|
||||
func infoTitle(doc *reader.Document) string {
|
||||
trailer := doc.Trailer()
|
||||
if trailer == nil {
|
||||
return ""
|
||||
}
|
||||
info, err := doc.Resolve(trailer.Get("Info"))
|
||||
if err != nil || info == nil {
|
||||
return ""
|
||||
}
|
||||
d, ok := reader.ToDict(info)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return textString(d.Get("Title"))
|
||||
}
|
||||
|
||||
// textString decodes a reader.String (raw bytes) as a PDF text string per rule 3.
|
||||
func textString(o reader.Object) string {
|
||||
s, ok := reader.ToString(o)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return decodePDFString(s)
|
||||
}
|
||||
|
||||
// decodePDFString follows rule 3: bytes starting FE FF are UTF-16BE (the rest is
|
||||
// decoded as such); otherwise each byte is treated as a Latin-1 code point.
|
||||
func decodePDFString(b []byte) string {
|
||||
if len(b) >= 2 && b[0] == 0xFE && b[1] == 0xFF {
|
||||
return strings.TrimSpace(string(utf16.Decode(utf16BE(b[2:]))))
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
// utf16BE packs big-endian byte pairs into uint16s for utf16.Decode.
|
||||
func utf16BE(b []byte) []uint16 {
|
||||
n := len(b) / 2
|
||||
u := make([]uint16, n)
|
||||
for i := 0; i < 2*n; i += 2 {
|
||||
u[i/2] = uint16(b[i])<<8 | uint16(b[i+1])
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// ChildMain is `sift pdftext`: flags (-max-pages N, default 2000), the PDF on
|
||||
// stdin (read with a 200 MiB cap), Extract to stdout. Exit codes: 0 ok, 3
|
||||
// ErrEncrypted, 4 ErrUnreadable, 1 anything else. It runs with a soft memory
|
||||
// limit, reads no environment variable, and opens no network connection.
|
||||
func ChildMain(args []string, stdin io.Reader, stdout io.Writer) int {
|
||||
debug.SetMemoryLimit(768 << 20)
|
||||
|
||||
fs := flag.NewFlagSet("pdftext", flag.ContinueOnError)
|
||||
maxPages := fs.Int("max-pages", 2000, "maximum number of pages to extract")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
pdf, err := io.ReadAll(io.LimitReader(stdin, 200<<20))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "pdftext: read input:", err)
|
||||
return 4
|
||||
}
|
||||
if len(pdf) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "pdftext: empty input")
|
||||
return 4
|
||||
}
|
||||
|
||||
if err := Extract(pdf, *maxPages, stdout); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, ErrEncrypted):
|
||||
return 3
|
||||
case errors.Is(err, ErrUnreadable):
|
||||
return 4
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -0,0 +1,252 @@
|
||||
package pdftext_test
|
||||
|
||||
// P5a task 02: Extract (the child's work), ChildMain (the `sift pdftext` entry point) and Runner
|
||||
// (the parent: exec, deadline, kill, parse).
|
||||
//
|
||||
// The Runner tests run the test binary itself as the child: TestMain turns into the child when
|
||||
// SIFT_PDFTEXT_CHILD=1 is set, exactly as `sift pdftext` would. The two other variables are test
|
||||
// hooks that live only here: SIFT_PDFTEXT_SLEEP delays the child (for the deadline test) and
|
||||
// SIFT_PDFTEXT_PIDFILE records its pid (to prove it is gone afterwards).
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"strings"
|
||||
"syscall"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/pdftest"
|
||||
"git.wntrmute.dev/kyle/sift/internal/pdftext"
|
||||
)
|
||||
|
||||
func TestMain(m *testing.M) {
|
||||
if os.Getenv("SIFT_PDFTEXT_CHILD") == "1" {
|
||||
if p := os.Getenv("SIFT_PDFTEXT_PIDFILE"); p != "" {
|
||||
_ = os.WriteFile(p, []byte(strconv.Itoa(os.Getpid())), 0o600)
|
||||
}
|
||||
if d, err := time.ParseDuration(os.Getenv("SIFT_PDFTEXT_SLEEP")); err == nil {
|
||||
time.Sleep(d)
|
||||
}
|
||||
os.Exit(pdftext.ChildMain(os.Args[1:], os.Stdin, os.Stdout))
|
||||
}
|
||||
os.Exit(m.Run())
|
||||
}
|
||||
|
||||
func three() []byte {
|
||||
return pdftest.Build(pdftest.Doc{
|
||||
Title: "Three Pages",
|
||||
Pages: []string{
|
||||
"Chapter one begins here.\nSecond line (with parens).",
|
||||
"The quick brown fox is on page two.",
|
||||
pdftest.Lorem(3, 5),
|
||||
},
|
||||
Outline: []pdftest.Outline{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "3 Filler", Page: 3, Level: 1}},
|
||||
})
|
||||
}
|
||||
|
||||
type line struct {
|
||||
Meta *pdftext.Meta `json:"meta"`
|
||||
Page int `json:"page"`
|
||||
Text *string `json:"text"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
func lines(t *testing.T, out []byte) []line {
|
||||
t.Helper()
|
||||
var ls []line
|
||||
sc := bufio.NewScanner(bytes.NewReader(out))
|
||||
sc.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
for sc.Scan() {
|
||||
var l line
|
||||
if err := json.Unmarshal(sc.Bytes(), &l); err != nil {
|
||||
t.Fatalf("output line is not JSON: %q (%v)", sc.Text(), err)
|
||||
}
|
||||
ls = append(ls, l)
|
||||
}
|
||||
return ls
|
||||
}
|
||||
|
||||
func TestExtractWritesMetaThenPages(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := pdftext.Extract(three(), 2000, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ls := lines(t, out.Bytes())
|
||||
if len(ls) != 4 || ls[0].Meta == nil {
|
||||
t.Fatalf("got %d lines, first meta=%v; want a meta line then 3 page lines:\n%s", len(ls), ls[0].Meta, out.String())
|
||||
}
|
||||
m := ls[0].Meta
|
||||
if m.Pages != 3 || m.Title != "Three Pages" {
|
||||
t.Errorf("meta = %+v, want 3 pages titled Three Pages", *m)
|
||||
}
|
||||
want := []pdftext.OutlineItem{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "3 Filler", Page: 3, Level: 1}}
|
||||
if len(m.Outline) != 3 {
|
||||
t.Fatalf("outline = %+v, want %+v", m.Outline, want)
|
||||
}
|
||||
for i := range want {
|
||||
if m.Outline[i] != want[i] {
|
||||
t.Errorf("outline[%d] = %+v, want %+v", i, m.Outline[i], want[i])
|
||||
}
|
||||
}
|
||||
for i, l := range ls[1:] {
|
||||
if l.Page != i+1 || l.Text == nil || l.Error != "" {
|
||||
t.Errorf("line %d = %+v, want page %d with text", i+1, l, i+1)
|
||||
}
|
||||
}
|
||||
if !strings.Contains(*ls[1].Text, "Chapter one begins here.") || !strings.Contains(*ls[2].Text, "quick brown fox") || !strings.Contains(*ls[3].Text, "p3 l5") {
|
||||
t.Errorf("page texts wrong: %q / %q / %q", *ls[1].Text, *ls[2].Text, *ls[3].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractPageCap(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := pdftext.Extract(three(), 2, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ls := lines(t, out.Bytes())
|
||||
if len(ls) != 3 || ls[0].Meta.Pages != 3 || ls[2].Page != 2 {
|
||||
t.Errorf("maxPages 2: %d lines, meta %+v; want meta (pages 3) and pages 1-2", len(ls), ls[0].Meta)
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractNoOutline(t *testing.T) {
|
||||
var out bytes.Buffer
|
||||
if err := pdftext.Extract(pdftest.Build(pdftest.Doc{Pages: []string{"a", "b"}}), 10, &out); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ls := lines(t, out.Bytes())
|
||||
if ls[0].Meta.Outline == nil || len(ls[0].Meta.Outline) != 0 {
|
||||
t.Errorf("outline = %#v, want an empty non-nil slice (JSON [])", ls[0].Meta.Outline)
|
||||
}
|
||||
if !strings.Contains(out.String(), `"outline":[]`) {
|
||||
t.Errorf("meta line must carry \"outline\":[]: %s", out.String())
|
||||
}
|
||||
}
|
||||
|
||||
func TestExtractErrors(t *testing.T) {
|
||||
enc, err := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := pdftext.Extract(enc, 10, &bytes.Buffer{}); !errors.Is(err, pdftext.ErrEncrypted) {
|
||||
t.Errorf("user-password pdf: %v, want ErrEncrypted", err)
|
||||
}
|
||||
owner, err := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-owner.pdf"))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var out bytes.Buffer
|
||||
if err := pdftext.Extract(owner, 10, &out); err != nil || !strings.Contains(out.String(), "quick brown fox") {
|
||||
t.Errorf("owner-password-only pdf: err %v; want its text", err)
|
||||
}
|
||||
if err := pdftext.Extract([]byte("not a pdf at all"), 10, &bytes.Buffer{}); !errors.Is(err, pdftext.ErrUnreadable) {
|
||||
t.Errorf("garbage: %v, want ErrUnreadable", err)
|
||||
}
|
||||
// A truncated file either yields what it can or ErrUnreadable; it never panics.
|
||||
b := three()
|
||||
_ = pdftext.Extract(b[:len(b)*6/10], 10, &bytes.Buffer{})
|
||||
}
|
||||
|
||||
func TestChildMainExitCodes(t *testing.T) {
|
||||
enc, _ := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf"))
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
in []byte
|
||||
code int
|
||||
}{
|
||||
{"ok", three(), 0},
|
||||
{"encrypted", enc, 3},
|
||||
{"unreadable", []byte("nope"), 4},
|
||||
} {
|
||||
var out bytes.Buffer
|
||||
if code := pdftext.ChildMain([]string{"-max-pages", "5"}, bytes.NewReader(tc.in), &out); code != tc.code {
|
||||
t.Errorf("%s: exit %d, want %d", tc.name, code, tc.code)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func runner(env ...string) *pdftext.Runner {
|
||||
return &pdftext.Runner{Exe: os.Args[0], Env: append([]string{"SIFT_PDFTEXT_CHILD=1"}, env...), Deadline: 20 * time.Second}
|
||||
}
|
||||
|
||||
func TestRunnerRun(t *testing.T) {
|
||||
doc, err := runner().Run(context.Background(), three())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if doc.Pages != 3 || doc.Title != "Three Pages" || len(doc.Outline) != 3 || len(doc.Text) != 3 {
|
||||
t.Fatalf("doc = %+v", doc)
|
||||
}
|
||||
if !strings.Contains(doc.Text[1], "quick brown fox") {
|
||||
t.Errorf("page 2 = %q", doc.Text[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunnerErrors(t *testing.T) {
|
||||
enc, _ := os.ReadFile(filepath.Join("..", "..", "tests", "testdata", "pdf", "encrypted-user.pdf"))
|
||||
if _, err := runner().Run(context.Background(), enc); !errors.Is(err, pdftext.ErrEncrypted) {
|
||||
t.Errorf("encrypted: %v, want ErrEncrypted", err)
|
||||
}
|
||||
if _, err := runner().Run(context.Background(), []byte("nope")); !errors.Is(err, pdftext.ErrUnreadable) {
|
||||
t.Errorf("garbage: %v, want ErrUnreadable", err)
|
||||
}
|
||||
scanned := pdftest.Build(pdftest.Doc{Pages: []string{"", "", ""}, Scanned: []int{1, 2, 3}})
|
||||
if _, err := runner().Run(context.Background(), scanned); !errors.Is(err, pdftext.ErrNoText) {
|
||||
t.Errorf("scanned: %v, want ErrNoText", err)
|
||||
}
|
||||
r := runner()
|
||||
r.MaxText = 100 // bytes of child output
|
||||
if _, err := r.Run(context.Background(), three()); !errors.Is(err, pdftext.ErrTooMuchText) {
|
||||
t.Errorf("output over MaxText: %v, want ErrTooMuchText", err)
|
||||
}
|
||||
}
|
||||
|
||||
// The child is killed at the deadline and reaped: no process is left behind.
|
||||
func TestRunnerDeadlineKillsTheChild(t *testing.T) {
|
||||
pidfile := filepath.Join(t.TempDir(), "pid")
|
||||
r := runner("SIFT_PDFTEXT_SLEEP=30s", "SIFT_PDFTEXT_PIDFILE="+pidfile)
|
||||
r.Deadline = 300 * time.Millisecond
|
||||
start := time.Now()
|
||||
_, err := r.Run(context.Background(), three())
|
||||
if !errors.Is(err, pdftext.ErrTimeout) {
|
||||
t.Fatalf("err %v, want ErrTimeout", err)
|
||||
}
|
||||
if d := time.Since(start); d > 3*time.Second {
|
||||
t.Errorf("Run returned after %v; the 300 ms deadline was not enforced", d)
|
||||
}
|
||||
b, err := os.ReadFile(pidfile)
|
||||
if err != nil {
|
||||
t.Fatalf("child never wrote its pid: %v", err)
|
||||
}
|
||||
pid, _ := strconv.Atoi(string(b))
|
||||
if err := syscall.Kill(pid, 0); !errors.Is(err, syscall.ESRCH) {
|
||||
t.Errorf("child %d still exists after Run returned (kill -0: %v)", pid, err)
|
||||
}
|
||||
}
|
||||
|
||||
// The caller's context cancels the child too.
|
||||
func TestRunnerContextCancel(t *testing.T) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 300*time.Millisecond)
|
||||
defer cancel()
|
||||
_, err := runner("SIFT_PDFTEXT_SLEEP=30s").Run(ctx, three())
|
||||
if err == nil {
|
||||
t.Fatal("cancelled run succeeded")
|
||||
}
|
||||
}
|
||||
|
||||
// Runner passes only Env to the child: nothing of the parent's environment leaks in.
|
||||
func TestRunnerEnvIsExplicit(t *testing.T) {
|
||||
t.Setenv("SIFT_PDFTEXT_SLEEP", "30s") // in the parent only
|
||||
r := runner()
|
||||
r.Deadline = 5 * time.Second
|
||||
if _, err := r.Run(context.Background(), three()); err != nil {
|
||||
t.Errorf("parent env leaked into the child (it slept): %v", err)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package pdftext
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
// defaultDeadline bounds a child that never finishes.
|
||||
defaultDeadline = 60 * time.Second
|
||||
// defaultMaxText caps child output before it is rejected.
|
||||
defaultMaxText = 64 << 20 // 64 MiB
|
||||
// defaultMaxPages caps how many pages the child extracts.
|
||||
defaultMaxPages = 2000
|
||||
// stderrCap keeps only the first part of the child's stderr, for messages.
|
||||
stderrCap = 4096
|
||||
)
|
||||
|
||||
// Runner executes the extraction child and parses its output into a Doc.
|
||||
type Runner struct {
|
||||
Exe string // "" = /proc/self/exe
|
||||
Args []string // nil = []string{"pdftext"}; -max-pages is appended
|
||||
Env []string // the child's whole environment; nil = none
|
||||
Deadline time.Duration // 0 = 60 s
|
||||
MaxText int64 // bytes of child output; 0 = 64 MiB
|
||||
MaxPages int // 0 = 2000
|
||||
}
|
||||
|
||||
// Run executes the child, writes pdf to its stdin, and parses its output into a
|
||||
// Doc. The child is killed at the deadline (or when ctx is done, whichever is
|
||||
// sooner) and always reaped before Run returns.
|
||||
func (r *Runner) Run(ctx context.Context, pdf []byte) (*Doc, error) {
|
||||
deadline := r.Deadline
|
||||
if deadline == 0 {
|
||||
deadline = defaultDeadline
|
||||
}
|
||||
maxText := r.MaxText
|
||||
if maxText == 0 {
|
||||
maxText = defaultMaxText
|
||||
}
|
||||
maxPages := r.MaxPages
|
||||
if maxPages == 0 {
|
||||
maxPages = defaultMaxPages
|
||||
}
|
||||
|
||||
ctx, cancel := context.WithDeadline(ctx, time.Now().Add(deadline))
|
||||
defer cancel()
|
||||
|
||||
cmd := exec.CommandContext(ctx, r.exe(), r.args(maxPages)...)
|
||||
if r.Env == nil {
|
||||
cmd.Env = []string{}
|
||||
} else {
|
||||
cmd.Env = r.Env
|
||||
}
|
||||
|
||||
stdin, err := cmd.StdinPipe()
|
||||
if err != nil {
|
||||
return nil, wrapUnreadable(err)
|
||||
}
|
||||
stdout, err := cmd.StdoutPipe()
|
||||
if err != nil {
|
||||
return nil, wrapUnreadable(err)
|
||||
}
|
||||
stderr, err := cmd.StderrPipe()
|
||||
if err != nil {
|
||||
return nil, wrapUnreadable(err)
|
||||
}
|
||||
|
||||
var stdoutBuf, stderrBuf bytes.Buffer
|
||||
done := make(chan error, 1)
|
||||
go func() {
|
||||
// One extra byte so output over the limit is distinguishable from
|
||||
// output exactly at it.
|
||||
_, e := io.Copy(&stdoutBuf, io.LimitReader(stdout, maxText+1))
|
||||
done <- e
|
||||
}()
|
||||
stderrDone := make(chan struct{})
|
||||
go func() {
|
||||
io.Copy(&limitedWriter{n: stderrCap, w: &stderrBuf}, stderr)
|
||||
close(stderrDone)
|
||||
}()
|
||||
go func() {
|
||||
if _, err := stdin.Write(pdf); err != nil {
|
||||
return
|
||||
}
|
||||
_ = stdin.Close()
|
||||
}()
|
||||
|
||||
if err := cmd.Start(); err != nil {
|
||||
cancel()
|
||||
_ = stdout.Close()
|
||||
_ = stderr.Close()
|
||||
<-done
|
||||
<-stderrDone
|
||||
return nil, wrapUnreadable(err)
|
||||
}
|
||||
|
||||
readErr := <-done
|
||||
<-stderrDone
|
||||
|
||||
// Output over the limit: kill the child and report.
|
||||
if int64(stdoutBuf.Len()) > maxText {
|
||||
cancel()
|
||||
_ = cmd.Wait()
|
||||
return nil, ErrTooMuchText
|
||||
}
|
||||
|
||||
waitErr := cmd.Wait()
|
||||
|
||||
// A deadline reached while the child ran is a timeout, even if the kill
|
||||
// surfaced as some other error.
|
||||
if ctx.Err() == context.DeadlineExceeded {
|
||||
return nil, ErrTimeout
|
||||
}
|
||||
if readErr != nil && !errors.Is(readErr, io.EOF) {
|
||||
return nil, wrapUnreadable(fmt.Errorf("read output: %w", readErr))
|
||||
}
|
||||
if waitErr != nil {
|
||||
exitErr, ok := waitErr.(*exec.ExitError)
|
||||
if !ok {
|
||||
return nil, wrapUnreadable(fmt.Errorf("child: %v", waitErr))
|
||||
}
|
||||
switch exitErr.ExitCode() {
|
||||
case 3:
|
||||
return nil, ErrEncrypted
|
||||
case 4:
|
||||
return nil, ErrUnreadable
|
||||
default:
|
||||
return nil, wrapUnreadable([]byte(firstLine(stderrBuf.Bytes())))
|
||||
}
|
||||
}
|
||||
|
||||
doc, err := parseDoc(stdoutBuf.Bytes(), maxPages)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
// exe resolves the executable to run, defaulting to this process's own path.
|
||||
func (r *Runner) exe() string {
|
||||
if r.Exe != "" {
|
||||
return r.Exe
|
||||
}
|
||||
if e, err := os.Readlink("/proc/self/exe"); err == nil {
|
||||
return e
|
||||
}
|
||||
return "sift"
|
||||
}
|
||||
|
||||
// args builds the child's argument list: the configured args, or "pdftext",
|
||||
// with -max-pages appended.
|
||||
func (r *Runner) args(maxPages int) []string {
|
||||
args := []string{"pdftext"}
|
||||
if r.Args != nil {
|
||||
args = r.Args
|
||||
}
|
||||
return append(args, "-max-pages", strconv.Itoa(maxPages))
|
||||
}
|
||||
|
||||
// parseDoc turns the child's JSON Lines into a Doc. The first line is the meta;
|
||||
// each following line is a page's text or its error. It returns ErrNoText when
|
||||
// no page was extracted or every extracted page's text is blank, and
|
||||
// ErrUnreadable when the output is not what the child should have produced.
|
||||
func parseDoc(data []byte, maxPages int) (*Doc, error) {
|
||||
sc := bufio.NewScanner(bytes.NewReader(data))
|
||||
sc.Buffer(make([]byte, 1<<20), 64<<20)
|
||||
|
||||
var doc Doc
|
||||
for sc.Scan() {
|
||||
var line struct {
|
||||
Meta *Meta `json:"meta"`
|
||||
Page int `json:"page"`
|
||||
Text *string `json:"text"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
if err := json.Unmarshal(sc.Bytes(), &line); err != nil {
|
||||
return nil, wrapUnreadable(nil)
|
||||
}
|
||||
if line.Meta != nil {
|
||||
doc.Meta = *line.Meta
|
||||
continue
|
||||
}
|
||||
appendPage(&doc, line.Page, line.Text, line.Error)
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, wrapUnreadable(nil)
|
||||
}
|
||||
|
||||
if len(doc.Text) == 0 {
|
||||
return nil, ErrNoText
|
||||
}
|
||||
for _, t := range doc.Text {
|
||||
if strings.TrimSpace(t) != "" {
|
||||
return &doc, nil
|
||||
}
|
||||
}
|
||||
return nil, ErrNoText
|
||||
}
|
||||
|
||||
// appendPage records one page's result, growing Text to fit page n.
|
||||
func appendPage(doc *Doc, n int, text *string, errMsg string) {
|
||||
for len(doc.Text) < n {
|
||||
doc.Text = append(doc.Text, "")
|
||||
}
|
||||
if text != nil {
|
||||
doc.Text[n-1] = *text
|
||||
return
|
||||
}
|
||||
doc.Text[n-1] = ""
|
||||
if errMsg != "" {
|
||||
doc.Failed = append(doc.Failed, n)
|
||||
}
|
||||
}
|
||||
|
||||
// wrapUnreadable wraps a cause (an error or a chunk of stderr) as ErrUnreadable.
|
||||
func wrapUnreadable(v any) error {
|
||||
switch s := v.(type) {
|
||||
case nil:
|
||||
return ErrUnreadable
|
||||
case error:
|
||||
return fmt.Errorf("%w: %v", ErrUnreadable, s)
|
||||
case []byte:
|
||||
return fmt.Errorf("%w: %s", ErrUnreadable, firstLine(s))
|
||||
case string:
|
||||
return fmt.Errorf("%w: %s", ErrUnreadable, s)
|
||||
}
|
||||
return ErrUnreadable
|
||||
}
|
||||
|
||||
// firstLine returns the first line of b, or b when it has no newline.
|
||||
func firstLine(b []byte) string {
|
||||
if i := bytes.IndexByte(b, '\n'); i >= 0 {
|
||||
return string(b[:i])
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
// limitedWriter writes at most n bytes, dropping the rest so a child that
|
||||
// overflows stderr cannot block the parent.
|
||||
type limitedWriter struct {
|
||||
n int64
|
||||
w io.Writer
|
||||
}
|
||||
|
||||
func (l *limitedWriter) Write(p []byte) (int, error) {
|
||||
if l.n <= 0 {
|
||||
return 0, nil
|
||||
}
|
||||
if int64(len(p)) > l.n {
|
||||
p = p[:l.n]
|
||||
}
|
||||
n, err := l.w.Write(p)
|
||||
l.n -= int64(n)
|
||||
if l.n < 0 {
|
||||
l.n = 0
|
||||
}
|
||||
return n, err
|
||||
}
|
||||
Reference in New Issue
Block a user