pdftext: extract PDF text in a re-exec'd child (sift pdftext) with a deadline
The parser is pure Go but a goroutine cannot be killed, so extraction runs in a child process: the binary re-executed as `sift pdftext`, reading the PDF on stdin and writing JSON Lines on stdout, killed at a deadline by the parent. pdftext.go holds the types, the five errors, Extract (reader.Open with ErrWrongPassword -> ErrEncrypted, per-page recover, trailer /Info /Title, and rule-3 UTF-16BE/Latin-1 text decode) and ChildMain (soft 768 MiB memory limit, -no env read, no network, exit codes 0/3/4/1). outline.go walks /Outlines -> /First -> /Next with /First children at level+1, resolving /Dest and /GoTo /A dests to 1-based pages, bounded at 10,000 items and never visiting an object twice. runner.go is the Runner: env-isolated child (nil env is an empty environment), 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code to error mapping, and Doc parsing with ErrNoText. cmd/sift/main.go adds the unlisted `pdftext` case. tools.go is removed; the go-pdfkit modules stay vendored. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
ornith-1.5-35b-a3b
@@ -0,0 +1,216 @@
|
||||
// Package pdftext extracts text, title and outline from a PDF. Extraction runs
|
||||
// in a child process (see Runner) so a page that hangs or panics cannot take
|
||||
// the parent down; Extract is the child's own work, usable directly in tests.
|
||||
package pdftext
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"runtime/debug"
|
||||
"strings"
|
||||
"unicode/utf16"
|
||||
|
||||
"github.com/go-pdfkit/extract"
|
||||
"github.com/go-pdfkit/reader"
|
||||
)
|
||||
|
||||
// The errors Extract and Runner can return.
|
||||
var (
|
||||
ErrEncrypted = errors.New("encrypted PDF (password required)")
|
||||
ErrUnreadable = errors.New("unreadable PDF")
|
||||
ErrNoText = errors.New("no text layer (scanned PDF?)")
|
||||
ErrTimeout = errors.New("pdf extraction timed out")
|
||||
ErrTooMuchText = errors.New("pdf text over the output limit")
|
||||
)
|
||||
|
||||
// OutlineItem is one bookmark: its title, the 1-based page it points at, and
|
||||
// its nesting level (1 = top level).
|
||||
type OutlineItem struct {
|
||||
Title string `json:"title"`
|
||||
Page int `json:"page"` // 1-based
|
||||
Level int `json:"level"` // 1 = top level
|
||||
}
|
||||
|
||||
// Meta is the document-level summary written as the first output line.
|
||||
type Meta struct {
|
||||
Pages int `json:"pages"` // pages in the document (not only those extracted)
|
||||
Title string `json:"title"` // /Info /Title, "" when absent
|
||||
Outline []OutlineItem `json:"outline"` // never nil: [] when the document has none
|
||||
}
|
||||
|
||||
// Doc is an extracted document: Text[i] is page i+1's text, for the pages
|
||||
// extracted (len(Text) ≤ Pages). Failed lists pages whose extraction failed
|
||||
// (their Text is "").
|
||||
type Doc struct {
|
||||
Meta
|
||||
Text []string
|
||||
Failed []int
|
||||
}
|
||||
|
||||
// metaLine is the first output line: the document summary.
|
||||
type metaLine struct {
|
||||
Meta Meta `json:"meta"`
|
||||
}
|
||||
|
||||
// pageOK and pageErr are the per-page lines. A page that extracts yields pageOK
|
||||
// even when its text is empty (an image-only page); one that fails yields
|
||||
// pageErr and extraction continues.
|
||||
type pageOK struct {
|
||||
Page int `json:"page"`
|
||||
Text string `json:"text"`
|
||||
}
|
||||
|
||||
type pageErr struct {
|
||||
Page int `json:"page"`
|
||||
Error string `json:"error"`
|
||||
}
|
||||
|
||||
// Extract writes one {"meta":{…}} line, then one {"page":n,"text":"…"} line per
|
||||
// page from 1 up to min(Pages, maxPages); a page that fails yields
|
||||
// {"page":n,"error":"…"} and extraction goes on. It returns ErrEncrypted when
|
||||
// the document needs a user password, ErrUnreadable when it cannot be opened or
|
||||
// has no pages, and never panics.
|
||||
func Extract(pdf []byte, maxPages int, w io.Writer) error {
|
||||
doc, err := reader.Open(pdf)
|
||||
if err != nil {
|
||||
if errors.Is(err, reader.ErrWrongPassword) {
|
||||
return ErrEncrypted
|
||||
}
|
||||
return ErrUnreadable
|
||||
}
|
||||
if doc.PageCount() == 0 {
|
||||
return ErrUnreadable
|
||||
}
|
||||
|
||||
meta := Meta{
|
||||
Pages: doc.PageCount(),
|
||||
Title: infoTitle(doc),
|
||||
Outline: walkOutline(doc),
|
||||
}
|
||||
if meta.Outline == nil {
|
||||
meta.Outline = []OutlineItem{}
|
||||
}
|
||||
if err := writeJSONLine(w, metaLine{Meta: meta}); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
n := doc.PageCount()
|
||||
if maxPages > 0 && n > maxPages {
|
||||
n = maxPages
|
||||
}
|
||||
for i := 1; i <= n; i++ {
|
||||
func() {
|
||||
defer func() {
|
||||
if r := recover(); r != nil {
|
||||
_ = writeJSONLine(w, pageErr{Page: i, Error: fmt.Sprintf("panic: %v", r)})
|
||||
}
|
||||
}()
|
||||
text, err := extract.Text(doc, i)
|
||||
if err != nil {
|
||||
_ = writeJSONLine(w, pageErr{Page: i, Error: err.Error()})
|
||||
return
|
||||
}
|
||||
_ = writeJSONLine(w, pageOK{Page: i, Text: text})
|
||||
}()
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// writeJSONLine marshals v and writes it as one JSON line.
|
||||
func writeJSONLine(w io.Writer, v any) error {
|
||||
b, err := json.Marshal(v)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if _, err := w.Write(b); err != nil {
|
||||
return err
|
||||
}
|
||||
_, err = w.Write([]byte{'\n'})
|
||||
return err
|
||||
}
|
||||
|
||||
// infoTitle returns the document title from the trailer's /Info /Title, decoded
|
||||
// as a PDF text string (rule 3), "" when absent.
|
||||
func infoTitle(doc *reader.Document) string {
|
||||
trailer := doc.Trailer()
|
||||
if trailer == nil {
|
||||
return ""
|
||||
}
|
||||
info, err := doc.Resolve(trailer.Get("Info"))
|
||||
if err != nil || info == nil {
|
||||
return ""
|
||||
}
|
||||
d, ok := reader.ToDict(info)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return textString(d.Get("Title"))
|
||||
}
|
||||
|
||||
// textString decodes a reader.String (raw bytes) as a PDF text string per rule 3.
|
||||
func textString(o reader.Object) string {
|
||||
s, ok := reader.ToString(o)
|
||||
if !ok {
|
||||
return ""
|
||||
}
|
||||
return decodePDFString(s)
|
||||
}
|
||||
|
||||
// decodePDFString follows rule 3: bytes starting FE FF are UTF-16BE (the rest is
|
||||
// decoded as such); otherwise each byte is treated as a Latin-1 code point.
|
||||
func decodePDFString(b []byte) string {
|
||||
if len(b) >= 2 && b[0] == 0xFE && b[1] == 0xFF {
|
||||
return strings.TrimSpace(string(utf16.Decode(utf16BE(b[2:]))))
|
||||
}
|
||||
return strings.TrimSpace(string(b))
|
||||
}
|
||||
|
||||
// utf16BE packs big-endian byte pairs into uint16s for utf16.Decode.
|
||||
func utf16BE(b []byte) []uint16 {
|
||||
n := len(b) / 2
|
||||
u := make([]uint16, n)
|
||||
for i := 0; i < 2*n; i += 2 {
|
||||
u[i/2] = uint16(b[i])<<8 | uint16(b[i+1])
|
||||
}
|
||||
return u
|
||||
}
|
||||
|
||||
// ChildMain is `sift pdftext`: flags (-max-pages N, default 2000), the PDF on
|
||||
// stdin (read with a 200 MiB cap), Extract to stdout. Exit codes: 0 ok, 3
|
||||
// ErrEncrypted, 4 ErrUnreadable, 1 anything else. It runs with a soft memory
|
||||
// limit, reads no environment variable, and opens no network connection.
|
||||
func ChildMain(args []string, stdin io.Reader, stdout io.Writer) int {
|
||||
debug.SetMemoryLimit(768 << 20)
|
||||
|
||||
fs := flag.NewFlagSet("pdftext", flag.ContinueOnError)
|
||||
maxPages := fs.Int("max-pages", 2000, "maximum number of pages to extract")
|
||||
if err := fs.Parse(args); err != nil {
|
||||
return 1
|
||||
}
|
||||
|
||||
pdf, err := io.ReadAll(io.LimitReader(stdin, 200<<20))
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, "pdftext: read input:", err)
|
||||
return 4
|
||||
}
|
||||
if len(pdf) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "pdftext: empty input")
|
||||
return 4
|
||||
}
|
||||
|
||||
if err := Extract(pdf, *maxPages, stdout); err != nil {
|
||||
switch {
|
||||
case errors.Is(err, ErrEncrypted):
|
||||
return 3
|
||||
case errors.Is(err, ErrUnreadable):
|
||||
return 4
|
||||
default:
|
||||
return 1
|
||||
}
|
||||
}
|
||||
return 0
|
||||
}
|
||||
Reference in New Issue
Block a user