P5a plan: PDF documents — seven Ornith tasks with acceptance tests first
docs/plans/p5a: 01 kind+fetch limits, 02 pdftext child + runner, 03 dcache PDF tables, 04 pdfpool, 05 pdfdoc render (map/pages/grep), 06 api, 07 digest citations, metrics, CLI, skill, unit, README. Given tests compile against a panic-only skeleton and fail there for the intended reasons; PDFs for tests are built by internal/pdftest (given), plus two qpdf-encrypted fixtures. Owner changes for the run: vendor go-pdfkit/extract v0.3.0 + reader v0.6.0 (pinned via tools.go until task 02); AGENTS.md, tools/run-plan.sh (enforces a Co-Authored-By trailer for the model), docs/implementer-log.md, `make gate`; gofmt (whitespace) on clean.go and guard; TestUnsupportedType uses a zip now that PDFs are supported; requestDeadline 90→210 s and writeTimeout 120→240 s for the PDF path's worst case. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5.5
+273
@@ -0,0 +1,273 @@
|
||||
package extract
|
||||
|
||||
import (
|
||||
"strings"
|
||||
|
||||
"github.com/go-pdfkit/pdffont"
|
||||
"github.com/go-pdfkit/reader"
|
||||
)
|
||||
|
||||
// maxFormDepth is how deeply one form may draw another before a page is taken
|
||||
// to be drawing itself.
|
||||
const maxFormDepth = 12
|
||||
|
||||
// An extractor walks a content stream keeping just enough state to say where
|
||||
// each piece of text was drawn.
|
||||
type extractor struct {
|
||||
doc *reader.Document
|
||||
runs []Run
|
||||
fonts map[int]*pdffont.Font
|
||||
// tm is where the next glyph goes and tlm where the line began; both
|
||||
// belong to the page rather than to the graphics state.
|
||||
tm, tlm matrix
|
||||
images []Image
|
||||
// wantPictures says whether anybody is going to read Image.Data. When
|
||||
// nobody is, the filters are left undone.
|
||||
wantPictures bool
|
||||
}
|
||||
|
||||
// run walks a content stream.
|
||||
func (e *extractor) run(content []byte, resources reader.Dict, g state, depth int) {
|
||||
if depth > maxFormDepth {
|
||||
return
|
||||
}
|
||||
// The operations are taken one at a time rather than all at once. The
|
||||
// walk only ever goes forwards, so nothing needs the list; keeping it
|
||||
// costs a struct and a slice header for every operator on the page, and
|
||||
// pages are not always small. One arXiv figure holds a single page of
|
||||
// 84.8 MB of content stream and 4 579 973 operations: holding them cost
|
||||
// 2 166 MB where reading them one by one costs 935 MB.
|
||||
scan := reader.NewContentScanner(content)
|
||||
var stack []state
|
||||
for {
|
||||
op, more := scan.Next()
|
||||
if !more {
|
||||
break
|
||||
}
|
||||
n := numbers(op.Operands)
|
||||
switch op.Operator {
|
||||
case "q":
|
||||
stack = append(stack, g)
|
||||
case "Q":
|
||||
if len(stack) > 0 {
|
||||
g = stack[len(stack)-1]
|
||||
stack = stack[:len(stack)-1]
|
||||
}
|
||||
case "cm":
|
||||
if len(n) >= 6 {
|
||||
g.ctm = matrix{n[0], n[1], n[2], n[3], n[4], n[5]}.mul(g.ctm)
|
||||
}
|
||||
case "BT":
|
||||
e.tm, e.tlm = identity, identity
|
||||
case "ET":
|
||||
e.tm, e.tlm = identity, identity
|
||||
case "Tf":
|
||||
e.setFont(&g, op.Operands, n, resources)
|
||||
case "Tc":
|
||||
if len(n) >= 1 {
|
||||
g.text.charSpace = n[0]
|
||||
}
|
||||
case "Tw":
|
||||
if len(n) >= 1 {
|
||||
g.text.wordSpace = n[0]
|
||||
}
|
||||
case "Tz":
|
||||
if len(n) >= 1 {
|
||||
g.text.scale = n[0] / 100
|
||||
}
|
||||
case "TL":
|
||||
if len(n) >= 1 {
|
||||
g.text.leading = n[0]
|
||||
}
|
||||
case "Ts":
|
||||
if len(n) >= 1 {
|
||||
g.text.rise = n[0]
|
||||
}
|
||||
case "Tr":
|
||||
if len(n) >= 1 {
|
||||
g.text.mode = int(n[0])
|
||||
}
|
||||
case "Td":
|
||||
if len(n) >= 2 {
|
||||
e.tlm = matrix{1, 0, 0, 1, n[0], n[1]}.mul(e.tlm)
|
||||
e.tm = e.tlm
|
||||
}
|
||||
case "TD":
|
||||
if len(n) >= 2 {
|
||||
g.text.leading = -n[1]
|
||||
e.tlm = matrix{1, 0, 0, 1, n[0], n[1]}.mul(e.tlm)
|
||||
e.tm = e.tlm
|
||||
}
|
||||
case "Tm":
|
||||
if len(n) >= 6 {
|
||||
e.tlm = matrix{n[0], n[1], n[2], n[3], n[4], n[5]}
|
||||
e.tm = e.tlm
|
||||
}
|
||||
case "T*":
|
||||
e.nextLine(&g)
|
||||
case "Tj":
|
||||
e.showOperands(&g, op.Operands)
|
||||
case "'":
|
||||
e.nextLine(&g)
|
||||
e.showOperands(&g, op.Operands)
|
||||
case "\"":
|
||||
if len(n) >= 2 {
|
||||
g.text.wordSpace, g.text.charSpace = n[0], n[1]
|
||||
}
|
||||
e.nextLine(&g)
|
||||
e.showOperands(&g, op.Operands)
|
||||
case "TJ":
|
||||
e.showArray(&g, op.Operands)
|
||||
case "Do":
|
||||
e.doXObject(&g, op.Operands, resources, depth)
|
||||
case "BI":
|
||||
e.inlineImage(&g, op.Image)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// setFont reads the font a Tf operator names.
|
||||
func (e *extractor) setFont(g *state, operands []reader.Object, n []float64, resources reader.Dict) {
|
||||
if len(operands) < 2 {
|
||||
return
|
||||
}
|
||||
name, ok := reader.ToName(operands[0])
|
||||
if !ok {
|
||||
return
|
||||
}
|
||||
if len(n) >= 1 {
|
||||
g.text.size = n[len(n)-1]
|
||||
}
|
||||
g.text.fontName = string(name)
|
||||
g.text.font = e.font(name, resources)
|
||||
}
|
||||
|
||||
// font reads a font out of the page's resources, remembering the ones already
|
||||
// read so that a page setting the same font a thousand times reads it once.
|
||||
func (e *extractor) font(name reader.Name, resources reader.Dict) *pdffont.Font {
|
||||
fonts, ok := e.doc.GetDict(resources, "Font")
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
if ref, ok := fonts.Get(name).(reader.Ref); ok {
|
||||
if f, seen := e.fonts[ref.Num]; seen {
|
||||
return f
|
||||
}
|
||||
dict, ok := e.doc.GetDict(fonts, name)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
f := pdffont.Read(e.doc, dict)
|
||||
attachProgram(f)
|
||||
e.fonts[ref.Num] = f
|
||||
return f
|
||||
}
|
||||
dict, ok := e.doc.GetDict(fonts, name)
|
||||
if !ok {
|
||||
return nil
|
||||
}
|
||||
f := pdffont.Read(e.doc, dict)
|
||||
attachProgram(f)
|
||||
return f
|
||||
}
|
||||
|
||||
// nextLine moves down by the leading, which is what T* and the quote
|
||||
// operators do.
|
||||
func (e *extractor) nextLine(g *state) {
|
||||
e.tlm = matrix{1, 0, 0, 1, 0, -g.text.leading}.mul(e.tlm)
|
||||
e.tm = e.tlm
|
||||
}
|
||||
|
||||
// showOperands shows the string a Tj-like operator carries.
|
||||
func (e *extractor) showOperands(g *state, operands []reader.Object) {
|
||||
for _, o := range operands {
|
||||
if s, ok := reader.ToString(o); ok {
|
||||
e.show(g, s)
|
||||
return
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// showArray shows the pieces a TJ operator carries, moving the pen by the
|
||||
// numbers between them.
|
||||
func (e *extractor) showArray(g *state, operands []reader.Object) {
|
||||
for _, o := range operands {
|
||||
arr, ok := reader.ToArray(o)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
for _, item := range arr {
|
||||
if s, ok := reader.ToString(item); ok {
|
||||
e.show(g, s)
|
||||
continue
|
||||
}
|
||||
if v, ok := reader.ToFloat(item); ok {
|
||||
e.advance(g, -v/1000*g.text.size*g.text.scale)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// show reads one string as text and records where it was drawn.
|
||||
func (e *extractor) show(g *state, s []byte) {
|
||||
f := g.text.font
|
||||
if f == nil || len(s) == 0 {
|
||||
return
|
||||
}
|
||||
start := e.tm.mul(g.ctm)
|
||||
x, y := start.apply(0, g.text.rise)
|
||||
var text strings.Builder
|
||||
unreadable := false
|
||||
width := 0.0
|
||||
for _, code := range f.Codes(s) {
|
||||
if t, ok := f.Text(code); ok {
|
||||
text.WriteString(pdffont.TrimText(t))
|
||||
} else {
|
||||
unreadable = true
|
||||
}
|
||||
adv := (f.Width(code)*g.text.size + g.text.charSpace) * g.text.scale
|
||||
if f.Kind() != pdffont.Composite && code == ' ' {
|
||||
adv += g.text.wordSpace * g.text.scale
|
||||
}
|
||||
width += adv
|
||||
}
|
||||
e.advance(g, width)
|
||||
if text.Len() == 0 && !unreadable {
|
||||
return
|
||||
}
|
||||
// How far a length in text space reaches on the page: the text matrix and
|
||||
// the page's transform together, which is the same matrix the run's own
|
||||
// position came through.
|
||||
//
|
||||
// Only the page's transform used to be counted here, and the text matrix
|
||||
// was left out. A document is free to put the whole of its scale there —
|
||||
// "/F1 1 Tf" and then a text matrix of ten is how TeX writes every PDF it
|
||||
// has ever produced — and such a document reported its text as one point
|
||||
// tall with letters half a point wide, while saying quite correctly where
|
||||
// on the page each run began.
|
||||
//
|
||||
// Those are the two numbers a word break is decided by. A gap measured in
|
||||
// points was being weighed against a space width ten times too small, so
|
||||
// every ordinary kern between two letters looked like a space: "Original
|
||||
// Domain" came back as "Or ig inal D omain". Across four thousand arXiv
|
||||
// figures, 57% of the words a reader got back were one or two letters
|
||||
// long. Nothing failed, and nothing was slow; the text was simply no
|
||||
// longer text.
|
||||
scale := start.scale()
|
||||
e.runs = append(e.runs, Run{
|
||||
Text: text.String(),
|
||||
X: x,
|
||||
Y: y,
|
||||
Width: width * scale,
|
||||
Size: g.text.size * scale,
|
||||
Font: g.text.fontName,
|
||||
Space: f.Width(' ') * g.text.size * g.text.scale * scale,
|
||||
Invisible: g.text.mode == 3,
|
||||
Unreadable: unreadable,
|
||||
})
|
||||
}
|
||||
|
||||
// advance moves the pen along the line by a distance in text space.
|
||||
func (e *extractor) advance(g *state, by float64) {
|
||||
e.tm = matrix{1, 0, 0, 1, by, 0}.mul(e.tm)
|
||||
}
|
||||
Reference in New Issue
Block a user