P5a plan: PDF documents — seven Ornith tasks with acceptance tests first

docs/plans/p5a: 01 kind+fetch limits, 02 pdftext child + runner, 03 dcache PDF tables,
04 pdfpool, 05 pdfdoc render (map/pages/grep), 06 api, 07 digest citations, metrics, CLI,
skill, unit, README. Given tests compile against a panic-only skeleton and fail there for the
intended reasons; PDFs for tests are built by internal/pdftest (given), plus two qpdf-encrypted
fixtures.

Owner changes for the run: vendor go-pdfkit/extract v0.3.0 + reader v0.6.0 (pinned via
tools.go until task 02); AGENTS.md, tools/run-plan.sh (enforces a Co-Authored-By trailer for
the model), docs/implementer-log.md, `make gate`; gofmt (whitespace) on clean.go and guard;
TestUnsupportedType uses a zip now that PDFs are supported; requestDeadline 90→210 s and
writeTimeout 120→240 s for the PDF path's worst case.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
2026-09-25 20:47:38 -07:00
co-authored by Claude Opus 5.5
parent 3f2b78d4aa
commit cfde35397a
115 changed files with 26422 additions and 43 deletions
+579
View File
@@ -0,0 +1,579 @@
package reader
import "fmt"
// CCITT Group 3 and Group 4 fax decoding, as used by /CCITTFaxDecode.
//
// WHY THIS IS HERE
//
// A scanned form is a fax. 67 of the 1 633 real forms in the corpus — the
// eleven issuing bodies, not the vendor test suites — carry a CCITT-encoded
// image, 263 images between them, and 6 of their pages have nothing else on
// them at all. Those six pages drew entirely blank, and a blank page is the
// failure a reader notices before any other.
//
// WHY IT BELONGS IN THE READER RATHER THAN IN A RENDERER
//
// The other filters this package stops at — DCT, JPX, JBIG2 — carry an image
// with its own idea of how many components it has and how deep they are. This
// one does not: it produces bilevel samples, one bit a pixel, rows padded to a
// byte, and the stream dictionary says what they mean. That is a byte stream,
// so it is a filter, and every caller gets it rather than each writing its own.
//
// THE REFERENCE READ BEFORE WRITING THIS
//
// ITU-T Recommendations T.4 (Group 3) and T.6 (Group 4), by way of the tables
// and the changing-element algorithm in golang.org/x/image/ccitt — read, not
// imported, because this package has no dependencies outside the standard
// library and gains none. The code tables below were extracted mechanically
// from that package's gen.go rather than retyped, because a table of 218
// variable-length codes transcribed by hand is a table with a mistake in it.
//
// The algorithm is the one T.6 Figure 1 describes. A row is decoded into one
// byte a pixel — 0xFF white, 0x00 black — and the row before it is the
// reference against which the two-dimensional modes are resolved:
//
// b1 b2
// v v
// prev: BBBBBwwwwwBBBwwwww
// curr: BBBwwwwwBBBBBBwwww
// ^ ^ ^
// a0 a1 a2
//
// a0 is where the pen is; a1 the next colour change to its right on this row;
// b1 the first change on the row above that is to the right of a0 and of the
// opposite colour to it; b2 the next change after b1. Pass mode says a1 is at
// or beyond b2; horizontal mode reads two runs and ignores the row above;
// vertical mode puts a1 at b1 plus an offset between -3 and +3.
// The two-dimensional modes of T.4 Table 1.
const (
modePass = iota
modeH
modeV0
modeVR1
modeVR2
modeVR3
modeVL1
modeVL2
modeVL3
modeExt
)
// A ccittCode is one row of a code table: the value a code names, and the bits
// that name it, most significant first.
type ccittCode struct {
value int
bits string
}
// ccittModeCodes is Table 1 of ITU-T T.4: the two-dimensional modes.
var ccittModeCodes = []ccittCode{
{modePass, "0001"}, {modeH, "001"}, {modeV0, "1"}, {modeVR1, "011"},
{modeVR2, "000011"}, {modeVR3, "0000011"}, {modeVL1, "010"}, {modeVL2, "000010"},
{modeVL3, "0000010"}, {modeExt, "0000001"},
}
// ccittWhiteCodes is Tables 2 and 3 of ITU-T T.4 for a white run: the
// terminating codes 0 to 63, then the make-up codes in steps of 64.
var ccittWhiteCodes = []ccittCode{
{0, "00110101"}, {1, "000111"}, {2, "0111"}, {3, "1000"},
{4, "1011"}, {5, "1100"}, {6, "1110"}, {7, "1111"},
{8, "10011"}, {9, "10100"}, {10, "00111"}, {11, "01000"},
{12, "001000"}, {13, "000011"}, {14, "110100"}, {15, "110101"},
{16, "101010"}, {17, "101011"}, {18, "0100111"}, {19, "0001100"},
{20, "0001000"}, {21, "0010111"}, {22, "0000011"}, {23, "0000100"},
{24, "0101000"}, {25, "0101011"}, {26, "0010011"}, {27, "0100100"},
{28, "0011000"}, {29, "00000010"}, {30, "00000011"}, {31, "00011010"},
{32, "00011011"}, {33, "00010010"}, {34, "00010011"}, {35, "00010100"},
{36, "00010101"}, {37, "00010110"}, {38, "00010111"}, {39, "00101000"},
{40, "00101001"}, {41, "00101010"}, {42, "00101011"}, {43, "00101100"},
{44, "00101101"}, {45, "00000100"}, {46, "00000101"}, {47, "00001010"},
{48, "00001011"}, {49, "01010010"}, {50, "01010011"}, {51, "01010100"},
{52, "01010101"}, {53, "00100100"}, {54, "00100101"}, {55, "01011000"},
{56, "01011001"}, {57, "01011010"}, {58, "01011011"}, {59, "01001010"},
{60, "01001011"}, {61, "00110010"}, {62, "00110011"}, {63, "00110100"},
{64, "11011"}, {128, "10010"}, {192, "010111"}, {256, "0110111"},
{320, "00110110"}, {384, "00110111"}, {448, "01100100"}, {512, "01100101"},
{576, "01101000"}, {640, "01100111"}, {704, "011001100"}, {768, "011001101"},
{832, "011010010"}, {896, "011010011"}, {960, "011010100"}, {1024, "011010101"},
{1088, "011010110"}, {1152, "011010111"}, {1216, "011011000"}, {1280, "011011001"},
{1344, "011011010"}, {1408, "011011011"}, {1472, "010011000"}, {1536, "010011001"},
{1600, "010011010"}, {1664, "011000"}, {1728, "010011011"}, {1792, "00000001000"},
{1856, "00000001100"}, {1920, "00000001101"}, {1984, "000000010010"}, {2048, "000000010011"},
{2112, "000000010100"}, {2176, "000000010101"}, {2240, "000000010110"}, {2304, "000000010111"},
{2368, "000000011100"}, {2432, "000000011101"}, {2496, "000000011110"}, {2560, "000000011111"},
}
// ccittBlackCodes is Tables 2 and 3 of ITU-T T.4 for a black run.
var ccittBlackCodes = []ccittCode{
{0, "0000110111"}, {1, "010"}, {2, "11"}, {3, "10"},
{4, "011"}, {5, "0011"}, {6, "0010"}, {7, "00011"},
{8, "000101"}, {9, "000100"}, {10, "0000100"}, {11, "0000101"},
{12, "0000111"}, {13, "00000100"}, {14, "00000111"}, {15, "000011000"},
{16, "0000010111"}, {17, "0000011000"}, {18, "0000001000"}, {19, "00001100111"},
{20, "00001101000"}, {21, "00001101100"}, {22, "00000110111"}, {23, "00000101000"},
{24, "00000010111"}, {25, "00000011000"}, {26, "000011001010"}, {27, "000011001011"},
{28, "000011001100"}, {29, "000011001101"}, {30, "000001101000"}, {31, "000001101001"},
{32, "000001101010"}, {33, "000001101011"}, {34, "000011010010"}, {35, "000011010011"},
{36, "000011010100"}, {37, "000011010101"}, {38, "000011010110"}, {39, "000011010111"},
{40, "000001101100"}, {41, "000001101101"}, {42, "000011011010"}, {43, "000011011011"},
{44, "000001010100"}, {45, "000001010101"}, {46, "000001010110"}, {47, "000001010111"},
{48, "000001100100"}, {49, "000001100101"}, {50, "000001010010"}, {51, "000001010011"},
{52, "000000100100"}, {53, "000000110111"}, {54, "000000111000"}, {55, "000000100111"},
{56, "000000101000"}, {57, "000001011000"}, {58, "000001011001"}, {59, "000000101011"},
{60, "000000101100"}, {61, "000001011010"}, {62, "000001100110"}, {63, "000001100111"},
{64, "0000001111"}, {128, "000011001000"}, {192, "000011001001"}, {256, "000001011011"},
{320, "000000110011"}, {384, "000000110100"}, {448, "000000110101"}, {512, "0000001101100"},
{576, "0000001101101"}, {640, "0000001001010"}, {704, "0000001001011"}, {768, "0000001001100"},
{832, "0000001001101"}, {896, "0000001110010"}, {960, "0000001110011"}, {1024, "0000001110100"},
{1088, "0000001110101"}, {1152, "0000001110110"}, {1216, "0000001110111"}, {1280, "0000001010010"},
{1344, "0000001010011"}, {1408, "0000001010100"}, {1472, "0000001010101"}, {1536, "0000001011010"},
{1600, "0000001011011"}, {1664, "0000001100100"}, {1728, "0000001100101"}, {1792, "00000001000"},
{1856, "00000001100"}, {1920, "00000001101"}, {1984, "000000010010"}, {2048, "000000010011"},
{2112, "000000010100"}, {2176, "000000010101"}, {2240, "000000010110"}, {2304, "000000010111"},
{2368, "000000011100"}, {2432, "000000011101"}, {2496, "000000011110"}, {2560, "000000011111"},
}
// A ccittTable decodes one of the code tables. Codes are grouped by their
// length, which is the whole of the decoding: read a bit, lengthen the code by
// one, and ask whether a code of that length says anything. A prefix-free code
// cannot answer twice, so the first answer is the only one.
type ccittTable struct {
byLength []map[uint32]int
}
// newCCITTTable groups a table's codes by length. The bits are read from the
// strings the tables are written in, so what the code says and what the
// specification says are the same text.
func newCCITTTable(codes []ccittCode) *ccittTable {
longest := 0
for _, c := range codes {
if len(c.bits) > longest {
longest = len(c.bits)
}
}
t := &ccittTable{byLength: make([]map[uint32]int, longest+1)}
for _, c := range codes {
var v uint32
for _, b := range c.bits {
v <<= 1
if b == '1' {
v |= 1
}
}
n := len(c.bits)
if t.byLength[n] == nil {
t.byLength[n] = map[uint32]int{}
}
t.byLength[n][v] = c.value
}
return t
}
var (
ccittModes = newCCITTTable(ccittModeCodes)
ccittWhite = newCCITTTable(ccittWhiteCodes)
ccittBlack = newCCITTTable(ccittBlackCodes)
)
// A ccittBits reads one bit at a time, most significant first, which is the
// order a fax is written in.
type ccittBits struct {
data []byte
pos int // in bits
}
func (b *ccittBits) next() (uint32, bool) {
if b.pos >= 8*len(b.data) {
return 0, false
}
bit := (b.data[b.pos/8] >> (7 - uint(b.pos%8))) & 1
b.pos++
return uint32(bit), true
}
// align moves to the next byte boundary, which /EncodedByteAlign asks for at
// the start of every row.
func (b *ccittBits) align() { b.pos = (b.pos + 7) &^ 7 }
func (b *ccittBits) exhausted() bool { return b.pos >= 8*len(b.data) }
// restIsFill says whether nothing but zero bits remain. A fax is padded to a
// byte, and often to rather more than a byte, and zeros decode as perfectly
// good two-dimensional modes: pass mode is 0001 and vertical-left-3 is 0000010,
// so a decoder that does not stop here goes on inventing rows that look like
// the last real one. One French form gave 1 636 rows for a 208-row image that
// way, and every one of the surplus rows had plausible ink on it.
func (b *ccittBits) restIsFill() bool {
for i := b.pos; i < 8*len(b.data); i++ {
if b.data[i/8]&(1<<(7-uint(i%8))) != 0 {
return false
}
}
return true
}
// read decodes one code. It returns false at the end of the data or on a code
// the table does not name — the caller decides which of those is fatal, since a
// truncated fax is common and worth showing as far as it goes.
func (t *ccittTable) read(b *ccittBits) (int, bool) {
var code uint32
for n := 1; n < len(t.byLength); n++ {
bit, ok := b.next()
if !ok {
return 0, false
}
code = code<<1 | bit
if v, ok := t.byLength[n][code]; ok {
return v, true
}
}
return 0, false
}
// skipEOL consumes an end-of-line code, 000000000001, if one is next, and any
// fill bits before it. A Group 3 fax may have one at the end of every row, may
// have none at all, and may have several together at the end; all three are
// found in the corpus, so none of them may be an error.
func (b *ccittBits) skipEOL() bool {
start := b.pos
zeros := 0
for {
bit, ok := b.next()
if !ok {
b.pos = start
return false
}
if bit == 0 {
zeros++
continue
}
if zeros >= 11 {
return true
}
b.pos = start
return false
}
}
// ccittParams are the decode parameters of /CCITTFaxDecode, with the defaults
// the specification gives.
type ccittParams struct {
k int // <0 Group 4, 0 Group 3 one-dimensional, >0 Group 3 mixed
columns int // default 1728
rows int // 0 means as many as the data holds
blackIs1 bool // false: a 0 bit is black
encodedByteAlign bool
endOfBlock bool // default true: the data ends with a block marker
}
func ccittParamsOf(parm Dict, r Resolver) ccittParams {
return ccittParams{
k: intParm(parm, "K", 0, r),
columns: intParm(parm, "Columns", 1728, r),
rows: intParm(parm, "Rows", 0, r),
blackIs1: boolParm(parm, "BlackIs1", false, r),
encodedByteAlign: boolParm(parm, "EncodedByteAlign", false, r),
endOfBlock: boolParm(parm, "EndOfBlock", true, r),
}
}
// endOfBlockEOLs is how many end-of-line codes in a row mean the data has
// ended: two for Group 4, which has none between its rows, and six for Group 3,
// where every row may legitimately be followed by one.
func endOfBlockEOLs(k int) int {
if k < 0 {
return 2
}
return 6
}
// maxCCITTPixels bounds what one image may decode to, the way maxDecodedSize
// bounds a Flate stream. A fax names its own width and height in the decode
// parameters, so a file can ask for an image of any size it likes; 2^28 pixels
// is more than four times A4 at 600 dots to the inch.
var maxCCITTPixels = 1 << 28
// ccittDecode turns a fax into bilevel samples: one bit a pixel, each row
// padded to a byte boundary, which is what every other image filter in this
// package produces and what a stream dictionary's /Width, /Height and
// /BitsPerComponent describe.
func ccittDecode(data []byte, p ccittParams) ([]byte, error) {
if p.columns <= 0 {
return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d columns", p.columns)
}
if p.rows < 0 {
return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d rows", p.rows)
}
if p.rows > 0 && p.columns > maxCCITTPixels/p.rows {
return nil, fmt.Errorf("reader: /CCITTFaxDecode names %d by %d pixels, "+
"past the limit of %d", p.columns, p.rows, maxCCITTPixels)
}
b := &ccittBits{data: data}
stride := (p.columns + 7) / 8
curr := make([]byte, p.columns)
prev := []byte(nil)
out := []byte(nil)
for row := 0; p.rows == 0 || row < p.rows; row++ {
if p.rows == 0 && len(out)/stride*p.columns >= maxCCITTPixels {
return nil, fmt.Errorf("reader: /CCITTFaxDecode ran past %d pixels "+
"without naming how many rows it has", maxCCITTPixels)
}
if p.encodedByteAlign {
b.align()
}
// A Group 3 row may be introduced by an end-of-line code. Consuming it
// here rather than after the row means a file that has them and a file
// that does not are read the same way — and counting them is how the
// end of the data announces itself, since the block marker is nothing
// but end-of-line codes in a row.
eols := 0
for b.skipEOL() {
eols++
}
if p.endOfBlock && eols >= endOfBlockEOLs(p.k) {
break
}
if b.exhausted() || b.restIsFill() {
break
}
twoDimensional := p.k < 0
if p.k > 0 {
// In mixed mode a bit after the end-of-line says which the row is:
// 1 for one-dimensional, 0 for two.
// The data was checked for bits a moment ago, so this cannot
// fail; the value is taken rather than the error handled, because
// a branch that cannot be reached cannot be tested.
bit, _ := b.next()
twoDimensional = bit == 0
}
if err := ccittRow(b, curr, prev, twoDimensional); err != nil {
if len(out) == 0 {
return nil, err
}
// A truncated fax is worth showing as far as it got: the rows
// already decoded are real, and refusing them turns a damaged
// scan into a blank page. Leaving the loop rather than returning
// here matters — the padding below is what makes the answer as
// long as /Rows promised, and a fuzz case of two bytes found that
// returning early handed the caller one row where it had asked
// for four, with no error to say so.
break
}
out = append(out, ccittPack(curr, stride, p.blackIs1)...)
if prev == nil {
prev = make([]byte, p.columns)
}
copy(prev, curr)
}
// A file that says how many rows it has gets that many, padded with white
// if the data ran out, so the samples match the /Height the dictionary
// promises rather than being short by a row.
if p.rows > 0 {
white := make([]byte, stride)
if !p.blackIs1 {
for i := range white {
white[i] = 0xFF
}
}
for len(out) < p.rows*stride {
out = append(out, white...)
}
}
return out, nil
}
// ccittPack turns a row of one byte a pixel into one bit a pixel. With
// /BlackIs1 false — the default — a 0 bit is black, so white becomes a 1.
func ccittPack(row []byte, stride int, blackIs1 bool) []byte {
out := make([]byte, stride)
for x, v := range row {
white := v != 0
if white != blackIs1 {
out[x/8] |= 1 << (7 - uint(x%8))
}
}
return out
}
// ccittRow decodes one row into curr, one byte a pixel: 0xFF white, 0x00 black.
func ccittRow(b *ccittBits, curr, prev []byte, twoDimensional bool) error {
for i := range curr {
curr[i] = 0
}
at, white, first := 0, true, true
for at < len(curr) {
if !twoDimensional {
n, err := ccittRun(b, white)
if err != nil {
return err
}
if at+n > len(curr) {
return fmt.Errorf("reader: a fax row of %d pixels named %d",
len(curr), at+n)
}
fill(curr[at:at+n], white)
at += n
white = !white
first = false
continue
}
mode, ok := ccittModes.read(b)
if !ok {
return fmt.Errorf("reader: a fax row named no mode this reader knows")
}
switch mode {
case modePass:
// ccittFindB starts where the pen is and only walks forward,
// stopping at the end of the row, so b2 is always a pixel of this
// row at or after the pen. There is nothing to check.
b2 := ccittFindB(prev, curr, at, white, first, true)
fill(curr[at:b2], white)
at = b2
case modeH:
// Two runs, of the current colour and then the other, neither of
// them looking at the row above.
for i := 0; i < 2; i++ {
n, err := ccittRun(b, white)
if err != nil {
return err
}
if at+n > len(curr) {
return fmt.Errorf("reader: a fax row of %d pixels named %d",
len(curr), at+n)
}
fill(curr[at:at+n], white)
at += n
white = !white
}
// Horizontal mode reads a pair, so the two flips above have put
// the colour back where it started; there is nothing to undo.
case modeV0, modeVR1, modeVR2, modeVR3, modeVL1, modeVL2, modeVL3:
a1 := ccittFindB(prev, curr, at, white, first, false) + ccittOffset(mode)
if a1 < at || a1 > len(curr) {
return fmt.Errorf("reader: a fax vertical mode named pixel %d of %d",
a1, len(curr))
}
fill(curr[at:a1], white)
at = a1
white = !white
default:
return fmt.Errorf("reader: a fax named an extension mode, which " +
"this reader does not decode")
}
first = false
}
return nil
}
// ccittOffset is how far a vertical mode moves a1 from b1.
func ccittOffset(mode int) int {
switch mode {
case modeVR1:
return 1
case modeVR2:
return 2
case modeVR3:
return 3
case modeVL1:
return -1
case modeVL2:
return -2
case modeVL3:
return -3
}
return 0
}
// ccittRun reads one run length, which is a sequence of make-up codes ending in
// a terminating code below 64.
func ccittRun(b *ccittBits, white bool) (int, error) {
table := ccittBlack
if white {
table = ccittWhite
}
total := 0
for {
n, ok := table.read(b)
if !ok {
return 0, fmt.Errorf("reader: a fax run named no length this reader knows")
}
total += n
if total > maxCCITTPixels {
return 0, fmt.Errorf("reader: a fax named a run of %d pixels", total)
}
if n <= 63 {
return total, nil
}
}
}
// ccittFindB finds b1, or b2 when second is set: the changing elements on the
// row above, as T.6 Figure 1 defines them.
func ccittFindB(prev, curr []byte, at int, white, first, second bool) int {
// The row above the first row is implicitly all white, so it has no
// changing elements and both b values are at the end of the row.
if prev == nil {
return len(curr)
}
i := at
if first {
// a0 is implicitly one pixel before the row, on white. b1 is the first
// black pixel above; b2 the first white pixel after that.
for i < len(prev) && prev[i] != 0 {
i++
}
if second {
for i < len(prev) && prev[i] == 0 {
i++
}
}
return i
}
// Walk past the run above that is of the opposite colour to the pen, then
// past the run of the pen's own colour: what follows is b1.
opposite := byte(0xFF)
if white {
opposite = 0
}
for i < len(prev) && prev[i] == opposite {
i++
}
same := ^opposite
for i < len(prev) && prev[i] == same {
i++
}
if second {
for i < len(prev) && prev[i] == opposite {
i++
}
}
return i
}
// boolParm reads a boolean decode parameter, falling back to its default.
func boolParm(parm Dict, key Name, def bool, r Resolver) bool {
if parm == nil {
return def
}
v, err := Resolve(parm.Get(key), r)
if err != nil {
return def
}
if b, ok := v.(Bool); ok {
return bool(b)
}
return def
}
// fill paints a run of pixels, one byte each.
func fill(row []byte, white bool) {
v := byte(0)
if white {
v = 0xFF
}
for i := range row {
row[i] = v
}
}