kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

P5a plan: PDF documents — seven Ornith tasks with acceptance tests first

Browse Source 
docs/plans/p5a: 01 kind+fetch limits, 02 pdftext child + runner, 03 dcache PDF tables,
04 pdfpool, 05 pdfdoc render (map/pages/grep), 06 api, 07 digest citations, metrics, CLI,
skill, unit, README. Given tests compile against a panic-only skeleton and fail there for the
intended reasons; PDFs for tests are built by internal/pdftest (given), plus two qpdf-encrypted
fixtures.

Owner changes for the run: vendor go-pdfkit/extract v0.3.0 + reader v0.6.0 (pinned via
tools.go until task 02); AGENTS.md, tools/run-plan.sh (enforces a Co-Authored-By trailer for
the model), docs/implementer-log.md, `make gate`; gofmt (whitespace) on clean.go and guard;
TestUnsupportedType uses a zip now that PDFs are supported; requestDeadline 90→210 s and
writeTimeout 120→240 s for the PDF path's worst case.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
kyle
2026-09-25 20:47:38 -07:00
co-authored by Claude Opus 5.5
parent 3f2b78d4aa
commit cfde35397a
115 changed files with 26422 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files
vendor/github.com/go-pdfkit/pdffont/encoding.go
Generated Vendored
+302
View File
@@ -0,0 +1,302 @@
package pdffont
import "strings"
// The two encodings a simple font is addressed through when it does not carry
// one of its own, as a name per code. Only the codes that name a glyph appear;
// the rest are blank.
//
// These are the tables the specification prints. They matter because a font
// program is addressed by glyph name or by character, and a content stream is
// written in single bytes: without them, every byte would have to be guessed
// at.
// StandardEncoding is Adobe's own, which most Type 1 fonts assume, and what a
// simple font is read with when it says nothing else.
var StandardEncoding = [256]string{
32: "space", 33: "exclam", 34: "quotedbl", 35: "numbersign", 36: "dollar",
37: "percent", 38: "ampersand", 39: "quoteright", 40: "parenleft",
41: "parenright", 42: "asterisk", 43: "plus", 44: "comma", 45: "hyphen",
46: "period", 47: "slash", 48: "zero", 49: "one", 50: "two", 51: "three",
52: "four", 53: "five", 54: "six", 55: "seven", 56: "eight", 57: "nine",
58: "colon", 59: "semicolon", 60: "less", 61: "equal", 62: "greater",
63: "question", 64: "at", 65: "A", 66: "B", 67: "C", 68: "D", 69: "E",
70: "F", 71: "G", 72: "H", 73: "I", 74: "J", 75: "K", 76: "L", 77: "M",
78: "N", 79: "O", 80: "P", 81: "Q", 82: "R", 83: "S", 84: "T", 85: "U",
86: "V", 87: "W", 88: "X", 89: "Y", 90: "Z", 91: "bracketleft",
92: "backslash", 93: "bracketright", 94: "asciicircum", 95: "underscore",
96: "quoteleft", 97: "a", 98: "b", 99: "c", 100: "d", 101: "e", 102: "f",
103: "g", 104: "h", 105: "i", 106: "j", 107: "k", 108: "l", 109: "m",
110: "n", 111: "o", 112: "p", 113: "q", 114: "r", 115: "s", 116: "t",
117: "u", 118: "v", 119: "w", 120: "x", 121: "y", 122: "z",
123: "braceleft", 124: "bar", 125: "braceright", 126: "asciitilde",
161: "exclamdown", 162: "cent", 163: "sterling", 164: "fraction",
165: "yen", 166: "florin", 167: "section", 168: "currency",
169: "quotesingle", 170: "quotedblleft", 171: "guillemotleft",
172: "guilsinglleft", 173: "guilsinglright", 174: "fi", 175: "fl",
177: "endash", 178: "dagger", 179: "daggerdbl", 180: "periodcentered",
182: "paragraph", 183: "bullet", 184: "quotesinglbase",
185: "quotedblbase", 186: "quotedblright", 187: "guillemotright",
188: "ellipsis", 189: "perthousand", 191: "questiondown", 193: "grave",
194: "acute", 195: "circumflex", 196: "tilde", 197: "macron", 198: "breve",
199: "dotaccent", 200: "dieresis", 202: "ring", 203: "cedilla",
205: "hungarumlaut", 206: "ogonek", 207: "caron", 208: "emdash",
225: "AE", 227: "ordfeminine", 232: "Lslash", 233: "Oslash", 234: "OE",
235: "ordmasculine", 241: "ae", 245: "dotlessi", 248: "lslash",
249: "oslash", 250: "oe", 251: "germandbls",
}
// WinAnsiEncoding is the one a file written on Windows assumes, and the one
// most producers name.
var WinAnsiEncoding = [256]string{
32: "space", 33: "exclam", 34: "quotedbl", 35: "numbersign", 36: "dollar",
37: "percent", 38: "ampersand", 39: "quotesingle", 40: "parenleft",
41: "parenright", 42: "asterisk", 43: "plus", 44: "comma", 45: "hyphen",
46: "period", 47: "slash", 48: "zero", 49: "one", 50: "two", 51: "three",
52: "four", 53: "five", 54: "six", 55: "seven", 56: "eight", 57: "nine",
58: "colon", 59: "semicolon", 60: "less", 61: "equal", 62: "greater",
63: "question", 64: "at", 65: "A", 66: "B", 67: "C", 68: "D", 69: "E",
70: "F", 71: "G", 72: "H", 73: "I", 74: "J", 75: "K", 76: "L", 77: "M",
78: "N", 79: "O", 80: "P", 81: "Q", 82: "R", 83: "S", 84: "T", 85: "U",
86: "V", 87: "W", 88: "X", 89: "Y", 90: "Z", 91: "bracketleft",
92: "backslash", 93: "bracketright", 94: "asciicircum", 95: "underscore",
96: "grave", 97: "a", 98: "b", 99: "c", 100: "d", 101: "e", 102: "f",
103: "g", 104: "h", 105: "i", 106: "j", 107: "k", 108: "l", 109: "m",
110: "n", 111: "o", 112: "p", 113: "q", 114: "r", 115: "s", 116: "t",
117: "u", 118: "v", 119: "w", 120: "x", 121: "y", 122: "z",
123: "braceleft", 124: "bar", 125: "braceright", 126: "asciitilde",
128: "Euro", 130: "quotesinglbase", 131: "florin", 132: "quotedblbase",
133: "ellipsis", 134: "dagger", 135: "daggerdbl", 136: "circumflex",
137: "perthousand", 138: "Scaron", 139: "guilsinglleft", 140: "OE",
142: "Zcaron", 145: "quoteleft", 146: "quoteright", 147: "quotedblleft",
148: "quotedblright", 149: "bullet", 150: "endash", 151: "emdash",
152: "tilde", 153: "trademark", 154: "scaron", 155: "guilsinglright",
156: "oe", 158: "zcaron", 159: "Ydieresis", 160: "space",
161: "exclamdown", 162: "cent", 163: "sterling", 164: "currency",
165: "yen", 166: "brokenbar", 167: "section", 168: "dieresis",
169: "copyright", 170: "ordfeminine", 171: "guillemotleft",
172: "logicalnot", 173: "hyphen", 174: "registered", 175: "macron",
176: "degree", 177: "plusminus", 178: "twosuperior",
179: "threesuperior", 180: "acute", 181: "mu", 182: "paragraph",
183: "periodcentered", 184: "cedilla", 185: "onesuperior",
186: "ordmasculine", 187: "guillemotright", 188: "onequarter",
189: "onehalf", 190: "threequarters", 191: "questiondown", 192: "Agrave",
193: "Aacute", 194: "Acircumflex", 195: "Atilde", 196: "Adieresis",
197: "Aring", 198: "AE", 199: "Ccedilla", 200: "Egrave", 201: "Eacute",
202: "Ecircumflex", 203: "Edieresis", 204: "Igrave", 205: "Iacute",
206: "Icircumflex", 207: "Idieresis", 208: "Eth", 209: "Ntilde",
210: "Ograve", 211: "Oacute", 212: "Ocircumflex", 213: "Otilde",
214: "Odieresis", 215: "multiply", 216: "Oslash", 217: "Ugrave",
218: "Uacute", 219: "Ucircumflex", 220: "Udieresis", 221: "Yacute",
222: "Thorn", 223: "germandbls", 224: "agrave", 225: "aacute",
226: "acircumflex", 227: "atilde", 228: "adieresis", 229: "aring",
230: "ae", 231: "ccedilla", 232: "egrave", 233: "eacute",
234: "ecircumflex", 235: "edieresis", 236: "igrave", 237: "iacute",
238: "icircumflex", 239: "idieresis", 240: "eth", 241: "ntilde",
242: "ograve", 243: "oacute", 244: "ocircumflex", 245: "otilde",
246: "odieresis", 247: "divide", 248: "oslash", 249: "ugrave",
250: "uacute", 251: "ucircumflex", 252: "udieresis", 253: "yacute",
254: "thorn", 255: "ydieresis",
}
// glyphRunes is what the names above stand for. A font program is looked up by
// character, so a name has to become one; anything not here, and not written
// in one of the two conventional forms, cannot be looked up that way.
var glyphRunes = map[string]rune{
"space": ' ', "exclam": '!', "quotedbl": '"', "numbersign": '#',
"dollar": '$', "percent": '%', "ampersand": '&', "quotesingle": '\'',
"quoteright": '’', "quoteleft": '‘', "parenleft": '(',
"parenright": ')', "asterisk": '*', "plus": '+', "comma": ',',
"hyphen": '-', "period": '.', "slash": '/', "zero": '0', "one": '1',
"two": '2', "three": '3', "four": '4', "five": '5', "six": '6',
"seven": '7', "eight": '8', "nine": '9', "colon": ':', "semicolon": ';',
"less": '<', "equal": '=', "greater": '>', "question": '?', "at": '@',
"bracketleft": '[', "backslash": '\\', "bracketright": ']',
"asciicircum": '^', "underscore": '_', "braceleft": '{', "bar": '|',
"braceright": '}', "asciitilde": '~', "exclamdown": '¡', "cent": '¢',
"sterling": '£', "fraction": '⁄', "yen": '¥', "florin": 'ƒ',
"section": '§', "currency": '¤', "quotedblleft": '“',
"quotedblright": '”', "guillemotleft": '«', "guillemotright": '»',
"guilsinglleft": '‹', "guilsinglright": '›', "fi": 'fi', "fl": 'fl',
"endash": '–', "emdash": '—', "dagger": '†', "daggerdbl": '‡',
"periodcentered": '·', "paragraph": '¶', "bullet": '•',
"quotesinglbase": '‚', "quotedblbase": '„', "ellipsis": '…',
"perthousand": '‰', "questiondown": '¿', "grave": '`', "acute": '´',
"circumflex": 'ˆ', "tilde": '˜', "macron": '¯', "breve": '˘',
"dotaccent": '˙', "dieresis": '¨', "ring": '˚', "cedilla": '¸',
"hungarumlaut": '˝', "ogonek": '˛', "caron": 'ˇ', "AE": 'Æ', "ae": 'æ',
"ordfeminine": 'ª', "ordmasculine": 'º', "Lslash": 'Ł', "lslash": 'ł',
"Oslash": 'Ø', "oslash": 'ø', "OE": 'Œ', "oe": 'œ', "dotlessi": 'ı',
"germandbls": 'ß', "Euro": '€', "Scaron": 'Š', "scaron": 'š',
"Zcaron": 'Ž', "zcaron": 'ž', "Ydieresis": 'Ÿ', "trademark": '™',
"brokenbar": '¦', "copyright": '©', "logicalnot": '¬', "registered": '®',
"degree": '°', "plusminus": '±', "twosuperior": '²',
"threesuperior": '³', "mu": 'µ', "onesuperior": '¹', "onequarter": '¼',
"onehalf": '½', "threequarters": '¾', "multiply": '×', "divide": '÷',
"Agrave": 'À', "Aacute": 'Á', "Acircumflex": 'Â', "Atilde": 'Ã',
"Adieresis": 'Ä', "Aring": 'Å', "Ccedilla": 'Ç', "Egrave": 'È',
"Eacute": 'É', "Ecircumflex": 'Ê', "Edieresis": 'Ë', "Igrave": 'Ì',
"Iacute": 'Í', "Icircumflex": 'Î', "Idieresis": 'Ï', "Eth": 'Ð',
"Ntilde": 'Ñ', "Ograve": 'Ò', "Oacute": 'Ó', "Ocircumflex": 'Ô',
"Otilde": 'Õ', "Odieresis": 'Ö', "Ugrave": 'Ù', "Uacute": 'Ú',
"Ucircumflex": 'Û', "Udieresis": 'Ü', "Yacute": 'Ý', "Thorn": 'Þ',
"agrave": 'à', "aacute": 'á', "acircumflex": 'â', "atilde": 'ã',
"adieresis": 'ä', "aring": 'å', "ccedilla": 'ç', "egrave": 'è',
"eacute": 'é', "ecircumflex": 'ê', "edieresis": 'ë', "igrave": 'ì',
"iacute": 'í', "icircumflex": 'î', "idieresis": 'ï', "eth": 'ð',
"ntilde": 'ñ', "ograve": 'ò', "oacute": 'ó', "ocircumflex": 'ô',
"otilde": 'õ', "odieresis": 'ö', "ugrave": 'ù', "uacute": 'ú',
"ucircumflex": 'û', "udieresis": 'ü', "yacute": 'ý', "thorn": 'þ',
"ydieresis": 'ÿ',
}
// RuneOfGlyphName turns a glyph name into the character it stands for, through
// the table above or through the two conventions the specification defines for
// naming a character directly. ok is false for a name that says nothing about
// which character it is, which a subsetted font's own names often do not.
func RuneOfGlyphName(name string) (rune, bool) {
s, ok := TextOfGlyphName(name)
if !ok {
return 0, false
}
r := []rune(s)
if len(r) != 1 {
return 0, false
}
return r[0], true
}
// TextOfGlyphName turns a glyph name into the text it stands for, which is not
// always one character: a name may say it is a ligature of several.
//
// Three of the rules here are the ones the Adobe Glyph List lays down for
// reading a name nobody has listed, and each was costing real text. A name may
// carry a variant after a full stop — "a.sc" is a small-capital A and is still
// an A, 7 272 of them in the corpus. A name may be several component names
// with underscores between them — "f_i" is the fi ligature, and without this
// "Definition" comes back "Denition", which is not a missing character so much
// as a misspelt word. And a name may simply be one nobody thought to list.
func TextOfGlyphName(name string) (string, bool) {
if name == "" {
return "", false
}
if r, ok := oneGlyphRune(name); ok {
return string(r), true
}
if seq, ok := unicodeSequence(name); ok {
return seq, true
}
// A variant of a character is still that character: the part before the
// first full stop is the name, and what follows says which cut of it.
if base, _, found := strings.Cut(name, "."); found {
if base == "" {
return "", false
}
return TextOfGlyphName(base)
}
// A name made of parts with underscores between them is the parts, in
// order — which is how a ligature is named when it has no name of its own.
if strings.Contains(name, "_") {
// Every part that can be read contributes at least one character, and
// a part that cannot gives up on the whole name, so what comes out of
// this is never empty.
var out strings.Builder
for _, part := range strings.Split(name, "_") {
piece, ok := TextOfGlyphName(part)
if !ok {
return "", false
}
out.WriteString(piece)
}
return out.String(), true
}
return "", false
}
// oneGlyphRune is a name that stands for exactly one character, by any of the
// ways a name can.
func oneGlyphRune(name string) (rune, bool) {
if r, ok := glyphRunes[name]; ok {
return r, true
}
if r, ok := latinExtendedRunes[name]; ok {
return r, true
}
if r, ok := greekAndMathRunes[name]; ok {
return r, true
}
if len(name) == 1 {
return rune(name[0]), true
}
if r, ok := parseHexName(name, "uni", 4); ok {
return r, true
}
if r, ok := parseHexName(name, "u", 4); ok {
return r, true
}
return 0, false
}
// unicodeSequence reads the uniXXXXYYYY form, which names several characters
// at once: a letter and the accent that goes over it, or the three pieces a
// Hebrew cluster is written in.
//
// It has to be told apart from something that looks exactly like it. Producers
// write a glyph's own number in the same shape — uni00000048 is glyph 72, not
// U+0000 followed by U+0048 — and 49 740 names in the corpus are that. The two
// are separable by one observation: a real sequence never begins with U+0000,
// because nothing is written after a character that does not exist. Of the
// 3 875 genuine sequences found across 14 823 embedded fonts, not one starts
// with a zero group; of the disguised glyph numbers, all of them do.
func unicodeSequence(name string) (string, bool) {
digits, ok := strings.CutPrefix(name, "uni")
if !ok || len(digits) < 8 || len(digits)%4 != 0 {
return "", false
}
var out strings.Builder
for i := 0; i < len(digits); i += 4 {
r, ok := parseHexName("uni"+digits[i:i+4], "uni", 4)
if !ok {
return "", false
}
if r == 0 {
// A glyph number wearing a character's clothes.
return "", false
}
out.WriteRune(r)
}
return out.String(), true
}
// parseHexName reads the uniXXXX and uXXXX conventions.
func parseHexName(name, prefix string, minDigits int) (rune, bool) {
if len(name) <= len(prefix) || name[:len(prefix)] != prefix {
return 0, false
}
digits := name[len(prefix):]
if len(digits) < minDigits || len(digits) > 6 {
return 0, false
}
var v rune
for i := 0; i < len(digits); i++ {
d := hexDigit(digits[i])
if d < 0 {
return 0, false
}
v = v<<4 | rune(d)
}
return v, true
}
// hexDigit reads one hexadecimal digit, or reports -1.
func hexDigit(c byte) int {
switch {
case c >= '0' && c <= '9':
return int(c - '0')
case c >= 'A' && c <= 'F':
return int(c-'A') + 10
case c >= 'a' && c <= 'f':
return int(c-'a') + 10
}
return -1
}