From 41a68feb063b46fad71e13c5fb75f1874942e143 Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Fri, 25 Sep 2026 23:52:11 -0700 Subject: [PATCH] pdfdoc: page markers, the map for long documents, page ranges, grep Pure render function on a *pdftext.Doc with no I/O: the whole text with page markers when it fits, otherwise a map (title, outline with page numbers, leading pages and how to ask for more), a page range, or grep results with page numbers. Every length is in runes. Co-Authored-By: ornith-1.5-35b-a3b --- docs/implementer-log.md | 1 + internal/pdfdoc/grep.go | 153 ++++++++++++++++++ internal/pdfdoc/pdfdoc.go | 285 +++++++++++++++++++++++++++++++++ internal/pdfdoc/pdfdoc_test.go | 249 ++++++++++++++++++++++++++++ 4 files changed, 688 insertions(+) create mode 100644 internal/pdfdoc/grep.go create mode 100644 internal/pdfdoc/pdfdoc.go create mode 100644 internal/pdfdoc/pdfdoc_test.go diff --git a/docs/implementer-log.md b/docs/implementer-log.md index ffd4218..1ebeb42 100644 --- a/docs/implementer-log.md +++ b/docs/implementer-log.md @@ -8,3 +8,4 @@ One row per task session that ends in a commit. Newest last. | p5a/02-pdftext | 2026-09-25 | done | 1 | gate: ok | none | Built the child-extraction package: `Extract` (reader.Open with ErrWrongPassword→ErrEncrypted, per-page recover, trailer /Info /Title, rule-3 UTF-16BE/Latin-1 decode), `outline.go` (/Dest and /GoTo /A dests, 10k-item and no-double-visit bounds, never-nil outline), `Runner` (re-exec'd child, env-isolated via empty env, 60 s deadline, 64 MiB output cap, 200 MiB stdin cap, stderr capped at 4 KiB, exit-code→error mapping, Doc parse plus ErrNoText), and the `sift pdftext` case in main.go. Removed the tools.go module pin; go.mod/go.sum/vendor/modules.txt confirmed unchanged. All 10 given tests pass under -race -count=3 including the deadline-kill test; make gate ok; given files byte-identical; `printf nope | bin/sift pdftext` exits 4. | | p5a/03-dcache | 2026-09-25 | done | 1 | gate: ok | evict() tie-break | Added the pdf_doc/pdf_page tables, `PDFEntry`/`PDFKey`/`PutPDF`/`GetPDF`/`TouchPDF` (one-tx write deleting old pages first, page-ordered read, 304 touch moves only fetched_at), PDFH/PDFM counters, `Size()` including pdf_doc nbytes. One deviation: extended the eviction union with `pdf_doc` and added `rowid ASC` to the ORDER BY so same-second puts evict in insertion order (deterministic whole-document eviction); without it TestPDFEviction is order-dependent. Left Stats() signature unchanged for task 07. Given test byte-identical. | | p5a/04-pdfpool | 2026-09-25 | done | 1 | gate: ok | none | Implemented the generic scheduler. `Pool` holds one mutex, a `jobs` map (key→in-flight job), a `callerJobs` map (caller→their job), and a FIFO `waiters` line. `Do` applies the six rules in order: join (counted in `Waiting`, no slot, allowed while the caller holds another job, `fn` may be nil), caller-busy, then slot/queue/refuse. A job's result is broadcast to every sharer via a `done` channel; results are not cached (key forgotten on finish). `fn` runs under `context.WithoutCancel` so the flight outlives a caller that leaves. Panic in `fn` is recovered into an error, freed the slot and caller. One implementation finding (not a plan deviation): queued jobs must not be counted in `Running()`, so `startLocked` only registers a job and a separate `takeSlotLocked`/`grant` increments the slot count — abandoning a queued starter (`abandon`) releases sharers without touching the slot. Given test byte-identical. | +| p5a/05-pdfdoc | 2026-09-25 | done | 1 | gate: ok | none | Implemented the render package with no I/O on a `*pdftext.Doc`. `Marker`/`Full` print each extracted page as `--- page N ---`, text trimmed, joined by a blank line. `Render` dispatches grep → page range → whole. Whole fits in OutCap runes, else a map: title (or `Untitled PDF`)/pages/`~tokens` line with `pages 1-E extracted` only when fewer extracted than `Pages`, a cuttable `## Outline` (two-space indent per level below 1, cut before the header/instruction), the instruction line, and whole leading pages within OutCap/4 (at least page 1, cut to fit). Range parses `a-b`/`a`/`a-`, wraps out-of-range in `ErrBadPages` with `pages out of range (1-E)`, clamps `b`, caps at 100 pages and OutCap runes, and notes the rest as `pages="k-b"`. Grep is RE2 with smart-case `(?i)` prefix, 200-byte cap, per-line matches with previous/next page context in `p. P-`/`p. P:` form, first 100 shown, total counted; RE2 keeps the `(a+)+$` test linear. Every length is in runes. Given test byte-identical; make gate ok. | diff --git a/internal/pdfdoc/grep.go b/internal/pdfdoc/grep.go new file mode 100644 index 0000000..6f34bfd --- /dev/null +++ b/internal/pdfdoc/grep.go @@ -0,0 +1,153 @@ +package pdfdoc + +import ( + "fmt" + "regexp" + "strconv" + "strings" + "unicode/utf8" + + "git.wntrmute.dev/kyle/sift/internal/pdftext" +) + +// maxGrepMatches bounds how many match blocks are shown; the count reported is +// always the true total. +const maxGrepMatches = 100 + +// maxGrepPatternBytes bounds a grep pattern; a longer one is rejected before it +// can compile. +const maxGrepPatternBytes = 200 + +// hit is one match: the 1-based page and the 0-based line index within it. +type hit struct{ page, line int } + +// renderGrep searches every line of every page in the range (all pages when no +// range is given) and returns the matches with their page numbers. +func renderGrep(doc *pdftext.Doc, opt Options) (Rendered, error) { + pattern := opt.Grep + if len(pattern) > maxGrepPatternBytes { + return Rendered{}, fmt.Errorf("%w: pattern over %d bytes", ErrBadGrep, maxGrepPatternBytes) + } + re, err := regexp.Compile(grepPattern(pattern)) + if err != nil { + return Rendered{}, fmt.Errorf("%w: %v", ErrBadGrep, err) + } + + start, end := 1, len(doc.Text) + if opt.Pages != "" { + s, e, err := parsePages(opt.Pages, len(doc.Text)) + if err != nil { + return Rendered{}, err + } + start, end = s, e + } + + var hits []hit + linesByPage := make(map[int][]string, end-start+1) + for p := start; p <= end; p++ { + lines := splitLines(doc.Text[p-1]) + linesByPage[p] = lines + for j := range lines { + if re.MatchString(lines[j]) { + hits = append(hits, hit{p, j}) + } + } + } + + n := len(hits) + var header string + switch { + case n == 0: + header = `grep "` + pattern + `": no matches` + case n == 1: + header = `grep "` + pattern + `": 1 match` + default: + header = fmt.Sprintf(`grep "%s": %d matches`, pattern, n) + } + if n > maxGrepMatches { + header += ", showing the first " + strconv.Itoa(maxGrepMatches) + } + + maxBlocks := n + if maxBlocks > maxGrepMatches { + maxBlocks = maxGrepMatches + } + + outLines := make([]string, 0, 16+len(hits)) + outLines = append(outLines, header) + running := utf8.RuneCountInString(header) + shown := 0 + for i := 0; i < maxBlocks; i++ { + block := matchBlock(hits[i], linesByPage) + add := utf8.RuneCountInString(strings.Join(block, "\n")) + if i == 0 { + if running+add > opt.OutCap { + break + } + } else if running+1+add > opt.OutCap { + break + } + outLines = append(outLines, block...) + running += add + if i > 0 { + running++ + } + shown++ + } + content := strings.Join(outLines, "\n") + if utf8.RuneCountInString(content) > opt.OutCap { + content = truncateRunes(content, opt.OutCap) + } + + note := "" + if shown < maxBlocks { + note = fmt.Sprintf("showing %d of %d matches", shown, n) + } + return Rendered{Content: content, GrepHits: n, Note: note}, nil +} + +// grepPattern applies smart case: it lowercases-sensitively prefixes "(?i)" +// unless the pattern already contains an upper-case letter, which asks for a +// case-sensitive search. +func grepPattern(pattern string) string { + if hasUpper(pattern) { + return pattern + } + return "(?i)" + pattern +} + +// hasUpper reports whether s contains an upper-case ASCII letter. +func hasUpper(s string) bool { + for _, r := range s { + if r >= 'A' && r <= 'Z' { + return true + } + } + return false +} + +// matchBlock renders one hit as its previous line, the matching line, the next +// line and a separator. Context lines that do not exist (the match is on the +// first or last line) are omitted. +func matchBlock(h hit, linesByPage map[int][]string) []string { + lines := linesByPage[h.page] + var b []string + if h.line > 0 { + b = append(b, fmt.Sprintf("p. %d- %s", h.page, strings.TrimSpace(lines[h.line-1]))) + } + b = append(b, fmt.Sprintf("p. %d: %s", h.page, strings.TrimSpace(lines[h.line]))) + if h.line+1 < len(lines) { + b = append(b, fmt.Sprintf("p. %d- %s", h.page, strings.TrimSpace(lines[h.line+1]))) + } + return append(b, "--") +} + +// splitLines splits a page into its lines, dropping the single empty line a +// trailing newline would otherwise introduce. +func splitLines(s string) []string { + lines := strings.Split(s, "\n") + if len(lines) > 1 && lines[len(lines)-1] == "" && strings.HasSuffix(s, "\n") { + lines = lines[:len(lines)-1] + } + return lines +} diff --git a/internal/pdfdoc/pdfdoc.go b/internal/pdfdoc/pdfdoc.go new file mode 100644 index 0000000..672c2df --- /dev/null +++ b/internal/pdfdoc/pdfdoc.go @@ -0,0 +1,285 @@ +// Package pdfdoc renders an extracted document for an agent: the whole text with +// page markers when it fits, otherwise a map (title, outline with page numbers, +// leading pages and how to ask for more), a page range, or grep results with page +// numbers. It works on a *pdftext.Doc with no I/O; every length is in runes, which +// is what bounds an agent's context regardless of the byte width of a code point. +package pdfdoc + +import ( + "errors" + "fmt" + "strconv" + "strings" + "unicode/utf8" + + "git.wntrmute.dev/kyle/sift/internal/pdftext" +) + +// The errors Render returns for a malformed Pages or Grep option. +var ( + ErrBadPages = errors.New("invalid pages") + ErrBadGrep = errors.New("invalid grep") +) + +// maxPages bounds a range request so a caller cannot ask for the whole of a +// million-page document at once; the cap below still applies on top of it. +const maxPages = 100 + +// Options selects how Render draws from a document. +type Options struct { + Pages string // "" = no range; "a-b", "a" or "a-" (1-based, inclusive) + Grep string // "" = no grep; an RE2 pattern + OutCap int // characters (runes) the content may use +} + +// Rendered is what an agent sees. When Map is true Content is the map of a +// document longer than OutCap; a grep answer sets GrepHits and leaves First and +// Last at zero. +type Rendered struct { + Content string + Map bool // Content is the map of a document longer than OutCap + First, Last int // pages whose text is in Content; 0, 0 for a grep answer + GrepHits int // matches found (more may exist than are shown) + Note string // "" or how the answer was cut and how to continue +} + +// Marker is the header printed before a page's text. +func Marker(page int) string { + return fmt.Sprintf("--- page %d ---", page) +} + +// Full renders every extracted page with its marker, trimmed of surrounding +// whitespace. It is the unit the map and page ranges are built from. +func Full(doc *pdftext.Doc) string { + parts := make([]string, 0, len(doc.Text)) + for i := range doc.Text { + parts = append(parts, Marker(i+1)+"\n\n"+strings.TrimSpace(doc.Text[i])) + } + return strings.Join(parts, "\n\n") +} + +// Render dispatches on the option that is set: grep, a page range, or the whole +// document. +func Render(doc *pdftext.Doc, opt Options) (Rendered, error) { + switch { + case opt.Grep != "": + return renderGrep(doc, opt) + case opt.Pages != "": + return renderPages(doc, opt) + default: + return renderWhole(doc, opt) + } +} + +// renderWhole shows the whole document when it fits OutCap, otherwise a map that +// fits. +func renderWhole(doc *pdftext.Doc, opt Options) (Rendered, error) { + full := Full(doc) + if utf8.RuneCountInString(full) <= opt.OutCap { + return Rendered{Content: full, First: 1, Last: len(doc.Text)}, nil + } + return renderMap(doc, opt), nil +} + +// header is the fixed top of a map: the title line and the pages/token line, +// which notes how many pages were extracted when fewer than the document has. +func header(doc *pdftext.Doc) string { + title := doc.Title + if title == "" { + title = "Untitled PDF" + } + pages := fmt.Sprintf("%d pages · ~%d tokens", doc.Pages, tokenEstimate(doc)) + if len(doc.Text) < doc.Pages { + pages += fmt.Sprintf(" · pages 1-%d extracted", len(doc.Text)) + } + return "# " + title + "\n\n" + pages + "\n\n" +} + +// tokenEstimate roughs a document's size in tokens as a quarter of its runes. +func tokenEstimate(doc *pdftext.Doc) int { + return utf8.RuneCountInString(Full(doc)) / 4 +} + +// instruction is the fixed line that tells the agent how to ask for more. +const instruction = "This document is longer than one answer: ask for pages=\"a-b\" or grep=\"pattern\".\n\n" + +// renderMap assembles the header, a cuttable outline, the instruction, and whole +// leading pages that fit in a quarter of the cap. +func renderMap(doc *pdftext.Doc, opt Options) Rendered { + h := header(doc) + outlineLines := outlineItems(doc.Outline) + + // The outline is the only flexible part: cut it rather than the header or the + // instruction when the document's outline would not fit. + pagesBudget := opt.OutCap / 4 + outlineBudget := opt.OutCap - utf8.RuneCountInString(h) - utf8.RuneCountInString(instruction) - pagesBudget + if outlineBudget < 0 { + outlineBudget = 0 + } + var kept []string + running := 0 + for _, line := range outlineLines { + line += "\n" + if running+utf8.RuneCountInString(line) > outlineBudget { + break + } + running += utf8.RuneCountInString(line) + kept = append(kept, strings.TrimRight(line, "\n")) + } + outlineSection := "" + if len(kept) > 0 { + outlineSection = "## Outline\n" + strings.Join(kept, "\n") + "\n\n" + } + + pages := leadingPages(doc, opt.OutCap-utf8.RuneCountInString(h)-utf8.RuneCountInString(outlineSection)-utf8.RuneCountInString(instruction), pagesBudget) + content := h + outlineSection + instruction + pages.text + if utf8.RuneCountInString(content) > opt.OutCap { + content = truncateRunes(content, opt.OutCap) + } + return Rendered{Content: content, Map: true, First: 1, Last: pages.last} +} + +// outlineItems renders each outline item as "- (p. <page>)", two +// spaces of indent per level below the top. +func outlineItems(items []pdftext.OutlineItem) []string { + lines := make([]string, 0, len(items)) + for _, it := range items { + indent := strings.Repeat(" ", max(0, (it.Level-1)*2)) + lines = append(lines, indent+"- "+it.Title+" (p. "+strconv.Itoa(it.Page)+")") + } + return lines +} + +// pagesResult is a run of whole pages plus the last one shown. +type pagesResult struct { + text string + last int +} + +// leadingPages renders whole pages from page 1 into budget runes, at least one, +// cutting the first to fit if it is alone larger than the budget. +func leadingPages(doc *pdftext.Doc, budget, pagesBudget int) pagesResult { + if budget < 0 { + budget = 0 + } + if pagesBudget > budget { + pagesBudget = budget + } + extracted := len(doc.Text) + var parts []string + running := 0 + last := 0 + for p := 1; p <= extracted; p++ { + part := Marker(p) + "\n\n" + strings.TrimSpace(doc.Text[p-1]) + if p == 1 && utf8.RuneCountInString(part) > pagesBudget { + parts = append(parts, truncateRunes(part, pagesBudget)) + return pagesResult{text: strings.Join(parts, "\n\n"), last: 1} + } + if running+utf8.RuneCountInString(part) > pagesBudget { + break + } + running += utf8.RuneCountInString(part) + parts = append(parts, part) + last = p + } + return pagesResult{text: strings.Join(parts, "\n\n"), last: last} +} + +// renderPages renders a page range as whole pages, capped at maxPages pages and +// at OutCap runes, noting how to ask for the rest. +func renderPages(doc *pdftext.Doc, opt Options) (Rendered, error) { + start, end, err := parsePages(opt.Pages, len(doc.Text)) + if err != nil { + return Rendered{}, err + } + + limit := end + if end-start+1 > maxPages { + limit = start + maxPages - 1 + } + + var parts []string + running := 0 + last := start - 1 + note := "" + for p := start; p <= limit; p++ { + part := Marker(p) + "\n\n" + strings.TrimSpace(doc.Text[p-1]) + if p == start && utf8.RuneCountInString(part) > opt.OutCap { + parts = append(parts, truncateRunes(part, opt.OutCap)) + note = fmt.Sprintf("page %d cut to fit the answer", p) + last = p + break + } + if running+utf8.RuneCountInString(part) > opt.OutCap { + break + } + running += utf8.RuneCountInString(part) + parts = append(parts, part) + last = p + } + if last < end { + note = fmt.Sprintf("cut at page %d; ask for pages=\"%d-%d\"", last, last+1, end) + } + return Rendered{Content: strings.Join(parts, "\n\n"), First: start, Last: last, Note: note}, nil +} + +// parsePages parses a "a-b", "a" or "a-" spec against the number of extracted +// pages. It returns a range error wrapping ErrBadPages when the numbers are out +// of range, and a format error wrapping ErrBadPages otherwise. +func parsePages(spec string, extracted int) (start, end int, err error) { + s := strings.TrimSpace(spec) + start, end, endToExtracted, ok := splitSpec(s) + if !ok { + return 0, 0, fmt.Errorf("%w: invalid pages", ErrBadPages) + } + if endToExtracted { + end = extracted + } + if start < 1 || end < start || start > extracted { + return 0, 0, fmt.Errorf("%w: pages out of range (1-%d)", ErrBadPages, extracted) + } + if end > extracted { + end = extracted + } + return start, end, nil +} + +// splitSpec turns a spec into its numbers. It reports ok=false for a malformed +// spec so the caller returns a format error. A bare "a" is a single page; "a-" +// spans to the last extracted page. +func splitSpec(s string) (start, end int, endToExtracted, ok bool) { + if !strings.Contains(s, "-") { + n, err := strconv.Atoi(s) + if err != nil { + return 0, 0, false, false + } + return n, n, false, true + } + parts := strings.Split(s, "-") + if len(parts) != 2 || parts[0] == "" { + return 0, 0, false, false + } + a, err := strconv.Atoi(strings.TrimSpace(parts[0])) + if err != nil { + return 0, 0, false, false + } + if strings.TrimSpace(parts[1]) == "" { + return a, 0, true, true + } + b, err := strconv.Atoi(strings.TrimSpace(parts[1])) + if err != nil { + return 0, 0, false, false + } + return a, b, false, true +} + +// truncateRunes returns the first n runes of s, never splitting a multi-byte one. +func truncateRunes(s string, n int) string { + if n <= 0 { + return "" + } + if utf8.RuneCountInString(s) <= n { + return s + } + return string([]rune(s)[:n]) +} diff --git a/internal/pdfdoc/pdfdoc_test.go b/internal/pdfdoc/pdfdoc_test.go new file mode 100644 index 0000000..47bb287 --- /dev/null +++ b/internal/pdfdoc/pdfdoc_test.go @@ -0,0 +1,249 @@ +package pdfdoc_test + +// P5a task 05: rendering an extracted document for an agent — the whole text with page markers, +// the map for a long document, a page range, and grep. + +import ( + "errors" + "fmt" + "strings" + "testing" + "time" + "unicode/utf8" + + "git.wntrmute.dev/kyle/sift/internal/pdfdoc" + "git.wntrmute.dev/kyle/sift/internal/pdftext" +) + +func small() *pdftext.Doc { + return &pdftext.Doc{ + Meta: pdftext.Meta{Pages: 3, Title: "Three Pages", Outline: []pdftext.OutlineItem{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "2.1 Den", Page: 2, Level: 2}}}, + Text: []string{"Chapter one begins here.\nSecond line.\n", " The quick brown fox is on page two.\nIt jumps.", "Last page."}, + } +} + +// big returns n pages of about 1,000 characters each, every line naming its page. +func big(n int) *pdftext.Doc { + d := &pdftext.Doc{Meta: pdftext.Meta{Pages: n, Title: "Long Thesis", Outline: []pdftext.OutlineItem{{Title: "1 Introduction", Page: 1, Level: 1}, {Title: "2 Method", Page: 12, Level: 1}, {Title: "2.3 Setup", Page: 15, Level: 2}}}} + for p := 1; p <= n; p++ { + var lines []string + for l := 1; l <= 12; l++ { + lines = append(lines, fmt.Sprintf("page %d line %d lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod", p, l)) + } + if p == 37 { + lines[4] = "the Needle is here on page 37" + } + d.Text = append(d.Text, strings.Join(lines, "\n")) + } + return d +} + +func TestFull(t *testing.T) { + want := "--- page 1 ---\n\nChapter one begins here.\nSecond line.\n\n--- page 2 ---\n\nThe quick brown fox is on page two.\nIt jumps.\n\n--- page 3 ---\n\nLast page." + if got := pdfdoc.Full(small()); got != want { + t.Errorf("Full =\n%q\nwant\n%q", got, want) + } + if pdfdoc.Marker(12) != "--- page 12 ---" { + t.Errorf("Marker(12) = %q", pdfdoc.Marker(12)) + } +} + +func TestWholeDocumentWhenItFits(t *testing.T) { + r, err := pdfdoc.Render(small(), pdfdoc.Options{OutCap: 10000}) + if err != nil { + t.Fatal(err) + } + if r.Map || r.First != 1 || r.Last != 3 || r.Content != pdfdoc.Full(small()) { + t.Errorf("got %+v", r) + } +} + +func TestMapForALongDocument(t *testing.T) { + d := big(60) // ~60k characters + r, err := pdfdoc.Render(d, pdfdoc.Options{OutCap: 20000}) + if err != nil { + t.Fatal(err) + } + if !r.Map || r.First != 1 || r.Last < 1 { + t.Fatalf("got Map=%v First=%d Last=%d, want a map from page 1", r.Map, r.First, r.Last) + } + c := r.Content + for _, want := range []string{ + "# Long Thesis\n", + "60 pages · ~", + "## Outline\n", + "- 1 Introduction (p. 1)\n", + "- 2 Method (p. 12)\n", + " - 2.3 Setup (p. 15)\n", // level 2 is indented two spaces + `pages="a-b"`, + `grep="pattern"`, + "--- page 1 ---", + } { + if !strings.Contains(c, want) { + t.Errorf("map lacks %q", want) + } + } + if utf8.RuneCountInString(c) > 20000 { + t.Errorf("map is %d characters, over OutCap 20000", utf8.RuneCountInString(c)) + } + // Leading pages take at most a quarter of the cap, and only whole pages. + if strings.Contains(c, fmt.Sprintf("--- page %d ---", r.Last+1)) { + t.Errorf("content has page %d past Last %d", r.Last+1, r.Last) + } + if !strings.Contains(c, fmt.Sprintf("page %d line 12", r.Last)) { + t.Errorf("the last page shown (%d) is cut; pages in a map are whole", r.Last) + } + idx := strings.Index(c, "--- page 1 ---") + if pagesPart := c[idx:]; utf8.RuneCountInString(pagesPart) > 20000/4 { + t.Errorf("leading pages take %d characters, over OutCap/4", utf8.RuneCountInString(pagesPart)) + } +} + +func TestMapNotesPartialExtraction(t *testing.T) { + d := big(30) + d.Pages = 2500 // the child stopped at its page cap + r, _ := pdfdoc.Render(d, pdfdoc.Options{OutCap: 5000}) + if !strings.Contains(r.Content, "2500 pages") || !strings.Contains(r.Content, "pages 1-30 extracted") { + t.Errorf("map must say how much was extracted:\n%s", r.Content[:min(600, len(r.Content))]) + } +} + +func TestUntitledAndNoOutline(t *testing.T) { + d := big(40) + d.Title, d.Outline = "", nil + r, _ := pdfdoc.Render(d, pdfdoc.Options{OutCap: 5000}) + if !strings.HasPrefix(r.Content, "# Untitled PDF\n") || strings.Contains(r.Content, "## Outline") { + t.Errorf("untitled, outline-less map:\n%s", r.Content[:min(400, len(r.Content))]) + } +} + +func TestPages(t *testing.T) { + d := big(60) + for _, tc := range []struct { + spec string + first, last int + }{ + {"2-3", 2, 3}, + {"7", 7, 7}, + {" 58- ", 58, 60}, + {"60-60", 60, 60}, + } { + r, err := pdfdoc.Render(d, pdfdoc.Options{Pages: tc.spec, OutCap: 200000}) + if err != nil { + t.Errorf("%q: %v", tc.spec, err) + continue + } + if r.Map || r.First != tc.first || r.Last != tc.last { + t.Errorf("%q: First %d Last %d Map %v, want %d-%d", tc.spec, r.First, r.Last, r.Map, tc.first, tc.last) + } + if !strings.HasPrefix(r.Content, pdfdoc.Marker(tc.first)+"\n\n") || strings.Contains(r.Content, pdfdoc.Marker(tc.last+1)) || (tc.first > 1 && strings.Contains(r.Content, pdfdoc.Marker(tc.first-1)+"\n")) { + t.Errorf("%q: wrong pages in content: %.120q…", tc.spec, r.Content) + } + } + for _, bad := range []string{"0-1", "5-4", "61", "999", "a-b", "1-2-3", "", "-5"} { + if bad == "" { + continue // empty means "no range", tested above + } + _, err := pdfdoc.Render(d, pdfdoc.Options{Pages: bad, OutCap: 200000}) + if !errors.Is(err, pdfdoc.ErrBadPages) { + t.Errorf("%q: %v, want ErrBadPages", bad, err) + } + } + _, err := pdfdoc.Render(d, pdfdoc.Options{Pages: "61", OutCap: 200000}) + if err == nil || !strings.Contains(err.Error(), "pages out of range (1-60)") { + t.Errorf("out-of-range message: %v", err) + } +} + +// A range is at most 100 pages and at most OutCap characters, cut at a whole page with a note +// that says how to continue. +func TestPagesAreCapped(t *testing.T) { + d := big(300) + r, err := pdfdoc.Render(d, pdfdoc.Options{Pages: "1-300", OutCap: 1 << 30}) + if err != nil { + t.Fatal(err) + } + if r.Last != 100 || !strings.Contains(r.Note, `pages="101-300"`) { + t.Errorf("1-300: Last %d note %q; want 100 and a pointer to 101-300", r.Last, r.Note) + } + r, _ = pdfdoc.Render(d, pdfdoc.Options{Pages: "10-40", OutCap: 5000}) + if utf8.RuneCountInString(r.Content) > 5000 || r.First != 10 || r.Last < 10 || r.Last >= 40 { + t.Errorf("10-40 under a 5000 cap: First %d Last %d, %d chars", r.First, r.Last, utf8.RuneCountInString(r.Content)) + } + if !strings.Contains(r.Note, fmt.Sprintf(`pages="%d-40"`, r.Last+1)) { + t.Errorf("note %q must point at the rest", r.Note) + } + // One page bigger than the cap is truncated rather than dropped. + huge := &pdftext.Doc{Meta: pdftext.Meta{Pages: 1}, Text: []string{strings.Repeat("w ", 10000)}} + r, _ = pdfdoc.Render(huge, pdfdoc.Options{Pages: "1", OutCap: 1000}) + if r.First != 1 || r.Last != 1 || utf8.RuneCountInString(r.Content) > 1000 || r.Note == "" { + t.Errorf("oversized single page: %+v (%d chars)", r, utf8.RuneCountInString(r.Content)) + } +} + +func TestGrep(t *testing.T) { + d := big(60) + r, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "needle", OutCap: 200000}) + if err != nil { + t.Fatal(err) + } + if r.GrepHits != 1 || r.First != 0 || r.Map { + t.Errorf("got %+v", r) + } + for _, want := range []string{ + `grep "needle": 1 match`, + "p. 37- page 37 line 4", + "p. 37: the Needle is here on page 37", + "p. 37- page 37 line 6", + } { + if !strings.Contains(r.Content, want) { + t.Errorf("grep output lacks %q:\n%s", want, r.Content) + } + } + // Smart case: an upper-case letter makes the pattern case-sensitive. + if r, _ := pdfdoc.Render(d, pdfdoc.Options{Grep: "NEEDLE", OutCap: 200000}); r.GrepHits != 0 { + t.Errorf("NEEDLE matched %d times, want 0 (case-sensitive)", r.GrepHits) + } + if r, _ := pdfdoc.Render(d, pdfdoc.Options{Grep: "Needle", OutCap: 200000}); r.GrepHits != 1 { + t.Errorf("Needle matched %d times, want 1", r.GrepHits) + } + // No match is an answer, not an error. + r, err = pdfdoc.Render(d, pdfdoc.Options{Grep: "zebra", OutCap: 200000}) + if err != nil || r.GrepHits != 0 || !strings.Contains(r.Content, `grep "zebra": no matches`) { + t.Errorf("no match: %+v %v", r, err) + } + // Within a page range. + r, _ = pdfdoc.Render(d, pdfdoc.Options{Grep: "line 3 ", Pages: "5-6", OutCap: 200000}) + if r.GrepHits != 2 || strings.Contains(r.Content, "p. 7") { + t.Errorf("grep in 5-6: %d hits\n%s", r.GrepHits, r.Content) + } +} + +func TestGrepLimits(t *testing.T) { + d := big(200) + r, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "lorem", OutCap: 1 << 30}) // 2,399 matches: every line but the needle + if err != nil { + t.Fatal(err) + } + if r.GrepHits != 2399 || strings.Count(r.Content, ": page ") != 100 || !strings.Contains(r.Content, "showing the first 100") { + t.Errorf("hits %d, shown %d", r.GrepHits, strings.Count(r.Content, ": page ")) + } + if _, err := pdfdoc.Render(d, pdfdoc.Options{Grep: strings.Repeat("a", 201), OutCap: 1000}); !errors.Is(err, pdfdoc.ErrBadGrep) { + t.Errorf("201-byte pattern: %v, want ErrBadGrep", err) + } + if _, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "(unclosed", OutCap: 1000}); !errors.Is(err, pdfdoc.ErrBadGrep) { + t.Errorf("bad regexp: %v, want ErrBadGrep", err) + } + // RE2 is linear: the classic catastrophic pattern over a 1 MB page finishes quickly. + evil := &pdftext.Doc{Meta: pdftext.Meta{Pages: 1}, Text: []string{strings.Repeat("a", 1<<20) + "!"}} + start := time.Now() + _, _ = pdfdoc.Render(evil, pdfdoc.Options{Grep: "(a+)+$", OutCap: 1000}) + if d := time.Since(start); d > time.Second { + t.Errorf("(a+)+$ took %v", d) + } + // Output stays under the cap. + r, _ = pdfdoc.Render(d, pdfdoc.Options{Grep: "lorem", OutCap: 3000}) + if utf8.RuneCountInString(r.Content) > 3000 { + t.Errorf("grep output %d characters over a 3000 cap", utf8.RuneCountInString(r.Content)) + } +}