pdfdoc: page markers, the map for long documents, page ranges, grep
Pure render function on a *pdftext.Doc with no I/O: the whole text with page markers when it fits, otherwise a map (title, outline with page numbers, leading pages and how to ask for more), a page range, or grep results with page numbers. Every length is in runes. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
ornith-1.5-35b-a3b
@@ -0,0 +1,153 @@
|
||||
package pdfdoc
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/pdftext"
|
||||
)
|
||||
|
||||
// maxGrepMatches bounds how many match blocks are shown; the count reported is
|
||||
// always the true total.
|
||||
const maxGrepMatches = 100
|
||||
|
||||
// maxGrepPatternBytes bounds a grep pattern; a longer one is rejected before it
|
||||
// can compile.
|
||||
const maxGrepPatternBytes = 200
|
||||
|
||||
// hit is one match: the 1-based page and the 0-based line index within it.
|
||||
type hit struct{ page, line int }
|
||||
|
||||
// renderGrep searches every line of every page in the range (all pages when no
|
||||
// range is given) and returns the matches with their page numbers.
|
||||
func renderGrep(doc *pdftext.Doc, opt Options) (Rendered, error) {
|
||||
pattern := opt.Grep
|
||||
if len(pattern) > maxGrepPatternBytes {
|
||||
return Rendered{}, fmt.Errorf("%w: pattern over %d bytes", ErrBadGrep, maxGrepPatternBytes)
|
||||
}
|
||||
re, err := regexp.Compile(grepPattern(pattern))
|
||||
if err != nil {
|
||||
return Rendered{}, fmt.Errorf("%w: %v", ErrBadGrep, err)
|
||||
}
|
||||
|
||||
start, end := 1, len(doc.Text)
|
||||
if opt.Pages != "" {
|
||||
s, e, err := parsePages(opt.Pages, len(doc.Text))
|
||||
if err != nil {
|
||||
return Rendered{}, err
|
||||
}
|
||||
start, end = s, e
|
||||
}
|
||||
|
||||
var hits []hit
|
||||
linesByPage := make(map[int][]string, end-start+1)
|
||||
for p := start; p <= end; p++ {
|
||||
lines := splitLines(doc.Text[p-1])
|
||||
linesByPage[p] = lines
|
||||
for j := range lines {
|
||||
if re.MatchString(lines[j]) {
|
||||
hits = append(hits, hit{p, j})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
n := len(hits)
|
||||
var header string
|
||||
switch {
|
||||
case n == 0:
|
||||
header = `grep "` + pattern + `": no matches`
|
||||
case n == 1:
|
||||
header = `grep "` + pattern + `": 1 match`
|
||||
default:
|
||||
header = fmt.Sprintf(`grep "%s": %d matches`, pattern, n)
|
||||
}
|
||||
if n > maxGrepMatches {
|
||||
header += ", showing the first " + strconv.Itoa(maxGrepMatches)
|
||||
}
|
||||
|
||||
maxBlocks := n
|
||||
if maxBlocks > maxGrepMatches {
|
||||
maxBlocks = maxGrepMatches
|
||||
}
|
||||
|
||||
outLines := make([]string, 0, 16+len(hits))
|
||||
outLines = append(outLines, header)
|
||||
running := utf8.RuneCountInString(header)
|
||||
shown := 0
|
||||
for i := 0; i < maxBlocks; i++ {
|
||||
block := matchBlock(hits[i], linesByPage)
|
||||
add := utf8.RuneCountInString(strings.Join(block, "\n"))
|
||||
if i == 0 {
|
||||
if running+add > opt.OutCap {
|
||||
break
|
||||
}
|
||||
} else if running+1+add > opt.OutCap {
|
||||
break
|
||||
}
|
||||
outLines = append(outLines, block...)
|
||||
running += add
|
||||
if i > 0 {
|
||||
running++
|
||||
}
|
||||
shown++
|
||||
}
|
||||
content := strings.Join(outLines, "\n")
|
||||
if utf8.RuneCountInString(content) > opt.OutCap {
|
||||
content = truncateRunes(content, opt.OutCap)
|
||||
}
|
||||
|
||||
note := ""
|
||||
if shown < maxBlocks {
|
||||
note = fmt.Sprintf("showing %d of %d matches", shown, n)
|
||||
}
|
||||
return Rendered{Content: content, GrepHits: n, Note: note}, nil
|
||||
}
|
||||
|
||||
// grepPattern applies smart case: it lowercases-sensitively prefixes "(?i)"
|
||||
// unless the pattern already contains an upper-case letter, which asks for a
|
||||
// case-sensitive search.
|
||||
func grepPattern(pattern string) string {
|
||||
if hasUpper(pattern) {
|
||||
return pattern
|
||||
}
|
||||
return "(?i)" + pattern
|
||||
}
|
||||
|
||||
// hasUpper reports whether s contains an upper-case ASCII letter.
|
||||
func hasUpper(s string) bool {
|
||||
for _, r := range s {
|
||||
if r >= 'A' && r <= 'Z' {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// matchBlock renders one hit as its previous line, the matching line, the next
|
||||
// line and a separator. Context lines that do not exist (the match is on the
|
||||
// first or last line) are omitted.
|
||||
func matchBlock(h hit, linesByPage map[int][]string) []string {
|
||||
lines := linesByPage[h.page]
|
||||
var b []string
|
||||
if h.line > 0 {
|
||||
b = append(b, fmt.Sprintf("p. %d- %s", h.page, strings.TrimSpace(lines[h.line-1])))
|
||||
}
|
||||
b = append(b, fmt.Sprintf("p. %d: %s", h.page, strings.TrimSpace(lines[h.line])))
|
||||
if h.line+1 < len(lines) {
|
||||
b = append(b, fmt.Sprintf("p. %d- %s", h.page, strings.TrimSpace(lines[h.line+1])))
|
||||
}
|
||||
return append(b, "--")
|
||||
}
|
||||
|
||||
// splitLines splits a page into its lines, dropping the single empty line a
|
||||
// trailing newline would otherwise introduce.
|
||||
func splitLines(s string) []string {
|
||||
lines := strings.Split(s, "\n")
|
||||
if len(lines) > 1 && lines[len(lines)-1] == "" && strings.HasSuffix(s, "\n") {
|
||||
lines = lines[:len(lines)-1]
|
||||
}
|
||||
return lines
|
||||
}
|
||||
@@ -0,0 +1,285 @@
|
||||
// Package pdfdoc renders an extracted document for an agent: the whole text with
|
||||
// page markers when it fits, otherwise a map (title, outline with page numbers,
|
||||
// leading pages and how to ask for more), a page range, or grep results with page
|
||||
// numbers. It works on a *pdftext.Doc with no I/O; every length is in runes, which
|
||||
// is what bounds an agent's context regardless of the byte width of a code point.
|
||||
package pdfdoc
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/pdftext"
|
||||
)
|
||||
|
||||
// The errors Render returns for a malformed Pages or Grep option.
|
||||
var (
|
||||
ErrBadPages = errors.New("invalid pages")
|
||||
ErrBadGrep = errors.New("invalid grep")
|
||||
)
|
||||
|
||||
// maxPages bounds a range request so a caller cannot ask for the whole of a
|
||||
// million-page document at once; the cap below still applies on top of it.
|
||||
const maxPages = 100
|
||||
|
||||
// Options selects how Render draws from a document.
|
||||
type Options struct {
|
||||
Pages string // "" = no range; "a-b", "a" or "a-" (1-based, inclusive)
|
||||
Grep string // "" = no grep; an RE2 pattern
|
||||
OutCap int // characters (runes) the content may use
|
||||
}
|
||||
|
||||
// Rendered is what an agent sees. When Map is true Content is the map of a
|
||||
// document longer than OutCap; a grep answer sets GrepHits and leaves First and
|
||||
// Last at zero.
|
||||
type Rendered struct {
|
||||
Content string
|
||||
Map bool // Content is the map of a document longer than OutCap
|
||||
First, Last int // pages whose text is in Content; 0, 0 for a grep answer
|
||||
GrepHits int // matches found (more may exist than are shown)
|
||||
Note string // "" or how the answer was cut and how to continue
|
||||
}
|
||||
|
||||
// Marker is the header printed before a page's text.
|
||||
func Marker(page int) string {
|
||||
return fmt.Sprintf("--- page %d ---", page)
|
||||
}
|
||||
|
||||
// Full renders every extracted page with its marker, trimmed of surrounding
|
||||
// whitespace. It is the unit the map and page ranges are built from.
|
||||
func Full(doc *pdftext.Doc) string {
|
||||
parts := make([]string, 0, len(doc.Text))
|
||||
for i := range doc.Text {
|
||||
parts = append(parts, Marker(i+1)+"\n\n"+strings.TrimSpace(doc.Text[i]))
|
||||
}
|
||||
return strings.Join(parts, "\n\n")
|
||||
}
|
||||
|
||||
// Render dispatches on the option that is set: grep, a page range, or the whole
|
||||
// document.
|
||||
func Render(doc *pdftext.Doc, opt Options) (Rendered, error) {
|
||||
switch {
|
||||
case opt.Grep != "":
|
||||
return renderGrep(doc, opt)
|
||||
case opt.Pages != "":
|
||||
return renderPages(doc, opt)
|
||||
default:
|
||||
return renderWhole(doc, opt)
|
||||
}
|
||||
}
|
||||
|
||||
// renderWhole shows the whole document when it fits OutCap, otherwise a map that
|
||||
// fits.
|
||||
func renderWhole(doc *pdftext.Doc, opt Options) (Rendered, error) {
|
||||
full := Full(doc)
|
||||
if utf8.RuneCountInString(full) <= opt.OutCap {
|
||||
return Rendered{Content: full, First: 1, Last: len(doc.Text)}, nil
|
||||
}
|
||||
return renderMap(doc, opt), nil
|
||||
}
|
||||
|
||||
// header is the fixed top of a map: the title line and the pages/token line,
|
||||
// which notes how many pages were extracted when fewer than the document has.
|
||||
func header(doc *pdftext.Doc) string {
|
||||
title := doc.Title
|
||||
if title == "" {
|
||||
title = "Untitled PDF"
|
||||
}
|
||||
pages := fmt.Sprintf("%d pages · ~%d tokens", doc.Pages, tokenEstimate(doc))
|
||||
if len(doc.Text) < doc.Pages {
|
||||
pages += fmt.Sprintf(" · pages 1-%d extracted", len(doc.Text))
|
||||
}
|
||||
return "# " + title + "\n\n" + pages + "\n\n"
|
||||
}
|
||||
|
||||
// tokenEstimate roughs a document's size in tokens as a quarter of its runes.
|
||||
func tokenEstimate(doc *pdftext.Doc) int {
|
||||
return utf8.RuneCountInString(Full(doc)) / 4
|
||||
}
|
||||
|
||||
// instruction is the fixed line that tells the agent how to ask for more.
|
||||
const instruction = "This document is longer than one answer: ask for pages=\"a-b\" or grep=\"pattern\".\n\n"
|
||||
|
||||
// renderMap assembles the header, a cuttable outline, the instruction, and whole
|
||||
// leading pages that fit in a quarter of the cap.
|
||||
func renderMap(doc *pdftext.Doc, opt Options) Rendered {
|
||||
h := header(doc)
|
||||
outlineLines := outlineItems(doc.Outline)
|
||||
|
||||
// The outline is the only flexible part: cut it rather than the header or the
|
||||
// instruction when the document's outline would not fit.
|
||||
pagesBudget := opt.OutCap / 4
|
||||
outlineBudget := opt.OutCap - utf8.RuneCountInString(h) - utf8.RuneCountInString(instruction) - pagesBudget
|
||||
if outlineBudget < 0 {
|
||||
outlineBudget = 0
|
||||
}
|
||||
var kept []string
|
||||
running := 0
|
||||
for _, line := range outlineLines {
|
||||
line += "\n"
|
||||
if running+utf8.RuneCountInString(line) > outlineBudget {
|
||||
break
|
||||
}
|
||||
running += utf8.RuneCountInString(line)
|
||||
kept = append(kept, strings.TrimRight(line, "\n"))
|
||||
}
|
||||
outlineSection := ""
|
||||
if len(kept) > 0 {
|
||||
outlineSection = "## Outline\n" + strings.Join(kept, "\n") + "\n\n"
|
||||
}
|
||||
|
||||
pages := leadingPages(doc, opt.OutCap-utf8.RuneCountInString(h)-utf8.RuneCountInString(outlineSection)-utf8.RuneCountInString(instruction), pagesBudget)
|
||||
content := h + outlineSection + instruction + pages.text
|
||||
if utf8.RuneCountInString(content) > opt.OutCap {
|
||||
content = truncateRunes(content, opt.OutCap)
|
||||
}
|
||||
return Rendered{Content: content, Map: true, First: 1, Last: pages.last}
|
||||
}
|
||||
|
||||
// outlineItems renders each outline item as "- <indent><title> (p. <page>)", two
|
||||
// spaces of indent per level below the top.
|
||||
func outlineItems(items []pdftext.OutlineItem) []string {
|
||||
lines := make([]string, 0, len(items))
|
||||
for _, it := range items {
|
||||
indent := strings.Repeat(" ", max(0, (it.Level-1)*2))
|
||||
lines = append(lines, indent+"- "+it.Title+" (p. "+strconv.Itoa(it.Page)+")")
|
||||
}
|
||||
return lines
|
||||
}
|
||||
|
||||
// pagesResult is a run of whole pages plus the last one shown.
|
||||
type pagesResult struct {
|
||||
text string
|
||||
last int
|
||||
}
|
||||
|
||||
// leadingPages renders whole pages from page 1 into budget runes, at least one,
|
||||
// cutting the first to fit if it is alone larger than the budget.
|
||||
func leadingPages(doc *pdftext.Doc, budget, pagesBudget int) pagesResult {
|
||||
if budget < 0 {
|
||||
budget = 0
|
||||
}
|
||||
if pagesBudget > budget {
|
||||
pagesBudget = budget
|
||||
}
|
||||
extracted := len(doc.Text)
|
||||
var parts []string
|
||||
running := 0
|
||||
last := 0
|
||||
for p := 1; p <= extracted; p++ {
|
||||
part := Marker(p) + "\n\n" + strings.TrimSpace(doc.Text[p-1])
|
||||
if p == 1 && utf8.RuneCountInString(part) > pagesBudget {
|
||||
parts = append(parts, truncateRunes(part, pagesBudget))
|
||||
return pagesResult{text: strings.Join(parts, "\n\n"), last: 1}
|
||||
}
|
||||
if running+utf8.RuneCountInString(part) > pagesBudget {
|
||||
break
|
||||
}
|
||||
running += utf8.RuneCountInString(part)
|
||||
parts = append(parts, part)
|
||||
last = p
|
||||
}
|
||||
return pagesResult{text: strings.Join(parts, "\n\n"), last: last}
|
||||
}
|
||||
|
||||
// renderPages renders a page range as whole pages, capped at maxPages pages and
|
||||
// at OutCap runes, noting how to ask for the rest.
|
||||
func renderPages(doc *pdftext.Doc, opt Options) (Rendered, error) {
|
||||
start, end, err := parsePages(opt.Pages, len(doc.Text))
|
||||
if err != nil {
|
||||
return Rendered{}, err
|
||||
}
|
||||
|
||||
limit := end
|
||||
if end-start+1 > maxPages {
|
||||
limit = start + maxPages - 1
|
||||
}
|
||||
|
||||
var parts []string
|
||||
running := 0
|
||||
last := start - 1
|
||||
note := ""
|
||||
for p := start; p <= limit; p++ {
|
||||
part := Marker(p) + "\n\n" + strings.TrimSpace(doc.Text[p-1])
|
||||
if p == start && utf8.RuneCountInString(part) > opt.OutCap {
|
||||
parts = append(parts, truncateRunes(part, opt.OutCap))
|
||||
note = fmt.Sprintf("page %d cut to fit the answer", p)
|
||||
last = p
|
||||
break
|
||||
}
|
||||
if running+utf8.RuneCountInString(part) > opt.OutCap {
|
||||
break
|
||||
}
|
||||
running += utf8.RuneCountInString(part)
|
||||
parts = append(parts, part)
|
||||
last = p
|
||||
}
|
||||
if last < end {
|
||||
note = fmt.Sprintf("cut at page %d; ask for pages=\"%d-%d\"", last, last+1, end)
|
||||
}
|
||||
return Rendered{Content: strings.Join(parts, "\n\n"), First: start, Last: last, Note: note}, nil
|
||||
}
|
||||
|
||||
// parsePages parses a "a-b", "a" or "a-" spec against the number of extracted
|
||||
// pages. It returns a range error wrapping ErrBadPages when the numbers are out
|
||||
// of range, and a format error wrapping ErrBadPages otherwise.
|
||||
func parsePages(spec string, extracted int) (start, end int, err error) {
|
||||
s := strings.TrimSpace(spec)
|
||||
start, end, endToExtracted, ok := splitSpec(s)
|
||||
if !ok {
|
||||
return 0, 0, fmt.Errorf("%w: invalid pages", ErrBadPages)
|
||||
}
|
||||
if endToExtracted {
|
||||
end = extracted
|
||||
}
|
||||
if start < 1 || end < start || start > extracted {
|
||||
return 0, 0, fmt.Errorf("%w: pages out of range (1-%d)", ErrBadPages, extracted)
|
||||
}
|
||||
if end > extracted {
|
||||
end = extracted
|
||||
}
|
||||
return start, end, nil
|
||||
}
|
||||
|
||||
// splitSpec turns a spec into its numbers. It reports ok=false for a malformed
|
||||
// spec so the caller returns a format error. A bare "a" is a single page; "a-"
|
||||
// spans to the last extracted page.
|
||||
func splitSpec(s string) (start, end int, endToExtracted, ok bool) {
|
||||
if !strings.Contains(s, "-") {
|
||||
n, err := strconv.Atoi(s)
|
||||
if err != nil {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
return n, n, false, true
|
||||
}
|
||||
parts := strings.Split(s, "-")
|
||||
if len(parts) != 2 || parts[0] == "" {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
a, err := strconv.Atoi(strings.TrimSpace(parts[0]))
|
||||
if err != nil {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
if strings.TrimSpace(parts[1]) == "" {
|
||||
return a, 0, true, true
|
||||
}
|
||||
b, err := strconv.Atoi(strings.TrimSpace(parts[1]))
|
||||
if err != nil {
|
||||
return 0, 0, false, false
|
||||
}
|
||||
return a, b, false, true
|
||||
}
|
||||
|
||||
// truncateRunes returns the first n runes of s, never splitting a multi-byte one.
|
||||
func truncateRunes(s string, n int) string {
|
||||
if n <= 0 {
|
||||
return ""
|
||||
}
|
||||
if utf8.RuneCountInString(s) <= n {
|
||||
return s
|
||||
}
|
||||
return string([]rune(s)[:n])
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package pdfdoc_test
|
||||
|
||||
// P5a task 05: rendering an extracted document for an agent — the whole text with page markers,
|
||||
// the map for a long document, a page range, and grep.
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/pdfdoc"
|
||||
"git.wntrmute.dev/kyle/sift/internal/pdftext"
|
||||
)
|
||||
|
||||
func small() *pdftext.Doc {
|
||||
return &pdftext.Doc{
|
||||
Meta: pdftext.Meta{Pages: 3, Title: "Three Pages", Outline: []pdftext.OutlineItem{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "2.1 Den", Page: 2, Level: 2}}},
|
||||
Text: []string{"Chapter one begins here.\nSecond line.\n", " The quick brown fox is on page two.\nIt jumps.", "Last page."},
|
||||
}
|
||||
}
|
||||
|
||||
// big returns n pages of about 1,000 characters each, every line naming its page.
|
||||
func big(n int) *pdftext.Doc {
|
||||
d := &pdftext.Doc{Meta: pdftext.Meta{Pages: n, Title: "Long Thesis", Outline: []pdftext.OutlineItem{{Title: "1 Introduction", Page: 1, Level: 1}, {Title: "2 Method", Page: 12, Level: 1}, {Title: "2.3 Setup", Page: 15, Level: 2}}}}
|
||||
for p := 1; p <= n; p++ {
|
||||
var lines []string
|
||||
for l := 1; l <= 12; l++ {
|
||||
lines = append(lines, fmt.Sprintf("page %d line %d lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod", p, l))
|
||||
}
|
||||
if p == 37 {
|
||||
lines[4] = "the Needle is here on page 37"
|
||||
}
|
||||
d.Text = append(d.Text, strings.Join(lines, "\n"))
|
||||
}
|
||||
return d
|
||||
}
|
||||
|
||||
func TestFull(t *testing.T) {
|
||||
want := "--- page 1 ---\n\nChapter one begins here.\nSecond line.\n\n--- page 2 ---\n\nThe quick brown fox is on page two.\nIt jumps.\n\n--- page 3 ---\n\nLast page."
|
||||
if got := pdfdoc.Full(small()); got != want {
|
||||
t.Errorf("Full =\n%q\nwant\n%q", got, want)
|
||||
}
|
||||
if pdfdoc.Marker(12) != "--- page 12 ---" {
|
||||
t.Errorf("Marker(12) = %q", pdfdoc.Marker(12))
|
||||
}
|
||||
}
|
||||
|
||||
func TestWholeDocumentWhenItFits(t *testing.T) {
|
||||
r, err := pdfdoc.Render(small(), pdfdoc.Options{OutCap: 10000})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Map || r.First != 1 || r.Last != 3 || r.Content != pdfdoc.Full(small()) {
|
||||
t.Errorf("got %+v", r)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapForALongDocument(t *testing.T) {
|
||||
d := big(60) // ~60k characters
|
||||
r, err := pdfdoc.Render(d, pdfdoc.Options{OutCap: 20000})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !r.Map || r.First != 1 || r.Last < 1 {
|
||||
t.Fatalf("got Map=%v First=%d Last=%d, want a map from page 1", r.Map, r.First, r.Last)
|
||||
}
|
||||
c := r.Content
|
||||
for _, want := range []string{
|
||||
"# Long Thesis\n",
|
||||
"60 pages · ~",
|
||||
"## Outline\n",
|
||||
"- 1 Introduction (p. 1)\n",
|
||||
"- 2 Method (p. 12)\n",
|
||||
" - 2.3 Setup (p. 15)\n", // level 2 is indented two spaces
|
||||
`pages="a-b"`,
|
||||
`grep="pattern"`,
|
||||
"--- page 1 ---",
|
||||
} {
|
||||
if !strings.Contains(c, want) {
|
||||
t.Errorf("map lacks %q", want)
|
||||
}
|
||||
}
|
||||
if utf8.RuneCountInString(c) > 20000 {
|
||||
t.Errorf("map is %d characters, over OutCap 20000", utf8.RuneCountInString(c))
|
||||
}
|
||||
// Leading pages take at most a quarter of the cap, and only whole pages.
|
||||
if strings.Contains(c, fmt.Sprintf("--- page %d ---", r.Last+1)) {
|
||||
t.Errorf("content has page %d past Last %d", r.Last+1, r.Last)
|
||||
}
|
||||
if !strings.Contains(c, fmt.Sprintf("page %d line 12", r.Last)) {
|
||||
t.Errorf("the last page shown (%d) is cut; pages in a map are whole", r.Last)
|
||||
}
|
||||
idx := strings.Index(c, "--- page 1 ---")
|
||||
if pagesPart := c[idx:]; utf8.RuneCountInString(pagesPart) > 20000/4 {
|
||||
t.Errorf("leading pages take %d characters, over OutCap/4", utf8.RuneCountInString(pagesPart))
|
||||
}
|
||||
}
|
||||
|
||||
func TestMapNotesPartialExtraction(t *testing.T) {
|
||||
d := big(30)
|
||||
d.Pages = 2500 // the child stopped at its page cap
|
||||
r, _ := pdfdoc.Render(d, pdfdoc.Options{OutCap: 5000})
|
||||
if !strings.Contains(r.Content, "2500 pages") || !strings.Contains(r.Content, "pages 1-30 extracted") {
|
||||
t.Errorf("map must say how much was extracted:\n%s", r.Content[:min(600, len(r.Content))])
|
||||
}
|
||||
}
|
||||
|
||||
func TestUntitledAndNoOutline(t *testing.T) {
|
||||
d := big(40)
|
||||
d.Title, d.Outline = "", nil
|
||||
r, _ := pdfdoc.Render(d, pdfdoc.Options{OutCap: 5000})
|
||||
if !strings.HasPrefix(r.Content, "# Untitled PDF\n") || strings.Contains(r.Content, "## Outline") {
|
||||
t.Errorf("untitled, outline-less map:\n%s", r.Content[:min(400, len(r.Content))])
|
||||
}
|
||||
}
|
||||
|
||||
func TestPages(t *testing.T) {
|
||||
d := big(60)
|
||||
for _, tc := range []struct {
|
||||
spec string
|
||||
first, last int
|
||||
}{
|
||||
{"2-3", 2, 3},
|
||||
{"7", 7, 7},
|
||||
{" 58- ", 58, 60},
|
||||
{"60-60", 60, 60},
|
||||
} {
|
||||
r, err := pdfdoc.Render(d, pdfdoc.Options{Pages: tc.spec, OutCap: 200000})
|
||||
if err != nil {
|
||||
t.Errorf("%q: %v", tc.spec, err)
|
||||
continue
|
||||
}
|
||||
if r.Map || r.First != tc.first || r.Last != tc.last {
|
||||
t.Errorf("%q: First %d Last %d Map %v, want %d-%d", tc.spec, r.First, r.Last, r.Map, tc.first, tc.last)
|
||||
}
|
||||
if !strings.HasPrefix(r.Content, pdfdoc.Marker(tc.first)+"\n\n") || strings.Contains(r.Content, pdfdoc.Marker(tc.last+1)) || (tc.first > 1 && strings.Contains(r.Content, pdfdoc.Marker(tc.first-1)+"\n")) {
|
||||
t.Errorf("%q: wrong pages in content: %.120q…", tc.spec, r.Content)
|
||||
}
|
||||
}
|
||||
for _, bad := range []string{"0-1", "5-4", "61", "999", "a-b", "1-2-3", "", "-5"} {
|
||||
if bad == "" {
|
||||
continue // empty means "no range", tested above
|
||||
}
|
||||
_, err := pdfdoc.Render(d, pdfdoc.Options{Pages: bad, OutCap: 200000})
|
||||
if !errors.Is(err, pdfdoc.ErrBadPages) {
|
||||
t.Errorf("%q: %v, want ErrBadPages", bad, err)
|
||||
}
|
||||
}
|
||||
_, err := pdfdoc.Render(d, pdfdoc.Options{Pages: "61", OutCap: 200000})
|
||||
if err == nil || !strings.Contains(err.Error(), "pages out of range (1-60)") {
|
||||
t.Errorf("out-of-range message: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
// A range is at most 100 pages and at most OutCap characters, cut at a whole page with a note
|
||||
// that says how to continue.
|
||||
func TestPagesAreCapped(t *testing.T) {
|
||||
d := big(300)
|
||||
r, err := pdfdoc.Render(d, pdfdoc.Options{Pages: "1-300", OutCap: 1 << 30})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.Last != 100 || !strings.Contains(r.Note, `pages="101-300"`) {
|
||||
t.Errorf("1-300: Last %d note %q; want 100 and a pointer to 101-300", r.Last, r.Note)
|
||||
}
|
||||
r, _ = pdfdoc.Render(d, pdfdoc.Options{Pages: "10-40", OutCap: 5000})
|
||||
if utf8.RuneCountInString(r.Content) > 5000 || r.First != 10 || r.Last < 10 || r.Last >= 40 {
|
||||
t.Errorf("10-40 under a 5000 cap: First %d Last %d, %d chars", r.First, r.Last, utf8.RuneCountInString(r.Content))
|
||||
}
|
||||
if !strings.Contains(r.Note, fmt.Sprintf(`pages="%d-40"`, r.Last+1)) {
|
||||
t.Errorf("note %q must point at the rest", r.Note)
|
||||
}
|
||||
// One page bigger than the cap is truncated rather than dropped.
|
||||
huge := &pdftext.Doc{Meta: pdftext.Meta{Pages: 1}, Text: []string{strings.Repeat("w ", 10000)}}
|
||||
r, _ = pdfdoc.Render(huge, pdfdoc.Options{Pages: "1", OutCap: 1000})
|
||||
if r.First != 1 || r.Last != 1 || utf8.RuneCountInString(r.Content) > 1000 || r.Note == "" {
|
||||
t.Errorf("oversized single page: %+v (%d chars)", r, utf8.RuneCountInString(r.Content))
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrep(t *testing.T) {
|
||||
d := big(60)
|
||||
r, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "needle", OutCap: 200000})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.GrepHits != 1 || r.First != 0 || r.Map {
|
||||
t.Errorf("got %+v", r)
|
||||
}
|
||||
for _, want := range []string{
|
||||
`grep "needle": 1 match`,
|
||||
"p. 37- page 37 line 4",
|
||||
"p. 37: the Needle is here on page 37",
|
||||
"p. 37- page 37 line 6",
|
||||
} {
|
||||
if !strings.Contains(r.Content, want) {
|
||||
t.Errorf("grep output lacks %q:\n%s", want, r.Content)
|
||||
}
|
||||
}
|
||||
// Smart case: an upper-case letter makes the pattern case-sensitive.
|
||||
if r, _ := pdfdoc.Render(d, pdfdoc.Options{Grep: "NEEDLE", OutCap: 200000}); r.GrepHits != 0 {
|
||||
t.Errorf("NEEDLE matched %d times, want 0 (case-sensitive)", r.GrepHits)
|
||||
}
|
||||
if r, _ := pdfdoc.Render(d, pdfdoc.Options{Grep: "Needle", OutCap: 200000}); r.GrepHits != 1 {
|
||||
t.Errorf("Needle matched %d times, want 1", r.GrepHits)
|
||||
}
|
||||
// No match is an answer, not an error.
|
||||
r, err = pdfdoc.Render(d, pdfdoc.Options{Grep: "zebra", OutCap: 200000})
|
||||
if err != nil || r.GrepHits != 0 || !strings.Contains(r.Content, `grep "zebra": no matches`) {
|
||||
t.Errorf("no match: %+v %v", r, err)
|
||||
}
|
||||
// Within a page range.
|
||||
r, _ = pdfdoc.Render(d, pdfdoc.Options{Grep: "line 3 ", Pages: "5-6", OutCap: 200000})
|
||||
if r.GrepHits != 2 || strings.Contains(r.Content, "p. 7") {
|
||||
t.Errorf("grep in 5-6: %d hits\n%s", r.GrepHits, r.Content)
|
||||
}
|
||||
}
|
||||
|
||||
func TestGrepLimits(t *testing.T) {
|
||||
d := big(200)
|
||||
r, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "lorem", OutCap: 1 << 30}) // 2,399 matches: every line but the needle
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if r.GrepHits != 2399 || strings.Count(r.Content, ": page ") != 100 || !strings.Contains(r.Content, "showing the first 100") {
|
||||
t.Errorf("hits %d, shown %d", r.GrepHits, strings.Count(r.Content, ": page "))
|
||||
}
|
||||
if _, err := pdfdoc.Render(d, pdfdoc.Options{Grep: strings.Repeat("a", 201), OutCap: 1000}); !errors.Is(err, pdfdoc.ErrBadGrep) {
|
||||
t.Errorf("201-byte pattern: %v, want ErrBadGrep", err)
|
||||
}
|
||||
if _, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "(unclosed", OutCap: 1000}); !errors.Is(err, pdfdoc.ErrBadGrep) {
|
||||
t.Errorf("bad regexp: %v, want ErrBadGrep", err)
|
||||
}
|
||||
// RE2 is linear: the classic catastrophic pattern over a 1 MB page finishes quickly.
|
||||
evil := &pdftext.Doc{Meta: pdftext.Meta{Pages: 1}, Text: []string{strings.Repeat("a", 1<<20) + "!"}}
|
||||
start := time.Now()
|
||||
_, _ = pdfdoc.Render(evil, pdfdoc.Options{Grep: "(a+)+$", OutCap: 1000})
|
||||
if d := time.Since(start); d > time.Second {
|
||||
t.Errorf("(a+)+$ took %v", d)
|
||||
}
|
||||
// Output stays under the cap.
|
||||
r, _ = pdfdoc.Render(d, pdfdoc.Options{Grep: "lorem", OutCap: 3000})
|
||||
if utf8.RuneCountInString(r.Content) > 3000 {
|
||||
t.Errorf("grep output %d characters over a 3000 cap", utf8.RuneCountInString(r.Content))
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user