sift/internal/pdfdoc/pdfdoc_test.go

package pdfdoc_test

// P5a task 05: rendering an extracted document for an agent — the whole text with page markers,
// the map for a long document, a page range, and grep.

import (
	"errors"
	"fmt"
	"strings"
	"testing"
	"time"
	"unicode/utf8"

	"git.wntrmute.dev/kyle/sift/internal/pdfdoc"
	"git.wntrmute.dev/kyle/sift/internal/pdftext"
)

func small() *pdftext.Doc {
	return &pdftext.Doc{
		Meta: pdftext.Meta{Pages: 3, Title: "Three Pages", Outline: []pdftext.OutlineItem{{Title: "1 Intro", Page: 1, Level: 1}, {Title: "2 Fox", Page: 2, Level: 1}, {Title: "2.1 Den", Page: 2, Level: 2}}},
		Text: []string{"Chapter one begins here.\nSecond line.\n", "  The quick brown fox is on page two.\nIt jumps.", "Last page."},
	}
}

// big returns n pages of about 1,000 characters each, every line naming its page.
func big(n int) *pdftext.Doc {
	d := &pdftext.Doc{Meta: pdftext.Meta{Pages: n, Title: "Long Thesis", Outline: []pdftext.OutlineItem{{Title: "1 Introduction", Page: 1, Level: 1}, {Title: "2 Method", Page: 12, Level: 1}, {Title: "2.3 Setup", Page: 15, Level: 2}}}}
	for p := 1; p <= n; p++ {
		var lines []string
		for l := 1; l <= 12; l++ {
			lines = append(lines, fmt.Sprintf("page %d line %d lorem ipsum dolor sit amet consectetur adipiscing elit sed do eiusmod", p, l))
		}
		if p == 37 {
			lines[4] = "the Needle is here on page 37"
		}
		d.Text = append(d.Text, strings.Join(lines, "\n"))
	}
	return d
}

func TestFull(t *testing.T) {
	want := "--- page 1 ---\n\nChapter one begins here.\nSecond line.\n\n--- page 2 ---\n\nThe quick brown fox is on page two.\nIt jumps.\n\n--- page 3 ---\n\nLast page."
	if got := pdfdoc.Full(small()); got != want {
		t.Errorf("Full =\n%q\nwant\n%q", got, want)
	}
	if pdfdoc.Marker(12) != "--- page 12 ---" {
		t.Errorf("Marker(12) = %q", pdfdoc.Marker(12))
	}
}

func TestWholeDocumentWhenItFits(t *testing.T) {
	r, err := pdfdoc.Render(small(), pdfdoc.Options{OutCap: 10000})
	if err != nil {
		t.Fatal(err)
	}
	if r.Map || r.First != 1 || r.Last != 3 || r.Content != pdfdoc.Full(small()) {
		t.Errorf("got %+v", r)
	}
}

func TestMapForALongDocument(t *testing.T) {
	d := big(60) // ~60k characters
	r, err := pdfdoc.Render(d, pdfdoc.Options{OutCap: 20000})
	if err != nil {
		t.Fatal(err)
	}
	if !r.Map || r.First != 1 || r.Last < 1 {
		t.Fatalf("got Map=%v First=%d Last=%d, want a map from page 1", r.Map, r.First, r.Last)
	}
	c := r.Content
	for _, want := range []string{
		"# Long Thesis\n",
		"60 pages · ~",
		"## Outline\n",
		"- 1 Introduction (p. 1)\n",
		"- 2 Method (p. 12)\n",
		"  - 2.3 Setup (p. 15)\n", // level 2 is indented two spaces
		`pages="a-b"`,
		`grep="pattern"`,
		"--- page 1 ---",
	} {
		if !strings.Contains(c, want) {
			t.Errorf("map lacks %q", want)
		}
	}
	if utf8.RuneCountInString(c) > 20000 {
		t.Errorf("map is %d characters, over OutCap 20000", utf8.RuneCountInString(c))
	}
	// Leading pages take at most a quarter of the cap, and only whole pages.
	if strings.Contains(c, fmt.Sprintf("--- page %d ---", r.Last+1)) {
		t.Errorf("content has page %d past Last %d", r.Last+1, r.Last)
	}
	if !strings.Contains(c, fmt.Sprintf("page %d line 12", r.Last)) {
		t.Errorf("the last page shown (%d) is cut; pages in a map are whole", r.Last)
	}
	idx := strings.Index(c, "--- page 1 ---")
	if pagesPart := c[idx:]; utf8.RuneCountInString(pagesPart) > 20000/4 {
		t.Errorf("leading pages take %d characters, over OutCap/4", utf8.RuneCountInString(pagesPart))
	}
}

func TestMapNotesPartialExtraction(t *testing.T) {
	d := big(30)
	d.Pages = 2500 // the child stopped at its page cap
	r, _ := pdfdoc.Render(d, pdfdoc.Options{OutCap: 5000})
	if !strings.Contains(r.Content, "2500 pages") || !strings.Contains(r.Content, "pages 1-30 extracted") {
		t.Errorf("map must say how much was extracted:\n%s", r.Content[:min(600, len(r.Content))])
	}
}

func TestUntitledAndNoOutline(t *testing.T) {
	d := big(40)
	d.Title, d.Outline = "", nil
	r, _ := pdfdoc.Render(d, pdfdoc.Options{OutCap: 5000})
	if !strings.HasPrefix(r.Content, "# Untitled PDF\n") || strings.Contains(r.Content, "## Outline") {
		t.Errorf("untitled, outline-less map:\n%s", r.Content[:min(400, len(r.Content))])
	}
}

func TestPages(t *testing.T) {
	d := big(60)
	for _, tc := range []struct {
		spec        string
		first, last int
	}{
		{"2-3", 2, 3},
		{"7", 7, 7},
		{" 58- ", 58, 60},
		{"60-60", 60, 60},
	} {
		r, err := pdfdoc.Render(d, pdfdoc.Options{Pages: tc.spec, OutCap: 200000})
		if err != nil {
			t.Errorf("%q: %v", tc.spec, err)
			continue
		}
		if r.Map || r.First != tc.first || r.Last != tc.last {
			t.Errorf("%q: First %d Last %d Map %v, want %d-%d", tc.spec, r.First, r.Last, r.Map, tc.first, tc.last)
		}
		if !strings.HasPrefix(r.Content, pdfdoc.Marker(tc.first)+"\n\n") || strings.Contains(r.Content, pdfdoc.Marker(tc.last+1)) || (tc.first > 1 && strings.Contains(r.Content, pdfdoc.Marker(tc.first-1)+"\n")) {
			t.Errorf("%q: wrong pages in content: %.120q…", tc.spec, r.Content)
		}
	}
	for _, bad := range []string{"0-1", "5-4", "61", "999", "a-b", "1-2-3", "", "-5"} {
		if bad == "" {
			continue // empty means "no range", tested above
		}
		_, err := pdfdoc.Render(d, pdfdoc.Options{Pages: bad, OutCap: 200000})
		if !errors.Is(err, pdfdoc.ErrBadPages) {
			t.Errorf("%q: %v, want ErrBadPages", bad, err)
		}
	}
	_, err := pdfdoc.Render(d, pdfdoc.Options{Pages: "61", OutCap: 200000})
	if err == nil || !strings.Contains(err.Error(), "pages out of range (1-60)") {
		t.Errorf("out-of-range message: %v", err)
	}
}

// A range is at most 100 pages and at most OutCap characters, cut at a whole page with a note
// that says how to continue.
func TestPagesAreCapped(t *testing.T) {
	d := big(300)
	r, err := pdfdoc.Render(d, pdfdoc.Options{Pages: "1-300", OutCap: 1 << 30})
	if err != nil {
		t.Fatal(err)
	}
	if r.Last != 100 || !strings.Contains(r.Note, `pages="101-300"`) {
		t.Errorf("1-300: Last %d note %q; want 100 and a pointer to 101-300", r.Last, r.Note)
	}
	r, _ = pdfdoc.Render(d, pdfdoc.Options{Pages: "10-40", OutCap: 5000})
	if utf8.RuneCountInString(r.Content) > 5000 || r.First != 10 || r.Last < 10 || r.Last >= 40 {
		t.Errorf("10-40 under a 5000 cap: First %d Last %d, %d chars", r.First, r.Last, utf8.RuneCountInString(r.Content))
	}
	if !strings.Contains(r.Note, fmt.Sprintf(`pages="%d-40"`, r.Last+1)) {
		t.Errorf("note %q must point at the rest", r.Note)
	}
	// One page bigger than the cap is truncated rather than dropped.
	huge := &pdftext.Doc{Meta: pdftext.Meta{Pages: 1}, Text: []string{strings.Repeat("w ", 10000)}}
	r, _ = pdfdoc.Render(huge, pdfdoc.Options{Pages: "1", OutCap: 1000})
	if r.First != 1 || r.Last != 1 || utf8.RuneCountInString(r.Content) > 1000 || r.Note == "" {
		t.Errorf("oversized single page: %+v (%d chars)", r, utf8.RuneCountInString(r.Content))
	}
}

func TestGrep(t *testing.T) {
	d := big(60)
	r, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "needle", OutCap: 200000})
	if err != nil {
		t.Fatal(err)
	}
	if r.GrepHits != 1 || r.First != 0 || r.Map {
		t.Errorf("got %+v", r)
	}
	for _, want := range []string{
		`grep "needle": 1 match`,
		"p. 37- page 37 line 4",
		"p. 37: the Needle is here on page 37",
		"p. 37- page 37 line 6",
	} {
		if !strings.Contains(r.Content, want) {
			t.Errorf("grep output lacks %q:\n%s", want, r.Content)
		}
	}
	// Smart case: an upper-case letter makes the pattern case-sensitive.
	if r, _ := pdfdoc.Render(d, pdfdoc.Options{Grep: "NEEDLE", OutCap: 200000}); r.GrepHits != 0 {
		t.Errorf("NEEDLE matched %d times, want 0 (case-sensitive)", r.GrepHits)
	}
	if r, _ := pdfdoc.Render(d, pdfdoc.Options{Grep: "Needle", OutCap: 200000}); r.GrepHits != 1 {
		t.Errorf("Needle matched %d times, want 1", r.GrepHits)
	}
	// No match is an answer, not an error.
	r, err = pdfdoc.Render(d, pdfdoc.Options{Grep: "zebra", OutCap: 200000})
	if err != nil || r.GrepHits != 0 || !strings.Contains(r.Content, `grep "zebra": no matches`) {
		t.Errorf("no match: %+v %v", r, err)
	}
	// Within a page range.
	r, _ = pdfdoc.Render(d, pdfdoc.Options{Grep: "line 3 ", Pages: "5-6", OutCap: 200000})
	if r.GrepHits != 2 || strings.Contains(r.Content, "p. 7") {
		t.Errorf("grep in 5-6: %d hits\n%s", r.GrepHits, r.Content)
	}
}

func TestGrepLimits(t *testing.T) {
	d := big(200)
	r, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "lorem", OutCap: 1 << 30}) // 2,399 matches: every line but the needle
	if err != nil {
		t.Fatal(err)
	}
	if r.GrepHits != 2399 || strings.Count(r.Content, ": page ") != 100 || !strings.Contains(r.Content, "showing the first 100") {
		t.Errorf("hits %d, shown %d", r.GrepHits, strings.Count(r.Content, ": page "))
	}
	if _, err := pdfdoc.Render(d, pdfdoc.Options{Grep: strings.Repeat("a", 201), OutCap: 1000}); !errors.Is(err, pdfdoc.ErrBadGrep) {
		t.Errorf("201-byte pattern: %v, want ErrBadGrep", err)
	}
	if _, err := pdfdoc.Render(d, pdfdoc.Options{Grep: "(unclosed", OutCap: 1000}); !errors.Is(err, pdfdoc.ErrBadGrep) {
		t.Errorf("bad regexp: %v, want ErrBadGrep", err)
	}
	// RE2 is linear: the classic catastrophic pattern over a 1 MB page finishes. A backtracking
	// engine would not finish at all on this input, so the bound only has to be finite and far
	// above normal: 10 s. (1 s was too tight under -race, which slows regexp ~15–20x: talos
	// measured 1.5–1.9 s, 2026-09-26.)
	evil := &pdftext.Doc{Meta: pdftext.Meta{Pages: 1}, Text: []string{strings.Repeat("a", 1<<20) + "!"}}
	start := time.Now()
	_, _ = pdfdoc.Render(evil, pdfdoc.Options{Grep: "(a+)+$", OutCap: 1000})
	if d := time.Since(start); d > 10*time.Second {
		t.Errorf("(a+)+$ took %v", d)
	}
	// Output stays under the cap.
	r, _ = pdfdoc.Render(d, pdfdoc.Options{Grep: "lorem", OutCap: 3000})
	if utf8.RuneCountInString(r.Content) > 3000 {
		t.Errorf("grep output %d characters over a 3000 cap", utf8.RuneCountInString(r.Content))
	}
}