api: PDF cache with revalidation; pages and grep; refuse questions on very long documents

Answer extracted PDFs from the document cache: fresh for PDFTTL, then a
conditional GET (304 reuses the stored text, a 200 is extracted and stored
without a second GET). Store after extraction with the response hash, ETag,
fetch time and extracted pages. Route pages/grep through pdfdoc.Options with
400s for bad ranges and non-PDFs, and refuse a question when the document
exceeds PDFMaxQuestionChunks with a pointer to grep and pages.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>

internal/api/api.go

@@ -6,6 +6,7 @@ import (
 	"encoding/hex"
 	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"log/slog"
 	"net/http"
@@ -381,7 +382,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
 	md, htmlOut := cl.Markdown, ""
 	pages := 0
 	if doc != nil {
-		rp, err := s.renderPDF(doc)
+		rp, err := s.renderPDF(doc, pdfdoc.Options{OutCap: s.OutCap})
 		if err != nil {
 			s.writeFetchErr(w, c, req.URL, err)
 			return
@@ -472,6 +473,11 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 	}
 	fetchMS := time.Since(t0).Milliseconds()
 
+	if doc == nil && (req.Pages != "" || req.Grep != "") {
+		s.writeFetchErr(w, c, req.URL, &fetch.Error{Status: 400, Msg: "pages and grep apply to PDFs only"})
+		return
+	}
+
 	var docLen, pdfPages int
 	var content string
 	var pdfMap map[string]any
@@ -481,8 +487,12 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 			writeErr(w, 400, "raw mode does not apply to PDFs", "")
 			return
 		}
-		rp, err := s.renderPDF(doc)
+		rp, err := s.renderPDF(doc, pdfdoc.Options{Pages: req.Pages, Grep: req.Grep, OutCap: s.OutCap})
 		if err != nil {
+			if errors.Is(err, pdfdoc.ErrBadPages) || errors.Is(err, pdfdoc.ErrBadGrep) {
+				s.writeFetchErr(w, c, req.URL, &fetch.Error{Status: 400, Msg: err.Error()})
+				return
+			}
 			s.writeFetchErr(w, c, req.URL, err)
 			return
 		}
@@ -526,6 +536,17 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 	var digMeta map[string]any
 	var digestMS int64
 	if mode == "digest" {
+		if doc != nil {
+			n := len(digest.Split(pdfdoc.Full(doc), digest.ChunkTokens))
+			limit := s.PDFMaxQuestionChunks
+			if limit == 0 {
+				limit = DefaultPDFMaxQuestionChunks
+			}
+			if n > limit {
+				s.writeFetchErr(w, c, req.URL, &fetch.Error{Status: 422, Msg: fmt.Sprintf("document too long for question (%d chunks); use grep or pages", n)})
+				return
+			}
+		}
 		if s.Digest == nil {
 			warn = append(warn, "digest fell back to clean: unset")
 			mode = "clean"
@@ -590,7 +611,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
 		"fetched_at":   res.FetchedAt.Format(time.RFC3339),
 		"hash":         res.SHA256(),
 		"bytes":        len(res.Body),
-		"cached":       false,
+		"cached":       len(res.Body) == 0,
 		"kind":         "live",
 	}
 	if snap != nil {
@@ -673,15 +694,24 @@ func (s *Server) cleanFetched(res *fetch.Resource, jq string, onlyMain bool) (*f
 }
 
 func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, *pdftext.Doc, error) {
-	if err := s.acquire(c.Name, rawURL); err != nil {
-		return nil, nil, nil, err
-	}
-	defer s.release(c.Name, rawURL)
-
-	res, err := s.Fetch.Fetch(r.Context(), rawURL)
+	res, hitDoc, ok, err := s.pdfFromCache(r.Context(), c, rawURL)
 	if err != nil {
 		return nil, nil, nil, err
 	}
+	if ok {
+		return res, &clean.Result{Kind: "pdf", Title: hitDoc.Title}, hitDoc, nil
+	}
+	if res == nil {
+		if err := s.acquire(c.Name, rawURL); err != nil {
+			return nil, nil, nil, err
+		}
+		defer s.release(c.Name, rawURL)
+
+		res, err = s.Fetch.Fetch(r.Context(), rawURL)
+		if err != nil {
+			return nil, nil, nil, err
+		}
+	}
 	kind := clean.Kind(res.ContentType, res.Body)
 	if kind == "other" {
 		ct := res.ContentType
@@ -713,6 +743,7 @@ func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onl
 		if err != nil {
 			return nil, nil, nil, err
 		}
+		s.storePDF(dcache.PDFKey(rawURL, s.callerClass(c)), res, doc)
 		cl = &clean.Result{Kind: "pdf", Title: doc.Title}
 	}
 	return res, cl, doc, nil