kyleandornith-1.5-35b-a3b
93ec310113
Route "pdf" through clean.Kind in doFetch/cleanFetched: extract under the pool (keyed by caller class + document hash, one per caller), map the five pdftext errors plus a pool panic to 422, ErrBusy to 503+Retry-After 10, ErrCallerBusy to 429, and answer with the whole text or the map. Adds RetryAfter to fetch.Error and the matching header. No cache, pages or grep — those are task 07. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
87 lines
2.8 KiB
Go
87 lines
2.8 KiB
Go
package api
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
"git.wntrmute.dev/kyle/sift/internal/auth"
|
|
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
|
"git.wntrmute.dev/kyle/sift/internal/pdfdoc"
|
|
"git.wntrmute.dev/kyle/sift/internal/pdfpool"
|
|
"git.wntrmute.dev/kyle/sift/internal/pdftext"
|
|
)
|
|
|
|
const (
|
|
// DefaultPDFTTL bounds a cached PDF's life. Task 07; declared now so the test compiles.
|
|
DefaultPDFTTL = 24 * time.Hour
|
|
// DefaultPDFMaxQuestionChunks caps a digest question's length. Task 07; declared now.
|
|
DefaultPDFMaxQuestionChunks = 200
|
|
)
|
|
|
|
// runner returns the configured extraction child, or a zero Runner that runs
|
|
// /proc/self/exe pdftext.
|
|
func (s *Server) runner() *pdftext.Runner {
|
|
if s.PDFRunner != nil {
|
|
return s.PDFRunner
|
|
}
|
|
return &pdftext.Runner{}
|
|
}
|
|
|
|
// pool returns the extraction pool, built once and only when none is configured.
|
|
func (s *Server) pool() *pdfpool.Pool {
|
|
if s.PDFPool == nil {
|
|
s.pdfPoolOnce.Do(func() {
|
|
s.PDFPool = pdfpool.New(2, 4, 20*time.Second)
|
|
})
|
|
}
|
|
return s.PDFPool
|
|
}
|
|
|
|
// extractPDF runs one extraction under the pool, coalesced by document hash and
|
|
// bounded to one per caller.
|
|
func (s *Server) extractPDF(ctx context.Context, c *auth.Caller, res *fetch.Resource) (*pdftext.Doc, error) {
|
|
fn := func(jobCtx context.Context) (any, error) {
|
|
return s.runner().Run(jobCtx, res.Body)
|
|
}
|
|
v, err := s.pool().Do(ctx, s.callerClass(c)+"|"+res.SHA256(), c.Name, fn)
|
|
if err != nil {
|
|
return nil, pdfError(err)
|
|
}
|
|
doc, ok := v.(*pdftext.Doc)
|
|
if !ok {
|
|
return nil, fmt.Errorf("pdf: bad result %T", v)
|
|
}
|
|
return doc, nil
|
|
}
|
|
|
|
// pdfError maps every PDF-path error to the status sift returns to its caller;
|
|
// a PDF never produces a 500.
|
|
func pdfError(err error) error {
|
|
switch {
|
|
case errors.Is(err, pdftext.ErrNoText):
|
|
return &fetch.Error{Status: 422, Msg: "no text layer (scanned PDF?)", Hint: "ocr"}
|
|
case errors.Is(err, pdftext.ErrEncrypted):
|
|
return &fetch.Error{Status: 422, Msg: "encrypted PDF (password required)"}
|
|
case errors.Is(err, pdftext.ErrTimeout):
|
|
return &fetch.Error{Status: 422, Msg: "pdf extraction timed out"}
|
|
case errors.Is(err, pdftext.ErrTooMuchText):
|
|
return &fetch.Error{Status: 422, Msg: "pdf text over the output limit"}
|
|
case errors.Is(err, pdfpool.ErrBusy):
|
|
return &fetch.Error{Status: 503, Msg: "pdf workers busy", RetryAfter: 10}
|
|
case errors.Is(err, pdfpool.ErrCallerBusy):
|
|
return &fetch.Error{Status: 429, Msg: "busy"}
|
|
case errors.Is(err, pdftext.ErrUnreadable):
|
|
return &fetch.Error{Status: 422, Msg: "unreadable PDF"}
|
|
default:
|
|
return &fetch.Error{Status: 422, Msg: "unreadable PDF"}
|
|
}
|
|
}
|
|
|
|
// renderPDF draws the document for an agent: the whole text when it fits the
|
|
// output cap, otherwise the map.
|
|
func (s *Server) renderPDF(doc *pdftext.Doc) (pdfdoc.Rendered, error) {
|
|
return pdfdoc.Render(doc, pdfdoc.Options{OutCap: s.OutCap})
|
|
}
|