kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

api: serve PDFs — extraction in the child under the pool, full text or map, errors

Browse Source 
Route "pdf" through clean.Kind in doFetch/cleanFetched: extract under the pool
(keyed by caller class + document hash, one per caller), map the five pdftext
errors plus a pool panic to 422, ErrBusy to 503+Retry-After 10, ErrCallerBusy to
429, and answer with the whole text or the map. Adds RetryAfter to fetch.Error
and the matching header. No cache, pages or grep — those are task 07.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
kyle
2026-09-26 04:04:11 -07:00
co-authored by ornith-1.5-35b-a3b
parent 9c858aded9
commit 93ec310113
5 changed files with 603 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+110 -39
View File
@@ -10,10 +10,12 @@ import (
"log/slog"
"net/http"
"net/url"
"strconv"
"strings"
"sync"
"sync/atomic"
"time"
"unicode/utf8"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/clean"
@@ -23,6 +25,9 @@ import (
"git.wntrmute.dev/kyle/sift/internal/fleetctx"
"git.wntrmute.dev/kyle/sift/internal/guard"
"git.wntrmute.dev/kyle/sift/internal/metrics"
"git.wntrmute.dev/kyle/sift/internal/pdfdoc"
"git.wntrmute.dev/kyle/sift/internal/pdfpool"
"git.wntrmute.dev/kyle/sift/internal/pdftext"
"git.wntrmute.dev/kyle/sift/internal/recover"
"git.wntrmute.dev/kyle/sift/internal/search"
"git.wntrmute.dev/kyle/sift/internal/selfip"
@@ -44,8 +49,13 @@ type Server struct {
OutCap int
SelfIP *selfip.Discoverer
Digest *digest.Engine
Search *search.Engine
Ctx *fleetctx.Client
PDFRunner *pdftext.Runner
PDFPool *pdfpool.Pool
PDFTTL time.Duration
PDFMaxQuestionChunks int
Search *search.Engine
Ctx *fleetctx.Client
// PermitRestricted is context.permit_restricted: those callers never
// have a fetch host written to the log (host=-). Independent of Honcho.
PermitRestricted []string
@@ -56,6 +66,8 @@ type Server struct {
per map[string]chan struct{}
reqN, blockN, scrapeN atomic.Uint64
pdfPoolOnce sync.Once
}
func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log *slog.Logger) *Server {
@@ -300,6 +312,10 @@ type v1Req struct {
Context *bool `json:"context"`
JQ string `json:"jq"`
Recover bool `json:"recover"`
// Pages narrows a PDF digest to "2-3"; Grep matches lines in it. Optional; task-07.
Pages string `json:"pages"`
Grep string `json:"grep"`
}
func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
@@ -357,12 +373,23 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
}
}
s.scrapeN.Add(1)
res, cl, ferr := s.doFetch(r, c, req.URL, "", onlyMain)
res, cl, doc, ferr := s.doFetch(r, c, req.URL, "", onlyMain)
if ferr != nil {
s.writeFetchErr(w, c, req.URL, ferr)
return
}
md, htmlOut := cl.Markdown, ""
pages := 0
if doc != nil {
rp, err := s.renderPDF(doc)
if err != nil {
s.writeFetchErr(w, c, req.URL, err)
return
}
md = rp.Content
htmlOut = ""
pages = doc.Pages
}
if wantHTML {
htmlOut = cl.HTML
}
@@ -371,7 +398,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
}
rawTok := len(res.Body) / 4
retTok := len(md) / 4
s.Log.Info("scrape", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "status", res.Status, "bytes", len(res.Body))
s.Log.Info("scrape", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "status", res.Status, "bytes", len(res.Body), "pdf_pages", pages)
writeJSON(w, 200, map[string]any{
"success": true,
"data": map[string]any{
@@ -382,6 +409,7 @@ func (s *Server) scrape(w http.ResponseWriter, r *http.Request) {
"sourceURL": res.FinalURL,
"statusCode": res.Status,
"contentType": res.ContentType,
"numPages": pages,
"sift": map[string]any{
"hash": res.SHA256(),
"fetchedAt": res.FetchedAt.Format(time.RFC3339),
@@ -429,12 +457,12 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
t0 := time.Now()
onlyMain := true
res, cl, ferr := s.doFetch(r, c, req.URL, req.JQ, onlyMain)
res, cl, doc, ferr := s.doFetch(r, c, req.URL, req.JQ, onlyMain)
var snap *recover.Result
if ferr != nil && req.Recover && recoverWorthy(ferr) {
sr, rerr := recover.Try(r.Context(), s.Fetch, req.URL)
if rerr == nil && sr != nil && sr.Res != nil {
res, cl, ferr = s.cleanFetched(sr.Res, req.JQ, onlyMain)
res, cl, doc, ferr = s.cleanFetched(sr.Res, req.JQ, onlyMain)
snap = sr
}
}
@@ -444,16 +472,50 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
}
fetchMS := time.Since(t0).Milliseconds()
content := cl.Markdown
if cl.Kind == "json" {
content = cl.HTML
}
if mode == "raw" {
content = string(res.Body)
if len(content) > s.OutCap {
content = clean.TruncateUTF8(content, s.OutCap)
cl.Trunc = true
cl.Warning = "truncated to output cap"
var docLen, pdfPages int
var content string
var pdfMap map[string]any
if doc != nil {
pdfPages = doc.Pages
if mode == "raw" {
writeErr(w, 400, "raw mode does not apply to PDFs", "")
return
}
rp, err := s.renderPDF(doc)
if err != nil {
s.writeFetchErr(w, c, req.URL, err)
return
}
full := pdfdoc.Full(doc)
docLen = utf8.RuneCountInString(full)
if mode == "digest" {
content = full
} else {
content = rp.Content
}
pdfMap = map[string]any{
"pages": doc.Pages,
"title": doc.Title,
"outline": doc.Outline,
"extracted": len(doc.Text),
"first": rp.First,
"last": rp.Last,
"map": rp.Map,
"grep_hits": rp.GrepHits,
"note": rp.Note,
}
} else {
content = cl.Markdown
if cl.Kind == "json" {
content = cl.HTML
}
if mode == "raw" {
content = string(res.Body)
if len(content) > s.OutCap {
content = clean.TruncateUTF8(content, s.OutCap)
cl.Trunc = true
cl.Warning = "truncated to output cap"
}
}
}
warn := []string{}
@@ -518,7 +580,7 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
}
s.Log.Info("fetch", "caller", c.Name, "host", s.logHost(c, res.FinalURL), "mode", mode,
"q_len", qLen, "raw_tokens", len(res.Body)/4, "returned_tokens", len(content)/4,
"digest_ms", digestMS, "fallback", fb)
"pdf_pages", pdfPages, "digest_ms", digestMS, "fallback", fb)
src := map[string]any{
"url": res.URL,
@@ -549,12 +611,15 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
"source": src,
"warnings": warn,
"sizes": map[string]int{
"raw_tokens": len(res.Body) / 4,
"raw_tokens": docLen / 4,
"clean_tokens": digest.EstTokens(cl.Markdown),
"returned_tokens": digest.EstTokens(content),
},
"timings": map[string]int64{"fetch_ms": fetchMS, "digest_ms": digestMS, "context_ms": ctxMS},
}
if pdfMap != nil {
out["pdf"] = pdfMap
}
if digMeta != nil {
out["digest"] = digMeta
}
@@ -585,13 +650,11 @@ func headings(md string) []string {
return h
}
func (s *Server) cleanFetched(res *fetch.Resource, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, error) {
kind := clean.MediaKind(res.ContentType)
if kind == "sniff" {
kind = clean.Sniff(res.Body)
}
if kind == "other" {
return nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type"}
func (s *Server) cleanFetched(res *fetch.Resource, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, *pdftext.Doc, error) {
kind := clean.Kind(res.ContentType, res.Body)
if kind == "other" || kind == "pdf" {
// The recover path (archive snapshots) never serves a PDF.
return nil, nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type"}
}
var cl *clean.Result
var cerr error
@@ -604,31 +667,28 @@ func (s *Server) cleanFetched(res *fetch.Resource, jq string, onlyMain bool) (*f
cl, cerr = clean.Text(res, s.OutCap)
}
if cerr != nil {
return nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
return nil, nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
}
return res, cl, nil
return res, cl, nil, nil
}
func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, error) {
func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onlyMain bool) (*fetch.Resource, *clean.Result, *pdftext.Doc, error) {
if err := s.acquire(c.Name, rawURL); err != nil {
return nil, nil, err
return nil, nil, nil, err
}
defer s.release(c.Name, rawURL)
res, err := s.Fetch.Fetch(r.Context(), rawURL)
if err != nil {
return nil, nil, err
}
kind := clean.MediaKind(res.ContentType)
if kind == "sniff" {
kind = clean.Sniff(res.Body)
return nil, nil, nil, err
}
kind := clean.Kind(res.ContentType, res.Body)
if kind == "other" {
ct := res.ContentType
if ct == "" {
ct = "unknown"
}
return nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type " + ct + "; hint: browser"}
return nil, nil, nil, &fetch.Error{Status: 415, Hint: "browser", Msg: "unsupported content type " + ct + "; hint: browser"}
}
var cl *clean.Result
var cerr error
@@ -637,17 +697,25 @@ func (s *Server) doFetch(r *http.Request, c *auth.Caller, rawURL, jq string, onl
cl, cerr = clean.HTML(res, onlyMain, s.OutCap)
case "json":
cl, cerr = clean.JSON(res, jq, s.OutCap)
default:
case "text":
cl, cerr = clean.Text(res, s.OutCap)
}
if cerr != nil {
var tl *clean.TooLargeError
if errors.As(cerr, &tl) {
return nil, nil, &fetch.Error{Status: 400, Msg: tl.Error()}
return nil, nil, nil, &fetch.Error{Status: 400, Msg: tl.Error()}
}
return nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
return nil, nil, nil, &fetch.Error{Status: 422, Msg: cerr.Error()}
}
return res, cl, nil
var doc *pdftext.Doc
if kind == "pdf" {
doc, err = s.extractPDF(r.Context(), c, res)
if err != nil {
return nil, nil, nil, err
}
cl = &clean.Result{Kind: "pdf", Title: doc.Title}
}
return res, cl, doc, nil
}
func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL string, err error) {
@@ -670,6 +738,9 @@ func (s *Server) writeFetchErr(w http.ResponseWriter, c *auth.Caller, rawURL str
} else {
s.Log.Warn("fetch failed", "caller", name, "host", host, "reason", s.logReason(c, fe), "status", fe.Status)
}
if fe.RetryAfter > 0 {
w.Header().Set("Retry-After", strconv.Itoa(fe.RetryAfter))
}
writeErr(w, fe.Status, fe.Msg, fe.Hint)
return
}