kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

api: POST /v1/extract answers an uploaded PDF like a fetched one (content-addressed cache)

Browse Source 
Add POST /v1/extract: auth the caller, read the PDF body, hash it, and answer
it on the PDF fetch path exactly as /v1/fetch answers the same PDF by URL. The
hash is the content-addressed cache key ("upload:"+hash under the caller class),
so identical content is extracted once with no TTL or revalidation. The api.go
diff is scoped to the rule-1 changes: the DefaultPDFMaxUpload const, the
PDFMaxUpload Server field, the route, and the two ServeHTTP lines.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
kyle
2026-09-26 19:37:45 -07:00
co-authored by ornith-1.5-35b-a3b
parent 0481b6e32f
commit 7c40531891
4 changed files with 336 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+12 -2
View File
@@ -38,6 +38,10 @@ const (
maxReqBody = 64 << 10
writeTimeout = 240 * time.Second // P5a: a PDF may take 120 s to download + 20 s queued + 60 s to extract
requestDeadline = 210 * time.Second // P5a: the PDF path's worst case (see writeTimeout); other paths keep their own shorter timeouts
// DefaultPDFMaxUpload is the largest body POST /v1/extract accepts: an uploaded
// PDF, addressed by its hash and extracted once per content.
DefaultPDFMaxUpload = 100 << 20
)
type Server struct {
@@ -55,6 +59,7 @@ type Server struct {
PDFPool *pdfpool.Pool
PDFTTL time.Duration
PDFMaxQuestionChunks int
PDFMaxUpload int64 // 0 = DefaultPDFMaxUpload; the /v1/extract upload cap
Search *search.Engine
Ctx *fleetctx.Client
// PermitRestricted is context.permit_restricted: those callers never
@@ -93,6 +98,7 @@ func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log
s.Mux.HandleFunc("GET /metrics.json", s.metrics)
s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
s.Mux.HandleFunc("POST /v1/extract", s.v1extract)
s.Mux.HandleFunc("POST /v1/search", s.v1search)
s.Mux.HandleFunc("POST /v2/search", s.v2search)
return s
@@ -111,9 +117,13 @@ func (s *Server) HTTPServer(addr string) *http.Server {
}
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxReqBody)
var limit int64 = maxReqBody
if r.URL.Path == "/v1/extract" {
limit = int64(s.pdfMaxUpload())
}
r.Body = http.MaxBytesReader(w, r.Body, limit)
switch r.URL.Path {
case "/v1/fetch", "/v2/scrape", "/v1/search", "/v2/search":
case "/v1/fetch", "/v2/scrape", "/v1/search", "/v2/search", "/v1/extract":
ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
defer cancel()
r = r.WithContext(ctx)