P5a: digests cite pages; sift_pdf_* metrics; sift fetch --pages/--grep; skill, unit and README
Digest systemPrompt gains a page-cite instruction and PromptVersion is 3.
Add the sift_pdf_* metric series (extract_total{outcome}, extract_seconds,
pages_total, pool_wait_seconds, cache_hits_total) to internal/metrics and wire
ObservePDF/ObservePDFWait/PDFCacheHit into internal/api/pdf.go. Refactor
cmd/sift/fetchcmd.go to a fetchOpts struct with parseFetchOpts and fetchBody,
adding --pages/--grep. Update skill/SKILL.md, set nix MemoryMax to 1G, and add a
PDF documents section to the README.
Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
+50
-1
@@ -6,6 +6,7 @@ import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"sync/atomic"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/auth"
|
||||
@@ -45,10 +46,32 @@ func (s *Server) pool() *pdfpool.Pool {
|
||||
// extractPDF runs one extraction under the pool, coalesced by document hash and
|
||||
// bounded to one per caller.
|
||||
func (s *Server) extractPDF(ctx context.Context, c *auth.Caller, res *fetch.Resource) (*pdftext.Doc, error) {
|
||||
var execNano atomic.Int64
|
||||
fn := func(jobCtx context.Context) (any, error) {
|
||||
return s.runner().Run(jobCtx, res.Body)
|
||||
start := time.Now()
|
||||
v, err := s.runner().Run(jobCtx, res.Body)
|
||||
execNano.Store(int64(time.Since(start)))
|
||||
return v, err
|
||||
}
|
||||
start := time.Now()
|
||||
v, err := s.pool().Do(ctx, s.callerClass(c)+"|"+res.SHA256(), c.Name, fn)
|
||||
total := time.Since(start)
|
||||
// The job's own extraction time is what this call measured it to be; the
|
||||
// rest of Do is time spent waiting for a slot.
|
||||
if exec := execNano.Load(); exec > 0 {
|
||||
wait := total - time.Duration(exec)
|
||||
if wait < 0 {
|
||||
wait = 0
|
||||
}
|
||||
s.Prom.ObservePDFWait(wait)
|
||||
}
|
||||
pages := 0
|
||||
if err == nil {
|
||||
if doc, ok := v.(*pdftext.Doc); ok {
|
||||
pages = doc.Pages
|
||||
}
|
||||
}
|
||||
s.Prom.ObservePDF(pdfOutcome(err), total, pages)
|
||||
if err != nil {
|
||||
return nil, pdfError(err)
|
||||
}
|
||||
@@ -59,6 +82,30 @@ func (s *Server) extractPDF(ctx context.Context, c *auth.Caller, res *fetch.Reso
|
||||
return doc, nil
|
||||
}
|
||||
|
||||
// pdfOutcome classifies an extraction error into the single value the
|
||||
// sift_pdf_extract_total label carries; a nil error is a success and anything
|
||||
// unrecognised is "other" so raw text never reaches a label.
|
||||
func pdfOutcome(err error) string {
|
||||
switch {
|
||||
case err == nil:
|
||||
return "ok"
|
||||
case errors.Is(err, pdftext.ErrNoText):
|
||||
return "no_text"
|
||||
case errors.Is(err, pdftext.ErrEncrypted):
|
||||
return "encrypted"
|
||||
case errors.Is(err, pdftext.ErrUnreadable):
|
||||
return "unreadable"
|
||||
case errors.Is(err, pdftext.ErrTimeout):
|
||||
return "timeout"
|
||||
case errors.Is(err, pdftext.ErrTooMuchText):
|
||||
return "too_large"
|
||||
case errors.Is(err, pdfpool.ErrBusy), errors.Is(err, pdfpool.ErrCallerBusy):
|
||||
return "busy"
|
||||
default:
|
||||
return "other"
|
||||
}
|
||||
}
|
||||
|
||||
// pdfError maps every PDF-path error to the status sift returns to its caller;
|
||||
// a PDF never produces a 500.
|
||||
func pdfError(err error) error {
|
||||
@@ -107,6 +154,7 @@ func (s *Server) pdfFromCache(ctx context.Context, c *auth.Caller, rawURL string
|
||||
ttl = DefaultPDFTTL
|
||||
}
|
||||
if time.Since(e.FetchedAt) < ttl {
|
||||
s.Prom.PDFCacheHit()
|
||||
res, doc := s.pdfFromEntry(rawURL, e)
|
||||
return res, doc, true, nil
|
||||
}
|
||||
@@ -116,6 +164,7 @@ func (s *Server) pdfFromCache(ctx context.Context, c *auth.Caller, rawURL string
|
||||
}
|
||||
if res.Status == http.StatusNotModified {
|
||||
cache.TouchPDF(key, time.Now().UTC())
|
||||
s.Prom.PDFCacheHit()
|
||||
res, doc := s.pdfFromEntry(rawURL, e)
|
||||
return res, doc, true, nil
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user