P5a: digests cite pages; sift_pdf_* metrics; sift fetch --pages/--grep; skill, unit and README

Digest systemPrompt gains a page-cite instruction and PromptVersion is 3.
Add the sift_pdf_* metric series (extract_total{outcome}, extract_seconds,
pages_total, pool_wait_seconds, cache_hits_total) to internal/metrics and wire
ObservePDF/ObservePDFWait/PDFCacheHit into internal/api/pdf.go. Refactor
cmd/sift/fetchcmd.go to a fetchOpts struct with parseFetchOpts and fetchBody,
adding --pages/--grep. Update skill/SKILL.md, set nix MemoryMax to 1G, and add a
PDF documents section to the README.

Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
2026-09-26 04:36:49 -07:00
co-authored by ornith-1.5-35b-a3b
parent 09f738bcac
commit 7e78de633f
11 changed files with 391 additions and 41 deletions
+50 -1
View File
@@ -6,6 +6,7 @@ import (
"errors"
"fmt"
"net/http"
"sync/atomic"
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
@@ -45,10 +46,32 @@ func (s *Server) pool() *pdfpool.Pool {
// extractPDF runs one extraction under the pool, coalesced by document hash and
// bounded to one per caller.
func (s *Server) extractPDF(ctx context.Context, c *auth.Caller, res *fetch.Resource) (*pdftext.Doc, error) {
var execNano atomic.Int64
fn := func(jobCtx context.Context) (any, error) {
return s.runner().Run(jobCtx, res.Body)
start := time.Now()
v, err := s.runner().Run(jobCtx, res.Body)
execNano.Store(int64(time.Since(start)))
return v, err
}
start := time.Now()
v, err := s.pool().Do(ctx, s.callerClass(c)+"|"+res.SHA256(), c.Name, fn)
total := time.Since(start)
// The job's own extraction time is what this call measured it to be; the
// rest of Do is time spent waiting for a slot.
if exec := execNano.Load(); exec > 0 {
wait := total - time.Duration(exec)
if wait < 0 {
wait = 0
}
s.Prom.ObservePDFWait(wait)
}
pages := 0
if err == nil {
if doc, ok := v.(*pdftext.Doc); ok {
pages = doc.Pages
}
}
s.Prom.ObservePDF(pdfOutcome(err), total, pages)
if err != nil {
return nil, pdfError(err)
}
@@ -59,6 +82,30 @@ func (s *Server) extractPDF(ctx context.Context, c *auth.Caller, res *fetch.Reso
return doc, nil
}
// pdfOutcome classifies an extraction error into the single value the
// sift_pdf_extract_total label carries; a nil error is a success and anything
// unrecognised is "other" so raw text never reaches a label.
func pdfOutcome(err error) string {
switch {
case err == nil:
return "ok"
case errors.Is(err, pdftext.ErrNoText):
return "no_text"
case errors.Is(err, pdftext.ErrEncrypted):
return "encrypted"
case errors.Is(err, pdftext.ErrUnreadable):
return "unreadable"
case errors.Is(err, pdftext.ErrTimeout):
return "timeout"
case errors.Is(err, pdftext.ErrTooMuchText):
return "too_large"
case errors.Is(err, pdfpool.ErrBusy), errors.Is(err, pdfpool.ErrCallerBusy):
return "busy"
default:
return "other"
}
}
// pdfError maps every PDF-path error to the status sift returns to its caller;
// a PDF never produces a 500.
func pdfError(err error) error {
@@ -107,6 +154,7 @@ func (s *Server) pdfFromCache(ctx context.Context, c *auth.Caller, rawURL string
ttl = DefaultPDFTTL
}
if time.Since(e.FetchedAt) < ttl {
s.Prom.PDFCacheHit()
res, doc := s.pdfFromEntry(rawURL, e)
return res, doc, true, nil
}
@@ -116,6 +164,7 @@ func (s *Server) pdfFromCache(ctx context.Context, c *auth.Caller, rawURL string
}
if res.Status == http.StatusNotModified {
cache.TouchPDF(key, time.Now().UTC())
s.Prom.PDFCacheHit()
res, doc := s.pdfFromEntry(rawURL, e)
return res, doc, true, nil
}