P5a: digests cite pages; sift_pdf_* metrics; sift fetch --pages/--grep; skill, unit and README
Digest systemPrompt gains a page-cite instruction and PromptVersion is 3.
Add the sift_pdf_* metric series (extract_total{outcome}, extract_seconds,
pages_total, pool_wait_seconds, cache_hits_total) to internal/metrics and wire
ObservePDF/ObservePDFWait/PDFCacheHit into internal/api/pdf.go. Refactor
cmd/sift/fetchcmd.go to a fetchOpts struct with parseFetchOpts and fetchBody,
adding --pages/--grep. Update skill/SKILL.md, set nix MemoryMax to 1G, and add a
PDF documents section to the README.
Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
@@ -41,12 +41,24 @@ type Source interface {
|
||||
Snapshot() Snapshot
|
||||
}
|
||||
|
||||
// pdfOutcomes are the outcomes ObservePDF counts directly; anything else is
|
||||
// folded into "other" so a raw outcome string never becomes a label.
|
||||
var pdfOutcomes = map[string]struct{}{
|
||||
"ok": {}, "no_text": {}, "encrypted": {}, "unreadable": {},
|
||||
"timeout": {}, "too_large": {}, "busy": {},
|
||||
}
|
||||
|
||||
// Set is a private registry of sift_* series.
|
||||
type Set struct {
|
||||
Reg *prometheus.Registry
|
||||
Requests *prometheus.CounterVec
|
||||
Duration *prometheus.HistogramVec
|
||||
handler http.Handler
|
||||
Reg *prometheus.Registry
|
||||
Requests *prometheus.CounterVec
|
||||
Duration *prometheus.HistogramVec
|
||||
PDFTotal *prometheus.CounterVec
|
||||
PDFSeconds prometheus.Histogram
|
||||
PDFPages prometheus.Counter
|
||||
PDFWait prometheus.Histogram
|
||||
PDFCacheHits prometheus.Counter
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func New(src Source) *Set {
|
||||
@@ -60,12 +72,39 @@ func New(src Source) *Set {
|
||||
Help: "API wall time (fetch/search) or digest time (digest).",
|
||||
Buckets: durationBuckets,
|
||||
}, []string{"op"})
|
||||
reg.MustRegister(requests, duration, &collector{src: src})
|
||||
pdfTotal := prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "sift_pdf_extract_total",
|
||||
Help: "PDF extraction attempts by outcome.",
|
||||
}, []string{"outcome"})
|
||||
pdfSeconds := prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "sift_pdf_extract_seconds",
|
||||
Help: "PDF extraction wall time.",
|
||||
Buckets: durationBuckets,
|
||||
})
|
||||
pdfPages := prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sift_pdf_pages_total",
|
||||
Help: "PDF pages extracted.",
|
||||
})
|
||||
pdfWait := prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "sift_pdf_pool_wait_seconds",
|
||||
Help: "Time a PDF job waited in the pool before its extraction started.",
|
||||
Buckets: durationBuckets,
|
||||
})
|
||||
pdfCacheHits := prometheus.NewCounter(prometheus.CounterOpts{
|
||||
Name: "sift_pdf_cache_hits_total",
|
||||
Help: "PDF answers served from the document cache.",
|
||||
})
|
||||
reg.MustRegister(requests, duration, pdfTotal, pdfSeconds, pdfPages, pdfWait, pdfCacheHits, &collector{src: src})
|
||||
return &Set{
|
||||
Reg: reg,
|
||||
Requests: requests,
|
||||
Duration: duration,
|
||||
handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{EnableOpenMetrics: false}),
|
||||
Reg: reg,
|
||||
Requests: requests,
|
||||
Duration: duration,
|
||||
PDFTotal: pdfTotal,
|
||||
PDFSeconds: pdfSeconds,
|
||||
PDFPages: pdfPages,
|
||||
PDFWait: pdfWait,
|
||||
PDFCacheHits: pdfCacheHits,
|
||||
handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{EnableOpenMetrics: false}),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -89,6 +128,38 @@ func (s *Set) ObserveDigest(d time.Duration) {
|
||||
s.Duration.WithLabelValues("digest").Observe(d.Seconds())
|
||||
}
|
||||
|
||||
// ObservePDF records one PDF extraction attempt: the outcome (folded to "other"
|
||||
// when it is not a known value, so a raw string never becomes a label), the wall
|
||||
// time, and the pages extracted.
|
||||
func (s *Set) ObservePDF(outcome string, d time.Duration, pages int) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if _, ok := pdfOutcomes[outcome]; !ok {
|
||||
outcome = "other"
|
||||
}
|
||||
s.PDFTotal.WithLabelValues(outcome).Inc()
|
||||
s.PDFSeconds.Observe(d.Seconds())
|
||||
s.PDFPages.Add(float64(pages))
|
||||
}
|
||||
|
||||
// ObservePDFWait records the time a PDF job spent in the pool before its
|
||||
// extraction started.
|
||||
func (s *Set) ObservePDFWait(d time.Duration) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.PDFWait.Observe(d.Seconds())
|
||||
}
|
||||
|
||||
// PDFCacheHit records one PDF answer served from the document cache.
|
||||
func (s *Set) PDFCacheHit() {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.PDFCacheHits.Inc()
|
||||
}
|
||||
|
||||
func (s *Set) Handler() http.Handler {
|
||||
if s == nil {
|
||||
return http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
|
||||
|
||||
Reference in New Issue
Block a user