metrics: sift_batch_* series (jobs, targets, depth, in-flight, durations, bytes)
New internal/metrics/batch.go adds the batch series to metrics.Set, registered in New: BatchJobsTotal and BatchTargetsTotal counters (label "outcome", any value outside the known set folded to "other" so a raw string never becomes a label), BatchQueueDepth and BatchTargetsInflight gauges, BatchJobSeconds and BatchTargetSeconds histograms (the existing request buckets) and BatchResultBytes (1K,4K,16K,64K,256K,1M). The Observe…/Set… methods are nil-receiver safe like ObservePDF and flow through the existing /metrics handler; no new endpoint or Snapshot change. Co-Authored-By: ornith-1.5-35b-a3b <ornith-1.5-35b-a3b@llama.cpp.invalid>
This commit is contained in:
ornith-1.5-35b-a3b
@@ -0,0 +1,80 @@
|
||||
package metrics
|
||||
|
||||
import "time"
|
||||
|
||||
var batchResultBytesBuckets = []float64{1024, 4096, 16384, 65536, 262144, 1048576}
|
||||
|
||||
var batchJobOutcomes = map[string]struct{}{
|
||||
"created": {}, "reused": {}, "done": {}, "partial": {},
|
||||
"failed": {}, "cancelled": {}, "restart": {},
|
||||
}
|
||||
|
||||
var batchTargetOutcomes = map[string]struct{}{
|
||||
"ok": {}, "blocked": {}, "timeout": {}, "busy": {},
|
||||
"upstream_4xx": {}, "upstream_5xx": {}, "unsupported": {},
|
||||
"too_long": {}, "cancel": {},
|
||||
}
|
||||
|
||||
// ObserveBatchJob records one batch outcome. Values outside the known set fold
|
||||
// to "other" so a raw string never becomes a label.
|
||||
func (s *Set) ObserveBatchJob(outcome string) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if _, ok := batchJobOutcomes[outcome]; !ok {
|
||||
outcome = "other"
|
||||
}
|
||||
s.BatchJobsTotal.WithLabelValues(outcome).Inc()
|
||||
}
|
||||
|
||||
// ObserveBatchTarget records one target outcome. Values outside the known set
|
||||
// fold to "other" so a raw string never becomes a label.
|
||||
func (s *Set) ObserveBatchTarget(outcome string) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
if _, ok := batchTargetOutcomes[outcome]; !ok {
|
||||
outcome = "other"
|
||||
}
|
||||
s.BatchTargetsTotal.WithLabelValues(outcome).Inc()
|
||||
}
|
||||
|
||||
// SetBatchQueueDepth sets the live job count gauge (queued+running).
|
||||
func (s *Set) SetBatchQueueDepth(n int) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.BatchQueueDepth.Set(float64(n))
|
||||
}
|
||||
|
||||
// SetBatchTargetsInflight sets the in-flight targets gauge.
|
||||
func (s *Set) SetBatchTargetsInflight(n int) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.BatchTargetsInflight.Set(float64(n))
|
||||
}
|
||||
|
||||
// ObserveBatchJobSeconds records the wall time from submit to a batch's terminal status.
|
||||
func (s *Set) ObserveBatchJobSeconds(d time.Duration) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.BatchJobSeconds.Observe(d.Seconds())
|
||||
}
|
||||
|
||||
// ObserveBatchTargetSeconds records the wall time for one target fetch.
|
||||
func (s *Set) ObserveBatchTargetSeconds(d time.Duration) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.BatchTargetSeconds.Observe(d.Seconds())
|
||||
}
|
||||
|
||||
// ObserveBatchResultBytes records the stored byte count of one target result.
|
||||
func (s *Set) ObserveBatchResultBytes(n int) {
|
||||
if s == nil {
|
||||
return
|
||||
}
|
||||
s.BatchResultBytes.Observe(float64(n))
|
||||
}
|
||||
@@ -0,0 +1,61 @@
|
||||
package metrics
|
||||
|
||||
import (
|
||||
"io"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
type batchSrc struct{}
|
||||
|
||||
func (batchSrc) Snapshot() Snapshot { return Snapshot{} }
|
||||
|
||||
func TestBatchSeries(t *testing.T) {
|
||||
s := New(batchSrc{})
|
||||
s.ObserveBatchJob("created")
|
||||
s.ObserveBatchJob("done")
|
||||
s.ObserveBatchJob("done")
|
||||
s.ObserveBatchJob("reused")
|
||||
s.ObserveBatchJob("some raw string") // folded to "other", never a label
|
||||
s.ObserveBatchTarget("ok")
|
||||
s.ObserveBatchTarget("ok")
|
||||
s.ObserveBatchTarget("blocked")
|
||||
s.ObserveBatchTarget("upstream_5xx")
|
||||
s.ObserveBatchTarget("leak me") // folded to "other"
|
||||
s.SetBatchQueueDepth(3)
|
||||
s.SetBatchTargetsInflight(1)
|
||||
s.ObserveBatchJobSeconds(4 * time.Second)
|
||||
s.ObserveBatchTargetSeconds(1500 * time.Millisecond)
|
||||
s.ObserveBatchResultBytes(4096)
|
||||
|
||||
req := httptest.NewRequest("GET", "/metrics", nil)
|
||||
req.RemoteAddr, req.Host = "127.0.0.1:9", "127.0.0.1:8890"
|
||||
rec := httptest.NewRecorder()
|
||||
s.Handler().ServeHTTP(rec, req)
|
||||
body, _ := io.ReadAll(rec.Body)
|
||||
out := string(body)
|
||||
for _, want := range []string{
|
||||
`sift_batch_jobs_total{outcome="created"} 1`,
|
||||
`sift_batch_jobs_total{outcome="done"} 2`,
|
||||
`sift_batch_jobs_total{outcome="reused"} 1`,
|
||||
`sift_batch_jobs_total{outcome="other"} 1`,
|
||||
`sift_batch_targets_total{outcome="ok"} 2`,
|
||||
`sift_batch_targets_total{outcome="blocked"} 1`,
|
||||
`sift_batch_targets_total{outcome="upstream_5xx"} 1`,
|
||||
`sift_batch_targets_total{outcome="other"} 1`,
|
||||
`sift_batch_queue_depth 3`,
|
||||
`sift_batch_targets_inflight 1`,
|
||||
`sift_batch_job_seconds_count 1`,
|
||||
`sift_batch_target_seconds_count 1`,
|
||||
`sift_batch_result_bytes_count 1`,
|
||||
} {
|
||||
if !strings.Contains(out, want) {
|
||||
t.Errorf("metrics lack %q", want)
|
||||
}
|
||||
}
|
||||
if strings.Contains(out, "raw") || strings.Contains(out, "leak") {
|
||||
t.Error("a raw outcome string leaked into a label")
|
||||
}
|
||||
}
|
||||
+66
-19
@@ -50,15 +50,22 @@ var pdfOutcomes = map[string]struct{}{
|
||||
|
||||
// Set is a private registry of sift_* series.
|
||||
type Set struct {
|
||||
Reg *prometheus.Registry
|
||||
Requests *prometheus.CounterVec
|
||||
Duration *prometheus.HistogramVec
|
||||
PDFTotal *prometheus.CounterVec
|
||||
PDFSeconds prometheus.Histogram
|
||||
PDFPages prometheus.Counter
|
||||
PDFWait prometheus.Histogram
|
||||
PDFCacheHits prometheus.Counter
|
||||
handler http.Handler
|
||||
Reg *prometheus.Registry
|
||||
Requests *prometheus.CounterVec
|
||||
Duration *prometheus.HistogramVec
|
||||
PDFTotal *prometheus.CounterVec
|
||||
PDFSeconds prometheus.Histogram
|
||||
PDFPages prometheus.Counter
|
||||
PDFWait prometheus.Histogram
|
||||
PDFCacheHits prometheus.Counter
|
||||
BatchJobsTotal *prometheus.CounterVec
|
||||
BatchTargetsTotal *prometheus.CounterVec
|
||||
BatchQueueDepth prometheus.Gauge
|
||||
BatchTargetsInflight prometheus.Gauge
|
||||
BatchJobSeconds prometheus.Histogram
|
||||
BatchTargetSeconds prometheus.Histogram
|
||||
BatchResultBytes prometheus.Histogram
|
||||
handler http.Handler
|
||||
}
|
||||
|
||||
func New(src Source) *Set {
|
||||
@@ -94,17 +101,57 @@ func New(src Source) *Set {
|
||||
Name: "sift_pdf_cache_hits_total",
|
||||
Help: "PDF answers served from the document cache.",
|
||||
})
|
||||
reg.MustRegister(requests, duration, pdfTotal, pdfSeconds, pdfPages, pdfWait, pdfCacheHits, &collector{src: src})
|
||||
batchJobs := prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "sift_batch_jobs_total",
|
||||
Help: "Batch jobs by outcome.",
|
||||
}, []string{"outcome"})
|
||||
batchTargets := prometheus.NewCounterVec(prometheus.CounterOpts{
|
||||
Name: "sift_batch_targets_total",
|
||||
Help: "Batch targets by outcome.",
|
||||
}, []string{"outcome"})
|
||||
batchQueueDepth := prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "sift_batch_queue_depth",
|
||||
Help: "Live batch jobs (queued+running).",
|
||||
})
|
||||
batchTargetsInflight := prometheus.NewGauge(prometheus.GaugeOpts{
|
||||
Name: "sift_batch_targets_inflight",
|
||||
Help: "Batch targets currently being fetched.",
|
||||
})
|
||||
batchJobSeconds := prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "sift_batch_job_seconds",
|
||||
Help: "Batch wall time from submit to terminal status.",
|
||||
Buckets: durationBuckets,
|
||||
})
|
||||
batchTargetSeconds := prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "sift_batch_target_seconds",
|
||||
Help: "Batch target fetch wall time.",
|
||||
Buckets: durationBuckets,
|
||||
})
|
||||
batchResultBytes := prometheus.NewHistogram(prometheus.HistogramOpts{
|
||||
Name: "sift_batch_result_bytes",
|
||||
Help: "Stored bytes per batch target result.",
|
||||
Buckets: batchResultBytesBuckets,
|
||||
})
|
||||
reg.MustRegister(requests, duration, pdfTotal, pdfSeconds, pdfPages, pdfWait, pdfCacheHits,
|
||||
batchJobs, batchTargets, batchQueueDepth, batchTargetsInflight,
|
||||
batchJobSeconds, batchTargetSeconds, batchResultBytes, &collector{src: src})
|
||||
return &Set{
|
||||
Reg: reg,
|
||||
Requests: requests,
|
||||
Duration: duration,
|
||||
PDFTotal: pdfTotal,
|
||||
PDFSeconds: pdfSeconds,
|
||||
PDFPages: pdfPages,
|
||||
PDFWait: pdfWait,
|
||||
PDFCacheHits: pdfCacheHits,
|
||||
handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{EnableOpenMetrics: false}),
|
||||
Reg: reg,
|
||||
Requests: requests,
|
||||
Duration: duration,
|
||||
PDFTotal: pdfTotal,
|
||||
PDFSeconds: pdfSeconds,
|
||||
PDFPages: pdfPages,
|
||||
PDFWait: pdfWait,
|
||||
PDFCacheHits: pdfCacheHits,
|
||||
BatchJobsTotal: batchJobs,
|
||||
BatchTargetsTotal: batchTargets,
|
||||
BatchQueueDepth: batchQueueDepth,
|
||||
BatchTargetsInflight: batchTargetsInflight,
|
||||
BatchJobSeconds: batchJobSeconds,
|
||||
BatchTargetSeconds: batchTargetSeconds,
|
||||
BatchResultBytes: batchResultBytes,
|
||||
handler: promhttp.HandlerFor(reg, promhttp.HandlerOpts{EnableOpenMetrics: false}),
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user