p2.7: prometheus /metrics
Default GET /metrics is Prometheus text; JSON stays at /metrics.json and ?format=json. :8443 needs the metrics caller bearer; loopback stays open. Restricted callers have no per-caller series and no tavily:restricted gauge.
This commit is contained in:
+149
-1
@@ -22,6 +22,7 @@ import (
|
||||
"git.wntrmute.dev/kyle/sift/internal/fetch"
|
||||
"git.wntrmute.dev/kyle/sift/internal/fleetctx"
|
||||
"git.wntrmute.dev/kyle/sift/internal/guard"
|
||||
"git.wntrmute.dev/kyle/sift/internal/metrics"
|
||||
"git.wntrmute.dev/kyle/sift/internal/recover"
|
||||
"git.wntrmute.dev/kyle/sift/internal/search"
|
||||
"git.wntrmute.dev/kyle/sift/internal/selfip"
|
||||
@@ -48,6 +49,7 @@ type Server struct {
|
||||
// PermitRestricted is context.permit_restricted: those callers never
|
||||
// have a fetch host written to the log (host=-). Independent of Honcho.
|
||||
PermitRestricted []string
|
||||
Prom *metrics.Set
|
||||
|
||||
global chan struct{}
|
||||
perMu sync.Mutex
|
||||
@@ -71,9 +73,11 @@ func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log
|
||||
global: make(chan struct{}, 8),
|
||||
per: map[string]chan struct{}{},
|
||||
}
|
||||
s.Prom = metrics.New(s)
|
||||
s.Mux.HandleFunc("GET /healthz", s.healthz)
|
||||
s.Mux.HandleFunc("GET /healthz/", s.healthz)
|
||||
s.Mux.HandleFunc("GET /metrics", s.metrics)
|
||||
s.Mux.HandleFunc("GET /metrics.json", s.metrics)
|
||||
s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
|
||||
s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
|
||||
s.Mux.HandleFunc("POST /v1/search", s.v1search)
|
||||
@@ -100,10 +104,38 @@ func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
|
||||
defer cancel()
|
||||
r = r.WithContext(ctx)
|
||||
cw := &codeWriter{ResponseWriter: w}
|
||||
start := time.Now()
|
||||
s.Mux.ServeHTTP(cw, r)
|
||||
code := cw.code
|
||||
if code == 0 {
|
||||
code = http.StatusOK
|
||||
}
|
||||
s.Prom.Observe(r.URL.Path, time.Since(start), code)
|
||||
return
|
||||
}
|
||||
s.Mux.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
type codeWriter struct {
|
||||
http.ResponseWriter
|
||||
code int
|
||||
}
|
||||
|
||||
func (c *codeWriter) WriteHeader(status int) {
|
||||
if c.code == 0 {
|
||||
c.code = status
|
||||
}
|
||||
c.ResponseWriter.WriteHeader(status)
|
||||
}
|
||||
|
||||
func (c *codeWriter) Write(b []byte) (int, error) {
|
||||
if c.code == 0 {
|
||||
c.code = http.StatusOK
|
||||
}
|
||||
return c.ResponseWriter.Write(b)
|
||||
}
|
||||
|
||||
func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
out := map[string]any{"status": "ok", "version": s.Version}
|
||||
@@ -119,7 +151,44 @@ func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
}
|
||||
|
||||
func (s *Server) metrics(w http.ResponseWriter, _ *http.Request) {
|
||||
func (s *Server) metrics(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.allowMetrics(w, r) {
|
||||
return
|
||||
}
|
||||
if r.URL.Path == "/metrics.json" || r.URL.Query().Get("format") == "json" {
|
||||
s.writeMetricsJSON(w)
|
||||
return
|
||||
}
|
||||
s.Prom.Handler().ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
func (s *Server) allowMetrics(w http.ResponseWriter, r *http.Request) bool {
|
||||
if metrics.LoopbackOpen(r) {
|
||||
return true
|
||||
}
|
||||
h := r.Header.Get("Authorization")
|
||||
tok := strings.TrimSpace(strings.TrimPrefix(h, "Bearer"))
|
||||
tok = strings.TrimSpace(tok)
|
||||
if tok == "" || s.Auth == nil {
|
||||
s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "missing metrics key")
|
||||
writeErr(w, 401, "unauthorized", "")
|
||||
return false
|
||||
}
|
||||
c := s.Auth.LookupBearer(tok)
|
||||
if c == nil {
|
||||
s.Log.Warn("unauthorized", "caller", "-", "host", "-", "reason", "unknown metrics key")
|
||||
writeErr(w, 401, "unauthorized", "")
|
||||
return false
|
||||
}
|
||||
if !c.MetricsOnly() {
|
||||
s.Log.Warn("unauthorized", "caller", c.Name, "host", "-", "reason", "not metrics caller")
|
||||
writeErr(w, 403, "unauthorized", "")
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func (s *Server) writeMetricsJSON(w http.ResponseWriter) {
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
fails := uint64(0)
|
||||
if s.SelfIP != nil {
|
||||
@@ -136,9 +205,80 @@ func (s *Server) metrics(w http.ResponseWriter, _ *http.Request) {
|
||||
m[k] = v
|
||||
}
|
||||
}
|
||||
calls, fb := uint64(0), map[string]uint64{}
|
||||
var inflight, waiters int
|
||||
if s.Digest != nil {
|
||||
calls, fb = s.Digest.Stats()
|
||||
inflight, waiters = s.Digest.Gate.Snapshot()
|
||||
}
|
||||
m["digest_calls"] = calls
|
||||
m["digest_fallbacks"] = fb
|
||||
m["dixie_inflight"] = inflight
|
||||
m["dixie_queue_waiters"] = waiters
|
||||
var dH, dM, eH, eM uint64
|
||||
if cache := s.cache(); cache != nil {
|
||||
dH, dM, eH, eM, _, _, _ = cache.Stats()
|
||||
}
|
||||
m["dcache_digest_hit"] = dH
|
||||
m["dcache_digest_miss"] = dM
|
||||
m["dcache_embed_hit"] = eH
|
||||
m["dcache_embed_miss"] = eM
|
||||
_ = json.NewEncoder(w).Encode(m)
|
||||
}
|
||||
|
||||
func (s *Server) cache() *dcache.DB {
|
||||
if s.Digest != nil && s.Digest.Cache != nil {
|
||||
return s.Digest.Cache
|
||||
}
|
||||
if s.Search != nil {
|
||||
return s.Search.Cache
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Server) Snapshot() metrics.Snapshot {
|
||||
out := metrics.Snapshot{
|
||||
Blocks: s.blockN.Load(),
|
||||
Scrapes: s.scrapeN.Load(),
|
||||
Restricted: map[string]struct{}{},
|
||||
}
|
||||
for _, n := range s.PermitRestricted {
|
||||
out.Restricted[n] = struct{}{}
|
||||
}
|
||||
if s.SelfIP != nil {
|
||||
out.SelfIPFailures = s.SelfIP.Failures.Load()
|
||||
}
|
||||
if s.Digest != nil {
|
||||
out.DigestCalls, out.DigestFallbacks = s.Digest.Stats()
|
||||
out.DixieInflight, out.DixieWaiters = s.Digest.Gate.Snapshot()
|
||||
}
|
||||
if cache := s.cache(); cache != nil {
|
||||
out.DigestHit, out.DigestMiss, out.EmbedHit, out.EmbedMiss, _, _, _ = cache.Stats()
|
||||
}
|
||||
if s.Search != nil {
|
||||
m := s.Search.Metrics()
|
||||
if v, ok := m["search_queries"].(map[string]map[string]uint64); ok {
|
||||
out.SearchQueries = v
|
||||
}
|
||||
if v, ok := m["search_credits"].(map[string]map[string]uint64); ok {
|
||||
out.SearchCredits = v
|
||||
}
|
||||
if v, ok := m["search_daily_credits"].(map[string]uint64); ok {
|
||||
out.SearchDaily = v
|
||||
}
|
||||
if v, ok := m["search_capped"].(map[string]uint64); ok {
|
||||
out.SearchCapped = v
|
||||
}
|
||||
if v, ok := m["search_cache_hit"].(uint64); ok {
|
||||
out.SearchHit = v
|
||||
}
|
||||
if v, ok := m["search_cache_miss"].(uint64); ok {
|
||||
out.SearchMiss = v
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
type scrapeReq struct {
|
||||
URL string `json:"url"`
|
||||
Formats []string `json:"formats"`
|
||||
@@ -177,6 +317,11 @@ func (s *Server) caller(w http.ResponseWriter, r *http.Request) *auth.Caller {
|
||||
writeErr(w, 401, "unauthorized", "")
|
||||
return nil
|
||||
}
|
||||
if c.MetricsOnly() {
|
||||
s.Log.Warn("unauthorized", "caller", c.Name, "host", "-", "reason", "metrics caller")
|
||||
writeErr(w, 403, "unauthorized", "")
|
||||
return nil
|
||||
}
|
||||
return c
|
||||
}
|
||||
|
||||
@@ -327,6 +472,9 @@ func (s *Server) v1fetch(w http.ResponseWriter, r *http.Request) {
|
||||
} else {
|
||||
dr := s.Digest.Run(r.Context(), content, q, res.FinalURL, s.callerClass(c))
|
||||
digestMS = dr.DigestMS
|
||||
if !dr.Cached {
|
||||
s.Prom.ObserveDigest(time.Duration(dr.DigestMS) * time.Millisecond)
|
||||
}
|
||||
if dr.Fallback != "" {
|
||||
warn = append(warn, "digest fell back to clean: "+dr.Fallback)
|
||||
mode = "clean"
|
||||
|
||||
Reference in New Issue
Block a user