p2.7: prometheus /metrics
Default GET /metrics is Prometheus text; JSON stays at /metrics.json and ?format=json. :8443 needs the metrics caller bearer; loopback stays open. Restricted callers have no per-caller series and no tavily:restricted gauge.
This commit is contained in:
@@ -0,0 +1,256 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/auth"
|
||||
"git.wntrmute.dev/kyle/sift/internal/digest"
|
||||
dto "github.com/prometheus/client_model/go"
|
||||
"github.com/prometheus/common/expfmt"
|
||||
)
|
||||
|
||||
func loopbackGET(path string) *http.Request {
|
||||
req := httptest.NewRequest(http.MethodGet, path, nil)
|
||||
req.Host = "127.0.0.1:8890"
|
||||
req.RemoteAddr = "127.0.0.1:9"
|
||||
return req
|
||||
}
|
||||
|
||||
func serveMetrics(t *testing.T, s *Server, req *http.Request) *httptest.ResponseRecorder {
|
||||
t.Helper()
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, req)
|
||||
return rec
|
||||
}
|
||||
|
||||
func parseProm(t *testing.T, body string) map[string]*dto.MetricFamily {
|
||||
t.Helper()
|
||||
parser := expfmt.NewDecoder(strings.NewReader(body), expfmt.NewFormat(expfmt.TypeTextPlain))
|
||||
out := map[string]*dto.MetricFamily{}
|
||||
for {
|
||||
var mf dto.MetricFamily
|
||||
if err := parser.Decode(&mf); err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
t.Fatalf("expfmt: %v\n%s", err, body)
|
||||
}
|
||||
out[mf.GetName()] = &mf
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestMetricsJSONSidePath(t *testing.T) {
|
||||
s, key, _, _, _ := searchServer(t)
|
||||
postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
|
||||
postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
|
||||
for _, path := range []string{"/metrics.json", "/metrics?format=json"} {
|
||||
rec := serveMetrics(t, s, loopbackGET(path))
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%s %d %s", path, rec.Code, rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Header().Get("Content-Type"), "json") {
|
||||
t.Fatalf("%s ctype %s", path, rec.Header().Get("Content-Type"))
|
||||
}
|
||||
var m map[string]any
|
||||
if err := json.Unmarshal(rec.Body.Bytes(), &m); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if m["search_cache_hit"] == 0 && m["search_cache_miss"] == 0 {
|
||||
t.Fatalf("%s %v", path, m)
|
||||
}
|
||||
if _, ok := m["requests"]; !ok {
|
||||
t.Fatalf("missing requests: %v", m)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsPrometheusLoopback(t *testing.T) {
|
||||
s, key, _, _, _ := searchServer(t)
|
||||
postJSON(t, s, "/v1/search", key, map[string]any{"query": "fleet q", "k": 5, "rerank": false})
|
||||
rec := serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("%d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
ct := rec.Header().Get("Content-Type")
|
||||
if !strings.Contains(ct, "text/plain") {
|
||||
t.Fatalf("ctype %s", ct)
|
||||
}
|
||||
fams := parseProm(t, rec.Body.String())
|
||||
if fams["sift_search_queries_total"] == nil {
|
||||
t.Fatalf("missing queries total: %v", rec.Body.String())
|
||||
}
|
||||
if fams["sift_search_queries_caller_total"] == nil {
|
||||
t.Fatalf("missing caller total")
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), `caller="test"`) {
|
||||
t.Fatal("fleet caller label")
|
||||
}
|
||||
if strings.Contains(rec.Body.String(), "fleet q") {
|
||||
t.Fatal("query leaked")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsAuth(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
keys := filepath.Join(dir, "keys.toml")
|
||||
fleetOut := filepath.Join(dir, "fleet")
|
||||
metOut := filepath.Join(dir, "met")
|
||||
if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := auth.Add(keys, auth.NameMetrics, "should-be-cleared", metOut); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
store, err := auth.Load(keys)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
mc := store.LookupBearer(strings.TrimSpace(mustRead(t, metOut)))
|
||||
if mc == nil || mc.Workspace != "" || !mc.MetricsOnly() {
|
||||
t.Fatalf("metrics caller %+v", mc)
|
||||
}
|
||||
s, fleetKey := setupAPI(t, nil)
|
||||
s.Auth = store
|
||||
fleetKey = strings.TrimSpace(mustRead(t, fleetOut))
|
||||
metKey := strings.TrimSpace(mustRead(t, metOut))
|
||||
|
||||
req := httptest.NewRequest(http.MethodGet, "/metrics", nil) // 192.0.2.1 + example.com
|
||||
rec := serveMetrics(t, s, req)
|
||||
if rec.Code != 401 {
|
||||
t.Fatalf("no bearer: %d", rec.Code)
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+fleetKey)
|
||||
rec = serveMetrics(t, s, req)
|
||||
if rec.Code != 403 {
|
||||
t.Fatalf("fleet bearer: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
req = httptest.NewRequest(http.MethodGet, "/metrics", nil)
|
||||
req.Header.Set("Authorization", "Bearer "+metKey)
|
||||
rec = serveMetrics(t, s, req)
|
||||
if rec.Code != 200 {
|
||||
t.Fatalf("metrics bearer: %d %s", rec.Code, rec.Body.String())
|
||||
}
|
||||
|
||||
fwd := loopbackGET("/metrics")
|
||||
fwd.Header.Set("X-Forwarded-For", "100.1.2.3")
|
||||
rec = serveMetrics(t, s, fwd)
|
||||
if rec.Code != 401 {
|
||||
t.Fatalf("forwarded loopback: %d", rec.Code)
|
||||
}
|
||||
|
||||
rec = postJSON(t, s, "/v1/fetch", metKey, map[string]string{"url": "https://example.com/"})
|
||||
if rec.Code != 403 {
|
||||
t.Fatalf("metrics fetch: %d", rec.Code)
|
||||
}
|
||||
rec = postJSON(t, s, "/v1/search", metKey, map[string]any{"query": "x"})
|
||||
if rec.Code != 403 {
|
||||
t.Fatalf("metrics search: %d", rec.Code)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsRestrictedNoCallerLabel(t *testing.T) {
|
||||
s, fleetKey, gsKey, _, _ := searchServer(t)
|
||||
q := "secret-restricted-query-42"
|
||||
postJSON(t, s, "/v1/search", fleetKey, map[string]any{"query": "fleet-only-q", "k": 5, "rerank": false})
|
||||
postJSON(t, s, "/v1/search", gsKey, map[string]any{"query": q, "k": 5, "rerank": false})
|
||||
rec := serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
body := rec.Body.String()
|
||||
if rec.Code != 200 {
|
||||
t.Fatal(body)
|
||||
}
|
||||
if strings.Contains(body, q) || strings.Contains(body, "fleet-only-q") {
|
||||
t.Fatal("query in prometheus")
|
||||
}
|
||||
if strings.Contains(body, "example.com") {
|
||||
t.Fatal("host in prometheus")
|
||||
}
|
||||
if strings.Contains(body, `caller="straylight-gs"`) {
|
||||
t.Fatal("restricted caller series")
|
||||
}
|
||||
if strings.Contains(body, "tavily:restricted") {
|
||||
t.Fatal("restricted cap partition")
|
||||
}
|
||||
if !strings.Contains(body, `caller="test"`) {
|
||||
t.Fatal("fleet caller missing")
|
||||
}
|
||||
js := serveMetrics(t, s, loopbackGET("/metrics.json"))
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(js.Body.Bytes(), &m)
|
||||
queries, _ := m["search_queries"].(map[string]any)
|
||||
if queries["straylight-gs"] == nil {
|
||||
t.Fatalf("json still has restricted caller: %v", m["search_queries"])
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsDigestFallbackAndGauges(t *testing.T) {
|
||||
s, _ := setupAPI(t, nil)
|
||||
g := digest.NewGate(1, 0, time.Second)
|
||||
s.Digest = &digest.Engine{Gate: g}
|
||||
ok, _ := g.Acquire(context.Background())
|
||||
if !ok {
|
||||
t.Fatal("hold permit")
|
||||
}
|
||||
s.Digest.Run(context.Background(), "doc", "q", "https://ex.test/p", "fleet")
|
||||
rec := serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
body := rec.Body.String()
|
||||
if !strings.Contains(body, `sift_digest_fallbacks_total{reason="queue_full"}`) &&
|
||||
!strings.Contains(body, `reason="queue_full"`) {
|
||||
t.Fatal(body)
|
||||
}
|
||||
if !strings.Contains(body, "sift_dixie_inflight") || !strings.Contains(body, "sift_dixie_queue_waiters") {
|
||||
t.Fatal(body)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsDoesNotCountScrapes(t *testing.T) {
|
||||
s, key := setupAPI(t, nil)
|
||||
before := serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
after := serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
if before.Body.String() != after.Body.String() && strings.Contains(after.Body.String(), `endpoint="/metrics"`) {
|
||||
t.Fatal(after.Body.String())
|
||||
}
|
||||
postJSON(t, s, "/v2/scrape", key, map[string]string{"url": "http://127.0.0.1/"})
|
||||
rec := serveMetrics(t, s, loopbackGET("/metrics"))
|
||||
if !strings.Contains(rec.Body.String(), `sift_requests_total{code="403",endpoint="/v2/scrape"}`) &&
|
||||
!strings.Contains(rec.Body.String(), `endpoint="/v2/scrape"`) {
|
||||
t.Fatal(rec.Body.String())
|
||||
}
|
||||
if !strings.Contains(rec.Body.String(), "sift_request_duration_seconds") {
|
||||
t.Fatal("histogram missing")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMetricsCacheHitMiss(t *testing.T) {
|
||||
s, key, _, _, _ := searchServer(t)
|
||||
postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
|
||||
postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
|
||||
req := loopbackGET("/metrics.json")
|
||||
rec := httptest.NewRecorder()
|
||||
s.ServeHTTP(rec, req)
|
||||
var m map[string]any
|
||||
_ = json.Unmarshal(rec.Body.Bytes(), &m)
|
||||
if m["search_cache_hit"] == 0 && m["search_cache_miss"] == 0 {
|
||||
t.Fatalf("%v", m)
|
||||
}
|
||||
}
|
||||
|
||||
func mustRead(t *testing.T, path string) string {
|
||||
t.Helper()
|
||||
b, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
Reference in New Issue
Block a user