p3: search API, pinned sources, cache, CLI

POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
2026-09-23 12:17:51 -07:00
parent 3687636ed7
commit d7873403ff
27 changed files with 2361 additions and 21 deletions
@@ -0,0 +1,175 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"log/slog"
+	"net/http"
+	"net/http/httptest"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/auth"
+	"git.wntrmute.dev/kyle/sift/internal/dcache"
+	"git.wntrmute.dev/kyle/sift/internal/fetch"
+	"git.wntrmute.dev/kyle/sift/internal/guard"
+	"git.wntrmute.dev/kyle/sift/internal/search"
+	"git.wntrmute.dev/kyle/sift/internal/search/dial"
+)
+
+func searchServer(t *testing.T) (*Server, string, string, *recHandler, *httptest.Server) {
+	t.Helper()
+	dir := t.TempDir()
+	keys := filepath.Join(dir, "keys.toml")
+	fleetOut := filepath.Join(dir, "fleet")
+	gsOut := filepath.Join(dir, "gs")
+	if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
+		t.Fatal(err)
+	}
+	if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil {
+		t.Fatal(err)
+	}
+	fleetKey, _ := os.ReadFile(fleetOut)
+	gsKey, _ := os.ReadFile(gsOut)
+	store, err := auth.Load(keys)
+	if err != nil {
+		t.Fatal(err)
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if strings.Contains(r.URL.Path, "search") {
+			_, _ = w.Write([]byte(`{"results":[{"url":"https://example.com/hit","title":"Hit","content":"snippet about q","score":1}]}`))
+			return
+		}
+		w.WriteHeader(404)
+	}))
+	t.Cleanup(ts.Close)
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	db, err := dcache.Open(filepath.Join(dir, "c.db"), 1<<20, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { _ = db.Close() })
+	h := &recHandler{}
+	s := New("test", &guard.Guard{}, &fetch.Fetcher{Guard: &guard.Guard{}, UA: "sift-test"}, store, slog.New(h))
+	s.PermitRestricted = []string{"straylight-gs"}
+	s.Search = &search.Engine{
+		Sources:  []search.Source{&search.SearxNG{Client: c}},
+		DefaultK: 8,
+		RerankOn: false,
+		Cache:    db,
+	}
+	return s, strings.TrimSpace(string(fleetKey)), strings.TrimSpace(string(gsKey)), h, ts
+}
+
+func TestV2SearchGolden(t *testing.T) {
+	s, key, _, _, _ := searchServer(t)
+	rec := postJSON(t, s, "/v2/search", key, map[string]any{"query": "contract", "limit": 3, "url": "http://127.0.0.1:11434"})
+	if rec.Code != 200 {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+	var env map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
+		t.Fatal(err)
+	}
+	if env["success"] != true {
+		t.Fatalf("%v", env)
+	}
+	data, _ := env["data"].(map[string]any)
+	web, _ := data["web"].([]any)
+	if len(web) < 1 {
+		t.Fatalf("web %v", env)
+	}
+	row, _ := web[0].(map[string]any)
+	if row["url"] == "" || row["title"] == "" || row["description"] == nil {
+		t.Fatalf("%v", row)
+	}
+}
+
+func TestSearchCachePartitionAndRestrictedLogs(t *testing.T) {
+	s, fleet, gs, h, _ := searchServer(t)
+	rec := postJSON(t, s, "/v1/search", gs, map[string]any{"query": "secret topic", "k": 5, "rerank": false})
+	if rec.Code != 200 {
+		t.Fatalf("gs %d %s", rec.Code, rec.Body.String())
+	}
+	rec = postJSON(t, s, "/v1/search", fleet, map[string]any{"query": "secret topic", "k": 5, "rerank": false})
+	if rec.Code != 200 {
+		t.Fatalf("fleet %d %s", rec.Code, rec.Body.String())
+	}
+	var env map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
+		t.Fatal(err)
+	}
+	if env["cached"] == true {
+		t.Fatal("fleet observed restricted cache")
+	}
+	dump := h.dump()
+	if strings.Contains(dump, "secret topic") || strings.Contains(dump, "example.com") || strings.Contains(dump, "127.0.0.1") {
+		t.Fatalf("restricted log leak:\n%s", dump)
+	}
+}
+
+func TestSearchInvalidQuery(t *testing.T) {
+	s, key, _, _, _ := searchServer(t)
+	rec := postJSON(t, s, "/v1/search", key, map[string]any{"query": ""})
+	if rec.Code != 400 {
+		t.Fatalf("%d", rec.Code)
+	}
+}
+
+func TestHealthzMissingKey(t *testing.T) {
+	s, _ := setupAPI(t, nil)
+	s.Search = &search.Engine{Degraded: []string{"search.tavily: missing key"}}
+	req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
+	rec := httptest.NewRecorder()
+	s.ServeHTTP(rec, req)
+	if rec.Code != 200 || !strings.Contains(rec.Body.String(), "missing key") {
+		t.Fatalf("%s", rec.Body.String())
+	}
+}
+
+func TestMetricsCacheHitMiss(t *testing.T) {
+	s, key, _, _, _ := searchServer(t)
+	postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
+	postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
+	req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
+	rec := httptest.NewRecorder()
+	s.ServeHTTP(rec, req)
+	var m map[string]any
+	_ = json.Unmarshal(rec.Body.Bytes(), &m)
+	if m["search_cache_hit"] == 0 && m["search_cache_miss"] == 0 {
+		t.Fatalf("%v", m)
+	}
+}
+
+func TestNoSearchResults422(t *testing.T) {
+	s, key := setupAPI(t, nil)
+	s.Search = &search.Engine{DefaultK: 8}
+	rec := postJSON(t, s, "/v1/search", key, map[string]any{"query": "none"})
+	if rec.Code != 422 {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func (h *recHandler) dump() string {
+	h.mu.Lock()
+	defer h.mu.Unlock()
+	var b bytes.Buffer
+	for _, r := range h.recs {
+		b.WriteString(recMsg(r))
+		r.Attrs(func(a slog.Attr) bool {
+			b.WriteByte(' ')
+			b.WriteString(a.Key)
+			b.WriteByte('=')
+			b.WriteString(a.Value.String())
+			return true
+		})
+		b.WriteByte('\n')
+	}
+	return b.String()
+}