kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p3: search API, pinned sources, cache, CLI

Browse Source 
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and
Tavily from config-fixed endpoints. Public sources refuse private A
records; SearXNG is a loopback-only client, not the fetch guard.

Cache keys include caller class. Restricted callers search like the
fleet; logs omit query, hosts, URLs, and upstream errors. Degraded
results are not stored. Tavily sends search_depth=basic and drops
answer. Missing keys surface in /healthz.

Default-on sources stay off until the bake-off. Query harvest is not
in this commit.
This commit is contained in:
kyle
2026-09-23 12:17:51 -07:00
parent 3687636ed7
commit d7873403ff
27 changed files with 2361 additions and 21 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/api.go
+17 -3
View File
@@ -23,6 +23,7 @@ import (
"git.wntrmute.dev/kyle/sift/internal/fleetctx"
"git.wntrmute.dev/kyle/sift/internal/guard"
"git.wntrmute.dev/kyle/sift/internal/recover"
"git.wntrmute.dev/kyle/sift/internal/search"
"git.wntrmute.dev/kyle/sift/internal/selfip"
)
@@ -42,6 +43,7 @@ type Server struct {
OutCap int
SelfIP *selfip.Discoverer
Digest *digest.Engine
Search *search.Engine
Ctx *fleetctx.Client
// PermitRestricted is context.permit_restricted: those callers never
// have a fetch host written to the log (host=-). Independent of Honcho.
@@ -74,6 +76,8 @@ func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log
s.Mux.HandleFunc("GET /metrics", s.metrics)
s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
s.Mux.HandleFunc("POST /v1/search", s.v1search)
s.Mux.HandleFunc("POST /v2/search", s.v2search)
return s
}
@@ -92,7 +96,7 @@ func (s *Server) HTTPServer(addr string) *http.Server {
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxReqBody)
switch r.URL.Path {
case "/v1/fetch", "/v2/scrape":
case "/v1/fetch", "/v2/scrape", "/v1/search", "/v2/search":
ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
defer cancel()
r = r.WithContext(ctx)
@@ -106,6 +110,10 @@ func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
if s.SelfIP != nil {
out["self_ip"] = s.SelfIP.Status()
}
if s.Search != nil && len(s.Search.Degraded) > 0 {
out["status"] = "degraded"
out["search"] = s.Search.Degraded
}
_ = json.NewEncoder(w).Encode(out)
}
@@ -115,12 +123,18 @@ func (s *Server) metrics(w http.ResponseWriter, _ *http.Request) {
if s.SelfIP != nil {
fails = s.SelfIP.Failures.Load()
}
_ = json.NewEncoder(w).Encode(map[string]any{
m := map[string]any{
"requests": s.reqN.Load(),
"blocks": s.blockN.Load(),
"scrapes": s.scrapeN.Load(),
"self_ip_failures": fails,
})
}
if s.Search != nil {
for k, v := range s.Search.Metrics() {
m[k] = v
}
}
_ = json.NewEncoder(w).Encode(m)
}
type scrapeReq struct {
internal/api/search.go
+147
View File
@@ -0,0 +1,147 @@
package api
import (
"encoding/json"
"net/http"
"strings"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/search"
)
type v1SearchReq struct {
Query string `json:"query"`
Question string `json:"question"`
K int `json:"k"`
Rerank *bool `json:"rerank"`
Digest bool `json:"digest"`
URL string `json:"url"`
Limit int `json:"limit"`
}
type v2SearchReq struct {
Query string `json:"query"`
Limit int `json:"limit"`
URL string `json:"url"`
Digest bool `json:"digest"`
}
func (s *Server) v1search(w http.ResponseWriter, r *http.Request) {
s.handleSearch(w, r, false)
}
func (s *Server) v2search(w http.ResponseWriter, r *http.Request) {
s.handleSearch(w, r, true)
}
func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
s.reqN.Add(1)
c := s.caller(w, r)
if c == nil {
return
}
dec := json.NewDecoder(r.Body)
query := ""
k := 0
doRerank := true
digest := false
if v2 {
var req v2SearchReq
if err := dec.Decode(&req); err != nil && err.Error() != "EOF" {
writeErr(w, 400, "invalid query", "")
return
}
query = req.Query
k = req.Limit
digest = req.Digest
_ = req.URL
} else {
var req v1SearchReq
if err := dec.Decode(&req); err != nil && err.Error() != "EOF" {
writeErr(w, 400, "invalid query", "")
return
}
query = req.Query
k = req.K
if k == 0 {
k = req.Limit
}
if req.Rerank != nil {
doRerank = *req.Rerank
}
digest = req.Digest
_ = req.URL
_ = req.Question
}
if msg := search.ValidateQuery(query); msg != "" {
writeErr(w, 400, msg, "")
return
}
defK := search.DefaultK
if v2 {
defK = search.DefaultV2K
}
k = search.ClampK(k, defK)
if s.Search == nil {
writeErr(w, 422, "no search results", "")
return
}
res := s.Search.Run(r.Context(), strings.TrimSpace(query), k, doRerank, c.Name, s.callerClass(c))
warns := append([]string(nil), res.Warnings...)
if digest {
warns = append(warns, "digest ignored")
}
if !res.Cached && len(res.SourcesOK) == 0 && len(res.Hits) == 0 {
writeErr(w, 422, "no search results", "")
return
}
s.logSearch(c, query, res)
if v2 {
web := make([]map[string]any, 0, len(res.Hits))
for i, h := range res.Hits {
web = append(web, map[string]any{
"url": h.URL,
"title": h.Title,
"description": h.Snippet,
"position": i + 1,
})
}
writeJSON(w, 200, map[string]any{"success": true, "data": map[string]any{"web": web}})
return
}
results := make([]map[string]any, 0, len(res.Hits))
for _, h := range res.Hits {
results = append(results, map[string]any{
"url": h.URL,
"title": h.Title,
"snippet": h.Snippet,
"score": h.Score,
"source": h.Source,
})
}
writeJSON(w, 200, map[string]any{
"results": results,
"warnings": warns,
"sizes": map[string]any{
"results": len(results),
"sources_ok": res.SourcesOK,
"sources_failed": res.SourcesFailed,
},
"timings": map[string]any{"search_ms": res.SearchMS, "rerank_ms": res.RerankMS},
"cached": res.Cached,
})
}
func (s *Server) logSearch(c *auth.Caller, query string, res search.RunResult) {
qLen := len([]rune(query))
cache := "miss"
if res.Cached {
cache = "hit"
}
src := strings.Join(res.Attempted, ",")
if s.restrictedLog(c) {
s.Log.Info("search", "caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache)
return
}
s.Log.Info("search", "caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache, "rerank_ms", res.RerankMS)
}
internal/api/search_test.go
+175
View File
@@ -0,0 +1,175 @@
package api
import (
"bytes"
"encoding/json"
"log/slog"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"time"
"git.wntrmute.dev/kyle/sift/internal/auth"
"git.wntrmute.dev/kyle/sift/internal/dcache"
"git.wntrmute.dev/kyle/sift/internal/fetch"
"git.wntrmute.dev/kyle/sift/internal/guard"
"git.wntrmute.dev/kyle/sift/internal/search"
"git.wntrmute.dev/kyle/sift/internal/search/dial"
)
func searchServer(t *testing.T) (*Server, string, string, *recHandler, *httptest.Server) {
t.Helper()
dir := t.TempDir()
keys := filepath.Join(dir, "keys.toml")
fleetOut := filepath.Join(dir, "fleet")
gsOut := filepath.Join(dir, "gs")
if err := auth.Add(keys, "test", "hermes-fleet", fleetOut); err != nil {
t.Fatal(err)
}
if err := auth.Add(keys, "straylight-gs", "guardian-star", gsOut); err != nil {
t.Fatal(err)
}
fleetKey, _ := os.ReadFile(fleetOut)
gsKey, _ := os.ReadFile(gsOut)
store, err := auth.Load(keys)
if err != nil {
t.Fatal(err)
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if strings.Contains(r.URL.Path, "search") {
_, _ = w.Write([]byte(`{"results":[{"url":"https://example.com/hit","title":"Hit","content":"snippet about q","score":1}]}`))
return
}
w.WriteHeader(404)
}))
t.Cleanup(ts.Close)
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
db, err := dcache.Open(filepath.Join(dir, "c.db"), 1<<20, nil)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = db.Close() })
h := &recHandler{}
s := New("test", &guard.Guard{}, &fetch.Fetcher{Guard: &guard.Guard{}, UA: "sift-test"}, store, slog.New(h))
s.PermitRestricted = []string{"straylight-gs"}
s.Search = &search.Engine{
Sources: []search.Source{&search.SearxNG{Client: c}},
DefaultK: 8,
RerankOn: false,
Cache: db,
}
return s, strings.TrimSpace(string(fleetKey)), strings.TrimSpace(string(gsKey)), h, ts
}
func TestV2SearchGolden(t *testing.T) {
s, key, _, _, _ := searchServer(t)
rec := postJSON(t, s, "/v2/search", key, map[string]any{"query": "contract", "limit": 3, "url": "http://127.0.0.1:11434"})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
var env map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
t.Fatal(err)
}
if env["success"] != true {
t.Fatalf("%v", env)
}
data, _ := env["data"].(map[string]any)
web, _ := data["web"].([]any)
if len(web) < 1 {
t.Fatalf("web %v", env)
}
row, _ := web[0].(map[string]any)
if row["url"] == "" || row["title"] == "" || row["description"] == nil {
t.Fatalf("%v", row)
}
}
func TestSearchCachePartitionAndRestrictedLogs(t *testing.T) {
s, fleet, gs, h, _ := searchServer(t)
rec := postJSON(t, s, "/v1/search", gs, map[string]any{"query": "secret topic", "k": 5, "rerank": false})
if rec.Code != 200 {
t.Fatalf("gs %d %s", rec.Code, rec.Body.String())
}
rec = postJSON(t, s, "/v1/search", fleet, map[string]any{"query": "secret topic", "k": 5, "rerank": false})
if rec.Code != 200 {
t.Fatalf("fleet %d %s", rec.Code, rec.Body.String())
}
var env map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
t.Fatal(err)
}
if env["cached"] == true {
t.Fatal("fleet observed restricted cache")
}
dump := h.dump()
if strings.Contains(dump, "secret topic") || strings.Contains(dump, "example.com") || strings.Contains(dump, "127.0.0.1") {
t.Fatalf("restricted log leak:\n%s", dump)
}
}
func TestSearchInvalidQuery(t *testing.T) {
s, key, _, _, _ := searchServer(t)
rec := postJSON(t, s, "/v1/search", key, map[string]any{"query": ""})
if rec.Code != 400 {
t.Fatalf("%d", rec.Code)
}
}
func TestHealthzMissingKey(t *testing.T) {
s, _ := setupAPI(t, nil)
s.Search = &search.Engine{Degraded: []string{"search.tavily: missing key"}}
req := httptest.NewRequest(http.MethodGet, "/healthz", nil)
rec := httptest.NewRecorder()
s.ServeHTTP(rec, req)
if rec.Code != 200 || !strings.Contains(rec.Body.String(), "missing key") {
t.Fatalf("%s", rec.Body.String())
}
}
func TestMetricsCacheHitMiss(t *testing.T) {
s, key, _, _, _ := searchServer(t)
postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
postJSON(t, s, "/v1/search", key, map[string]any{"query": "same q", "k": 5, "rerank": false})
req := httptest.NewRequest(http.MethodGet, "/metrics", nil)
rec := httptest.NewRecorder()
s.ServeHTTP(rec, req)
var m map[string]any
_ = json.Unmarshal(rec.Body.Bytes(), &m)
if m["search_cache_hit"] == 0 && m["search_cache_miss"] == 0 {
t.Fatalf("%v", m)
}
}
func TestNoSearchResults422(t *testing.T) {
s, key := setupAPI(t, nil)
s.Search = &search.Engine{DefaultK: 8}
rec := postJSON(t, s, "/v1/search", key, map[string]any{"query": "none"})
if rec.Code != 422 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
}
func (h *recHandler) dump() string {
h.mu.Lock()
defer h.mu.Unlock()
var b bytes.Buffer
for _, r := range h.recs {
b.WriteString(recMsg(r))
r.Attrs(func(a slog.Attr) bool {
b.WriteByte(' ')
b.WriteString(a.Key)
b.WriteByte('=')
b.WriteString(a.Value.String())
return true
})
b.WriteByte('\n')
}
return b.String()
}