p3: search API, pinned sources, cache, CLI

POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and
Tavily from config-fixed endpoints. Public sources refuse private A
records; SearXNG is a loopback-only client, not the fetch guard.

Cache keys include caller class. Restricted callers search like the
fleet; logs omit query, hosts, URLs, and upstream errors. Degraded
results are not stored. Tavily sends search_depth=basic and drops
answer. Missing keys surface in /healthz.

Default-on sources stay off until the bake-off. Query harvest is not
in this commit.
This commit is contained in:
2026-09-23 12:17:51 -07:00
parent 3687636ed7
commit d7873403ff
27 changed files with 2361 additions and 21 deletions
+17 -3
View File
@@ -23,6 +23,7 @@ import (
"git.wntrmute.dev/kyle/sift/internal/fleetctx"
"git.wntrmute.dev/kyle/sift/internal/guard"
"git.wntrmute.dev/kyle/sift/internal/recover"
"git.wntrmute.dev/kyle/sift/internal/search"
"git.wntrmute.dev/kyle/sift/internal/selfip"
)
@@ -42,6 +43,7 @@ type Server struct {
OutCap int
SelfIP *selfip.Discoverer
Digest *digest.Engine
Search *search.Engine
Ctx *fleetctx.Client
// PermitRestricted is context.permit_restricted: those callers never
// have a fetch host written to the log (host=-). Independent of Honcho.
@@ -74,6 +76,8 @@ func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log
s.Mux.HandleFunc("GET /metrics", s.metrics)
s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
s.Mux.HandleFunc("POST /v1/search", s.v1search)
s.Mux.HandleFunc("POST /v2/search", s.v2search)
return s
}
@@ -92,7 +96,7 @@ func (s *Server) HTTPServer(addr string) *http.Server {
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
r.Body = http.MaxBytesReader(w, r.Body, maxReqBody)
switch r.URL.Path {
case "/v1/fetch", "/v2/scrape":
case "/v1/fetch", "/v2/scrape", "/v1/search", "/v2/search":
ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
defer cancel()
r = r.WithContext(ctx)
@@ -106,6 +110,10 @@ func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
if s.SelfIP != nil {
out["self_ip"] = s.SelfIP.Status()
}
if s.Search != nil && len(s.Search.Degraded) > 0 {
out["status"] = "degraded"
out["search"] = s.Search.Degraded
}
_ = json.NewEncoder(w).Encode(out)
}
@@ -115,12 +123,18 @@ func (s *Server) metrics(w http.ResponseWriter, _ *http.Request) {
if s.SelfIP != nil {
fails = s.SelfIP.Failures.Load()
}
_ = json.NewEncoder(w).Encode(map[string]any{
m := map[string]any{
"requests": s.reqN.Load(),
"blocks": s.blockN.Load(),
"scrapes": s.scrapeN.Load(),
"self_ip_failures": fails,
})
}
if s.Search != nil {
for k, v := range s.Search.Metrics() {
m[k] = v
}
}
_ = json.NewEncoder(w).Encode(m)
}
type scrapeReq struct {