p3: search API, pinned sources, cache, CLI
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
This commit is contained in:
+17
-3
@@ -23,6 +23,7 @@ import (
|
||||
"git.wntrmute.dev/kyle/sift/internal/fleetctx"
|
||||
"git.wntrmute.dev/kyle/sift/internal/guard"
|
||||
"git.wntrmute.dev/kyle/sift/internal/recover"
|
||||
"git.wntrmute.dev/kyle/sift/internal/search"
|
||||
"git.wntrmute.dev/kyle/sift/internal/selfip"
|
||||
)
|
||||
|
||||
@@ -42,6 +43,7 @@ type Server struct {
|
||||
OutCap int
|
||||
SelfIP *selfip.Discoverer
|
||||
Digest *digest.Engine
|
||||
Search *search.Engine
|
||||
Ctx *fleetctx.Client
|
||||
// PermitRestricted is context.permit_restricted: those callers never
|
||||
// have a fetch host written to the log (host=-). Independent of Honcho.
|
||||
@@ -74,6 +76,8 @@ func New(version string, g *guard.Guard, f *fetch.Fetcher, keys *auth.Store, log
|
||||
s.Mux.HandleFunc("GET /metrics", s.metrics)
|
||||
s.Mux.HandleFunc("POST /v2/scrape", s.scrape)
|
||||
s.Mux.HandleFunc("POST /v1/fetch", s.v1fetch)
|
||||
s.Mux.HandleFunc("POST /v1/search", s.v1search)
|
||||
s.Mux.HandleFunc("POST /v2/search", s.v2search)
|
||||
return s
|
||||
}
|
||||
|
||||
@@ -92,7 +96,7 @@ func (s *Server) HTTPServer(addr string) *http.Server {
|
||||
func (s *Server) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
r.Body = http.MaxBytesReader(w, r.Body, maxReqBody)
|
||||
switch r.URL.Path {
|
||||
case "/v1/fetch", "/v2/scrape":
|
||||
case "/v1/fetch", "/v2/scrape", "/v1/search", "/v2/search":
|
||||
ctx, cancel := context.WithTimeout(r.Context(), requestDeadline)
|
||||
defer cancel()
|
||||
r = r.WithContext(ctx)
|
||||
@@ -106,6 +110,10 @@ func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) {
|
||||
if s.SelfIP != nil {
|
||||
out["self_ip"] = s.SelfIP.Status()
|
||||
}
|
||||
if s.Search != nil && len(s.Search.Degraded) > 0 {
|
||||
out["status"] = "degraded"
|
||||
out["search"] = s.Search.Degraded
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(out)
|
||||
}
|
||||
|
||||
@@ -115,12 +123,18 @@ func (s *Server) metrics(w http.ResponseWriter, _ *http.Request) {
|
||||
if s.SelfIP != nil {
|
||||
fails = s.SelfIP.Failures.Load()
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||
m := map[string]any{
|
||||
"requests": s.reqN.Load(),
|
||||
"blocks": s.blockN.Load(),
|
||||
"scrapes": s.scrapeN.Load(),
|
||||
"self_ip_failures": fails,
|
||||
})
|
||||
}
|
||||
if s.Search != nil {
|
||||
for k, v := range s.Search.Metrics() {
|
||||
m[k] = v
|
||||
}
|
||||
}
|
||||
_ = json.NewEncoder(w).Encode(m)
|
||||
}
|
||||
|
||||
type scrapeReq struct {
|
||||
|
||||
Reference in New Issue
Block a user