p3: search API, pinned sources, cache, CLI

POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
2026-09-23 12:17:51 -07:00
parent 3687636ed7
commit d7873403ff
27 changed files with 2361 additions and 21 deletions
@@ -0,0 +1,100 @@
+package search
+
+import (
+	"log/slog"
+	"os"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/config"
+	"git.wntrmute.dev/kyle/sift/internal/dcache"
+	"git.wntrmute.dev/kyle/sift/internal/search/dial"
+)
+
+const (
+	defaultSearx = "http://127.0.0.1:8888"
+	defaultBrave = "https://api.search.brave.com/res/v1/llm/context"
+	defaultTav   = "https://api.tavily.com/search"
+)
+
+// Build constructs sources from config. Missing keys are listed on Engine.Degraded
+// and logged once; those sources are not attempted.
+func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) *Engine {
+	timeout := config.ParseDuration(cfg.Timeout, 8*time.Second)
+	ttl := config.ParseDuration(cfg.CacheTTL, time.Hour)
+	k := cfg.K
+	if k <= 0 {
+		k = DefaultK
+	}
+	e := &Engine{
+		Timeout:   timeout,
+		DefaultK:  k,
+		RerankOn:  cfg.RerankOn(),
+		Cache:     cache,
+		CacheTTL:  ttl,
+		Embedder:  emb,
+		Log:       log,
+		TavilyCap: cfg.Tavily.DailyCreditCap,
+	}
+	if cfg.SearxNG.Enabled {
+		u := cfg.SearxNG.URL
+		if u == "" {
+			u = defaultSearx
+		}
+		c, err := dial.New(u, dial.Options{Loopback: true, Timeout: timeout})
+		if err != nil {
+			e.Degraded = append(e.Degraded, "search.searxng: "+err.Error())
+			if log != nil {
+				log.Error("search source", "source", NameSearxNG, "err", err)
+			}
+		} else {
+			e.Sources = append(e.Sources, &SearxNG{Client: c})
+		}
+	}
+	if cfg.BraveLLM.Enabled {
+		key := os.Getenv("BRAVE_API_KEY")
+		if key == "" {
+			e.Degraded = append(e.Degraded, "search.brave_llm: missing key")
+			if log != nil {
+				log.Error("search source", "source", NameBraveLLM, "err", "missing key")
+			}
+		} else {
+			u := cfg.BraveLLM.URL
+			if u == "" {
+				u = defaultBrave
+			}
+			c, err := dial.New(u, dial.Options{Timeout: timeout})
+			if err != nil {
+				e.Degraded = append(e.Degraded, "search.brave_llm: "+err.Error())
+				if log != nil {
+					log.Error("search source", "source", NameBraveLLM, "err", err)
+				}
+			} else {
+				e.Sources = append(e.Sources, &BraveLLM{Client: c, Key: key})
+			}
+		}
+	}
+	if cfg.Tavily.Enabled {
+		key := os.Getenv("TAVILY_API_KEY")
+		if key == "" {
+			e.Degraded = append(e.Degraded, "search.tavily: missing key")
+			if log != nil {
+				log.Error("search source", "source", NameTavily, "err", "missing key")
+			}
+		} else {
+			u := cfg.Tavily.URL
+			if u == "" {
+				u = defaultTav
+			}
+			c, err := dial.New(u, dial.Options{Timeout: timeout})
+			if err != nil {
+				e.Degraded = append(e.Degraded, "search.tavily: "+err.Error())
+				if log != nil {
+					log.Error("search source", "source", NameTavily, "err", err)
+				}
+			} else {
+				e.Sources = append(e.Sources, &Tavily{Client: c, Key: key})
+			}
+		}
+	}
+	return e
+}