kyle
b6e1cdf825
Live llm/context hits live in grounding.generic plus a URL-keyed sources map, not a sources array. Fan-in uses reciprocal-rank fusion instead of cosine. no_rerank keeps that source's own order as a prefix.
109 lines
2.9 KiB
Go
109 lines
2.9 KiB
Go
package search
|
|
|
|
import (
|
|
"log/slog"
|
|
"os"
|
|
"time"
|
|
|
|
"git.wntrmute.dev/kyle/sift/internal/config"
|
|
"git.wntrmute.dev/kyle/sift/internal/dcache"
|
|
"git.wntrmute.dev/kyle/sift/internal/search/dial"
|
|
)
|
|
|
|
const (
|
|
defaultSearx = "http://127.0.0.1:8888"
|
|
defaultBrave = "https://api.search.brave.com/res/v1/llm/context"
|
|
defaultTav = "https://api.tavily.com/search"
|
|
)
|
|
|
|
// Build constructs sources from config. Missing keys are listed on Engine.Degraded
|
|
// and logged once; those sources are not attempted. A source whose DNS is down
|
|
// at boot stays in the list and is retried on Search.
|
|
func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) *Engine {
|
|
timeout := config.ParseDuration(cfg.Timeout, 8*time.Second)
|
|
ttl := config.ParseDuration(cfg.CacheTTL, time.Hour)
|
|
k := cfg.K
|
|
if k <= 0 {
|
|
k = DefaultK
|
|
}
|
|
e := &Engine{
|
|
Timeout: timeout,
|
|
DefaultK: k,
|
|
RerankOn: cfg.RerankOn(),
|
|
Cache: cache,
|
|
CacheTTL: ttl,
|
|
Embedder: emb,
|
|
Log: log,
|
|
TavilyCap: cfg.Tavily.DailyCreditCap,
|
|
NoRerank: map[string]bool{},
|
|
}
|
|
if cfg.SearxNG.NoRerank {
|
|
e.NoRerank[NameSearxNG] = true
|
|
}
|
|
if cfg.BraveLLM.NoRerank {
|
|
e.NoRerank[NameBraveLLM] = true
|
|
}
|
|
if cfg.Tavily.NoRerank {
|
|
e.NoRerank[NameTavily] = true
|
|
}
|
|
if cfg.SearxNG.Enabled {
|
|
u := cfg.SearxNG.URL
|
|
if u == "" {
|
|
u = defaultSearx
|
|
}
|
|
addPinned(e, log, NameSearxNG, u, dial.Options{Loopback: true, Timeout: timeout}, func(c *dial.Client) Source {
|
|
return &SearxNG{Client: c}
|
|
})
|
|
}
|
|
if cfg.BraveLLM.Enabled {
|
|
key := os.Getenv("BRAVE_API_KEY")
|
|
if key == "" {
|
|
e.Degraded = append(e.Degraded, "search.brave_llm: missing key")
|
|
if log != nil {
|
|
log.Error("search source", "source", NameBraveLLM, "err", "missing key")
|
|
}
|
|
} else {
|
|
u := cfg.BraveLLM.URL
|
|
if u == "" {
|
|
u = defaultBrave
|
|
}
|
|
addPinned(e, log, NameBraveLLM, u, dial.Options{Timeout: timeout}, func(c *dial.Client) Source {
|
|
return &BraveLLM{Client: c, Key: key}
|
|
})
|
|
}
|
|
}
|
|
if cfg.Tavily.Enabled {
|
|
key := os.Getenv("TAVILY_API_KEY")
|
|
if key == "" {
|
|
e.Degraded = append(e.Degraded, "search.tavily: missing key")
|
|
if log != nil {
|
|
log.Error("search source", "source", NameTavily, "err", "missing key")
|
|
}
|
|
} else {
|
|
u := cfg.Tavily.URL
|
|
if u == "" {
|
|
u = defaultTav
|
|
}
|
|
addPinned(e, log, NameTavily, u, dial.Options{Timeout: timeout}, func(c *dial.Client) Source {
|
|
return &Tavily{Client: c, Key: key}
|
|
})
|
|
}
|
|
}
|
|
return e
|
|
}
|
|
|
|
func addPinned(e *Engine, log *slog.Logger, name, raw string, opt dial.Options, wrap func(*dial.Client) Source) {
|
|
c, err := dial.New(raw, opt)
|
|
if err != nil {
|
|
e.Degraded = append(e.Degraded, "search."+name+": unavailable")
|
|
if log != nil {
|
|
log.Error("search source", "source", name, "err", "unavailable")
|
|
}
|
|
return
|
|
}
|
|
e.Sources = append(e.Sources, wrap(c))
|
|
if !c.Ready() && log != nil {
|
|
log.Error("search source unresolved, will retry", "source", name)
|
|
}
|
|
}
|