kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p3: tavily parse errors degrade; retry DNS on search

Browse Source 
A 200 with a bad Tavily body is tavily: error, not an empty success
that would cache for the hour. Upstream clients stay in the source
list if DNS is down at boot and retry the pin on Search; /healthz
lists them as unresolved until a pin lands.
This commit is contained in:
kyle
2026-09-23 12:22:31 -07:00
parent f748e695b1
commit 429f6b207e
8 changed files with 177 additions and 51 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/search/build.go
+26 -28
View File
@@ -17,7 +17,8 @@ const (
)
// Build constructs sources from config. Missing keys are listed on Engine.Degraded
// and logged once; those sources are not attempted.
// and logged once; those sources are not attempted. A source whose DNS is down
// at boot stays in the list and is retried on Search.
func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) *Engine {
timeout := config.ParseDuration(cfg.Timeout, 8*time.Second)
ttl := config.ParseDuration(cfg.CacheTTL, time.Hour)
@@ -40,15 +41,9 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger)
if u == "" {
u = defaultSearx
}
c, err := dial.New(u, dial.Options{Loopback: true, Timeout: timeout})
if err != nil {
e.Degraded = append(e.Degraded, "search.searxng: "+err.Error())
if log != nil {
log.Error("search source", "source", NameSearxNG, "err", err)
}
} else {
e.Sources = append(e.Sources, &SearxNG{Client: c})
}
addPinned(e, log, NameSearxNG, u, dial.Options{Loopback: true, Timeout: timeout}, func(c *dial.Client) Source {
return &SearxNG{Client: c}
})
}
if cfg.BraveLLM.Enabled {
key := os.Getenv("BRAVE_API_KEY")
@@ -62,15 +57,9 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger)
if u == "" {
u = defaultBrave
}
c, err := dial.New(u, dial.Options{Timeout: timeout})
if err != nil {
e.Degraded = append(e.Degraded, "search.brave_llm: "+err.Error())
if log != nil {
log.Error("search source", "source", NameBraveLLM, "err", err)
}
} else {
e.Sources = append(e.Sources, &BraveLLM{Client: c, Key: key})
}
addPinned(e, log, NameBraveLLM, u, dial.Options{Timeout: timeout}, func(c *dial.Client) Source {
return &BraveLLM{Client: c, Key: key}
})
}
}
if cfg.Tavily.Enabled {
@@ -85,16 +74,25 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger)
if u == "" {
u = defaultTav
}
c, err := dial.New(u, dial.Options{Timeout: timeout})
if err != nil {
e.Degraded = append(e.Degraded, "search.tavily: "+err.Error())
if log != nil {
log.Error("search source", "source", NameTavily, "err", err)
}
} else {
e.Sources = append(e.Sources, &Tavily{Client: c, Key: key})
}
addPinned(e, log, NameTavily, u, dial.Options{Timeout: timeout}, func(c *dial.Client) Source {
return &Tavily{Client: c, Key: key}
})
}
}
return e
}
func addPinned(e *Engine, log *slog.Logger, name, raw string, opt dial.Options, wrap func(*dial.Client) Source) {
c, err := dial.New(raw, opt)
if err != nil {
e.Degraded = append(e.Degraded, "search."+name+": unavailable")
if log != nil {
log.Error("search source", "source", name, "err", "unavailable")
}
return
}
e.Sources = append(e.Sources, wrap(c))
if !c.Ready() && log != nil {
log.Error("search source unresolved, will retry", "source", name)
}
}