diff --git a/internal/api/api.go b/internal/api/api.go index 10d6569..439e54d 100644 --- a/internal/api/api.go +++ b/internal/api/api.go @@ -110,9 +110,11 @@ func (s *Server) healthz(w http.ResponseWriter, _ *http.Request) { if s.SelfIP != nil { out["self_ip"] = s.SelfIP.Status() } - if s.Search != nil && len(s.Search.Degraded) > 0 { - out["status"] = "degraded" - out["search"] = s.Search.Degraded + if s.Search != nil { + if d := s.Search.Health(); len(d) > 0 { + out["status"] = "degraded" + out["search"] = d + } } _ = json.NewEncoder(w).Encode(out) } diff --git a/internal/search/build.go b/internal/search/build.go index b9275aa..b20476c 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -17,7 +17,8 @@ const ( ) // Build constructs sources from config. Missing keys are listed on Engine.Degraded -// and logged once; those sources are not attempted. +// and logged once; those sources are not attempted. A source whose DNS is down +// at boot stays in the list and is retried on Search. func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) *Engine { timeout := config.ParseDuration(cfg.Timeout, 8*time.Second) ttl := config.ParseDuration(cfg.CacheTTL, time.Hour) @@ -40,15 +41,9 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) if u == "" { u = defaultSearx } - c, err := dial.New(u, dial.Options{Loopback: true, Timeout: timeout}) - if err != nil { - e.Degraded = append(e.Degraded, "search.searxng: "+err.Error()) - if log != nil { - log.Error("search source", "source", NameSearxNG, "err", err) - } - } else { - e.Sources = append(e.Sources, &SearxNG{Client: c}) - } + addPinned(e, log, NameSearxNG, u, dial.Options{Loopback: true, Timeout: timeout}, func(c *dial.Client) Source { + return &SearxNG{Client: c} + }) } if cfg.BraveLLM.Enabled { key := os.Getenv("BRAVE_API_KEY") @@ -62,15 +57,9 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) if u == "" { u = defaultBrave } - c, err := dial.New(u, dial.Options{Timeout: timeout}) - if err != nil { - e.Degraded = append(e.Degraded, "search.brave_llm: "+err.Error()) - if log != nil { - log.Error("search source", "source", NameBraveLLM, "err", err) - } - } else { - e.Sources = append(e.Sources, &BraveLLM{Client: c, Key: key}) - } + addPinned(e, log, NameBraveLLM, u, dial.Options{Timeout: timeout}, func(c *dial.Client) Source { + return &BraveLLM{Client: c, Key: key} + }) } } if cfg.Tavily.Enabled { @@ -85,16 +74,25 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) if u == "" { u = defaultTav } - c, err := dial.New(u, dial.Options{Timeout: timeout}) - if err != nil { - e.Degraded = append(e.Degraded, "search.tavily: "+err.Error()) - if log != nil { - log.Error("search source", "source", NameTavily, "err", err) - } - } else { - e.Sources = append(e.Sources, &Tavily{Client: c, Key: key}) - } + addPinned(e, log, NameTavily, u, dial.Options{Timeout: timeout}, func(c *dial.Client) Source { + return &Tavily{Client: c, Key: key} + }) } } return e } + +func addPinned(e *Engine, log *slog.Logger, name, raw string, opt dial.Options, wrap func(*dial.Client) Source) { + c, err := dial.New(raw, opt) + if err != nil { + e.Degraded = append(e.Degraded, "search."+name+": unavailable") + if log != nil { + log.Error("search source", "source", name, "err", "unavailable") + } + return + } + e.Sources = append(e.Sources, wrap(c)) + if !c.Ready() && log != nil { + log.Error("search source unresolved, will retry", "source", name) + } +} diff --git a/internal/search/dial/dial.go b/internal/search/dial/dial.go index 9b3873f..2b54e5c 100644 --- a/internal/search/dial/dial.go +++ b/internal/search/dial/dial.go @@ -77,9 +77,7 @@ func New(raw string, opt Options) (*Client, error) { lookup: lookup, refresh: opt.Refresh, } - if err := c.resolve(context.Background()); err != nil { - return nil, err - } + _ = c.resolve(context.Background()) // DNS down: keep the client, retry in Search d := &net.Dialer{ Timeout: opt.Timeout, Control: c.control, @@ -204,17 +202,38 @@ func (c *Client) checkRedirect(req *http.Request, via []*http.Request) error { return nil } -// RefreshIfStale re-resolves when the pin is older than Refresh. +// RefreshIfStale re-resolves when unpinned or the pin is older than Refresh. +// A failed refresh leaves existing pins in place. func (c *Client) RefreshIfStale(ctx context.Context) error { c.mu.Lock() - stale := time.Since(c.resolved) >= c.refresh + need := len(c.ips) == 0 || time.Since(c.resolved) >= c.refresh c.mu.Unlock() - if !stale { + if !need { return nil } return c.resolve(ctx) } +// Ensure resolves if needed. Error only when still unpinned afterwards. +func (c *Client) Ensure(ctx context.Context) error { + if c == nil { + return fmt.Errorf("search dial: nil") + } + if err := c.RefreshIfStale(ctx); err != nil && !c.Ready() { + return err + } + return nil +} + +func (c *Client) Ready() bool { + if c == nil { + return false + } + c.mu.Lock() + defer c.mu.Unlock() + return len(c.ips) > 0 +} + func (c *Client) IPs() []netip.Addr { c.mu.Lock() defer c.mu.Unlock() diff --git a/internal/search/dial/dial_test.go b/internal/search/dial/dial_test.go index eb6faa6..64cd2c2 100644 --- a/internal/search/dial/dial_test.go +++ b/internal/search/dial/dial_test.go @@ -2,16 +2,20 @@ package dial_test import ( "context" + "errors" "net/http" "net/http/httptest" "net/netip" "net/url" "strings" + "sync/atomic" "testing" "git.wntrmute.dev/kyle/sift/internal/search/dial" ) +var errDNS = errors.New("dns down") + func TestLoopbackAllowsConfiguredPort(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.WriteHeader(200) @@ -53,25 +57,31 @@ func TestLoopbackRefusesOtherPort(t *testing.T) { } func TestPublicPoisonedPrivateRefused(t *testing.T) { - _, err := dial.New("https://api.tavily.com/search", dial.Options{ + c, err := dial.New("https://api.tavily.com/search", dial.Options{ Loopback: false, Lookup: func(_ context.Context, host string) ([]netip.Addr, error) { return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil }, }) - if err == nil { - t.Fatal("poisoned A record allowed") + if err != nil { + t.Fatal(err) + } + if c.Ready() { + t.Fatal("poisoned A record pinned") } } func TestPublicPrivateRFC1918Refused(t *testing.T) { - _, err := dial.New("https://api.search.brave.com/res/v1/llm/context", dial.Options{ + c, err := dial.New("https://api.search.brave.com/res/v1/llm/context", dial.Options{ Lookup: func(_ context.Context, _ string) ([]netip.Addr, error) { return []netip.Addr{netip.MustParseAddr("10.0.0.5")}, nil }, }) - if err == nil { - t.Fatal("private A allowed") + if err != nil { + t.Fatal(err) + } + if c.Ready() { + t.Fatal("private A pinned") } } @@ -110,3 +120,39 @@ func TestParseRejectsNonHTTP(t *testing.T) { } _ = url.URL{} } + +func TestResolveRetriesAfterDNSDown(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(200) + })) + defer ts.Close() + u, _ := url.Parse(ts.URL) + ip := netip.MustParseAddr(u.Hostname()) + var n atomic.Int32 + c, err := dial.New(ts.URL, dial.Options{ + Loopback: true, + Lookup: func(_ context.Context, host string) ([]netip.Addr, error) { + if n.Add(1) == 1 { + return nil, errDNS + } + return []netip.Addr{ip}, nil + }, + }) + if err != nil { + t.Fatal(err) + } + if c.Ready() { + t.Fatal("first lookup should fail") + } + if err := c.Ensure(context.Background()); err != nil { + t.Fatal(err) + } + if !c.Ready() { + t.Fatal("retry") + } + resp, err := c.HTTP.Get(ts.URL + "/") + if err != nil { + t.Fatal(err) + } + resp.Body.Close() +} diff --git a/internal/search/engine.go b/internal/search/engine.go index c501019..aa7bdf9 100644 --- a/internal/search/engine.go +++ b/internal/search/engine.go @@ -20,7 +20,7 @@ type Engine struct { CacheTTL time.Duration Embedder Embedder Log *slog.Logger - Degraded []string // healthz: "search.brave_llm: missing key" + Degraded []string // static: missing keys TavilyCap int Now func() time.Time @@ -46,6 +46,27 @@ type RunResult struct { Attempted []string } +type readySrc interface { + Name() string + Ready() bool +} + +// Health is missing keys plus sources that still have no pin. +func (e *Engine) Health() []string { + if e == nil { + return nil + } + out := append([]string(nil), e.Degraded...) + for _, s := range e.Sources { + r, ok := s.(readySrc) + if !ok || r.Ready() { + continue + } + out = append(out, "search."+r.Name()+": unresolved") + } + return out +} + func (e *Engine) now() time.Time { if e != nil && e.Now != nil { return e.Now() diff --git a/internal/search/sources.go b/internal/search/sources.go index 4dfceab..fdd5e95 100644 --- a/internal/search/sources.go +++ b/internal/search/sources.go @@ -30,6 +30,16 @@ func warnErr(name string, err error) string { return name + ": error" } +func pinErr(ctx context.Context, c *dial.Client, name string) string { + if c == nil { + return name + ": error" + } + if err := c.Ensure(ctx); err != nil { + return warnErr(name, err) + } + return "" +} + func readJSON(resp *http.Response, dest any) error { defer resp.Body.Close() if resp.StatusCode != 200 { @@ -46,12 +56,16 @@ type SearxNG struct { } func (s *SearxNG) Name() string { return NameSearxNG } +func (s *SearxNG) Ready() bool { return s != nil && s.Client.Ready() } +func (s *BraveLLM) Ready() bool { return s != nil && s.Client.Ready() } func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut { if s == nil || s.Client == nil { return SourceOut{Warn: NameSearxNG + ": error"} } - _ = s.Client.RefreshIfStale(ctx) + if w := pinErr(ctx, s.Client, NameSearxNG); w != "" { + return SourceOut{Warn: w} + } base := strings.TrimRight(s.Client.URL.String(), "/") u, err := url.Parse(base + "/search") if err != nil { @@ -110,7 +124,9 @@ func (s *BraveLLM) Search(ctx context.Context, query string, k int) SourceOut { if s == nil || s.Client == nil { return SourceOut{Warn: NameBraveLLM + ": error"} } - _ = s.Client.RefreshIfStale(ctx) + if w := pinErr(ctx, s.Client, NameBraveLLM); w != "" { + return SourceOut{Warn: w} + } u, err := url.Parse(s.Client.URL.String()) if err != nil { return SourceOut{Warn: warnErr(NameBraveLLM, err)} diff --git a/internal/search/sources_test.go b/internal/search/sources_test.go index eda21fb..655ea40 100644 --- a/internal/search/sources_test.go +++ b/internal/search/sources_test.go @@ -95,3 +95,21 @@ func TestBraveMapsSources(t *testing.T) { t.Fatal("grounding used as snippet") } } + +func TestTavilyBadJSONIsError(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write([]byte(`not-json`)) + })) + defer ts.Close() + c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5) + if out.Warn != "tavily: error" { + t.Fatalf("warn %q", out.Warn) + } + if len(out.Hits) != 0 || out.Credits != 0 { + t.Fatalf("%+v", out) + } +} diff --git a/internal/search/tavily.go b/internal/search/tavily.go index 4315806..06959f5 100644 --- a/internal/search/tavily.go +++ b/internal/search/tavily.go @@ -17,6 +17,7 @@ type Tavily struct { } func (s *Tavily) Name() string { return NameTavily } +func (s *Tavily) Ready() bool { return s != nil && s.Client.Ready() } type tavilyReq struct { Query string `json:"query"` @@ -31,7 +32,9 @@ func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut { if s == nil || s.Client == nil { return SourceOut{Warn: NameTavily + ": error"} } - _ = s.Client.RefreshIfStale(ctx) + if w := pinErr(ctx, s.Client, NameTavily); w != "" { + return SourceOut{Warn: w} + } if k <= 0 { k = DefaultK } @@ -66,11 +69,14 @@ func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut { if err != nil { return SourceOut{Warn: warnErr(NameTavily, err)} } - hits, credits := mapTavily(raw) + hits, credits, err := mapTavily(raw) + if err != nil { + return SourceOut{Warn: NameTavily + ": error"} + } return SourceOut{Hits: hits, Credits: credits} } -func mapTavily(raw []byte) ([]Hit, int) { +func mapTavily(raw []byte) ([]Hit, int, error) { var env struct { Answer string `json:"answer"` Results []struct { @@ -84,7 +90,7 @@ func mapTavily(raw []byte) ([]Hit, int) { } `json:"usage"` } if err := json.Unmarshal(raw, &env); err != nil { - return nil, 0 + return nil, 0, err } _ = env.Answer // dropped; never returned hits := make([]Hit, 0, len(env.Results)) @@ -98,5 +104,5 @@ func mapTavily(raw []byte) ([]Hit, int) { if credits <= 0 { credits = 1 } - return hits, credits + return hits, credits, nil }