kyle
d7873403ff
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
103 lines
2.6 KiB
Go
103 lines
2.6 KiB
Go
package search
|
|
|
|
import (
|
|
"bytes"
|
|
"context"
|
|
"encoding/json"
|
|
"io"
|
|
"net/http"
|
|
|
|
"git.wntrmute.dev/kyle/sift/internal/search/dial"
|
|
)
|
|
|
|
// Tavily is POST {url} JSON. Never requests or returns answer.
|
|
type Tavily struct {
|
|
Client *dial.Client
|
|
Key string
|
|
}
|
|
|
|
func (s *Tavily) Name() string { return NameTavily }
|
|
|
|
type tavilyReq struct {
|
|
Query string `json:"query"`
|
|
MaxResults int `json:"max_results"`
|
|
IncludeAnswer bool `json:"include_answer"`
|
|
IncludeRawContent bool `json:"include_raw_content"`
|
|
IncludeImages bool `json:"include_images"`
|
|
SearchDepth string `json:"search_depth"`
|
|
}
|
|
|
|
func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
|
|
if s == nil || s.Client == nil {
|
|
return SourceOut{Warn: NameTavily + ": error"}
|
|
}
|
|
_ = s.Client.RefreshIfStale(ctx)
|
|
if k <= 0 {
|
|
k = DefaultK
|
|
}
|
|
if k > 20 {
|
|
k = 20
|
|
}
|
|
payload, _ := json.Marshal(tavilyReq{
|
|
Query: query,
|
|
MaxResults: k,
|
|
IncludeAnswer: false,
|
|
IncludeRawContent: false,
|
|
IncludeImages: false,
|
|
SearchDepth: "basic",
|
|
})
|
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.Client.URL.String(), bytes.NewReader(payload))
|
|
if err != nil {
|
|
return SourceOut{Warn: warnErr(NameTavily, err)}
|
|
}
|
|
req.Header.Set("Content-Type", "application/json")
|
|
req.Header.Set("Authorization", "Bearer "+s.Key)
|
|
resp, err := s.Client.HTTP.Do(req)
|
|
if err != nil {
|
|
return SourceOut{Warn: warnErr(NameTavily, err)}
|
|
}
|
|
if resp.StatusCode != 200 {
|
|
code := resp.StatusCode
|
|
resp.Body.Close()
|
|
return SourceOut{Warn: warnStatus(NameTavily, code)}
|
|
}
|
|
raw, err := io.ReadAll(io.LimitReader(resp.Body, 1<<20))
|
|
resp.Body.Close()
|
|
if err != nil {
|
|
return SourceOut{Warn: warnErr(NameTavily, err)}
|
|
}
|
|
hits, credits := mapTavily(raw)
|
|
return SourceOut{Hits: hits, Credits: credits}
|
|
}
|
|
|
|
func mapTavily(raw []byte) ([]Hit, int) {
|
|
var env struct {
|
|
Answer string `json:"answer"`
|
|
Results []struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Content string `json:"content"`
|
|
Score float64 `json:"score"`
|
|
} `json:"results"`
|
|
Usage struct {
|
|
Credits int `json:"credits"`
|
|
} `json:"usage"`
|
|
}
|
|
if err := json.Unmarshal(raw, &env); err != nil {
|
|
return nil, 0
|
|
}
|
|
_ = env.Answer // dropped; never returned
|
|
hits := make([]Hit, 0, len(env.Results))
|
|
for _, r := range env.Results {
|
|
if r.URL == "" {
|
|
continue
|
|
}
|
|
hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameTavily})
|
|
}
|
|
credits := env.Usage.Credits
|
|
if credits <= 0 {
|
|
credits = 1
|
|
}
|
|
return hits, credits
|
|
}
|