kyle
d7873403ff
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
115 lines
2.1 KiB
Go
115 lines
2.1 KiB
Go
package search
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
const (
|
|
MaxQueryRunes = 1500
|
|
MaxK = 20
|
|
DefaultK = 8
|
|
DefaultV2K = 5
|
|
NameSearxNG = "searxng"
|
|
NameBraveLLM = "brave_llm"
|
|
NameTavily = "tavily"
|
|
)
|
|
|
|
var sourceRank = map[string]int{
|
|
NameSearxNG: 0,
|
|
NameBraveLLM: 1,
|
|
NameTavily: 2,
|
|
}
|
|
|
|
// Hit is one search result after mapping. No page body, no vendor answers.
|
|
type Hit struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Snippet string `json:"snippet"`
|
|
Score float64 `json:"score"`
|
|
Source string `json:"source"`
|
|
}
|
|
|
|
// Source is one upstream. Config-fixed endpoint; never a request URL.
|
|
type Source interface {
|
|
Name() string
|
|
Search(ctx context.Context, query string, k int) SourceOut
|
|
}
|
|
|
|
type SourceOut struct {
|
|
Hits []Hit
|
|
Warn string
|
|
Credits int
|
|
}
|
|
|
|
func ValidateQuery(q string) string {
|
|
q = strings.TrimSpace(q)
|
|
if q == "" {
|
|
return "invalid query"
|
|
}
|
|
if utf8.RuneCountInString(q) > MaxQueryRunes {
|
|
return "invalid query"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func ClampK(k, def int) int {
|
|
if k <= 0 {
|
|
k = def
|
|
}
|
|
if k > MaxK {
|
|
return MaxK
|
|
}
|
|
return k
|
|
}
|
|
|
|
// Dedupe by canonical URL. First source (in fan-in order) wins the Source
|
|
// field; a later longer snippet replaces the text.
|
|
func Dedupe(in []Hit) []Hit {
|
|
type slot struct {
|
|
h Hit
|
|
i int
|
|
}
|
|
order := make([]string, 0)
|
|
seen := map[string]*slot{}
|
|
for _, h := range in {
|
|
canon := CanonicalURL(h.URL)
|
|
if canon == "" {
|
|
continue
|
|
}
|
|
h.URL = canon
|
|
if s, ok := seen[canon]; ok {
|
|
if len(h.Snippet) > len(s.h.Snippet) {
|
|
src := s.h.Source
|
|
sc := s.h.Score
|
|
s.h = h
|
|
s.h.Source = src
|
|
if sc > s.h.Score {
|
|
s.h.Score = sc
|
|
}
|
|
}
|
|
continue
|
|
}
|
|
cp := h
|
|
seen[canon] = &slot{h: cp, i: len(order)}
|
|
order = append(order, canon)
|
|
}
|
|
out := make([]Hit, 0, len(order))
|
|
for _, c := range order {
|
|
out = append(out, seen[c].h)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func sortBySourceOrder(hits []Hit) {
|
|
sort.SliceStable(hits, func(i, j int) bool {
|
|
ri, rj := sourceRank[hits[i].Source], sourceRank[hits[j].Source]
|
|
if ri != rj {
|
|
return ri < rj
|
|
}
|
|
return false
|
|
})
|
|
}
|