p3: search API, pinned sources, cache, CLI
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
This commit is contained in:
@@ -0,0 +1,114 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sort"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
MaxQueryRunes = 1500
|
||||
MaxK = 20
|
||||
DefaultK = 8
|
||||
DefaultV2K = 5
|
||||
NameSearxNG = "searxng"
|
||||
NameBraveLLM = "brave_llm"
|
||||
NameTavily = "tavily"
|
||||
)
|
||||
|
||||
var sourceRank = map[string]int{
|
||||
NameSearxNG: 0,
|
||||
NameBraveLLM: 1,
|
||||
NameTavily: 2,
|
||||
}
|
||||
|
||||
// Hit is one search result after mapping. No page body, no vendor answers.
|
||||
type Hit struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Score float64 `json:"score"`
|
||||
Source string `json:"source"`
|
||||
}
|
||||
|
||||
// Source is one upstream. Config-fixed endpoint; never a request URL.
|
||||
type Source interface {
|
||||
Name() string
|
||||
Search(ctx context.Context, query string, k int) SourceOut
|
||||
}
|
||||
|
||||
type SourceOut struct {
|
||||
Hits []Hit
|
||||
Warn string
|
||||
Credits int
|
||||
}
|
||||
|
||||
func ValidateQuery(q string) string {
|
||||
q = strings.TrimSpace(q)
|
||||
if q == "" {
|
||||
return "invalid query"
|
||||
}
|
||||
if utf8.RuneCountInString(q) > MaxQueryRunes {
|
||||
return "invalid query"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func ClampK(k, def int) int {
|
||||
if k <= 0 {
|
||||
k = def
|
||||
}
|
||||
if k > MaxK {
|
||||
return MaxK
|
||||
}
|
||||
return k
|
||||
}
|
||||
|
||||
// Dedupe by canonical URL. First source (in fan-in order) wins the Source
|
||||
// field; a later longer snippet replaces the text.
|
||||
func Dedupe(in []Hit) []Hit {
|
||||
type slot struct {
|
||||
h Hit
|
||||
i int
|
||||
}
|
||||
order := make([]string, 0)
|
||||
seen := map[string]*slot{}
|
||||
for _, h := range in {
|
||||
canon := CanonicalURL(h.URL)
|
||||
if canon == "" {
|
||||
continue
|
||||
}
|
||||
h.URL = canon
|
||||
if s, ok := seen[canon]; ok {
|
||||
if len(h.Snippet) > len(s.h.Snippet) {
|
||||
src := s.h.Source
|
||||
sc := s.h.Score
|
||||
s.h = h
|
||||
s.h.Source = src
|
||||
if sc > s.h.Score {
|
||||
s.h.Score = sc
|
||||
}
|
||||
}
|
||||
continue
|
||||
}
|
||||
cp := h
|
||||
seen[canon] = &slot{h: cp, i: len(order)}
|
||||
order = append(order, canon)
|
||||
}
|
||||
out := make([]Hit, 0, len(order))
|
||||
for _, c := range order {
|
||||
out = append(out, seen[c].h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func sortBySourceOrder(hits []Hit) {
|
||||
sort.SliceStable(hits, func(i, j int) bool {
|
||||
ri, rj := sourceRank[hits[i].Source], sourceRank[hits[j].Source]
|
||||
if ri != rj {
|
||||
return ri < rj
|
||||
}
|
||||
return false
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user