p3: search API, pinned sources, cache, CLI

POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and
Tavily from config-fixed endpoints. Public sources refuse private A
records; SearXNG is a loopback-only client, not the fetch guard.

Cache keys include caller class. Restricted callers search like the
fleet; logs omit query, hosts, URLs, and upstream errors. Degraded
results are not stored. Tavily sends search_depth=basic and drops
answer. Missing keys surface in /healthz.

Default-on sources stay off until the bake-off. Query harvest is not
in this commit.
This commit is contained in:
2026-09-23 12:17:51 -07:00
parent 3687636ed7
commit d7873403ff
27 changed files with 2361 additions and 21 deletions
+114
View File
@@ -0,0 +1,114 @@
package search
import (
"context"
"sort"
"strings"
"unicode/utf8"
)
const (
MaxQueryRunes = 1500
MaxK = 20
DefaultK = 8
DefaultV2K = 5
NameSearxNG = "searxng"
NameBraveLLM = "brave_llm"
NameTavily = "tavily"
)
var sourceRank = map[string]int{
NameSearxNG: 0,
NameBraveLLM: 1,
NameTavily: 2,
}
// Hit is one search result after mapping. No page body, no vendor answers.
type Hit struct {
URL string `json:"url"`
Title string `json:"title"`
Snippet string `json:"snippet"`
Score float64 `json:"score"`
Source string `json:"source"`
}
// Source is one upstream. Config-fixed endpoint; never a request URL.
type Source interface {
Name() string
Search(ctx context.Context, query string, k int) SourceOut
}
type SourceOut struct {
Hits []Hit
Warn string
Credits int
}
func ValidateQuery(q string) string {
q = strings.TrimSpace(q)
if q == "" {
return "invalid query"
}
if utf8.RuneCountInString(q) > MaxQueryRunes {
return "invalid query"
}
return ""
}
func ClampK(k, def int) int {
if k <= 0 {
k = def
}
if k > MaxK {
return MaxK
}
return k
}
// Dedupe by canonical URL. First source (in fan-in order) wins the Source
// field; a later longer snippet replaces the text.
func Dedupe(in []Hit) []Hit {
type slot struct {
h Hit
i int
}
order := make([]string, 0)
seen := map[string]*slot{}
for _, h := range in {
canon := CanonicalURL(h.URL)
if canon == "" {
continue
}
h.URL = canon
if s, ok := seen[canon]; ok {
if len(h.Snippet) > len(s.h.Snippet) {
src := s.h.Source
sc := s.h.Score
s.h = h
s.h.Source = src
if sc > s.h.Score {
s.h.Score = sc
}
}
continue
}
cp := h
seen[canon] = &slot{h: cp, i: len(order)}
order = append(order, canon)
}
out := make([]Hit, 0, len(order))
for _, c := range order {
out = append(out, seen[c].h)
}
return out
}
func sortBySourceOrder(hits []Hit) {
sort.SliceStable(hits, func(i, j int) bool {
ri, rj := sourceRank[hits[i].Source], sourceRank[hits[j].Source]
if ri != rj {
return ri < rj
}
return false
})
}