kyle
8b26776cdd
Native include/exclude domains, time_range, topic, exact_match, min_score, and up to 4 queries fused with RRF. Honesty notes stay off the Warn/fail path. Tavily keeps include_answer false and sets auto_parameters false. Cache key is canonical JSON. CLI flags and skill recipe included.
140 lines
2.9 KiB
Go
140 lines
2.9 KiB
Go
package search
|
|
|
|
import (
|
|
"context"
|
|
"sort"
|
|
"strings"
|
|
"unicode/utf8"
|
|
)
|
|
|
|
const (
|
|
MaxQueryRunes = 1500
|
|
MaxK = 20
|
|
DefaultK = 8
|
|
DefaultV2K = 5
|
|
NameSearxNG = "searxng"
|
|
NameBraveLLM = "brave_llm"
|
|
NameTavily = "tavily"
|
|
)
|
|
|
|
var sourceRank = map[string]int{
|
|
NameSearxNG: 0,
|
|
NameBraveLLM: 1,
|
|
NameTavily: 2,
|
|
}
|
|
|
|
// Hit is one search result after mapping. No page body, no vendor answers.
|
|
type Hit struct {
|
|
URL string `json:"url"`
|
|
Title string `json:"title"`
|
|
Snippet string `json:"snippet"`
|
|
Score float64 `json:"score"`
|
|
Source string `json:"source"`
|
|
PublishedDate string `json:"published_date,omitempty"`
|
|
}
|
|
|
|
// Filter is native search knobs. The zero value is P3 behaviour.
|
|
type Filter struct {
|
|
IncludeDomains []string
|
|
ExcludeDomains []string
|
|
IncludeDomainsMode string // restrict | prefer; empty means restrict when include is set
|
|
TimeRange string // day|week|month|year
|
|
StartDate string
|
|
EndDate string
|
|
Topic string // general|news|finance; empty means general
|
|
ExactMatch bool
|
|
MinScore float64
|
|
}
|
|
|
|
// Source is one upstream. Config-fixed endpoint; never a request URL.
|
|
type Source interface {
|
|
Name() string
|
|
Search(ctx context.Context, query string, k int, f Filter) SourceOut
|
|
}
|
|
|
|
type SourceOut struct {
|
|
Hits []Hit
|
|
Warn string // failure only (timeout, HTTP error, pin)
|
|
Notes []string // honesty: filters not applied, recall loss
|
|
Credits int
|
|
}
|
|
|
|
func ValidateQuery(q string) string {
|
|
q = strings.TrimSpace(q)
|
|
if q == "" {
|
|
return "invalid query"
|
|
}
|
|
if utf8.RuneCountInString(q) > MaxQueryRunes {
|
|
return "invalid query"
|
|
}
|
|
return ""
|
|
}
|
|
|
|
func ClampK(k, def int) int {
|
|
if k <= 0 {
|
|
k = def
|
|
}
|
|
if k > MaxK {
|
|
return MaxK
|
|
}
|
|
return k
|
|
}
|
|
|
|
// Dedupe by canonical URL. First source (in fan-in order) wins the Source
|
|
// field; a later longer snippet replaces the text.
|
|
func Dedupe(in []Hit) []Hit {
|
|
type slot struct {
|
|
h Hit
|
|
i int
|
|
}
|
|
order := make([]string, 0)
|
|
seen := map[string]*slot{}
|
|
for _, h := range in {
|
|
canon := CanonicalURL(h.URL)
|
|
if canon == "" {
|
|
continue
|
|
}
|
|
if s, ok := seen[canon]; ok {
|
|
keepDate := firstDate(s.h.PublishedDate, h.PublishedDate)
|
|
if len(h.Snippet) > len(s.h.Snippet) {
|
|
src := s.h.Source
|
|
keep := s.h.URL
|
|
sc := s.h.Score
|
|
s.h = h
|
|
s.h.Source = src
|
|
s.h.URL = keep
|
|
if sc > s.h.Score {
|
|
s.h.Score = sc
|
|
}
|
|
}
|
|
s.h.PublishedDate = keepDate
|
|
continue
|
|
}
|
|
cp := h
|
|
seen[canon] = &slot{h: cp, i: len(order)}
|
|
order = append(order, canon)
|
|
}
|
|
out := make([]Hit, 0, len(order))
|
|
for _, c := range order {
|
|
out = append(out, seen[c].h)
|
|
}
|
|
return out
|
|
}
|
|
|
|
func firstDate(a, b string) string {
|
|
if a != "" {
|
|
return a
|
|
}
|
|
return b
|
|
}
|
|
|
|
func sortBySourceOrder(hits []Hit) {
|
|
sort.SliceStable(hits, func(i, j int) bool {
|
|
ri, rj := sourceRank[hits[i].Source], sourceRank[hits[j].Source]
|
|
if ri != rj {
|
|
return ri < rj
|
|
}
|
|
return false
|
|
})
|
|
}
|