feat(search): P3.1 filters and multi-query merge
Native include/exclude domains, time_range, topic, exact_match, min_score, and up to 4 queries fused with RRF. Honesty notes stay off the Warn/fail path. Tavily keeps include_answer false and sets auto_parameters false. Cache key is canonical JSON. CLI flags and skill recipe included.
This commit is contained in:
@@ -0,0 +1,166 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"net/url"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func resultHost(raw string) string {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return ""
|
||||
}
|
||||
h := strings.ToLower(u.Hostname())
|
||||
return strings.TrimSuffix(h, ".")
|
||||
}
|
||||
|
||||
func hostMatch(host, d string) bool {
|
||||
return host == d || strings.HasSuffix(host, "."+d)
|
||||
}
|
||||
|
||||
func filterDomains(hits []Hit, f Filter) []Hit {
|
||||
if len(f.IncludeDomains) == 0 && len(f.ExcludeDomains) == 0 {
|
||||
return hits
|
||||
}
|
||||
mode := f.IncludeDomainsMode
|
||||
if mode == "" {
|
||||
mode = "restrict"
|
||||
}
|
||||
restrict := mode != "prefer" && len(f.IncludeDomains) > 0
|
||||
out := make([]Hit, 0, len(hits))
|
||||
for _, h := range hits {
|
||||
host := resultHost(h.URL)
|
||||
if host == "" {
|
||||
if restrict {
|
||||
continue
|
||||
}
|
||||
} else {
|
||||
drop := false
|
||||
for _, d := range f.ExcludeDomains {
|
||||
if hostMatch(host, d) {
|
||||
drop = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if drop {
|
||||
continue
|
||||
}
|
||||
if restrict {
|
||||
ok := false
|
||||
for _, d := range f.IncludeDomains {
|
||||
if hostMatch(host, d) {
|
||||
ok = true
|
||||
break
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
out = append(out, h)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func filterMinScore(hits []Hit, min float64) []Hit {
|
||||
if min <= 0 {
|
||||
return hits
|
||||
}
|
||||
out := make([]Hit, 0, len(hits))
|
||||
for _, h := range hits {
|
||||
if h.Score >= min {
|
||||
out = append(out, h)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func dedupeStrings(in []string) []string {
|
||||
seen := map[string]bool{}
|
||||
out := make([]string, 0, len(in))
|
||||
for _, s := range in {
|
||||
if s == "" || seen[s] {
|
||||
continue
|
||||
}
|
||||
seen[s] = true
|
||||
out = append(out, s)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func searxQueryAndNotes(q string, f Filter) (string, []string) {
|
||||
var notes []string
|
||||
if f.IncludeDomainsMode == "prefer" && len(f.IncludeDomains) > 0 {
|
||||
notes = append(notes, "searxng: include_domains_mode=prefer not applied")
|
||||
} else if n := len(f.IncludeDomains); n == 1 {
|
||||
q = "site:" + f.IncludeDomains[0] + " " + q
|
||||
} else if n > 1 {
|
||||
max := n
|
||||
if max > searxSiteCap {
|
||||
max = searxSiteCap
|
||||
notes = append(notes, "searxng: extra include_domains dropped")
|
||||
}
|
||||
var b strings.Builder
|
||||
for i := 0; i < max; i++ {
|
||||
if i > 0 {
|
||||
b.WriteString(" OR ")
|
||||
}
|
||||
b.WriteString("site:")
|
||||
b.WriteString(f.IncludeDomains[i])
|
||||
}
|
||||
b.WriteByte(' ')
|
||||
b.WriteString(q)
|
||||
q = b.String()
|
||||
}
|
||||
if f.Topic == "finance" {
|
||||
notes = append(notes, "searxng: topic ignored")
|
||||
}
|
||||
if f.StartDate != "" || f.EndDate != "" {
|
||||
notes = append(notes, "searxng: start_date/end_date not applied")
|
||||
}
|
||||
if f.MinScore > 0 {
|
||||
notes = append(notes, "searxng: min_score not applied")
|
||||
}
|
||||
return q, notes
|
||||
}
|
||||
|
||||
func braveNotes(f Filter) []string {
|
||||
var notes []string
|
||||
if f.TimeRange != "" {
|
||||
notes = append(notes, "brave_llm: time_range not applied")
|
||||
}
|
||||
if f.Topic != "" && f.Topic != "general" {
|
||||
notes = append(notes, "brave_llm: topic not applied")
|
||||
}
|
||||
if f.ExactMatch {
|
||||
notes = append(notes, "brave_llm: exact_match not applied")
|
||||
}
|
||||
if f.IncludeDomainsMode == "prefer" && len(f.IncludeDomains) > 0 {
|
||||
notes = append(notes, "brave_llm: include_domains_mode=prefer not applied")
|
||||
}
|
||||
if f.MinScore > 0 {
|
||||
notes = append(notes, "brave_llm: min_score not applied")
|
||||
}
|
||||
return notes
|
||||
}
|
||||
|
||||
func FilterKinds(f Filter) string {
|
||||
var k []string
|
||||
if len(f.IncludeDomains) > 0 || len(f.ExcludeDomains) > 0 {
|
||||
k = append(k, "domains")
|
||||
}
|
||||
if f.TimeRange != "" || f.StartDate != "" || f.EndDate != "" {
|
||||
k = append(k, "time")
|
||||
}
|
||||
if f.Topic != "" && f.Topic != "general" {
|
||||
k = append(k, "topic")
|
||||
}
|
||||
if f.ExactMatch {
|
||||
k = append(k, "exact")
|
||||
}
|
||||
if f.MinScore > 0 {
|
||||
k = append(k, "score")
|
||||
}
|
||||
return strings.Join(k, ",")
|
||||
}
|
||||
@@ -2,7 +2,6 @@ package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"log/slog"
|
||||
"sort"
|
||||
"sync"
|
||||
@@ -90,104 +89,7 @@ func (e *Engine) timeout() time.Duration {
|
||||
}
|
||||
|
||||
func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, caller, class string) RunResult {
|
||||
start := e.now()
|
||||
k = ClampK(k, e.DefaultK)
|
||||
if e.DefaultK <= 0 {
|
||||
k = ClampK(k, DefaultK)
|
||||
}
|
||||
if class == "" {
|
||||
class = dcache.ClassFleet
|
||||
}
|
||||
|
||||
attempted := e.attemptedNames(class)
|
||||
key := dcache.SearchKey(query, attempted, k, e.now(), class)
|
||||
if b, ok := e.Cache.GetSearch(key); ok {
|
||||
var hits []Hit
|
||||
if json.Unmarshal(b, &hits) == nil {
|
||||
e.mu.Lock()
|
||||
e.CacheHit++
|
||||
e.mu.Unlock()
|
||||
return RunResult{Hits: clampHits(hits, k), Cached: true, Attempted: attempted, SearchMS: e.now().Sub(start).Milliseconds()}
|
||||
}
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.CacheMiss++
|
||||
e.mu.Unlock()
|
||||
|
||||
type one struct {
|
||||
name string
|
||||
out SourceOut
|
||||
skip bool
|
||||
}
|
||||
outs := make([]one, len(e.Sources))
|
||||
var wg sync.WaitGroup
|
||||
for i, src := range e.Sources {
|
||||
name := src.Name()
|
||||
if name == NameTavily && e.tavilyCapped(class) {
|
||||
outs[i] = one{name: name, out: SourceOut{Warn: "tavily: daily cap"}, skip: true}
|
||||
e.addCapped(NameTavily)
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, src Source, name string) {
|
||||
defer wg.Done()
|
||||
sctx, cancel := context.WithTimeout(ctx, e.timeout())
|
||||
defer cancel()
|
||||
outs[i] = one{name: name, out: src.Search(sctx, query, k)}
|
||||
}(i, src, name)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var lists []namedHits
|
||||
var hits []Hit
|
||||
var warns []string
|
||||
var okNames, failNames, att []string
|
||||
degraded := false
|
||||
for _, o := range outs {
|
||||
if o.skip {
|
||||
if o.out.Warn != "" {
|
||||
warns = append(warns, o.out.Warn)
|
||||
}
|
||||
continue
|
||||
}
|
||||
att = append(att, o.name)
|
||||
e.addQuery(caller, o.name)
|
||||
if o.out.Warn != "" {
|
||||
warns = append(warns, o.out.Warn)
|
||||
failNames = append(failNames, o.name)
|
||||
degraded = true
|
||||
e.addErr(o.name, o.out.Warn)
|
||||
continue
|
||||
}
|
||||
okNames = append(okNames, o.name)
|
||||
lists = append(lists, namedHits{name: o.name, hits: o.out.Hits})
|
||||
if o.out.Credits > 0 {
|
||||
e.addCredits(caller, o.name, o.out.Credits)
|
||||
if o.name == NameTavily && e.Cache != nil {
|
||||
e.Cache.AddSearchCredits(tavilyCapSource(class), e.now().UTC().Format("2006-01-02"), o.out.Credits)
|
||||
}
|
||||
}
|
||||
}
|
||||
hits = mergeSearch(e, lists, doRerank)
|
||||
|
||||
var rerankMS int64
|
||||
hits = clampHits(hits, k)
|
||||
|
||||
if !degraded && e.Cache != nil && len(att) > 0 {
|
||||
payload, _ := json.Marshal(hits)
|
||||
ck := dcache.SearchKey(query, att, k, e.now(), class)
|
||||
e.Cache.PutSearch(ck, payload, e.ttl())
|
||||
}
|
||||
|
||||
return RunResult{
|
||||
Hits: hits,
|
||||
Warnings: warns,
|
||||
SourcesOK: okNames,
|
||||
SourcesFailed: failNames,
|
||||
SearchMS: e.now().Sub(start).Milliseconds(),
|
||||
RerankMS: rerankMS,
|
||||
Attempted: att,
|
||||
}
|
||||
return e.RunFilter(ctx, []string{query}, k, doRerank, caller, class, Filter{})
|
||||
}
|
||||
|
||||
func (e *Engine) attemptedNames(class string) []string {
|
||||
|
||||
@@ -14,12 +14,13 @@ type fakeSrc struct {
|
||||
name string
|
||||
hits []Hit
|
||||
warn string
|
||||
notes []string
|
||||
delay time.Duration
|
||||
credits int
|
||||
}
|
||||
|
||||
func (f fakeSrc) Name() string { return f.name }
|
||||
func (f fakeSrc) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
func (f fakeSrc) Search(ctx context.Context, query string, k int, _ Filter) SourceOut {
|
||||
if f.delay > 0 {
|
||||
select {
|
||||
case <-time.After(f.delay):
|
||||
@@ -27,7 +28,7 @@ func (f fakeSrc) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
return SourceOut{Warn: f.name + ": timeout"}
|
||||
}
|
||||
}
|
||||
return SourceOut{Hits: f.hits, Warn: f.warn, Credits: f.credits}
|
||||
return SourceOut{Hits: f.hits, Warn: f.warn, Credits: f.credits, Notes: f.notes}
|
||||
}
|
||||
|
||||
type stubEmb struct {
|
||||
|
||||
@@ -0,0 +1,254 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"time"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
const (
|
||||
maxDomainList = 20
|
||||
maxHostLen = 253
|
||||
searxSiteCap = 3
|
||||
)
|
||||
|
||||
var timeRangeLong = map[string]string{
|
||||
"day": "day", "d": "day",
|
||||
"week": "week", "w": "week",
|
||||
"month": "month", "m": "month",
|
||||
"year": "year", "y": "year",
|
||||
}
|
||||
|
||||
// NormalizeSearch checks query-vs-queries and filter fields. errMsg is
|
||||
// "invalid query" or "invalid search"; hint is empty or a field name.
|
||||
func NormalizeSearch(query string, queries []string, f Filter) (qs []string, out Filter, errMsg, hint string) {
|
||||
qs, errMsg = normalizeQueries(query, queries)
|
||||
if errMsg != "" {
|
||||
return nil, Filter{}, errMsg, ""
|
||||
}
|
||||
out, hint = normalizeFilter(f)
|
||||
if hint != "" {
|
||||
return nil, Filter{}, "invalid search", hint
|
||||
}
|
||||
if out.ExactMatch {
|
||||
for _, q := range qs {
|
||||
if !hasQuotedPhrase(q) {
|
||||
return nil, Filter{}, "invalid search", "exact_match"
|
||||
}
|
||||
}
|
||||
}
|
||||
return qs, out, "", ""
|
||||
}
|
||||
|
||||
func normalizeQueries(query string, queries []string) ([]string, string) {
|
||||
q := strings.TrimSpace(query)
|
||||
qs := make([]string, 0, len(queries))
|
||||
for _, s := range queries {
|
||||
qs = append(qs, strings.TrimSpace(s))
|
||||
}
|
||||
hasQ := q != ""
|
||||
hasQS := len(qs) > 0
|
||||
if hasQ && hasQS {
|
||||
return nil, "invalid query"
|
||||
}
|
||||
if !hasQ && !hasQS {
|
||||
return nil, "invalid query"
|
||||
}
|
||||
if hasQ {
|
||||
qs = []string{q}
|
||||
}
|
||||
if len(qs) < 1 || len(qs) > 4 {
|
||||
return nil, "invalid query"
|
||||
}
|
||||
for _, s := range qs {
|
||||
if s == "" || utf8.RuneCountInString(s) > MaxQueryRunes {
|
||||
return nil, "invalid query"
|
||||
}
|
||||
}
|
||||
return qs, ""
|
||||
}
|
||||
|
||||
func normalizeFilter(f Filter) (Filter, string) {
|
||||
inc, hint := normalizeDomains(f.IncludeDomains, "include_domains")
|
||||
if hint != "" {
|
||||
return Filter{}, hint
|
||||
}
|
||||
exc, hint := normalizeDomains(f.ExcludeDomains, "exclude_domains")
|
||||
if hint != "" {
|
||||
return Filter{}, hint
|
||||
}
|
||||
mode := strings.TrimSpace(f.IncludeDomainsMode)
|
||||
switch mode {
|
||||
case "", "restrict", "prefer":
|
||||
default:
|
||||
return Filter{}, "include_domains_mode"
|
||||
}
|
||||
if mode != "" && len(inc) == 0 {
|
||||
return Filter{}, "include_domains_mode"
|
||||
}
|
||||
if len(inc) > 0 && mode == "" {
|
||||
mode = "restrict"
|
||||
}
|
||||
|
||||
tr := strings.TrimSpace(f.TimeRange)
|
||||
if tr != "" {
|
||||
long, ok := timeRangeLong[tr]
|
||||
if !ok {
|
||||
return Filter{}, "time_range"
|
||||
}
|
||||
tr = long
|
||||
}
|
||||
start := strings.TrimSpace(f.StartDate)
|
||||
end := strings.TrimSpace(f.EndDate)
|
||||
if start != "" {
|
||||
if !validDay(start) {
|
||||
return Filter{}, "start_date"
|
||||
}
|
||||
}
|
||||
if end != "" {
|
||||
if !validDay(end) {
|
||||
return Filter{}, "end_date"
|
||||
}
|
||||
}
|
||||
if (start != "" || end != "") && tr != "" {
|
||||
return Filter{}, "time_range"
|
||||
}
|
||||
if start != "" && end != "" {
|
||||
st, _ := time.Parse("2006-01-02", start)
|
||||
en, _ := time.Parse("2006-01-02", end)
|
||||
if en.Before(st) {
|
||||
return Filter{}, "end_date"
|
||||
}
|
||||
}
|
||||
|
||||
topic := strings.TrimSpace(f.Topic)
|
||||
switch topic {
|
||||
case "", "general", "news", "finance":
|
||||
default:
|
||||
return Filter{}, "topic"
|
||||
}
|
||||
|
||||
ms := f.MinScore
|
||||
if ms < 0 || ms > 1 {
|
||||
return Filter{}, "min_score"
|
||||
}
|
||||
|
||||
return Filter{
|
||||
IncludeDomains: inc,
|
||||
ExcludeDomains: exc,
|
||||
IncludeDomainsMode: mode,
|
||||
TimeRange: tr,
|
||||
StartDate: start,
|
||||
EndDate: end,
|
||||
Topic: topic,
|
||||
ExactMatch: f.ExactMatch,
|
||||
MinScore: ms,
|
||||
}, ""
|
||||
}
|
||||
|
||||
func normalizeDomains(in []string, hint string) ([]string, string) {
|
||||
if len(in) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
out := make([]string, 0, len(in))
|
||||
seen := map[string]bool{}
|
||||
for _, raw := range in {
|
||||
s := strings.TrimSpace(raw)
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
d, ok := normalizeDomain(s)
|
||||
if !ok {
|
||||
return nil, hint
|
||||
}
|
||||
if seen[d] {
|
||||
continue
|
||||
}
|
||||
seen[d] = true
|
||||
out = append(out, d)
|
||||
}
|
||||
if len(out) == 0 {
|
||||
return nil, ""
|
||||
}
|
||||
if len(out) > maxDomainList {
|
||||
return nil, hint
|
||||
}
|
||||
return out, ""
|
||||
}
|
||||
|
||||
func normalizeDomain(s string) (string, bool) {
|
||||
s = strings.ToLower(strings.TrimSpace(s))
|
||||
s = strings.TrimSuffix(s, ".")
|
||||
if s == "" || len(s) > maxHostLen {
|
||||
return "", false
|
||||
}
|
||||
if strings.ContainsAny(s, " :/@*?#[]") {
|
||||
return "", false
|
||||
}
|
||||
labels := strings.Split(s, ".")
|
||||
if len(labels) == 0 {
|
||||
return "", false
|
||||
}
|
||||
for _, lab := range labels {
|
||||
if !validLabel(lab) {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return s, true
|
||||
}
|
||||
|
||||
func validLabel(lab string) bool {
|
||||
if lab == "" || len(lab) > 63 {
|
||||
return false
|
||||
}
|
||||
if lab[0] == '-' || lab[len(lab)-1] == '-' {
|
||||
return false
|
||||
}
|
||||
for i := 0; i < len(lab); i++ {
|
||||
c := lab[i]
|
||||
if c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' {
|
||||
continue
|
||||
}
|
||||
return false
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func validDay(s string) bool {
|
||||
t, err := time.Parse("2006-01-02", s)
|
||||
return err == nil && t.Format("2006-01-02") == s
|
||||
}
|
||||
|
||||
// hasQuotedPhrase reports a balanced ASCII "…" pair with ≥1 rune inside and no unpaired quotes.
|
||||
func hasQuotedPhrase(q string) bool {
|
||||
in := false
|
||||
n := 0
|
||||
pairs := 0
|
||||
for _, r := range q {
|
||||
if r != '"' {
|
||||
if in {
|
||||
n++
|
||||
}
|
||||
continue
|
||||
}
|
||||
if !in {
|
||||
in = true
|
||||
n = 0
|
||||
continue
|
||||
}
|
||||
if n == 0 {
|
||||
return false
|
||||
}
|
||||
in = false
|
||||
pairs++
|
||||
}
|
||||
return !in && pairs >= 1
|
||||
}
|
||||
|
||||
func QueryRuneLen(qs []string) int {
|
||||
n := 0
|
||||
for _, q := range qs {
|
||||
n += utf8.RuneCountInString(q)
|
||||
}
|
||||
return n
|
||||
}
|
||||
@@ -0,0 +1,159 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/search/dial"
|
||||
)
|
||||
|
||||
func TestTavilyFiltersOutbound(t *testing.T) {
|
||||
var gotBody []byte
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotBody, _ = io.ReadAll(r.Body)
|
||||
_, _ = w.Write([]byte(`{"answer":"SECRET","results":[{"url":"https://ex.com/a","title":"A","content":"snip","score":0.9,"published_date":"2026-09-01"}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := Filter{
|
||||
IncludeDomains: []string{"ex.com"},
|
||||
IncludeDomainsMode: "restrict",
|
||||
TimeRange: "month",
|
||||
Topic: "news",
|
||||
ExactMatch: true,
|
||||
}
|
||||
out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), `docs "phrase"`, 5, f)
|
||||
if out.Warn != "" {
|
||||
t.Fatal(out.Warn)
|
||||
}
|
||||
var req map[string]any
|
||||
if err := json.Unmarshal(gotBody, &req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req["include_answer"] != false || req["auto_parameters"] != false || req["search_depth"] != "basic" {
|
||||
t.Fatalf("%+v", req)
|
||||
}
|
||||
if req["query"] != `docs "phrase"` {
|
||||
t.Fatalf("query rewritten: %v", req["query"])
|
||||
}
|
||||
if req["exact_match"] != true {
|
||||
t.Fatalf("exact_match %v", req["exact_match"])
|
||||
}
|
||||
if _, ok := req["start_date"]; ok {
|
||||
t.Fatal("time_range with dates")
|
||||
}
|
||||
if req["time_range"] != "month" || req["topic"] != "news" {
|
||||
t.Fatalf("%+v", req)
|
||||
}
|
||||
if req["include_published_date"] != true {
|
||||
t.Fatal("want include_published_date")
|
||||
}
|
||||
if _, ok := req["include_domains_mode"]; !ok {
|
||||
t.Fatal("mode missing")
|
||||
}
|
||||
if strings.Contains(string(gotBody), "extract") || strings.Contains(string(gotBody), "/crawl") {
|
||||
t.Fatal(string(gotBody))
|
||||
}
|
||||
if len(out.Hits) != 1 || out.Hits[0].PublishedDate != "2026-09-01" {
|
||||
t.Fatalf("%+v", out.Hits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestTavilyModeOmittedWithoutInclude(t *testing.T) {
|
||||
var gotBody []byte
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotBody, _ = io.ReadAll(r.Body)
|
||||
_, _ = w.Write([]byte(`{"results":[{"url":"https://ex.com/a","title":"A","content":"s","score":0.2}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{IncludeDomainsMode: "prefer"})
|
||||
if out.Warn != "" {
|
||||
t.Fatal(out.Warn)
|
||||
}
|
||||
var req map[string]any
|
||||
_ = json.Unmarshal(gotBody, &req)
|
||||
if _, ok := req["include_domains_mode"]; ok {
|
||||
t.Fatalf("mode without include: %+v", req)
|
||||
}
|
||||
if out.Hits[0].PublishedDate != "" {
|
||||
t.Fatal("missing date must omit")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSearxNGSiteAndParams(t *testing.T) {
|
||||
var gotQ, gotTR, gotCat string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotQ = r.URL.Query().Get("q")
|
||||
gotTR = r.URL.Query().Get("time_range")
|
||||
gotCat = r.URL.Query().Get("categories")
|
||||
if strings.Contains(gotQ, "!news") || strings.Contains(gotQ, "!month") {
|
||||
t.Errorf("bang in q: %s", gotQ)
|
||||
}
|
||||
_, _ = w.Write([]byte(`{"results":[{"url":"https://a.example/","title":"a","content":"aa","score":1,"publishedDate":"Mon, 01 Sep 2026 00:00:00 GMT"}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := Filter{IncludeDomains: []string{"a.example"}, TimeRange: "month", Topic: "news", ExactMatch: true}
|
||||
out := (&SearxNG{Client: c}).Search(context.Background(), `"phrase" here`, 8, f)
|
||||
if !strings.HasPrefix(gotQ, "site:a.example ") {
|
||||
t.Fatalf("q %q", gotQ)
|
||||
}
|
||||
if !strings.Contains(gotQ, `"phrase" here`) || strings.Count(gotQ, `"phrase"`) != 1 {
|
||||
t.Fatalf("wrapped q %q", gotQ)
|
||||
}
|
||||
if gotTR != "month" || gotCat != "news" {
|
||||
t.Fatalf("params tr=%q cat=%q", gotTR, gotCat)
|
||||
}
|
||||
if out.Hits[0].PublishedDate == "" {
|
||||
t.Fatal("publishedDate")
|
||||
}
|
||||
}
|
||||
|
||||
func TestBraveNotesAndUnchangedURL(t *testing.T) {
|
||||
var gotURL string
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
gotURL = r.URL.String()
|
||||
_, _ = w.Write([]byte(`{"sources":[{"url":"https://off.example/x","title":"S","snippet":"hello"}]}`))
|
||||
}))
|
||||
defer ts.Close()
|
||||
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
f := Filter{TimeRange: "month", Topic: "news", ExactMatch: true, MinScore: 0.5, IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"}
|
||||
out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), `"phrase"`, 5, f)
|
||||
if !strings.Contains(gotURL, "q=") {
|
||||
t.Fatalf("url %s", gotURL)
|
||||
}
|
||||
if strings.Contains(gotURL, "time_range") || strings.Contains(gotURL, "site:") {
|
||||
t.Fatalf("brave grew params: %s", gotURL)
|
||||
}
|
||||
note := strings.Join(out.Notes, " ")
|
||||
for _, want := range []string{
|
||||
"brave_llm: time_range not applied",
|
||||
"brave_llm: topic not applied",
|
||||
"brave_llm: exact_match not applied",
|
||||
"brave_llm: include_domains_mode=prefer not applied",
|
||||
"brave_llm: min_score not applied",
|
||||
} {
|
||||
if !strings.Contains(note, want) {
|
||||
t.Fatalf("missing %q in %v", want, out.Notes)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,269 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/dcache"
|
||||
)
|
||||
|
||||
type qSrc struct {
|
||||
name string
|
||||
byQuery map[string][]Hit
|
||||
notes []string
|
||||
credits int
|
||||
}
|
||||
|
||||
func (q qSrc) Name() string { return q.name }
|
||||
func (q qSrc) Search(_ context.Context, query string, _ int, _ Filter) SourceOut {
|
||||
return SourceOut{Hits: q.byQuery[query], Notes: q.notes, Credits: q.credits}
|
||||
}
|
||||
|
||||
func TestNotesDoNotFailSource(t *testing.T) {
|
||||
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
e := &Engine{
|
||||
Sources: []Source{fakeSrc{
|
||||
name: NameSearxNG,
|
||||
hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}},
|
||||
notes: []string{"searxng: min_score not applied"},
|
||||
}},
|
||||
DefaultK: 8, Cache: db, RerankOn: false,
|
||||
}
|
||||
r1 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
|
||||
if len(r1.Hits) != 1 {
|
||||
t.Fatalf("hits %+v", r1.Hits)
|
||||
}
|
||||
if len(r1.SourcesOK) != 1 || r1.SourcesOK[0] != NameSearxNG {
|
||||
t.Fatalf("ok %v fail %v", r1.SourcesOK, r1.SourcesFailed)
|
||||
}
|
||||
if r1.Cached {
|
||||
t.Fatal("first miss")
|
||||
}
|
||||
joined := strings.Join(r1.Warnings, " ")
|
||||
if !strings.Contains(joined, "min_score not applied") {
|
||||
t.Fatalf("warnings %v", r1.Warnings)
|
||||
}
|
||||
r2 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
|
||||
if !r2.Cached {
|
||||
t.Fatal("notes must still cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestWarnStillDisablesCache(t *testing.T) {
|
||||
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
e := &Engine{
|
||||
Sources: []Source{
|
||||
fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}},
|
||||
fakeSrc{name: NameBraveLLM, warn: "brave_llm: 500"},
|
||||
},
|
||||
DefaultK: 8, Cache: db, RerankOn: false,
|
||||
}
|
||||
e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet)
|
||||
r := e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet)
|
||||
if r.Cached {
|
||||
t.Fatal("warn cached")
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinScoreTavilyOnly(t *testing.T) {
|
||||
e := &Engine{
|
||||
Sources: []Source{
|
||||
fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://s.example/a", Score: 0.01, Source: NameSearxNG}}, notes: []string{"searxng: min_score not applied"}},
|
||||
fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://b.example/a", Source: NameBraveLLM}}, notes: []string{"brave_llm: min_score not applied"}},
|
||||
fakeSrc{name: NameTavily, hits: []Hit{
|
||||
{URL: "https://t.example/low", Score: 0.4, Source: NameTavily},
|
||||
{URL: "https://t.example/high", Score: 0.9, Source: NameTavily},
|
||||
}},
|
||||
},
|
||||
DefaultK: 8, RerankOn: false,
|
||||
}
|
||||
r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
|
||||
var urls []string
|
||||
for _, h := range r.Hits {
|
||||
urls = append(urls, h.URL)
|
||||
if h.URL == "https://t.example/low" {
|
||||
t.Fatal("tavily 0.4 kept")
|
||||
}
|
||||
}
|
||||
joined := strings.Join(urls, " ")
|
||||
if !strings.Contains(joined, "s.example") || !strings.Contains(joined, "b.example") || !strings.Contains(joined, "t.example/high") {
|
||||
t.Fatalf("hits %v", urls)
|
||||
}
|
||||
w := strings.Join(r.Warnings, " ")
|
||||
if !strings.Contains(w, "searxng: min_score not applied") || !strings.Contains(w, "brave_llm: min_score not applied") {
|
||||
t.Fatalf("warnings %v", r.Warnings)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDomainPostFilter(t *testing.T) {
|
||||
hits := []Hit{
|
||||
{URL: "https://openai.com/blog", Source: NameBraveLLM},
|
||||
{URL: "https://sub.openai.com/x", Source: NameSearxNG},
|
||||
{URL: "https://pinterest.com/pin", Source: NameBraveLLM},
|
||||
{URL: "https://example.com/a", Source: NameSearxNG},
|
||||
}
|
||||
f := Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}
|
||||
got := filterDomains(hits, f)
|
||||
if len(got) != 2 {
|
||||
t.Fatalf("%+v", got)
|
||||
}
|
||||
f.IncludeDomainsMode = "prefer"
|
||||
got = filterDomains(hits, f)
|
||||
if len(got) != 4 {
|
||||
t.Fatalf("prefer %+v", got)
|
||||
}
|
||||
got = filterDomains(hits, Filter{ExcludeDomains: []string{"pinterest.com"}})
|
||||
if len(got) != 3 {
|
||||
t.Fatalf("exclude %+v", got)
|
||||
}
|
||||
e := &Engine{
|
||||
Sources: []Source{fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://off.example/x", Source: NameBraveLLM}}}},
|
||||
DefaultK: 8, RerankOn: false,
|
||||
}
|
||||
r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"})
|
||||
if len(r.Hits) != 0 {
|
||||
t.Fatalf("brave off-domain kept %+v", r.Hits)
|
||||
}
|
||||
r = e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"})
|
||||
if len(r.Hits) != 1 {
|
||||
t.Fatalf("prefer dropped brave %+v", r.Hits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeDomainOR(t *testing.T) {
|
||||
_, _, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"a.com OR b.com"}})
|
||||
if msg != "invalid search" || hint != "include_domains" {
|
||||
t.Fatalf("%q %q", msg, hint)
|
||||
}
|
||||
_, out, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"oracle.com"}})
|
||||
if msg != "" || hint != "" || len(out.IncludeDomains) != 1 || out.IncludeDomains[0] != "oracle.com" {
|
||||
t.Fatalf("%q %q %+v", msg, hint, out)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMultiQueryRRF(t *testing.T) {
|
||||
q1 := make([]Hit, 0, 3)
|
||||
for _, u := range []string{"https://a.example/1", "https://a.example/2", "https://a.example/3"} {
|
||||
q1 = append(q1, Hit{URL: u, Source: NameSearxNG})
|
||||
}
|
||||
e := &Engine{
|
||||
Sources: []Source{qSrc{
|
||||
name: NameSearxNG,
|
||||
byQuery: map[string][]Hit{
|
||||
"one": q1,
|
||||
"two": {{URL: "https://b.example/only", Source: NameSearxNG}},
|
||||
},
|
||||
}},
|
||||
DefaultK: 3, RerankOn: true,
|
||||
}
|
||||
r := e.RunFilter(context.Background(), []string{"one", "two"}, 3, true, "titan", dcache.ClassFleet, Filter{})
|
||||
found := false
|
||||
for _, h := range r.Hits {
|
||||
if h.URL == "https://b.example/only" {
|
||||
found = true
|
||||
}
|
||||
}
|
||||
if !found {
|
||||
t.Fatalf("q2 hit missing: %+v", r.Hits)
|
||||
}
|
||||
}
|
||||
|
||||
func TestCapMidBatchNoCache(t *testing.T) {
|
||||
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
e := &Engine{
|
||||
Sources: []Source{qSrc{
|
||||
name: NameTavily,
|
||||
byQuery: map[string][]Hit{
|
||||
"one": {{URL: "https://t.example/1", Source: NameTavily}},
|
||||
"two": {{URL: "https://t.example/2", Source: NameTavily}},
|
||||
},
|
||||
credits: 1,
|
||||
}},
|
||||
DefaultK: 8, Cache: db, TavilyCap: 1, RerankOn: false,
|
||||
}
|
||||
r := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{})
|
||||
n := 0
|
||||
for _, w := range r.Warnings {
|
||||
if strings.Contains(w, "daily cap") {
|
||||
n++
|
||||
}
|
||||
}
|
||||
if n != 1 {
|
||||
t.Fatalf("warnings %v", r.Warnings)
|
||||
}
|
||||
r2 := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{})
|
||||
if r2.Cached {
|
||||
t.Fatal("mid-batch cap must not cache")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCacheKeyFilters(t *testing.T) {
|
||||
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer db.Close()
|
||||
now := time.Date(2026, 9, 29, 10, 0, 0, 0, time.UTC)
|
||||
src := fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}}
|
||||
e := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
|
||||
base := Filter{}
|
||||
e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, base)
|
||||
if e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached {
|
||||
t.Fatal("different include_domains must miss")
|
||||
}
|
||||
e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"})
|
||||
if !e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached {
|
||||
t.Fatal("same filters must hit")
|
||||
}
|
||||
e2 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
|
||||
e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{})
|
||||
if !e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{Topic: "general"}).Cached {
|
||||
t.Fatal("omitted topic and general must share a key")
|
||||
}
|
||||
_, day, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "d"})
|
||||
_, long, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "day"})
|
||||
if day.TimeRange != long.TimeRange || day.TimeRange != "day" {
|
||||
t.Fatalf("%q %q", day.TimeRange, long.TimeRange)
|
||||
}
|
||||
e3 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
|
||||
e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, day)
|
||||
if !e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, long).Cached {
|
||||
t.Fatal("d and day must share a key")
|
||||
}
|
||||
}
|
||||
|
||||
func TestQuotedPhrase(t *testing.T) {
|
||||
if !hasQuotedPhrase(`docs "phrase" x`) || hasQuotedPhrase(`docs phrase`) || hasQuotedPhrase(`"unclosed`) || hasQuotedPhrase(`""`) {
|
||||
t.Fatal("quoted phrase")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeQueries(t *testing.T) {
|
||||
_, _, msg, _ := NormalizeSearch("a", []string{"b"}, Filter{})
|
||||
if msg != "invalid query" {
|
||||
t.Fatalf("both: %q", msg)
|
||||
}
|
||||
qs, _, msg, _ := NormalizeSearch("", []string{"a", "b", "c", "d", "e"}, Filter{})
|
||||
if msg != "invalid query" || qs != nil {
|
||||
t.Fatalf("five: %q", msg)
|
||||
}
|
||||
_, _, msg, hint := NormalizeSearch(`no quotes`, nil, Filter{ExactMatch: true})
|
||||
if msg != "invalid search" || hint != "exact_match" {
|
||||
t.Fatalf("%q %q", msg, hint)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,223 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"sync"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/dcache"
|
||||
)
|
||||
|
||||
type srcOut struct {
|
||||
name string
|
||||
out SourceOut
|
||||
skip bool
|
||||
}
|
||||
|
||||
type queryFan struct {
|
||||
hits []Hit
|
||||
warns []string
|
||||
notes []string
|
||||
okNames []string
|
||||
failNames []string
|
||||
att []string
|
||||
degraded bool
|
||||
tavSkip bool
|
||||
tavUsed bool
|
||||
}
|
||||
|
||||
func (e *Engine) RunFilter(ctx context.Context, queries []string, k int, doRerank bool, caller, class string, f Filter) RunResult {
|
||||
start := e.now()
|
||||
k = ClampK(k, e.DefaultK)
|
||||
if e.DefaultK <= 0 {
|
||||
k = ClampK(k, DefaultK)
|
||||
}
|
||||
if class == "" {
|
||||
class = dcache.ClassFleet
|
||||
}
|
||||
if len(queries) == 0 {
|
||||
queries = []string{""}
|
||||
}
|
||||
|
||||
attempted := e.attemptedNames(class)
|
||||
key := e.searchKey(queries, attempted, k, class, f)
|
||||
if b, ok := e.Cache.GetSearch(key); ok {
|
||||
var hits []Hit
|
||||
if json.Unmarshal(b, &hits) == nil {
|
||||
e.mu.Lock()
|
||||
e.CacheHit++
|
||||
e.mu.Unlock()
|
||||
return RunResult{Hits: clampHits(hits, k), Cached: true, Attempted: attempted, SearchMS: e.now().Sub(start).Milliseconds()}
|
||||
}
|
||||
}
|
||||
e.mu.Lock()
|
||||
e.CacheMiss++
|
||||
e.mu.Unlock()
|
||||
|
||||
var lists [][]Hit
|
||||
var warns, notes []string
|
||||
okSeen, failSeen := map[string]bool{}, map[string]bool{}
|
||||
var okNames, failNames []string
|
||||
degraded := false
|
||||
capMid := false
|
||||
sawTavily := false
|
||||
var lastAtt []string
|
||||
|
||||
for _, q := range queries {
|
||||
one := e.fanInQuery(ctx, q, k, doRerank, caller, class, f)
|
||||
if one.tavUsed {
|
||||
sawTavily = true
|
||||
}
|
||||
if one.tavSkip && sawTavily {
|
||||
capMid = true
|
||||
}
|
||||
if one.degraded {
|
||||
degraded = true
|
||||
}
|
||||
warns = append(warns, one.warns...)
|
||||
notes = append(notes, one.notes...)
|
||||
for _, n := range one.okNames {
|
||||
if !okSeen[n] {
|
||||
okSeen[n] = true
|
||||
okNames = append(okNames, n)
|
||||
}
|
||||
}
|
||||
for _, n := range one.failNames {
|
||||
if !failSeen[n] {
|
||||
failSeen[n] = true
|
||||
failNames = append(failNames, n)
|
||||
}
|
||||
}
|
||||
if len(one.att) > 0 {
|
||||
lastAtt = one.att
|
||||
}
|
||||
lists = append(lists, one.hits)
|
||||
}
|
||||
|
||||
var hits []Hit
|
||||
if len(lists) <= 1 {
|
||||
if len(lists) == 1 {
|
||||
hits = lists[0]
|
||||
}
|
||||
} else {
|
||||
hits = rrfMerge(lists)
|
||||
hits = Dedupe(hits)
|
||||
}
|
||||
hits = clampHits(hits, k)
|
||||
|
||||
warns = append(warns, notes...)
|
||||
warns = dedupeStrings(warns)
|
||||
|
||||
att := lastAtt
|
||||
if len(att) == 0 {
|
||||
att = attempted
|
||||
}
|
||||
if !degraded && !capMid && e.Cache != nil && len(att) > 0 {
|
||||
payload, _ := json.Marshal(hits)
|
||||
ck := e.searchKey(queries, att, k, class, f)
|
||||
e.Cache.PutSearch(ck, payload, e.ttl())
|
||||
}
|
||||
|
||||
return RunResult{
|
||||
Hits: hits,
|
||||
Warnings: warns,
|
||||
SourcesOK: okNames,
|
||||
SourcesFailed: failNames,
|
||||
SearchMS: e.now().Sub(start).Milliseconds(),
|
||||
Attempted: att,
|
||||
}
|
||||
}
|
||||
|
||||
func (e *Engine) searchKey(queries []string, sources []string, k int, class string, f Filter) string {
|
||||
return dcache.EncodeSearchKey(dcache.SearchKeyInput{
|
||||
Queries: queries,
|
||||
Sources: sources,
|
||||
K: k,
|
||||
Hour: e.now(),
|
||||
CallerClass: class,
|
||||
IncludeDomains: f.IncludeDomains,
|
||||
ExcludeDomains: f.ExcludeDomains,
|
||||
IncludeDomainsMode: f.IncludeDomainsMode,
|
||||
TimeRange: f.TimeRange,
|
||||
StartDate: f.StartDate,
|
||||
EndDate: f.EndDate,
|
||||
Topic: f.Topic,
|
||||
ExactMatch: f.ExactMatch,
|
||||
MinScore: f.MinScore,
|
||||
})
|
||||
}
|
||||
|
||||
func (e *Engine) fanInQuery(ctx context.Context, query string, k int, doRerank bool, caller, class string, f Filter) queryFan {
|
||||
outs := make([]srcOut, len(e.Sources))
|
||||
var wg sync.WaitGroup
|
||||
tavSkip := false
|
||||
for i, src := range e.Sources {
|
||||
name := src.Name()
|
||||
if name == NameTavily && e.tavilyCapped(class) {
|
||||
outs[i] = srcOut{name: name, out: SourceOut{Warn: "tavily: daily cap"}, skip: true}
|
||||
e.addCapped(NameTavily)
|
||||
tavSkip = true
|
||||
continue
|
||||
}
|
||||
wg.Add(1)
|
||||
go func(i int, src Source, name string) {
|
||||
defer wg.Done()
|
||||
sctx, cancel := context.WithTimeout(ctx, e.timeout())
|
||||
defer cancel()
|
||||
outs[i] = srcOut{name: name, out: src.Search(sctx, query, k, f)}
|
||||
}(i, src, name)
|
||||
}
|
||||
wg.Wait()
|
||||
|
||||
var lists []namedHits
|
||||
var warns, notes []string
|
||||
var okNames, failNames, att []string
|
||||
degraded := false
|
||||
tavUsed := false
|
||||
for _, o := range outs {
|
||||
if o.skip {
|
||||
if o.out.Warn != "" {
|
||||
warns = append(warns, o.out.Warn)
|
||||
}
|
||||
continue
|
||||
}
|
||||
att = append(att, o.name)
|
||||
e.addQuery(caller, o.name)
|
||||
if o.name == NameTavily {
|
||||
tavUsed = true
|
||||
}
|
||||
if o.out.Warn != "" {
|
||||
warns = append(warns, o.out.Warn)
|
||||
failNames = append(failNames, o.name)
|
||||
degraded = true
|
||||
e.addErr(o.name, o.out.Warn)
|
||||
continue
|
||||
}
|
||||
okNames = append(okNames, o.name)
|
||||
notes = append(notes, o.out.Notes...)
|
||||
hits := o.out.Hits
|
||||
if o.name == NameTavily && f.MinScore > 0 {
|
||||
hits = filterMinScore(hits, f.MinScore)
|
||||
}
|
||||
lists = append(lists, namedHits{name: o.name, hits: hits})
|
||||
if o.out.Credits > 0 {
|
||||
e.addCredits(caller, o.name, o.out.Credits)
|
||||
if o.name == NameTavily && e.Cache != nil {
|
||||
e.Cache.AddSearchCredits(tavilyCapSource(class), e.now().UTC().Format("2006-01-02"), o.out.Credits)
|
||||
}
|
||||
}
|
||||
}
|
||||
merged := mergeSearch(e, lists, doRerank)
|
||||
merged = filterDomains(merged, f)
|
||||
return queryFan{
|
||||
hits: merged,
|
||||
warns: warns,
|
||||
notes: notes,
|
||||
okNames: okNames,
|
||||
failNames: failNames,
|
||||
att: att,
|
||||
degraded: degraded,
|
||||
tavSkip: tavSkip,
|
||||
tavUsed: tavUsed,
|
||||
}
|
||||
}
|
||||
@@ -105,6 +105,7 @@ func rrfMerge(lists [][]Hit) []Hit {
|
||||
sc := 1.0 / float64(rrfK+r+1)
|
||||
if a, ok := seen[c]; ok {
|
||||
a.s += sc
|
||||
keepDate := firstDate(a.h.PublishedDate, h.PublishedDate)
|
||||
if len(h.Snippet) > len(a.h.Snippet) {
|
||||
src := a.h.Source
|
||||
keep := a.h.URL
|
||||
@@ -112,6 +113,7 @@ func rrfMerge(lists [][]Hit) []Hit {
|
||||
a.h.Source = src
|
||||
a.h.URL = keep
|
||||
}
|
||||
a.h.PublishedDate = keepDate
|
||||
continue
|
||||
}
|
||||
cp := h
|
||||
|
||||
+18
-10
@@ -59,7 +59,7 @@ func (s *SearxNG) Name() string { return NameSearxNG }
|
||||
func (s *SearxNG) Ready() bool { return s != nil && s.Client.Ready() }
|
||||
func (s *BraveLLM) Ready() bool { return s != nil && s.Client.Ready() }
|
||||
|
||||
func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
func (s *SearxNG) Search(ctx context.Context, query string, k int, f Filter) SourceOut {
|
||||
if s == nil || s.Client == nil {
|
||||
return SourceOut{Warn: NameSearxNG + ": error"}
|
||||
}
|
||||
@@ -71,10 +71,17 @@ func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
if err != nil {
|
||||
return SourceOut{Warn: warnErr(NameSearxNG, err)}
|
||||
}
|
||||
qtext, notes := searxQueryAndNotes(query, f)
|
||||
q := u.Query()
|
||||
q.Set("q", query)
|
||||
q.Set("q", qtext)
|
||||
q.Set("format", "json")
|
||||
q.Set("pageno", "1")
|
||||
if f.TimeRange != "" {
|
||||
q.Set("time_range", f.TimeRange)
|
||||
}
|
||||
if f.Topic == "news" {
|
||||
q.Set("categories", "news")
|
||||
}
|
||||
u.RawQuery = q.Encode()
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
@@ -92,10 +99,11 @@ func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
}
|
||||
var env struct {
|
||||
Results []struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Score float64 `json:"score"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Score float64 `json:"score"`
|
||||
PublishedDate string `json:"publishedDate"`
|
||||
} `json:"results"`
|
||||
}
|
||||
if err := readJSON(resp, &env); err != nil {
|
||||
@@ -107,9 +115,9 @@ func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
if r.URL == "" {
|
||||
continue
|
||||
}
|
||||
hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameSearxNG})
|
||||
hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameSearxNG, PublishedDate: r.PublishedDate})
|
||||
}
|
||||
return SourceOut{Hits: hits}
|
||||
return SourceOut{Hits: hits, Notes: notes}
|
||||
}
|
||||
|
||||
// BraveLLM is GET {url}?q= with X-Subscription-Token.
|
||||
@@ -120,7 +128,7 @@ type BraveLLM struct {
|
||||
|
||||
func (s *BraveLLM) Name() string { return NameBraveLLM }
|
||||
|
||||
func (s *BraveLLM) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
func (s *BraveLLM) Search(ctx context.Context, query string, k int, f Filter) SourceOut {
|
||||
if s == nil || s.Client == nil {
|
||||
return SourceOut{Warn: NameBraveLLM + ": error"}
|
||||
}
|
||||
@@ -159,7 +167,7 @@ func (s *BraveLLM) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
if len(hits) >= 0 {
|
||||
credits = 1
|
||||
}
|
||||
return SourceOut{Hits: hits, Credits: credits}
|
||||
return SourceOut{Hits: hits, Credits: credits, Notes: braveNotes(f)}
|
||||
}
|
||||
|
||||
func mapBrave(raw []byte) []Hit {
|
||||
|
||||
@@ -31,17 +31,20 @@ func TestTavilyOutboundAndDropsAnswer(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := (&Tavily{Client: c, Key: "test-key"}).Search(context.Background(), "q", 5)
|
||||
out := (&Tavily{Client: c, Key: "test-key"}).Search(context.Background(), "q", 5, Filter{})
|
||||
if out.Warn != "" {
|
||||
t.Fatal(out.Warn)
|
||||
}
|
||||
var req tavilyReq
|
||||
var req map[string]any
|
||||
if err := json.Unmarshal(gotBody, &req); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if req.IncludeAnswer || req.IncludeRawContent || req.IncludeImages || req.SearchDepth != "basic" {
|
||||
if req["include_answer"] != false || req["include_raw_content"] != false || req["include_images"] != false || req["search_depth"] != "basic" {
|
||||
t.Fatalf("%+v", req)
|
||||
}
|
||||
if req["auto_parameters"] != false {
|
||||
t.Fatalf("auto_parameters %+v", req["auto_parameters"])
|
||||
}
|
||||
if strings.Contains(string(gotBody), "extract") || strings.Contains(string(gotBody), "crawl") {
|
||||
t.Fatal(string(gotBody))
|
||||
}
|
||||
@@ -70,7 +73,7 @@ func TestSearxNGMapsAndSorts(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := (&SearxNG{Client: c}).Search(context.Background(), "q", 8)
|
||||
out := (&SearxNG{Client: c}).Search(context.Background(), "q", 8, Filter{})
|
||||
if out.Warn != "" || len(out.Hits) != 2 || out.Hits[0].Title != "a" {
|
||||
t.Fatalf("%+v", out)
|
||||
}
|
||||
@@ -88,7 +91,7 @@ func TestBraveMapsSources(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
|
||||
out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{})
|
||||
if len(out.Hits) != 1 || out.Hits[0].URL != "https://s.example/x" {
|
||||
t.Fatalf("%+v", out)
|
||||
}
|
||||
@@ -110,7 +113,7 @@ func TestBraveMapsGroundingGeneric(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
|
||||
out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{})
|
||||
if len(out.Hits) != 3 {
|
||||
t.Fatalf("hits %d %+v", len(out.Hits), out.Hits)
|
||||
}
|
||||
@@ -139,7 +142,7 @@ func TestTavilyBadJSONIsError(t *testing.T) {
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
|
||||
out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{})
|
||||
if out.Warn != "tavily: error" {
|
||||
t.Fatalf("warn %q", out.Warn)
|
||||
}
|
||||
|
||||
+54
-23
@@ -19,16 +19,7 @@ type Tavily struct {
|
||||
func (s *Tavily) Name() string { return NameTavily }
|
||||
func (s *Tavily) Ready() bool { return s != nil && s.Client.Ready() }
|
||||
|
||||
type tavilyReq struct {
|
||||
Query string `json:"query"`
|
||||
MaxResults int `json:"max_results"`
|
||||
IncludeAnswer bool `json:"include_answer"`
|
||||
IncludeRawContent bool `json:"include_raw_content"`
|
||||
IncludeImages bool `json:"include_images"`
|
||||
SearchDepth string `json:"search_depth"`
|
||||
}
|
||||
|
||||
func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
func (s *Tavily) Search(ctx context.Context, query string, k int, f Filter) SourceOut {
|
||||
if s == nil || s.Client == nil {
|
||||
return SourceOut{Warn: NameTavily + ": error"}
|
||||
}
|
||||
@@ -41,14 +32,7 @@ func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
if k > 20 {
|
||||
k = 20
|
||||
}
|
||||
payload, _ := json.Marshal(tavilyReq{
|
||||
Query: query,
|
||||
MaxResults: k,
|
||||
IncludeAnswer: false,
|
||||
IncludeRawContent: false,
|
||||
IncludeImages: false,
|
||||
SearchDepth: "basic",
|
||||
})
|
||||
payload, _ := json.Marshal(tavilyBody(query, k, f))
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.Client.URL.String(), bytes.NewReader(payload))
|
||||
if err != nil {
|
||||
return SourceOut{Warn: warnErr(NameTavily, err)}
|
||||
@@ -76,14 +60,58 @@ func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
|
||||
return SourceOut{Hits: hits, Credits: credits}
|
||||
}
|
||||
|
||||
func tavilyBody(query string, k int, f Filter) map[string]any {
|
||||
body := map[string]any{
|
||||
"query": query,
|
||||
"max_results": k,
|
||||
"include_answer": false,
|
||||
"include_raw_content": false,
|
||||
"include_images": false,
|
||||
"search_depth": "basic",
|
||||
"auto_parameters": false,
|
||||
}
|
||||
if len(f.IncludeDomains) > 0 {
|
||||
body["include_domains"] = f.IncludeDomains
|
||||
mode := f.IncludeDomainsMode
|
||||
if mode == "" {
|
||||
mode = "restrict"
|
||||
}
|
||||
body["include_domains_mode"] = mode
|
||||
}
|
||||
if len(f.ExcludeDomains) > 0 {
|
||||
body["exclude_domains"] = f.ExcludeDomains
|
||||
}
|
||||
if f.TimeRange != "" {
|
||||
body["time_range"] = f.TimeRange
|
||||
} else {
|
||||
if f.StartDate != "" {
|
||||
body["start_date"] = f.StartDate
|
||||
}
|
||||
if f.EndDate != "" {
|
||||
body["end_date"] = f.EndDate
|
||||
}
|
||||
}
|
||||
if f.Topic != "" && f.Topic != "general" {
|
||||
body["topic"] = f.Topic
|
||||
}
|
||||
if f.ExactMatch {
|
||||
body["exact_match"] = true
|
||||
}
|
||||
if f.Topic == "news" || f.TimeRange != "" || f.StartDate != "" || f.EndDate != "" {
|
||||
body["include_published_date"] = true
|
||||
}
|
||||
return body
|
||||
}
|
||||
|
||||
func mapTavily(raw []byte) ([]Hit, int, error) {
|
||||
var env struct {
|
||||
Answer string `json:"answer"`
|
||||
Results []struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Score float64 `json:"score"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Content string `json:"content"`
|
||||
Score float64 `json:"score"`
|
||||
PublishedDate string `json:"published_date"`
|
||||
} `json:"results"`
|
||||
Usage struct {
|
||||
Credits int `json:"credits"`
|
||||
@@ -98,7 +126,10 @@ func mapTavily(raw []byte) ([]Hit, int, error) {
|
||||
if r.URL == "" {
|
||||
continue
|
||||
}
|
||||
hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameTavily})
|
||||
hits = append(hits, Hit{
|
||||
URL: r.URL, Title: r.Title, Snippet: r.Content,
|
||||
Score: r.Score, Source: NameTavily, PublishedDate: r.PublishedDate,
|
||||
})
|
||||
}
|
||||
credits := env.Usage.Credits
|
||||
if credits <= 0 {
|
||||
|
||||
@@ -25,22 +25,37 @@ var sourceRank = map[string]int{
|
||||
|
||||
// Hit is one search result after mapping. No page body, no vendor answers.
|
||||
type Hit struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Score float64 `json:"score"`
|
||||
Source string `json:"source"`
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Snippet string `json:"snippet"`
|
||||
Score float64 `json:"score"`
|
||||
Source string `json:"source"`
|
||||
PublishedDate string `json:"published_date,omitempty"`
|
||||
}
|
||||
|
||||
// Filter is native search knobs. The zero value is P3 behaviour.
|
||||
type Filter struct {
|
||||
IncludeDomains []string
|
||||
ExcludeDomains []string
|
||||
IncludeDomainsMode string // restrict | prefer; empty means restrict when include is set
|
||||
TimeRange string // day|week|month|year
|
||||
StartDate string
|
||||
EndDate string
|
||||
Topic string // general|news|finance; empty means general
|
||||
ExactMatch bool
|
||||
MinScore float64
|
||||
}
|
||||
|
||||
// Source is one upstream. Config-fixed endpoint; never a request URL.
|
||||
type Source interface {
|
||||
Name() string
|
||||
Search(ctx context.Context, query string, k int) SourceOut
|
||||
Search(ctx context.Context, query string, k int, f Filter) SourceOut
|
||||
}
|
||||
|
||||
type SourceOut struct {
|
||||
Hits []Hit
|
||||
Warn string
|
||||
Warn string // failure only (timeout, HTTP error, pin)
|
||||
Notes []string // honesty: filters not applied, recall loss
|
||||
Credits int
|
||||
}
|
||||
|
||||
@@ -80,6 +95,7 @@ func Dedupe(in []Hit) []Hit {
|
||||
continue
|
||||
}
|
||||
if s, ok := seen[canon]; ok {
|
||||
keepDate := firstDate(s.h.PublishedDate, h.PublishedDate)
|
||||
if len(h.Snippet) > len(s.h.Snippet) {
|
||||
src := s.h.Source
|
||||
keep := s.h.URL
|
||||
@@ -91,6 +107,7 @@ func Dedupe(in []Hit) []Hit {
|
||||
s.h.Score = sc
|
||||
}
|
||||
}
|
||||
s.h.PublishedDate = keepDate
|
||||
continue
|
||||
}
|
||||
cp := h
|
||||
@@ -104,6 +121,13 @@ func Dedupe(in []Hit) []Hit {
|
||||
return out
|
||||
}
|
||||
|
||||
func firstDate(a, b string) string {
|
||||
if a != "" {
|
||||
return a
|
||||
}
|
||||
return b
|
||||
}
|
||||
|
||||
func sortBySourceOrder(hits []Hit) {
|
||||
sort.SliceStable(hits, func(i, j int) bool {
|
||||
ri, rj := sourceRank[hits[i].Source], sourceRank[hits[j].Source]
|
||||
|
||||
Reference in New Issue
Block a user