package search import ( "context" "path/filepath" "strings" "testing" "time" "git.wntrmute.dev/kyle/sift/internal/dcache" ) type qSrc struct { name string byQuery map[string][]Hit notes []string credits int } func (q qSrc) Name() string { return q.name } func (q qSrc) Search(_ context.Context, query string, _ int, _ Filter) SourceOut { return SourceOut{Hits: q.byQuery[query], Notes: q.notes, Credits: q.credits} } func TestNotesDoNotFailSource(t *testing.T) { db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil) if err != nil { t.Fatal(err) } defer db.Close() e := &Engine{ Sources: []Source{fakeSrc{ name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}, notes: []string{"searxng: min_score not applied"}, }}, DefaultK: 8, Cache: db, RerankOn: false, } r1 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5}) if len(r1.Hits) != 1 { t.Fatalf("hits %+v", r1.Hits) } if len(r1.SourcesOK) != 1 || r1.SourcesOK[0] != NameSearxNG { t.Fatalf("ok %v fail %v", r1.SourcesOK, r1.SourcesFailed) } if r1.Cached { t.Fatal("first miss") } joined := strings.Join(r1.Warnings, " ") if !strings.Contains(joined, "min_score not applied") { t.Fatalf("warnings %v", r1.Warnings) } r2 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5}) if !r2.Cached { t.Fatal("notes must still cache") } } func TestWarnStillDisablesCache(t *testing.T) { db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil) if err != nil { t.Fatal(err) } defer db.Close() e := &Engine{ Sources: []Source{ fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}}, fakeSrc{name: NameBraveLLM, warn: "brave_llm: 500"}, }, DefaultK: 8, Cache: db, RerankOn: false, } e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet) r := e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet) if r.Cached { t.Fatal("warn cached") } } func TestMinScoreTavilyOnly(t *testing.T) { e := &Engine{ Sources: []Source{ fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://s.example/a", Score: 0.01, Source: NameSearxNG}}, notes: []string{"searxng: min_score not applied"}}, fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://b.example/a", Source: NameBraveLLM}}, notes: []string{"brave_llm: min_score not applied"}}, fakeSrc{name: NameTavily, hits: []Hit{ {URL: "https://t.example/low", Score: 0.4, Source: NameTavily}, {URL: "https://t.example/high", Score: 0.9, Source: NameTavily}, }}, }, DefaultK: 8, RerankOn: false, } r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5}) var urls []string for _, h := range r.Hits { urls = append(urls, h.URL) if h.URL == "https://t.example/low" { t.Fatal("tavily 0.4 kept") } } joined := strings.Join(urls, " ") if !strings.Contains(joined, "s.example") || !strings.Contains(joined, "b.example") || !strings.Contains(joined, "t.example/high") { t.Fatalf("hits %v", urls) } w := strings.Join(r.Warnings, " ") if !strings.Contains(w, "searxng: min_score not applied") || !strings.Contains(w, "brave_llm: min_score not applied") { t.Fatalf("warnings %v", r.Warnings) } } func TestDomainPostFilter(t *testing.T) { hits := []Hit{ {URL: "https://openai.com/blog", Source: NameBraveLLM}, {URL: "https://sub.openai.com/x", Source: NameSearxNG}, {URL: "https://pinterest.com/pin", Source: NameBraveLLM}, {URL: "https://example.com/a", Source: NameSearxNG}, } f := Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"} got := filterDomains(hits, f) if len(got) != 2 { t.Fatalf("%+v", got) } f.IncludeDomainsMode = "prefer" got = filterDomains(hits, f) if len(got) != 4 { t.Fatalf("prefer %+v", got) } got = filterDomains(hits, Filter{ExcludeDomains: []string{"pinterest.com"}}) if len(got) != 3 { t.Fatalf("exclude %+v", got) } e := &Engine{ Sources: []Source{fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://off.example/x", Source: NameBraveLLM}}}}, DefaultK: 8, RerankOn: false, } r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}) if len(r.Hits) != 0 { t.Fatalf("brave off-domain kept %+v", r.Hits) } r = e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"}) if len(r.Hits) != 1 { t.Fatalf("prefer dropped brave %+v", r.Hits) } } func TestNormalizeDomainOR(t *testing.T) { _, _, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"a.com OR b.com"}}) if msg != "invalid search" || hint != "include_domains" { t.Fatalf("%q %q", msg, hint) } _, out, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"oracle.com"}}) if msg != "" || hint != "" || len(out.IncludeDomains) != 1 || out.IncludeDomains[0] != "oracle.com" { t.Fatalf("%q %q %+v", msg, hint, out) } _, out, msg, _ = NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"z.com", "a.com", "m.com"}}) if msg != "" || strings.Join(out.IncludeDomains, ",") != "a.com,m.com,z.com" { t.Fatalf("sorted domains %+v", out.IncludeDomains) } _, four, _, _ := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"d.com", "c.com", "b.com", "a.com"}}) q, notes := searxQueryAndNotes("q", four) if !strings.HasPrefix(q, "site:a.com OR site:b.com OR site:c.com ") { t.Fatalf("site subset %q", q) } if !strings.Contains(strings.Join(notes, " "), "extra include_domains dropped") { t.Fatalf("notes %v", notes) } } func TestMultiQueryRRF(t *testing.T) { q1 := make([]Hit, 0, 3) for _, u := range []string{"https://a.example/1", "https://a.example/2", "https://a.example/3"} { q1 = append(q1, Hit{URL: u, Source: NameSearxNG}) } e := &Engine{ Sources: []Source{qSrc{ name: NameSearxNG, byQuery: map[string][]Hit{ "one": q1, "two": {{URL: "https://b.example/only", Source: NameSearxNG}}, }, }}, DefaultK: 3, RerankOn: true, } r := e.RunFilter(context.Background(), []string{"one", "two"}, 3, true, "titan", dcache.ClassFleet, Filter{}) found := false for _, h := range r.Hits { if h.URL == "https://b.example/only" { found = true } } if !found { t.Fatalf("q2 hit missing: %+v", r.Hits) } } func TestCapMidBatchNoCache(t *testing.T) { db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil) if err != nil { t.Fatal(err) } defer db.Close() e := &Engine{ Sources: []Source{qSrc{ name: NameTavily, byQuery: map[string][]Hit{ "one": {{URL: "https://t.example/1", Source: NameTavily}}, "two": {{URL: "https://t.example/2", Source: NameTavily}}, }, credits: 1, }}, DefaultK: 8, Cache: db, TavilyCap: 1, RerankOn: false, } r := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{}) n := 0 for _, w := range r.Warnings { if strings.Contains(w, "daily cap") { n++ } } if n != 1 { t.Fatalf("warnings %v", r.Warnings) } r2 := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{}) if r2.Cached { t.Fatal("mid-batch cap must not cache") } } func TestCacheKeyFilters(t *testing.T) { db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil) if err != nil { t.Fatal(err) } defer db.Close() now := time.Date(2026, 9, 29, 10, 0, 0, 0, time.UTC) src := fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}} e := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }} base := Filter{} e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, base) if e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached { t.Fatal("different include_domains must miss") } e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}) if !e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached { t.Fatal("same filters must hit") } e2 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }} e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{}) if !e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{Topic: "general"}).Cached { t.Fatal("omitted topic and general must share a key") } _, day, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "d"}) _, long, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "day"}) if day.TimeRange != long.TimeRange || day.TimeRange != "day" { t.Fatalf("%q %q", day.TimeRange, long.TimeRange) } e3 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }} e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, day) if !e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, long).Cached { t.Fatal("d and day must share a key") } } func TestQuotedPhrase(t *testing.T) { if !hasQuotedPhrase(`docs "phrase" x`) || hasQuotedPhrase(`docs phrase`) || hasQuotedPhrase(`"unclosed`) || hasQuotedPhrase(`""`) { t.Fatal("quoted phrase") } } func TestNormalizeQueries(t *testing.T) { _, _, msg, _ := NormalizeSearch("a", []string{"b"}, Filter{}) if msg != "invalid query" { t.Fatalf("both: %q", msg) } qs, _, msg, _ := NormalizeSearch("", []string{"a", "b", "c", "d", "e"}, Filter{}) if msg != "invalid query" || qs != nil { t.Fatalf("five: %q", msg) } _, _, msg, hint := NormalizeSearch(`no quotes`, nil, Filter{ExactMatch: true}) if msg != "invalid search" || hint != "exact_match" { t.Fatalf("%q %q", msg, hint) } }