sift/internal/search/p31_test.go

package search

import (
	"context"
	"path/filepath"
	"strings"
	"testing"
	"time"

	"git.wntrmute.dev/kyle/sift/internal/dcache"
)

type qSrc struct {
	name    string
	byQuery map[string][]Hit
	notes   []string
	credits int
}

func (q qSrc) Name() string { return q.name }
func (q qSrc) Search(_ context.Context, query string, _ int, _ Filter) SourceOut {
	return SourceOut{Hits: q.byQuery[query], Notes: q.notes, Credits: q.credits}
}

func TestNotesDoNotFailSource(t *testing.T) {
	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	e := &Engine{
		Sources: []Source{fakeSrc{
			name:  NameSearxNG,
			hits:  []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}},
			notes: []string{"searxng: min_score not applied"},
		}},
		DefaultK: 8, Cache: db, RerankOn: false,
	}
	r1 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
	if len(r1.Hits) != 1 {
		t.Fatalf("hits %+v", r1.Hits)
	}
	if len(r1.SourcesOK) != 1 || r1.SourcesOK[0] != NameSearxNG {
		t.Fatalf("ok %v fail %v", r1.SourcesOK, r1.SourcesFailed)
	}
	if r1.Cached {
		t.Fatal("first miss")
	}
	joined := strings.Join(r1.Warnings, " ")
	if !strings.Contains(joined, "min_score not applied") {
		t.Fatalf("warnings %v", r1.Warnings)
	}
	r2 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
	if !r2.Cached {
		t.Fatal("notes must still cache")
	}
}

func TestWarnStillDisablesCache(t *testing.T) {
	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	e := &Engine{
		Sources: []Source{
			fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}},
			fakeSrc{name: NameBraveLLM, warn: "brave_llm: 500"},
		},
		DefaultK: 8, Cache: db, RerankOn: false,
	}
	e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet)
	r := e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet)
	if r.Cached {
		t.Fatal("warn cached")
	}
}

func TestMinScoreTavilyOnly(t *testing.T) {
	e := &Engine{
		Sources: []Source{
			fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://s.example/a", Score: 0.01, Source: NameSearxNG}}, notes: []string{"searxng: min_score not applied"}},
			fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://b.example/a", Source: NameBraveLLM}}, notes: []string{"brave_llm: min_score not applied"}},
			fakeSrc{name: NameTavily, hits: []Hit{
				{URL: "https://t.example/low", Score: 0.4, Source: NameTavily},
				{URL: "https://t.example/high", Score: 0.9, Source: NameTavily},
			}},
		},
		DefaultK: 8, RerankOn: false,
	}
	r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
	var urls []string
	for _, h := range r.Hits {
		urls = append(urls, h.URL)
		if h.URL == "https://t.example/low" {
			t.Fatal("tavily 0.4 kept")
		}
	}
	joined := strings.Join(urls, " ")
	if !strings.Contains(joined, "s.example") || !strings.Contains(joined, "b.example") || !strings.Contains(joined, "t.example/high") {
		t.Fatalf("hits %v", urls)
	}
	w := strings.Join(r.Warnings, " ")
	if !strings.Contains(w, "searxng: min_score not applied") || !strings.Contains(w, "brave_llm: min_score not applied") {
		t.Fatalf("warnings %v", r.Warnings)
	}
}

func TestDomainPostFilter(t *testing.T) {
	hits := []Hit{
		{URL: "https://openai.com/blog", Source: NameBraveLLM},
		{URL: "https://sub.openai.com/x", Source: NameSearxNG},
		{URL: "https://pinterest.com/pin", Source: NameBraveLLM},
		{URL: "https://example.com/a", Source: NameSearxNG},
	}
	f := Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}
	got := filterDomains(hits, f)
	if len(got) != 2 {
		t.Fatalf("%+v", got)
	}
	f.IncludeDomainsMode = "prefer"
	got = filterDomains(hits, f)
	if len(got) != 4 {
		t.Fatalf("prefer %+v", got)
	}
	got = filterDomains(hits, Filter{ExcludeDomains: []string{"pinterest.com"}})
	if len(got) != 3 {
		t.Fatalf("exclude %+v", got)
	}
	e := &Engine{
		Sources:  []Source{fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://off.example/x", Source: NameBraveLLM}}}},
		DefaultK: 8, RerankOn: false,
	}
	r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"})
	if len(r.Hits) != 0 {
		t.Fatalf("brave off-domain kept %+v", r.Hits)
	}
	r = e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"})
	if len(r.Hits) != 1 {
		t.Fatalf("prefer dropped brave %+v", r.Hits)
	}
}

func TestNormalizeDomainOR(t *testing.T) {
	_, _, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"a.com OR b.com"}})
	if msg != "invalid search" || hint != "include_domains" {
		t.Fatalf("%q %q", msg, hint)
	}
	_, out, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"oracle.com"}})
	if msg != "" || hint != "" || len(out.IncludeDomains) != 1 || out.IncludeDomains[0] != "oracle.com" {
		t.Fatalf("%q %q %+v", msg, hint, out)
	}
	_, out, msg, _ = NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"z.com", "a.com", "m.com"}})
	if msg != "" || strings.Join(out.IncludeDomains, ",") != "a.com,m.com,z.com" {
		t.Fatalf("sorted domains %+v", out.IncludeDomains)
	}
	_, four, _, _ := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"d.com", "c.com", "b.com", "a.com"}})
	q, notes := searxQueryAndNotes("q", four)
	if !strings.HasPrefix(q, "site:a.com OR site:b.com OR site:c.com ") {
		t.Fatalf("site subset %q", q)
	}
	if !strings.Contains(strings.Join(notes, " "), "extra include_domains dropped") {
		t.Fatalf("notes %v", notes)
	}
}

func TestMultiQueryRRF(t *testing.T) {
	q1 := make([]Hit, 0, 3)
	for _, u := range []string{"https://a.example/1", "https://a.example/2", "https://a.example/3"} {
		q1 = append(q1, Hit{URL: u, Source: NameSearxNG})
	}
	e := &Engine{
		Sources: []Source{qSrc{
			name: NameSearxNG,
			byQuery: map[string][]Hit{
				"one": q1,
				"two": {{URL: "https://b.example/only", Source: NameSearxNG}},
			},
		}},
		DefaultK: 3, RerankOn: true,
	}
	r := e.RunFilter(context.Background(), []string{"one", "two"}, 3, true, "titan", dcache.ClassFleet, Filter{})
	found := false
	for _, h := range r.Hits {
		if h.URL == "https://b.example/only" {
			found = true
		}
	}
	if !found {
		t.Fatalf("q2 hit missing: %+v", r.Hits)
	}
}

func TestCapMidBatchNoCache(t *testing.T) {
	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	e := &Engine{
		Sources: []Source{qSrc{
			name: NameTavily,
			byQuery: map[string][]Hit{
				"one": {{URL: "https://t.example/1", Source: NameTavily}},
				"two": {{URL: "https://t.example/2", Source: NameTavily}},
			},
			credits: 1,
		}},
		DefaultK: 8, Cache: db, TavilyCap: 1, RerankOn: false,
	}
	r := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{})
	n := 0
	for _, w := range r.Warnings {
		if strings.Contains(w, "daily cap") {
			n++
		}
	}
	if n != 1 {
		t.Fatalf("warnings %v", r.Warnings)
	}
	r2 := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{})
	if r2.Cached {
		t.Fatal("mid-batch cap must not cache")
	}
}

func TestCacheKeyFilters(t *testing.T) {
	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
	if err != nil {
		t.Fatal(err)
	}
	defer db.Close()
	now := time.Date(2026, 9, 29, 10, 0, 0, 0, time.UTC)
	src := fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}}
	e := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
	base := Filter{}
	e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, base)
	if e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached {
		t.Fatal("different include_domains must miss")
	}
	e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"})
	if !e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached {
		t.Fatal("same filters must hit")
	}
	e2 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
	e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{})
	if !e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{Topic: "general"}).Cached {
		t.Fatal("omitted topic and general must share a key")
	}
	_, day, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "d"})
	_, long, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "day"})
	if day.TimeRange != long.TimeRange || day.TimeRange != "day" {
		t.Fatalf("%q %q", day.TimeRange, long.TimeRange)
	}
	e3 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
	e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, day)
	if !e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, long).Cached {
		t.Fatal("d and day must share a key")
	}
}

func TestQuotedPhrase(t *testing.T) {
	if !hasQuotedPhrase(`docs "phrase" x`) || hasQuotedPhrase(`docs phrase`) || hasQuotedPhrase(`"unclosed`) || hasQuotedPhrase(`""`) {
		t.Fatal("quoted phrase")
	}
}

func TestNormalizeQueries(t *testing.T) {
	_, _, msg, _ := NormalizeSearch("a", []string{"b"}, Filter{})
	if msg != "invalid query" {
		t.Fatalf("both: %q", msg)
	}
	qs, _, msg, _ := NormalizeSearch("", []string{"a", "b", "c", "d", "e"}, Filter{})
	if msg != "invalid query" || qs != nil {
		t.Fatalf("five: %q", msg)
	}
	_, _, msg, hint := NormalizeSearch(`no quotes`, nil, Filter{ExactMatch: true})
	if msg != "invalid search" || hint != "exact_match" {
		t.Fatalf("%q %q", msg, hint)
	}
}