feat(search): P3.1 filters and multi-query merge

Native include/exclude domains, time_range, topic, exact_match, min_score, and up to 4 queries fused with RRF. Honesty notes stay off the Warn/fail path. Tavily keeps include_answer false and sets auto_parameters false. Cache key is canonical JSON. CLI flags and skill recipe included.
2026-09-29 15:30:19 -07:00
parent 5c6a4a40ec
commit 8b26776cdd
21 changed files with 1606 additions and 195 deletions
@@ -0,0 +1,159 @@
+package search
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/search/dial"
+)
+
+func TestTavilyFiltersOutbound(t *testing.T) {
+	var gotBody []byte
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotBody, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{"answer":"SECRET","results":[{"url":"https://ex.com/a","title":"A","content":"snip","score":0.9,"published_date":"2026-09-01"}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := Filter{
+		IncludeDomains:     []string{"ex.com"},
+		IncludeDomainsMode: "restrict",
+		TimeRange:          "month",
+		Topic:              "news",
+		ExactMatch:         true,
+	}
+	out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), `docs "phrase"`, 5, f)
+	if out.Warn != "" {
+		t.Fatal(out.Warn)
+	}
+	var req map[string]any
+	if err := json.Unmarshal(gotBody, &req); err != nil {
+		t.Fatal(err)
+	}
+	if req["include_answer"] != false || req["auto_parameters"] != false || req["search_depth"] != "basic" {
+		t.Fatalf("%+v", req)
+	}
+	if req["query"] != `docs "phrase"` {
+		t.Fatalf("query rewritten: %v", req["query"])
+	}
+	if req["exact_match"] != true {
+		t.Fatalf("exact_match %v", req["exact_match"])
+	}
+	if _, ok := req["start_date"]; ok {
+		t.Fatal("time_range with dates")
+	}
+	if req["time_range"] != "month" || req["topic"] != "news" {
+		t.Fatalf("%+v", req)
+	}
+	if req["include_published_date"] != true {
+		t.Fatal("want include_published_date")
+	}
+	if _, ok := req["include_domains_mode"]; !ok {
+		t.Fatal("mode missing")
+	}
+	if strings.Contains(string(gotBody), "extract") || strings.Contains(string(gotBody), "/crawl") {
+		t.Fatal(string(gotBody))
+	}
+	if len(out.Hits) != 1 || out.Hits[0].PublishedDate != "2026-09-01" {
+		t.Fatalf("%+v", out.Hits)
+	}
+}
+
+func TestTavilyModeOmittedWithoutInclude(t *testing.T) {
+	var gotBody []byte
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotBody, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{"results":[{"url":"https://ex.com/a","title":"A","content":"s","score":0.2}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{IncludeDomainsMode: "prefer"})
+	if out.Warn != "" {
+		t.Fatal(out.Warn)
+	}
+	var req map[string]any
+	_ = json.Unmarshal(gotBody, &req)
+	if _, ok := req["include_domains_mode"]; ok {
+		t.Fatalf("mode without include: %+v", req)
+	}
+	if out.Hits[0].PublishedDate != "" {
+		t.Fatal("missing date must omit")
+	}
+}
+
+func TestSearxNGSiteAndParams(t *testing.T) {
+	var gotQ, gotTR, gotCat string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotQ = r.URL.Query().Get("q")
+		gotTR = r.URL.Query().Get("time_range")
+		gotCat = r.URL.Query().Get("categories")
+		if strings.Contains(gotQ, "!news") || strings.Contains(gotQ, "!month") {
+			t.Errorf("bang in q: %s", gotQ)
+		}
+		_, _ = w.Write([]byte(`{"results":[{"url":"https://a.example/","title":"a","content":"aa","score":1,"publishedDate":"Mon, 01 Sep 2026 00:00:00 GMT"}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := Filter{IncludeDomains: []string{"a.example"}, TimeRange: "month", Topic: "news", ExactMatch: true}
+	out := (&SearxNG{Client: c}).Search(context.Background(), `"phrase" here`, 8, f)
+	if !strings.HasPrefix(gotQ, "site:a.example ") {
+		t.Fatalf("q %q", gotQ)
+	}
+	if !strings.Contains(gotQ, `"phrase" here`) || strings.Count(gotQ, `"phrase"`) != 1 {
+		t.Fatalf("wrapped q %q", gotQ)
+	}
+	if gotTR != "month" || gotCat != "news" {
+		t.Fatalf("params tr=%q cat=%q", gotTR, gotCat)
+	}
+	if out.Hits[0].PublishedDate == "" {
+		t.Fatal("publishedDate")
+	}
+}
+
+func TestBraveNotesAndUnchangedURL(t *testing.T) {
+	var gotURL string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotURL = r.URL.String()
+		_, _ = w.Write([]byte(`{"sources":[{"url":"https://off.example/x","title":"S","snippet":"hello"}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := Filter{TimeRange: "month", Topic: "news", ExactMatch: true, MinScore: 0.5, IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"}
+	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), `"phrase"`, 5, f)
+	if !strings.Contains(gotURL, "q=") {
+		t.Fatalf("url %s", gotURL)
+	}
+	if strings.Contains(gotURL, "time_range") || strings.Contains(gotURL, "site:") {
+		t.Fatalf("brave grew params: %s", gotURL)
+	}
+	note := strings.Join(out.Notes, " ")
+	for _, want := range []string{
+		"brave_llm: time_range not applied",
+		"brave_llm: topic not applied",
+		"brave_llm: exact_match not applied",
+		"brave_llm: include_domains_mode=prefer not applied",
+		"brave_llm: min_score not applied",
+	} {
+		if !strings.Contains(note, want) {
+			t.Fatalf("missing %q in %v", want, out.Notes)
+		}
+	}
+}