kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(search): P3.1 filters and multi-query merge

Browse Source 
Native include/exclude domains, time_range, topic, exact_match, min_score,
and up to 4 queries fused with RRF. Honesty notes stay off the Warn/fail
path. Tavily keeps include_answer false and sets auto_parameters false.
Cache key is canonical JSON. CLI flags and skill recipe included.
This commit is contained in:
kyle
2026-09-29 15:30:19 -07:00
parent 5c6a4a40ec
commit 8b26776cdd
21 changed files with 1606 additions and 195 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/api/search.go
+59 -19
View File
@@ -10,13 +10,23 @@ import (
)
type v1SearchReq struct {
Query string `json:"query"`
Question string `json:"question"`
K int `json:"k"`
Rerank *bool `json:"rerank"`
Digest bool `json:"digest"`
URL string `json:"url"`
Limit int `json:"limit"`
Query string `json:"query"`
Queries []string `json:"queries"`
Question string `json:"question"`
K int `json:"k"`
Rerank *bool `json:"rerank"`
Digest bool `json:"digest"`
URL string `json:"url"`
Limit int `json:"limit"`
IncludeDomains []string `json:"include_domains"`
ExcludeDomains []string `json:"exclude_domains"`
IncludeDomainsMode string `json:"include_domains_mode"`
TimeRange string `json:"time_range"`
StartDate string `json:"start_date"`
EndDate string `json:"end_date"`
Topic string `json:"topic"`
ExactMatch bool `json:"exact_match"`
MinScore *float64 `json:"min_score"`
}
type v2SearchReq struct {
@@ -42,9 +52,11 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
}
dec := json.NewDecoder(r.Body)
query := ""
var queries []string
k := 0
doRerank := true
digest := false
var f search.Filter
if v2 {
var req v2SearchReq
if err := dec.Decode(&req); err != nil && err.Error() != "EOF" {
@@ -55,13 +67,17 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
k = req.Limit
digest = req.Digest
_ = req.URL
if msg := search.ValidateQuery(query); msg != "" {
writeErr(w, 400, msg, "")
return
}
queries = []string{strings.TrimSpace(query)}
} else {
var req v1SearchReq
if err := dec.Decode(&req); err != nil && err.Error() != "EOF" {
writeErr(w, 400, "invalid query", "")
return
}
query = req.Query
k = req.K
if k == 0 {
k = req.Limit
@@ -72,10 +88,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
digest = req.Digest
_ = req.URL
_ = req.Question
}
if msg := search.ValidateQuery(query); msg != "" {
writeErr(w, 400, msg, "")
return
raw := search.Filter{
IncludeDomains: req.IncludeDomains,
ExcludeDomains: req.ExcludeDomains,
IncludeDomainsMode: req.IncludeDomainsMode,
TimeRange: req.TimeRange,
StartDate: req.StartDate,
EndDate: req.EndDate,
Topic: req.Topic,
ExactMatch: req.ExactMatch,
}
if req.MinScore != nil {
raw.MinScore = *req.MinScore
}
var errMsg, hint string
queries, f, errMsg, hint = search.NormalizeSearch(req.Query, req.Queries, raw)
if errMsg != "" {
writeErr(w, 400, errMsg, hint)
return
}
}
defK := search.DefaultK
if v2 {
@@ -86,7 +117,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
writeErr(w, 422, "no search results", "")
return
}
res := s.Search.Run(r.Context(), strings.TrimSpace(query), k, doRerank, c.Name, s.callerClass(c))
res := s.Search.RunFilter(r.Context(), queries, k, doRerank, c.Name, s.callerClass(c), f)
warns := append([]string(nil), res.Warnings...)
if digest {
warns = append(warns, "digest ignored")
@@ -95,7 +126,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
writeErr(w, 422, "no search results", "")
return
}
s.logSearch(c, query, res)
s.logSearch(c, queries, f, res)
if v2 {
web := make([]map[string]any, 0, len(res.Hits))
for i, h := range res.Hits {
@@ -111,13 +142,17 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
}
results := make([]map[string]any, 0, len(res.Hits))
for _, h := range res.Hits {
results = append(results, map[string]any{
row := map[string]any{
"url": h.URL,
"title": h.Title,
"snippet": h.Snippet,
"score": h.Score,
"source": h.Source,
})
}
if h.PublishedDate != "" {
row["published_date"] = h.PublishedDate
}
results = append(results, row)
}
writeJSON(w, 200, map[string]any{
"results": results,
@@ -132,8 +167,8 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
})
}
func (s *Server) logSearch(c *auth.Caller, query string, res search.RunResult) {
qLen := len([]rune(query))
func (s *Server) logSearch(c *auth.Caller, queries []string, f search.Filter, res search.RunResult) {
qLen := search.QueryRuneLen(queries)
cache := "miss"
if res.Cached {
cache = "hit"
@@ -143,5 +178,10 @@ func (s *Server) logSearch(c *auth.Caller, query string, res search.RunResult) {
s.Log.Info("search", "caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache)
return
}
s.Log.Info("search", "caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache, "rerank_ms", res.RerankMS)
kinds := search.FilterKinds(f)
attrs := []any{"caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache, "rerank_ms", res.RerankMS, "n_queries", len(queries)}
if kinds != "" {
attrs = append(attrs, "filters", kinds)
}
s.Log.Info("search", attrs...)
}
internal/api/search_filter_test.go
+117
View File
@@ -0,0 +1,117 @@
package api
import (
"context"
"encoding/json"
"strings"
"testing"
"git.wntrmute.dev/kyle/sift/internal/search"
)
type stubSearch struct {
hits []search.Hit
}
func (s stubSearch) Name() string { return search.NameSearxNG }
func (s stubSearch) Search(_ context.Context, _ string, _ int, _ search.Filter) search.SourceOut {
hits := s.hits
if hits == nil {
hits = []search.Hit{{URL: "https://ok.example/a", Title: "t", Snippet: "s", Source: search.NameSearxNG, PublishedDate: "2026-09-01"}}
}
return search.SourceOut{Hits: hits}
}
func TestSearchValidationTable(t *testing.T) {
s, key, _, _, _ := searchServer(t)
s.Search.Sources = []search.Source{stubSearch{}}
cases := []struct {
body map[string]any
err string
hint string
}{
{map[string]any{"query": ""}, "invalid query", ""},
{map[string]any{"query": "a", "queries": []string{"b"}}, "invalid query", ""},
{map[string]any{"queries": []string{"a", "b", "c", "d", "e"}}, "invalid query", ""},
{map[string]any{"queries": []string{"a", ""}}, "invalid query", ""},
{map[string]any{"query": "q", "include_domains": []string{"a.com OR b.com"}}, "invalid search", "include_domains"},
{map[string]any{"query": "q", "include_domains_mode": "prefer"}, "invalid search", "include_domains_mode"},
{map[string]any{"query": "q", "time_range": "hour"}, "invalid search", "time_range"},
{map[string]any{"query": "q", "time_range": "day", "start_date": "2026-09-01"}, "invalid search", "time_range"},
{map[string]any{"query": "q", "start_date": "2026-09-10", "end_date": "2026-09-01"}, "invalid search", "end_date"},
{map[string]any{"query": "q", "topic": "sports"}, "invalid search", "topic"},
{map[string]any{"query": "q", "min_score": -1}, "invalid search", "min_score"},
{map[string]any{"query": "q", "min_score": 1.1}, "invalid search", "min_score"},
{map[string]any{"query": "no quotes", "exact_match": true}, "invalid search", "exact_match"},
}
for _, tc := range cases {
rec := postJSON(t, s, "/v1/search", key, tc.body)
if rec.Code != 400 {
t.Fatalf("%v: %d %s", tc.body, rec.Code, rec.Body.String())
}
var env map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
t.Fatal(err)
}
if env["error"] != tc.err {
t.Fatalf("%v error %v want %q", tc.body, env["error"], tc.err)
}
h, _ := env["hint"].(string)
if h != tc.hint {
t.Fatalf("%v hint %q want %q", tc.body, h, tc.hint)
}
}
rec := postJSON(t, s, "/v1/search", key, map[string]any{"query": "q", "include_domains": []string{"oracle.com"}})
if rec.Code != 200 {
t.Fatalf("oracle.com %d %s", rec.Code, rec.Body.String())
}
rec = postJSON(t, s, "/v1/search", key, map[string]any{"query": `find "phrase"`, "exact_match": true})
if rec.Code != 200 {
t.Fatalf("exact_match quoted %d %s", rec.Code, rec.Body.String())
}
}
func TestV2SearchNoPublishedDate(t *testing.T) {
s, key, _, _, _ := searchServer(t)
s.Search.Sources = []search.Source{stubSearch{}}
rec := postJSON(t, s, "/v2/search", key, map[string]any{"query": "contract", "limit": 3, "include_domains": []string{"secret.example"}})
if rec.Code != 200 {
t.Fatalf("%d %s", rec.Code, rec.Body.String())
}
if strings.Contains(rec.Body.String(), "published_date") {
t.Fatalf("v2 leaked published_date: %s", rec.Body.String())
}
var env map[string]any
if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
t.Fatal(err)
}
data, _ := env["data"].(map[string]any)
web, _ := data["web"].([]any)
if len(web) < 1 {
t.Fatalf("%v", env)
}
}
func TestSearchFilterLogsNoSecrets(t *testing.T) {
s, fleet, gs, h, _ := searchServer(t)
s.Search.Sources = []search.Source{stubSearch{}}
body := map[string]any{
"queries": []string{"alpha secret", "beta secret", "gamma secret"},
"include_domains": []string{"secret.example"},
"rerank": false,
}
rec := postJSON(t, s, "/v1/search", gs, body)
if rec.Code != 200 {
t.Fatalf("gs %d %s", rec.Code, rec.Body.String())
}
rec = postJSON(t, s, "/v1/search", fleet, body)
if rec.Code != 200 {
t.Fatalf("fleet %d %s", rec.Code, rec.Body.String())
}
dump := h.dump()
for _, bad := range []string{"secret.example", "alpha secret", "beta secret", "gamma secret"} {
if strings.Contains(dump, bad) {
t.Fatalf("log leak %q:\n%s", bad, dump)
}
}
}