feat(search): P3.1 filters and multi-query merge

Native include/exclude domains, time_range, topic, exact_match, min_score,
and up to 4 queries fused with RRF. Honesty notes stay off the Warn/fail
path. Tavily keeps include_answer false and sets auto_parameters false.
Cache key is canonical JSON. CLI flags and skill recipe included.

README.md

@@ -19,7 +19,7 @@ The server binds `127.0.0.1:8890` and is published on the tailnet by `tailscale
 | `POST /v2/scrape` | Firecrawl-compatible fetch (`url`, `formats`) — what Hermes `web_extract` calls |
 | `POST /v2/search` | Firecrawl-compatible search (`query`, `limit`) — what Hermes `web_search` calls |
 | `POST /v1/fetch` | native fetch: `url`, `mode` (`clean`/`raw`/`digest`), `question`, `jq`, `context`, `recover`, and for PDFs `pages`, `grep` |
-| `POST /v1/search` | native search: `query`, `question`, `k`, `rerank`, `digest` |
+| `POST /v1/search` | native search: `query` or `queries`, `k`, `rerank`, `digest`, domain/time/topic/`exact_match`/`min_score` |
 | `GET /metrics`, `/metrics.json` | Prometheus text or JSON; open on loopback, bearer elsewhere |
 | `GET /healthz` | liveness |
 
@@ -34,7 +34,7 @@ sift serve  [-listen addr] [-config /etc/sift/config.toml] [-keys /var/lib/sift/
 sift fetch  <url> [--question q] [--mode clean|raw|digest] [--jq filter] [--pages a-b] [--grep re]
                   [--no-context] [--recover] [--json]
 sift extract <file.pdf> [--pages a-b] [--grep re] [--question q] [--mode clean|digest] [--json]
-  sift search   "<query>" [--k n] [--json]
+  sift search   "<query>" [--k n] [--json] [--include-domain host]... [--topic news] [--time-range month]
   sift batch    submit|status|get|cancel
   sift keys   add --caller name [--workspace ws] [--out file] | list | revoke --caller name
   sift version

cmd/sift/searchcmd.go

@@ -8,15 +8,35 @@ import (
 	"io"
 	"net/http"
 	"os"
+	"strings"
 )
 
+type stringList []string
+
+func (s *stringList) String() string { return strings.Join(*s, ",") }
+func (s *stringList) Set(v string) error {
+	*s = append(*s, v)
+	return nil
+}
+
 func searchMain() {
 	fs := flag.NewFlagSet("search", flag.ContinueOnError)
 	fs.SetOutput(io.Discard)
 	k := fs.Int("k", 0, "result cap")
 	asJSON := fs.Bool("json", false, "native envelope")
+	var include, exclude stringList
+	fs.Var(&include, "include-domain", "restrict or prefer this host")
+	fs.Var(&exclude, "exclude-domain", "drop this host")
+	domainsMode := fs.String("domains-mode", "", "restrict|prefer")
+	timeRange := fs.String("time-range", "", "day|week|month|year")
+	startDate := fs.String("start-date", "", "YYYY-MM-DD")
+	endDate := fs.String("end-date", "", "YYYY-MM-DD")
+	topic := fs.String("topic", "", "general|news|finance")
+	exact := fs.Bool("exact-match", false, "require quoted phrases")
+	minScore := fs.Float64("min-score", 0, "Tavily score cut")
+	queriesFile := fs.String("queries-file", "", "JSON array of 1-4 queries")
 	args := os.Args[1:]
-	flagArgs, pos, err := splitAroundPositional(args, map[string]bool{"k": true})
+	flagArgs, pos, err := splitAroundPositional(args, searchValueFlags)
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(2)
@@ -25,20 +45,65 @@ func searchMain() {
 		fmt.Fprintln(os.Stderr, searchUsage)
 		os.Exit(2)
 	}
-	if len(pos) != 1 {
+	body := map[string]any{}
-		fmt.Fprintln(os.Stderr, searchUsage)
+	if *queriesFile != "" {
-		os.Exit(2)
+		if len(pos) != 0 {
+			fmt.Fprintln(os.Stderr, searchUsage)
+			os.Exit(2)
+		}
+		raw, err := os.ReadFile(*queriesFile)
+		if err != nil {
+			fmt.Fprintln(os.Stderr, err)
+			os.Exit(2)
+		}
+		var qs []string
+		if err := json.Unmarshal(raw, &qs); err != nil {
+			fmt.Fprintln(os.Stderr, "queries-file: want JSON array of strings")
+			os.Exit(2)
+		}
+		body["queries"] = qs
+	} else {
+		if len(pos) != 1 {
+			fmt.Fprintln(os.Stderr, searchUsage)
+			os.Exit(2)
+		}
+		body["query"] = pos[0]
+	}
+	if *k > 0 {
+		body["k"] = *k
+	}
+	if len(include) > 0 {
+		body["include_domains"] = []string(include)
+	}
+	if len(exclude) > 0 {
+		body["exclude_domains"] = []string(exclude)
+	}
+	if *domainsMode != "" {
+		body["include_domains_mode"] = *domainsMode
+	}
+	if *timeRange != "" {
+		body["time_range"] = *timeRange
+	}
+	if *startDate != "" {
+		body["start_date"] = *startDate
+	}
+	if *endDate != "" {
+		body["end_date"] = *endDate
+	}
+	if *topic != "" {
+		body["topic"] = *topic
+	}
+	if *exact {
+		body["exact_match"] = true
+	}
+	if *minScore > 0 {
+		body["min_score"] = *minScore
 	}
-	query := pos[0]
 	base, key, err := loadClient()
 	if err != nil {
 		fmt.Fprintln(os.Stderr, err)
 		os.Exit(5)
 	}
-	body := map[string]any{"query": query}
-	if *k > 0 {
-		body["k"] = *k
-	}
 	raw, _ := json.Marshal(body)
 	req, err := http.NewRequest(http.MethodPost, base+"/v1/search", bytes.NewReader(raw))
 	if err != nil {
@@ -80,7 +145,24 @@ func searchMain() {
 		if sn, _ := m["snippet"].(string); sn != "" {
 			fmt.Println(sn)
 		}
+		if d, _ := m["published_date"].(string); d != "" {
+			fmt.Println(d)
+		}
 	}
 }
 
-const searchUsage = `usage: sift search "<query>" [--k N] [--json]`
+var searchValueFlags = map[string]bool{
+	"k": true, "include-domain": true, "exclude-domain": true,
+	"domains-mode": true, "time-range": true, "start-date": true, "end-date": true,
+	"topic": true, "min-score": true, "queries-file": true,
+}
+
+const searchUsage = `usage: sift search "<query>" [--k n] [--json]
+            [--include-domain host]... [--exclude-domain host]...
+            [--domains-mode restrict|prefer]
+            [--time-range day|week|month|year]
+            [--start-date YYYY-MM-DD] [--end-date YYYY-MM-DD]
+            [--topic general|news|finance]
+            [--exact-match]
+            [--min-score 0.5]
+            [--queries-file path]`

docs/implementer-log.md

@@ -26,6 +26,7 @@ One row per task session that ends in a commit. Newest last.
 | p6/07-api-endpoints | 2026-09-26 | done | 1 | gate: ok | none | The four /v1/batch handlers in new `internal/api/batch.go`: `batchSubmit` (POST) decodes `{"targets":[{url,mode,question,pages,grep,jq,context,recover}]}` flat like `/v1/fetch`, 400 "invalid batch" on bad json/empty list/empty url/bad mode (only empty,clean,raw,digest), then `PreCheck` per target → 422 "blocked: target <idx>: <reason>" with no job, then `Submit` mapping ErrTooManyTargets→422 "too many targets (max 32)", ErrCallerBusy→429+Retry-After 30, ErrBusy→429+Retry-After 60, ErrNoTargets→400, and 202 {success,job_id,status,targets,reused} (status "queued" new, current status reused). `batchStatus` (GET) and `batchResult` (GET {id}/{idx}) and `batchCancel` (DELETE) share `ownership` (unknown or not `c.Name` → 404, never 403; nil batch → 503); status is a summary with no results unless `?results=1` inlines the envelope (Meta+content) on done targets, idx validated 0..targets_total-1 else 404, cancel of a terminal job → 409 "job already finished" else 200 {success,status:cancelled}. `api.go` scoped to the rule-2 changes: `batchMaxBody` (256 KiB) const, the four routes in `New`, and `ServeHTTP` applying the per-path cap once (`/v1/batch`→256 KiB, `/v1/extract`→pdfMaxUpload, else 64 KiB) and collapsing every batch path to a fixed metrics label via `batchObservePath` (`/v1/batch` or `/v1/batch/{id}`, never a job id). Each handler logs one `batch` line with caller/job_id/targets/status-or-reused and never a URL/hostname/idem_key. Given test byte-identical; all api tests pass under -race -count=3; make gate ok. |
 | p6/08-serve-config | 2026-09-26 | done | 1 | gate: ok | none | `[batch]` config in `internal/config`: the `Batch` type (disabled/targets_per_job/jobs_per_caller/jobs_total/workers/ttl/max_bytes) and `File.Batch`, plus `ParseSize` (trims spaces, accepts only binary KiB/MiB/GiB suffixes or a plain byte count, zero/negative/other → def). serve.go opens the dcache store when batch is on (`!cfg.Batch.Disabled`) alongside dixie/search — same path, same call, Warn on failure — then, before building the runner, reaps in-flight jobs a restart killed via `MarkInFlightFailed("restart", now)` with one `ObserveBatchJob("restart")` per reaped job and a `batch reaped` info log when n>0; builds `srv.Batch` from the config (TTL via `ParseDuration`, MaxBytes via `ParseSize`, runner defaults fill the zeros) only when enabled and the store opened; closes `srv.Batch` in both shutdown branches when non-nil. Added a commented `[batch]` block to dist/sift.toml.example. Given test byte-identical; all config and cmd tests pass under -race -count=3. |
 | p6/09-cli-docs | 2026-09-26 | done | 1 | gate: ok | none | `cmd/sift/batchcmd.go`: `parseBatchArgs` (verb + unchanged `rest`; submit takes one arg, status one job id optionally preceded by `--results`, get a job id plus a non-negative-integer idx, cancel one job id, else a usage error), `batchMain` dispatch, one small function per verb, and `batchDo` (bearer header, decode the JSON envelope, print it indented on stdout, on non-2xx print `env["error"]` to stderr and `exitFor`). `submitMain` reads the file or stdin and `buildTargetsBody` accepts a JSON object with a non-empty `targets` array, a bare array of target objects, or a bare array of URL strings (each wrapped `{"url": s}`), validating locally before the call. `main.go`: one `case "batch"` and one help line. README "Batch" section (four endpoints, serial-per-job/one-worker/round-robin model, 32 targets/1-per-caller/8-total caps, 256 KiB body, idempotency, lease/TTL/restart, summary-vs-results, restricted redaction, `sift_batch_*` metrics); SKILL.md "Batch" section. Given test byte-identical; all cmd/sift tests pass under -race -count=3. |
+| p3.1 | 2026-09-29 | done | 1 | gofmt/vet/race ok; make gate blocked by pre-existing staticcheck 0.6.1 vs go 1.26 | none | Native filters + multi-query RRF: Filter on Source.Search, SourceOut.Notes, Tavily auto_parameters false, SearXNG site:/time_range/categories, Brave notes, domain post-filter, Tavily min_score, sequential queries, JSON cache key, CLI flags, skill recipe. §11 tests in p31_*.go and search_filter_test.go. Guard/auth/orion pin untouched. |
 
 ## P5a review (owner, 2026-09-26)
 

internal/api/search.go

@@ -10,13 +10,23 @@ import (
 )
 
 type v1SearchReq struct {
-	Query    string `json:"query"`
+	Query              string   `json:"query"`
-	Question string `json:"question"`
+	Queries            []string `json:"queries"`
-	K        int    `json:"k"`
+	Question           string   `json:"question"`
-	Rerank   *bool  `json:"rerank"`
+	K                  int      `json:"k"`
-	Digest   bool   `json:"digest"`
+	Rerank             *bool    `json:"rerank"`
-	URL      string `json:"url"`
+	Digest             bool     `json:"digest"`
-	Limit    int    `json:"limit"`
+	URL                string   `json:"url"`
+	Limit              int      `json:"limit"`
+	IncludeDomains     []string `json:"include_domains"`
+	ExcludeDomains     []string `json:"exclude_domains"`
+	IncludeDomainsMode string   `json:"include_domains_mode"`
+	TimeRange          string   `json:"time_range"`
+	StartDate          string   `json:"start_date"`
+	EndDate            string   `json:"end_date"`
+	Topic              string   `json:"topic"`
+	ExactMatch         bool     `json:"exact_match"`
+	MinScore           *float64 `json:"min_score"`
 }
 
 type v2SearchReq struct {
@@ -42,9 +52,11 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 	}
 	dec := json.NewDecoder(r.Body)
 	query := ""
+	var queries []string
 	k := 0
 	doRerank := true
 	digest := false
+	var f search.Filter
 	if v2 {
 		var req v2SearchReq
 		if err := dec.Decode(&req); err != nil && err.Error() != "EOF" {
@@ -55,13 +67,17 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 		k = req.Limit
 		digest = req.Digest
 		_ = req.URL
+		if msg := search.ValidateQuery(query); msg != "" {
+			writeErr(w, 400, msg, "")
+			return
+		}
+		queries = []string{strings.TrimSpace(query)}
 	} else {
 		var req v1SearchReq
 		if err := dec.Decode(&req); err != nil && err.Error() != "EOF" {
 			writeErr(w, 400, "invalid query", "")
 			return
 		}
-		query = req.Query
 		k = req.K
 		if k == 0 {
 			k = req.Limit
@@ -72,10 +88,25 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 		digest = req.Digest
 		_ = req.URL
 		_ = req.Question
-	}
+		raw := search.Filter{
-	if msg := search.ValidateQuery(query); msg != "" {
+			IncludeDomains:     req.IncludeDomains,
-		writeErr(w, 400, msg, "")
+			ExcludeDomains:     req.ExcludeDomains,
-		return
+			IncludeDomainsMode: req.IncludeDomainsMode,
+			TimeRange:          req.TimeRange,
+			StartDate:          req.StartDate,
+			EndDate:            req.EndDate,
+			Topic:              req.Topic,
+			ExactMatch:         req.ExactMatch,
+		}
+		if req.MinScore != nil {
+			raw.MinScore = *req.MinScore
+		}
+		var errMsg, hint string
+		queries, f, errMsg, hint = search.NormalizeSearch(req.Query, req.Queries, raw)
+		if errMsg != "" {
+			writeErr(w, 400, errMsg, hint)
+			return
+		}
 	}
 	defK := search.DefaultK
 	if v2 {
@@ -86,7 +117,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 		writeErr(w, 422, "no search results", "")
 		return
 	}
-	res := s.Search.Run(r.Context(), strings.TrimSpace(query), k, doRerank, c.Name, s.callerClass(c))
+	res := s.Search.RunFilter(r.Context(), queries, k, doRerank, c.Name, s.callerClass(c), f)
 	warns := append([]string(nil), res.Warnings...)
 	if digest {
 		warns = append(warns, "digest ignored")
@@ -95,7 +126,7 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 		writeErr(w, 422, "no search results", "")
 		return
 	}
-	s.logSearch(c, query, res)
+	s.logSearch(c, queries, f, res)
 	if v2 {
 		web := make([]map[string]any, 0, len(res.Hits))
 		for i, h := range res.Hits {
@@ -111,13 +142,17 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 	}
 	results := make([]map[string]any, 0, len(res.Hits))
 	for _, h := range res.Hits {
-		results = append(results, map[string]any{
+		row := map[string]any{
 			"url":     h.URL,
 			"title":   h.Title,
 			"snippet": h.Snippet,
 			"score":   h.Score,
 			"source":  h.Source,
-		})
+		}
+		if h.PublishedDate != "" {
+			row["published_date"] = h.PublishedDate
+		}
+		results = append(results, row)
 	}
 	writeJSON(w, 200, map[string]any{
 		"results":  results,
@@ -132,8 +167,8 @@ func (s *Server) handleSearch(w http.ResponseWriter, r *http.Request, v2 bool) {
 	})
 }
 
-func (s *Server) logSearch(c *auth.Caller, query string, res search.RunResult) {
+func (s *Server) logSearch(c *auth.Caller, queries []string, f search.Filter, res search.RunResult) {
-	qLen := len([]rune(query))
+	qLen := search.QueryRuneLen(queries)
 	cache := "miss"
 	if res.Cached {
 		cache = "hit"
@@ -143,5 +178,10 @@ func (s *Server) logSearch(c *auth.Caller, query string, res search.RunResult) {
 		s.Log.Info("search", "caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache)
 		return
 	}
-	s.Log.Info("search", "caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache, "rerank_ms", res.RerankMS)
+	kinds := search.FilterKinds(f)
+	attrs := []any{"caller", c.Name, "q_len", qLen, "sources", src, "n", len(res.Hits), "cache", cache, "rerank_ms", res.RerankMS, "n_queries", len(queries)}
+	if kinds != "" {
+		attrs = append(attrs, "filters", kinds)
+	}
+	s.Log.Info("search", attrs...)
 }

internal/api/search_filter_test.go

@@ -0,0 +1,117 @@
+package api
+
+import (
+	"context"
+	"encoding/json"
+	"strings"
+	"testing"
+
+	"git.wntrmute.dev/kyle/sift/internal/search"
+)
+
+type stubSearch struct {
+	hits []search.Hit
+}
+
+func (s stubSearch) Name() string { return search.NameSearxNG }
+func (s stubSearch) Search(_ context.Context, _ string, _ int, _ search.Filter) search.SourceOut {
+	hits := s.hits
+	if hits == nil {
+		hits = []search.Hit{{URL: "https://ok.example/a", Title: "t", Snippet: "s", Source: search.NameSearxNG, PublishedDate: "2026-09-01"}}
+	}
+	return search.SourceOut{Hits: hits}
+}
+
+func TestSearchValidationTable(t *testing.T) {
+	s, key, _, _, _ := searchServer(t)
+	s.Search.Sources = []search.Source{stubSearch{}}
+	cases := []struct {
+		body map[string]any
+		err  string
+		hint string
+	}{
+		{map[string]any{"query": ""}, "invalid query", ""},
+		{map[string]any{"query": "a", "queries": []string{"b"}}, "invalid query", ""},
+		{map[string]any{"queries": []string{"a", "b", "c", "d", "e"}}, "invalid query", ""},
+		{map[string]any{"queries": []string{"a", ""}}, "invalid query", ""},
+		{map[string]any{"query": "q", "include_domains": []string{"a.com OR b.com"}}, "invalid search", "include_domains"},
+		{map[string]any{"query": "q", "include_domains_mode": "prefer"}, "invalid search", "include_domains_mode"},
+		{map[string]any{"query": "q", "time_range": "hour"}, "invalid search", "time_range"},
+		{map[string]any{"query": "q", "time_range": "day", "start_date": "2026-09-01"}, "invalid search", "time_range"},
+		{map[string]any{"query": "q", "start_date": "2026-09-10", "end_date": "2026-09-01"}, "invalid search", "end_date"},
+		{map[string]any{"query": "q", "topic": "sports"}, "invalid search", "topic"},
+		{map[string]any{"query": "q", "min_score": -1}, "invalid search", "min_score"},
+		{map[string]any{"query": "q", "min_score": 1.1}, "invalid search", "min_score"},
+		{map[string]any{"query": "no quotes", "exact_match": true}, "invalid search", "exact_match"},
+	}
+	for _, tc := range cases {
+		rec := postJSON(t, s, "/v1/search", key, tc.body)
+		if rec.Code != 400 {
+			t.Fatalf("%v: %d %s", tc.body, rec.Code, rec.Body.String())
+		}
+		var env map[string]any
+		if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
+			t.Fatal(err)
+		}
+		if env["error"] != tc.err {
+			t.Fatalf("%v error %v want %q", tc.body, env["error"], tc.err)
+		}
+		h, _ := env["hint"].(string)
+		if h != tc.hint {
+			t.Fatalf("%v hint %q want %q", tc.body, h, tc.hint)
+		}
+	}
+	rec := postJSON(t, s, "/v1/search", key, map[string]any{"query": "q", "include_domains": []string{"oracle.com"}})
+	if rec.Code != 200 {
+		t.Fatalf("oracle.com %d %s", rec.Code, rec.Body.String())
+	}
+	rec = postJSON(t, s, "/v1/search", key, map[string]any{"query": `find "phrase"`, "exact_match": true})
+	if rec.Code != 200 {
+		t.Fatalf("exact_match quoted %d %s", rec.Code, rec.Body.String())
+	}
+}
+
+func TestV2SearchNoPublishedDate(t *testing.T) {
+	s, key, _, _, _ := searchServer(t)
+	s.Search.Sources = []search.Source{stubSearch{}}
+	rec := postJSON(t, s, "/v2/search", key, map[string]any{"query": "contract", "limit": 3, "include_domains": []string{"secret.example"}})
+	if rec.Code != 200 {
+		t.Fatalf("%d %s", rec.Code, rec.Body.String())
+	}
+	if strings.Contains(rec.Body.String(), "published_date") {
+		t.Fatalf("v2 leaked published_date: %s", rec.Body.String())
+	}
+	var env map[string]any
+	if err := json.Unmarshal(rec.Body.Bytes(), &env); err != nil {
+		t.Fatal(err)
+	}
+	data, _ := env["data"].(map[string]any)
+	web, _ := data["web"].([]any)
+	if len(web) < 1 {
+		t.Fatalf("%v", env)
+	}
+}
+
+func TestSearchFilterLogsNoSecrets(t *testing.T) {
+	s, fleet, gs, h, _ := searchServer(t)
+	s.Search.Sources = []search.Source{stubSearch{}}
+	body := map[string]any{
+		"queries":         []string{"alpha secret", "beta secret", "gamma secret"},
+		"include_domains": []string{"secret.example"},
+		"rerank":          false,
+	}
+	rec := postJSON(t, s, "/v1/search", gs, body)
+	if rec.Code != 200 {
+		t.Fatalf("gs %d %s", rec.Code, rec.Body.String())
+	}
+	rec = postJSON(t, s, "/v1/search", fleet, body)
+	if rec.Code != 200 {
+		t.Fatalf("fleet %d %s", rec.Code, rec.Body.String())
+	}
+	dump := h.dump()
+	for _, bad := range []string{"secret.example", "alpha secret", "beta secret", "gamma secret"} {
+		if strings.Contains(dump, bad) {
+			t.Fatalf("log leak %q:\n%s", bad, dump)
+		}
+	}
+}

internal/dcache/dcache.go

@@ -11,8 +11,6 @@ import (
 	"log/slog"
 	"math"
 	"os"
-	"sort"
-	"strconv"
 	"strings"
 	"sync"
 	"time"
@@ -181,17 +179,6 @@ func QuestionEmbedKey(question, embedModel, callerClass string) string {
 	return hex.EncodeToString(sum[:])
 }
 
-func SearchKey(query string, sources []string, k int, hour time.Time, callerClass string) string {
-	if callerClass == "" {
-		callerClass = ClassFleet
-	}
-	src := append([]string(nil), sources...)
-	sort.Strings(src)
-	bucket := hour.UTC().Format("2006-01-02T15")
-	sum := sha256.Sum256([]byte(NormalizeQuestion(query) + "\x00" + strings.Join(src, ",") + "\x00" + strconv.Itoa(k) + "\x00" + bucket + "\x00" + callerClass))
-	return hex.EncodeToString(sum[:])
-}
-
 type DigestHit struct {
 	Text       string
 	ChunksUsed int

internal/dcache/dcache_test.go

@@ -109,6 +109,12 @@ func TestSearchKeyPartitions(t *testing.T) {
 	if SearchKey("hello", []string{"searxng", "tavily"}, 8, now.Add(time.Hour), ClassFleet) == a {
 		t.Fatal("hour")
 	}
+	if EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet}) == EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet, IncludeDomains: []string{"a.com"}}) {
+		t.Fatal("include_domains")
+	}
+	if EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet}) != EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet, Topic: "general"}) {
+		t.Fatal("topic general")
+	}
 }
 
 func TestCorruptDBBypass(t *testing.T) {

internal/dcache/searchkey.go

@@ -0,0 +1,108 @@
+package dcache
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"encoding/json"
+	"sort"
+	"strconv"
+	"time"
+)
+
+// SearchKeyInput is the canonical cache identity for a search. Callers pass
+// already-normalised filter fields (long time_range, sorted domains optional).
+type SearchKeyInput struct {
+	Queries            []string
+	Sources            []string
+	K                  int
+	Hour               time.Time
+	CallerClass        string
+	IncludeDomains     []string
+	ExcludeDomains     []string
+	IncludeDomainsMode string
+	TimeRange          string
+	StartDate          string
+	EndDate            string
+	Topic              string
+	ExactMatch         bool
+	MinScore           float64
+}
+
+func SearchKey(query string, sources []string, k int, hour time.Time, callerClass string) string {
+	return EncodeSearchKey(SearchKeyInput{
+		Queries:     []string{query},
+		Sources:     sources,
+		K:           k,
+		Hour:        hour,
+		CallerClass: callerClass,
+	})
+}
+
+func EncodeSearchKey(in SearchKeyInput) string {
+	if in.CallerClass == "" {
+		in.CallerClass = ClassFleet
+	}
+	src := append([]string(nil), in.Sources...)
+	sort.Strings(src)
+	qs := make([]string, len(in.Queries))
+	for i, q := range in.Queries {
+		qs[i] = NormalizeQuestion(q)
+	}
+	topic := in.Topic
+	if topic == "" {
+		topic = "general"
+	}
+	m := map[string]any{
+		"caller_class": in.CallerClass,
+		"exact_match":  in.ExactMatch,
+		"hour":         in.Hour.UTC().Format("2006-01-02T15"),
+		"k":            in.K,
+		"min_score":    strconv.FormatFloat(in.MinScore, 'f', -1, 64),
+		"queries":      qs,
+		"sources":      src,
+		"topic":        topic,
+	}
+	if inc := sortedUnique(in.IncludeDomains); len(inc) > 0 {
+		m["include_domains"] = inc
+		mode := in.IncludeDomainsMode
+		if mode == "" {
+			mode = "restrict"
+		}
+		m["include_domains_mode"] = mode
+	}
+	if exc := sortedUnique(in.ExcludeDomains); len(exc) > 0 {
+		m["exclude_domains"] = exc
+	}
+	if in.TimeRange != "" {
+		m["time_range"] = in.TimeRange
+	}
+	if in.StartDate != "" {
+		m["start_date"] = in.StartDate
+	}
+	if in.EndDate != "" {
+		m["end_date"] = in.EndDate
+	}
+	b, _ := json.Marshal(m)
+	sum := sha256.Sum256(b)
+	return hex.EncodeToString(sum[:])
+}
+
+func sortedUnique(in []string) []string {
+	if len(in) == 0 {
+		return nil
+	}
+	out := append([]string(nil), in...)
+	sort.Strings(out)
+	n := 0
+	for _, s := range out {
+		if s == "" {
+			continue
+		}
+		if n > 0 && out[n-1] == s {
+			continue
+		}
+		out[n] = s
+		n++
+	}
+	return out[:n]
+}

internal/search/apply.go

@@ -0,0 +1,166 @@
+package search
+
+import (
+	"net/url"
+	"strings"
+)
+
+func resultHost(raw string) string {
+	u, err := url.Parse(raw)
+	if err != nil {
+		return ""
+	}
+	h := strings.ToLower(u.Hostname())
+	return strings.TrimSuffix(h, ".")
+}
+
+func hostMatch(host, d string) bool {
+	return host == d || strings.HasSuffix(host, "."+d)
+}
+
+func filterDomains(hits []Hit, f Filter) []Hit {
+	if len(f.IncludeDomains) == 0 && len(f.ExcludeDomains) == 0 {
+		return hits
+	}
+	mode := f.IncludeDomainsMode
+	if mode == "" {
+		mode = "restrict"
+	}
+	restrict := mode != "prefer" && len(f.IncludeDomains) > 0
+	out := make([]Hit, 0, len(hits))
+	for _, h := range hits {
+		host := resultHost(h.URL)
+		if host == "" {
+			if restrict {
+				continue
+			}
+		} else {
+			drop := false
+			for _, d := range f.ExcludeDomains {
+				if hostMatch(host, d) {
+					drop = true
+					break
+				}
+			}
+			if drop {
+				continue
+			}
+			if restrict {
+				ok := false
+				for _, d := range f.IncludeDomains {
+					if hostMatch(host, d) {
+						ok = true
+						break
+					}
+				}
+				if !ok {
+					continue
+				}
+			}
+		}
+		out = append(out, h)
+	}
+	return out
+}
+
+func filterMinScore(hits []Hit, min float64) []Hit {
+	if min <= 0 {
+		return hits
+	}
+	out := make([]Hit, 0, len(hits))
+	for _, h := range hits {
+		if h.Score >= min {
+			out = append(out, h)
+		}
+	}
+	return out
+}
+
+func dedupeStrings(in []string) []string {
+	seen := map[string]bool{}
+	out := make([]string, 0, len(in))
+	for _, s := range in {
+		if s == "" || seen[s] {
+			continue
+		}
+		seen[s] = true
+		out = append(out, s)
+	}
+	return out
+}
+
+func searxQueryAndNotes(q string, f Filter) (string, []string) {
+	var notes []string
+	if f.IncludeDomainsMode == "prefer" && len(f.IncludeDomains) > 0 {
+		notes = append(notes, "searxng: include_domains_mode=prefer not applied")
+	} else if n := len(f.IncludeDomains); n == 1 {
+		q = "site:" + f.IncludeDomains[0] + " " + q
+	} else if n > 1 {
+		max := n
+		if max > searxSiteCap {
+			max = searxSiteCap
+			notes = append(notes, "searxng: extra include_domains dropped")
+		}
+		var b strings.Builder
+		for i := 0; i < max; i++ {
+			if i > 0 {
+				b.WriteString(" OR ")
+			}
+			b.WriteString("site:")
+			b.WriteString(f.IncludeDomains[i])
+		}
+		b.WriteByte(' ')
+		b.WriteString(q)
+		q = b.String()
+	}
+	if f.Topic == "finance" {
+		notes = append(notes, "searxng: topic ignored")
+	}
+	if f.StartDate != "" || f.EndDate != "" {
+		notes = append(notes, "searxng: start_date/end_date not applied")
+	}
+	if f.MinScore > 0 {
+		notes = append(notes, "searxng: min_score not applied")
+	}
+	return q, notes
+}
+
+func braveNotes(f Filter) []string {
+	var notes []string
+	if f.TimeRange != "" {
+		notes = append(notes, "brave_llm: time_range not applied")
+	}
+	if f.Topic != "" && f.Topic != "general" {
+		notes = append(notes, "brave_llm: topic not applied")
+	}
+	if f.ExactMatch {
+		notes = append(notes, "brave_llm: exact_match not applied")
+	}
+	if f.IncludeDomainsMode == "prefer" && len(f.IncludeDomains) > 0 {
+		notes = append(notes, "brave_llm: include_domains_mode=prefer not applied")
+	}
+	if f.MinScore > 0 {
+		notes = append(notes, "brave_llm: min_score not applied")
+	}
+	return notes
+}
+
+func FilterKinds(f Filter) string {
+	var k []string
+	if len(f.IncludeDomains) > 0 || len(f.ExcludeDomains) > 0 {
+		k = append(k, "domains")
+	}
+	if f.TimeRange != "" || f.StartDate != "" || f.EndDate != "" {
+		k = append(k, "time")
+	}
+	if f.Topic != "" && f.Topic != "general" {
+		k = append(k, "topic")
+	}
+	if f.ExactMatch {
+		k = append(k, "exact")
+	}
+	if f.MinScore > 0 {
+		k = append(k, "score")
+	}
+	return strings.Join(k, ",")
+}

internal/search/engine.go

@@ -2,7 +2,6 @@ package search
 
 import (
 	"context"
-	"encoding/json"
 	"log/slog"
 	"sort"
 	"sync"
@@ -90,104 +89,7 @@ func (e *Engine) timeout() time.Duration {
 }
 
 func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, caller, class string) RunResult {
-	start := e.now()
+	return e.RunFilter(ctx, []string{query}, k, doRerank, caller, class, Filter{})
-	k = ClampK(k, e.DefaultK)
-	if e.DefaultK <= 0 {
-		k = ClampK(k, DefaultK)
-	}
-	if class == "" {
-		class = dcache.ClassFleet
-	}
-
-	attempted := e.attemptedNames(class)
-	key := dcache.SearchKey(query, attempted, k, e.now(), class)
-	if b, ok := e.Cache.GetSearch(key); ok {
-		var hits []Hit
-		if json.Unmarshal(b, &hits) == nil {
-			e.mu.Lock()
-			e.CacheHit++
-			e.mu.Unlock()
-			return RunResult{Hits: clampHits(hits, k), Cached: true, Attempted: attempted, SearchMS: e.now().Sub(start).Milliseconds()}
-		}
-	}
-	e.mu.Lock()
-	e.CacheMiss++
-	e.mu.Unlock()
-
-	type one struct {
-		name string
-		out  SourceOut
-		skip bool
-	}
-	outs := make([]one, len(e.Sources))
-	var wg sync.WaitGroup
-	for i, src := range e.Sources {
-		name := src.Name()
-		if name == NameTavily && e.tavilyCapped(class) {
-			outs[i] = one{name: name, out: SourceOut{Warn: "tavily: daily cap"}, skip: true}
-			e.addCapped(NameTavily)
-			continue
-		}
-		wg.Add(1)
-		go func(i int, src Source, name string) {
-			defer wg.Done()
-			sctx, cancel := context.WithTimeout(ctx, e.timeout())
-			defer cancel()
-			outs[i] = one{name: name, out: src.Search(sctx, query, k)}
-		}(i, src, name)
-	}
-	wg.Wait()
-
-	var lists []namedHits
-	var hits []Hit
-	var warns []string
-	var okNames, failNames, att []string
-	degraded := false
-	for _, o := range outs {
-		if o.skip {
-			if o.out.Warn != "" {
-				warns = append(warns, o.out.Warn)
-			}
-			continue
-		}
-		att = append(att, o.name)
-		e.addQuery(caller, o.name)
-		if o.out.Warn != "" {
-			warns = append(warns, o.out.Warn)
-			failNames = append(failNames, o.name)
-			degraded = true
-			e.addErr(o.name, o.out.Warn)
-			continue
-		}
-		okNames = append(okNames, o.name)
-		lists = append(lists, namedHits{name: o.name, hits: o.out.Hits})
-		if o.out.Credits > 0 {
-			e.addCredits(caller, o.name, o.out.Credits)
-			if o.name == NameTavily && e.Cache != nil {
-				e.Cache.AddSearchCredits(tavilyCapSource(class), e.now().UTC().Format("2006-01-02"), o.out.Credits)
-			}
-		}
-	}
-	hits = mergeSearch(e, lists, doRerank)
-
-	var rerankMS int64
-	hits = clampHits(hits, k)
-
-	if !degraded && e.Cache != nil && len(att) > 0 {
-		payload, _ := json.Marshal(hits)
-		ck := dcache.SearchKey(query, att, k, e.now(), class)
-		e.Cache.PutSearch(ck, payload, e.ttl())
-	}
-
-	return RunResult{
-		Hits:          hits,
-		Warnings:      warns,
-		SourcesOK:     okNames,
-		SourcesFailed: failNames,
-		SearchMS:      e.now().Sub(start).Milliseconds(),
-		RerankMS:      rerankMS,
-		Attempted:     att,
-	}
 }
 
 func (e *Engine) attemptedNames(class string) []string {

internal/search/engine_test.go

@@ -14,12 +14,13 @@ type fakeSrc struct {
 	name    string
 	hits    []Hit
 	warn    string
+	notes   []string
 	delay   time.Duration
 	credits int
 }
 
 func (f fakeSrc) Name() string { return f.name }
-func (f fakeSrc) Search(ctx context.Context, query string, k int) SourceOut {
+func (f fakeSrc) Search(ctx context.Context, query string, k int, _ Filter) SourceOut {
 	if f.delay > 0 {
 		select {
 		case <-time.After(f.delay):
@@ -27,7 +28,7 @@ func (f fakeSrc) Search(ctx context.Context, query string, k int) SourceOut {
 			return SourceOut{Warn: f.name + ": timeout"}
 		}
 	}
-	return SourceOut{Hits: f.hits, Warn: f.warn, Credits: f.credits}
+	return SourceOut{Hits: f.hits, Warn: f.warn, Credits: f.credits, Notes: f.notes}
 }
 
 type stubEmb struct {

internal/search/filter.go

@@ -0,0 +1,254 @@
+package search
+
+import (
+	"strings"
+	"time"
+	"unicode/utf8"
+)
+
+const (
+	maxDomainList = 20
+	maxHostLen    = 253
+	searxSiteCap  = 3
+)
+
+var timeRangeLong = map[string]string{
+	"day": "day", "d": "day",
+	"week": "week", "w": "week",
+	"month": "month", "m": "month",
+	"year": "year", "y": "year",
+}
+
+// NormalizeSearch checks query-vs-queries and filter fields. errMsg is
+// "invalid query" or "invalid search"; hint is empty or a field name.
+func NormalizeSearch(query string, queries []string, f Filter) (qs []string, out Filter, errMsg, hint string) {
+	qs, errMsg = normalizeQueries(query, queries)
+	if errMsg != "" {
+		return nil, Filter{}, errMsg, ""
+	}
+	out, hint = normalizeFilter(f)
+	if hint != "" {
+		return nil, Filter{}, "invalid search", hint
+	}
+	if out.ExactMatch {
+		for _, q := range qs {
+			if !hasQuotedPhrase(q) {
+				return nil, Filter{}, "invalid search", "exact_match"
+			}
+		}
+	}
+	return qs, out, "", ""
+}
+
+func normalizeQueries(query string, queries []string) ([]string, string) {
+	q := strings.TrimSpace(query)
+	qs := make([]string, 0, len(queries))
+	for _, s := range queries {
+		qs = append(qs, strings.TrimSpace(s))
+	}
+	hasQ := q != ""
+	hasQS := len(qs) > 0
+	if hasQ && hasQS {
+		return nil, "invalid query"
+	}
+	if !hasQ && !hasQS {
+		return nil, "invalid query"
+	}
+	if hasQ {
+		qs = []string{q}
+	}
+	if len(qs) < 1 || len(qs) > 4 {
+		return nil, "invalid query"
+	}
+	for _, s := range qs {
+		if s == "" || utf8.RuneCountInString(s) > MaxQueryRunes {
+			return nil, "invalid query"
+		}
+	}
+	return qs, ""
+}
+
+func normalizeFilter(f Filter) (Filter, string) {
+	inc, hint := normalizeDomains(f.IncludeDomains, "include_domains")
+	if hint != "" {
+		return Filter{}, hint
+	}
+	exc, hint := normalizeDomains(f.ExcludeDomains, "exclude_domains")
+	if hint != "" {
+		return Filter{}, hint
+	}
+	mode := strings.TrimSpace(f.IncludeDomainsMode)
+	switch mode {
+	case "", "restrict", "prefer":
+	default:
+		return Filter{}, "include_domains_mode"
+	}
+	if mode != "" && len(inc) == 0 {
+		return Filter{}, "include_domains_mode"
+	}
+	if len(inc) > 0 && mode == "" {
+		mode = "restrict"
+	}
+
+	tr := strings.TrimSpace(f.TimeRange)
+	if tr != "" {
+		long, ok := timeRangeLong[tr]
+		if !ok {
+			return Filter{}, "time_range"
+		}
+		tr = long
+	}
+	start := strings.TrimSpace(f.StartDate)
+	end := strings.TrimSpace(f.EndDate)
+	if start != "" {
+		if !validDay(start) {
+			return Filter{}, "start_date"
+		}
+	}
+	if end != "" {
+		if !validDay(end) {
+			return Filter{}, "end_date"
+		}
+	}
+	if (start != "" || end != "") && tr != "" {
+		return Filter{}, "time_range"
+	}
+	if start != "" && end != "" {
+		st, _ := time.Parse("2006-01-02", start)
+		en, _ := time.Parse("2006-01-02", end)
+		if en.Before(st) {
+			return Filter{}, "end_date"
+		}
+	}
+
+	topic := strings.TrimSpace(f.Topic)
+	switch topic {
+	case "", "general", "news", "finance":
+	default:
+		return Filter{}, "topic"
+	}
+
+	ms := f.MinScore
+	if ms < 0 || ms > 1 {
+		return Filter{}, "min_score"
+	}
+
+	return Filter{
+		IncludeDomains:     inc,
+		ExcludeDomains:     exc,
+		IncludeDomainsMode: mode,
+		TimeRange:          tr,
+		StartDate:          start,
+		EndDate:            end,
+		Topic:              topic,
+		ExactMatch:         f.ExactMatch,
+		MinScore:           ms,
+	}, ""
+}
+
+func normalizeDomains(in []string, hint string) ([]string, string) {
+	if len(in) == 0 {
+		return nil, ""
+	}
+	out := make([]string, 0, len(in))
+	seen := map[string]bool{}
+	for _, raw := range in {
+		s := strings.TrimSpace(raw)
+		if s == "" {
+			continue
+		}
+		d, ok := normalizeDomain(s)
+		if !ok {
+			return nil, hint
+		}
+		if seen[d] {
+			continue
+		}
+		seen[d] = true
+		out = append(out, d)
+	}
+	if len(out) == 0 {
+		return nil, ""
+	}
+	if len(out) > maxDomainList {
+		return nil, hint
+	}
+	return out, ""
+}
+
+func normalizeDomain(s string) (string, bool) {
+	s = strings.ToLower(strings.TrimSpace(s))
+	s = strings.TrimSuffix(s, ".")
+	if s == "" || len(s) > maxHostLen {
+		return "", false
+	}
+	if strings.ContainsAny(s, " :/@*?#[]") {
+		return "", false
+	}
+	labels := strings.Split(s, ".")
+	if len(labels) == 0 {
+		return "", false
+	}
+	for _, lab := range labels {
+		if !validLabel(lab) {
+			return "", false
+		}
+	}
+	return s, true
+}
+
+func validLabel(lab string) bool {
+	if lab == "" || len(lab) > 63 {
+		return false
+	}
+	if lab[0] == '-' || lab[len(lab)-1] == '-' {
+		return false
+	}
+	for i := 0; i < len(lab); i++ {
+		c := lab[i]
+		if c >= 'a' && c <= 'z' || c >= '0' && c <= '9' || c == '-' {
+			continue
+		}
+		return false
+	}
+	return true
+}
+
+func validDay(s string) bool {
+	t, err := time.Parse("2006-01-02", s)
+	return err == nil && t.Format("2006-01-02") == s
+}
+
+// hasQuotedPhrase reports a balanced ASCII "…" pair with ≥1 rune inside and no unpaired quotes.
+func hasQuotedPhrase(q string) bool {
+	in := false
+	n := 0
+	pairs := 0
+	for _, r := range q {
+		if r != '"' {
+			if in {
+				n++
+			}
+			continue
+		}
+		if !in {
+			in = true
+			n = 0
+			continue
+		}
+		if n == 0 {
+			return false
+		}
+		in = false
+		pairs++
+	}
+	return !in && pairs >= 1
+}
+
+func QueryRuneLen(qs []string) int {
+	n := 0
+	for _, q := range qs {
+		n += utf8.RuneCountInString(q)
+	}
+	return n
+}

internal/search/p31_sources_test.go

@@ -0,0 +1,159 @@
+package search
+
+import (
+	"context"
+	"encoding/json"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/search/dial"
+)
+
+func TestTavilyFiltersOutbound(t *testing.T) {
+	var gotBody []byte
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotBody, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{"answer":"SECRET","results":[{"url":"https://ex.com/a","title":"A","content":"snip","score":0.9,"published_date":"2026-09-01"}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := Filter{
+		IncludeDomains:     []string{"ex.com"},
+		IncludeDomainsMode: "restrict",
+		TimeRange:          "month",
+		Topic:              "news",
+		ExactMatch:         true,
+	}
+	out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), `docs "phrase"`, 5, f)
+	if out.Warn != "" {
+		t.Fatal(out.Warn)
+	}
+	var req map[string]any
+	if err := json.Unmarshal(gotBody, &req); err != nil {
+		t.Fatal(err)
+	}
+	if req["include_answer"] != false || req["auto_parameters"] != false || req["search_depth"] != "basic" {
+		t.Fatalf("%+v", req)
+	}
+	if req["query"] != `docs "phrase"` {
+		t.Fatalf("query rewritten: %v", req["query"])
+	}
+	if req["exact_match"] != true {
+		t.Fatalf("exact_match %v", req["exact_match"])
+	}
+	if _, ok := req["start_date"]; ok {
+		t.Fatal("time_range with dates")
+	}
+	if req["time_range"] != "month" || req["topic"] != "news" {
+		t.Fatalf("%+v", req)
+	}
+	if req["include_published_date"] != true {
+		t.Fatal("want include_published_date")
+	}
+	if _, ok := req["include_domains_mode"]; !ok {
+		t.Fatal("mode missing")
+	}
+	if strings.Contains(string(gotBody), "extract") || strings.Contains(string(gotBody), "/crawl") {
+		t.Fatal(string(gotBody))
+	}
+	if len(out.Hits) != 1 || out.Hits[0].PublishedDate != "2026-09-01" {
+		t.Fatalf("%+v", out.Hits)
+	}
+}
+
+func TestTavilyModeOmittedWithoutInclude(t *testing.T) {
+	var gotBody []byte
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotBody, _ = io.ReadAll(r.Body)
+		_, _ = w.Write([]byte(`{"results":[{"url":"https://ex.com/a","title":"A","content":"s","score":0.2}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{IncludeDomainsMode: "prefer"})
+	if out.Warn != "" {
+		t.Fatal(out.Warn)
+	}
+	var req map[string]any
+	_ = json.Unmarshal(gotBody, &req)
+	if _, ok := req["include_domains_mode"]; ok {
+		t.Fatalf("mode without include: %+v", req)
+	}
+	if out.Hits[0].PublishedDate != "" {
+		t.Fatal("missing date must omit")
+	}
+}
+
+func TestSearxNGSiteAndParams(t *testing.T) {
+	var gotQ, gotTR, gotCat string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotQ = r.URL.Query().Get("q")
+		gotTR = r.URL.Query().Get("time_range")
+		gotCat = r.URL.Query().Get("categories")
+		if strings.Contains(gotQ, "!news") || strings.Contains(gotQ, "!month") {
+			t.Errorf("bang in q: %s", gotQ)
+		}
+		_, _ = w.Write([]byte(`{"results":[{"url":"https://a.example/","title":"a","content":"aa","score":1,"publishedDate":"Mon, 01 Sep 2026 00:00:00 GMT"}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := Filter{IncludeDomains: []string{"a.example"}, TimeRange: "month", Topic: "news", ExactMatch: true}
+	out := (&SearxNG{Client: c}).Search(context.Background(), `"phrase" here`, 8, f)
+	if !strings.HasPrefix(gotQ, "site:a.example ") {
+		t.Fatalf("q %q", gotQ)
+	}
+	if !strings.Contains(gotQ, `"phrase" here`) || strings.Count(gotQ, `"phrase"`) != 1 {
+		t.Fatalf("wrapped q %q", gotQ)
+	}
+	if gotTR != "month" || gotCat != "news" {
+		t.Fatalf("params tr=%q cat=%q", gotTR, gotCat)
+	}
+	if out.Hits[0].PublishedDate == "" {
+		t.Fatal("publishedDate")
+	}
+}
+
+func TestBraveNotesAndUnchangedURL(t *testing.T) {
+	var gotURL string
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		gotURL = r.URL.String()
+		_, _ = w.Write([]byte(`{"sources":[{"url":"https://off.example/x","title":"S","snippet":"hello"}]}`))
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	f := Filter{TimeRange: "month", Topic: "news", ExactMatch: true, MinScore: 0.5, IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"}
+	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), `"phrase"`, 5, f)
+	if !strings.Contains(gotURL, "q=") {
+		t.Fatalf("url %s", gotURL)
+	}
+	if strings.Contains(gotURL, "time_range") || strings.Contains(gotURL, "site:") {
+		t.Fatalf("brave grew params: %s", gotURL)
+	}
+	note := strings.Join(out.Notes, " ")
+	for _, want := range []string{
+		"brave_llm: time_range not applied",
+		"brave_llm: topic not applied",
+		"brave_llm: exact_match not applied",
+		"brave_llm: include_domains_mode=prefer not applied",
+		"brave_llm: min_score not applied",
+	} {
+		if !strings.Contains(note, want) {
+			t.Fatalf("missing %q in %v", want, out.Notes)
+		}
+	}
+}

internal/search/p31_test.go

@@ -0,0 +1,269 @@
+package search
+
+import (
+	"context"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"git.wntrmute.dev/kyle/sift/internal/dcache"
+)
+
+type qSrc struct {
+	name    string
+	byQuery map[string][]Hit
+	notes   []string
+	credits int
+}
+
+func (q qSrc) Name() string { return q.name }
+func (q qSrc) Search(_ context.Context, query string, _ int, _ Filter) SourceOut {
+	return SourceOut{Hits: q.byQuery[query], Notes: q.notes, Credits: q.credits}
+}
+
+func TestNotesDoNotFailSource(t *testing.T) {
+	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	e := &Engine{
+		Sources: []Source{fakeSrc{
+			name:  NameSearxNG,
+			hits:  []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}},
+			notes: []string{"searxng: min_score not applied"},
+		}},
+		DefaultK: 8, Cache: db, RerankOn: false,
+	}
+	r1 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
+	if len(r1.Hits) != 1 {
+		t.Fatalf("hits %+v", r1.Hits)
+	}
+	if len(r1.SourcesOK) != 1 || r1.SourcesOK[0] != NameSearxNG {
+		t.Fatalf("ok %v fail %v", r1.SourcesOK, r1.SourcesFailed)
+	}
+	if r1.Cached {
+		t.Fatal("first miss")
+	}
+	joined := strings.Join(r1.Warnings, " ")
+	if !strings.Contains(joined, "min_score not applied") {
+		t.Fatalf("warnings %v", r1.Warnings)
+	}
+	r2 := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
+	if !r2.Cached {
+		t.Fatal("notes must still cache")
+	}
+}
+
+func TestWarnStillDisablesCache(t *testing.T) {
+	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	e := &Engine{
+		Sources: []Source{
+			fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}},
+			fakeSrc{name: NameBraveLLM, warn: "brave_llm: 500"},
+		},
+		DefaultK: 8, Cache: db, RerankOn: false,
+	}
+	e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet)
+	r := e.Run(context.Background(), "q", 8, false, "titan", dcache.ClassFleet)
+	if r.Cached {
+		t.Fatal("warn cached")
+	}
+}
+
+func TestMinScoreTavilyOnly(t *testing.T) {
+	e := &Engine{
+		Sources: []Source{
+			fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://s.example/a", Score: 0.01, Source: NameSearxNG}}, notes: []string{"searxng: min_score not applied"}},
+			fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://b.example/a", Source: NameBraveLLM}}, notes: []string{"brave_llm: min_score not applied"}},
+			fakeSrc{name: NameTavily, hits: []Hit{
+				{URL: "https://t.example/low", Score: 0.4, Source: NameTavily},
+				{URL: "https://t.example/high", Score: 0.9, Source: NameTavily},
+			}},
+		},
+		DefaultK: 8, RerankOn: false,
+	}
+	r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{MinScore: 0.5})
+	var urls []string
+	for _, h := range r.Hits {
+		urls = append(urls, h.URL)
+		if h.URL == "https://t.example/low" {
+			t.Fatal("tavily 0.4 kept")
+		}
+	}
+	joined := strings.Join(urls, " ")
+	if !strings.Contains(joined, "s.example") || !strings.Contains(joined, "b.example") || !strings.Contains(joined, "t.example/high") {
+		t.Fatalf("hits %v", urls)
+	}
+	w := strings.Join(r.Warnings, " ")
+	if !strings.Contains(w, "searxng: min_score not applied") || !strings.Contains(w, "brave_llm: min_score not applied") {
+		t.Fatalf("warnings %v", r.Warnings)
+	}
+}
+
+func TestDomainPostFilter(t *testing.T) {
+	hits := []Hit{
+		{URL: "https://openai.com/blog", Source: NameBraveLLM},
+		{URL: "https://sub.openai.com/x", Source: NameSearxNG},
+		{URL: "https://pinterest.com/pin", Source: NameBraveLLM},
+		{URL: "https://example.com/a", Source: NameSearxNG},
+	}
+	f := Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}
+	got := filterDomains(hits, f)
+	if len(got) != 2 {
+		t.Fatalf("%+v", got)
+	}
+	f.IncludeDomainsMode = "prefer"
+	got = filterDomains(hits, f)
+	if len(got) != 4 {
+		t.Fatalf("prefer %+v", got)
+	}
+	got = filterDomains(hits, Filter{ExcludeDomains: []string{"pinterest.com"}})
+	if len(got) != 3 {
+		t.Fatalf("exclude %+v", got)
+	}
+	e := &Engine{
+		Sources:  []Source{fakeSrc{name: NameBraveLLM, hits: []Hit{{URL: "https://off.example/x", Source: NameBraveLLM}}}},
+		DefaultK: 8, RerankOn: false,
+	}
+	r := e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"})
+	if len(r.Hits) != 0 {
+		t.Fatalf("brave off-domain kept %+v", r.Hits)
+	}
+	r = e.RunFilter(context.Background(), []string{"q"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "prefer"})
+	if len(r.Hits) != 1 {
+		t.Fatalf("prefer dropped brave %+v", r.Hits)
+	}
+}
+
+func TestNormalizeDomainOR(t *testing.T) {
+	_, _, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"a.com OR b.com"}})
+	if msg != "invalid search" || hint != "include_domains" {
+		t.Fatalf("%q %q", msg, hint)
+	}
+	_, out, msg, hint := NormalizeSearch("q", nil, Filter{IncludeDomains: []string{"oracle.com"}})
+	if msg != "" || hint != "" || len(out.IncludeDomains) != 1 || out.IncludeDomains[0] != "oracle.com" {
+		t.Fatalf("%q %q %+v", msg, hint, out)
+	}
+}
+
+func TestMultiQueryRRF(t *testing.T) {
+	q1 := make([]Hit, 0, 3)
+	for _, u := range []string{"https://a.example/1", "https://a.example/2", "https://a.example/3"} {
+		q1 = append(q1, Hit{URL: u, Source: NameSearxNG})
+	}
+	e := &Engine{
+		Sources: []Source{qSrc{
+			name: NameSearxNG,
+			byQuery: map[string][]Hit{
+				"one": q1,
+				"two": {{URL: "https://b.example/only", Source: NameSearxNG}},
+			},
+		}},
+		DefaultK: 3, RerankOn: true,
+	}
+	r := e.RunFilter(context.Background(), []string{"one", "two"}, 3, true, "titan", dcache.ClassFleet, Filter{})
+	found := false
+	for _, h := range r.Hits {
+		if h.URL == "https://b.example/only" {
+			found = true
+		}
+	}
+	if !found {
+		t.Fatalf("q2 hit missing: %+v", r.Hits)
+	}
+}
+
+func TestCapMidBatchNoCache(t *testing.T) {
+	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	e := &Engine{
+		Sources: []Source{qSrc{
+			name: NameTavily,
+			byQuery: map[string][]Hit{
+				"one": {{URL: "https://t.example/1", Source: NameTavily}},
+				"two": {{URL: "https://t.example/2", Source: NameTavily}},
+			},
+			credits: 1,
+		}},
+		DefaultK: 8, Cache: db, TavilyCap: 1, RerankOn: false,
+	}
+	r := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{})
+	n := 0
+	for _, w := range r.Warnings {
+		if strings.Contains(w, "daily cap") {
+			n++
+		}
+	}
+	if n != 1 {
+		t.Fatalf("warnings %v", r.Warnings)
+	}
+	r2 := e.RunFilter(context.Background(), []string{"one", "two"}, 8, false, "titan", dcache.ClassFleet, Filter{})
+	if r2.Cached {
+		t.Fatal("mid-batch cap must not cache")
+	}
+}
+
+func TestCacheKeyFilters(t *testing.T) {
+	db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+	now := time.Date(2026, 9, 29, 10, 0, 0, 0, time.UTC)
+	src := fakeSrc{name: NameSearxNG, hits: []Hit{{URL: "https://ok.example/a", Source: NameSearxNG}}}
+	e := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
+	base := Filter{}
+	e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, base)
+	if e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached {
+		t.Fatal("different include_domains must miss")
+	}
+	e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"})
+	if !e.RunFilter(context.Background(), []string{"hello"}, 8, false, "titan", dcache.ClassFleet, Filter{IncludeDomains: []string{"openai.com"}, IncludeDomainsMode: "restrict"}).Cached {
+		t.Fatal("same filters must hit")
+	}
+	e2 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
+	e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{})
+	if !e2.RunFilter(context.Background(), []string{"topicq"}, 8, false, "titan", dcache.ClassFleet, Filter{Topic: "general"}).Cached {
+		t.Fatal("omitted topic and general must share a key")
+	}
+	_, day, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "d"})
+	_, long, _, _ := NormalizeSearch("q", nil, Filter{TimeRange: "day"})
+	if day.TimeRange != long.TimeRange || day.TimeRange != "day" {
+		t.Fatalf("%q %q", day.TimeRange, long.TimeRange)
+	}
+	e3 := &Engine{Sources: []Source{src}, DefaultK: 8, Cache: db, RerankOn: false, Now: func() time.Time { return now }}
+	e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, day)
+	if !e3.RunFilter(context.Background(), []string{"timeq"}, 8, false, "titan", dcache.ClassFleet, long).Cached {
+		t.Fatal("d and day must share a key")
+	}
+}
+
+func TestQuotedPhrase(t *testing.T) {
+	if !hasQuotedPhrase(`docs "phrase" x`) || hasQuotedPhrase(`docs phrase`) || hasQuotedPhrase(`"unclosed`) || hasQuotedPhrase(`""`) {
+		t.Fatal("quoted phrase")
+	}
+}
+
+func TestNormalizeQueries(t *testing.T) {
+	_, _, msg, _ := NormalizeSearch("a", []string{"b"}, Filter{})
+	if msg != "invalid query" {
+		t.Fatalf("both: %q", msg)
+	}
+	qs, _, msg, _ := NormalizeSearch("", []string{"a", "b", "c", "d", "e"}, Filter{})
+	if msg != "invalid query" || qs != nil {
+		t.Fatalf("five: %q", msg)
+	}
+	_, _, msg, hint := NormalizeSearch(`no quotes`, nil, Filter{ExactMatch: true})
+	if msg != "invalid search" || hint != "exact_match" {
+		t.Fatalf("%q %q", msg, hint)
+	}
+}

internal/search/pipeline.go

@@ -0,0 +1,223 @@
+package search
+
+import (
+	"context"
+	"encoding/json"
+	"sync"
+
+	"git.wntrmute.dev/kyle/sift/internal/dcache"
+)
+
+type srcOut struct {
+	name string
+	out  SourceOut
+	skip bool
+}
+
+type queryFan struct {
+	hits      []Hit
+	warns     []string
+	notes     []string
+	okNames   []string
+	failNames []string
+	att       []string
+	degraded  bool
+	tavSkip   bool
+	tavUsed   bool
+}
+
+func (e *Engine) RunFilter(ctx context.Context, queries []string, k int, doRerank bool, caller, class string, f Filter) RunResult {
+	start := e.now()
+	k = ClampK(k, e.DefaultK)
+	if e.DefaultK <= 0 {
+		k = ClampK(k, DefaultK)
+	}
+	if class == "" {
+		class = dcache.ClassFleet
+	}
+	if len(queries) == 0 {
+		queries = []string{""}
+	}
+
+	attempted := e.attemptedNames(class)
+	key := e.searchKey(queries, attempted, k, class, f)
+	if b, ok := e.Cache.GetSearch(key); ok {
+		var hits []Hit
+		if json.Unmarshal(b, &hits) == nil {
+			e.mu.Lock()
+			e.CacheHit++
+			e.mu.Unlock()
+			return RunResult{Hits: clampHits(hits, k), Cached: true, Attempted: attempted, SearchMS: e.now().Sub(start).Milliseconds()}
+		}
+	}
+	e.mu.Lock()
+	e.CacheMiss++
+	e.mu.Unlock()
+
+	var lists [][]Hit
+	var warns, notes []string
+	okSeen, failSeen := map[string]bool{}, map[string]bool{}
+	var okNames, failNames []string
+	degraded := false
+	capMid := false
+	sawTavily := false
+	var lastAtt []string
+
+	for _, q := range queries {
+		one := e.fanInQuery(ctx, q, k, doRerank, caller, class, f)
+		if one.tavUsed {
+			sawTavily = true
+		}
+		if one.tavSkip && sawTavily {
+			capMid = true
+		}
+		if one.degraded {
+			degraded = true
+		}
+		warns = append(warns, one.warns...)
+		notes = append(notes, one.notes...)
+		for _, n := range one.okNames {
+			if !okSeen[n] {
+				okSeen[n] = true
+				okNames = append(okNames, n)
+			}
+		}
+		for _, n := range one.failNames {
+			if !failSeen[n] {
+				failSeen[n] = true
+				failNames = append(failNames, n)
+			}
+		}
+		if len(one.att) > 0 {
+			lastAtt = one.att
+		}
+		lists = append(lists, one.hits)
+	}
+
+	var hits []Hit
+	if len(lists) <= 1 {
+		if len(lists) == 1 {
+			hits = lists[0]
+		}
+	} else {
+		hits = rrfMerge(lists)
+		hits = Dedupe(hits)
+	}
+	hits = clampHits(hits, k)
+
+	warns = append(warns, notes...)
+	warns = dedupeStrings(warns)
+
+	att := lastAtt
+	if len(att) == 0 {
+		att = attempted
+	}
+	if !degraded && !capMid && e.Cache != nil && len(att) > 0 {
+		payload, _ := json.Marshal(hits)
+		ck := e.searchKey(queries, att, k, class, f)
+		e.Cache.PutSearch(ck, payload, e.ttl())
+	}
+
+	return RunResult{
+		Hits:          hits,
+		Warnings:      warns,
+		SourcesOK:     okNames,
+		SourcesFailed: failNames,
+		SearchMS:      e.now().Sub(start).Milliseconds(),
+		Attempted:     att,
+	}
+}
+
+func (e *Engine) searchKey(queries []string, sources []string, k int, class string, f Filter) string {
+	return dcache.EncodeSearchKey(dcache.SearchKeyInput{
+		Queries:            queries,
+		Sources:            sources,
+		K:                  k,
+		Hour:               e.now(),
+		CallerClass:        class,
+		IncludeDomains:     f.IncludeDomains,
+		ExcludeDomains:     f.ExcludeDomains,
+		IncludeDomainsMode: f.IncludeDomainsMode,
+		TimeRange:          f.TimeRange,
+		StartDate:          f.StartDate,
+		EndDate:            f.EndDate,
+		Topic:              f.Topic,
+		ExactMatch:         f.ExactMatch,
+		MinScore:           f.MinScore,
+	})
+}
+
+func (e *Engine) fanInQuery(ctx context.Context, query string, k int, doRerank bool, caller, class string, f Filter) queryFan {
+	outs := make([]srcOut, len(e.Sources))
+	var wg sync.WaitGroup
+	tavSkip := false
+	for i, src := range e.Sources {
+		name := src.Name()
+		if name == NameTavily && e.tavilyCapped(class) {
+			outs[i] = srcOut{name: name, out: SourceOut{Warn: "tavily: daily cap"}, skip: true}
+			e.addCapped(NameTavily)
+			tavSkip = true
+			continue
+		}
+		wg.Add(1)
+		go func(i int, src Source, name string) {
+			defer wg.Done()
+			sctx, cancel := context.WithTimeout(ctx, e.timeout())
+			defer cancel()
+			outs[i] = srcOut{name: name, out: src.Search(sctx, query, k, f)}
+		}(i, src, name)
+	}
+	wg.Wait()
+
+	var lists []namedHits
+	var warns, notes []string
+	var okNames, failNames, att []string
+	degraded := false
+	tavUsed := false
+	for _, o := range outs {
+		if o.skip {
+			if o.out.Warn != "" {
+				warns = append(warns, o.out.Warn)
+			}
+			continue
+		}
+		att = append(att, o.name)
+		e.addQuery(caller, o.name)
+		if o.name == NameTavily {
+			tavUsed = true
+		}
+		if o.out.Warn != "" {
+			warns = append(warns, o.out.Warn)
+			failNames = append(failNames, o.name)
+			degraded = true
+			e.addErr(o.name, o.out.Warn)
+			continue
+		}
+		okNames = append(okNames, o.name)
+		notes = append(notes, o.out.Notes...)
+		hits := o.out.Hits
+		if o.name == NameTavily && f.MinScore > 0 {
+			hits = filterMinScore(hits, f.MinScore)
+		}
+		lists = append(lists, namedHits{name: o.name, hits: hits})
+		if o.out.Credits > 0 {
+			e.addCredits(caller, o.name, o.out.Credits)
+			if o.name == NameTavily && e.Cache != nil {
+				e.Cache.AddSearchCredits(tavilyCapSource(class), e.now().UTC().Format("2006-01-02"), o.out.Credits)
+			}
+		}
+	}
+	merged := mergeSearch(e, lists, doRerank)
+	merged = filterDomains(merged, f)
+	return queryFan{
+		hits:      merged,
+		warns:     warns,
+		notes:     notes,
+		okNames:   okNames,
+		failNames: failNames,
+		att:       att,
+		degraded:  degraded,
+		tavSkip:   tavSkip,
+		tavUsed:   tavUsed,
+	}
+}

internal/search/rank.go

@@ -105,6 +105,7 @@ func rrfMerge(lists [][]Hit) []Hit {
 			sc := 1.0 / float64(rrfK+r+1)
 			if a, ok := seen[c]; ok {
 				a.s += sc
+				keepDate := firstDate(a.h.PublishedDate, h.PublishedDate)
 				if len(h.Snippet) > len(a.h.Snippet) {
 					src := a.h.Source
 					keep := a.h.URL
@@ -112,6 +113,7 @@ func rrfMerge(lists [][]Hit) []Hit {
 					a.h.Source = src
 					a.h.URL = keep
 				}
+				a.h.PublishedDate = keepDate
 				continue
 			}
 			cp := h

internal/search/sources.go

@@ -59,7 +59,7 @@ func (s *SearxNG) Name() string { return NameSearxNG }
 func (s *SearxNG) Ready() bool  { return s != nil && s.Client.Ready() }
 func (s *BraveLLM) Ready() bool { return s != nil && s.Client.Ready() }
 
-func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
+func (s *SearxNG) Search(ctx context.Context, query string, k int, f Filter) SourceOut {
 	if s == nil || s.Client == nil {
 		return SourceOut{Warn: NameSearxNG + ": error"}
 	}
@@ -71,10 +71,17 @@ func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
 	if err != nil {
 		return SourceOut{Warn: warnErr(NameSearxNG, err)}
 	}
+	qtext, notes := searxQueryAndNotes(query, f)
 	q := u.Query()
-	q.Set("q", query)
+	q.Set("q", qtext)
 	q.Set("format", "json")
 	q.Set("pageno", "1")
+	if f.TimeRange != "" {
+		q.Set("time_range", f.TimeRange)
+	}
+	if f.Topic == "news" {
+		q.Set("categories", "news")
+	}
 	u.RawQuery = q.Encode()
 	req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
 	if err != nil {
@@ -92,10 +99,11 @@ func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
 	}
 	var env struct {
 		Results []struct {
-			URL     string  `json:"url"`
+			URL           string  `json:"url"`
-			Title   string  `json:"title"`
+			Title         string  `json:"title"`
-			Content string  `json:"content"`
+			Content       string  `json:"content"`
-			Score   float64 `json:"score"`
+			Score         float64 `json:"score"`
+			PublishedDate string  `json:"publishedDate"`
 		} `json:"results"`
 	}
 	if err := readJSON(resp, &env); err != nil {
@@ -107,9 +115,9 @@ func (s *SearxNG) Search(ctx context.Context, query string, k int) SourceOut {
 		if r.URL == "" {
 			continue
 		}
-		hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameSearxNG})
+		hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameSearxNG, PublishedDate: r.PublishedDate})
 	}
-	return SourceOut{Hits: hits}
+	return SourceOut{Hits: hits, Notes: notes}
 }
 
 // BraveLLM is GET {url}?q= with X-Subscription-Token.
@@ -120,7 +128,7 @@ type BraveLLM struct {
 
 func (s *BraveLLM) Name() string { return NameBraveLLM }
 
-func (s *BraveLLM) Search(ctx context.Context, query string, k int) SourceOut {
+func (s *BraveLLM) Search(ctx context.Context, query string, k int, f Filter) SourceOut {
 	if s == nil || s.Client == nil {
 		return SourceOut{Warn: NameBraveLLM + ": error"}
 	}
@@ -159,7 +167,7 @@ func (s *BraveLLM) Search(ctx context.Context, query string, k int) SourceOut {
 	if len(hits) >= 0 {
 		credits = 1
 	}
-	return SourceOut{Hits: hits, Credits: credits}
+	return SourceOut{Hits: hits, Credits: credits, Notes: braveNotes(f)}
 }
 
 func mapBrave(raw []byte) []Hit {

internal/search/sources_test.go

@@ -31,17 +31,20 @@ func TestTavilyOutboundAndDropsAnswer(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	out := (&Tavily{Client: c, Key: "test-key"}).Search(context.Background(), "q", 5)
+	out := (&Tavily{Client: c, Key: "test-key"}).Search(context.Background(), "q", 5, Filter{})
 	if out.Warn != "" {
 		t.Fatal(out.Warn)
 	}
-	var req tavilyReq
+	var req map[string]any
 	if err := json.Unmarshal(gotBody, &req); err != nil {
 		t.Fatal(err)
 	}
-	if req.IncludeAnswer || req.IncludeRawContent || req.IncludeImages || req.SearchDepth != "basic" {
+	if req["include_answer"] != false || req["include_raw_content"] != false || req["include_images"] != false || req["search_depth"] != "basic" {
 		t.Fatalf("%+v", req)
 	}
+	if req["auto_parameters"] != false {
+		t.Fatalf("auto_parameters %+v", req["auto_parameters"])
+	}
 	if strings.Contains(string(gotBody), "extract") || strings.Contains(string(gotBody), "crawl") {
 		t.Fatal(string(gotBody))
 	}
@@ -70,7 +73,7 @@ func TestSearxNGMapsAndSorts(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	out := (&SearxNG{Client: c}).Search(context.Background(), "q", 8)
+	out := (&SearxNG{Client: c}).Search(context.Background(), "q", 8, Filter{})
 	if out.Warn != "" || len(out.Hits) != 2 || out.Hits[0].Title != "a" {
 		t.Fatalf("%+v", out)
 	}
@@ -88,7 +91,7 @@ func TestBraveMapsSources(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
+	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{})
 	if len(out.Hits) != 1 || out.Hits[0].URL != "https://s.example/x" {
 		t.Fatalf("%+v", out)
 	}
@@ -110,7 +113,7 @@ func TestBraveMapsGroundingGeneric(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
+	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{})
 	if len(out.Hits) != 3 {
 		t.Fatalf("hits %d %+v", len(out.Hits), out.Hits)
 	}
@@ -139,7 +142,7 @@ func TestTavilyBadJSONIsError(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
+	out := (&Tavily{Client: c, Key: "k"}).Search(context.Background(), "q", 5, Filter{})
 	if out.Warn != "tavily: error" {
 		t.Fatalf("warn %q", out.Warn)
 	}

internal/search/tavily.go

@@ -19,16 +19,7 @@ type Tavily struct {
 func (s *Tavily) Name() string { return NameTavily }
 func (s *Tavily) Ready() bool  { return s != nil && s.Client.Ready() }
 
-type tavilyReq struct {
+func (s *Tavily) Search(ctx context.Context, query string, k int, f Filter) SourceOut {
-	Query             string `json:"query"`
-	MaxResults        int    `json:"max_results"`
-	IncludeAnswer     bool   `json:"include_answer"`
-	IncludeRawContent bool   `json:"include_raw_content"`
-	IncludeImages     bool   `json:"include_images"`
-	SearchDepth       string `json:"search_depth"`
-}
-
-func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
 	if s == nil || s.Client == nil {
 		return SourceOut{Warn: NameTavily + ": error"}
 	}
@@ -41,14 +32,7 @@ func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
 	if k > 20 {
 		k = 20
 	}
-	payload, _ := json.Marshal(tavilyReq{
+	payload, _ := json.Marshal(tavilyBody(query, k, f))
-		Query:             query,
-		MaxResults:        k,
-		IncludeAnswer:     false,
-		IncludeRawContent: false,
-		IncludeImages:     false,
-		SearchDepth:       "basic",
-	})
 	req, err := http.NewRequestWithContext(ctx, http.MethodPost, s.Client.URL.String(), bytes.NewReader(payload))
 	if err != nil {
 		return SourceOut{Warn: warnErr(NameTavily, err)}
@@ -76,14 +60,58 @@ func (s *Tavily) Search(ctx context.Context, query string, k int) SourceOut {
 	return SourceOut{Hits: hits, Credits: credits}
 }
 
+func tavilyBody(query string, k int, f Filter) map[string]any {
+	body := map[string]any{
+		"query":               query,
+		"max_results":         k,
+		"include_answer":      false,
+		"include_raw_content": false,
+		"include_images":      false,
+		"search_depth":        "basic",
+		"auto_parameters":     false,
+	}
+	if len(f.IncludeDomains) > 0 {
+		body["include_domains"] = f.IncludeDomains
+		mode := f.IncludeDomainsMode
+		if mode == "" {
+			mode = "restrict"
+		}
+		body["include_domains_mode"] = mode
+	}
+	if len(f.ExcludeDomains) > 0 {
+		body["exclude_domains"] = f.ExcludeDomains
+	}
+	if f.TimeRange != "" {
+		body["time_range"] = f.TimeRange
+	} else {
+		if f.StartDate != "" {
+			body["start_date"] = f.StartDate
+		}
+		if f.EndDate != "" {
+			body["end_date"] = f.EndDate
+		}
+	}
+	if f.Topic != "" && f.Topic != "general" {
+		body["topic"] = f.Topic
+	}
+	if f.ExactMatch {
+		body["exact_match"] = true
+	}
+	if f.Topic == "news" || f.TimeRange != "" || f.StartDate != "" || f.EndDate != "" {
+		body["include_published_date"] = true
+	}
+	return body
+}
+
 func mapTavily(raw []byte) ([]Hit, int, error) {
 	var env struct {
 		Answer  string `json:"answer"`
 		Results []struct {
-			URL     string  `json:"url"`
+			URL           string  `json:"url"`
-			Title   string  `json:"title"`
+			Title         string  `json:"title"`
-			Content string  `json:"content"`
+			Content       string  `json:"content"`
-			Score   float64 `json:"score"`
+			Score         float64 `json:"score"`
+			PublishedDate string  `json:"published_date"`
 		} `json:"results"`
 		Usage struct {
 			Credits int `json:"credits"`
@@ -98,7 +126,10 @@ func mapTavily(raw []byte) ([]Hit, int, error) {
 		if r.URL == "" {
 			continue
 		}
-		hits = append(hits, Hit{URL: r.URL, Title: r.Title, Snippet: r.Content, Score: r.Score, Source: NameTavily})
+		hits = append(hits, Hit{
+			URL: r.URL, Title: r.Title, Snippet: r.Content,
+			Score: r.Score, Source: NameTavily, PublishedDate: r.PublishedDate,
+		})
 	}
 	credits := env.Usage.Credits
 	if credits <= 0 {

internal/search/types.go

@@ -25,22 +25,37 @@ var sourceRank = map[string]int{
 
 // Hit is one search result after mapping. No page body, no vendor answers.
 type Hit struct {
-	URL     string  `json:"url"`
+	URL           string  `json:"url"`
-	Title   string  `json:"title"`
+	Title         string  `json:"title"`
-	Snippet string  `json:"snippet"`
+	Snippet       string  `json:"snippet"`
-	Score   float64 `json:"score"`
+	Score         float64 `json:"score"`
-	Source  string  `json:"source"`
+	Source        string  `json:"source"`
+	PublishedDate string  `json:"published_date,omitempty"`
+}
+
+// Filter is native search knobs. The zero value is P3 behaviour.
+type Filter struct {
+	IncludeDomains     []string
+	ExcludeDomains     []string
+	IncludeDomainsMode string // restrict | prefer; empty means restrict when include is set
+	TimeRange          string // day|week|month|year
+	StartDate          string
+	EndDate            string
+	Topic              string // general|news|finance; empty means general
+	ExactMatch         bool
+	MinScore           float64
 }
 
 // Source is one upstream. Config-fixed endpoint; never a request URL.
 type Source interface {
 	Name() string
-	Search(ctx context.Context, query string, k int) SourceOut
+	Search(ctx context.Context, query string, k int, f Filter) SourceOut
 }
 
 type SourceOut struct {
 	Hits    []Hit
-	Warn    string
+	Warn    string   // failure only (timeout, HTTP error, pin)
+	Notes   []string // honesty: filters not applied, recall loss
 	Credits int
 }
 
@@ -80,6 +95,7 @@ func Dedupe(in []Hit) []Hit {
 			continue
 		}
 		if s, ok := seen[canon]; ok {
+			keepDate := firstDate(s.h.PublishedDate, h.PublishedDate)
 			if len(h.Snippet) > len(s.h.Snippet) {
 				src := s.h.Source
 				keep := s.h.URL
@@ -91,6 +107,7 @@ func Dedupe(in []Hit) []Hit {
 					s.h.Score = sc
 				}
 			}
+			s.h.PublishedDate = keepDate
 			continue
 		}
 		cp := h
@@ -104,6 +121,13 @@ func Dedupe(in []Hit) []Hit {
 	return out
 }
 
+func firstDate(a, b string) string {
+	if a != "" {
+		return a
+	}
+	return b
+}
+
 func sortBySourceOrder(hits []Hit) {
 	sort.SliceStable(hits, func(i, j int) bool {
 		ri, rj := sourceRank[hits[i].Source], sourceRank[hits[j].Source]

skill/SKILL.md

@@ -21,7 +21,9 @@ markdown, JSON → JSON), optionally **digested** for your question by a local m
 **what the fleet already knows** attached from Honcho — so you read 1k tokens, not 20k.
 
 `sift search` is the first step for web search. Use `web_search` only if sift is
-unreachable. Then `sift fetch` the URLs you need. Restricted callers use the same path.
+unreachable. Hermes `web_search` hits `/v2/search` and does **not** get native
+filters. Then `sift fetch` / `sift batch` the URLs you need — do not fetch every
+hit. Restricted callers use the same path.
 
 ## Rule: sift is the explicit first step
 
@@ -38,8 +40,9 @@ Whenever you need the content of a URL:
 ## Commands
 
 ```bash
-# Web search (native). Then sift fetch the URLs you need.
+# Web search (native). Then score/filter, then batch-fetch — not every hit.
-sift search "HBM wafer share 2026" --k 8
+sift search "HBM wafer share 2026" --k 8 --json
+sift search "OpenAI" --include-domain openai.com --topic news --time-range month --json
 
 # Page or document → cleaned content. Add --question to get a focused digest.
 sift fetch "https://example.com/article" --question "what does it say about HBM wafer share?"
@@ -55,6 +58,31 @@ Useful flags: `--mode digest|clean|raw` (default `digest` when `--question` is g
 else `clean`), `--no-context` (skip Honcho lookup), `--recover` (server-side archive ladder
 for blocked pages), `--json` (full envelope: content, source, context, sizes, warnings).
 
+## Search
+
+```bash
+sift search "<query>" [--k n] [--json]
+            [--include-domain host]... [--exclude-domain host]...
+            [--domains-mode restrict|prefer]
+            [--time-range day|week|month|year]
+            [--start-date YYYY-MM-DD] [--end-date YYYY-MM-DD]
+            [--topic general|news|finance]
+            [--exact-match]
+            [--min-score 0.5]
+            [--queries-file path]
+```
+
+Product News Tracker shape (official domain + news + last month):
+
+```bash
+sift search "product launch" --include-domain openai.com --topic news --time-range month --json
+```
+
+**Recipe:** after `sift search --json`, do not `sift fetch` every URL. Drop hits below a
+Tavily score you care about (`--min-score`, Tavily's 0–1 scale) or take top N, then
+`sift batch submit`. `web_search` is only the fallback when sift is unreachable, and it
+still hits `/v2/search` with no filters.
+
 ## Batch
 
 When you will read the content of several URLs later (a shortlist to compare), submit them once