feat(search): P3.1 filters and multi-query merge
Native include/exclude domains, time_range, topic, exact_match, min_score, and up to 4 queries fused with RRF. Honesty notes stay off the Warn/fail path. Tavily keeps include_answer false and sets auto_parameters false. Cache key is canonical JSON. CLI flags and skill recipe included.
This commit is contained in:
@@ -11,8 +11,6 @@ import (
|
||||
"log/slog"
|
||||
"math"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
@@ -181,17 +179,6 @@ func QuestionEmbedKey(question, embedModel, callerClass string) string {
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func SearchKey(query string, sources []string, k int, hour time.Time, callerClass string) string {
|
||||
if callerClass == "" {
|
||||
callerClass = ClassFleet
|
||||
}
|
||||
src := append([]string(nil), sources...)
|
||||
sort.Strings(src)
|
||||
bucket := hour.UTC().Format("2006-01-02T15")
|
||||
sum := sha256.Sum256([]byte(NormalizeQuestion(query) + "\x00" + strings.Join(src, ",") + "\x00" + strconv.Itoa(k) + "\x00" + bucket + "\x00" + callerClass))
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
type DigestHit struct {
|
||||
Text string
|
||||
ChunksUsed int
|
||||
|
||||
@@ -109,6 +109,12 @@ func TestSearchKeyPartitions(t *testing.T) {
|
||||
if SearchKey("hello", []string{"searxng", "tavily"}, 8, now.Add(time.Hour), ClassFleet) == a {
|
||||
t.Fatal("hour")
|
||||
}
|
||||
if EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet}) == EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet, IncludeDomains: []string{"a.com"}}) {
|
||||
t.Fatal("include_domains")
|
||||
}
|
||||
if EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet}) != EncodeSearchKey(SearchKeyInput{Queries: []string{"hello"}, Sources: []string{"searxng"}, K: 8, Hour: now, CallerClass: ClassFleet, Topic: "general"}) {
|
||||
t.Fatal("topic general")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCorruptDBBypass(t *testing.T) {
|
||||
|
||||
@@ -0,0 +1,108 @@
|
||||
package dcache
|
||||
|
||||
import (
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"encoding/json"
|
||||
"sort"
|
||||
"strconv"
|
||||
"time"
|
||||
)
|
||||
|
||||
// SearchKeyInput is the canonical cache identity for a search. Callers pass
|
||||
// already-normalised filter fields (long time_range, sorted domains optional).
|
||||
type SearchKeyInput struct {
|
||||
Queries []string
|
||||
Sources []string
|
||||
K int
|
||||
Hour time.Time
|
||||
CallerClass string
|
||||
IncludeDomains []string
|
||||
ExcludeDomains []string
|
||||
IncludeDomainsMode string
|
||||
TimeRange string
|
||||
StartDate string
|
||||
EndDate string
|
||||
Topic string
|
||||
ExactMatch bool
|
||||
MinScore float64
|
||||
}
|
||||
|
||||
func SearchKey(query string, sources []string, k int, hour time.Time, callerClass string) string {
|
||||
return EncodeSearchKey(SearchKeyInput{
|
||||
Queries: []string{query},
|
||||
Sources: sources,
|
||||
K: k,
|
||||
Hour: hour,
|
||||
CallerClass: callerClass,
|
||||
})
|
||||
}
|
||||
|
||||
func EncodeSearchKey(in SearchKeyInput) string {
|
||||
if in.CallerClass == "" {
|
||||
in.CallerClass = ClassFleet
|
||||
}
|
||||
src := append([]string(nil), in.Sources...)
|
||||
sort.Strings(src)
|
||||
qs := make([]string, len(in.Queries))
|
||||
for i, q := range in.Queries {
|
||||
qs[i] = NormalizeQuestion(q)
|
||||
}
|
||||
topic := in.Topic
|
||||
if topic == "" {
|
||||
topic = "general"
|
||||
}
|
||||
m := map[string]any{
|
||||
"caller_class": in.CallerClass,
|
||||
"exact_match": in.ExactMatch,
|
||||
"hour": in.Hour.UTC().Format("2006-01-02T15"),
|
||||
"k": in.K,
|
||||
"min_score": strconv.FormatFloat(in.MinScore, 'f', -1, 64),
|
||||
"queries": qs,
|
||||
"sources": src,
|
||||
"topic": topic,
|
||||
}
|
||||
if inc := sortedUnique(in.IncludeDomains); len(inc) > 0 {
|
||||
m["include_domains"] = inc
|
||||
mode := in.IncludeDomainsMode
|
||||
if mode == "" {
|
||||
mode = "restrict"
|
||||
}
|
||||
m["include_domains_mode"] = mode
|
||||
}
|
||||
if exc := sortedUnique(in.ExcludeDomains); len(exc) > 0 {
|
||||
m["exclude_domains"] = exc
|
||||
}
|
||||
if in.TimeRange != "" {
|
||||
m["time_range"] = in.TimeRange
|
||||
}
|
||||
if in.StartDate != "" {
|
||||
m["start_date"] = in.StartDate
|
||||
}
|
||||
if in.EndDate != "" {
|
||||
m["end_date"] = in.EndDate
|
||||
}
|
||||
b, _ := json.Marshal(m)
|
||||
sum := sha256.Sum256(b)
|
||||
return hex.EncodeToString(sum[:])
|
||||
}
|
||||
|
||||
func sortedUnique(in []string) []string {
|
||||
if len(in) == 0 {
|
||||
return nil
|
||||
}
|
||||
out := append([]string(nil), in...)
|
||||
sort.Strings(out)
|
||||
n := 0
|
||||
for _, s := range out {
|
||||
if s == "" {
|
||||
continue
|
||||
}
|
||||
if n > 0 && out[n-1] == s {
|
||||
continue
|
||||
}
|
||||
out[n] = s
|
||||
n++
|
||||
}
|
||||
return out[:n]
|
||||
}
|
||||
Reference in New Issue
Block a user