p3: map Brave LLM Context; RRF merge; per-source no_rerank
Live llm/context hits live in grounding.generic plus a URL-keyed sources map, not a sources array. Fan-in uses reciprocal-rank fusion instead of cosine. no_rerank keeps that source's own order as a prefix.
This commit is contained in:
@@ -3,6 +3,7 @@ package search
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"sort"
|
||||
"strings"
|
||||
)
|
||||
|
||||
@@ -79,3 +80,77 @@ func rankHits(ctx context.Context, query string, hits []Hit, emb Embedder) (out
|
||||
}
|
||||
return out, ""
|
||||
}
|
||||
|
||||
const rrfK = 60
|
||||
|
||||
type namedHits struct {
|
||||
name string
|
||||
hits []Hit
|
||||
}
|
||||
|
||||
func rrfMerge(lists [][]Hit) []Hit {
|
||||
type acc struct {
|
||||
h Hit
|
||||
s float64
|
||||
}
|
||||
seen := map[string]*acc{}
|
||||
var order []string
|
||||
for _, list := range lists {
|
||||
for r, h := range list {
|
||||
c := CanonicalURL(h.URL)
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
h.URL = c
|
||||
sc := 1.0 / float64(rrfK+r+1)
|
||||
if a, ok := seen[c]; ok {
|
||||
a.s += sc
|
||||
if len(h.Snippet) > len(a.h.Snippet) {
|
||||
src := a.h.Source
|
||||
a.h = h
|
||||
a.h.Source = src
|
||||
}
|
||||
continue
|
||||
}
|
||||
cp := h
|
||||
seen[c] = &acc{h: cp, s: sc}
|
||||
order = append(order, c)
|
||||
}
|
||||
}
|
||||
out := make([]Hit, 0, len(seen))
|
||||
for _, c := range order {
|
||||
a := seen[c]
|
||||
a.h.Score = a.s
|
||||
out = append(out, a.h)
|
||||
}
|
||||
sort.SliceStable(out, func(i, j int) bool {
|
||||
return out[i].Score > out[j].Score
|
||||
})
|
||||
return out
|
||||
}
|
||||
|
||||
func mergeSearch(e *Engine, lists []namedHits, doRerank bool) []Hit {
|
||||
if doRerank && e != nil && e.RerankOn {
|
||||
var hold, rest [][]Hit
|
||||
for _, l := range lists {
|
||||
if e.NoRerank[l.name] {
|
||||
hold = append(hold, l.hits)
|
||||
} else {
|
||||
rest = append(rest, l.hits)
|
||||
}
|
||||
}
|
||||
var out []Hit
|
||||
for _, h := range hold {
|
||||
out = append(out, h...)
|
||||
}
|
||||
out = append(out, rrfMerge(rest)...)
|
||||
return Dedupe(out)
|
||||
}
|
||||
var all []Hit
|
||||
for _, l := range lists {
|
||||
all = append(all, l.hits...)
|
||||
}
|
||||
all = Dedupe(all)
|
||||
sortBySourceOrder(all)
|
||||
return all
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user