p3: keep original URLs on RRF/dedupe; rerank=false is not a merge
CanonicalURL is the dedupe key only. Hits keep the first source's original URL, including a trailing slash. Dedupe has the same rule.
This commit is contained in:
+10
-6
@@ -146,10 +146,10 @@ placeholder line the same way (empty, on-box fill).
|
|||||||
3. Fan-in: one goroutine per enabled source, each with `search.timeout`. Collect whatever
|
3. Fan-in: one goroutine per enabled source, each with `search.timeout`. Collect whatever
|
||||||
arrived; a timeout or non-2xx is a `warnings` entry (`searxng: timeout`, `tavily: 429`)
|
arrived; a timeout or non-2xx is a `warnings` entry (`searxng: timeout`, `tavily: 429`)
|
||||||
and that source contributes nothing.
|
and that source contributes nothing.
|
||||||
4. Dedupe on **canonical URL**: scheme+host lowercased, default ports stripped, fragment
|
4. Dedupe on **canonical URL as a key only** (scheme+host lowercased, default ports
|
||||||
dropped, trailing slash dropped on the path, `www.` left as-is (different sites). First
|
stripped, fragment dropped, trailing slash dropped on the path; `www.` left as-is).
|
||||||
snippet wins; `source` becomes the first source that produced it; later sources only
|
The hit returned to the agent keeps the **first source's original URL**. First
|
||||||
affect rerank input if they brought a longer snippet (keep the longer one).
|
source wins attribution; a later longer snippet replaces the text only.
|
||||||
5. Merge: reciprocal-rank fusion across source lists unless `rerank: false`.
|
5. Merge: reciprocal-rank fusion across source lists unless `rerank: false`.
|
||||||
A source with `no_rerank = true` keeps its own order as a prefix (first-source
|
A source with `no_rerank = true` keeps its own order as a prefix (first-source
|
||||||
wins on URL). Cosine/honcho-embed is not used for search.
|
wins on URL). Cosine/honcho-embed is not used for search.
|
||||||
@@ -165,8 +165,12 @@ Fan-in uses reciprocal-rank fusion (`1/(60+rank)`) over per-source lists. Search
|
|||||||
does not call `honcho-embed`; Dixie saturation cannot fail a search.
|
does not call `honcho-embed`; Dixie saturation cannot fail a search.
|
||||||
|
|
||||||
`[search.tavily] no_rerank = true` keeps Tavily's own order at the front of the
|
`[search.tavily] no_rerank = true` keeps Tavily's own order at the front of the
|
||||||
result list; remaining sources still RRF. `rerank = false` skips RRF and keeps
|
result list; remaining sources still RRF.
|
||||||
source-fan-in order (SearXNG, then Brave, then Tavily).
|
|
||||||
|
**`rerank = false` is not a merge.** It concatenates in source order (SearXNG,
|
||||||
|
then Brave, then Tavily) and dedupes. Do not enable two sources with
|
||||||
|
`rerank = false` expecting RRF or Tavily-first. That only happens with
|
||||||
|
`rerank = true` (and `no_rerank` on Tavily if you want Tavily's order).
|
||||||
|
|
||||||
## 6. Guard boundary (SSRF)
|
## 6. Guard boundary (SSRF)
|
||||||
|
|
||||||
|
|||||||
@@ -157,6 +157,23 @@ func TestRRFInterleaveAndNoRerank(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func TestRRFKeepsOriginalURL(t *testing.T) {
|
||||||
|
orig := "https://ex.com/docs/"
|
||||||
|
merged := rrfMerge([][]Hit{
|
||||||
|
{{URL: orig, Title: "slash", Source: NameSearxNG}},
|
||||||
|
{{URL: "https://ex.com/docs", Title: "stripped", Snippet: "longer", Source: NameTavily}},
|
||||||
|
})
|
||||||
|
if len(merged) != 1 {
|
||||||
|
t.Fatalf("%+v", merged)
|
||||||
|
}
|
||||||
|
if merged[0].URL != orig {
|
||||||
|
t.Fatalf("want original URL %q got %q", orig, merged[0].URL)
|
||||||
|
}
|
||||||
|
if CanonicalURL(orig) != "https://ex.com/docs" {
|
||||||
|
t.Fatalf("canonical %q", CanonicalURL(orig))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func TestDailyCapSkipsTavily(t *testing.T) {
|
func TestDailyCapSkipsTavily(t *testing.T) {
|
||||||
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@@ -187,4 +204,7 @@ func TestDedupeLongerSnippet(t *testing.T) {
|
|||||||
if len(out) != 1 || out[0].Source != NameSearxNG || out[0].Snippet != "aabbcc" {
|
if len(out) != 1 || out[0].Source != NameSearxNG || out[0].Snippet != "aabbcc" {
|
||||||
t.Fatalf("%+v", out)
|
t.Fatalf("%+v", out)
|
||||||
}
|
}
|
||||||
|
if out[0].URL != "https://Ex.com/a#x" {
|
||||||
|
t.Fatalf("dedupe must keep first original URL, got %q", out[0].URL)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -97,18 +97,20 @@ func rrfMerge(lists [][]Hit) []Hit {
|
|||||||
var order []string
|
var order []string
|
||||||
for _, list := range lists {
|
for _, list := range lists {
|
||||||
for r, h := range list {
|
for r, h := range list {
|
||||||
c := CanonicalURL(h.URL)
|
orig := h.URL
|
||||||
|
c := CanonicalURL(orig)
|
||||||
if c == "" {
|
if c == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
h.URL = c
|
|
||||||
sc := 1.0 / float64(rrfK+r+1)
|
sc := 1.0 / float64(rrfK+r+1)
|
||||||
if a, ok := seen[c]; ok {
|
if a, ok := seen[c]; ok {
|
||||||
a.s += sc
|
a.s += sc
|
||||||
if len(h.Snippet) > len(a.h.Snippet) {
|
if len(h.Snippet) > len(a.h.Snippet) {
|
||||||
src := a.h.Source
|
src := a.h.Source
|
||||||
|
keep := a.h.URL
|
||||||
a.h = h
|
a.h = h
|
||||||
a.h.Source = src
|
a.h.Source = src
|
||||||
|
a.h.URL = keep
|
||||||
}
|
}
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -79,13 +79,14 @@ func Dedupe(in []Hit) []Hit {
|
|||||||
if canon == "" {
|
if canon == "" {
|
||||||
continue
|
continue
|
||||||
}
|
}
|
||||||
h.URL = canon
|
|
||||||
if s, ok := seen[canon]; ok {
|
if s, ok := seen[canon]; ok {
|
||||||
if len(h.Snippet) > len(s.h.Snippet) {
|
if len(h.Snippet) > len(s.h.Snippet) {
|
||||||
src := s.h.Source
|
src := s.h.Source
|
||||||
|
keep := s.h.URL
|
||||||
sc := s.h.Score
|
sc := s.h.Score
|
||||||
s.h = h
|
s.h = h
|
||||||
s.h.Source = src
|
s.h.Source = src
|
||||||
|
s.h.URL = keep
|
||||||
if sc > s.h.Score {
|
if sc > s.h.Score {
|
||||||
s.h.Score = sc
|
s.h.Score = sc
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user