p3: keep original URLs on RRF/dedupe; rerank=false is not a merge
CanonicalURL is the dedupe key only. Hits keep the first source's original URL, including a trailing slash. Dedupe has the same rule.
This commit is contained in:
@@ -157,6 +157,23 @@ func TestRRFInterleaveAndNoRerank(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestRRFKeepsOriginalURL(t *testing.T) {
|
||||
orig := "https://ex.com/docs/"
|
||||
merged := rrfMerge([][]Hit{
|
||||
{{URL: orig, Title: "slash", Source: NameSearxNG}},
|
||||
{{URL: "https://ex.com/docs", Title: "stripped", Snippet: "longer", Source: NameTavily}},
|
||||
})
|
||||
if len(merged) != 1 {
|
||||
t.Fatalf("%+v", merged)
|
||||
}
|
||||
if merged[0].URL != orig {
|
||||
t.Fatalf("want original URL %q got %q", orig, merged[0].URL)
|
||||
}
|
||||
if CanonicalURL(orig) != "https://ex.com/docs" {
|
||||
t.Fatalf("canonical %q", CanonicalURL(orig))
|
||||
}
|
||||
}
|
||||
|
||||
func TestDailyCapSkipsTavily(t *testing.T) {
|
||||
db, err := dcache.Open(filepath.Join(t.TempDir(), "c.db"), 1<<20, nil)
|
||||
if err != nil {
|
||||
@@ -187,4 +204,7 @@ func TestDedupeLongerSnippet(t *testing.T) {
|
||||
if len(out) != 1 || out[0].Source != NameSearxNG || out[0].Snippet != "aabbcc" {
|
||||
t.Fatalf("%+v", out)
|
||||
}
|
||||
if out[0].URL != "https://Ex.com/a#x" {
|
||||
t.Fatalf("dedupe must keep first original URL, got %q", out[0].URL)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -97,18 +97,20 @@ func rrfMerge(lists [][]Hit) []Hit {
|
||||
var order []string
|
||||
for _, list := range lists {
|
||||
for r, h := range list {
|
||||
c := CanonicalURL(h.URL)
|
||||
orig := h.URL
|
||||
c := CanonicalURL(orig)
|
||||
if c == "" {
|
||||
continue
|
||||
}
|
||||
h.URL = c
|
||||
sc := 1.0 / float64(rrfK+r+1)
|
||||
if a, ok := seen[c]; ok {
|
||||
a.s += sc
|
||||
if len(h.Snippet) > len(a.h.Snippet) {
|
||||
src := a.h.Source
|
||||
keep := a.h.URL
|
||||
a.h = h
|
||||
a.h.Source = src
|
||||
a.h.URL = keep
|
||||
}
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -79,13 +79,14 @@ func Dedupe(in []Hit) []Hit {
|
||||
if canon == "" {
|
||||
continue
|
||||
}
|
||||
h.URL = canon
|
||||
if s, ok := seen[canon]; ok {
|
||||
if len(h.Snippet) > len(s.h.Snippet) {
|
||||
src := s.h.Source
|
||||
keep := s.h.URL
|
||||
sc := s.h.Score
|
||||
s.h = h
|
||||
s.h.Source = src
|
||||
s.h.URL = keep
|
||||
if sc > s.h.Score {
|
||||
s.h.Score = sc
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user