p3: map Brave LLM Context; RRF merge; per-source no_rerank

Live llm/context hits live in grounding.generic plus a URL-keyed sources map, not a sources array. Fan-in uses reciprocal-rank fusion instead of cosine. no_rerank keeps that source's own order as a prefix.
2026-09-23 16:02:21 -07:00
parent eb4e46d036
commit b6e1cdf825
10 changed files with 264 additions and 58 deletions
@@ -35,6 +35,16 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger)
 		Embedder:  emb,
 		Log:       log,
 		TavilyCap: cfg.Tavily.DailyCreditCap,
+		NoRerank:  map[string]bool{},
+	}
+	if cfg.SearxNG.NoRerank {
+		e.NoRerank[NameSearxNG] = true
+	}
+	if cfg.BraveLLM.NoRerank {
+		e.NoRerank[NameBraveLLM] = true
+	}
+	if cfg.Tavily.NoRerank {
+		e.NoRerank[NameTavily] = true
 	}
 	if cfg.SearxNG.Enabled {
 		u := cfg.SearxNG.URL
@@ -23,6 +23,7 @@ type Engine struct {
 	Degraded  []string // static: missing keys
 	TavilyCap int
 	Now       func() time.Time
+	NoRerank  map[string]bool

 	mu             sync.Mutex
 	Queries        map[string]map[string]uint64
@@ -137,6 +138,7 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca
 	}
 	wg.Wait()

+	var lists []namedHits
 	var hits []Hit
 	var warns []string
 	var okNames, failNames, att []string
@@ -158,7 +160,7 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca
 			continue
 		}
 		okNames = append(okNames, o.name)
-		hits = append(hits, o.out.Hits...)
+		lists = append(lists, namedHits{name: o.name, hits: o.out.Hits})
 		if o.out.Credits > 0 {
 			e.addCredits(caller, o.name, o.out.Credits)
 			if o.name == NameTavily && e.Cache != nil {
@@ -166,24 +168,9 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca
 			}
 		}
 	}
-	hits = Dedupe(hits)
-	sortBySourceOrder(hits)
+	hits = mergeSearch(e, lists, doRerank)

 	var rerankMS int64
-	if doRerank && e.RerankOn {
-		rs := e.now()
-		ranked, fb := rankHits(ctx, query, hits, e.Embedder)
-		rerankMS = e.now().Sub(rs).Milliseconds()
-		if fb != "" {
-			warns = append(warns, "rerank fell back: "+fb)
-			degraded = true
-			e.mu.Lock()
-			e.RerankFallback++
-			e.mu.Unlock()
-		} else {
-			hits = ranked
-		}
-	}
 	hits = clampHits(hits, k)

 	if !degraded && e.Cache != nil && len(att) > 0 {
@@ -121,8 +121,39 @@ func TestRerankFallbackStill200(t *testing.T) {
 	if len(r.Hits) != 1 {
 		t.Fatal(r.Hits)
 	}
-	if !strings.Contains(strings.Join(r.Warnings, " "), "rerank fell back") {
-		t.Fatalf("%v", r.Warnings)
+	if strings.Contains(strings.Join(r.Warnings, " "), "rerank fell back") {
+		t.Fatalf("RRF must not depend on embed: %v", r.Warnings)
+	}
+}
+
+func TestRRFInterleaveAndNoRerank(t *testing.T) {
+	searx := []Hit{
+		{URL: "https://a.example/s", Title: "s1", Source: NameSearxNG},
+		{URL: "https://b.example/s", Title: "s2", Source: NameSearxNG},
+	}
+	tav := []Hit{
+		{URL: "https://tavily.example/1", Title: "t1", Source: NameTavily},
+		{URL: "https://a.example/s", Title: "t-dup", Snippet: "longer tavily snippet here", Source: NameTavily},
+	}
+	merged := rrfMerge([][]Hit{searx, tav})
+	if len(merged) != 3 {
+		t.Fatalf("%+v", merged)
+	}
+	if merged[0].URL != "https://a.example/s" {
+		t.Fatalf("shared URL should rank first, got %+v", merged[0])
+	}
+	e := &Engine{
+		Sources: []Source{
+			fakeSrc{name: NameSearxNG, hits: searx},
+			fakeSrc{name: NameTavily, hits: tav},
+		},
+		DefaultK: 8,
+		RerankOn: true,
+		NoRerank: map[string]bool{NameTavily: true},
+	}
+	r := e.Run(context.Background(), "q", 8, true, "titan", dcache.ClassFleet)
+	if r.Hits[0].URL != "https://tavily.example/1" {
+		t.Fatalf("no_rerank tavily should lead: %+v", r.Hits)
 	}
 }

@@ -3,6 +3,7 @@ package search
 import (
 	"context"
 	"math"
+	"sort"
 	"strings"
 )

@@ -79,3 +80,77 @@ func rankHits(ctx context.Context, query string, hits []Hit, emb Embedder) (out
 	}
 	return out, ""
 }
+
+const rrfK = 60
+
+type namedHits struct {
+	name string
+	hits []Hit
+}
+
+func rrfMerge(lists [][]Hit) []Hit {
+	type acc struct {
+		h Hit
+		s float64
+	}
+	seen := map[string]*acc{}
+	var order []string
+	for _, list := range lists {
+		for r, h := range list {
+			c := CanonicalURL(h.URL)
+			if c == "" {
+				continue
+			}
+			h.URL = c
+			sc := 1.0 / float64(rrfK+r+1)
+			if a, ok := seen[c]; ok {
+				a.s += sc
+				if len(h.Snippet) > len(a.h.Snippet) {
+					src := a.h.Source
+					a.h = h
+					a.h.Source = src
+				}
+				continue
+			}
+			cp := h
+			seen[c] = &acc{h: cp, s: sc}
+			order = append(order, c)
+		}
+	}
+	out := make([]Hit, 0, len(seen))
+	for _, c := range order {
+		a := seen[c]
+		a.h.Score = a.s
+		out = append(out, a.h)
+	}
+	sort.SliceStable(out, func(i, j int) bool {
+		return out[i].Score > out[j].Score
+	})
+	return out
+}
+
+func mergeSearch(e *Engine, lists []namedHits, doRerank bool) []Hit {
+	if doRerank && e != nil && e.RerankOn {
+		var hold, rest [][]Hit
+		for _, l := range lists {
+			if e.NoRerank[l.name] {
+				hold = append(hold, l.hits)
+			} else {
+				rest = append(rest, l.hits)
+			}
+		}
+		var out []Hit
+		for _, h := range hold {
+			out = append(out, h...)
+		}
+		out = append(out, rrfMerge(rest)...)
+		return Dedupe(out)
+	}
+	var all []Hit
+	for _, l := range lists {
+		all = append(all, l.hits...)
+	}
+	all = Dedupe(all)
+	sortBySourceOrder(all)
+	return all
+}
@@ -167,14 +167,34 @@ func mapBrave(raw []byte) []Hit {
 	if err := json.Unmarshal(raw, &top); err != nil {
 		return nil
 	}
-	var buckets []any
-	for _, key := range []string{"sources", "results", "web"} {
-		if v, ok := top[key]; ok {
-			buckets = append(buckets, v)
-		}
-	}
 	var hits []Hit
 	seen := map[string]bool{}
+	add := func(u, title, snippet string) {
+		if u == "" || seen[u] {
+			return
+		}
+		seen[u] = true
+		hits = append(hits, Hit{URL: u, Title: title, Snippet: snippet, Source: NameBraveLLM})
+	}
+
+	// Live LLM Context: grounding.generic[] (and poi) with snippets[].
+	if g, ok := top["grounding"].(map[string]any); ok {
+		for _, key := range []string{"generic", "poi"} {
+			arr, _ := g[key].([]any)
+			for _, item := range arr {
+				m, _ := item.(map[string]any)
+				if m == nil {
+					continue
+				}
+				snip := joinSnippets(m["snippets"])
+				if snip == "" {
+					snip = strField(m, "snippet", "description", "content", "text")
+				}
+				add(strField(m, "url", "link", "href"), strField(m, "title", "name"), snip)
+			}
+		}
+	}
+
 	var walk func(any)
 	walk = func(v any) {
 		switch t := v.(type) {
@@ -185,29 +205,49 @@ func mapBrave(raw []byte) []Hit {
 		case map[string]any:
 			u := strField(t, "url", "link", "href")
 			if u == "" {
-				for _, x := range t {
+				// URL-keyed object: sources["https://…"] = {title, hostname}
+				for k, x := range t {
+					if strings.HasPrefix(k, "http://") || strings.HasPrefix(k, "https://") {
+						title, snip := "", ""
+						if m, ok := x.(map[string]any); ok {
+							title = strField(m, "title", "name", "site_name")
+							snip = strField(m, "description", "snippet")
+						}
+						add(k, title, snip)
+						continue
+					}
 					walk(x)
 				}
 				return
 			}
-			if seen[u] {
-				return
-			}
-			seen[u] = true
-			hits = append(hits, Hit{
-				URL:     u,
-				Title:   strField(t, "title", "name"),
-				Snippet: strField(t, "snippet", "description", "content", "text"),
-				Source:  NameBraveLLM,
-			})
+			add(u, strField(t, "title", "name"), strField(t, "snippet", "description", "content", "text"))
 		}
 	}
-	for _, b := range buckets {
-		walk(b)
+	for _, key := range []string{"sources", "results", "web"} {
+		if v, ok := top[key]; ok {
+			walk(v)
+		}
 	}
 	return hits
 }

+func joinSnippets(v any) string {
+	switch t := v.(type) {
+	case []any:
+		parts := make([]string, 0, len(t))
+		for _, x := range t {
+			if s, ok := x.(string); ok && s != "" {
+				parts = append(parts, s)
+			}
+		}
+		return strings.Join(parts, "\n\n")
+	case string:
+		return t
+	default:
+		return ""
+	}
+}
+
 func strField(m map[string]any, keys ...string) string {
 	for _, k := range keys {
 		if v, ok := m[k]; ok {
@@ -6,6 +6,7 @@ import (
 	"io"
 	"net/http"
 	"net/http/httptest"
+	"os"
 	"strings"
 	"testing"
 	"time"
@@ -96,6 +97,39 @@ func TestBraveMapsSources(t *testing.T) {
 	}
 }

+func TestBraveMapsGroundingGeneric(t *testing.T) {
+	raw, err := os.ReadFile("../../tests/testdata/brave_llm_context.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		_, _ = w.Write(raw)
+	}))
+	defer ts.Close()
+	c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
+	if err != nil {
+		t.Fatal(err)
+	}
+	out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
+	if len(out.Hits) != 3 {
+		t.Fatalf("hits %d %+v", len(out.Hits), out.Hits)
+	}
+	if out.Hits[0].URL != "https://docs.tavily.com/documentation/api-reference/endpoint/search" {
+		t.Fatalf("first %+v", out.Hits[0])
+	}
+	if !strings.Contains(out.Hits[0].Snippet, "max_results") {
+		t.Fatalf("snippet %q", out.Hits[0].Snippet)
+	}
+	var urls []string
+	for _, h := range out.Hits {
+		urls = append(urls, h.URL)
+	}
+	joined := strings.Join(urls, " ")
+	if !strings.Contains(joined, "orphan.example") {
+		t.Fatalf("sources map missed: %v", urls)
+	}
+}
+
 func TestTavilyBadJSONIsError(t *testing.T) {
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		_, _ = w.Write([]byte(`not-json`))