From b6e1cdf8251a25772f8352ccc67198d1af5ff541 Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Wed, 23 Sep 2026 16:02:21 -0700 Subject: [PATCH] p3: map Brave LLM Context; RRF merge; per-source no_rerank Live llm/context hits live in grounding.generic plus a URL-keyed sources map, not a sources array. Fan-in uses reciprocal-rank fusion instead of cosine. no_rerank keeps that source's own order as a prefix. --- dist/sift.toml.example | 1 + docs/P3.md | 36 ++++++------- internal/config/config.go | 6 ++- internal/search/build.go | 10 ++++ internal/search/engine.go | 21 ++------ internal/search/engine_test.go | 35 +++++++++++- internal/search/rank.go | 75 ++++++++++++++++++++++++++ internal/search/sources.go | 78 ++++++++++++++++++++------- internal/search/sources_test.go | 34 ++++++++++++ tests/testdata/brave_llm_context.json | 26 +++++++++ 10 files changed, 264 insertions(+), 58 deletions(-) create mode 100644 tests/testdata/brave_llm_context.json diff --git a/dist/sift.toml.example b/dist/sift.toml.example index 8b9f245..edbc904 100644 --- a/dist/sift.toml.example +++ b/dist/sift.toml.example @@ -57,3 +57,4 @@ listen = "127.0.0.1:8890" # enabled = false # url = "https://api.tavily.com/search" # daily_credit_cap = 500 +# no_rerank = true diff --git a/docs/P3.md b/docs/P3.md index 4dda133..e9a1ced 100644 --- a/docs/P3.md +++ b/docs/P3.md @@ -112,12 +112,14 @@ the shared fan-in. This is the Hermes SearXNG provider's request (`plugins/web/s ### 3.2 Brave LLM Context -PLAN §2, verified earlier: `GET /res/v1/llm/context?q=…` with `BRAVE_API_KEY` from +PLAN §2: `GET /res/v1/llm/context?q=…` with `BRAVE_API_KEY` from `/var/lib/sift/secret.env` (0640 root:sift). **Do not read SearXNG's secret.env.** -Header `X-Subscription-Token`. Map result URLs and snippets from the live JSON -(`sources` and any URL-bearing fields). Do **not** assume a model-written -`grounding` answer exists — confirm against a live response before dropping or -keeping any field that is not a URL/title/snippet. +Header `X-Subscription-Token`. Live response (Brave docs + Open WebUI #24120): +hits are `grounding.generic[]` (`url`, `title`, `snippets[]`); top-level `sources` +is a URL-keyed metadata map, not a result array. Map generic first, join +`snippets`, then URL keys in `sources` for anything generic missed. Do **not** +treat a model-written `grounding` string as a snippet. Fixture: +`tests/testdata/brave_llm_context.json`. This is **not** Brave Web Search (`/res/v1/web/search`). SearXNG already runs `braveapi` as an engine; doubling that is wasted spend. @@ -148,25 +150,23 @@ placeholder line the same way (empty, on-box fill). dropped, trailing slash dropped on the path, `www.` left as-is (different sites). First snippet wins; `source` becomes the first source that produced it; later sources only affect rerank input if they brought a longer snippet (keep the longer one). -5. Rerank (§5) unless `rerank: false` or Dixie cannot take the call. +5. Merge: reciprocal-rank fusion across source lists unless `rerank: false`. + A source with `no_rerank = true` keeps its own order as a prefix (first-source + wins on URL). Cosine/honcho-embed is not used for search. 6. Truncate to `k` (native) or `limit` (`/v2/search`, default 5, max 20). -7. Store in the search cache **only if** every attempted source succeeded and - rerank did not fall back. A timeout or fallback must not pin a partial result - for the rest of the hour (skip the write, or use a short TTL such as 2 m — skip - is the default). Return. +7. Store in the search cache **only if** every attempted source succeeded. + A timeout must not pin a partial result for the rest of the hour. Search **never GETs a result URL**. Agents `sift fetch` those through the P1 guard. -## 5. Rerank and Dixie +## 5. Merge (RRF) -Embed the query and each remaining snippet with `honcho-embed` (same engine, same -`dixie.max_inflight` FIFO as digest). Cosine similarity → `score`. Restore nothing to -source order except on fallback. +Fan-in uses reciprocal-rank fusion (`1/(60+rank)`) over per-source lists. Search +does not call `honcho-embed`; Dixie saturation cannot fail a search. -**Dixie saturation must not fail a search:** if the embed permit cannot be had within -`dixie.queue_wait`, or embed errors, or the queue is full → keep source-fan-in order -(SearXNG, then Brave, then Tavily, then original rank inside a source), warning -`rerank fell back: `. Never call chat/digest for search in P3. +`[search.tavily] no_rerank = true` keeps Tavily's own order at the front of the +result list; remaining sources still RRF. `rerank = false` skips RRF and keeps +source-fan-in order (SearXNG, then Brave, then Tavily). ## 6. Guard boundary (SSRF) diff --git a/internal/config/config.go b/internal/config/config.go index e48fac1..e0205de 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -53,14 +53,16 @@ type Search struct { } type SearchSource struct { - Enabled bool `toml:"enabled"` - URL string `toml:"url"` + Enabled bool `toml:"enabled"` + URL string `toml:"url"` + NoRerank bool `toml:"no_rerank"` } type TavilySource struct { Enabled bool `toml:"enabled"` URL string `toml:"url"` DailyCreditCap int `toml:"daily_credit_cap"` + NoRerank bool `toml:"no_rerank"` } func (s Search) RerankOn() bool { diff --git a/internal/search/build.go b/internal/search/build.go index b20476c..230c83e 100644 --- a/internal/search/build.go +++ b/internal/search/build.go @@ -35,6 +35,16 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger) Embedder: emb, Log: log, TavilyCap: cfg.Tavily.DailyCreditCap, + NoRerank: map[string]bool{}, + } + if cfg.SearxNG.NoRerank { + e.NoRerank[NameSearxNG] = true + } + if cfg.BraveLLM.NoRerank { + e.NoRerank[NameBraveLLM] = true + } + if cfg.Tavily.NoRerank { + e.NoRerank[NameTavily] = true } if cfg.SearxNG.Enabled { u := cfg.SearxNG.URL diff --git a/internal/search/engine.go b/internal/search/engine.go index aa7bdf9..f4baf24 100644 --- a/internal/search/engine.go +++ b/internal/search/engine.go @@ -23,6 +23,7 @@ type Engine struct { Degraded []string // static: missing keys TavilyCap int Now func() time.Time + NoRerank map[string]bool mu sync.Mutex Queries map[string]map[string]uint64 @@ -137,6 +138,7 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca } wg.Wait() + var lists []namedHits var hits []Hit var warns []string var okNames, failNames, att []string @@ -158,7 +160,7 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca continue } okNames = append(okNames, o.name) - hits = append(hits, o.out.Hits...) + lists = append(lists, namedHits{name: o.name, hits: o.out.Hits}) if o.out.Credits > 0 { e.addCredits(caller, o.name, o.out.Credits) if o.name == NameTavily && e.Cache != nil { @@ -166,24 +168,9 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca } } } - hits = Dedupe(hits) - sortBySourceOrder(hits) + hits = mergeSearch(e, lists, doRerank) var rerankMS int64 - if doRerank && e.RerankOn { - rs := e.now() - ranked, fb := rankHits(ctx, query, hits, e.Embedder) - rerankMS = e.now().Sub(rs).Milliseconds() - if fb != "" { - warns = append(warns, "rerank fell back: "+fb) - degraded = true - e.mu.Lock() - e.RerankFallback++ - e.mu.Unlock() - } else { - hits = ranked - } - } hits = clampHits(hits, k) if !degraded && e.Cache != nil && len(att) > 0 { diff --git a/internal/search/engine_test.go b/internal/search/engine_test.go index 7d9017e..cabdc2d 100644 --- a/internal/search/engine_test.go +++ b/internal/search/engine_test.go @@ -121,8 +121,39 @@ func TestRerankFallbackStill200(t *testing.T) { if len(r.Hits) != 1 { t.Fatal(r.Hits) } - if !strings.Contains(strings.Join(r.Warnings, " "), "rerank fell back") { - t.Fatalf("%v", r.Warnings) + if strings.Contains(strings.Join(r.Warnings, " "), "rerank fell back") { + t.Fatalf("RRF must not depend on embed: %v", r.Warnings) + } +} + +func TestRRFInterleaveAndNoRerank(t *testing.T) { + searx := []Hit{ + {URL: "https://a.example/s", Title: "s1", Source: NameSearxNG}, + {URL: "https://b.example/s", Title: "s2", Source: NameSearxNG}, + } + tav := []Hit{ + {URL: "https://tavily.example/1", Title: "t1", Source: NameTavily}, + {URL: "https://a.example/s", Title: "t-dup", Snippet: "longer tavily snippet here", Source: NameTavily}, + } + merged := rrfMerge([][]Hit{searx, tav}) + if len(merged) != 3 { + t.Fatalf("%+v", merged) + } + if merged[0].URL != "https://a.example/s" { + t.Fatalf("shared URL should rank first, got %+v", merged[0]) + } + e := &Engine{ + Sources: []Source{ + fakeSrc{name: NameSearxNG, hits: searx}, + fakeSrc{name: NameTavily, hits: tav}, + }, + DefaultK: 8, + RerankOn: true, + NoRerank: map[string]bool{NameTavily: true}, + } + r := e.Run(context.Background(), "q", 8, true, "titan", dcache.ClassFleet) + if r.Hits[0].URL != "https://tavily.example/1" { + t.Fatalf("no_rerank tavily should lead: %+v", r.Hits) } } diff --git a/internal/search/rank.go b/internal/search/rank.go index f50dfcb..c939328 100644 --- a/internal/search/rank.go +++ b/internal/search/rank.go @@ -3,6 +3,7 @@ package search import ( "context" "math" + "sort" "strings" ) @@ -79,3 +80,77 @@ func rankHits(ctx context.Context, query string, hits []Hit, emb Embedder) (out } return out, "" } + +const rrfK = 60 + +type namedHits struct { + name string + hits []Hit +} + +func rrfMerge(lists [][]Hit) []Hit { + type acc struct { + h Hit + s float64 + } + seen := map[string]*acc{} + var order []string + for _, list := range lists { + for r, h := range list { + c := CanonicalURL(h.URL) + if c == "" { + continue + } + h.URL = c + sc := 1.0 / float64(rrfK+r+1) + if a, ok := seen[c]; ok { + a.s += sc + if len(h.Snippet) > len(a.h.Snippet) { + src := a.h.Source + a.h = h + a.h.Source = src + } + continue + } + cp := h + seen[c] = &acc{h: cp, s: sc} + order = append(order, c) + } + } + out := make([]Hit, 0, len(seen)) + for _, c := range order { + a := seen[c] + a.h.Score = a.s + out = append(out, a.h) + } + sort.SliceStable(out, func(i, j int) bool { + return out[i].Score > out[j].Score + }) + return out +} + +func mergeSearch(e *Engine, lists []namedHits, doRerank bool) []Hit { + if doRerank && e != nil && e.RerankOn { + var hold, rest [][]Hit + for _, l := range lists { + if e.NoRerank[l.name] { + hold = append(hold, l.hits) + } else { + rest = append(rest, l.hits) + } + } + var out []Hit + for _, h := range hold { + out = append(out, h...) + } + out = append(out, rrfMerge(rest)...) + return Dedupe(out) + } + var all []Hit + for _, l := range lists { + all = append(all, l.hits...) + } + all = Dedupe(all) + sortBySourceOrder(all) + return all +} diff --git a/internal/search/sources.go b/internal/search/sources.go index fdd5e95..3455b68 100644 --- a/internal/search/sources.go +++ b/internal/search/sources.go @@ -167,14 +167,34 @@ func mapBrave(raw []byte) []Hit { if err := json.Unmarshal(raw, &top); err != nil { return nil } - var buckets []any - for _, key := range []string{"sources", "results", "web"} { - if v, ok := top[key]; ok { - buckets = append(buckets, v) - } - } var hits []Hit seen := map[string]bool{} + add := func(u, title, snippet string) { + if u == "" || seen[u] { + return + } + seen[u] = true + hits = append(hits, Hit{URL: u, Title: title, Snippet: snippet, Source: NameBraveLLM}) + } + + // Live LLM Context: grounding.generic[] (and poi) with snippets[]. + if g, ok := top["grounding"].(map[string]any); ok { + for _, key := range []string{"generic", "poi"} { + arr, _ := g[key].([]any) + for _, item := range arr { + m, _ := item.(map[string]any) + if m == nil { + continue + } + snip := joinSnippets(m["snippets"]) + if snip == "" { + snip = strField(m, "snippet", "description", "content", "text") + } + add(strField(m, "url", "link", "href"), strField(m, "title", "name"), snip) + } + } + } + var walk func(any) walk = func(v any) { switch t := v.(type) { @@ -185,29 +205,49 @@ func mapBrave(raw []byte) []Hit { case map[string]any: u := strField(t, "url", "link", "href") if u == "" { - for _, x := range t { + // URL-keyed object: sources["https://…"] = {title, hostname} + for k, x := range t { + if strings.HasPrefix(k, "http://") || strings.HasPrefix(k, "https://") { + title, snip := "", "" + if m, ok := x.(map[string]any); ok { + title = strField(m, "title", "name", "site_name") + snip = strField(m, "description", "snippet") + } + add(k, title, snip) + continue + } walk(x) } return } - if seen[u] { - return - } - seen[u] = true - hits = append(hits, Hit{ - URL: u, - Title: strField(t, "title", "name"), - Snippet: strField(t, "snippet", "description", "content", "text"), - Source: NameBraveLLM, - }) + add(u, strField(t, "title", "name"), strField(t, "snippet", "description", "content", "text")) } } - for _, b := range buckets { - walk(b) + for _, key := range []string{"sources", "results", "web"} { + if v, ok := top[key]; ok { + walk(v) + } } return hits } +func joinSnippets(v any) string { + switch t := v.(type) { + case []any: + parts := make([]string, 0, len(t)) + for _, x := range t { + if s, ok := x.(string); ok && s != "" { + parts = append(parts, s) + } + } + return strings.Join(parts, "\n\n") + case string: + return t + default: + return "" + } +} + func strField(m map[string]any, keys ...string) string { for _, k := range keys { if v, ok := m[k]; ok { diff --git a/internal/search/sources_test.go b/internal/search/sources_test.go index 655ea40..d78a586 100644 --- a/internal/search/sources_test.go +++ b/internal/search/sources_test.go @@ -6,6 +6,7 @@ import ( "io" "net/http" "net/http/httptest" + "os" "strings" "testing" "time" @@ -96,6 +97,39 @@ func TestBraveMapsSources(t *testing.T) { } } +func TestBraveMapsGroundingGeneric(t *testing.T) { + raw, err := os.ReadFile("../../tests/testdata/brave_llm_context.json") + if err != nil { + t.Fatal(err) + } + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + _, _ = w.Write(raw) + })) + defer ts.Close() + c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second}) + if err != nil { + t.Fatal(err) + } + out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5) + if len(out.Hits) != 3 { + t.Fatalf("hits %d %+v", len(out.Hits), out.Hits) + } + if out.Hits[0].URL != "https://docs.tavily.com/documentation/api-reference/endpoint/search" { + t.Fatalf("first %+v", out.Hits[0]) + } + if !strings.Contains(out.Hits[0].Snippet, "max_results") { + t.Fatalf("snippet %q", out.Hits[0].Snippet) + } + var urls []string + for _, h := range out.Hits { + urls = append(urls, h.URL) + } + joined := strings.Join(urls, " ") + if !strings.Contains(joined, "orphan.example") { + t.Fatalf("sources map missed: %v", urls) + } +} + func TestTavilyBadJSONIsError(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { _, _ = w.Write([]byte(`not-json`)) diff --git a/tests/testdata/brave_llm_context.json b/tests/testdata/brave_llm_context.json new file mode 100644 index 0000000..c646477 --- /dev/null +++ b/tests/testdata/brave_llm_context.json @@ -0,0 +1,26 @@ +{ + "grounding": { + "generic": [ + { + "url": "https://docs.tavily.com/documentation/api-reference/endpoint/search", + "title": "Tavily Search API", + "snippets": ["POST /search with query and max_results.", "search_depth basic is one credit."] + }, + { + "url": "https://api.search.brave.com/help", + "title": "Brave Search API", + "snippets": ["LLM Context returns grounding.generic."] + } + ] + }, + "sources": { + "https://docs.tavily.com/documentation/api-reference/endpoint/search": { + "title": "Tavily Search API", + "hostname": "docs.tavily.com" + }, + "https://orphan.example/only-in-sources": { + "title": "Orphan", + "hostname": "orphan.example" + } + } +}