kyle/sift
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

p3: map Brave LLM Context; RRF merge; per-source no_rerank

Browse Source 
Live llm/context hits live in grounding.generic plus a URL-keyed
sources map, not a sources array. Fan-in uses reciprocal-rank fusion
instead of cosine. no_rerank keeps that source's own order as a prefix.
This commit is contained in:
kyle
2026-09-23 16:02:21 -07:00
parent eb4e46d036
commit b6e1cdf825
10 changed files with 264 additions and 58 deletions
Download Patch File Download Diff File Expand all files Collapse all files
dist/sift.toml.example
Vendored
+1
View File
@@ -57,3 +57,4 @@ listen = "127.0.0.1:8890"
# enabled = false
# url = "https://api.tavily.com/search"
# daily_credit_cap = 500
# no_rerank = true
docs/P3.md
+18 -18
View File
@@ -112,12 +112,14 @@ the shared fan-in. This is the Hermes SearXNG provider's request (`plugins/web/s
### 3.2 Brave LLM Context
PLAN §2, verified earlier: `GET /res/v1/llm/context?q=…` with `BRAVE_API_KEY` from
PLAN §2: `GET /res/v1/llm/context?q=…` with `BRAVE_API_KEY` from
`/var/lib/sift/secret.env` (0640 root:sift). **Do not read SearXNG's secret.env.**
Header `X-Subscription-Token`. Map result URLs and snippets from the live JSON
(`sources` and any URL-bearing fields). Do **not** assume a model-written
`grounding` answer exists — confirm against a live response before dropping or
keeping any field that is not a URL/title/snippet.
Header `X-Subscription-Token`. Live response (Brave docs + Open WebUI #24120):
hits are `grounding.generic[]` (`url`, `title`, `snippets[]`); top-level `sources`
is a URL-keyed metadata map, not a result array. Map generic first, join
`snippets`, then URL keys in `sources` for anything generic missed. Do **not**
treat a model-written `grounding` string as a snippet. Fixture:
`tests/testdata/brave_llm_context.json`.
This is **not** Brave Web Search (`/res/v1/web/search`). SearXNG already runs `braveapi`
as an engine; doubling that is wasted spend.
@@ -148,25 +150,23 @@ placeholder line the same way (empty, on-box fill).
dropped, trailing slash dropped on the path, `www.` left as-is (different sites). First
snippet wins; `source` becomes the first source that produced it; later sources only
affect rerank input if they brought a longer snippet (keep the longer one).
5. Rerank (§5) unless `rerank: false` or Dixie cannot take the call.
5. Merge: reciprocal-rank fusion across source lists unless `rerank: false`.
A source with `no_rerank = true` keeps its own order as a prefix (first-source
wins on URL). Cosine/honcho-embed is not used for search.
6. Truncate to `k` (native) or `limit` (`/v2/search`, default 5, max 20).
7. Store in the search cache **only if** every attempted source succeeded and
rerank did not fall back. A timeout or fallback must not pin a partial result
for the rest of the hour (skip the write, or use a short TTL such as 2 m — skip
is the default). Return.
7. Store in the search cache **only if** every attempted source succeeded.
A timeout must not pin a partial result for the rest of the hour.
Search **never GETs a result URL**. Agents `sift fetch` those through the P1 guard.
## 5. Rerank and Dixie
## 5. Merge (RRF)
Embed the query and each remaining snippet with `honcho-embed` (same engine, same
`dixie.max_inflight` FIFO as digest). Cosine similarity → `score`. Restore nothing to
source order except on fallback.
Fan-in uses reciprocal-rank fusion (`1/(60+rank)`) over per-source lists. Search
does not call `honcho-embed`; Dixie saturation cannot fail a search.
**Dixie saturation must not fail a search:** if the embed permit cannot be had within
`dixie.queue_wait`, or embed errors, or the queue is full → keep source-fan-in order
(SearXNG, then Brave, then Tavily, then original rank inside a source), warning
`rerank fell back: <reason>`. Never call chat/digest for search in P3.
`[search.tavily] no_rerank = true` keeps Tavily's own order at the front of the
result list; remaining sources still RRF. `rerank = false` skips RRF and keeps
source-fan-in order (SearXNG, then Brave, then Tavily).
## 6. Guard boundary (SSRF)
internal/config/config.go
+2
View File
@@ -55,12 +55,14 @@ type Search struct {
type SearchSource struct {
Enabled bool `toml:"enabled"`
URL string `toml:"url"`
NoRerank bool `toml:"no_rerank"`
}
type TavilySource struct {
Enabled bool `toml:"enabled"`
URL string `toml:"url"`
DailyCreditCap int `toml:"daily_credit_cap"`
NoRerank bool `toml:"no_rerank"`
}
func (s Search) RerankOn() bool {
internal/search/build.go
+10
View File
@@ -35,6 +35,16 @@ func Build(cfg config.Search, cache *dcache.DB, emb Embedder, log *slog.Logger)
Embedder: emb,
Log: log,
TavilyCap: cfg.Tavily.DailyCreditCap,
NoRerank: map[string]bool{},
}
if cfg.SearxNG.NoRerank {
e.NoRerank[NameSearxNG] = true
}
if cfg.BraveLLM.NoRerank {
e.NoRerank[NameBraveLLM] = true
}
if cfg.Tavily.NoRerank {
e.NoRerank[NameTavily] = true
}
if cfg.SearxNG.Enabled {
u := cfg.SearxNG.URL
internal/search/engine.go
+4 -17
View File
@@ -23,6 +23,7 @@ type Engine struct {
Degraded []string // static: missing keys
TavilyCap int
Now func() time.Time
NoRerank map[string]bool
mu sync.Mutex
Queries map[string]map[string]uint64
@@ -137,6 +138,7 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca
}
wg.Wait()
var lists []namedHits
var hits []Hit
var warns []string
var okNames, failNames, att []string
@@ -158,7 +160,7 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca
continue
}
okNames = append(okNames, o.name)
hits = append(hits, o.out.Hits...)
lists = append(lists, namedHits{name: o.name, hits: o.out.Hits})
if o.out.Credits > 0 {
e.addCredits(caller, o.name, o.out.Credits)
if o.name == NameTavily && e.Cache != nil {
@@ -166,24 +168,9 @@ func (e *Engine) Run(ctx context.Context, query string, k int, doRerank bool, ca
}
}
}
hits = Dedupe(hits)
sortBySourceOrder(hits)
hits = mergeSearch(e, lists, doRerank)
var rerankMS int64
if doRerank && e.RerankOn {
rs := e.now()
ranked, fb := rankHits(ctx, query, hits, e.Embedder)
rerankMS = e.now().Sub(rs).Milliseconds()
if fb != "" {
warns = append(warns, "rerank fell back: "+fb)
degraded = true
e.mu.Lock()
e.RerankFallback++
e.mu.Unlock()
} else {
hits = ranked
}
}
hits = clampHits(hits, k)
if !degraded && e.Cache != nil && len(att) > 0 {
internal/search/engine_test.go
+33 -2
View File
@@ -121,8 +121,39 @@ func TestRerankFallbackStill200(t *testing.T) {
if len(r.Hits) != 1 {
t.Fatal(r.Hits)
}
if !strings.Contains(strings.Join(r.Warnings, " "), "rerank fell back") {
t.Fatalf("%v", r.Warnings)
if strings.Contains(strings.Join(r.Warnings, " "), "rerank fell back") {
t.Fatalf("RRF must not depend on embed: %v", r.Warnings)
}
}
func TestRRFInterleaveAndNoRerank(t *testing.T) {
searx := []Hit{
{URL: "https://a.example/s", Title: "s1", Source: NameSearxNG},
{URL: "https://b.example/s", Title: "s2", Source: NameSearxNG},
}
tav := []Hit{
{URL: "https://tavily.example/1", Title: "t1", Source: NameTavily},
{URL: "https://a.example/s", Title: "t-dup", Snippet: "longer tavily snippet here", Source: NameTavily},
}
merged := rrfMerge([][]Hit{searx, tav})
if len(merged) != 3 {
t.Fatalf("%+v", merged)
}
if merged[0].URL != "https://a.example/s" {
t.Fatalf("shared URL should rank first, got %+v", merged[0])
}
e := &Engine{
Sources: []Source{
fakeSrc{name: NameSearxNG, hits: searx},
fakeSrc{name: NameTavily, hits: tav},
},
DefaultK: 8,
RerankOn: true,
NoRerank: map[string]bool{NameTavily: true},
}
r := e.Run(context.Background(), "q", 8, true, "titan", dcache.ClassFleet)
if r.Hits[0].URL != "https://tavily.example/1" {
t.Fatalf("no_rerank tavily should lead: %+v", r.Hits)
}
}
internal/search/rank.go
+75
View File
@@ -3,6 +3,7 @@ package search
import (
"context"
"math"
"sort"
"strings"
)
@@ -79,3 +80,77 @@ func rankHits(ctx context.Context, query string, hits []Hit, emb Embedder) (out
}
return out, ""
}
const rrfK = 60
type namedHits struct {
name string
hits []Hit
}
func rrfMerge(lists [][]Hit) []Hit {
type acc struct {
h Hit
s float64
}
seen := map[string]*acc{}
var order []string
for _, list := range lists {
for r, h := range list {
c := CanonicalURL(h.URL)
if c == "" {
continue
}
h.URL = c
sc := 1.0 / float64(rrfK+r+1)
if a, ok := seen[c]; ok {
a.s += sc
if len(h.Snippet) > len(a.h.Snippet) {
src := a.h.Source
a.h = h
a.h.Source = src
}
continue
}
cp := h
seen[c] = &acc{h: cp, s: sc}
order = append(order, c)
}
}
out := make([]Hit, 0, len(seen))
for _, c := range order {
a := seen[c]
a.h.Score = a.s
out = append(out, a.h)
}
sort.SliceStable(out, func(i, j int) bool {
return out[i].Score > out[j].Score
})
return out
}
func mergeSearch(e *Engine, lists []namedHits, doRerank bool) []Hit {
if doRerank && e != nil && e.RerankOn {
var hold, rest [][]Hit
for _, l := range lists {
if e.NoRerank[l.name] {
hold = append(hold, l.hits)
} else {
rest = append(rest, l.hits)
}
}
var out []Hit
for _, h := range hold {
out = append(out, h...)
}
out = append(out, rrfMerge(rest)...)
return Dedupe(out)
}
var all []Hit
for _, l := range lists {
all = append(all, l.hits...)
}
all = Dedupe(all)
sortBySourceOrder(all)
return all
}
internal/search/sources.go
+59 -19
View File
@@ -167,14 +167,34 @@ func mapBrave(raw []byte) []Hit {
if err := json.Unmarshal(raw, &top); err != nil {
return nil
}
var buckets []any
for _, key := range []string{"sources", "results", "web"} {
if v, ok := top[key]; ok {
buckets = append(buckets, v)
}
}
var hits []Hit
seen := map[string]bool{}
add := func(u, title, snippet string) {
if u == "" || seen[u] {
return
}
seen[u] = true
hits = append(hits, Hit{URL: u, Title: title, Snippet: snippet, Source: NameBraveLLM})
}
// Live LLM Context: grounding.generic[] (and poi) with snippets[].
if g, ok := top["grounding"].(map[string]any); ok {
for _, key := range []string{"generic", "poi"} {
arr, _ := g[key].([]any)
for _, item := range arr {
m, _ := item.(map[string]any)
if m == nil {
continue
}
snip := joinSnippets(m["snippets"])
if snip == "" {
snip = strField(m, "snippet", "description", "content", "text")
}
add(strField(m, "url", "link", "href"), strField(m, "title", "name"), snip)
}
}
}
var walk func(any)
walk = func(v any) {
switch t := v.(type) {
@@ -185,29 +205,49 @@ func mapBrave(raw []byte) []Hit {
case map[string]any:
u := strField(t, "url", "link", "href")
if u == "" {
for _, x := range t {
// URL-keyed object: sources["https://…"] = {title, hostname}
for k, x := range t {
if strings.HasPrefix(k, "http://") || strings.HasPrefix(k, "https://") {
title, snip := "", ""
if m, ok := x.(map[string]any); ok {
title = strField(m, "title", "name", "site_name")
snip = strField(m, "description", "snippet")
}
add(k, title, snip)
continue
}
walk(x)
}
return
}
if seen[u] {
return
}
seen[u] = true
hits = append(hits, Hit{
URL: u,
Title: strField(t, "title", "name"),
Snippet: strField(t, "snippet", "description", "content", "text"),
Source: NameBraveLLM,
})
add(u, strField(t, "title", "name"), strField(t, "snippet", "description", "content", "text"))
}
}
for _, b := range buckets {
walk(b)
for _, key := range []string{"sources", "results", "web"} {
if v, ok := top[key]; ok {
walk(v)
}
}
return hits
}
func joinSnippets(v any) string {
switch t := v.(type) {
case []any:
parts := make([]string, 0, len(t))
for _, x := range t {
if s, ok := x.(string); ok && s != "" {
parts = append(parts, s)
}
}
return strings.Join(parts, "\n\n")
case string:
return t
default:
return ""
}
}
func strField(m map[string]any, keys ...string) string {
for _, k := range keys {
if v, ok := m[k]; ok {
internal/search/sources_test.go
+34
View File
@@ -6,6 +6,7 @@ import (
"io"
"net/http"
"net/http/httptest"
"os"
"strings"
"testing"
"time"
@@ -96,6 +97,39 @@ func TestBraveMapsSources(t *testing.T) {
}
}
func TestBraveMapsGroundingGeneric(t *testing.T) {
raw, err := os.ReadFile("../../tests/testdata/brave_llm_context.json")
if err != nil {
t.Fatal(err)
}
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write(raw)
}))
defer ts.Close()
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: time.Second})
if err != nil {
t.Fatal(err)
}
out := (&BraveLLM{Client: c, Key: "k"}).Search(context.Background(), "q", 5)
if len(out.Hits) != 3 {
t.Fatalf("hits %d %+v", len(out.Hits), out.Hits)
}
if out.Hits[0].URL != "https://docs.tavily.com/documentation/api-reference/endpoint/search" {
t.Fatalf("first %+v", out.Hits[0])
}
if !strings.Contains(out.Hits[0].Snippet, "max_results") {
t.Fatalf("snippet %q", out.Hits[0].Snippet)
}
var urls []string
for _, h := range out.Hits {
urls = append(urls, h.URL)
}
joined := strings.Join(urls, " ")
if !strings.Contains(joined, "orphan.example") {
t.Fatalf("sources map missed: %v", urls)
}
}
func TestTavilyBadJSONIsError(t *testing.T) {
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
_, _ = w.Write([]byte(`not-json`))
tests/testdata/brave_llm_context.json
Vendored
+26
View File
@@ -0,0 +1,26 @@
{
"grounding": {
"generic": [
{
"url": "https://docs.tavily.com/documentation/api-reference/endpoint/search",
"title": "Tavily Search API",
"snippets": ["POST /search with query and max_results.", "search_depth basic is one credit."]
},
{
"url": "https://api.search.brave.com/help",
"title": "Brave Search API",
"snippets": ["LLM Context returns grounding.generic."]
}
]
},
"sources": {
"https://docs.tavily.com/documentation/api-reference/endpoint/search": {
"title": "Tavily Search API",
"hostname": "docs.tavily.com"
},
"https://orphan.example/only-in-sources": {
"title": "Orphan",
"hostname": "orphan.example"
}
}
}