kyle
d7873403ff
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
82 lines
1.6 KiB
Go
82 lines
1.6 KiB
Go
package search
|
|
|
|
import (
|
|
"context"
|
|
"math"
|
|
"strings"
|
|
)
|
|
|
|
type Embedder interface {
|
|
Embed(ctx context.Context, inputs []string) ([][]float64, error)
|
|
Acquire(ctx context.Context) (ok bool, reason string)
|
|
Release()
|
|
}
|
|
|
|
func Cosine(a, b []float64) float64 {
|
|
if len(a) == 0 || len(a) != len(b) {
|
|
return 0
|
|
}
|
|
var dot, na, nb float64
|
|
for i := range a {
|
|
dot += a[i] * b[i]
|
|
na += a[i] * a[i]
|
|
nb += b[i] * b[i]
|
|
}
|
|
if na == 0 || nb == 0 {
|
|
return 0
|
|
}
|
|
return dot / (math.Sqrt(na) * math.Sqrt(nb))
|
|
}
|
|
|
|
func rankHits(ctx context.Context, query string, hits []Hit, emb Embedder) (out []Hit, fallback string) {
|
|
if emb == nil || len(hits) == 0 {
|
|
if emb == nil {
|
|
return hits, "no embed"
|
|
}
|
|
return hits, ""
|
|
}
|
|
ok, reason := emb.Acquire(ctx)
|
|
if !ok {
|
|
if reason == "" {
|
|
reason = "queue_full"
|
|
}
|
|
return hits, reason
|
|
}
|
|
defer emb.Release()
|
|
inputs := make([]string, 0, 1+len(hits))
|
|
inputs = append(inputs, query)
|
|
for _, h := range hits {
|
|
s := strings.TrimSpace(h.Title + " " + h.Snippet)
|
|
if s == "" {
|
|
s = h.URL
|
|
}
|
|
inputs = append(inputs, s)
|
|
}
|
|
vecs, err := emb.Embed(ctx, inputs)
|
|
if err != nil || len(vecs) != len(inputs) {
|
|
return hits, "embed error"
|
|
}
|
|
q := vecs[0]
|
|
type pair struct {
|
|
h Hit
|
|
s float64
|
|
}
|
|
ps := make([]pair, len(hits))
|
|
for i := range hits {
|
|
ps[i] = pair{h: hits[i], s: Cosine(q, vecs[i+1])}
|
|
ps[i].h.Score = ps[i].s
|
|
}
|
|
for i := 1; i < len(ps); i++ {
|
|
j := i
|
|
for j > 0 && ps[j].s > ps[j-1].s {
|
|
ps[j], ps[j-1] = ps[j-1], ps[j]
|
|
j--
|
|
}
|
|
}
|
|
out = make([]Hit, len(ps))
|
|
for i := range ps {
|
|
out[i] = ps[i].h
|
|
}
|
|
return out, ""
|
|
}
|