p3: search API, pinned sources, cache, CLI
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
This commit is contained in:
@@ -0,0 +1,81 @@
|
||||
package search
|
||||
|
||||
import (
|
||||
"context"
|
||||
"math"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Embedder interface {
|
||||
Embed(ctx context.Context, inputs []string) ([][]float64, error)
|
||||
Acquire(ctx context.Context) (ok bool, reason string)
|
||||
Release()
|
||||
}
|
||||
|
||||
func Cosine(a, b []float64) float64 {
|
||||
if len(a) == 0 || len(a) != len(b) {
|
||||
return 0
|
||||
}
|
||||
var dot, na, nb float64
|
||||
for i := range a {
|
||||
dot += a[i] * b[i]
|
||||
na += a[i] * a[i]
|
||||
nb += b[i] * b[i]
|
||||
}
|
||||
if na == 0 || nb == 0 {
|
||||
return 0
|
||||
}
|
||||
return dot / (math.Sqrt(na) * math.Sqrt(nb))
|
||||
}
|
||||
|
||||
func rankHits(ctx context.Context, query string, hits []Hit, emb Embedder) (out []Hit, fallback string) {
|
||||
if emb == nil || len(hits) == 0 {
|
||||
if emb == nil {
|
||||
return hits, "no embed"
|
||||
}
|
||||
return hits, ""
|
||||
}
|
||||
ok, reason := emb.Acquire(ctx)
|
||||
if !ok {
|
||||
if reason == "" {
|
||||
reason = "queue_full"
|
||||
}
|
||||
return hits, reason
|
||||
}
|
||||
defer emb.Release()
|
||||
inputs := make([]string, 0, 1+len(hits))
|
||||
inputs = append(inputs, query)
|
||||
for _, h := range hits {
|
||||
s := strings.TrimSpace(h.Title + " " + h.Snippet)
|
||||
if s == "" {
|
||||
s = h.URL
|
||||
}
|
||||
inputs = append(inputs, s)
|
||||
}
|
||||
vecs, err := emb.Embed(ctx, inputs)
|
||||
if err != nil || len(vecs) != len(inputs) {
|
||||
return hits, "embed error"
|
||||
}
|
||||
q := vecs[0]
|
||||
type pair struct {
|
||||
h Hit
|
||||
s float64
|
||||
}
|
||||
ps := make([]pair, len(hits))
|
||||
for i := range hits {
|
||||
ps[i] = pair{h: hits[i], s: Cosine(q, vecs[i+1])}
|
||||
ps[i].h.Score = ps[i].s
|
||||
}
|
||||
for i := 1; i < len(ps); i++ {
|
||||
j := i
|
||||
for j > 0 && ps[j].s > ps[j-1].s {
|
||||
ps[j], ps[j-1] = ps[j-1], ps[j]
|
||||
j--
|
||||
}
|
||||
}
|
||||
out = make([]Hit, len(ps))
|
||||
for i := range ps {
|
||||
out[i] = ps[i].h
|
||||
}
|
||||
return out, ""
|
||||
}
|
||||
Reference in New Issue
Block a user