kyle
d7873403ff
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
41 lines
880 B
Go
41 lines
880 B
Go
package search
|
|
|
|
import (
|
|
"net/url"
|
|
"strings"
|
|
)
|
|
|
|
// CanonicalURL lowercases scheme+host, strips default ports and fragments,
|
|
// and drops a trailing slash on the path. www. is left as-is.
|
|
func CanonicalURL(raw string) string {
|
|
raw = strings.TrimSpace(raw)
|
|
if raw == "" {
|
|
return ""
|
|
}
|
|
u, err := url.Parse(raw)
|
|
if err != nil || u.Host == "" && u.Opaque == "" {
|
|
return strings.TrimRight(raw, "/")
|
|
}
|
|
u.Fragment = ""
|
|
u.Scheme = strings.ToLower(u.Scheme)
|
|
host := strings.ToLower(u.Hostname())
|
|
port := u.Port()
|
|
switch {
|
|
case port == "80" && u.Scheme == "http":
|
|
port = ""
|
|
case port == "443" && u.Scheme == "https":
|
|
port = ""
|
|
}
|
|
if port != "" {
|
|
u.Host = host + ":" + port
|
|
} else {
|
|
u.Host = host
|
|
}
|
|
if u.Path == "/" {
|
|
u.Path = ""
|
|
} else if len(u.Path) > 1 && strings.HasSuffix(u.Path, "/") {
|
|
u.Path = strings.TrimSuffix(u.Path, "/")
|
|
}
|
|
return u.String()
|
|
}
|