p3: search API, pinned sources, cache, CLI
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
This commit is contained in:
@@ -0,0 +1,232 @@
|
||||
// Package dial builds an HTTP client pinned to one config URL.
|
||||
// The agent cannot name the target; only the operator-configured host:port is reachable.
|
||||
package dial
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"fmt"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"sync"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/guard"
|
||||
)
|
||||
|
||||
const DefaultRefresh = 10 * time.Minute
|
||||
|
||||
// Lookup resolves a hostname. Tests inject a fake.
|
||||
type Lookup func(ctx context.Context, host string) ([]netip.Addr, error)
|
||||
|
||||
type Options struct {
|
||||
// Loopback permits loopback (and only loopback) addresses. Public sources
|
||||
// must leave this false: every record must pass guard.CheckAddr.
|
||||
Loopback bool
|
||||
Lookup Lookup
|
||||
Refresh time.Duration
|
||||
Timeout time.Duration
|
||||
}
|
||||
|
||||
// Client is an http.Client that may only connect to the config URL's host:port.
|
||||
type Client struct {
|
||||
URL *url.URL
|
||||
HTTP *http.Client
|
||||
Port string
|
||||
|
||||
loopback bool
|
||||
lookup Lookup
|
||||
refresh time.Duration
|
||||
|
||||
mu sync.Mutex
|
||||
ips []netip.Addr
|
||||
resolved time.Time
|
||||
}
|
||||
|
||||
func New(raw string, opt Options) (*Client, error) {
|
||||
u, err := parseConfigURL(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
port := u.Port()
|
||||
if port == "" {
|
||||
if u.Scheme == "https" {
|
||||
port = "443"
|
||||
} else {
|
||||
port = "80"
|
||||
}
|
||||
}
|
||||
if opt.Refresh <= 0 {
|
||||
opt.Refresh = DefaultRefresh
|
||||
}
|
||||
if opt.Timeout <= 0 {
|
||||
opt.Timeout = 8 * time.Second
|
||||
}
|
||||
lookup := opt.Lookup
|
||||
if lookup == nil {
|
||||
lookup = defaultLookup
|
||||
}
|
||||
c := &Client{
|
||||
URL: u,
|
||||
Port: port,
|
||||
loopback: opt.Loopback,
|
||||
lookup: lookup,
|
||||
refresh: opt.Refresh,
|
||||
}
|
||||
if err := c.resolve(context.Background()); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
d := &net.Dialer{
|
||||
Timeout: opt.Timeout,
|
||||
Control: c.control,
|
||||
}
|
||||
tr := &http.Transport{
|
||||
Proxy: nil,
|
||||
DialContext: d.DialContext,
|
||||
ForceAttemptHTTP2: true,
|
||||
TLSHandshakeTimeout: 10 * time.Second,
|
||||
ResponseHeaderTimeout: opt.Timeout,
|
||||
TLSClientConfig: &tls.Config{MinVersion: tls.VersionTLS12}, // system roots; InsecureSkipVerify stays false
|
||||
DisableCompression: false,
|
||||
}
|
||||
c.HTTP = &http.Client{
|
||||
Timeout: opt.Timeout + time.Second,
|
||||
Transport: tr,
|
||||
CheckRedirect: c.checkRedirect,
|
||||
}
|
||||
return c, nil
|
||||
}
|
||||
|
||||
func parseConfigURL(raw string) (*url.URL, error) {
|
||||
u, err := url.Parse(raw)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if u.Scheme != "http" && u.Scheme != "https" {
|
||||
return nil, fmt.Errorf("search upstream: scheme %q", u.Scheme)
|
||||
}
|
||||
if u.User != nil {
|
||||
return nil, fmt.Errorf("search upstream: userinfo not allowed")
|
||||
}
|
||||
if u.Hostname() == "" {
|
||||
return nil, fmt.Errorf("search upstream: empty host")
|
||||
}
|
||||
return u, nil
|
||||
}
|
||||
|
||||
func defaultLookup(ctx context.Context, host string) ([]netip.Addr, error) {
|
||||
if ip, err := netip.ParseAddr(host); err == nil {
|
||||
return []netip.Addr{ip}, nil
|
||||
}
|
||||
return net.DefaultResolver.LookupNetIP(ctx, "ip", host)
|
||||
}
|
||||
|
||||
func (c *Client) resolve(ctx context.Context) error {
|
||||
addrs, err := c.lookup(ctx, c.URL.Hostname())
|
||||
if err != nil {
|
||||
return fmt.Errorf("search resolve: %w", err)
|
||||
}
|
||||
if len(addrs) == 0 {
|
||||
return fmt.Errorf("search resolve: no addresses")
|
||||
}
|
||||
for i, a := range addrs {
|
||||
addrs[i] = a.Unmap()
|
||||
if err := c.checkAddr(addrs[i]); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
c.mu.Lock()
|
||||
c.ips = addrs
|
||||
c.resolved = time.Now()
|
||||
c.mu.Unlock()
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) checkAddr(a netip.Addr) error {
|
||||
if c.loopback {
|
||||
if !a.IsLoopback() {
|
||||
return fmt.Errorf("search loopback source got non-loopback %s", a)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
if err := guard.CheckAddr(a); err != nil {
|
||||
return fmt.Errorf("search public source: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) allow(ip netip.Addr, port string) bool {
|
||||
if port != c.Port {
|
||||
return false
|
||||
}
|
||||
ip = ip.Unmap()
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
for _, a := range c.ips {
|
||||
if a == ip {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func (c *Client) control(network, address string, _ syscall.RawConn) error {
|
||||
switch network {
|
||||
case "tcp", "tcp4", "tcp6":
|
||||
default:
|
||||
return fmt.Errorf("search dial: network %s", network)
|
||||
}
|
||||
host, port, err := net.SplitHostPort(address)
|
||||
if err != nil {
|
||||
return fmt.Errorf("search dial: %w", err)
|
||||
}
|
||||
ip, err := netip.ParseAddr(host)
|
||||
if err != nil {
|
||||
return fmt.Errorf("search dial: not an IP")
|
||||
}
|
||||
if !c.allow(ip, port) {
|
||||
return fmt.Errorf("search dial refused")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (c *Client) checkRedirect(req *http.Request, via []*http.Request) error {
|
||||
if len(via) > 1 {
|
||||
return fmt.Errorf("search redirect: extra hop")
|
||||
}
|
||||
if len(via) == 1 && !strings.EqualFold(req.URL.Hostname(), via[0].URL.Hostname()) {
|
||||
return fmt.Errorf("search redirect: different host")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// RefreshIfStale re-resolves when the pin is older than Refresh.
|
||||
func (c *Client) RefreshIfStale(ctx context.Context) error {
|
||||
c.mu.Lock()
|
||||
stale := time.Since(c.resolved) >= c.refresh
|
||||
c.mu.Unlock()
|
||||
if !stale {
|
||||
return nil
|
||||
}
|
||||
return c.resolve(ctx)
|
||||
}
|
||||
|
||||
func (c *Client) IPs() []netip.Addr {
|
||||
c.mu.Lock()
|
||||
defer c.mu.Unlock()
|
||||
out := make([]netip.Addr, len(c.ips))
|
||||
copy(out, c.ips)
|
||||
return out
|
||||
}
|
||||
|
||||
func (c *Client) TLSConfig() *tls.Config {
|
||||
tr, _ := c.HTTP.Transport.(*http.Transport)
|
||||
if tr == nil {
|
||||
return nil
|
||||
}
|
||||
return tr.TLSClientConfig
|
||||
}
|
||||
@@ -0,0 +1,112 @@
|
||||
package dial_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/netip"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.wntrmute.dev/kyle/sift/internal/search/dial"
|
||||
)
|
||||
|
||||
func TestLoopbackAllowsConfiguredPort(t *testing.T) {
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(200)
|
||||
_, _ = w.Write([]byte("ok"))
|
||||
}))
|
||||
defer ts.Close()
|
||||
c, err := dial.New(ts.URL, dial.Options{Loopback: true, Timeout: 0})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := c.HTTP.Get(ts.URL + "/")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatalf("status %d", resp.StatusCode)
|
||||
}
|
||||
}
|
||||
|
||||
func TestLoopbackRefusesOtherPort(t *testing.T) {
|
||||
c, err := dial.New("http://127.0.0.1:8888", dial.Options{
|
||||
Loopback: true,
|
||||
Lookup: func(_ context.Context, _ string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := c.HTTP.Get("http://127.0.0.1:11434/")
|
||||
if err == nil {
|
||||
resp.Body.Close()
|
||||
t.Fatal("dialed other loopback port")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "refused") && !strings.Contains(err.Error(), "search dial") {
|
||||
t.Fatalf("err %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicPoisonedPrivateRefused(t *testing.T) {
|
||||
_, err := dial.New("https://api.tavily.com/search", dial.Options{
|
||||
Loopback: false,
|
||||
Lookup: func(_ context.Context, host string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("127.0.0.1")}, nil
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("poisoned A record allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestPublicPrivateRFC1918Refused(t *testing.T) {
|
||||
_, err := dial.New("https://api.search.brave.com/res/v1/llm/context", dial.Options{
|
||||
Lookup: func(_ context.Context, _ string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("10.0.0.5")}, nil
|
||||
},
|
||||
})
|
||||
if err == nil {
|
||||
t.Fatal("private A allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestTLSSystemRoots(t *testing.T) {
|
||||
c, err := dial.New("https://api.tavily.com/search", dial.Options{
|
||||
Lookup: func(_ context.Context, _ string) ([]netip.Addr, error) {
|
||||
return []netip.Addr{netip.MustParseAddr("1.1.1.1")}, nil
|
||||
},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
cfg := c.TLSConfig()
|
||||
if cfg == nil {
|
||||
t.Fatal("nil tls")
|
||||
}
|
||||
if cfg.InsecureSkipVerify {
|
||||
t.Fatal("InsecureSkipVerify")
|
||||
}
|
||||
if cfg.RootCAs != nil {
|
||||
t.Fatal("want system roots (nil RootCAs)")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNoUserinfo(t *testing.T) {
|
||||
_, err := dial.New("http://u:p@127.0.0.1:8888", dial.Options{Loopback: true})
|
||||
if err == nil {
|
||||
t.Fatal("userinfo allowed")
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseRejectsNonHTTP(t *testing.T) {
|
||||
_, err := dial.New("ftp://example.com/", dial.Options{})
|
||||
if err == nil {
|
||||
t.Fatal("ftp allowed")
|
||||
}
|
||||
_ = url.URL{}
|
||||
}
|
||||
Reference in New Issue
Block a user