p3: search API, pinned sources, cache, CLI

POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and
Tavily from config-fixed endpoints. Public sources refuse private A
records; SearXNG is a loopback-only client, not the fetch guard.

Cache keys include caller class. Restricted callers search like the
fleet; logs omit query, hosts, URLs, and upstream errors. Degraded
results are not stored. Tavily sends search_depth=basic and drops
answer. Missing keys surface in /healthz.

Default-on sources stay off until the bake-off. Query harvest is not
in this commit.

cmd/sift/main.go

@@ -20,6 +20,8 @@ func main() {
 		serveMain()
 	case "fetch":
 		fetchMain()
+	case "search":
+		searchMain()
 	case "keys":
 		keysMain()
 	case "version", "-version", "--version":
@@ -39,6 +41,7 @@ func usage() {
 Commands:
   sift serve    [flags]     run the service
   sift fetch    <url>       fetch via the service
+  sift search   "<query>"   search via the service
   sift keys     add|list|revoke
   sift version
 `, version)

cmd/sift/searchcmd.go

@@ -0,0 +1,86 @@
+package main
+
+import (
+	"bytes"
+	"encoding/json"
+	"flag"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+)
+
+func searchMain() {
+	fs := flag.NewFlagSet("search", flag.ContinueOnError)
+	fs.SetOutput(io.Discard)
+	k := fs.Int("k", 0, "result cap")
+	asJSON := fs.Bool("json", false, "native envelope")
+	args := os.Args[1:]
+	flagArgs, pos, err := splitAroundPositional(args, map[string]bool{"k": true})
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(2)
+	}
+	if err := fs.Parse(flagArgs); err != nil {
+		fmt.Fprintln(os.Stderr, searchUsage)
+		os.Exit(2)
+	}
+	if len(pos) != 1 {
+		fmt.Fprintln(os.Stderr, searchUsage)
+		os.Exit(2)
+	}
+	query := pos[0]
+	base, key, err := loadClient()
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	body := map[string]any{"query": query}
+	if *k > 0 {
+		body["k"] = *k
+	}
+	raw, _ := json.Marshal(body)
+	req, err := http.NewRequest(http.MethodPost, base+"/v1/search", bytes.NewReader(raw))
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	req.Header.Set("Authorization", "Bearer "+key)
+	req.Header.Set("Content-Type", "application/json")
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	defer resp.Body.Close()
+	var env map[string]any
+	if err := json.NewDecoder(resp.Body).Decode(&env); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(5)
+	}
+	if resp.StatusCode != 200 {
+		msg, _ := env["error"].(string)
+		fmt.Fprintln(os.Stderr, msg)
+		os.Exit(exitFor(resp.StatusCode, msg))
+	}
+	if *asJSON {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		_ = enc.Encode(env)
+		return
+	}
+	results, _ := env["results"].([]any)
+	for i, it := range results {
+		m, _ := it.(map[string]any)
+		if i > 0 {
+			fmt.Println()
+		}
+		fmt.Println(m["title"])
+		fmt.Println(m["url"])
+		if sn, _ := m["snippet"].(string); sn != "" {
+			fmt.Println(sn)
+		}
+	}
+}
+
+const searchUsage = `usage: sift search "<query>" [--k N] [--json]`

cmd/sift/serve.go

@@ -19,6 +19,7 @@ import (
 	"git.wntrmute.dev/kyle/sift/internal/fetch"
 	"git.wntrmute.dev/kyle/sift/internal/fleetctx"
 	"git.wntrmute.dev/kyle/sift/internal/guard"
+	"git.wntrmute.dev/kyle/sift/internal/search"
 	"git.wntrmute.dev/kyle/sift/internal/selfip"
 )
 
@@ -95,17 +96,25 @@ func serveMain() {
 	parent, stop := context.WithCancel(context.Background())
 	defer stop()
 
+	loadSecretEnv(secretEnvPath(), log)
+
 	srv := api.New(version, g, fetcher, store, log)
 	srv.PermitRestricted = cfg.Context.PermitRestricted
-	if cfg.Dixie.Chat != "" {
-		cachePath := cfg.Digest.Cache
-		if cachePath == "" {
-			cachePath = "/var/lib/sift/cache.db"
-		}
-		cache, err := dcache.Open(cachePath, 500<<20, log)
+
+	cachePath := cfg.Digest.Cache
+	if cachePath == "" {
+		cachePath = "/var/lib/sift/cache.db"
+	}
+	var cache *dcache.DB
+	if cfg.Dixie.Chat != "" || cfg.Search.AnyEnabled() {
+		c, err := dcache.Open(cachePath, 500<<20, log)
 		if err != nil {
-			log.Warn("digest cache", "err", err)
+			log.Warn("cache", "err", err)
+		} else {
+			cache = c
 		}
+	}
+	if cfg.Dixie.Chat != "" {
 		srv.Digest = &digest.Engine{
 			ChatURL:    cfg.Dixie.Chat,
 			EmbedURL:   cfg.Dixie.Embed,
@@ -117,6 +126,11 @@ func serveMain() {
 			Log:        log,
 		}
 	}
+	var emb search.Embedder
+	if srv.Digest != nil {
+		emb = srv.Digest
+	}
+	srv.Search = search.Build(cfg.Search, cache, emb, log)
 	if cfg.Context.Honcho != "" {
 		allow := cfg.Context.Workspaces
 		if len(allow) == 0 {
@@ -194,3 +208,37 @@ func serveMain() {
 		}
 	}
 }
+
+func secretEnvPath() string {
+	if p := os.Getenv("SIFT_SECRET"); p != "" {
+		return p
+	}
+	return "/var/lib/sift/secret.env"
+}
+
+// loadSecretEnv sets KEY=value from the on-box file when the process env
+// does not already have the key. Never logs values.
+func loadSecretEnv(path string, log *slog.Logger) {
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return
+	}
+	for _, line := range strings.Split(string(b), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" || strings.HasPrefix(line, "#") {
+			continue
+		}
+		k, v, ok := strings.Cut(line, "=")
+		if !ok {
+			continue
+		}
+		k = strings.TrimSpace(k)
+		v = strings.TrimSpace(v)
+		v = strings.Trim(v, `"'`)
+		if k == "" || os.Getenv(k) != "" {
+			continue
+		}
+		_ = os.Setenv(k, v)
+	}
+	_ = log
+}