p3: search API, pinned sources, cache, CLI

POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and
Tavily from config-fixed endpoints. Public sources refuse private A
records; SearXNG is a loopback-only client, not the fetch guard.

Cache keys include caller class. Restricted callers search like the
fleet; logs omit query, hosts, URLs, and upstream errors. Degraded
results are not stored. Tavily sends search_depth=basic and drops
answer. Missing keys surface in /healthz.

Default-on sources stay off until the bake-off. Query harvest is not
in this commit.
This commit is contained in:
2026-09-23 12:17:51 -07:00
parent 3687636ed7
commit d7873403ff
27 changed files with 2361 additions and 21 deletions
+74
View File
@@ -21,11 +21,52 @@ ROOT = Path(__file__).resolve().parents[2]
BIN = ROOT / "bin" / "sift-test"
# Replica of hermes-agent plugins/web/firecrawl/provider.py _extract_web_search_results
# (Hermes 0.21). Keep in sync if that function changes.
def _normalize_result_list(items):
if not isinstance(items, list):
return []
out = []
for it in items:
if isinstance(it, dict) and it.get("url"):
out.append(it)
return out
def extract_web_search_results(response):
plain = response
if hasattr(response, "model_dump"):
try:
plain = response.model_dump()
except Exception:
plain = response
if isinstance(plain, dict):
data = plain.get("data")
if isinstance(data, list):
return _normalize_result_list(data)
candidates = [data.get("web"), data.get("results")] if isinstance(data, dict) else []
for candidate in candidates + [plain.get("web"), plain.get("results")]:
normalized = _normalize_result_list(candidate)
if normalized:
return normalized
if hasattr(response, "web"):
return _normalize_result_list(getattr(response, "web", []))
return []
class Handler(BaseHTTPRequestHandler):
def log_message(self, *args):
pass
def do_GET(self):
if self.path.startswith("/search"):
body = b'{"results":[{"url":"https://example.com/contract","title":"Contract","content":"hello search","score":1}]}'
self.send_response(200)
self.send_header("Content-Type", "application/json")
self.send_header("Content-Length", str(len(body)))
self.end_headers()
self.wfile.write(body)
return
if self.path.startswith("/forbid"):
body = b"nope"
self.send_response(403)
@@ -81,6 +122,19 @@ def main() -> int:
keys.write_text(
f'[[caller]]\nname = "contract"\nworkspace = "hermes-fleet"\nkey_sha256 = "{digest}"\n'
)
cfg = tmp / "config.toml"
cfg.write_text(
f"""
[search]
k = 8
timeout = "8s"
rerank = false
[search.searxng]
enabled = true
url = "http://127.0.0.1:{fx_port}"
"""
)
listen = f"127.0.0.1:{free_port()}"
logf = tmp / "sift.log"
@@ -92,6 +146,8 @@ def main() -> int:
listen,
"-keys",
str(keys),
"-config",
str(cfg),
"-test-allow-prefix",
"127.0.0.0/8",
],
@@ -158,6 +214,24 @@ def main() -> int:
msg = scrape_err(page, key="wrong")
assert "401" in msg or "Unauthorized" in msg or "unauthorized" in msg.lower(), msg
sreq = urllib.request.Request(
f"http://{listen}/v2/search",
data=json.dumps({"query": "contract", "limit": 3}).encode(),
headers={
"Authorization": f"Bearer {raw}",
"Content-Type": "application/json",
},
method="POST",
)
with urllib.request.urlopen(sreq, timeout=10) as resp:
senv = json.loads(resp.read())
extracted = extract_web_search_results(senv)
assert extracted and extracted[0].get("url"), senv
found = app.search(query="contract", limit=3)
sdk_hits = extract_web_search_results(found)
assert sdk_hits, found
print("firecrawl-py SDK contract passed")
return 0
finally: