p3: search API, pinned sources, cache, CLI
POST /v1/search and /v2/search fan in SearXNG, Brave LLM Context, and Tavily from config-fixed endpoints. Public sources refuse private A records; SearXNG is a loopback-only client, not the fetch guard. Cache keys include caller class. Restricted callers search like the fleet; logs omit query, hosts, URLs, and upstream errors. Degraded results are not stored. Tavily sends search_depth=basic and drops answer. Missing keys surface in /healthz. Default-on sources stay off until the bake-off. Query harvest is not in this commit.
This commit is contained in:
@@ -21,11 +21,52 @@ ROOT = Path(__file__).resolve().parents[2]
|
||||
BIN = ROOT / "bin" / "sift-test"
|
||||
|
||||
|
||||
# Replica of hermes-agent plugins/web/firecrawl/provider.py _extract_web_search_results
|
||||
# (Hermes 0.21). Keep in sync if that function changes.
|
||||
def _normalize_result_list(items):
|
||||
if not isinstance(items, list):
|
||||
return []
|
||||
out = []
|
||||
for it in items:
|
||||
if isinstance(it, dict) and it.get("url"):
|
||||
out.append(it)
|
||||
return out
|
||||
|
||||
|
||||
def extract_web_search_results(response):
|
||||
plain = response
|
||||
if hasattr(response, "model_dump"):
|
||||
try:
|
||||
plain = response.model_dump()
|
||||
except Exception:
|
||||
plain = response
|
||||
if isinstance(plain, dict):
|
||||
data = plain.get("data")
|
||||
if isinstance(data, list):
|
||||
return _normalize_result_list(data)
|
||||
candidates = [data.get("web"), data.get("results")] if isinstance(data, dict) else []
|
||||
for candidate in candidates + [plain.get("web"), plain.get("results")]:
|
||||
normalized = _normalize_result_list(candidate)
|
||||
if normalized:
|
||||
return normalized
|
||||
if hasattr(response, "web"):
|
||||
return _normalize_result_list(getattr(response, "web", []))
|
||||
return []
|
||||
|
||||
|
||||
class Handler(BaseHTTPRequestHandler):
|
||||
def log_message(self, *args):
|
||||
pass
|
||||
|
||||
def do_GET(self):
|
||||
if self.path.startswith("/search"):
|
||||
body = b'{"results":[{"url":"https://example.com/contract","title":"Contract","content":"hello search","score":1}]}'
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", "application/json")
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
self.end_headers()
|
||||
self.wfile.write(body)
|
||||
return
|
||||
if self.path.startswith("/forbid"):
|
||||
body = b"nope"
|
||||
self.send_response(403)
|
||||
@@ -81,6 +122,19 @@ def main() -> int:
|
||||
keys.write_text(
|
||||
f'[[caller]]\nname = "contract"\nworkspace = "hermes-fleet"\nkey_sha256 = "{digest}"\n'
|
||||
)
|
||||
cfg = tmp / "config.toml"
|
||||
cfg.write_text(
|
||||
f"""
|
||||
[search]
|
||||
k = 8
|
||||
timeout = "8s"
|
||||
rerank = false
|
||||
|
||||
[search.searxng]
|
||||
enabled = true
|
||||
url = "http://127.0.0.1:{fx_port}"
|
||||
"""
|
||||
)
|
||||
|
||||
listen = f"127.0.0.1:{free_port()}"
|
||||
logf = tmp / "sift.log"
|
||||
@@ -92,6 +146,8 @@ def main() -> int:
|
||||
listen,
|
||||
"-keys",
|
||||
str(keys),
|
||||
"-config",
|
||||
str(cfg),
|
||||
"-test-allow-prefix",
|
||||
"127.0.0.0/8",
|
||||
],
|
||||
@@ -158,6 +214,24 @@ def main() -> int:
|
||||
msg = scrape_err(page, key="wrong")
|
||||
assert "401" in msg or "Unauthorized" in msg or "unauthorized" in msg.lower(), msg
|
||||
|
||||
sreq = urllib.request.Request(
|
||||
f"http://{listen}/v2/search",
|
||||
data=json.dumps({"query": "contract", "limit": 3}).encode(),
|
||||
headers={
|
||||
"Authorization": f"Bearer {raw}",
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
method="POST",
|
||||
)
|
||||
with urllib.request.urlopen(sreq, timeout=10) as resp:
|
||||
senv = json.loads(resp.read())
|
||||
extracted = extract_web_search_results(senv)
|
||||
assert extracted and extracted[0].get("url"), senv
|
||||
|
||||
found = app.search(query="contract", limit=3)
|
||||
sdk_hits = extract_web_search_results(found)
|
||||
assert sdk_hits, found
|
||||
|
||||
print("firecrawl-py SDK contract passed")
|
||||
return 0
|
||||
finally:
|
||||
|
||||
Reference in New Issue
Block a user