From 9a8dae3948d97e33afa6e415abd7a5b138a954fd Mon Sep 17 00:00:00 2001 From: Kyle Isom Date: Wed, 23 Sep 2026 14:09:42 -0700 Subject: [PATCH] p3: harvest 28 bake-off queries with targets and blind protocol Kyle-approved fleet harvest. Sibling targets (domain/page or open). Runner labels A-D; mapping stays in a sealed key file until scoring. --- .gitignore | 1 + docs/P3.md | 15 +++-- tests/bakeoff/blind.py | 95 +++++++++++++++++++++++++++++++ tests/testdata/BAKEOFF.md | 42 ++++++++++++++ tests/testdata/search_queries.txt | 28 +++++++++ tests/testdata/search_targets.txt | 31 ++++++++++ 6 files changed, 206 insertions(+), 6 deletions(-) create mode 100644 tests/bakeoff/blind.py create mode 100644 tests/testdata/BAKEOFF.md create mode 100644 tests/testdata/search_queries.txt create mode 100644 tests/testdata/search_targets.txt diff --git a/.gitignore b/.gitignore index 8ca64e5..d02678b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ /result-* *.test /sift +/tests/testdata/bakeoff-out/ diff --git a/docs/P3.md b/docs/P3.md index bb87926..216f705 100644 --- a/docs/P3.md +++ b/docs/P3.md @@ -266,11 +266,13 @@ Confidentiality: ## 10. Bake-off (acceptance) -A fixed set of **~30 real queries** harvested from recent **fleet** `web_search` use +A fixed set of **28 real queries** harvested from recent **fleet** `web_search` use (gateway/`agent.log` on straylight, talos, titan). **Exclude** the guardian-star profile and anything from `permit_restricted` callers. Deduped, no secrets. Kyle -sees the list before it is committed as `tests/testdata/search_queries.txt`. Do not -invent the set. +approved the list; it is `tests/testdata/search_queries.txt`. Do not invent the set. + +Sibling `tests/testdata/search_targets.txt`: for each query, the page or domain a +good result set must contain, or `open` when there isn't a clear one. For each query, run four configurations and record top-k URLs + latency + credits: @@ -279,9 +281,10 @@ For each query, run four configurations and record top-k URLs + latency + credit 3. SearXNG + Brave LLM Context + Tavily 4. Current Hermes Tavily `web_search` (the live tool, not sift) -Judged **blind** on whether the result set would get the agent to the right source -(would you `sift fetch` the correct URL from this list?). Scoring notes live next to -the query file; the judge does not see the configuration name. +Judged **blind**. The runner writes result packs labelled A–D with no config names. +The A–D → 1–4 mapping is a separate key file the judge does not read until scoring +is done. Protocol: `tests/testdata/BAKEOFF.md`. Shuffle helper: `tests/bakeoff/blind.py`. +Judges: talos or Claude. Score: would you `sift fetch` the correct URL from this list? Report in this thread: diff --git a/tests/bakeoff/blind.py b/tests/bakeoff/blind.py new file mode 100644 index 0000000..01df090 --- /dev/null +++ b/tests/bakeoff/blind.py @@ -0,0 +1,95 @@ +#!/usr/bin/env python3 +"""Shuffle four named result dumps into A–D and write a sealed key file.""" + +from __future__ import annotations + +import argparse +import json +import random +import sys +from pathlib import Path + +LABELS = ("A", "B", "C", "D") +CONFIGS = ( + "searxng", + "searxng_brave", + "searxng_brave_tavily", + "hermes_tavily", +) + + +def load_queries(path: Path) -> list[str]: + return [ln.strip() for ln in path.read_text().splitlines() if ln.strip()] + + +def load_targets(path: Path) -> list[str]: + out: list[str] = [] + for ln in path.read_text().splitlines(): + s = ln.strip() + if not s or s.startswith("#"): + continue + out.append(s) + return out + + +def main() -> int: + p = argparse.ArgumentParser(description=__doc__) + p.add_argument( + "--raw-dir", + type=Path, + required=True, + help="Directory with {searxng,searxng_brave,searxng_brave_tavily,hermes_tavily}.json", + ) + p.add_argument( + "--out-dir", + type=Path, + default=Path("tests/testdata/bakeoff-out"), + ) + p.add_argument( + "--queries", + type=Path, + default=Path("tests/testdata/search_queries.txt"), + ) + p.add_argument( + "--targets", + type=Path, + default=Path("tests/testdata/search_targets.txt"), + ) + p.add_argument("--seed", type=int, default=None) + args = p.parse_args() + + queries = load_queries(args.queries) + targets = load_targets(args.targets) + if len(queries) != len(targets): + print( + f"query/target length mismatch: {len(queries)} vs {len(targets)}", + file=sys.stderr, + ) + return 1 + + packs = {} + for name in CONFIGS: + f = args.raw_dir / f"{name}.json" + packs[name] = json.loads(f.read_text()) + + rng = random.Random(args.seed) + order = list(CONFIGS) + rng.shuffle(order) + key = {lab: cfg for lab, cfg in zip(LABELS, order)} + + args.out_dir.mkdir(parents=True, exist_ok=True) + for lab, cfg in key.items(): + payload = packs[cfg] + if isinstance(payload, dict): + payload = {k: v for k, v in payload.items() if k not in ("config", "sources")} + (args.out_dir / f"{lab}.json").write_text(json.dumps(payload, indent=2) + "\n") + + key_path = args.out_dir / "key.json" + key_path.write_text(json.dumps({"labels": key, "n": len(queries)}, indent=2) + "\n") + print(f"wrote {args.out_dir}/{{A,B,C,D}}.json") + print(f"key sealed at {key_path} — judges do not open this until scoring is done") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/testdata/BAKEOFF.md b/tests/testdata/BAKEOFF.md new file mode 100644 index 0000000..2a1513d --- /dev/null +++ b/tests/testdata/BAKEOFF.md @@ -0,0 +1,42 @@ +# P3 §10 bake-off protocol + +Query list: `search_queries.txt` (28, Kyle-approved 2026-09-23). +Targets: `search_targets.txt` (same order; `open` = no required domain). + +Do not enable sources or start runs until this commit is on `kyle/sift` master. +SearXNG journal redaction (`searx.service`) is a prerequisite for `search.searxng.enabled`. + +## Configurations (never shown to the judge) + +1. SearXNG alone +2. SearXNG + Brave LLM Context +3. SearXNG + Brave LLM Context + Tavily +4. Current Hermes Tavily `web_search` (live tool, not sift) + +Sift does not take source names from the request. Configs 1–3 are three orion +`[search]` settings (or three sequential enables), not query parameters. +Config 4 is a fleet-host Hermes call, same 28 queries, `limit` matching sift `k`. + +Record per query: top-k URLs, titles, snippets, latency, Tavily credits, Brave calls. + +## Blind judging + +The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is the +full 28-query result set for one configuration, with no source names, no +warnings that name a vendor, and no config id. + +The mapping `A–D → 1–4` goes in a **separate key file** that the judge does +not read until scoring is done. Default path (gitignored): + +``` +tests/testdata/bakeoff-out/key.json +tests/testdata/bakeoff-out/{A,B,C,D}.json +``` + +Shuffle with `tests/bakeoff/blind.py` after the four raw dumps exist. + +Judges: **talos or Claude**. Score each query: would you `sift fetch` the +correct URL from this list? Against `search_targets.txt` when it is not `open`. + +After scores land, open `key.json` and report: winner (becomes default-on), +cost/query, p90 latency. diff --git a/tests/testdata/search_queries.txt b/tests/testdata/search_queries.txt new file mode 100644 index 0000000..b64718b --- /dev/null +++ b/tests/testdata/search_queries.txt @@ -0,0 +1,28 @@ +firecrawl-py 4.17.0 search method POST /v2/search query limit +Tavily API search extract crawl features pricing 2026 +Brave Search API vs Brave LLM API web search agent 2026 +vLLM PagedAttention what is vLLM serving engine +Google A2A Agent2Agent protocol agent communication +Hermes Agent Nous Research prompt engineering skills AGENTS.md +site:docs.anthropic.com agentic coding prompt engineering Claude Code +Rozado PLOS ONE political preferences of LLMs 2024 +LunaDrive delay-compensated high-voltage GaN FET motor driver site:arxiv.org +"PQLN" Lightning Kurt 2609.13781 +site:arxiv.org Tim Dettmers 2026 +bitsandbytes2 Dettmers runtime dynamic compression +SK Hynix CEO 2027 worst supply year DRAM HBM shortage 2030 +NVIDIA DGX Spark official price 2026 +Grok 4.7 regression vs 4.5 OR 4.6 +Framework Desktop 128GB 192GB price 2026 Ryzen AI Max +Xiaomi MiMo-V2.6-Pro open weights Hugging Face +ASUS DUAL-RTX3060-012G-V2 specifications TDP length dual slot +MSI PRO B760-P WIFI DDR4 PCIe x16 slot layout dual GPU x4 +CubeMars RO80 frameless BLDC weight torque KV specifications +SanDisk Optimus GX 7100M 2TB M.2 2230 NVMe specs +vLLM Apple Silicon Metal support site:docs.vllm.ai +obsidian livesync headless couchdb sync without desktop app +Omarchy Linux ISO download 4.0 site:omarchy.org OR site:omacom.io +KittenML kitten-tts-nano huggingface model voice quality +Nemotron 3 Diarization Hugging Face NVIDIA +Qwen3.8-Flash-Next GGUF size Q4_K_M parameters Hugging Face +bendlang bend github agentic programming language diff --git a/tests/testdata/search_targets.txt b/tests/testdata/search_targets.txt new file mode 100644 index 0000000..b8d2e13 --- /dev/null +++ b/tests/testdata/search_targets.txt @@ -0,0 +1,31 @@ +# Sibling of search_queries.txt. Line N (ignoring this comment and blanks) is the +# target for query N. A good result set must contain this page or domain. +# `open` means no required hit — score usefulness only. +docs.firecrawl.dev +docs.tavily.com +brave.com +docs.vllm.ai +a2a-protocol.org +hermes-agent.nousresearch.com +docs.anthropic.com +journals.plos.org +arxiv.org +arxiv.org/abs/2609.13781 +arxiv.org +github.com/bitsandbytes-foundation +open +nvidia.com +open +frame.work +huggingface.co +asus.com +msi.com +cubemars.com +sandisk.com +docs.vllm.ai +github.com/vrtmrz/obsidian-livesync +omarchy.org +huggingface.co +huggingface.co +huggingface.co +github.com/bendlang/bend