p3: harvest 28 bake-off queries with targets and blind protocol

Kyle-approved fleet harvest. Sibling targets (domain/page or open).
Runner labels A-D; mapping stays in a sealed key file until scoring.

tests/bakeoff/blind.py

@@ -0,0 +1,95 @@
+#!/usr/bin/env python3
+"""Shuffle four named result dumps into A–D and write a sealed key file."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import random
+import sys
+from pathlib import Path
+
+LABELS = ("A", "B", "C", "D")
+CONFIGS = (
+    "searxng",
+    "searxng_brave",
+    "searxng_brave_tavily",
+    "hermes_tavily",
+)
+
+
+def load_queries(path: Path) -> list[str]:
+    return [ln.strip() for ln in path.read_text().splitlines() if ln.strip()]
+
+
+def load_targets(path: Path) -> list[str]:
+    out: list[str] = []
+    for ln in path.read_text().splitlines():
+        s = ln.strip()
+        if not s or s.startswith("#"):
+            continue
+        out.append(s)
+    return out
+
+
+def main() -> int:
+    p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument(
+        "--raw-dir",
+        type=Path,
+        required=True,
+        help="Directory with {searxng,searxng_brave,searxng_brave_tavily,hermes_tavily}.json",
+    )
+    p.add_argument(
+        "--out-dir",
+        type=Path,
+        default=Path("tests/testdata/bakeoff-out"),
+    )
+    p.add_argument(
+        "--queries",
+        type=Path,
+        default=Path("tests/testdata/search_queries.txt"),
+    )
+    p.add_argument(
+        "--targets",
+        type=Path,
+        default=Path("tests/testdata/search_targets.txt"),
+    )
+    p.add_argument("--seed", type=int, default=None)
+    args = p.parse_args()
+
+    queries = load_queries(args.queries)
+    targets = load_targets(args.targets)
+    if len(queries) != len(targets):
+        print(
+            f"query/target length mismatch: {len(queries)} vs {len(targets)}",
+            file=sys.stderr,
+        )
+        return 1
+
+    packs = {}
+    for name in CONFIGS:
+        f = args.raw_dir / f"{name}.json"
+        packs[name] = json.loads(f.read_text())
+
+    rng = random.Random(args.seed)
+    order = list(CONFIGS)
+    rng.shuffle(order)
+    key = {lab: cfg for lab, cfg in zip(LABELS, order)}
+
+    args.out_dir.mkdir(parents=True, exist_ok=True)
+    for lab, cfg in key.items():
+        payload = packs[cfg]
+        if isinstance(payload, dict):
+            payload = {k: v for k, v in payload.items() if k not in ("config", "sources")}
+        (args.out_dir / f"{lab}.json").write_text(json.dumps(payload, indent=2) + "\n")
+
+    key_path = args.out_dir / "key.json"
+    key_path.write_text(json.dumps({"labels": key, "n": len(queries)}, indent=2) + "\n")
+    print(f"wrote {args.out_dir}/{{A,B,C,D}}.json")
+    print(f"key sealed at {key_path} — judges do not open this until scoring is done")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())