p3: actually blind bake-off packs

Rewrite A-D to [{rank, url, title, snippet}] only. Fail closed if
searxng/brave/tavily/hermes appear outside url or snippet. Self-test
and a 28-query fake-dump dry run pass.

docs/P3.md

@@ -281,7 +281,8 @@ For each query, run four configurations and record top-k URLs + latency + credit
 3. SearXNG + Brave LLM Context + Tavily
 4. Current Hermes Tavily `web_search` (the live tool, not sift)
 
-Judged **blind**. The runner writes result packs labelled A–D with no config names.
+Judged **blind**. The runner writes result packs labelled A–D; each pack is
+only `[{rank, url, title, snippet}, …]` per query, no scores/sources/warnings.
 The A–D → 1–4 mapping is a separate key file the judge does not read until scoring
 is done. Protocol: `tests/testdata/BAKEOFF.md`. Shuffle helper: `tests/bakeoff/blind.py`.
 Judges: talos or Claude. Score: would you `sift fetch` the correct URL from this list?

tests/bakeoff/blind.py

@@ -1,12 +1,18 @@
 #!/usr/bin/env python3
-"""Shuffle four named result dumps into A–D and write a sealed key file."""
+"""Shuffle four named result dumps into A–D and write a sealed key file.
+
+Blinded packs are a list of hit-lists (one per query, same order as
+search_queries.txt). Each hit is only {rank, url, title, snippet}.
+"""
 
 from __future__ import annotations
 
 import argparse
 import json
 import random
+import re
 import sys
+import tempfile
 from pathlib import Path
 
 LABELS = ("A", "B", "C", "D")
@@ -16,6 +22,9 @@ CONFIGS = (
     "searxng_brave_tavily",
     "hermes_tavily",
 )
+HIT_KEYS = ("rank", "url", "title", "snippet")
+LEAK_RE = re.compile(r"searxng|brave|tavily|hermes", re.I)
+EXEMPT_FIELDS = frozenset({"url", "snippet"})
 
 
 def load_queries(path: Path) -> list[str]:
@@ -32,12 +41,219 @@ def load_targets(path: Path) -> list[str]:
     return out
 
 
+def _hit_list(block: object) -> list:
+    if isinstance(block, list):
+        if block and all(isinstance(x, dict) and ("url" in x or "title" in x) for x in block):
+            return block
+        return []
+    if not isinstance(block, dict):
+        return []
+    for key in ("results", "web", "hits"):
+        v = block.get(key)
+        if isinstance(v, list):
+            return v
+    data = block.get("data")
+    if isinstance(data, list):
+        return data
+    if isinstance(data, dict):
+        for key in ("web", "results", "hits"):
+            v = data.get(key)
+            if isinstance(v, list):
+                return v
+    return []
+
+
+def _items(raw: object) -> list:
+    if isinstance(raw, list):
+        return raw
+    if isinstance(raw, dict):
+        for key in ("queries", "results"):
+            v = raw.get(key)
+            if isinstance(v, list) and v and isinstance(v[0], (dict, list)):
+                if key == "results" and v and isinstance(v[0], dict) and "url" in v[0]:
+                    return [raw]
+                return v
+        return [raw]
+    raise ValueError(f"raw dump must be a list or object, got {type(raw).__name__}")
+
+
+def normalize(raw: object, n: int) -> list[list[dict]]:
+    items = _items(raw)
+    if len(items) != n:
+        raise ValueError(f"expected {n} queries in dump, got {len(items)}")
+    out: list[list[dict]] = []
+    for item in items:
+        hits = []
+        if isinstance(item, list):
+            raw_hits = item
+        else:
+            raw_hits = _hit_list(item)
+        for i, h in enumerate(raw_hits, 1):
+            if not isinstance(h, dict):
+                continue
+            rank = h.get("rank", h.get("position", i))
+            try:
+                rank_i = int(rank)
+            except (TypeError, ValueError):
+                rank_i = i
+            hits.append(
+                {
+                    "rank": rank_i,
+                    "url": str(h.get("url") or ""),
+                    "title": str(h.get("title") or ""),
+                    "snippet": str(
+                        h.get("snippet") or h.get("description") or h.get("content") or ""
+                    ),
+                }
+            )
+        out.append(hits)
+    return out
+
+
+def leak_hits(obj: object, path: str = "$"):
+    if isinstance(obj, dict):
+        for k, v in obj.items():
+            if k in EXEMPT_FIELDS and isinstance(v, str):
+                continue
+            yield from leak_hits(v, f"{path}.{k}")
+    elif isinstance(obj, list):
+        for i, v in enumerate(obj):
+            yield from leak_hits(v, f"{path}[{i}]")
+    elif isinstance(obj, str) and LEAK_RE.search(obj):
+        yield path, obj
+
+
+def assert_blind(pack: object) -> None:
+    if not isinstance(pack, list):
+        raise ValueError("blind pack must be a list of hit lists")
+    for qi, hits in enumerate(pack):
+        if not isinstance(hits, list):
+            raise ValueError(f"query {qi}: hits must be a list")
+        for hi, h in enumerate(hits):
+            if not isinstance(h, dict) or set(h) != set(HIT_KEYS):
+                raise ValueError(f"query {qi} hit {hi}: keys must be {HIT_KEYS}, got {h!r}")
+            if not isinstance(h["rank"], int):
+                raise ValueError(f"query {qi} hit {hi}: rank must be int")
+            for k in ("url", "title", "snippet"):
+                if not isinstance(h[k], str):
+                    raise ValueError(f"query {qi} hit {hi}: {k} must be str")
+    found = list(leak_hits(pack))
+    if found:
+        bits = "; ".join(f"{p}={v!r}" for p, v in found[:8])
+        raise ValueError(f"vendor token outside url/snippet: {bits}")
+
+
+def write_packs(packs: dict[str, list], out_dir: Path, n: int, seed: int | None) -> Path:
+    rng = random.Random(seed)
+    order = list(CONFIGS)
+    rng.shuffle(order)
+    key = {lab: cfg for lab, cfg in zip(LABELS, order)}
+    out_dir.mkdir(parents=True, exist_ok=True)
+    for lab, cfg in key.items():
+        pack = packs[cfg]
+        assert_blind(pack)
+        (out_dir / f"{lab}.json").write_text(json.dumps(pack, indent=2) + "\n")
+    key_path = out_dir / "key.json"
+    key_path.write_text(json.dumps({"labels": key, "n": n}, indent=2) + "\n")
+    return key_path
+
+
+def self_test() -> int:
+    n = 2
+    sift_q = {
+        "query": "example docs",
+        "results": [
+            {
+                "url": "https://docs.example.com/search",
+                "title": "Search method",
+                "snippet": "POST /v2/search",
+                "score": 0.91,
+                "source": "searxng",
+            }
+        ],
+        "warnings": ["tavily: 429"],
+        "timings": {"search_ms": 40},
+        "credits": 1,
+        "config": "searxng",
+        "sources": ["searxng"],
+    }
+    hermes_q = {
+        "query": "example docs",
+        "success": True,
+        "data": {
+            "web": [
+                {
+                    "url": "https://api.tavily.com/docs",
+                    "title": "Vendor docs",
+                    "description": "search_depth basic",
+                    "position": 1,
+                    "score": 0.4,
+                }
+            ]
+        },
+        "warnings": ["brave: timeout"],
+    }
+    dumps = {
+        "searxng": [sift_q, sift_q],
+        "searxng_brave": {
+            "config": "searxng_brave",
+            "queries": [sift_q, sift_q],
+        },
+        "searxng_brave_tavily": [sift_q, sift_q],
+        "hermes_tavily": [hermes_q, hermes_q],
+    }
+    with tempfile.TemporaryDirectory() as tmp:
+        raw = Path(tmp) / "raw"
+        out = Path(tmp) / "out"
+        raw.mkdir()
+        for name, payload in dumps.items():
+            (raw / f"{name}.json").write_text(json.dumps(payload) + "\n")
+        packs = {name: normalize(payload, n) for name, payload in dumps.items()}
+        for pack in packs.values():
+            assert_blind(pack)
+        key_path = write_packs(packs, out, n, seed=1)
+        for lab in LABELS:
+            blob = json.loads((out / f"{lab}.json").read_text())
+            assert_blind(blob)
+            assert len(blob) == n
+            assert blob[0][0]["url"].startswith("https://")
+            keys = set()
+
+            def walk(o):
+                if isinstance(o, dict):
+                    keys.update(o)
+                    for v in o.values():
+                        walk(v)
+                elif isinstance(o, list):
+                    for v in o:
+                        walk(v)
+
+            walk(blob)
+            if keys != {"rank", "url", "title", "snippet"}:
+                raise AssertionError(f"{lab} keys {keys}")
+        # snippet/url may name vendors; title must not
+        bad = [[{"rank": 1, "url": "https://x.example", "title": "tavily guide", "snippet": "ok"}]]
+        try:
+            assert_blind(bad)
+        except ValueError as e:
+            if "vendor token" not in str(e):
+                raise
+        else:
+            raise AssertionError("title leak not caught")
+        ok_snip = [
+            [{"rank": 1, "url": "https://api.tavily.com/x", "title": "Docs", "snippet": "uses tavily"}]
+        ]
+        assert_blind(ok_snip)
+        print("self-test ok", key_path)
+    return 0
+
+
 def main() -> int:
     p = argparse.ArgumentParser(description=__doc__)
+    p.add_argument("--self-test", action="store_true")
     p.add_argument(
         "--raw-dir",
         type=Path,
-        required=True,
         help="Directory with {searxng,searxng_brave,searxng_brave_tavily,hermes_tavily}.json",
     )
     p.add_argument(
@@ -57,6 +273,10 @@ def main() -> int:
     )
     p.add_argument("--seed", type=int, default=None)
     args = p.parse_args()
+    if args.self_test:
+        return self_test()
+    if args.raw_dir is None:
+        p.error("--raw-dir is required unless --self-test")
 
     queries = load_queries(args.queries)
     targets = load_targets(args.targets)
@@ -70,22 +290,13 @@ def main() -> int:
     packs = {}
     for name in CONFIGS:
         f = args.raw_dir / f"{name}.json"
-        packs[name] = json.loads(f.read_text())
+        packs[name] = normalize(json.loads(f.read_text()), len(queries))
 
-    rng = random.Random(args.seed)
-    order = list(CONFIGS)
-    rng.shuffle(order)
-    key = {lab: cfg for lab, cfg in zip(LABELS, order)}
-
-    args.out_dir.mkdir(parents=True, exist_ok=True)
-    for lab, cfg in key.items():
-        payload = packs[cfg]
-        if isinstance(payload, dict):
-            payload = {k: v for k, v in payload.items() if k not in ("config", "sources")}
-        (args.out_dir / f"{lab}.json").write_text(json.dumps(payload, indent=2) + "\n")
-
-    key_path = args.out_dir / "key.json"
-    key_path.write_text(json.dumps({"labels": key, "n": len(queries)}, indent=2) + "\n")
+    try:
+        key_path = write_packs(packs, args.out_dir, len(queries), args.seed)
+    except ValueError as e:
+        print(e, file=sys.stderr)
+        return 1
     print(f"wrote {args.out_dir}/{{A,B,C,D}}.json")
     print(f"key sealed at {key_path} — judges do not open this until scoring is done")
     return 0

tests/testdata/BAKEOFF.md

@@ -21,9 +21,12 @@ Record per query: top-k URLs, titles, snippets, latency, Tavily credits, Brave c
 
 ## Blind judging
 
-The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is the
-full 28-query result set for one configuration, with no source names, no
-warnings that name a vendor, and no config id.
+The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is a
+JSON list of 28 hit-lists (query order from `search_queries.txt`). A hit is
+exactly `{rank, url, title, snippet}` — no scores, sources, warnings, latency,
+credits, or Hermes `description`/`position`. Vendor tokens (`searxng`, `brave`,
+`tavily`, `hermes`) outside `url`/`snippet` fail the shuffle. Raw dumps keep
+the extra fields for unblinding.
 
 The mapping `A–D → 1–4` goes in a **separate key file** that the judge does
 not read until scoring is done. Default path (gitignored):