p3: blind titles, cap snippets, equalise k; abort cached dumps

Exempt title from the vendor-token check. Truncate snippets to 160
chars and clip all four packs to the same per-query k. run.py fails
cached replies with a rerun message instead of a config mismatch.

tests/bakeoff/blind.py

@@ -24,7 +24,9 @@ CONFIGS = (
 )
 HIT_KEYS = ("rank", "url", "title", "snippet")
 LEAK_RE = re.compile(r"searxng|brave|tavily|hermes", re.I)
-EXEMPT_FIELDS = frozenset({"url", "snippet"})
+EXEMPT_FIELDS = frozenset({"url", "title", "snippet"})
+SNIPPET_MAX = 160
+DEFAULT_K = 8
 
 
 def load_queries(path: Path) -> list[str]:
@@ -96,20 +98,44 @@ def normalize(raw: object, n: int) -> list[list[dict]]:
                 rank_i = int(rank)
             except (TypeError, ValueError):
                 rank_i = i
+            snip = str(h.get("snippet") or h.get("description") or h.get("content") or "")
+            if len(snip) > SNIPPET_MAX:
+                snip = snip[:SNIPPET_MAX]
             hits.append(
                 {
                     "rank": rank_i,
                     "url": str(h.get("url") or ""),
                     "title": str(h.get("title") or ""),
-                    "snippet": str(
+                    "snippet": snip,
-                        h.get("snippet") or h.get("description") or h.get("content") or ""
-                    ),
                 }
             )
         out.append(hits)
     return out
 
 
+def equalise(packs: dict[str, list], k: int) -> None:
+    n = None
+    for cfg in CONFIGS:
+        if cfg not in packs:
+            raise ValueError(f"missing pack {cfg}")
+        got = len(packs[cfg])
+        if n is None:
+            n = got
+        elif got != n:
+            raise ValueError(f"{cfg}: {got} queries, want {n}")
+    assert n is not None
+    for cfg in CONFIGS:
+        for i in range(n):
+            packs[cfg][i] = packs[cfg][i][:k]
+    for i in range(n):
+        m = min(len(packs[cfg][i]) for cfg in CONFIGS)
+        for cfg in CONFIGS:
+            hits = packs[cfg][i][:m]
+            for j, h in enumerate(hits, 1):
+                h["rank"] = j
+            packs[cfg][i] = hits
+
+
 def leak_hits(obj: object, path: str = "$"):
     if isinstance(obj, dict):
         for k, v in obj.items():
@@ -140,10 +166,11 @@ def assert_blind(pack: object) -> None:
     found = list(leak_hits(pack))
     if found:
         bits = "; ".join(f"{p}={v!r}" for p, v in found[:8])
-        raise ValueError(f"vendor token outside url/snippet: {bits}")
+        raise ValueError(f"vendor token outside url/title/snippet: {bits}")
 
 
-def write_packs(packs: dict[str, list], out_dir: Path, n: int, seed: int | None) -> Path:
+def write_packs(packs: dict[str, list], out_dir: Path, n: int, seed: int | None, k: int = DEFAULT_K) -> Path:
+    equalise(packs, k)
     rng = random.Random(seed)
     order = list(CONFIGS)
     rng.shuffle(order)
@@ -231,19 +258,34 @@ def self_test() -> int:
             walk(blob)
             if keys != {"rank", "url", "title", "snippet"}:
                 raise AssertionError(f"{lab} keys {keys}")
-        # snippet/url may name vendors; title must not
+        # titles of correct hits name vendors; url/title/snippet are exempt
-        bad = [[{"rank": 1, "url": "https://x.example", "title": "tavily guide", "snippet": "ok"}]]
+        assert_blind(
-        try:
+            [[{"rank": 1, "url": "https://x.example", "title": "tavily guide", "snippet": "ok"}]]
-            assert_blind(bad)
+        )
-        except ValueError as e:
-            if "vendor token" not in str(e):
-                raise
-        else:
-            raise AssertionError("title leak not caught")
         ok_snip = [
             [{"rank": 1, "url": "https://api.tavily.com/x", "title": "Docs", "snippet": "uses tavily"}]
         ]
         assert_blind(ok_snip)
+        long = "L" * (SNIPPET_MAX + 40)
+        fat = {
+            "results": [
+                {"url": "https://a.example/" + str(i), "title": "T", "snippet": long}
+                for i in range(5)
+            ]
+        }
+        thin = {"results": [{"url": "https://b.example/0", "title": "T", "snippet": "short"}]}
+        packs2 = {
+            "searxng": normalize([fat, fat], 2),
+            "searxng_brave": normalize([thin, thin], 2),
+            "searxng_brave_tavily": normalize([fat, fat], 2),
+            "hermes_tavily": normalize([thin, thin], 2),
+        }
+        equalise(packs2, k=8)
+        for cfg, pack in packs2.items():
+            if len(pack[0]) != 1:
+                raise AssertionError(f"{cfg} k={len(pack[0])} after equalise")
+            if any(len(h["snippet"]) > SNIPPET_MAX for h in pack[0]):
+                raise AssertionError("snippet not truncated")
         print("self-test ok", key_path)
     return 0
 
@@ -272,6 +314,7 @@ def main() -> int:
         default=Path("tests/testdata/search_targets.txt"),
     )
     p.add_argument("--seed", type=int, default=None)
+    p.add_argument("--k", type=int, default=DEFAULT_K)
     args = p.parse_args()
     if args.self_test:
         return self_test()
@@ -293,7 +336,7 @@ def main() -> int:
         packs[name] = normalize(json.loads(f.read_text()), len(queries))
 
     try:
-        key_path = write_packs(packs, args.out_dir, len(queries), args.seed)
+        key_path = write_packs(packs, args.out_dir, len(queries), args.seed, args.k)
     except ValueError as e:
         print(e, file=sys.stderr)
         return 1

tests/bakeoff/run.py

@@ -108,6 +108,8 @@ def sift_one(base: str, key: str, query: str, k: int, timeout: float) -> dict:
 def check_sources(config: str, rows: list[dict]) -> str | None:
     want = EXPECTED_SOURCES[config]
     for i, row in enumerate(rows):
+        if row.get("cached"):
+            return f"query {i}: cached, rerun after the hour or change k"
         sizes = row.get("sizes") or {}
         ok = sizes.get("sources_ok") or []
         failed = sizes.get("sources_failed") or []
@@ -228,6 +230,10 @@ def self_test() -> int:
             bad = check_sources("searxng_brave", rows)
             if not bad:
                 raise AssertionError("mismatch not caught")
+            cached_rows = [{**rows[0], "cached": True, "sizes": {}}]
+            msg = check_sources("searxng", cached_rows)
+            if not msg or "cached, rerun after the hour or change k" not in msg:
+                raise AssertionError(msg)
             ht = hermes_tavily_one(host + "/search", "k", "one", 8, 5)
             if ht["data"]["web"][0]["description"] != "snippet":
                 raise AssertionError(ht)

tests/testdata/BAKEOFF.md

@@ -25,8 +25,10 @@ The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is a
 JSON list of 28 hit-lists (query order from `search_queries.txt`). A hit is
 exactly `{rank, url, title, snippet}` — no scores, sources, warnings, latency,
 credits, or Hermes `description`/`position`. Vendor tokens (`searxng`, `brave`,
-`tavily`, `hermes`) outside `url`/`snippet` fail the shuffle. Raw dumps keep
+`tavily`, `hermes`) outside `url`/`title`/`snippet` fail the shuffle. Snippets
-the extra fields for unblinding.
+are capped at 160 characters. All four packs are clipped to the same per-query
+`k` (min across configs, then `--k`) so hit-count and snippet length cannot
+name the configuration. Raw dumps keep the extra fields for unblinding.
 
 The mapping `A–D → 1–4` goes in a **separate key file** that the judge does
 not read until scoring is done. Default path (gitignored):
@@ -41,15 +43,16 @@ Shuffle with `tests/bakeoff/blind.py` after the four raw dumps exist.
 Collect a dump after orion is set to that configuration:
 
 ```
-python3 tests/bakeoff/run.py --config searxng
-python3 tests/bakeoff/run.py --config searxng_brave
 python3 tests/bakeoff/run.py --config searxng_brave_tavily
+python3 tests/bakeoff/run.py --config searxng_brave
+python3 tests/bakeoff/run.py --config searxng
 TAVILY_API_KEY=… python3 tests/bakeoff/run.py --config hermes_tavily
 python3 tests/bakeoff/blind.py --raw-dir tests/testdata/bakeoff-raw
 ```
 
-`run.py` refuses to stamp a sift dump if `sizes.sources_ok` does not match `--config`.
+All three sources on → collect config 3 first, then turn off tavily (config 2),
-Titan does not enable sources or edit nixos.
+then brave_llm (config 1). `run.py` refuses a dump whose `sources_ok` does not
+match `--config`, and refuses `cached: true` (rerun after the hour or change k).
 
 Judges: **talos or Claude**. Score each query: would you `sift fetch` the
 correct URL from this list? Against `search_targets.txt` when it is not `open`.