diff --git a/docs/P3.md b/docs/P3.md index 216f705..4dda133 100644 --- a/docs/P3.md +++ b/docs/P3.md @@ -281,7 +281,8 @@ For each query, run four configurations and record top-k URLs + latency + credit 3. SearXNG + Brave LLM Context + Tavily 4. Current Hermes Tavily `web_search` (the live tool, not sift) -Judged **blind**. The runner writes result packs labelled A–D with no config names. +Judged **blind**. The runner writes result packs labelled A–D; each pack is +only `[{rank, url, title, snippet}, …]` per query, no scores/sources/warnings. The A–D → 1–4 mapping is a separate key file the judge does not read until scoring is done. Protocol: `tests/testdata/BAKEOFF.md`. Shuffle helper: `tests/bakeoff/blind.py`. Judges: talos or Claude. Score: would you `sift fetch` the correct URL from this list? diff --git a/tests/bakeoff/blind.py b/tests/bakeoff/blind.py index 01df090..f28b84d 100644 --- a/tests/bakeoff/blind.py +++ b/tests/bakeoff/blind.py @@ -1,12 +1,18 @@ #!/usr/bin/env python3 -"""Shuffle four named result dumps into A–D and write a sealed key file.""" +"""Shuffle four named result dumps into A–D and write a sealed key file. + +Blinded packs are a list of hit-lists (one per query, same order as +search_queries.txt). Each hit is only {rank, url, title, snippet}. +""" from __future__ import annotations import argparse import json import random +import re import sys +import tempfile from pathlib import Path LABELS = ("A", "B", "C", "D") @@ -16,6 +22,9 @@ CONFIGS = ( "searxng_brave_tavily", "hermes_tavily", ) +HIT_KEYS = ("rank", "url", "title", "snippet") +LEAK_RE = re.compile(r"searxng|brave|tavily|hermes", re.I) +EXEMPT_FIELDS = frozenset({"url", "snippet"}) def load_queries(path: Path) -> list[str]: @@ -32,12 +41,219 @@ def load_targets(path: Path) -> list[str]: return out +def _hit_list(block: object) -> list: + if isinstance(block, list): + if block and all(isinstance(x, dict) and ("url" in x or "title" in x) for x in block): + return block + return [] + if not isinstance(block, dict): + return [] + for key in ("results", "web", "hits"): + v = block.get(key) + if isinstance(v, list): + return v + data = block.get("data") + if isinstance(data, list): + return data + if isinstance(data, dict): + for key in ("web", "results", "hits"): + v = data.get(key) + if isinstance(v, list): + return v + return [] + + +def _items(raw: object) -> list: + if isinstance(raw, list): + return raw + if isinstance(raw, dict): + for key in ("queries", "results"): + v = raw.get(key) + if isinstance(v, list) and v and isinstance(v[0], (dict, list)): + if key == "results" and v and isinstance(v[0], dict) and "url" in v[0]: + return [raw] + return v + return [raw] + raise ValueError(f"raw dump must be a list or object, got {type(raw).__name__}") + + +def normalize(raw: object, n: int) -> list[list[dict]]: + items = _items(raw) + if len(items) != n: + raise ValueError(f"expected {n} queries in dump, got {len(items)}") + out: list[list[dict]] = [] + for item in items: + hits = [] + if isinstance(item, list): + raw_hits = item + else: + raw_hits = _hit_list(item) + for i, h in enumerate(raw_hits, 1): + if not isinstance(h, dict): + continue + rank = h.get("rank", h.get("position", i)) + try: + rank_i = int(rank) + except (TypeError, ValueError): + rank_i = i + hits.append( + { + "rank": rank_i, + "url": str(h.get("url") or ""), + "title": str(h.get("title") or ""), + "snippet": str( + h.get("snippet") or h.get("description") or h.get("content") or "" + ), + } + ) + out.append(hits) + return out + + +def leak_hits(obj: object, path: str = "$"): + if isinstance(obj, dict): + for k, v in obj.items(): + if k in EXEMPT_FIELDS and isinstance(v, str): + continue + yield from leak_hits(v, f"{path}.{k}") + elif isinstance(obj, list): + for i, v in enumerate(obj): + yield from leak_hits(v, f"{path}[{i}]") + elif isinstance(obj, str) and LEAK_RE.search(obj): + yield path, obj + + +def assert_blind(pack: object) -> None: + if not isinstance(pack, list): + raise ValueError("blind pack must be a list of hit lists") + for qi, hits in enumerate(pack): + if not isinstance(hits, list): + raise ValueError(f"query {qi}: hits must be a list") + for hi, h in enumerate(hits): + if not isinstance(h, dict) or set(h) != set(HIT_KEYS): + raise ValueError(f"query {qi} hit {hi}: keys must be {HIT_KEYS}, got {h!r}") + if not isinstance(h["rank"], int): + raise ValueError(f"query {qi} hit {hi}: rank must be int") + for k in ("url", "title", "snippet"): + if not isinstance(h[k], str): + raise ValueError(f"query {qi} hit {hi}: {k} must be str") + found = list(leak_hits(pack)) + if found: + bits = "; ".join(f"{p}={v!r}" for p, v in found[:8]) + raise ValueError(f"vendor token outside url/snippet: {bits}") + + +def write_packs(packs: dict[str, list], out_dir: Path, n: int, seed: int | None) -> Path: + rng = random.Random(seed) + order = list(CONFIGS) + rng.shuffle(order) + key = {lab: cfg for lab, cfg in zip(LABELS, order)} + out_dir.mkdir(parents=True, exist_ok=True) + for lab, cfg in key.items(): + pack = packs[cfg] + assert_blind(pack) + (out_dir / f"{lab}.json").write_text(json.dumps(pack, indent=2) + "\n") + key_path = out_dir / "key.json" + key_path.write_text(json.dumps({"labels": key, "n": n}, indent=2) + "\n") + return key_path + + +def self_test() -> int: + n = 2 + sift_q = { + "query": "example docs", + "results": [ + { + "url": "https://docs.example.com/search", + "title": "Search method", + "snippet": "POST /v2/search", + "score": 0.91, + "source": "searxng", + } + ], + "warnings": ["tavily: 429"], + "timings": {"search_ms": 40}, + "credits": 1, + "config": "searxng", + "sources": ["searxng"], + } + hermes_q = { + "query": "example docs", + "success": True, + "data": { + "web": [ + { + "url": "https://api.tavily.com/docs", + "title": "Vendor docs", + "description": "search_depth basic", + "position": 1, + "score": 0.4, + } + ] + }, + "warnings": ["brave: timeout"], + } + dumps = { + "searxng": [sift_q, sift_q], + "searxng_brave": { + "config": "searxng_brave", + "queries": [sift_q, sift_q], + }, + "searxng_brave_tavily": [sift_q, sift_q], + "hermes_tavily": [hermes_q, hermes_q], + } + with tempfile.TemporaryDirectory() as tmp: + raw = Path(tmp) / "raw" + out = Path(tmp) / "out" + raw.mkdir() + for name, payload in dumps.items(): + (raw / f"{name}.json").write_text(json.dumps(payload) + "\n") + packs = {name: normalize(payload, n) for name, payload in dumps.items()} + for pack in packs.values(): + assert_blind(pack) + key_path = write_packs(packs, out, n, seed=1) + for lab in LABELS: + blob = json.loads((out / f"{lab}.json").read_text()) + assert_blind(blob) + assert len(blob) == n + assert blob[0][0]["url"].startswith("https://") + keys = set() + + def walk(o): + if isinstance(o, dict): + keys.update(o) + for v in o.values(): + walk(v) + elif isinstance(o, list): + for v in o: + walk(v) + + walk(blob) + if keys != {"rank", "url", "title", "snippet"}: + raise AssertionError(f"{lab} keys {keys}") + # snippet/url may name vendors; title must not + bad = [[{"rank": 1, "url": "https://x.example", "title": "tavily guide", "snippet": "ok"}]] + try: + assert_blind(bad) + except ValueError as e: + if "vendor token" not in str(e): + raise + else: + raise AssertionError("title leak not caught") + ok_snip = [ + [{"rank": 1, "url": "https://api.tavily.com/x", "title": "Docs", "snippet": "uses tavily"}] + ] + assert_blind(ok_snip) + print("self-test ok", key_path) + return 0 + + def main() -> int: p = argparse.ArgumentParser(description=__doc__) + p.add_argument("--self-test", action="store_true") p.add_argument( "--raw-dir", type=Path, - required=True, help="Directory with {searxng,searxng_brave,searxng_brave_tavily,hermes_tavily}.json", ) p.add_argument( @@ -57,6 +273,10 @@ def main() -> int: ) p.add_argument("--seed", type=int, default=None) args = p.parse_args() + if args.self_test: + return self_test() + if args.raw_dir is None: + p.error("--raw-dir is required unless --self-test") queries = load_queries(args.queries) targets = load_targets(args.targets) @@ -70,22 +290,13 @@ def main() -> int: packs = {} for name in CONFIGS: f = args.raw_dir / f"{name}.json" - packs[name] = json.loads(f.read_text()) + packs[name] = normalize(json.loads(f.read_text()), len(queries)) - rng = random.Random(args.seed) - order = list(CONFIGS) - rng.shuffle(order) - key = {lab: cfg for lab, cfg in zip(LABELS, order)} - - args.out_dir.mkdir(parents=True, exist_ok=True) - for lab, cfg in key.items(): - payload = packs[cfg] - if isinstance(payload, dict): - payload = {k: v for k, v in payload.items() if k not in ("config", "sources")} - (args.out_dir / f"{lab}.json").write_text(json.dumps(payload, indent=2) + "\n") - - key_path = args.out_dir / "key.json" - key_path.write_text(json.dumps({"labels": key, "n": len(queries)}, indent=2) + "\n") + try: + key_path = write_packs(packs, args.out_dir, len(queries), args.seed) + except ValueError as e: + print(e, file=sys.stderr) + return 1 print(f"wrote {args.out_dir}/{{A,B,C,D}}.json") print(f"key sealed at {key_path} — judges do not open this until scoring is done") return 0 diff --git a/tests/testdata/BAKEOFF.md b/tests/testdata/BAKEOFF.md index 2a1513d..b778922 100644 --- a/tests/testdata/BAKEOFF.md +++ b/tests/testdata/BAKEOFF.md @@ -21,9 +21,12 @@ Record per query: top-k URLs, titles, snippets, latency, Tavily credits, Brave c ## Blind judging -The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is the -full 28-query result set for one configuration, with no source names, no -warnings that name a vendor, and no config id. +The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is a +JSON list of 28 hit-lists (query order from `search_queries.txt`). A hit is +exactly `{rank, url, title, snippet}` — no scores, sources, warnings, latency, +credits, or Hermes `description`/`position`. Vendor tokens (`searxng`, `brave`, +`tavily`, `hermes`) outside `url`/`snippet` fail the shuffle. Raw dumps keep +the extra fields for unblinding. The mapping `A–D → 1–4` goes in a **separate key file** that the judge does not read until scoring is done. Default path (gitignored):