p3: harvest 28 bake-off queries with targets and blind protocol
Kyle-approved fleet harvest. Sibling targets (domain/page or open). Runner labels A-D; mapping stays in a sealed key file until scoring.
This commit is contained in:
@@ -0,0 +1,95 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Shuffle four named result dumps into A–D and write a sealed key file."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import random
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
LABELS = ("A", "B", "C", "D")
|
||||
CONFIGS = (
|
||||
"searxng",
|
||||
"searxng_brave",
|
||||
"searxng_brave_tavily",
|
||||
"hermes_tavily",
|
||||
)
|
||||
|
||||
|
||||
def load_queries(path: Path) -> list[str]:
|
||||
return [ln.strip() for ln in path.read_text().splitlines() if ln.strip()]
|
||||
|
||||
|
||||
def load_targets(path: Path) -> list[str]:
|
||||
out: list[str] = []
|
||||
for ln in path.read_text().splitlines():
|
||||
s = ln.strip()
|
||||
if not s or s.startswith("#"):
|
||||
continue
|
||||
out.append(s)
|
||||
return out
|
||||
|
||||
|
||||
def main() -> int:
|
||||
p = argparse.ArgumentParser(description=__doc__)
|
||||
p.add_argument(
|
||||
"--raw-dir",
|
||||
type=Path,
|
||||
required=True,
|
||||
help="Directory with {searxng,searxng_brave,searxng_brave_tavily,hermes_tavily}.json",
|
||||
)
|
||||
p.add_argument(
|
||||
"--out-dir",
|
||||
type=Path,
|
||||
default=Path("tests/testdata/bakeoff-out"),
|
||||
)
|
||||
p.add_argument(
|
||||
"--queries",
|
||||
type=Path,
|
||||
default=Path("tests/testdata/search_queries.txt"),
|
||||
)
|
||||
p.add_argument(
|
||||
"--targets",
|
||||
type=Path,
|
||||
default=Path("tests/testdata/search_targets.txt"),
|
||||
)
|
||||
p.add_argument("--seed", type=int, default=None)
|
||||
args = p.parse_args()
|
||||
|
||||
queries = load_queries(args.queries)
|
||||
targets = load_targets(args.targets)
|
||||
if len(queries) != len(targets):
|
||||
print(
|
||||
f"query/target length mismatch: {len(queries)} vs {len(targets)}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
return 1
|
||||
|
||||
packs = {}
|
||||
for name in CONFIGS:
|
||||
f = args.raw_dir / f"{name}.json"
|
||||
packs[name] = json.loads(f.read_text())
|
||||
|
||||
rng = random.Random(args.seed)
|
||||
order = list(CONFIGS)
|
||||
rng.shuffle(order)
|
||||
key = {lab: cfg for lab, cfg in zip(LABELS, order)}
|
||||
|
||||
args.out_dir.mkdir(parents=True, exist_ok=True)
|
||||
for lab, cfg in key.items():
|
||||
payload = packs[cfg]
|
||||
if isinstance(payload, dict):
|
||||
payload = {k: v for k, v in payload.items() if k not in ("config", "sources")}
|
||||
(args.out_dir / f"{lab}.json").write_text(json.dumps(payload, indent=2) + "\n")
|
||||
|
||||
key_path = args.out_dir / "key.json"
|
||||
key_path.write_text(json.dumps({"labels": key, "n": len(queries)}, indent=2) + "\n")
|
||||
print(f"wrote {args.out_dir}/{{A,B,C,D}}.json")
|
||||
print(f"key sealed at {key_path} — judges do not open this until scoring is done")
|
||||
return 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
raise SystemExit(main())
|
||||
Vendored
+42
@@ -0,0 +1,42 @@
|
||||
# P3 §10 bake-off protocol
|
||||
|
||||
Query list: `search_queries.txt` (28, Kyle-approved 2026-09-23).
|
||||
Targets: `search_targets.txt` (same order; `open` = no required domain).
|
||||
|
||||
Do not enable sources or start runs until this commit is on `kyle/sift` master.
|
||||
SearXNG journal redaction (`searx.service`) is a prerequisite for `search.searxng.enabled`.
|
||||
|
||||
## Configurations (never shown to the judge)
|
||||
|
||||
1. SearXNG alone
|
||||
2. SearXNG + Brave LLM Context
|
||||
3. SearXNG + Brave LLM Context + Tavily
|
||||
4. Current Hermes Tavily `web_search` (live tool, not sift)
|
||||
|
||||
Sift does not take source names from the request. Configs 1–3 are three orion
|
||||
`[search]` settings (or three sequential enables), not query parameters.
|
||||
Config 4 is a fleet-host Hermes call, same 28 queries, `limit` matching sift `k`.
|
||||
|
||||
Record per query: top-k URLs, titles, snippets, latency, Tavily credits, Brave calls.
|
||||
|
||||
## Blind judging
|
||||
|
||||
The runner writes **only** labelled packs `A` `B` `C` `D`. Each pack is the
|
||||
full 28-query result set for one configuration, with no source names, no
|
||||
warnings that name a vendor, and no config id.
|
||||
|
||||
The mapping `A–D → 1–4` goes in a **separate key file** that the judge does
|
||||
not read until scoring is done. Default path (gitignored):
|
||||
|
||||
```
|
||||
tests/testdata/bakeoff-out/key.json
|
||||
tests/testdata/bakeoff-out/{A,B,C,D}.json
|
||||
```
|
||||
|
||||
Shuffle with `tests/bakeoff/blind.py` after the four raw dumps exist.
|
||||
|
||||
Judges: **talos or Claude**. Score each query: would you `sift fetch` the
|
||||
correct URL from this list? Against `search_targets.txt` when it is not `open`.
|
||||
|
||||
After scores land, open `key.json` and report: winner (becomes default-on),
|
||||
cost/query, p90 latency.
|
||||
Vendored
+28
@@ -0,0 +1,28 @@
|
||||
firecrawl-py 4.17.0 search method POST /v2/search query limit
|
||||
Tavily API search extract crawl features pricing 2026
|
||||
Brave Search API vs Brave LLM API web search agent 2026
|
||||
vLLM PagedAttention what is vLLM serving engine
|
||||
Google A2A Agent2Agent protocol agent communication
|
||||
Hermes Agent Nous Research prompt engineering skills AGENTS.md
|
||||
site:docs.anthropic.com agentic coding prompt engineering Claude Code
|
||||
Rozado PLOS ONE political preferences of LLMs 2024
|
||||
LunaDrive delay-compensated high-voltage GaN FET motor driver site:arxiv.org
|
||||
"PQLN" Lightning Kurt 2609.13781
|
||||
site:arxiv.org Tim Dettmers 2026
|
||||
bitsandbytes2 Dettmers runtime dynamic compression
|
||||
SK Hynix CEO 2027 worst supply year DRAM HBM shortage 2030
|
||||
NVIDIA DGX Spark official price 2026
|
||||
Grok 4.7 regression vs 4.5 OR 4.6
|
||||
Framework Desktop 128GB 192GB price 2026 Ryzen AI Max
|
||||
Xiaomi MiMo-V2.6-Pro open weights Hugging Face
|
||||
ASUS DUAL-RTX3060-012G-V2 specifications TDP length dual slot
|
||||
MSI PRO B760-P WIFI DDR4 PCIe x16 slot layout dual GPU x4
|
||||
CubeMars RO80 frameless BLDC weight torque KV specifications
|
||||
SanDisk Optimus GX 7100M 2TB M.2 2230 NVMe specs
|
||||
vLLM Apple Silicon Metal support site:docs.vllm.ai
|
||||
obsidian livesync headless couchdb sync without desktop app
|
||||
Omarchy Linux ISO download 4.0 site:omarchy.org OR site:omacom.io
|
||||
KittenML kitten-tts-nano huggingface model voice quality
|
||||
Nemotron 3 Diarization Hugging Face NVIDIA
|
||||
Qwen3.8-Flash-Next GGUF size Q4_K_M parameters Hugging Face
|
||||
bendlang bend github agentic programming language
|
||||
Vendored
+31
@@ -0,0 +1,31 @@
|
||||
# Sibling of search_queries.txt. Line N (ignoring this comment and blanks) is the
|
||||
# target for query N. A good result set must contain this page or domain.
|
||||
# `open` means no required hit — score usefulness only.
|
||||
docs.firecrawl.dev
|
||||
docs.tavily.com
|
||||
brave.com
|
||||
docs.vllm.ai
|
||||
a2a-protocol.org
|
||||
hermes-agent.nousresearch.com
|
||||
docs.anthropic.com
|
||||
journals.plos.org
|
||||
arxiv.org
|
||||
arxiv.org/abs/2609.13781
|
||||
arxiv.org
|
||||
github.com/bitsandbytes-foundation
|
||||
open
|
||||
nvidia.com
|
||||
open
|
||||
frame.work
|
||||
huggingface.co
|
||||
asus.com
|
||||
msi.com
|
||||
cubemars.com
|
||||
sandisk.com
|
||||
docs.vllm.ai
|
||||
github.com/vrtmrz/obsidian-livesync
|
||||
omarchy.org
|
||||
huggingface.co
|
||||
huggingface.co
|
||||
huggingface.co
|
||||
github.com/bendlang/bend
|
||||
Reference in New Issue
Block a user