| """Build frozen adaptive T20 evaluation packs.""" |
|
|
| import argparse |
| import json |
| from pathlib import Path |
|
|
| _ROOT = Path(__file__).parent.parent |
| _EVAL_DIR = _ROOT / "data" / "eval_packs" |
|
|
|
|
| def _scenario(seed: int, start_state: str, over: int, score: int, wickets: int, target: int | None, tags: list[str]) -> dict: |
| return { |
| "seed": seed, |
| "task": "stage2_full", |
| "start_state": start_state, |
| "innings_type": "second" if target is not None else "first", |
| "over": over, |
| "ball": 0, |
| "score": score, |
| "wickets": wickets, |
| "target": target, |
| "max_overs": 20, |
| "scenario_tags": tags, |
| } |
|
|
|
|
| def build_pack(eval_pack_id: str = "adaptive_t20_v1") -> dict: |
| dev = [ |
| _scenario(101, "batting", 2, 18, 1, None, ["powerplay", "bat_first"]), |
| _scenario(102, "batting", 8, 64, 3, None, ["middle", "rebuild"]), |
| _scenario(103, "batting", 16, 128, 5, 172, ["death", "chase", "pressure"]), |
| _scenario(104, "bowling", 5, 48, 1, None, ["powerplay", "defend"]), |
| _scenario(105, "bowling", 17, 144, 4, 168, ["death", "defend", "wickets_in_hand"]), |
| ] |
|
|
| official = [] |
| seed = 1000 |
| templates = [ |
| ("batting", 1, 12, 0, None, ["powerplay", "bat_first"]), |
| ("batting", 6, 42, 2, None, ["middle", "collapse"]), |
| ("batting", 10, 78, 3, 164, ["middle", "chase", "par"]), |
| ("batting", 15, 118, 5, 186, ["death", "chase", "hard"]), |
| ("batting", 18, 152, 7, 174, ["death", "chase", "tail"]), |
| ("bowling", 3, 30, 0, None, ["powerplay", "defend"]), |
| ("bowling", 9, 74, 2, None, ["middle", "spin_matchup"]), |
| ("bowling", 14, 112, 4, 158, ["middle", "defend", "par"]), |
| ("bowling", 16, 136, 3, 172, ["death", "aggressive_opponent"]), |
| ("bowling", 19, 166, 7, 176, ["death", "close_finish"]), |
| ] |
| for cycle in range(6): |
| for start_state, over, score, wickets, target, tags in templates: |
| official.append( |
| _scenario( |
| seed, |
| start_state, |
| over, |
| score + cycle * 3, |
| min(9, wickets + (cycle % 2)), |
| target + cycle * 2 if target is not None else None, |
| tags, |
| ) |
| ) |
| seed += 1 |
|
|
| return { |
| "eval_pack_id": eval_pack_id, |
| "format": "T20", |
| "description": "Adaptive T20 benchmark with chase, defense, collapse, death-over, and matchup scenarios.", |
| "splits": { |
| "dev": dev, |
| "official": official, |
| }, |
| "opponent": { |
| "dev_mode": "heuristic", |
| "official_mode": "llm_cached", |
| "cache_path_env": "CRICKET_OPPONENT_CACHE", |
| "cache_file": f"data/opponent_cache/{eval_pack_id}.jsonl", |
| }, |
| "scoring": { |
| "result_quality": 0.35, |
| "strategy_bundle": 0.30, |
| "tool_efficiency": 0.20, |
| "format": 0.15, |
| }, |
| "scenario_tags": sorted({tag for s in dev + official for tag in s["scenario_tags"]}), |
| } |
|
|
|
|
| def main(): |
| parser = argparse.ArgumentParser() |
| parser.add_argument("--eval-pack-id", default="adaptive_t20_v1") |
| parser.add_argument("--output", default=None) |
| args = parser.parse_args() |
|
|
| pack = build_pack(args.eval_pack_id) |
| output = Path(args.output) if args.output else _EVAL_DIR / f"{args.eval_pack_id}.json" |
| output.parent.mkdir(parents=True, exist_ok=True) |
| with output.open("w") as f: |
| json.dump(pack, f, indent=2) |
| print( |
| f"Wrote {output}: {len(pack['splits']['dev'])} dev, " |
| f"{len(pack['splits']['official'])} official scenarios" |
| ) |
|
|
|
|
| if __name__ == "__main__": |
| main() |
|
|
