File size: 3,781 Bytes
d45f009 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 | """Build frozen adaptive T20 evaluation packs."""
import argparse
import json
from pathlib import Path
_ROOT = Path(__file__).parent.parent
_EVAL_DIR = _ROOT / "data" / "eval_packs"
def _scenario(seed: int, start_state: str, over: int, score: int, wickets: int, target: int | None, tags: list[str]) -> dict:
return {
"seed": seed,
"task": "stage2_full",
"start_state": start_state,
"innings_type": "second" if target is not None else "first",
"over": over,
"ball": 0,
"score": score,
"wickets": wickets,
"target": target,
"max_overs": 20,
"scenario_tags": tags,
}
def build_pack(eval_pack_id: str = "adaptive_t20_v1") -> dict:
dev = [
_scenario(101, "batting", 2, 18, 1, None, ["powerplay", "bat_first"]),
_scenario(102, "batting", 8, 64, 3, None, ["middle", "rebuild"]),
_scenario(103, "batting", 16, 128, 5, 172, ["death", "chase", "pressure"]),
_scenario(104, "bowling", 5, 48, 1, None, ["powerplay", "defend"]),
_scenario(105, "bowling", 17, 144, 4, 168, ["death", "defend", "wickets_in_hand"]),
]
official = []
seed = 1000
templates = [
("batting", 1, 12, 0, None, ["powerplay", "bat_first"]),
("batting", 6, 42, 2, None, ["middle", "collapse"]),
("batting", 10, 78, 3, 164, ["middle", "chase", "par"]),
("batting", 15, 118, 5, 186, ["death", "chase", "hard"]),
("batting", 18, 152, 7, 174, ["death", "chase", "tail"]),
("bowling", 3, 30, 0, None, ["powerplay", "defend"]),
("bowling", 9, 74, 2, None, ["middle", "spin_matchup"]),
("bowling", 14, 112, 4, 158, ["middle", "defend", "par"]),
("bowling", 16, 136, 3, 172, ["death", "aggressive_opponent"]),
("bowling", 19, 166, 7, 176, ["death", "close_finish"]),
]
for cycle in range(6):
for start_state, over, score, wickets, target, tags in templates:
official.append(
_scenario(
seed,
start_state,
over,
score + cycle * 3,
min(9, wickets + (cycle % 2)),
target + cycle * 2 if target is not None else None,
tags,
)
)
seed += 1
return {
"eval_pack_id": eval_pack_id,
"format": "T20",
"description": "Adaptive T20 benchmark with chase, defense, collapse, death-over, and matchup scenarios.",
"splits": {
"dev": dev,
"official": official,
},
"opponent": {
"dev_mode": "heuristic",
"official_mode": "llm_cached",
"cache_path_env": "CRICKET_OPPONENT_CACHE",
"cache_file": f"data/opponent_cache/{eval_pack_id}.jsonl",
},
"scoring": {
"result_quality": 0.35,
"strategy_bundle": 0.30,
"tool_efficiency": 0.20,
"format": 0.15,
},
"scenario_tags": sorted({tag for s in dev + official for tag in s["scenario_tags"]}),
}
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--eval-pack-id", default="adaptive_t20_v1")
parser.add_argument("--output", default=None)
args = parser.parse_args()
pack = build_pack(args.eval_pack_id)
output = Path(args.output) if args.output else _EVAL_DIR / f"{args.eval_pack_id}.json"
output.parent.mkdir(parents=True, exist_ok=True)
with output.open("w") as f:
json.dump(pack, f, indent=2)
print(
f"Wrote {output}: {len(pack['splits']['dev'])} dev, "
f"{len(pack['splits']['official'])} official scenarios"
)
if __name__ == "__main__":
main()
|