eval_results/scripts/sweep_diff.py

"""4-way per-episode comparison across stop_thr sweep."""
import re

def parse(p):
    s = open(p).read()
    pat = re.compile(
        r'"NE":\s*([\d.]+).*?"success":\s*([\d.]+).*?"TL":\s*([\d.]+).*?"trajectory_id":\s*"([^"]+)"',
        re.S,
    )
    return {m.group(4): (float(m.group(1)), float(m.group(2)), float(m.group(3))) for m in pat.finditer(s)}

import glob
runs = {}
for tag, glob_pat in [
    ("v5_088", "/tmp/*v5_stopdebug.log"),
    ("v7_092", "/tmp/*thr092_v7.log"),
    ("v6_095", "/tmp/*v6_stopdebug.log"),
    ("v8_097", "/tmp/*thr097_v8.log"),
]:
    files = glob.glob(glob_pat)
    if not files:
        print(f"MISSING {tag}: {glob_pat}")
        continue
    runs[tag] = parse(files[0])

# Use v6 tids as canonical (all runs have same 30 episodes)
keys = list(next(iter(runs.values())).keys())

print(f"{'tid':<20} | {'v5_088':>10} | {'v7_092':>10} | {'v6_095':>10} | {'v8_097':>10} | sweep pattern")
print("-" * 110)
patterns = {}
for k in keys:
    cells = []
    succ_str = ""
    for tag in ["v5_088", "v7_092", "v6_095", "v8_097"]:
        if k in runs[tag]:
            ne, s, tl = runs[tag][k]
            cells.append(f"NE{ne:>4.1f}{'✓' if s==1 else 'x'}")
            succ_str += "1" if s == 1 else "0"
        else:
            cells.append(" "*10)
            succ_str += "?"
    patterns.setdefault(succ_str, []).append(k)
    print(f"{k[:20]:<20} | {cells[0]:>10} | {cells[1]:>10} | {cells[2]:>10} | {cells[3]:>10} | {succ_str}")

print()
print("===pattern frequency (v5_v7_v6_v8 success bits)===")
labels = {"1111":"always succ","0000":"always fail","0010":"only v6","0011":"v6+v8",
          "0110":"v7+v6","0100":"only v7","0111":"v7+v6+v8","1110":"v5+v7+v6","1111":"all"}
for pat, eps in sorted(patterns.items(), key=lambda x: -len(x[1])):
    print(f"  {pat}: {len(eps)} ep  ({labels.get(pat,'')})  {eps[:5]}{'...' if len(eps)>5 else ''}")