"""Ground-truth per-episode agent.step analysis from eval.log.""" import re import sys from collections import Counter eval_log = open(sys.argv[1]).read() # Pattern for actual executed action (only count the unique one, not the dup INFO line) # Each agent.step produces 2 lines with same timestamp; use the one without file path action_re = re.compile( r"^\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]\[INFO\] get \d+ actions :\[\[(-?\d+)\]\]", re.M, ) actions = [(m.group(1), int(m.group(2))) for m in action_re.finditer(eval_log)] print(f"Total agent.step calls: {len(actions)}") # Episode boundaries finishes = list( re.finditer( r"\[(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\].*?\[(\d+)/30\]\[step_index:(\d+)\] " r"finish: \[trajectory_id:(\S+?)\]\[duration:([\d.]+) s\]\[step_count:(\d+)\]\[fps:([\d.]+)\]\[result:(\w+)\]", eval_log, ) ) print(f"Episodes finished: {len(finishes)}") # For each episode (1-indexed), assign actions whose timestamp <= episode finish timestamp # but > previous episode finish timestamp boundaries = [m.group(1) for m in finishes] # finish timestamps ep_info = [(m.group(2), m.group(4), m.group(6), m.group(8), m.group(5)) for m in finishes] # (idx, tid, step_count, result, duration) def in_episode(action_ts, ep_idx): """action_ts > prev_finish_ts AND action_ts <= this_finish_ts""" if ep_idx == 0: return action_ts <= boundaries[0] return boundaries[ep_idx - 1] < action_ts <= boundaries[ep_idx] # bucket ep_actions = {i: [] for i in range(len(finishes))} for ts, cls in actions: for i in range(len(finishes)): if in_episode(ts, i): ep_actions[i].append(cls) break print() print(f"{'ep':>3} {'tid':<25} {'result':<14} {'sim_step':>9} {'agent_step':>10} {'first_Stop':>11} {'F%':>5} {'TL%':>5} {'TR%':>5} {'Stop%':>5}") print("-" * 125) for i, (idx, tid, sim_step, result, dur) in enumerate(ep_info): acts = ep_actions[i] n = len(acts) if n == 0: print(f"{idx:>3} {tid[:25]:<25} {result[:14]:<14} NO ACTIONS") continue c = Counter(acts) pct = lambda k: 100 * c.get(k, 0) / n # first Stop position first_stop = next((j for j, a in enumerate(acts) if a == 0), None) fst = f"{first_stop+1}/{n}" if first_stop is not None else "never" short = "_".join(tid.split("_")[: len(tid.split("_")) // 2]) print( f"{idx:>3} {short[:25]:<25} {result[:14]:<14} {sim_step:>9} {n:>10} {fst:>11} " f"{pct(1):>5.1f} {pct(2):>5.1f} {pct(3):>5.1f} {pct(0):>5.1f}" ) # How many actions came AFTER the last episode finish (in-progress episode) last_finish_ts = boundaries[-1] if boundaries else None in_progress = [c for ts, c in actions if last_finish_ts is None or ts > last_finish_ts] print(f"\n[in-progress episode]: {len(in_progress)} agent.step calls so far") if in_progress: c = Counter(in_progress) print(f" class dist: {dict(c)}")