Sync refreshed episode0 metrics, README, and validation helpers

.gitattributes

@@ -424,3 +424,4 @@ artifacts/results/oven_episode0_iter5_visual_checks/visibility_232.png filter=lf
 artifacts/results/oven_episode0_iter5_visual_checks/visibility_234.png filter=lfs diff=lfs merge=lfs -text
 artifacts/results/oven_episode0_iter5_visual_checks/visibility_260.png filter=lfs diff=lfs merge=lfs -text
 artifacts/results/oven_episode0_iter6_visual_checks/boundary_rgb_contact_sheet.png filter=lfs diff=lfs merge=lfs -text
+artifacts/results/oven_episode0_iter6_visual_checks/early_visibility_contact_sheet.png filter=lfs diff=lfs merge=lfs -text

README.md

@@ -46,6 +46,7 @@ The old `iter4_*` outputs are still useful historical checkpoints, but the curre
   - Study runners and helpers.
   - `run_oven_label_study.py`: dense/keyframe study runner.
   - `launch_parallel_oven_label_study.py`: multi-display worker launcher.
+  - `refresh_saved_oven_study.py`: recompute keyframes, per-episode metrics, and summary JSONs from saved dense CSVs after metric-code changes.
   - `run_oven_single_frame.py`: single-frame recomputation helper.
   - `run_oven_frame_batch.py`: new sequential batch recomputation helper used to avoid late-frame drift.
   - `repair_oven_episode_dense.py`: batched repair pass for suspicious dense rows.
@@ -132,8 +133,9 @@ The current trustworthy artifacts are:
 - `artifacts/results/oven_episode0_iter6_independent_full/episode0.keyframes.csv`
 - `artifacts/results/oven_episode0_iter6_independent_full/episode0.metrics.json`
 - `artifacts/results/oven_episode0_iter6_independent_full/summary.json`
+- `artifacts/results/oven_episode0_iter6_visual_checks/early_visibility_contact_sheet.png`
 
-Important note: the `iter6_independent_full` dense CSV is the authoritative low-level artifact. The saved JSON summary in that directory was produced before the last `phase_score` retune, so the dense CSV should be treated as ground truth if you are re-deriving phase metrics.
+The `iter6_independent_full` CSVs and JSON summaries have been refreshed with the latest `phase_score` logic via `code/scripts/refresh_saved_oven_study.py`.
 
 ## Key Verified Findings
 
@@ -152,9 +154,10 @@ From the current independent-replay validation on episode 0:
   - frame `230`: `p_ext ≈ 0.0007`, `y_ext = 0`
   - frame `232`: `p_ext = 1.0`, `y_ext = 1`
   - frame `234`: `p_ext = 1.0`, `y_ext = 1`
-- In the full independent episode-0 run:
+- In the refreshed full independent episode-0 run:
   - `ppre_cross_frame = 216`
   - `pext_cross_frame = 232`
+  - `phase_cross_frame = 214`
   - `retrieve_cross_frame = 215`
   - `ready_cross_frame = 234`
   - `single_switch_rate = 1.0`
@@ -163,10 +166,15 @@ From the current independent-replay validation on episode 0:
   - `auprc_ppre_ypre ≈ 0.903`
   - `auroc_pext_yext = 1.0`
   - `auprc_pext_yext = 1.0`
-- After the final `phase_score` retune, recomputing phase columns on the dense CSV moves:
-  - `first phase_switch` to frame `214`
-  - `first y_retrieve` to frame `215`
-  - `first y_ready` to frame `234`
+  - `auroc_phase_yretrieve = 1.0`
+  - `auprc_phase_yretrieve = 1.0`
+  - `f1_phase_yretrieve ≈ 0.996`
+  - `auroc_phase_yready ≈ 0.998`
+  - `f1_phase_yready ≈ 0.905`
+- The refreshed phase columns now place:
+  - `first phase_switch` at frame `214`
+  - `first y_retrieve` at frame `215`
+  - `first y_ready` at frame `234`
 
 See:
 
@@ -222,7 +230,9 @@ Relevant current artifacts:
 - `artifacts/results/oven_episode0_iter4_batch/iter4_batch_comparison.csv`
 - `artifacts/results/oven_episode0_iter4_dense_geom_170_234.csv`
 - `artifacts/results/oven_episode0_iter6_independent_full/episode0.dense.csv`
+- `artifacts/results/oven_episode0_iter6_independent_full/summary.json`
 - `artifacts/results/oven_episode0_iter6_visual_checks/boundary_rgb_contact_sheet.png`
+- `artifacts/results/oven_episode0_iter6_visual_checks/early_visibility_contact_sheet.png`
 
 ## Environment
 

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_00/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 0
+}

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_00/worker.log

@@ -0,0 +1,28 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+[episode0] analyzed 25/329 dense frames
+[episode0] analyzed 50/329 dense frames
+[episode0] analyzed 75/329 dense frames
+[episode0] analyzed 100/329 dense frames
+The X11 connection broke (error 1). Did the X11 server die?
+QMutex: destroying locked mutex
+[CoppeliaSim:loadinfo]   done.
+
+
+Error: signal 11:
+
+/workspace/coppelia_sim/libcoppeliaSim.so.1(_Z11_segHandleri+0x30)[0x7dfddcb0aae0]
+/usr/lib/x86_64-linux-gnu/libc.so.6(+0x42520)[0x7dfed1961520]
+/workspace/coppelia_sim/libcoppeliaSim.so.1(_ZNK8_CJoint_11getPositionEv+0x4)[0x7dfddc97f724]
+/workspace/coppelia_sim/libcoppeliaSim.so.1(_Z19_validationCallbackPf+0x96)[0x7dfddccd9026]
+/workspace/coppelia_sim/libsimExtIK.so(_Z21ikGetConfigForTipPoseimPKidiPdPKdPFbS1_ES0_S3_S3_+0x100c)[0x7dfe4c4ae5bc]
+/workspace/coppelia_sim/libsimExtIK.so(ikPlugin_getConfigForTipPose+0x2af)[0x7dfe4c48a9ef]
+/workspace/coppelia_sim/libcoppeliaSim.so.1(_ZN16CPluginContainer28ikPlugin_getConfigForTipPoseEiiPKifiPfPKfPFbS2_ES1_S4_S4_RNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE+0x121)[0x7dfddccd8561]
+/workspace/coppelia_sim/libcoppeliaSim.so.1(_Z31simGetConfigForTipPose_internaliiPKifiPfPKfiS0_S0_S3_S3_Pv+0x174)[0x7dfddcb58b64]
+/workspace/pyrep/pyrep/backend/_sim_cffi.cpython-38-x86_64-linux-gnu.so(+0x47d4a)[0x7dfecde53d4a]
+/workspace/miniforge3/envs/rlbench/bin/python(PyCFunction_Call+0xde)[0x5f046db3064e]

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_01/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 1
+}

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_01/worker.log

@@ -0,0 +1,8 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.
+[episode1] analyzed 25/312 dense frames
+The X11 connection broke (error 1). Did the X11 server die?
+[CoppeliaSim:loadinfo]   done.

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_02/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 2
+}

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_02/worker.log

@@ -0,0 +1,13 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.
+ERROR:root:robot is in collision
+ERROR:root:Error when checking waypoints. Exception is: 
+[episode2] analyzed 25/325 dense frames
+[episode2] analyzed 50/325 dense frames
+[episode2] analyzed 75/325 dense frames
+[episode2] analyzed 100/325 dense frames
+The X11 connection broke (error 1). Did the X11 server die?
+[CoppeliaSim:loadinfo]   done.

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_03/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 3
+}

artifacts/results/iter6_parallel_smoke_4ep_independent/worker_03/worker.log

@@ -0,0 +1,8 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.
+[episode3] analyzed 25/286 dense frames
+The X11 connection broke (error 1). Did the X11 server die?
+[CoppeliaSim:loadinfo]   done.

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_00/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 0
+}

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_00/worker.log

@@ -0,0 +1,9 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+[episode0] analyzed 25/329 dense frames
+[episode0] analyzed 50/329 dense frames

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_01/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 1
+}

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_01/worker.log

@@ -0,0 +1,5 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_02/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 2
+}

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_02/worker.log

@@ -0,0 +1,9 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.
+ERROR:root:robot is in collision
+ERROR:root:Error when checking waypoints. Exception is: 
+[episode2] analyzed 25/325 dense frames
+[episode2] analyzed 50/325 dense frames

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_03/templates.json

@@ -0,0 +1,234 @@
+{
+  "templates": {
+    "approach_rel_poses": [
+      [
+        -0.3625197505146321,
+        0.08372429131714548,
+        0.13691903626392898,
+        0.6416348447407467,
+        0.6410872565393582,
+        0.29868034011062794,
+        0.2968196589661076
+      ],
+      [
+        -0.32979445983162176,
+        0.08659364792515764,
+        0.12045077578809749,
+        0.6287469690798895,
+        0.6287301318727057,
+        0.3243430018231259,
+        0.32276506520462594
+      ],
+      [
+        -0.2910925493361141,
+        0.08985276158129711,
+        0.10101096601604831,
+        0.6124184244028302,
+        0.6122709767290938,
+        0.35430518709849695,
+        0.35289624381998647
+      ],
+      [
+        -0.244818538450706,
+        0.09371070170750202,
+        0.07770067425331129,
+        0.5909841767637374,
+        0.5908549510320831,
+        0.3889344056030255,
+        0.38776043866540033
+      ],
+      [
+        -0.19036243621330934,
+        0.09824358579289771,
+        0.04990667629200307,
+        0.5635900638946065,
+        0.5631538374566398,
+        0.4277702591171187,
+        0.4268917903310006
+      ],
+      [
+        -0.12217129631756263,
+        0.09345070134195246,
+        -0.006546997831216039,
+        0.5503450945485194,
+        0.5264249147312487,
+        0.4450459711257041,
+        0.47109571176286236
+      ],
+      [
+        -0.11087643306877748,
+        0.09014108451750254,
+        -0.031580235354697894,
+        0.5622041544757288,
+        0.5226343775452266,
+        0.43323659021636735,
+        0.472319651291199
+      ]
+    ],
+    "pregrasp_rel_pose": [
+      -0.12217129631756263,
+      0.09345070134195246,
+      -0.006546997831216039,
+      0.5503450945485194,
+      0.5264249147312487,
+      0.4450459711257041,
+      0.47109571176286236
+    ],
+    "grasp_rel_pose": [
+      -0.09865615619553114,
+      0.09471180553609355,
+      -0.023587008474192483,
+      0.5452601219922814,
+      0.5150377871073674,
+      0.45379910210390734,
+      0.48113808012481835
+    ],
+    "retreat_rel_poses": [
+      [
+        0.2548525929450989,
+        0.23748670518398285,
+        0.292730450630188,
+        0.3300291728725241,
+        0.627796734434498,
+        0.6244628081014154,
+        0.3271058033489361
+      ],
+      [
+        0.22123366594314575,
+        0.20692090690135956,
+        0.3001290559768677,
+        0.3293369490711715,
+        0.6272174893562581,
+        0.6253236114995244,
+        0.3272701879538016
+      ],
+      [
+        0.17805954813957214,
+        0.16776257753372192,
+        0.3092261552810669,
+        0.3292203946981642,
+        0.627092205167199,
+        0.6254558193462786,
+        0.3273748859934127
+      ],
+      [
+        0.132161945104599,
+        0.1261904537677765,
+        0.31862151622772217,
+        0.32922833561062037,
+        0.6270130163004302,
+        0.6255370861132601,
+        0.32736330630941085
+      ],
+      [
+        0.05666536092758179,
+        0.058310166001319885,
+        0.33417391777038574,
+        0.3290531759368521,
+        0.6267400620289424,
+        0.6259557840834131,
+        0.3272617582705744
+      ],
+      [
+        0.0558415949344635,
+        0.05751470848917961,
+        0.3844491243362427,
+        0.32944931283048395,
+        0.6271554580797323,
+        0.6255767695203596,
+        0.3267918100454689
+      ],
+      [
+        0.05576580762863159,
+        0.053012214601039886,
+        0.5137358903884888,
+        0.3352571906446373,
+        0.6275228042160846,
+        0.6205178281888475,
+        0.329811114442256
+      ],
+      [
+        0.05594903230667114,
+        0.05725332349538803,
+        0.6016567945480347,
+        0.3290406683903203,
+        0.6269357648722236,
+        0.625839814295268,
+        0.3271212498634039
+      ],
+      [
+        0.11017268896102905,
+        0.09445947408676147,
+        0.6008111238479614,
+        0.3295905054305836,
+        0.6274618621218736,
+        0.6252874250360436,
+        0.3266149819043174
+      ],
+      [
+        0.1597837209701538,
+        0.1288578361272812,
+        0.6011183261871338,
+        0.3293548308249215,
+        0.627458650421648,
+        0.6253267712318655,
+        0.32678351642036935
+      ],
+      [
+        0.19115081429481506,
+        0.15017743408679962,
+        0.6012866497039795,
+        0.3296702831570499,
+        0.6273651178974856,
+        0.6251746471824625,
+        0.3269360392628933
+      ]
+    ],
+    "grasp_local_center": [
+      -0.09865615619553109,
+      0.09471180553609354,
+      -0.02358700847419226
+    ],
+    "grasp_region_extents": [
+      0.03,
+      0.015,
+      0.004
+    ],
+    "hold_open_angle": 0.7285879850387573,
+    "open_more_delta": 0.12,
+    "reference_tray_height": 1.0472617149353027,
+    "mask_handle_ids": [
+      163
+    ]
+  },
+  "template_episode": "episode0",
+  "template_frames": {
+    "pregrasp": 229,
+    "grasp": 234,
+    "right_close": 91,
+    "right_open": 130,
+    "approach": [
+      177,
+      187,
+      197,
+      208,
+      218,
+      229,
+      232
+    ],
+    "retreat": [
+      239,
+      244,
+      249,
+      254,
+      264,
+      274,
+      284,
+      295,
+      305,
+      310,
+      315
+    ]
+  },
+  "episode_offset": 3
+}

artifacts/results/iter7_parallel_smoke_4ep_independent/worker_03/worker.log

@@ -0,0 +1,5 @@
+WARNING:root:not sure how _robot_shapes are used is used.
+WARNING:root:Waypoints are not reachable right=(False, 'waypoint0 - DualPanda') left=(True, '')
+ERROR:root:Error when checking waypoints. Exception is: Error in task bimanual_take_tray_out_of_oven. Infeasible episode. Can't reach waypoint ((False, 'waypoint0 - DualPanda'), (True, '')).
+WARNING: QApplication was not created in the main() thread.
+WARNING:root:not sure how _robot_shapes are used is used.

artifacts/results/oven_episode0_iter6_independent_full/episode0.keyframes.csv

@@ -1,15 +1,15 @@
 frame_index,time_norm,door_angle,right_gripper_open,left_gripper_open,pregrasp_progress,pregrasp_distance,p_pre,p_ext,y_pre,y_ext,three_view_visibility,three_view_whole_tray_visibility,full_view_visibility,full_view_whole_tray_visibility,door_speed_abs,pregrasp_speed,phase_score,approach_active,y_retrieve,y_ready,phase_switch,keyframe_ordinal
-61,0.18597560975609756,1.5707963705062866,1.0,1.0,0.0,0.2815471545898242,0.05011632570066666,0.0007351239522298171,1.0,0.0,1.0,1.0,1.0,1.0,0.0,0.001988831083530873,0.020484405759290156,0.0,0.0,0.0,1.0,0
-91,0.2774390243902439,1.5705474615097046,0.0,1.0,0.0,0.2808195414862614,0.05034020214773165,0.0007343292236328118,1.0,0.0,1.0,1.0,1.0,1.0,0.0015521049499511719,0.003698527875334312,0.02737916203827016,0.0,0.0,0.0,1.0,1
-93,0.28353658536585363,1.5695279836654663,0.0,1.0,0.0,0.28105223514889954,0.050292342759892396,0.0007343292236328118,1.0,0.0,1.0,1.0,1.0,1.0,0.0032663345336914062,-0.0006522298561245643,0.012573085689973099,0.0,0.0,0.0,1.0,2
-130,0.39634146341463417,0.7299572229385376,1.0,1.0,0.0,0.2807313544459941,0.025190372580530272,0.0007351239522298171,0.0,0.0,1.0,1.0,1.0,1.0,0.07925748825073242,0.0010164967063264196,0.010363579970438247,0.0,0.0,0.0,1.0,3
-132,0.4024390243902439,0.7298687696456909,1.0,1.0,0.0,0.2810092232588846,0.02510434072988856,0.0007351239522298171,0.0,0.0,1.0,1.0,1.0,1.0,0.011527538299560547,-0.0006978082441660982,0.00627608518247214,0.0,0.0,0.0,1.0,4
-175,0.5335365853658537,0.7298847436904907,1.0,1.0,0.0,0.2810751229474485,0.025082568131946415,0.0007339318593343093,0.0,0.0,1.0,1.0,1.0,1.0,0.004901885986328125,0.004380962563844126,0.023794492288363108,0.0,0.0,0.0,1.0,5
-234,0.7134146341463414,0.5657719373703003,1.0,0.0,0.893594492894221,0.02980199367504531,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.03125190734863281,0.0022235016276432595,0.7503709776023946,0.0,1.0,1.0,1.0,6
-236,0.7195121951219512,0.5651155710220337,1.0,0.0,0.8942472865103727,0.029619159086471517,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.013582706451416016,-8.186486788797076e-05,0.741836007580705,0.0,1.0,1.0,1.0,7
-266,0.8109756097560976,0.39120233058929443,1.0,0.0,0.8942605091463285,0.02961545646280096,1.0,0.8942861239115393,1.0,1.0,1.0,0.9925977127057937,1.0,0.9958181793651616,0.005779266357421875,-0.001747388770690951,0.7418432800304807,0.0,1.0,1.0,1.0,8
-295,0.899390243902439,0.39143550395965576,1.0,0.0,0.8944821722383709,0.029553373615326353,1.0,1.0,1.0,1.0,0.01851851851851849,0.0,0.01851851851851849,0.3984725301798472,0.00179290771484375,-0.001257271121828142,0.7419651947311041,0.0,1.0,1.0,1.0,9
-315,0.9603658536585366,0.3914602994918823,1.0,1.0,0.9059970234001095,0.026328300305375845,0.7547424919077979,1.0,1.0,1.0,0.9142857142857143,0.031339031339031376,0.9261893004115226,0.5272011938678606,0.007107257843017578,0.048611156548003104,0.8814286120390221,1.0,1.0,1.0,1.0,10
-317,0.9664634146341463,0.39126694202423096,1.0,1.0,0.9114682917748704,0.024795909898429808,0.7485158670080193,1.0,1.0,1.0,0.9444444444444444,0.028490028490028463,0.9524798049077884,0.524831981883264,0.004470348358154297,-0.030190060249302673,0.6884365272281836,0.0,1.0,1.0,1.0,11
-322,0.9817073170731707,0.39166319370269775,1.0,1.0,0.8993793003031135,0.028181788579850032,0.7587438286971109,1.0,1.0,1.0,0.9238095238095239,0.03418803418803418,0.9333369913123,0.5257173382173383,0.003941059112548828,0.01527386607724876,0.7454400366499853,1.0,1.0,1.0,1.0,12
-328,1.0,0.3914250135421753,1.0,1.0,0.9016615670558527,0.02754257187710678,0.7553027732971624,1.0,1.0,1.0,0.9259259259259259,0.03703703703703698,0.9346441091297059,0.5261316872427984,0.0037717819213867188,-0.03662118395610786,0.6847395552050096,0.0,1.0,1.0,1.0,13
+61,0.1859756097560975,1.5707963705062866,1.0,1.0,0.0,0.2815471545898242,0.0501163257006666,0.0007351239522298,1.0,0.0,1.0,1.0,1.0,1.0,0.0,0.001988831083530873,0.015034897710199979,0.0,0.0,0.0,0.0,0
+91,0.2774390243902439,1.5705474615097046,0.0,1.0,0.0,0.2808195414862614,0.0503402021477316,0.0007343292236328,1.0,0.0,1.0,1.0,1.0,1.0,0.0015521049499489514,0.003698527875333757,0.015102060644319478,0.0,0.0,0.0,0.0,1
+93,0.2835365853658536,1.5695279836654663,0.0,1.0,0.0,0.2810522351488995,0.0502923427598923,0.0007343292236328,1.0,0.0,1.0,1.0,1.0,1.0,0.003266334533689186,-0.0006522298561245643,0.015087702827967689,0.0,0.0,0.0,0.0,2
+130,0.3963414634146341,0.7299572229385376,1.0,1.0,0.0,0.2807313544459941,0.0251903725805302,0.0007351239522298,0.0,0.0,1.0,1.0,1.0,1.0,0.07925748825073242,0.0010164967063258645,0.00755711177415906,0.0,0.0,0.0,0.0,3
+132,0.4024390243902439,0.7298687696456909,1.0,1.0,0.0,0.2810092232588846,0.0251043407298885,0.0007351239522298,0.0,0.0,1.0,1.0,1.0,1.0,0.011527538299560547,-0.0006978082441660982,0.00753130221896655,0.0,0.0,0.0,0.0,4
+175,0.5335365853658537,0.7298847436904907,1.0,1.0,0.0,0.2810751229474485,0.0250825681319464,0.0007339318593343,0.0,0.0,1.0,1.0,1.0,1.0,0.004901885986328125,0.004380962563844126,0.00752477043958392,0.0,0.0,0.0,0.0,5
+234,0.7134146341463414,0.5657719373703003,1.0,0.0,0.893594492894221,0.0298019936750453,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.03125190734863281,0.002223501627642982,0.9255161450259546,0.0,1.0,1.0,1.0,6
+236,0.7195121951219512,0.5651155710220337,1.0,0.0,0.8942472865103727,0.0296191590864715,1.0,1.0,1.0,1.0,1.0,1.0,1.0,1.0,0.013582706451416016,-8.186486788797076e-05,0.9259731005572609,0.0,1.0,1.0,1.0,7
+266,0.8109756097560976,0.3912023305892944,1.0,0.0,0.8942605091463285,0.0296154564628009,1.0,0.8942861239115393,1.0,1.0,1.0,0.9925977127057936,1.0,0.9958181793651616,0.005779266357421875,-0.0017473887706909857,0.9259823564024299,0.0,1.0,1.0,1.0,8
+295,0.899390243902439,0.3914355039596557,1.0,0.0,0.8944821722383709,0.0295533736153263,1.0,1.0,1.0,1.0,0.0185185185185184,0.0,0.0185185185185184,0.3984725301798472,0.0017929077148443051,-0.0012572711218280033,0.9261375205668596,0.0,1.0,1.0,1.0,9
+315,0.9603658536585366,0.3914602994918823,1.0,1.0,0.9059970234001096,0.0263283003053758,0.7547424919077979,1.0,1.0,1.0,0.9142857142857144,0.0313390313390313,0.9261893004115226,0.5272011938678606,0.007107257843017023,0.048611156548003,0.860620663952416,1.0,1.0,1.0,1.0,10
+317,0.9664634146341464,0.3912669420242309,1.0,1.0,0.9114682917748704,0.0247959098984298,0.7485158670080193,1.0,1.0,1.0,0.9444444444444444,0.0284900284900284,0.9524798049077884,0.524831981883264,0.004470348358153742,-0.03019006024930198,0.8625825643448151,0.0,1.0,1.0,1.0,11
+322,0.9817073170731708,0.3916631937026977,1.0,1.0,0.8993793003031135,0.02818178857985,0.7587438286971109,1.0,1.0,1.0,0.923809523809524,0.0341880341880341,0.9333369913123,0.5257173382173383,0.003941059112548273,0.015273866077249003,0.8571886588213127,1.0,1.0,1.0,1.0,12
+328,1.0,0.3914250135421753,1.0,1.0,0.9016615670558528,0.0275425718771067,0.7553027732971624,1.0,1.0,1.0,0.925925925925926,0.0370370370370369,0.934644109129706,0.5261316872427984,0.0037717819213856085,-0.036621183956108,0.8577539289282456,0.0,1.0,1.0,1.0,13

artifacts/results/oven_episode0_iter6_independent_full/episode0.metrics.json

@@ -7,15 +7,15 @@
   "phase_switch_falls": 0,
   "ppre_cross_frame": 216,
   "pext_cross_frame": 232,
-  "phase_cross_frame": 7,
+  "phase_cross_frame": 214,
   "retrieve_cross_frame": 215,
   "ready_cross_frame": 234,
   "ordering_ok": true,
-  "dense_boundary_error_to_retrieve_frames": 208.0,
-  "dense_boundary_error_frames": 227.0,
-  "dense_boundary_error_fraction": 0.6899696048632219,
-  "key_boundary_error_to_retrieve_keyframes": 6.0,
-  "key_boundary_error_keyframes": 6.0,
+  "dense_boundary_error_to_retrieve_frames": 1.0,
+  "dense_boundary_error_frames": 20.0,
+  "dense_boundary_error_fraction": 0.060790273556231005,
+  "key_boundary_error_to_retrieve_keyframes": 0.0,
+  "key_boundary_error_keyframes": 0.0,
   "auroc_vret_ypre_three": 0.37333333333333335,
   "auprc_vret_ypre_three": 0.6271326031029801,
   "auroc_vret_ypre_full": 0.37333333333333335,
@@ -26,10 +26,10 @@
   "auprc_pext_yext": 1.0,
   "auroc_phase_yretrieve": 1.0,
   "auprc_phase_yretrieve": 1.0,
-  "f1_phase_yretrieve": 0.5229357798165137,
-  "auroc_phase_yready": 0.9615384615384615,
-  "auprc_phase_yready": 0.7866105317416701,
-  "f1_phase_yready": 0.4556354916067146,
+  "f1_phase_yretrieve": 0.9956331877729258,
+  "auroc_phase_yready": 0.9982456140350877,
+  "auprc_phase_yready": 0.9955174255664128,
+  "f1_phase_yready": 0.9047619047619047,
   "baseline_auroc_door_yext": 0.0,
   "baseline_auprc_door_yext": 0.16605628948799386,
   "baseline_auroc_time_yext": 1.0,

artifacts/results/oven_episode0_iter6_independent_full/summary.json

@@ -7,14 +7,14 @@
     "phase_switch_falls": 0.0,
     "ppre_cross_frame": 216.0,
     "pext_cross_frame": 232.0,
-    "phase_cross_frame": 7.0,
+    "phase_cross_frame": 214.0,
     "retrieve_cross_frame": 215.0,
     "ready_cross_frame": 234.0,
-    "dense_boundary_error_to_retrieve_frames": 208.0,
-    "dense_boundary_error_frames": 227.0,
-    "dense_boundary_error_fraction": 0.6899696048632219,
-    "key_boundary_error_to_retrieve_keyframes": 6.0,
-    "key_boundary_error_keyframes": 6.0,
+    "dense_boundary_error_to_retrieve_frames": 1.0,
+    "dense_boundary_error_frames": 20.0,
+    "dense_boundary_error_fraction": 0.060790273556231005,
+    "key_boundary_error_to_retrieve_keyframes": 0.0,
+    "key_boundary_error_keyframes": 0.0,
     "auroc_vret_ypre_three": 0.37333333333333335,
     "auprc_vret_ypre_three": 0.6271326031029801,
     "auroc_vret_ypre_full": 0.37333333333333335,
@@ -25,10 +25,10 @@
     "auprc_pext_yext": 1.0,
     "auroc_phase_yretrieve": 1.0,
     "auprc_phase_yretrieve": 1.0,
-    "f1_phase_yretrieve": 0.5229357798165137,
-    "auroc_phase_yready": 0.9615384615384615,
-    "auprc_phase_yready": 0.7866105317416701,
-    "f1_phase_yready": 0.4556354916067146,
+    "f1_phase_yretrieve": 0.9956331877729258,
+    "auroc_phase_yready": 0.9982456140350877,
+    "auprc_phase_yready": 0.9955174255664128,
+    "f1_phase_yready": 0.9047619047619047,
     "baseline_auroc_door_yext": 0.0,
     "baseline_auprc_door_yext": 0.16605628948799386,
     "baseline_auroc_time_yext": 1.0,
@@ -57,14 +57,14 @@
     "phase_switch_falls": 0.0,
     "ppre_cross_frame": 216.0,
     "pext_cross_frame": 232.0,
-    "phase_cross_frame": 7.0,
+    "phase_cross_frame": 214.0,
     "retrieve_cross_frame": 215.0,
     "ready_cross_frame": 234.0,
-    "dense_boundary_error_to_retrieve_frames": 208.0,
-    "dense_boundary_error_frames": 227.0,
-    "dense_boundary_error_fraction": 0.6899696048632219,
-    "key_boundary_error_to_retrieve_keyframes": 6.0,
-    "key_boundary_error_keyframes": 6.0,
+    "dense_boundary_error_to_retrieve_frames": 1.0,
+    "dense_boundary_error_frames": 20.0,
+    "dense_boundary_error_fraction": 0.060790273556231005,
+    "key_boundary_error_to_retrieve_keyframes": 0.0,
+    "key_boundary_error_keyframes": 0.0,
     "auroc_vret_ypre_three": 0.37333333333333335,
     "auprc_vret_ypre_three": 0.6271326031029801,
     "auroc_vret_ypre_full": 0.37333333333333335,
@@ -75,10 +75,10 @@
     "auprc_pext_yext": 1.0,
     "auroc_phase_yretrieve": 1.0,
     "auprc_phase_yretrieve": 1.0,
-    "f1_phase_yretrieve": 0.5229357798165137,
-    "auroc_phase_yready": 0.9615384615384615,
-    "auprc_phase_yready": 0.7866105317416701,
-    "f1_phase_yready": 0.4556354916067146,
+    "f1_phase_yretrieve": 0.9956331877729258,
+    "auroc_phase_yready": 0.9982456140350877,
+    "auprc_phase_yready": 0.9955174255664128,
+    "f1_phase_yready": 0.9047619047619047,
     "baseline_auroc_door_yext": 0.0,
     "baseline_auprc_door_yext": 0.16605628948799386,
     "baseline_auroc_time_yext": 1.0,

code/rr_label_study/oven_study.py

@@ -1262,6 +1262,91 @@ def _annotate_phase_columns(frame_df: pd.DataFrame) -> pd.DataFrame:
     return frame_df
 
 
+def _episode_metrics_from_frames(
+    frame_df: pd.DataFrame,
+    key_df: pd.DataFrame,
+    episode_name: str,
+    description: str,
+    interventions: Dict[str, float],
+) -> Dict[str, object]:
+    y_pre_arr = frame_df["y_pre"].to_numpy(dtype=int)
+    y_ext_arr = frame_df["y_ext"].to_numpy(dtype=int)
+    y_retrieve_arr = frame_df["y_retrieve"].to_numpy(dtype=int)
+    y_ready_arr = frame_df["y_ready"].to_numpy(dtype=int)
+    p_pre_arr = frame_df["p_pre"].to_numpy(dtype=float)
+    p_ext_arr = frame_df["p_ext"].to_numpy(dtype=float)
+    phase_arr = frame_df["phase_switch"].to_numpy(dtype=int)
+    whole_vis = frame_df["full_view_whole_tray_visibility"].to_numpy(dtype=float)
+    door_angle_arr = frame_df["door_angle"].to_numpy(dtype=float)
+    time_arr = frame_df["time_norm"].to_numpy(dtype=float)
+
+    ppre_cross = _first_crossing(p_pre_arr, DEFAULT_PPRE_TAU)
+    pext_cross = _first_crossing(p_ext_arr, DEFAULT_PEXT_TAU)
+    phase_cross = _first_crossing(frame_df["phase_switch"].to_numpy(dtype=float), 0.5)
+    retrieve_cross = _first_crossing(y_retrieve_arr.astype(float), 0.5)
+    ready_cross = _first_crossing(y_ready_arr.astype(float), 0.5)
+    phase_rises, phase_falls = _transition_count(phase_arr)
+
+    key_phase_cross = _first_crossing(key_df["phase_switch"].to_numpy(dtype=float), 0.5)
+    key_retrieve_cross = _first_crossing(key_df["y_retrieve"].to_numpy(dtype=float), 0.5)
+    key_ready_cross = _first_crossing(key_df["y_ready"].to_numpy(dtype=float), 0.5)
+
+    return {
+        "episode_name": episode_name,
+        "description": description,
+        "num_dense_frames": int(len(frame_df)),
+        "num_keyframes": int(len(key_df)),
+        "phase_switch_rises": int(phase_rises),
+        "phase_switch_falls": int(phase_falls),
+        "ppre_cross_frame": int(ppre_cross),
+        "pext_cross_frame": int(pext_cross),
+        "phase_cross_frame": int(phase_cross),
+        "retrieve_cross_frame": int(retrieve_cross),
+        "ready_cross_frame": int(ready_cross),
+        "ordering_ok": bool(ppre_cross == -1 or pext_cross == -1 or ppre_cross <= pext_cross),
+        "dense_boundary_error_to_retrieve_frames": float(abs(phase_cross - retrieve_cross))
+        if phase_cross >= 0 and retrieve_cross >= 0
+        else float("nan"),
+        "dense_boundary_error_frames": float(abs(phase_cross - ready_cross))
+        if phase_cross >= 0 and ready_cross >= 0
+        else float("nan"),
+        "dense_boundary_error_fraction": float(abs(phase_cross - ready_cross) / len(frame_df))
+        if phase_cross >= 0 and ready_cross >= 0
+        else float("nan"),
+        "key_boundary_error_to_retrieve_keyframes": float(abs(key_phase_cross - key_retrieve_cross))
+        if key_phase_cross >= 0 and key_retrieve_cross >= 0
+        else float("nan"),
+        "key_boundary_error_keyframes": float(abs(key_phase_cross - key_ready_cross))
+        if key_phase_cross >= 0 and key_ready_cross >= 0
+        else float("nan"),
+        "auroc_vret_ypre_three": _safe_auc(y_pre_arr, frame_df["three_view_visibility"].to_numpy(dtype=float)),
+        "auprc_vret_ypre_three": _safe_auprc(y_pre_arr, frame_df["three_view_visibility"].to_numpy(dtype=float)),
+        "auroc_vret_ypre_full": _safe_auc(y_pre_arr, frame_df["full_view_visibility"].to_numpy(dtype=float)),
+        "auprc_vret_ypre_full": _safe_auprc(y_pre_arr, frame_df["full_view_visibility"].to_numpy(dtype=float)),
+        "auroc_ppre_ypre": _safe_auc(y_pre_arr, p_pre_arr),
+        "auprc_ppre_ypre": _safe_auprc(y_pre_arr, p_pre_arr),
+        "auroc_pext_yext": _safe_auc(y_ext_arr, p_ext_arr),
+        "auprc_pext_yext": _safe_auprc(y_ext_arr, p_ext_arr),
+        "auroc_phase_yretrieve": _safe_auc(y_retrieve_arr, frame_df["phase_score"].to_numpy(dtype=float)),
+        "auprc_phase_yretrieve": _safe_auprc(y_retrieve_arr, frame_df["phase_score"].to_numpy(dtype=float)),
+        "f1_phase_yretrieve": float(f1_score(y_retrieve_arr, phase_arr))
+        if np.any(y_retrieve_arr) and np.any(phase_arr)
+        else float("nan"),
+        "auroc_phase_yready": _safe_auc(y_ready_arr, frame_df["phase_score"].to_numpy(dtype=float)),
+        "auprc_phase_yready": _safe_auprc(y_ready_arr, frame_df["phase_score"].to_numpy(dtype=float)),
+        "f1_phase_yready": float(f1_score(y_ready_arr, phase_arr))
+        if np.any(y_ready_arr) and np.any(phase_arr)
+        else float("nan"),
+        "baseline_auroc_door_yext": _safe_auc(y_ext_arr, door_angle_arr),
+        "baseline_auprc_door_yext": _safe_auprc(y_ext_arr, door_angle_arr),
+        "baseline_auroc_time_yext": _safe_auc(y_ext_arr, time_arr),
+        "baseline_auprc_time_yext": _safe_auprc(y_ext_arr, time_arr),
+        "baseline_auroc_whole_vis_yext": _safe_auc(y_ext_arr, whole_vis),
+        "baseline_auprc_whole_vis_yext": _safe_auprc(y_ext_arr, whole_vis),
+        **interventions,
+    }
+
+
 def _interventional_validity(
     task,
     cache: ReplayCache,
@@ -1409,85 +1494,16 @@ def _analyze_episode(
         keyframes = [index for index in _keypoint_discovery(demo) if index < num_frames]
         key_df = _keyframe_subset(frame_df, keyframes)
 
-        y_pre_arr = frame_df["y_pre"].to_numpy(dtype=int)
-        y_ext_arr = frame_df["y_ext"].to_numpy(dtype=int)
-        y_retrieve_arr = frame_df["y_retrieve"].to_numpy(dtype=int)
-        y_ready_arr = frame_df["y_ready"].to_numpy(dtype=int)
-        p_pre_arr = frame_df["p_pre"].to_numpy(dtype=float)
-        p_ext_arr = frame_df["p_ext"].to_numpy(dtype=float)
-        phase_arr = frame_df["phase_switch"].to_numpy(dtype=int)
-        whole_vis = frame_df["full_view_whole_tray_visibility"].to_numpy(dtype=float)
-        door_angle_arr = frame_df["door_angle"].to_numpy(dtype=float)
-        time_arr = frame_df["time_norm"].to_numpy(dtype=float)
-
-        ppre_cross = _first_crossing(p_pre_arr, DEFAULT_PPRE_TAU)
-        pext_cross = _first_crossing(p_ext_arr, DEFAULT_PEXT_TAU)
-        phase_cross = _first_crossing(frame_df["phase_switch"].to_numpy(dtype=float), 0.5)
-        retrieve_cross = _first_crossing(y_retrieve_arr.astype(float), 0.5)
-        ready_cross = _first_crossing(y_ready_arr.astype(float), 0.5)
-        phase_rises, phase_falls = _transition_count(phase_arr)
-
-        key_phase_cross = _first_crossing(key_df["phase_switch"].to_numpy(dtype=float), 0.5)
-        key_retrieve_cross = _first_crossing(key_df["y_retrieve"].to_numpy(dtype=float), 0.5)
-        key_ready_cross = _first_crossing(key_df["y_ready"].to_numpy(dtype=float), 0.5)
-
         interventions = _interventional_validity(
             task, cache, episode_dir, demo, templates, frame_df
         )
-        metrics = {
-            "episode_name": episode_dir.name,
-            "description": descriptions[0],
-            "num_dense_frames": int(num_frames),
-            "num_keyframes": int(len(key_df)),
-            "phase_switch_rises": int(phase_rises),
-            "phase_switch_falls": int(phase_falls),
-            "ppre_cross_frame": int(ppre_cross),
-            "pext_cross_frame": int(pext_cross),
-            "phase_cross_frame": int(phase_cross),
-            "retrieve_cross_frame": int(retrieve_cross),
-            "ready_cross_frame": int(ready_cross),
-            "ordering_ok": bool(ppre_cross == -1 or pext_cross == -1 or ppre_cross <= pext_cross),
-            "dense_boundary_error_to_retrieve_frames": float(abs(phase_cross - retrieve_cross))
-            if phase_cross >= 0 and retrieve_cross >= 0
-            else float("nan"),
-            "dense_boundary_error_frames": float(abs(phase_cross - ready_cross))
-            if phase_cross >= 0 and ready_cross >= 0
-            else float("nan"),
-            "dense_boundary_error_fraction": float(abs(phase_cross - ready_cross) / len(frame_df))
-            if phase_cross >= 0 and ready_cross >= 0
-            else float("nan"),
-            "key_boundary_error_to_retrieve_keyframes": float(abs(key_phase_cross - key_retrieve_cross))
-            if key_phase_cross >= 0 and key_retrieve_cross >= 0
-            else float("nan"),
-            "key_boundary_error_keyframes": float(abs(key_phase_cross - key_ready_cross))
-            if key_phase_cross >= 0 and key_ready_cross >= 0
-            else float("nan"),
-            "auroc_vret_ypre_three": _safe_auc(y_pre_arr, frame_df["three_view_visibility"].to_numpy(dtype=float)),
-            "auprc_vret_ypre_three": _safe_auprc(y_pre_arr, frame_df["three_view_visibility"].to_numpy(dtype=float)),
-            "auroc_vret_ypre_full": _safe_auc(y_pre_arr, frame_df["full_view_visibility"].to_numpy(dtype=float)),
-            "auprc_vret_ypre_full": _safe_auprc(y_pre_arr, frame_df["full_view_visibility"].to_numpy(dtype=float)),
-            "auroc_ppre_ypre": _safe_auc(y_pre_arr, p_pre_arr),
-            "auprc_ppre_ypre": _safe_auprc(y_pre_arr, p_pre_arr),
-            "auroc_pext_yext": _safe_auc(y_ext_arr, p_ext_arr),
-            "auprc_pext_yext": _safe_auprc(y_ext_arr, p_ext_arr),
-            "auroc_phase_yretrieve": _safe_auc(y_retrieve_arr, frame_df["phase_score"].to_numpy(dtype=float)),
-            "auprc_phase_yretrieve": _safe_auprc(y_retrieve_arr, frame_df["phase_score"].to_numpy(dtype=float)),
-            "f1_phase_yretrieve": float(f1_score(y_retrieve_arr, phase_arr))
-            if np.any(y_retrieve_arr) and np.any(phase_arr)
-            else float("nan"),
-            "auroc_phase_yready": _safe_auc(y_ready_arr, frame_df["phase_score"].to_numpy(dtype=float)),
-            "auprc_phase_yready": _safe_auprc(y_ready_arr, frame_df["phase_score"].to_numpy(dtype=float)),
-            "f1_phase_yready": float(f1_score(y_ready_arr, phase_arr))
-            if np.any(y_ready_arr) and np.any(phase_arr)
-            else float("nan"),
-            "baseline_auroc_door_yext": _safe_auc(y_ext_arr, door_angle_arr),
-            "baseline_auprc_door_yext": _safe_auprc(y_ext_arr, door_angle_arr),
-            "baseline_auroc_time_yext": _safe_auc(y_ext_arr, time_arr),
-            "baseline_auprc_time_yext": _safe_auprc(y_ext_arr, time_arr),
-            "baseline_auroc_whole_vis_yext": _safe_auc(y_ext_arr, whole_vis),
-            "baseline_auprc_whole_vis_yext": _safe_auprc(y_ext_arr, whole_vis),
-            **interventions,
-        }
+        metrics = _episode_metrics_from_frames(
+            frame_df=frame_df,
+            key_df=key_df,
+            episode_name=episode_dir.name,
+            description=descriptions[0],
+            interventions=interventions,
+        )
         return EpisodeArtifacts(
             episode_name=episode_dir.name,
             dense=frame_df,

code/scripts/refresh_saved_oven_study.py

@@ -0,0 +1,100 @@
+import argparse
+import json
+from pathlib import Path
+import sys
+
+import pandas as pd
+
+
+PROJECT_ROOT = Path(__file__).resolve().parents[1]
+if str(PROJECT_ROOT) not in sys.path:
+    sys.path.insert(0, str(PROJECT_ROOT))
+
+from rr_label_study.oven_study import (  # noqa: E402
+    _aggregate_summary,
+    _annotate_phase_columns,
+    _episode_metrics_from_frames,
+)
+
+
+INTERVENTION_KEYS = [
+    "pre_ready_open_more_increases_pext",
+    "pre_ready_open_more_trials",
+    "pre_ready_hold_open_increases_pext",
+    "pre_ready_hold_open_trials",
+    "pre_ready_extract_success",
+    "pre_ready_extract_trials",
+    "pre_ready_wait_extract_success",
+    "pre_ready_wait_trials",
+    "post_ready_extract_success",
+    "post_ready_extract_trials",
+    "post_ready_open_more_low_gain",
+    "post_ready_open_more_trials",
+    "post_ready_hold_open_low_gain",
+    "post_ready_hold_open_trials",
+]
+
+
+def _refresh_episode(result_dir: Path, episode_name: str) -> dict:
+    dense_path = result_dir / f"{episode_name}.dense.csv"
+    keyframes_path = result_dir / f"{episode_name}.keyframes.csv"
+    metrics_path = result_dir / f"{episode_name}.metrics.json"
+
+    dense_df = pd.read_csv(dense_path)
+    dense_df = _annotate_phase_columns(dense_df)
+
+    old_key_df = pd.read_csv(keyframes_path)
+    keyframe_indices = old_key_df["frame_index"].astype(int).tolist()
+    key_df = dense_df[dense_df["frame_index"].isin(keyframe_indices)].copy()
+    key_df = key_df.sort_values("frame_index").reset_index(drop=True)
+    key_df["keyframe_ordinal"] = range(len(key_df))
+
+    with metrics_path.open("r", encoding="utf-8") as handle:
+        old_metrics = json.load(handle)
+    interventions = {
+        key: float(old_metrics[key]) for key in INTERVENTION_KEYS if key in old_metrics
+    }
+
+    metrics = _episode_metrics_from_frames(
+        frame_df=dense_df,
+        key_df=key_df,
+        episode_name=episode_name,
+        description=str(old_metrics.get("description", "")),
+        interventions=interventions,
+    )
+
+    dense_df.to_csv(dense_path, index=False)
+    key_df.to_csv(keyframes_path, index=False)
+    with metrics_path.open("w", encoding="utf-8") as handle:
+        json.dump(metrics, handle, indent=2)
+    return metrics
+
+
+def main(argv=None) -> int:
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--result-dir", required=True)
+    parser.add_argument("--episodes", nargs="*")
+    args = parser.parse_args(argv)
+
+    result_dir = Path(args.result_dir)
+    episode_metrics = []
+    if args.episodes:
+        episode_names = args.episodes
+    else:
+        episode_names = sorted(
+            path.stem.replace(".metrics", "")
+            for path in result_dir.glob("episode*.metrics.json")
+        )
+
+    for episode_name in episode_names:
+        episode_metrics.append(_refresh_episode(result_dir, episode_name))
+
+    summary = _aggregate_summary(episode_metrics)
+    with result_dir.joinpath("summary.json").open("w", encoding="utf-8") as handle:
+        json.dump(summary, handle, indent=2)
+    print(json.dumps(summary, indent=2))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())