Upload eval/eval_bert_partC.py

eval/eval_bert_partC.py

@@ -3,7 +3,9 @@
 policies = defaultdict(lambda: {"success":0,"cost":0.0,"n":0})
 
 print("\n[4] Evaluating all policies...")
-for iid, model_results in traces.items():
+for idx, (iid, model_results) in enumerate(traces.items()):
+    if idx % 100 == 0:
+        print(f"  Progress: {idx}/{len(traces)}")
     problem = next(iter(model_results.values()))['problem']
     task_type = classify_task(problem)
     floor = TASK_FLOOR.get(task_type, 2)
@@ -25,8 +27,8 @@ for iid, model_results in traces.items():
         policies['frontier']['cost'] += model_results[f_model]['cost']
     policies['frontier']['n'] += 1
 
-    # BERT
-    bert_tier, bert_conf = route_bert(problem)
+    # BERT (per-tier success prediction with cascade)
+    bert_tier, bert_conf, bert_probs = route_bert(problem)
     bert_tier = max(bert_tier, floor)
     m_bert = TIER_TO_SWE.get(bert_tier, f_model)
     if m_bert in model_results:
@@ -37,19 +39,19 @@ for iid, model_results in traces.items():
         policies['bert']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
     policies['bert']['n'] += 1
 
-    # v11 XGBoost
-    v11_tier, v11_conf, v11_probs = route_v11(problem)
-    v11_tier = max(v11_tier, floor)
-    m_v11 = TIER_TO_SWE.get(v11_tier, f_model)
-    if m_v11 in model_results:
-        policies['v11_xgboost']['success'] += int(model_results[m_v11]['resolved'])
-        policies['v11_xgboost']['cost'] += model_results[m_v11]['cost']
+    # v10 XGBoost
+    v10_tier, v10_conf, v10_probs = route_v10(problem)
+    v10_tier = max(v10_tier, floor)
+    m_v10 = TIER_TO_SWE.get(v10_tier, f_model)
+    if m_v10 in model_results:
+        policies['v10_xgboost']['success'] += int(model_results[m_v10]['resolved'])
+        policies['v10_xgboost']['cost'] += model_results[m_v10]['cost']
     else:
-        policies['v11_xgboost']['success'] += int(model_results.get(f_model,{}).get('resolved',0))
-        policies['v11_xgboost']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
-    policies['v11_xgboost']['n'] += 1
+        policies['v10_xgboost']['success'] += int(model_results.get(f_model,{}).get('resolved',0))
+        policies['v10_xgboost']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
+    policies['v10_xgboost']['n'] += 1
 
-    # BERT + feedback
+    # BERT + feedback (escalate on failure)
     if m_bert in model_results and model_results[m_bert]['resolved']:
         policies['bert_feedback']['success'] += 1
         policies['bert_feedback']['cost'] += model_results[m_bert]['cost']
@@ -66,22 +68,22 @@ for iid, model_results in traces.items():
             policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01)
     policies['bert_feedback']['n'] += 1
 
-    # v11 + feedback
-    if m_v11 in model_results and model_results[m_v11]['resolved']:
-        policies['v11_feedback']['success'] += 1
-        policies['v11_feedback']['cost'] += model_results[m_v11]['cost']
+    # v10 + feedback
+    if m_v10 in model_results and model_results[m_v10]['resolved']:
+        policies['v10_feedback']['success'] += 1
+        policies['v10_feedback']['cost'] += model_results[m_v10]['cost']
     else:
-        up_tier = min(v11_tier + 1, 5)
+        up_tier = min(v10_tier + 1, 5)
         m_up = TIER_TO_SWE.get(up_tier, f_model)
         if m_up in model_results and model_results[m_up]['resolved']:
-            policies['v11_feedback']['success'] += 1
-            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01) + model_results[m_up]['cost']
+            policies['v10_feedback']['success'] += 1
+            policies['v10_feedback']['cost'] += model_results.get(m_v10,{}).get('cost',0.01) + model_results[m_up]['cost']
         elif f_model in model_results and model_results[f_model]['resolved']:
-            policies['v11_feedback']['success'] += 1
-            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01) + model_results[f_model]['cost']
+            policies['v10_feedback']['success'] += 1
+            policies['v10_feedback']['cost'] += model_results.get(m_v10,{}).get('cost',0.01) + model_results[f_model]['cost']
         else:
-            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01)
-    policies['v11_feedback']['n'] += 1
+            policies['v10_feedback']['cost'] += model_results.get(m_v10,{}).get('cost',0.01)
+    policies['v10_feedback']['n'] += 1
 
     # Always cheap
     c_model = 'deepseek-v4-flash'
@@ -89,3 +91,5 @@ for iid, model_results in traces.items():
         policies['always_cheap']['success'] += int(model_results[c_model]['resolved'])
         policies['always_cheap']['cost'] += model_results[c_model]['cost']
     policies['always_cheap']['n'] += 1
+
+print(f"  Progress: {len(traces)}/{len(traces)} - DONE")