Upload eval/eval_bert_partC.py

eval/eval_bert_partC.py

@@ -0,0 +1,91 @@
+
+# ── Evaluate ──
+policies = defaultdict(lambda: {"success":0,"cost":0.0,"n":0})
+
+print("\n[4] Evaluating all policies...")
+for iid, model_results in traces.items():
+    problem = next(iter(model_results.values()))['problem']
+    task_type = classify_task(problem)
+    floor = TASK_FLOOR.get(task_type, 2)
+    f_model = 'claude-opus-4.7'
+
+    # Oracle
+    resolved = [(m, r) for m, r in model_results.items() if r['resolved']]
+    if resolved:
+        cheapest = min(resolved, key=lambda x: TIER_COST.get(MODEL_TIER[x[0]], 1.0))
+        policies['oracle']['success'] += 1
+        policies['oracle']['cost'] += cheapest[1]['cost']
+    else:
+        policies['oracle']['cost'] += min(r['cost'] for r in model_results.values())
+    policies['oracle']['n'] += 1
+
+    # Always frontier
+    if f_model in model_results:
+        policies['frontier']['success'] += int(model_results[f_model]['resolved'])
+        policies['frontier']['cost'] += model_results[f_model]['cost']
+    policies['frontier']['n'] += 1
+
+    # BERT
+    bert_tier, bert_conf = route_bert(problem)
+    bert_tier = max(bert_tier, floor)
+    m_bert = TIER_TO_SWE.get(bert_tier, f_model)
+    if m_bert in model_results:
+        policies['bert']['success'] += int(model_results[m_bert]['resolved'])
+        policies['bert']['cost'] += model_results[m_bert]['cost']
+    else:
+        policies['bert']['success'] += int(model_results.get(f_model,{}).get('resolved',0))
+        policies['bert']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
+    policies['bert']['n'] += 1
+
+    # v11 XGBoost
+    v11_tier, v11_conf, v11_probs = route_v11(problem)
+    v11_tier = max(v11_tier, floor)
+    m_v11 = TIER_TO_SWE.get(v11_tier, f_model)
+    if m_v11 in model_results:
+        policies['v11_xgboost']['success'] += int(model_results[m_v11]['resolved'])
+        policies['v11_xgboost']['cost'] += model_results[m_v11]['cost']
+    else:
+        policies['v11_xgboost']['success'] += int(model_results.get(f_model,{}).get('resolved',0))
+        policies['v11_xgboost']['cost'] += model_results.get(f_model,{}).get('cost',0.3)
+    policies['v11_xgboost']['n'] += 1
+
+    # BERT + feedback
+    if m_bert in model_results and model_results[m_bert]['resolved']:
+        policies['bert_feedback']['success'] += 1
+        policies['bert_feedback']['cost'] += model_results[m_bert]['cost']
+    else:
+        up_tier = min(bert_tier + 1, 5)
+        m_up = TIER_TO_SWE.get(up_tier, f_model)
+        if m_up in model_results and model_results[m_up]['resolved']:
+            policies['bert_feedback']['success'] += 1
+            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01) + model_results[m_up]['cost']
+        elif f_model in model_results and model_results[f_model]['resolved']:
+            policies['bert_feedback']['success'] += 1
+            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01) + model_results[f_model]['cost']
+        else:
+            policies['bert_feedback']['cost'] += model_results.get(m_bert,{}).get('cost',0.01)
+    policies['bert_feedback']['n'] += 1
+
+    # v11 + feedback
+    if m_v11 in model_results and model_results[m_v11]['resolved']:
+        policies['v11_feedback']['success'] += 1
+        policies['v11_feedback']['cost'] += model_results[m_v11]['cost']
+    else:
+        up_tier = min(v11_tier + 1, 5)
+        m_up = TIER_TO_SWE.get(up_tier, f_model)
+        if m_up in model_results and model_results[m_up]['resolved']:
+            policies['v11_feedback']['success'] += 1
+            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01) + model_results[m_up]['cost']
+        elif f_model in model_results and model_results[f_model]['resolved']:
+            policies['v11_feedback']['success'] += 1
+            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01) + model_results[f_model]['cost']
+        else:
+            policies['v11_feedback']['cost'] += model_results.get(m_v11,{}).get('cost',0.01)
+    policies['v11_feedback']['n'] += 1
+
+    # Always cheap
+    c_model = 'deepseek-v4-flash'
+    if c_model in model_results:
+        policies['always_cheap']['success'] += int(model_results[c_model]['resolved'])
+        policies['always_cheap']['cost'] += model_results[c_model]['cost']
+    policies['always_cheap']['n'] += 1