Upload eval/eval_bert_partD.py

eval/eval_bert_partD.py

@@ -0,0 +1,54 @@
+
+# ── Results ──
+fr = policies['frontier']
+fr_cost = fr['cost'] / fr['n']
+fr_succ = fr['success'] / fr['n']
+
+print(f"\n\n{'='*70}")
+print("BERT vs XGBoost ROUTER COMPARISON ON SWE-BENCH")
+print(f"{'='*70}")
+print(f"\n{'Policy':<20} {'Success':>10} {'AvgCost':>10} {'CostRed':>10}")
+print("-"*52)
+for name in ['oracle','bert_feedback','v11_feedback','bert','v11_xgboost','frontier','always_cheap']:
+    if name not in policies: continue
+    r = policies[name]
+    sr = r['success']/r['n'] if r['n'] > 0 else 0
+    ac = r['cost']/r['n'] if r['n'] > 0 else 0
+    cr = (1-ac/fr_cost)*100 if fr_cost > 0 else 0
+    print(f"{name:<20} {sr:>10.3f} {ac:>10.4f} {cr:>9.1f}%")
+
+print(f"\nQuality gap vs frontier:")
+for name in ['bert','bert_feedback','v11_xgboost','v11_feedback']:
+    r = policies[name]
+    sr = r['success']/r['n'] if r['n'] > 0 else 0
+    gap = (sr - fr_succ) * 100
+    print(f"  {name}: {gap:+.1f}pp vs frontier")
+
+# BERT tier distribution
+print(f"\nBERT tier distribution:")
+bert_tiers = defaultdict(int)
+for iid, model_results in list(traces.items())[:500]:
+    problem = next(iter(model_results.values()))['problem']
+    t, c = route_bert(problem)
+    bert_tiers[t] += 1
+for t in sorted(bert_tiers):
+    print(f"  Tier {t}: {bert_tiers[t]}")
+
+# Save results
+results = {}
+for name, r in policies.items():
+    sr = r['success']/r['n'] if r['n'] > 0 else 0
+    ac = r['cost']/r['n'] if r['n'] > 0 else 0
+    cr = (1-ac/fr_cost)*100 if fr_cost > 0 else 0
+    results[name] = {"success": round(sr, 4), "avg_cost": round(ac, 4), "costRed": round(cr, 1)}
+
+from huggingface_hub import HfApi
+import tempfile
+api = HfApi()
+with tempfile.NamedTemporaryFile(mode='w', suffix='.json', delete=False) as f:
+    json.dump(results, f, indent=2)
+    api.upload_file(path_or_fileobj=f.name, path_in_repo="eval/bert_vs_xgboost_results.json",
+                    repo_id="narcolepticchicken/agent-cost-optimizer", repo_type="model")
+    os.unlink(f.name)
+print(f"\nResults uploaded to eval/bert_vs_xgboost_results.json")
+print("DONE!")