Upload eval/run_bert_eval.py

eval/run_bert_eval.py

@@ -0,0 +1,47 @@
+#!/usr/bin/env python3
+"""Evaluate BERT router on SWE-Router and compare with v11 XGBoost.
+Combines eval_bert_router.py + eval_bert_part2.py + eval_bert_part3.py + eval_bert_part4.py
+"""
+import json, os, sys, random, pickle, math
+import numpy as np
+from collections import defaultdict
+
+MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2',
+          'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash']
+MODEL_TIER = {
+    'deepseek-v4-flash': 1, 'gpt-5-nano': 1,
+    'gpt-5-mini': 2, 'deepseek-v3.2': 2,
+    'gemini-2.5-pro': 3,
+    'claude-opus-4.7': 4, 'gpt-5.2': 4,
+    'gemini-3-pro': 5,
+}
+TIER_COST = {1:0.01, 2:0.05, 3:0.15, 4:0.30, 5:0.50}
+TIER_TO_SWE = {1:'deepseek-v4-flash',2:'gpt-5-mini',3:'gemini-2.5-pro',4:'claude-opus-4.7',5:'gemini-3-pro'}
+CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test","compile","runtime","segfault","thread","async","class","module","import","error","traceback"]
+LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
+RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
+TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
+CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
+SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
+LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
+MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
+
+def extract_features(pt):
+    r = pt.lower()
+    return {'req_len':len(pt),'num_words':len(pt.split()),'has_code':int(any(k in r for k in CODE_KW)),'n_code':sum(1 for k in CODE_KW if k in r),'has_legal':int(any(k in r for k in LEGAL_KW)),'has_research':int(any(k in r for k in RESEARCH_KW)),'has_tool':int(any(k in r for k in TOOL_KW)),'has_critical':int(any(k in r for k in CRITICAL_KW)),'has_simple':int(any(k in r for k in SIMPLE_KW)),'has_long':int(any(k in r for k in LONG_KW)),'has_math':int(any(k in r for k in MATH_KW)),'has_error_msg':int('error' in r or 'traceback' in r),'has_file_path':int('/' in r),'n_lines':pt.count('\n')+1,'has_version':int('version' in r or 'update' in r),'has_add':int('add' in r or 'new' in r),'has_fix':int('fix' in r or 'bug' in r),'has_change':int('change' in r or 'modify' in r),'has_remove':int('remove' in r),'has_test':int('test' in r or 'spec' in r),'has_doc':int('doc' in r)}
+
+TASK_FLOOR = {"quick_answer":1,"coding":3,"research":3,"document_drafting":2,"legal_regulated":4,"tool_heavy":2,"retrieval_heavy":2,"long_horizon":3,"unknown_ambiguous":3}
+
+def classify_task(text):
+    r = text.lower()
+    if any(k in r for k in ["contract","legal","compliance","gdpr","privacy"]): return "legal_regulated"
+    if any(k in r for k in ["debug","fix","bug","implement","refactor","code","function"]): return "coding"
+    if any(k in r for k in ["research","find sources","literature","investigate"]): return "research"
+    if any(k in r for k in ["search","fetch","api","query","database"]): return "tool_heavy"
+    if any(k in r for k in ["plan","roadmap","orchestrate","migrate","deploy"]): return "long_horizon"
+    if any(k in r for k in ["draft","write","compose","document"]): return "document_drafting"
+    if any(k in r for k in ["what is","explain","define","briefly"]): return "quick_answer"
+    return "unknown_ambiguous"
+
+print("BERT Router Evaluation on SWE-Router")
+print("="*60)