| """Evaluate BERT router on SWE-Router and compare with v11 XGBoost.""" |
| import json, os, sys, random, pickle, math |
| import numpy as np |
| from collections import defaultdict |
|
|
| MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2', |
| 'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash'] |
| MODEL_TIER = {'deepseek-v4-flash':1,'gpt-5-nano':1,'gpt-5-mini':2,'deepseek-v3.2':2, |
| 'gemini-2.5-pro':3,'claude-opus-4.7':4,'gpt-5.2':4,'gemini-3-pro':5} |
| TIER_COST = {1:0.01,2:0.05,3:0.15,4:0.30,5:0.50} |
| TIER_TO_SWE = {1:'deepseek-v4-flash',2:'gpt-5-mini',3:'gemini-2.5-pro',4:'claude-opus-4.7',5:'gemini-3-pro'} |
| TASK_FLOOR = {"quick_answer":1,"coding":3,"research":3,"document_drafting":2, |
| "legal_regulated":4,"tool_heavy":2,"retrieval_heavy":2,"long_horizon":3,"unknown_ambiguous":3} |
| CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test", |
| "compile","runtime","segfault","thread","async","class","module","import","error","traceback"] |
| LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"] |
| RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"] |
| TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"] |
| CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"] |
| SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"] |
| LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"] |
| MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"] |
|
|
| def extract_features(pt): |
| r = pt.lower() |
| return {'req_len':len(pt),'num_words':len(pt.split()), |
| 'has_code':int(any(k in r for k in CODE_KW)),'n_code':sum(1 for k in CODE_KW if k in r), |
| 'has_legal':int(any(k in r for k in LEGAL_KW)),'has_research':int(any(k in r for k in RESEARCH_KW)), |
| 'has_tool':int(any(k in r for k in TOOL_KW)),'has_critical':int(any(k in r for k in CRITICAL_KW)), |
| 'has_simple':int(any(k in r for k in SIMPLE_KW)),'has_long':int(any(k in r for k in LONG_KW)), |
| 'has_math':int(any(k in r for k in MATH_KW)),'has_error_msg':int('error' in r or 'traceback' in r), |
| 'has_file_path':int('/' in r),'n_lines':pt.count('\n')+1, |
| 'has_version':int('version' in r or 'update' in r),'has_add':int('add' in r or 'new' in r), |
| 'has_fix':int('fix' in r or 'bug' in r),'has_change':int('change' in r or 'modify' in r), |
| 'has_remove':int('remove' in r),'has_test':int('test' in r or 'spec' in r), |
| 'has_doc':int('doc' in r or 'readme' in r)} |
|
|
| def classify_task(text): |
| r = text.lower() |
| if any(k in r for k in ["contract","legal","compliance","gdpr","privacy"]): return "legal_regulated" |
| if any(k in r for k in ["debug","fix","bug","implement","refactor","code","function"]): return "coding" |
| if any(k in r for k in ["research","find sources","literature","investigate"]): return "research" |
| if any(k in r for k in ["search","fetch","api","query","database"]): return "tool_heavy" |
| if any(k in r for k in ["plan","roadmap","orchestrate","migrate","deploy"]): return "long_horizon" |
| if any(k in r for k in ["draft","write","compose","document"]): return "document_drafting" |
| if any(k in r for k in ["what is","explain","define","briefly"]): return "quick_answer" |
| return "unknown_ambiguous" |
|
|
| print("BERT Router Evaluation on SWE-Router") |
| print("="*60) |
|
|
| |
| print("\n[1] Loading SWE-Router data...") |
| from datasets import load_dataset |
| traces = defaultdict(dict) |
| for model in MODELS: |
| try: |
| ds = load_dataset(f'SWE-Router/swebench-verified-{model}', split='test') |
| for row in ds: |
| iid = row['instance_id'] |
| traces[iid][model] = {'resolved': row['resolved'], 'cost': float(row['instance_cost']), |
| 'problem': row['problem_statement']} |
| print(f" {model}: loaded") |
| except Exception as e: |
| print(f" {model}: FAILED - {e}") |
| print(f" Total tasks: {len(traces)}") |
|
|
