| """Evaluate BERT router on SWE-Router and compare with v11 XGBoost.""" |
| import json, os, sys, random, pickle, math |
| import numpy as np |
| from collections import defaultdict |
|
|
| |
| MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2', |
| 'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash'] |
|
|
| MODEL_TIER = { |
| 'deepseek-v4-flash': 1, 'gpt-5-nano': 1, |
| 'gpt-5-mini': 2, 'deepseek-v3.2': 2, |
| 'gemini-2.5-pro': 3, |
| 'claude-opus-4.7': 4, 'gpt-5.2': 4, |
| 'gemini-3-pro': 5, |
| } |
| TIER_COST = {1:0.01, 2:0.05, 3:0.15, 4:0.30, 5:0.50} |
| TIER_TO_SWE = { |
| 1: 'deepseek-v4-flash', 2: 'gpt-5-mini', |
| 3: 'gemini-2.5-pro', 4: 'claude-opus-4.7', 5: 'gemini-3-pro', |
| } |
|
|
| |
| CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test", |
| "compile","runtime","segfault","thread","async","class","module","import","error","traceback"] |
| LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"] |
| RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"] |
| TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"] |
| CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"] |
| SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"] |
| LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"] |
| MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"] |
|
|
| def extract_features(problem_text): |
| r = problem_text.lower() |
| feats = { |
| 'req_len': len(problem_text), 'num_words': len(problem_text.split()), |
| 'has_code': int(any(k in r for k in CODE_KW)), |
| 'n_code': sum(1 for k in CODE_KW if k in r), |
| 'has_legal': int(any(k in r for k in LEGAL_KW)), |
| 'has_research': int(any(k in r for k in RESEARCH_KW)), |
| 'has_tool': int(any(k in r for k in TOOL_KW)), |
| 'has_critical': int(any(k in r for k in CRITICAL_KW)), |
| 'has_simple': int(any(k in r for k in SIMPLE_KW)), |
| 'has_long': int(any(k in r for k in LONG_KW)), |
| 'has_math': int(any(k in r for k in MATH_KW)), |
| 'has_error_msg': int('error' in r or 'traceback' in r or 'exception' in r), |
| 'has_file_path': int('/' in r), |
| 'n_lines': problem_text.count('\n') + 1, |
| 'has_version': int('version' in r or 'update' in r or 'upgrade' in r), |
| 'has_add': int('add' in r or 'new' in r or 'create' in r), |
| 'has_fix': int('fix' in r or 'bug' in r or 'issue' in r or 'broken' in r), |
| 'has_change': int('change' in r or 'modify' in r or 'update' in r), |
| 'has_remove': int('remove' in r or 'delete' in r or 'drop' in r), |
| 'has_test': int('test' in r or 'spec' in r or 'assert' in r), |
| 'has_doc': int('doc' in r or 'readme' in r or 'comment' in r), |
| } |
| return feats |
|
|
| print("BERT Router Evaluation on SWE-Router") |
| print("="*60) |
|
|
