File size: 3,052 Bytes
ef25c39 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 | """Evaluate BERT router on SWE-Router and compare with v11 XGBoost."""
import json, os, sys, random, pickle, math
import numpy as np
from collections import defaultdict
# ── Constants ──
MODELS = ['claude-opus-4.7','gpt-5-mini','gpt-5-nano','gpt-5.2',
'gemini-2.5-pro','gemini-3-pro','deepseek-v3.2','deepseek-v4-flash']
MODEL_TIER = {
'deepseek-v4-flash': 1, 'gpt-5-nano': 1,
'gpt-5-mini': 2, 'deepseek-v3.2': 2,
'gemini-2.5-pro': 3,
'claude-opus-4.7': 4, 'gpt-5.2': 4,
'gemini-3-pro': 5,
}
TIER_COST = {1:0.01, 2:0.05, 3:0.15, 4:0.30, 5:0.50}
TIER_TO_SWE = {
1: 'deepseek-v4-flash', 2: 'gpt-5-mini',
3: 'gemini-2.5-pro', 4: 'claude-opus-4.7', 5: 'gemini-3-pro',
}
# ── Feature extraction (same as training) ──
CODE_KW = ["python","javascript","code","function","bug","debug","refactor","implement","test",
"compile","runtime","segfault","thread","async","class","module","import","error","traceback"]
LEGAL_KW = ["contract","legal","compliance","gdpr","privacy","policy","regulatory","liability"]
RESEARCH_KW = ["research","investigate","compare","analyze","survey","paper"]
TOOL_KW = ["search","fetch","retrieve","query","api","database","scrape","aggregate"]
CRITICAL_KW = ["critical","production","urgent","emergency","live","deployed","safety","security"]
SIMPLE_KW = ["typo","simple","quick","brief","minor","small","easy","trivial","just"]
LONG_KW = ["plan","project","roadmap","orchestrate","migrate","pipeline","deploy","architecture"]
MATH_KW = ["calculate","compute","solve","equation","formula","optimize","probability"]
def extract_features(problem_text):
r = problem_text.lower()
feats = {
'req_len': len(problem_text), 'num_words': len(problem_text.split()),
'has_code': int(any(k in r for k in CODE_KW)),
'n_code': sum(1 for k in CODE_KW if k in r),
'has_legal': int(any(k in r for k in LEGAL_KW)),
'has_research': int(any(k in r for k in RESEARCH_KW)),
'has_tool': int(any(k in r for k in TOOL_KW)),
'has_critical': int(any(k in r for k in CRITICAL_KW)),
'has_simple': int(any(k in r for k in SIMPLE_KW)),
'has_long': int(any(k in r for k in LONG_KW)),
'has_math': int(any(k in r for k in MATH_KW)),
'has_error_msg': int('error' in r or 'traceback' in r or 'exception' in r),
'has_file_path': int('/' in r),
'n_lines': problem_text.count('\n') + 1,
'has_version': int('version' in r or 'update' in r or 'upgrade' in r),
'has_add': int('add' in r or 'new' in r or 'create' in r),
'has_fix': int('fix' in r or 'bug' in r or 'issue' in r or 'broken' in r),
'has_change': int('change' in r or 'modify' in r or 'update' in r),
'has_remove': int('remove' in r or 'delete' in r or 'drop' in r),
'has_test': int('test' in r or 'spec' in r or 'assert' in r),
'has_doc': int('doc' in r or 'readme' in r or 'comment' in r),
}
return feats
print("BERT Router Evaluation on SWE-Router")
print("="*60)
|