narcolepticchicken commited on
Commit
fb62d8b
·
verified ·
1 Parent(s): c5c3bd4

Upload eval/swe_v10_results.json with huggingface_hub

Browse files
Files changed (1) hide show
  1. eval/swe_v10_results.json +38 -0
eval/swe_v10_results.json ADDED
@@ -0,0 +1,38 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "v10_direct": {
3
+ "success": 0.766,
4
+ "avg_cost": 0.1878,
5
+ "costRed": 40.7
6
+ },
7
+ "v10_cascade": {
8
+ "success": 0.756,
9
+ "avg_cost": 0.1767,
10
+ "costRed": 44.2
11
+ },
12
+ "v10_feedback": {
13
+ "success": 0.848,
14
+ "avg_cost": 0.2014,
15
+ "costRed": 36.4
16
+ },
17
+ "v8_synthetic": {
18
+ "success": 0.658,
19
+ "avg_cost": 0.3534,
20
+ "costRed": -11.6
21
+ },
22
+ "frontier": {
23
+ "success": 0.782,
24
+ "avg_cost": 0.3167,
25
+ "costRed": 0.0
26
+ },
27
+ "oracle": {
28
+ "success": 0.87,
29
+ "avg_cost": 0.0624,
30
+ "costRed": 80.3
31
+ },
32
+ "always_cheap": {
33
+ "success": 0.632,
34
+ "avg_cost": 0.0142,
35
+ "costRed": 95.5
36
+ },
37
+ "key_finding": "v10 trained on REAL SWE-Router data achieves 36-44% cost reduction vs 8% for synthetic-trained v8. v10_feedback achieves HIGHER success than always-frontier (84.8% vs 78.2%) at 36.4% cost reduction."
38
+ }