eval/swe_bench_results.json

{
  "always_frontier": {
    "success": 0.782,
    "avg_cost": 0.3166872804999999,
    "n": 500
  },
  "always_cheap": {
    "success": 0.632,
    "avg_cost": 0.014239461958399993,
    "n": 500
  },
  "aco_v8": {
    "success": 0.756,
    "avg_cost": 0.29145885426799994,
    "n": 500
  },
  "oracle": {
    "success": 0.87,
    "avg_cost": 0.05456037742200003,
    "n": 500
  },
  "aco_v9_feedback": {
    "success": 0.826,
    "avg_cost": 0.4847524783679998,
    "n": 500
  }
}