eval/bfcl_results.json

{
  "total_tasks": 800,
  "savings_opportunity_pct": 84.125,
  "opt_tier_distribution": {
    "2": 13,
    "1": 660
  },
  "model_success_rates": {
    "BitAgent/BitAgent-8B": 0.385,
    "NousResearch/Hermes-2-Pro-Llama-3-8B": 0.02375,
    "NousResearch/Hermes-2-Pro-Mistral-7B": 0.02625,
    "Qwen/QwQ-32B-Preview": 0.0,
    "Qwen/Qwen2-1.5B-Instruct": 0.005,
    "Qwen/Qwen2-7B-Instruct": 0.0325,
    "Qwen/Qwen2.5-1.5B-Instruct": 0.01125,
    "Qwen/Qwen2.5-7B-Instruct": 0.07625,
    "THUDM/glm-4-9b-chat": 0.035,
    "Team-ACE/ToolACE-8B": 0.0775,
    "ZJared/Haha-7B": 0.10375,
    "claude-3-5-sonnet-20241022": 0.075,
    "claude-3-opus-20240229": 0.07125,
    "deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct": 0.00125,
    "gemini-1.5-flash-001": 0.195,
    "gemini-1.5-flash-002": 0.125,
    "gemini-1.5-pro-001-FC": 0.16,
    "gemini-1.5-pro-001": 0.18875,
    "gemini-1.5-pro-002-FC": 0.21625,
    "gemini-2.0-flash-001-FC": 0.17875
  },
  "tool_error_rate": 1.8579889572641257
}