Upload training/aco_eval.py with huggingface_hub

training/aco_eval.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""ACO Benchmark Evaluation: Full system test with simulated agent traces."""
+import sys,json,random,pickle,time
+sys.path.insert(0,"/app")
+from collections import defaultdict
+
+TIER_STR={1:0.35,2:0.55,3:0.80,4:0.93,5:0.97}
+TIER_COST={1:0.05,2:0.15,3:0.75,4:1.0,5:1.5}
+TASK_FLOOR={"legal_regulated":4,"long_horizon":3,"research":3,"coding":3,
+             "unknown_ambiguous":3,"quick_answer":1,"document_drafting":2,
+             "tool_heavy":2,"retrieval_heavy":2}
+
+CODE_KW=["python","javascript","code","function","bug","debug","refactor","implement","test"]
+CRITICAL_KW=["critical","production","urgent","now","emergency","live","deployed","safety","security"]
+SIMPLE_KW=["typo","simple","quick","brief","briefly","just","minor","small","easy","trivial","clarification"]
+
+from aco.classifier import TaskCostClassifier
+from aco.router import ModelCascadeRouter
+from aco.context_budgeter import ContextBudgeter
+from aco.tool_gate import ToolCostGate
+from aco.verifier_budgeter import VerifierBudgeter
+from aco.retry_optimizer import RetryOptimizer
+from aco.meta_tool_miner import MetaToolMiner
+from aco.doom_detector import DoomDetector
+
+TASKS={
+  "quick_answer":["What is 2+2?","Explain quantum computing briefly.","Just tell me what 2+2 is."],
+  "coding":["Write a Python function to reverse a linked list.","Fix a typo in the README.","Debug this critical production segfault NOW.","Just fix the typo in line 42."],
+  "research":["Research latest transformer advances.","Find sources comparing LoRA and full FT briefly."],
+  "document_drafting":["Draft project proposal for ML pipeline.","Write email to team about deployment."],
+  "legal_regulated":["Review this contract for liability clauses.","Check GDPR compliance for data pipeline urgently."],
+  "tool_heavy":["Search open issues and create summary.","Fetch API docs and generate client code."],
+  "retrieval_heavy":["Answer based on 50-page document.","Find all payment processing mentions."],
+  "long_horizon":["Plan 3-month roadmap.","Orchestrate complete multi-region deployment."],
+  "unknown_ambiguous":["Help me with this thing.","I need something about the server."],
+}
+
+TOOL_LIST=["web_search","code_search","file_read","file_write","code_execute","verify"]
+TOOL_COST_ESTIMATES={"web_search":{"cost":0.01},"code_search":{"cost":0.005},"file_read":{"cost":0.001},"file_write":{"cost":0.001},"code_execute":{"cost":0.01},"verify":{"cost":0.02}}
+VERIFIER_COST=0.02
+
+print("="*80)
+print("ACO FULL SYSTEM BENCHMARK EVALUATION")
+print("="*80)
+
+# Initialize modules
+classifier=TaskCostClassifier()
+router=ModelCascadeRouter(model_path="/app/router_models/router_bundle_v8.pkl")
+context_budgeter=ContextBudgeter()
+tool_gate=ToolCostGate()
+verifier_budgeter=VerifierBudgeter()
+retry_optimizer=RetryOptimizer()
+meta_tool_miner=MetaToolMiner()
+doom_detector=DoomDetector()
+
+# Simulate 2000 agent runs
+rng=random.Random(42)
+N=2000
+results_aco=[]
+results_frontier=[]
+results_heuristic=[]
+results_cheap=[]
+
+for i in range(N):
+    tt=rng.choice(list(TASKS.keys()))
+    req=rng.choice(TASKS[tt])
+    
+    # Classify
+    pred=classifier.classify(req)
+    # Route
+    routing=router.route(req, pred["task_type"], pred["difficulty"], pred)
+    # Context budget
+    budget=context_budgeter.budget(pred["task_type"],pred["difficulty"],pred["needs_retrieval"],pred["needs_tools"])
+    # Tool decisions
+    tool_decisions={}
+    for tool in TOOL_LIST:
+        if pred["needs_tools"] or tt in ("coding","tool_heavy","retrieval_heavy","research"):
+            td=tool_gate.gate(tool,{"query":req},tt,1,5,routing.confidence)
+            tool_decisions[tool]=td
+    # Verifier
+    vd=verifier_budgeter.should_verify(tt,pred["risk"],routing.confidence,False,False,routing.tier)
+    # Simulate success
+    ps=TIER_STR[routing.tier]**(pred["difficulty"]*0.6)
+    success=rng.random()<ps
+    # Compute cost
+    model_cost=TIER_COST[routing.tier]
+    tool_cost=sum(TOOL_COST_ESTIMATES.get(t,{}).get("cost",0.02) for t,td in tool_decisions.items() if td.action=="use")
+    ver_cost=VERIFIER_COST if vd.should_verify else 0
+    total_cost=model_cost+tool_cost+ver_cost
+    
+    results_aco.append({"tt":tt,"tier":routing.tier,"success":success,"cost":total_cost,
+                       "model_cost":model_cost,"tool_cost":tool_cost,"ver_cost":ver_cost,
+                       "context_tokens":budget.total_tokens,"verified":vd.should_verify,
+                       "tools_used":sum(1 for td in tool_decisions.values() if td.action=="use"),
+                       "escalated":routing.escalated,"downgraded":routing.downgraded})
+    
+    # Baseline: always frontier
+    ps_f=TIER_STR[4]**(pred["difficulty"]*0.6)
+    s_f=rng.random()<ps_f
+    results_frontier.append({"tt":tt,"tier":4,"success":s_f,"cost":1.0+tool_cost+VERIFIER_COST})
+    
+    # Baseline: heuristic
+    h_tier=min(pred["difficulty"]+1,5)
+    h_tier=max(h_tier,TASK_FLOOR.get(tt,2))
+    ps_h=TIER_STR[h_tier]**(pred["difficulty"]*0.6)
+    s_h=rng.random()<ps_h
+    results_heuristic.append({"tt":tt,"tier":h_tier,"success":s_h,"cost":TIER_COST[h_tier]+tool_cost+ver_cost})
+    
+    # Baseline: always cheap
+    ps_c=TIER_STR[1]**(pred["difficulty"]*0.6)
+    s_c=rng.random()<ps_c
+    results_cheap.append({"tt":tt,"tier":1,"success":s_c,"cost":0.05+tool_cost})
+    
+    verifier_budgeter.reset_run()
+
+
+# Compute metrics
+def compute_metrics(results, name):
+    n=len(results)
+    succ=sum(1 for r in results if r["success"])
+    cost=sum(r["cost"] for r in results)
+    model_cost=sum(r.get("model_cost",r["cost"]) for r in results)
+    tool_cost=sum(r.get("tool_cost",0) for r in results)
+    ver_cost=sum(r.get("ver_cost",0) for r in results)
+    ctx=sum(r.get("context_tokens",8000) for r in results)/n
+    verified=sum(1 for r in results if r.get("verified",True))
+    tools=sum(r.get("tools_used",0) for r in results)/n
+    escalations=sum(1 for r in results if r.get("escalated",False))
+    downgrades=sum(1 for r in results if r.get("downgraded",False))
+    return {"name":name,"success_rate":succ/n,"avg_cost":cost/n,
+            "model_cost":model_cost/n,"tool_cost":tool_cost/n,"ver_cost":ver_cost/n,
+            "avg_context_tokens":ctx,"verifications":verified,
+            "avg_tools":tools,"escalations":escalations,"downgrades":downgrades}
+
+m=compute_metrics(results_aco,"aco_v8")
+m_f=compute_metrics(results_frontier,"always_frontier")
+m_h=compute_metrics(results_heuristic,"heuristic")
+m_c=compute_metrics(results_cheap,"always_cheap")
+
+print(f"\n{'Router':<20} {'Success':>10} {'AvgCost':>10} {'CostRed':>10} {'ModelCost':>10} {'ToolCost':>10} {'VerCost':>10} {'Context':>10} {'Verifs':>8}")
+print("-"*100)
+for r in [m_f,m_h,m,m_c]:
+    cr=(1-r["avg_cost"]/m_f["avg_cost"])*100
+    print(f"{r['name']:<20} {r['success_rate']:>10.3f} {r['avg_cost']:>10.4f} {cr:>9.1f}% {r['model_cost']:>10.4f} {r['tool_cost']:>10.4f} {r['ver_cost']:>10.4f} {r['avg_context_tokens']:>10.0f} {r['verifications']:>8d}")
+
+# Per-task breakdown
+print(f"\n\nPer-task breakdown:")
+for tt in sorted(set(r["tt"] for r in results_aco)):
+    aco_tt=[r for r in results_aco if r["tt"]==tt]
+    front_tt=[r for r in results_frontier if r["tt"]==tt]
+    n_tt=len(aco_tt)
+    a_s=sum(1 for r in aco_tt if r["success"])/n_tt
+    a_c=sum(r["cost"] for r in aco_tt)/n_tt
+    f_c=sum(r["cost"] for r in front_tt)/n_tt
+    f_s=sum(1 for r in front_tt if r["success"])/n_tt
+    cr=(1-a_c/f_c)*100
+    print(f"  {tt:<20} n={n_tt:>4} aco_success={a_s:.3f} frontier_success={f_s:.3f} aco_cost={a_c:.4f} costRed={cr:.1f}%")
+
+# Cost-quality frontier
+print(f"\n\nCost-Quality Frontier:")
+frontier_points=[]
+for r in [m_c,m_h,m,m_f]:
+    frontier_points.append((r["avg_cost"],r["success_rate"],r["name"]))
+frontier_points.sort(key=lambda x:x[0])
+for cost,succ,name in frontier_points:
+    print(f"  {name:<20} cost={cost:.4f} success={succ:.3f}")
+
+# Key findings
+print(f"\n\nKEY FINDINGS:")
+print(f"  ACO v8 success rate: {m['success_rate']:.3f}")
+print(f"  ACO v8 cost reduction: {(1-m['avg_cost']/m_f['avg_cost'])*100:.1f}%")
+print(f"  ACO v8 avg context: {m['avg_context_tokens']:.0f} tokens")
+print(f"  ACO v8 verifications: {m['verifications']}/{N}")
+print(f"  Escalations: {m['escalations']} ({m['escalations']/N*100:.1f}%)")
+print(f"  Downgrades: {m['downgrades']} ({m['downgrades']/N*100:.1f}%)")
+
+# Save
+with open("/app/aco_benchmark_results.json","w") as f:
+    json.dump({"aco_v8":m,"frontier":m_f,"heuristic":m_h,"cheap":m_c},f,indent=2)
+print(f"\nSaved to /app/aco_benchmark_results.json")
+print("DONE!")