Add run_eval.py and virtual_api_server.py

pipeline/run_eval.py

@@ -0,0 +1,101 @@
+"""Main evaluation runner for P2P StableToolBench experiments.
+
+Orchestrates: load queries -> load P2P data -> run ReAct inference -> save results.
+
+Usage:
+    python run_eval.py --condition baseline --groups G1_instruction --max_queries 5
+    python run_eval.py --condition all --groups all
+    python run_eval.py --condition p2p_desc --p2p_desc_dir /path/to/your/descriptions
+"""
+import os, sys, json, time, argparse
+from typing import Dict, List, Any, Optional
+from config import (TOOL_ROOT_DIR, SOLVABLE_QUERIES_DIR, OUTPUT_DIR, P2P_DESCRIPTIONS_DIR, P2P_EXAMPLES_DIR, TASK_MODEL, TEMPERATURE, ALL_GROUPS, CONDITION_NAMES, API_SERVER_URL, API_SERVER_PORT)
+from tool_utils import load_query_data, load_p2p_descriptions, load_p2p_examples
+from prompt_builder import build_initial_messages, get_condition_config, gather_examples_for_query
+from llm_client import LLMClient
+from react_loop import ReActRunner
+
+
+def run_condition(condition, group, llm, max_queries=None, output_dir=OUTPUT_DIR, p2p_descriptions=None, p2p_examples=None, service_url=None):
+    config = get_condition_config(condition, p2p_descriptions, p2p_examples)
+    query_path = os.path.join(SOLVABLE_QUERIES_DIR, f"{group}.json")
+    if not os.path.exists(query_path):
+        print(f"Query file not found: {query_path}"); return {}
+    custom_descs = config["custom_descriptions"] if config["use_custom_descriptions"] else None
+    queries = load_query_data(query_path, TOOL_ROOT_DIR, custom_descriptions=custom_descs)
+    if max_queries: queries = queries[:max_queries]
+    print(f"\n{'='*60}\nCondition: {condition} | Group: {group} | Queries: {len(queries)}\n{'='*60}")
+    condition_dir = os.path.join(output_dir, condition, group)
+    os.makedirs(condition_dir, exist_ok=True)
+    results = []
+    for i, query_data in enumerate(queries):
+        query_id = query_data["query_id"]
+        output_file = os.path.join(condition_dir, f"{query_id}_CoT@1.json")
+        if os.path.exists(output_file):
+            print(f"  [{i+1}/{len(queries)}] Query {query_id}: already done, skipping")
+            with open(output_file) as f: result = json.load(f)
+            results.append(result); continue
+        print(f"  [{i+1}/{len(queries)}] Query {query_id}: {query_data['query'][:80]}...")
+        examples = None
+        if config["use_examples"] and config["examples"]:
+            examples = gather_examples_for_query(query_data["tool_names"], query_data["api_name_reflect"], query_data["functions"], config["examples"], max_per_tool=1)
+        messages = build_initial_messages(query=query_data["query"], tool_descriptions=query_data["tool_descriptions"], examples=examples)
+        runner = ReActRunner(llm=llm, functions=query_data["functions"], tool_descriptions=query_data["tool_descriptions"], api_name_reflect=query_data["api_name_reflect"], tool_names=query_data["tool_names"], cate_names=query_data["cate_names"], service_url=service_url)
+        result = runner.run(messages)
+        result["query"], result["query_id"], result["condition"], result["group"] = query_data["query"], query_id, condition, group
+        with open(output_file, "w") as f:
+            save_data = {"query_id": query_id, "query": query_data["query"], "condition": condition, "group": group, "success": result["success"], "final_answer": result["final_answer"], "give_up": result["give_up"], "total_tokens": result["total_tokens"], "query_count": result["query_count"], "steps": result["steps"], "trajectory": result["trajectory"], "available_tools": [f_["function"]["name"] for f_ in query_data["functions"]]}
+            json.dump(save_data, f, indent=2)
+        results.append(result)
+        status = "solved" if result["success"] else ("gave up" if result["give_up"] else "failed")
+        print(f"    {status} | steps={result['steps']} | tokens={result['total_tokens']}")
+    total = len(results)
+    solved = sum(1 for r in results if r.get("success", False))
+    gave_up = sum(1 for r in results if r.get("give_up", False))
+    summary = {"condition": condition, "group": group, "total_queries": total, "solved": solved, "gave_up": gave_up, "failed": total - solved - gave_up, "pass_rate": solved / total * 100 if total > 0 else 0}
+    with open(os.path.join(condition_dir, "summary.json"), "w") as f: json.dump(summary, f, indent=2)
+    print(f"\n  Summary: {solved}/{total} solved ({summary['pass_rate']:.1f}% pass rate)")
+    return summary
+
+
+def main():
+    parser = argparse.ArgumentParser(description="P2P StableToolBench Evaluation")
+    parser.add_argument("--condition", type=str, default="baseline", choices=CONDITION_NAMES + ["all"])
+    parser.add_argument("--groups", type=str, nargs="+", default=["G1_instruction"])
+    parser.add_argument("--max_queries", type=int, default=None)
+    parser.add_argument("--model", type=str, default=TASK_MODEL)
+    parser.add_argument("--vllm_url", type=str, default="http://localhost:8000/v1")
+    parser.add_argument("--api_server_url", type=str, default=None)
+    parser.add_argument("--temperature", type=float, default=TEMPERATURE)
+    parser.add_argument("--output_dir", type=str, default=OUTPUT_DIR)
+    parser.add_argument("--p2p_desc_dir", type=str, default=P2P_DESCRIPTIONS_DIR)
+    parser.add_argument("--p2p_examples_dir", type=str, default=P2P_EXAMPLES_DIR)
+    args = parser.parse_args()
+    groups = ALL_GROUPS if "all" in args.groups else args.groups
+    conditions = CONDITION_NAMES if args.condition == "all" else [args.condition]
+    service_url = args.api_server_url or f"{API_SERVER_URL}:{API_SERVER_PORT}/virtual"
+    print(f"Connecting to vLLM at {args.vllm_url}\nModel: {args.model}")
+    llm = LLMClient(model=args.model, base_url=args.vllm_url, temperature=args.temperature)
+    p2p_descriptions, p2p_examples = None, None
+    if any(c in conditions for c in ["p2p_desc", "p2p_full"]):
+        p2p_descriptions = load_p2p_descriptions(args.p2p_desc_dir)
+        print(f"  Loaded {len(p2p_descriptions)} descriptions")
+    if any(c in conditions for c in ["p2p_demo", "p2p_full"]):
+        p2p_examples = load_p2p_examples(args.p2p_examples_dir)
+        print(f"  Loaded examples for {len(p2p_examples)} tools")
+    all_summaries = []
+    for condition in conditions:
+        for group in groups:
+            summary = run_condition(condition=condition, group=group, llm=llm, max_queries=args.max_queries, output_dir=args.output_dir, p2p_descriptions=p2p_descriptions, p2p_examples=p2p_examples, service_url=service_url)
+            all_summaries.append(summary)
+    print("\n" + "="*80 + "\nFINAL RESULTS\n" + "="*80)
+    print(f"{'Condition':<15} {'Group':<20} {'Solved':>8} {'Total':>8} {'Pass Rate':>10}")
+    print("-"*65)
+    for s in all_summaries:
+        if s: print(f"{s['condition']:<15} {s['group']:<20} {s['solved']:>8} {s['total_queries']:>8} {s['pass_rate']:>9.1f}%")
+    os.makedirs(args.output_dir, exist_ok=True)
+    with open(os.path.join(args.output_dir, "all_summaries.json"), "w") as f: json.dump(all_summaries, f, indent=2)
+    print(f"\nResults saved to {args.output_dir}")
+
+if __name__ == "__main__":
+    main()