fixed error

datasets/fixed_levels/leaderboard_fixed_levels.json

@@ -537,5 +537,263 @@
     },
     "run_id": "run_0013",
     "run_name": "fixed_levels_qwen_swarm"
+  },
+  {
+    "config": {
+      "llm_model": "gpt-5.4-mini",
+      "llm_provider": "openai",
+      "max_agents": 3,
+      "max_breadth": 2,
+      "max_depth": 2,
+      "max_steps": 24,
+      "max_width": 2,
+      "seed": 2026,
+      "seeded_questions": 30,
+      "swarm_enabled": true
+    },
+    "created_at": "2026-04-07T09:44:40+00:00",
+    "episodes": 1,
+    "metrics": {
+      "avg_compactness_reward": 0.0,
+      "avg_connectivity_gain_reward": 0.2,
+      "avg_connectivity_reward": -0.15,
+      "avg_diversity_reward": 0.12666666666666665,
+      "avg_entity_informativeness_reward": -0.018704290877944903,
+      "avg_format_reward": 0.15,
+      "avg_graph_f1": 0.5714285714285715,
+      "avg_knowledge_carrier_reward": 0.5,
+      "avg_knowledge_indexing_reward": 0.12272727272727273,
+      "avg_relation_informativeness_reward": 0.08056039127695382,
+      "avg_reward": 0.7135379106634446,
+      "avg_soft_shaping_reward": 0.3,
+      "avg_spawn_count": 4.0,
+      "avg_spawn_critical_steps": 6.0,
+      "avg_steps_to_solution": 9.0,
+      "deanonymization_accuracy": 0.0,
+      "leaderboard_score": 0.6633913226563717,
+      "retrieval_signal": 0.7179545454545455,
+      "spawn_completion_rate": 1.0,
+      "spawn_signal": 0.6666666666666666,
+      "structural_signal": 0.5138712200798018,
+      "task_success_rate": 1.0,
+      "tool_efficiency": 0.5
+    },
+    "run_id": "run_0014",
+    "run_name": "fixed_levels_qwen_swarm"
+  },
+  {
+    "config": {
+      "llm_model": "gpt-5.4-mini",
+      "llm_provider": "openai",
+      "max_agents": 3,
+      "max_breadth": 2,
+      "max_depth": 2,
+      "max_steps": 24,
+      "max_width": 2,
+      "seed": 2026,
+      "seeded_questions": 30,
+      "swarm_enabled": true
+    },
+    "created_at": "2026-04-07T09:55:19+00:00",
+    "episodes": 1,
+    "metrics": {
+      "avg_compactness_reward": 0.0,
+      "avg_connectivity_gain_reward": 0.2,
+      "avg_connectivity_reward": -0.15,
+      "avg_diversity_reward": 0.12666666666666665,
+      "avg_entity_informativeness_reward": -0.018704290877944903,
+      "avg_format_reward": 0.15,
+      "avg_graph_f1": 0.5714285714285715,
+      "avg_knowledge_carrier_reward": 0.5,
+      "avg_knowledge_indexing_reward": 0.12272727272727273,
+      "avg_relation_informativeness_reward": 0.08056039127695382,
+      "avg_reward": 0.7135379106634446,
+      "avg_soft_shaping_reward": 0.3,
+      "avg_spawn_count": 4.0,
+      "avg_spawn_critical_steps": 6.0,
+      "avg_steps_to_solution": 9.0,
+      "deanonymization_accuracy": 0.0,
+      "leaderboard_score": 0.6633913226563717,
+      "retrieval_signal": 0.7179545454545455,
+      "spawn_completion_rate": 1.0,
+      "spawn_signal": 0.6666666666666666,
+      "structural_signal": 0.5138712200798018,
+      "task_success_rate": 1.0,
+      "tool_efficiency": 0.5
+    },
+    "run_id": "run_0015",
+    "run_name": "fixed_levels_qwen_swarm"
+  },
+  {
+    "config": {
+      "llm_model": "gpt-5.4-mini",
+      "llm_provider": "openai",
+      "max_agents": 3,
+      "max_breadth": 2,
+      "max_depth": 2,
+      "max_steps": 24,
+      "max_width": 2,
+      "seed": 2026,
+      "seeded_questions": 30,
+      "swarm_enabled": true
+    },
+    "created_at": "2026-04-07T09:56:28+00:00",
+    "episodes": 30,
+    "metrics": {
+      "avg_compactness_reward": 0.0,
+      "avg_connectivity_gain_reward": 0.2000000000000001,
+      "avg_connectivity_reward": 0.12999999999999998,
+      "avg_diversity_reward": 0.12433333333333325,
+      "avg_entity_informativeness_reward": -0.02515191749984708,
+      "avg_format_reward": 0.15,
+      "avg_graph_f1": 0.2916528337385394,
+      "avg_knowledge_carrier_reward": 0.5,
+      "avg_knowledge_indexing_reward": 0.11539120363588044,
+      "avg_relation_informativeness_reward": 0.0769903534735767,
+      "avg_reward": 0.7150555461096118,
+      "avg_soft_shaping_reward": 0.3,
+      "avg_spawn_count": 4.0,
+      "avg_spawn_critical_steps": 6.0,
+      "avg_steps_to_solution": 9.0,
+      "deanonymization_accuracy": 0.0,
+      "leaderboard_score": 0.6132407715455404,
+      "retrieval_signal": 0.7153869212725582,
+      "spawn_completion_rate": 1.0,
+      "spawn_signal": 0.6666666666666666,
+      "structural_signal": 0.5815176871947458,
+      "task_success_rate": 1.0,
+      "tool_efficiency": 0.5
+    },
+    "run_id": "run_0016",
+    "run_name": "fixed_levels_qwen_swarm"
+  },
+  {
+    "config": {
+      "llm_model": "gpt-5.4-mini",
+      "llm_provider": "openai",
+      "max_agents": 3,
+      "max_breadth": 2,
+      "max_depth": 2,
+      "max_steps": 24,
+      "max_width": 2,
+      "seed": 2026,
+      "seeded_questions": 30,
+      "swarm_enabled": true
+    },
+    "created_at": "2026-04-07T10:02:32+00:00",
+    "episodes": 1,
+    "metrics": {
+      "avg_compactness_reward": 0.0,
+      "avg_connectivity_gain_reward": 0.2,
+      "avg_connectivity_reward": -0.15,
+      "avg_diversity_reward": 0.12666666666666665,
+      "avg_entity_informativeness_reward": -0.018704290877944903,
+      "avg_format_reward": 0.15,
+      "avg_graph_f1": 0.5714285714285715,
+      "avg_knowledge_carrier_reward": 0.5,
+      "avg_knowledge_indexing_reward": 0.12272727272727273,
+      "avg_relation_informativeness_reward": 0.08056039127695382,
+      "avg_reward": 0.7135379106634446,
+      "avg_soft_shaping_reward": 0.3,
+      "avg_spawn_count": 4.0,
+      "avg_spawn_critical_steps": 6.0,
+      "avg_steps_to_solution": 9.0,
+      "deanonymization_accuracy": 0.0,
+      "leaderboard_score": 0.6633913226563717,
+      "retrieval_signal": 0.7179545454545455,
+      "spawn_completion_rate": 1.0,
+      "spawn_signal": 0.6666666666666666,
+      "structural_signal": 0.5138712200798018,
+      "task_success_rate": 1.0,
+      "tool_efficiency": 0.5
+    },
+    "run_id": "run_0017",
+    "run_name": "fixed_levels_qwen_swarm"
+  },
+  {
+    "config": {
+      "llm_model": "gpt-5.4-mini",
+      "llm_provider": "openai",
+      "max_agents": 3,
+      "max_breadth": 2,
+      "max_depth": 2,
+      "max_steps": 24,
+      "max_width": 2,
+      "seed": 2026,
+      "seeded_questions": 30,
+      "swarm_enabled": true
+    },
+    "created_at": "2026-04-07T10:02:49+00:00",
+    "episodes": 3,
+    "metrics": {
+      "avg_compactness_reward": 0.0,
+      "avg_connectivity_gain_reward": 0.20000000000000004,
+      "avg_connectivity_reward": -0.06666666666666667,
+      "avg_diversity_reward": 0.13444444444444445,
+      "avg_entity_informativeness_reward": -0.029992009599206938,
+      "avg_format_reward": 0.15,
+      "avg_graph_f1": 0.5793650793650794,
+      "avg_knowledge_carrier_reward": 0.5,
+      "avg_knowledge_indexing_reward": 0.10372960372960373,
+      "avg_relation_informativeness_reward": 0.06898843512226,
+      "avg_reward": 0.7133699465240085,
+      "avg_soft_shaping_reward": 0.3,
+      "avg_spawn_count": 4.0,
+      "avg_spawn_critical_steps": 6.0,
+      "avg_steps_to_solution": 9.0,
+      "deanonymization_accuracy": 0.0,
+      "leaderboard_score": 0.6656078661080486,
+      "retrieval_signal": 0.7113053613053614,
+      "spawn_completion_rate": 1.0,
+      "spawn_signal": 0.6666666666666666,
+      "structural_signal": 0.5312992851046106,
+      "task_success_rate": 1.0,
+      "tool_efficiency": 0.5
+    },
+    "run_id": "run_0018",
+    "run_name": "fixed_levels_qwen_swarm"
+  },
+  {
+    "config": {
+      "llm_model": "gpt-5.4-mini",
+      "llm_provider": "openai",
+      "max_agents": 3,
+      "max_breadth": 2,
+      "max_depth": 2,
+      "max_steps": 24,
+      "max_width": 2,
+      "seed": 2026,
+      "seeded_questions": 30,
+      "swarm_enabled": true
+    },
+    "created_at": "2026-04-07T10:04:53+00:00",
+    "episodes": 3,
+    "metrics": {
+      "avg_compactness_reward": 0.0,
+      "avg_connectivity_gain_reward": 0.20000000000000004,
+      "avg_connectivity_reward": -0.06666666666666667,
+      "avg_diversity_reward": 0.13444444444444445,
+      "avg_entity_informativeness_reward": -0.029992009599206938,
+      "avg_format_reward": 0.15,
+      "avg_graph_f1": 0.5793650793650794,
+      "avg_knowledge_carrier_reward": 0.5,
+      "avg_knowledge_indexing_reward": 0.10372960372960373,
+      "avg_relation_informativeness_reward": 0.06898843512226,
+      "avg_reward": 0.7133699465240085,
+      "avg_soft_shaping_reward": 0.3,
+      "avg_spawn_count": 4.0,
+      "avg_spawn_critical_steps": 6.0,
+      "avg_steps_to_solution": 9.0,
+      "deanonymization_accuracy": 0.0,
+      "leaderboard_score": 0.6656078661080486,
+      "retrieval_signal": 0.7113053613053614,
+      "spawn_completion_rate": 1.0,
+      "spawn_signal": 0.6666666666666666,
+      "structural_signal": 0.5312992851046106,
+      "task_success_rate": 1.0,
+      "tool_efficiency": 0.5
+    },
+    "run_id": "run_0019",
+    "run_name": "fixed_levels_qwen_swarm"
   }
 ]

inference.py

@@ -20,12 +20,16 @@ from osint_env.viz import export_dashboard
 CONFIG_PATH = os.getenv("CONFIG_PATH", "datasets/fixed_levels/shared_config_fixed_levels.json")
 SEED_FILE = os.getenv("SEED_FILE", "datasets/fixed_levels/seed_fixed_levels.json")
 AGENT_MODE = os.getenv("AGENT_MODE", "swarm")
-LLM_PROVIDER = os.getenv("LLM_PROVIDER", "ollama")
-MODEL_NAME = os.getenv("MODEL_NAME", "qwen3:1.7b")
+LLM_PROVIDER = os.getenv("LLM_PROVIDER", "openai")
+MODEL_NAME = os.getenv("MODEL_NAME", "gpt-5.4-mini")
 OLLAMA_BASE_URL = os.getenv("OLLAMA_BASE_URL", "")
 OPENAI_BASE_URL = os.getenv("OPENAI_BASE_URL", "")
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
 OPENAI_API_KEY_ENV = os.getenv("OPENAI_API_KEY_ENV", "OPENAI_API_KEY")
+API_BASE_URL = os.getenv("API_BASE_URL", "")
+API_KEY = os.getenv("API_KEY", "")
+HF_SPACE_URL = os.getenv("HF_SPACE_URL", "")
+HF_TOKEN = os.getenv("HF_TOKEN", "")
 LLM_TIMEOUT_SECONDS = int(os.getenv("LLM_TIMEOUT_SECONDS", "0"))
 EPISODES = int(os.getenv("EPISODES", "1"))
 SUCCESS_SCORE_THRESHOLD = float(os.getenv("SUCCESS_SCORE_THRESHOLD", "0.67"))
@@ -66,6 +70,15 @@ def _normalize_ollama_base_url(url: str) -> str:
     return normalized or "http://127.0.0.1:11434"
 
 
+def _normalize_openai_base_url(url: str) -> str:
+    normalized = str(url or "").strip().rstrip("/")
+    if not normalized:
+        return ""
+    if normalized.endswith("/v1"):
+        return normalized
+    return f"{normalized}/v1"
+
+
 TASK_INDICES = _parse_task_indices(TASK_INDICES_RAW)
 
 
@@ -180,15 +193,20 @@ def _resolve_environment_config() -> EnvironmentConfig:
     if LLM_TIMEOUT_SECONDS > 0:
         env_cfg.llm.timeout_seconds = int(LLM_TIMEOUT_SECONDS)
 
-    api_base_override = os.getenv("API_BASE_URL", "")
-    if api_base_override.strip() or OLLAMA_BASE_URL.strip():
-        env_cfg.llm.ollama_base_url = _normalize_ollama_base_url(api_base_override or OLLAMA_BASE_URL)
-
-    if OPENAI_BASE_URL.strip():
-        env_cfg.llm.openai_base_url = OPENAI_BASE_URL.strip()
-
-    if OPENAI_API_KEY.strip():
-        env_cfg.llm.openai_api_key = OPENAI_API_KEY.strip()
+    if provider == "openai":
+        # Evaluation harnesses often inject API_BASE_URL/API_KEY for proxy enforcement.
+        resolved_openai_base = API_BASE_URL.strip() or OPENAI_BASE_URL.strip() or HF_SPACE_URL.strip()
+        if resolved_openai_base:
+            env_cfg.llm.openai_base_url = _normalize_openai_base_url(resolved_openai_base)
+
+        if API_KEY.strip():
+            env_cfg.llm.openai_api_key = API_KEY.strip()
+        elif OPENAI_API_KEY.strip():
+            env_cfg.llm.openai_api_key = OPENAI_API_KEY.strip()
+        elif HF_TOKEN.strip():
+            env_cfg.llm.openai_api_key = HF_TOKEN.strip()
+    elif API_BASE_URL.strip() or OLLAMA_BASE_URL.strip():
+        env_cfg.llm.ollama_base_url = _normalize_ollama_base_url(API_BASE_URL or OLLAMA_BASE_URL)
 
     if OPENAI_API_KEY_ENV.strip():
         env_cfg.llm.openai_api_key_env = OPENAI_API_KEY_ENV.strip()
@@ -387,7 +405,7 @@ def main() -> None:
         task_indices=TASK_INDICES,
     )
 
-    score = float(summary.get("task_success_rate", 0.0) or 0.0)
+    score = float(summary.get("avg_reward", 0.0) or 0.0)
     success = score >= SUCCESS_SCORE_THRESHOLD
     log_end(success=success, steps=steps_taken, score=score, rewards=rewards)