Update dashboard from inference runs

README.md

@@ -160,6 +160,9 @@ The script is designed to stay bounded enough for a normal benchmark pass to fin
 
 The submission-ready inference entrypoint is the root `inference.py` file. It talks to the deployed Hugging Face Space over HTTP, uses the OpenAI client for all model calls, and emits structured stdout logs in the `[START]`, `[STEP]`, and `[END]` format.
 
+The script accepts `HF_TOKEN` as the primary auth variable and also supports `OPENAI_API_KEY` or `API_KEY` as local fallbacks.
+After a successful run, `inference.py` also posts the evaluation summary back to the Space so the latest `/dashboard` view reflects that run.
+
 Required environment variables:
 
 - `API_BASE_URL`

inference.py

@@ -9,11 +9,13 @@ from openai import OpenAI
 from requests import RequestException
 
 from osint_env.baselines.openai_runner import SYSTEM_PROMPT, build_action_tools
+from osint_env.eval.metrics import EvalMetrics
 
 
 API_BASE_URL = os.getenv("API_BASE_URL", "https://api.openai.com/v1")
 MODEL_NAME = os.getenv("MODEL_NAME", "gpt-5.4-mini")
 HF_TOKEN = os.getenv("HF_TOKEN", "")
+API_KEY = os.getenv("API_KEY", "")
 OPENAI_API_KEY = os.getenv("OPENAI_API_KEY", "")
 SPACE_URL = os.getenv("SPACE_URL", "https://siddeshwar1625-osint.hf.space").rstrip("/")
 
@@ -32,19 +34,18 @@ def log_start(task: str, env: str, model: str) -> None:
     print(f"[START] task={task} env={env} model={model}", flush=True)
 
 
-def log_step(step: int, action: dict[str, Any], reward: float, done: bool, error: str | None) -> None:
-    action_text = json.dumps(action, sort_keys=True, separators=(",", ":"))
-    error_text = "null" if error is None else json.dumps(error)
+def log_step(step: int, action: str, reward: float, done: bool, error: str | None) -> None:
+    error_text = "null" if error is None else str(error)
     print(
-        f"[STEP] step={step} action={action_text} reward={reward:.4f} done={str(bool(done)).lower()} error={error_text}",
+        f"[STEP] step={step} action={action} reward={reward:.2f} done={str(bool(done)).lower()} error={error_text}",
         flush=True,
     )
 
 
 def log_end(success: bool, steps: int, score: float, rewards: list[float]) -> None:
-    rewards_text = json.dumps([round(value, 4) for value in rewards], separators=(",", ":"))
+    rewards_text = ",".join(f"{value:.2f}" for value in rewards)
     print(
-        f"[END] success={str(bool(success)).lower()} steps={steps} score={score:.4f} rewards={rewards_text}",
+        f"[END] success={str(bool(success)).lower()} steps={steps} score={score:.3f} rewards={rewards_text}",
         flush=True,
     )
 
@@ -135,6 +136,28 @@ def _decode_action(tool_name: str, args: dict[str, Any]) -> dict[str, Any]:
     return {"action_type": "CALL_TOOL", "payload": {"tool_name": tool_name, "args": dict(args)}}
 
 
+def _format_action(action: dict[str, Any]) -> str:
+    action_type = str(action.get("action_type", ""))
+    payload = dict(action.get("payload", {}))
+    if action_type == "ANSWER":
+        return f"answer({payload.get('answer', 'unknown')})"
+    if action_type == "ADD_EDGE":
+        return (
+            "add_edge("
+            f"{payload.get('src', '')},"
+            f"{payload.get('rel', '')},"
+            f"{payload.get('dst', '')},"
+            f"{float(payload.get('confidence', 1.0)):.2f}"
+            ")"
+        )
+    tool_name = str(payload.get("tool_name", "tool"))
+    args = dict(payload.get("args", {}))
+    if not args:
+        return f"{tool_name}()"
+    arg_str = ",".join(f"{key}={value}" for key, value in sorted(args.items()))
+    return f"{tool_name}({arg_str})"
+
+
 def _assistant_tool_call_id(message: dict[str, Any]) -> str | None:
     tool_calls = list(message.get("tool_calls", []))
     if not tool_calls:
@@ -192,10 +215,54 @@ def get_model_action(client: OpenAI, messages: list[dict[str, Any]], tools: list
         return {"action_type": "ANSWER", "payload": {"answer": "unknown"}}, {"role": "assistant", "content": ""}
 
 
+def _episode_row(result: dict[str, Any], task_meta: dict[str, Any]) -> dict[str, Any]:
+    info = dict(result.get("info", {}))
+    graph_snapshot = dict((result.get("observation") or {}).get("graph_snapshot", {}))
+    task_type = str(task_meta.get("task_type", "unknown"))
+    task_id = str(task_meta.get("task_id", "unknown"))
+    question = str(task_meta.get("question", ""))
+    task_answer = str(info.get("task_answer", ""))
+    agent_answer = str(info.get("agent_answer", ""))
+    graph_f1 = float(info.get("graph_f1", 0.0) or 0.0)
+    return {
+        "task_id": task_id,
+        "task_type": task_type,
+        "question": question,
+        "task_answer": task_answer,
+        "agent_answer": agent_answer,
+        "graph_f1": graph_f1,
+        "reward": float(info.get("total_reward", 0.0) or 0.0),
+        "steps": int(info.get("step_count", 0) or 0),
+        "tool_calls": int(info.get("tool_calls", 0) or 0),
+        "success": int(bool(agent_answer) and agent_answer == task_answer),
+        "reward_components": dict(info.get("reward_components", {})),
+        "pred_edges": list(graph_snapshot.get("edges", [])),
+        "truth_edges": [],
+    }
+
+
+def _publish_inference_report(summary: dict[str, Any], episodes: list[dict[str, Any]]) -> None:
+    payload = {
+        "run": {
+            "name": "inference_py_run",
+            "model": MODEL_NAME,
+            "space_url": SPACE_URL,
+            "task_indices": TASK_INDICES,
+            "max_steps": MAX_STEPS,
+        },
+        "summary": summary,
+        "episodes": episodes,
+    }
+    try:
+        _space_post("/openenv/report_inference", payload)
+    except RequestException as exc:
+        print(f"[DEBUG] Failed to publish inference report: {exc}", flush=True)
+
+
 def main() -> None:
-    api_key = OPENAI_API_KEY or HF_TOKEN
+    api_key = OPENAI_API_KEY or HF_TOKEN or API_KEY
     if not api_key:
-        raise SystemExit("Set OPENAI_API_KEY or HF_TOKEN before running inference.py.")
+        raise SystemExit("Set HF_TOKEN, OPENAI_API_KEY, or API_KEY before running inference.py.")
     if _looks_like_placeholder_api_key(api_key):
         raise SystemExit("Replace the placeholder with your real OpenAI API key.")
 
@@ -212,6 +279,8 @@ def main() -> None:
     history: list[str] = []
     rewards: list[float] = []
     task_scores: list[float] = []
+    episode_rows: list[dict[str, Any]] = []
+    metrics = EvalMetrics()
     steps_taken = 0
 
     log_start(task=TASK_NAME, env=BENCHMARK, model=MODEL_NAME)
@@ -220,6 +289,7 @@ def main() -> None:
         result = _space_post("/openenv/reset", {"task_index": task_index})
         session_id = str(result["session_id"])
         done = bool(result.get("done", False))
+        task_meta = dict((result.get("observation") or {}).get("task", {}))
         messages: list[dict[str, Any]] = [
             {"role": "system", "content": SYSTEM_PROMPT},
             {
@@ -249,7 +319,7 @@ def main() -> None:
             done = bool(result.get("done", False))
             rewards.append(reward)
             steps_taken += 1
-            log_step(step=steps_taken, action=action, reward=reward, done=done, error=error)
+            log_step(step=steps_taken, action=_format_action(action), reward=reward, done=done, error=error)
             history.append(f"step={steps_taken} task_index={task_index} reward={reward:+.4f}")
             messages.append(assistant_message)
             tool_message = _tool_result_message(assistant_message, result)
@@ -262,10 +332,14 @@ def main() -> None:
         task_answer = str(info.get("task_answer", ""))
         agent_answer = str(info.get("agent_answer", ""))
         task_scores.append(1.0 if agent_answer and agent_answer == task_answer else 0.0)
+        episode_row = _episode_row(result, task_meta)
+        episode_rows.append(episode_row)
+        metrics.add(info, task_type=episode_row["task_type"], graph_f1=float(episode_row["graph_f1"]))
 
     score = sum(task_scores) / max(1, len(task_scores))
     success = score >= SUCCESS_SCORE_THRESHOLD
     log_end(success=success, steps=steps_taken, score=score, rewards=rewards)
+    _publish_inference_report(metrics.summary(), episode_rows)
 
 
 if __name__ == "__main__":

server.py

@@ -14,6 +14,8 @@ from fastapi.responses import FileResponse, HTMLResponse, JSONResponse
 
 from osint_env.api import (
     OpenEnvActionRequest,
+    OpenEnvInferenceReportRequest,
+    OpenEnvInferenceReportResponse,
     OpenEnvObservationModel,
     OpenEnvResetRequest,
     OpenEnvResponseEnvelope,
@@ -22,6 +24,7 @@ from osint_env.api import (
 from osint_env.config import clone_environment_config, load_seeding_config, load_shared_config
 from osint_env.domain.models import Action, ActionType
 from osint_env.env.environment import OSINTEnvironment
+from osint_env.eval.leaderboard import load_leaderboard
 from osint_env.eval.runner import run_evaluation
 from osint_env.llm import build_llm_client
 from osint_env.viz import export_dashboard
@@ -134,6 +137,43 @@ def _store_session(session_id: str, env: OSINTEnvironment) -> None:
         _SESSIONS[session_id] = env
 
 
+def _task_lookup(env: OSINTEnvironment) -> dict[str, Any]:
+    return {task.task_id: task for task in env.tasks}
+
+
+def _normalize_episode_rows(env: OSINTEnvironment, episodes: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    tasks_by_id = _task_lookup(env)
+    normalized: list[dict[str, Any]] = []
+    for episode in episodes:
+        row = dict(episode)
+        task = tasks_by_id.get(str(row.get("task_id", "")))
+        if task is not None:
+            row.setdefault("task_type", task.task_type)
+            row.setdefault("question", task.question)
+            row.setdefault("task_answer", task.answer)
+            row.setdefault(
+                "truth_edges",
+                [
+                    {
+                        "src": edge.src,
+                        "rel": edge.rel,
+                        "dst": edge.dst,
+                        "confidence": float(edge.confidence),
+                    }
+                    for edge in task.supporting_edges
+                ],
+            )
+        row.setdefault("pred_edges", [])
+        row.setdefault("reward_components", {})
+        row.setdefault("graph_f1", 0.0)
+        row.setdefault("reward", 0.0)
+        row.setdefault("steps", 0)
+        row.setdefault("tool_calls", 0)
+        row.setdefault("success", 0)
+        normalized.append(row)
+    return normalized
+
+
 @lru_cache(maxsize=1)
 def _base_environment_snapshot() -> dict[str, Any]:
     env = _build_environment()
@@ -409,6 +449,30 @@ def openenv_state(session_id: str) -> OpenEnvResponseEnvelope:
     )
 
 
+@app.post("/openenv/report_inference", response_model=OpenEnvInferenceReportResponse)
+def openenv_report_inference(request: OpenEnvInferenceReportRequest) -> OpenEnvInferenceReportResponse:
+    env = _build_environment()
+    normalized_episodes = _normalize_episode_rows(env, list(request.episodes))
+    payload = {
+        "run": dict(request.run),
+        "summary": dict(request.summary),
+        "episodes": normalized_episodes,
+    }
+    LATEST_EVALUATION_OUTPUT.parent.mkdir(parents=True, exist_ok=True)
+    LATEST_EVALUATION_OUTPUT.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
+    dashboard_path = export_dashboard(
+        env=env,
+        evaluation=payload,
+        leaderboard_records=load_leaderboard("artifacts/baselines/openai_fixed_levels_leaderboard.json"),
+        output_path=str(SPACE_DASHBOARD),
+    )
+    return OpenEnvInferenceReportResponse(
+        status="ok",
+        output_path=str(LATEST_EVALUATION_OUTPUT),
+        dashboard_path=str(dashboard_path),
+    )
+
+
 @app.get("/dashboard")
 def dashboard() -> FileResponse:
     snapshot = _space_snapshot()

src/osint_env/api/__init__.py

@@ -1,5 +1,7 @@
 from osint_env.api.models import (
     OpenEnvActionRequest,
+    OpenEnvInferenceReportRequest,
+    OpenEnvInferenceReportResponse,
     OpenEnvObservationModel,
     OpenEnvResetRequest,
     OpenEnvResponseEnvelope,
@@ -8,6 +10,8 @@ from osint_env.api.models import (
 
 __all__ = [
     "OpenEnvActionRequest",
+    "OpenEnvInferenceReportRequest",
+    "OpenEnvInferenceReportResponse",
     "OpenEnvObservationModel",
     "OpenEnvResetRequest",
     "OpenEnvResponseEnvelope",

src/osint_env/api/models.py

@@ -36,3 +36,15 @@ class OpenEnvResponseEnvelope(BaseModel):
     reward: float
     done: bool
     info: dict[str, Any]
+
+
+class OpenEnvInferenceReportRequest(BaseModel):
+    run: dict[str, Any] = Field(default_factory=dict)
+    summary: dict[str, Any]
+    episodes: list[dict[str, Any]] = Field(default_factory=list)
+
+
+class OpenEnvInferenceReportResponse(BaseModel):
+    status: str
+    output_path: str
+    dashboard_path: str

tests/test_server.py

@@ -64,6 +64,43 @@ def test_openenv_reset_step_and_state_cycle():
     assert "task_answer" in step_body["info"]
 
 
+def test_report_inference_updates_latest_evaluation_and_dashboard(tmp_path, monkeypatch):
+    latest_evaluation = tmp_path / "latest_evaluation.json"
+    space_dashboard = tmp_path / "space_dashboard.html"
+
+    monkeypatch.setattr(server, "LATEST_EVALUATION_OUTPUT", latest_evaluation)
+    monkeypatch.setattr(server, "SPACE_DASHBOARD", space_dashboard)
+    monkeypatch.setattr(server, "load_leaderboard", lambda path: [])
+    monkeypatch.setattr(server, "export_dashboard", lambda env, evaluation, leaderboard_records, output_path: str(space_dashboard))
+
+    response = client.post(
+        "/openenv/report_inference",
+        json={
+            "run": {"name": "inference_py_run"},
+            "summary": {"leaderboard_score": 0.75, "task_success_rate": 1.0},
+            "episodes": [
+                {
+                    "task_id": "seed_task_0",
+                    "agent_answer": "user_bharat",
+                    "graph_f1": 0.5,
+                    "reward": 1.2,
+                    "steps": 5,
+                    "tool_calls": 4,
+                    "success": 1,
+                }
+            ],
+        },
+    )
+    assert response.status_code == 200
+    body = response.json()
+    assert body["status"] == "ok"
+    assert latest_evaluation.exists()
+    stored = json.loads(latest_evaluation.read_text(encoding="utf-8"))
+    assert stored["summary"]["leaderboard_score"] == 0.75
+    assert stored["episodes"][0]["task_id"] == "seed_task_0"
+    assert stored["episodes"][0]["truth_edges"]
+
+
 def test_space_snapshot_prefers_newer_evaluation_payload(tmp_path, monkeypatch):
     baseline_path = tmp_path / "baseline.json"
     evaluation_path = tmp_path / "evaluation.json"