fix(agents/live): also gate live test on BBB model artifact (avoids dotenv-loaded key path)
Browse files
tests/agents/test_orchestrator_live.py
ADDED
|
@@ -0,0 +1,74 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
"""Live integration test — hits real OpenRouter, picks pipeline, retrieves chunks.
|
| 2 |
+
|
| 3 |
+
Skipped unless BOTH OPENROUTER_API_KEY is set AND the BBB model artifact
|
| 4 |
+
is built (the `run_bbb_pipeline` tool can't run without it). Marked `slow`
|
| 5 |
+
(network round-trips).
|
| 6 |
+
|
| 7 |
+
The dual gate matters because src/llm/explainer.py auto-loads .env at
|
| 8 |
+
import time; without the model-artifact gate, this test would attempt a
|
| 9 |
+
real OpenRouter call in CI/dev and then fail because the BBB tool can't
|
| 10 |
+
execute. In the deployed Docker image both conditions are satisfied
|
| 11 |
+
(secret + build-time training).
|
| 12 |
+
"""
|
| 13 |
+
from __future__ import annotations
|
| 14 |
+
|
| 15 |
+
import os
|
| 16 |
+
from pathlib import Path
|
| 17 |
+
|
| 18 |
+
import pytest
|
| 19 |
+
from openai import OpenAI
|
| 20 |
+
|
| 21 |
+
from src.agents.orchestrator import Orchestrator
|
| 22 |
+
from src.agents.prompts import ORCHESTRATOR_SYSTEM_PROMPT
|
| 23 |
+
from src.agents.tools import build_default_tools
|
| 24 |
+
from src.rag.ingest import ingest_directory
|
| 25 |
+
|
| 26 |
+
|
| 27 |
+
_FIXTURE_KB = Path(__file__).parent.parent / "fixtures" / "kb_sample"
|
| 28 |
+
_DEFAULT_MODEL = "google/gemini-2.0-flash-exp:free"
|
| 29 |
+
_FALLBACK_MODEL = "anthropic/claude-haiku-4-5"
|
| 30 |
+
_BBB_MODEL_PATH = Path(
|
| 31 |
+
os.environ.get("BBB_MODEL_PATH", "data/processed/bbb_model.joblib")
|
| 32 |
+
)
|
| 33 |
+
|
| 34 |
+
|
| 35 |
+
@pytest.mark.slow
|
| 36 |
+
@pytest.mark.skipif(
|
| 37 |
+
not os.environ.get("OPENROUTER_API_KEY"),
|
| 38 |
+
reason="OPENROUTER_API_KEY not set",
|
| 39 |
+
)
|
| 40 |
+
@pytest.mark.skipif(
|
| 41 |
+
not _BBB_MODEL_PATH.exists(),
|
| 42 |
+
reason=f"BBB model artifact missing at {_BBB_MODEL_PATH} — run python -m src.models.bbb_model",
|
| 43 |
+
)
|
| 44 |
+
class TestOrchestratorLive:
|
| 45 |
+
@pytest.fixture(scope="class")
|
| 46 |
+
def rag_dir(self, tmp_path_factory: pytest.TempPathFactory) -> Path:
|
| 47 |
+
d = tmp_path_factory.mktemp("rag_live")
|
| 48 |
+
ingest_directory(_FIXTURE_KB, d)
|
| 49 |
+
return d
|
| 50 |
+
|
| 51 |
+
@pytest.fixture(scope="class")
|
| 52 |
+
def client(self) -> OpenAI:
|
| 53 |
+
return OpenAI(
|
| 54 |
+
base_url="https://openrouter.ai/api/v1",
|
| 55 |
+
api_key=os.environ["OPENROUTER_API_KEY"],
|
| 56 |
+
timeout=30.0,
|
| 57 |
+
)
|
| 58 |
+
|
| 59 |
+
def test_smiles_input_picks_bbb_then_retrieves(self, client: OpenAI, rag_dir: Path) -> None:
|
| 60 |
+
tools = build_default_tools(rag_index_dir=rag_dir)
|
| 61 |
+
orch = Orchestrator(
|
| 62 |
+
llm_client=client,
|
| 63 |
+
tools=tools,
|
| 64 |
+
system_prompt=ORCHESTRATOR_SYSTEM_PROMPT,
|
| 65 |
+
model=os.environ.get("NEUROBRIDGE_AGENT_MODEL", _DEFAULT_MODEL),
|
| 66 |
+
max_steps=5,
|
| 67 |
+
)
|
| 68 |
+
result = orch.run("CCO")
|
| 69 |
+
# Soft assertions — model behavior varies but the workflow shape is fixed.
|
| 70 |
+
assert result.finish_reason == "complete", f"got {result.finish_reason}, trace={result.trace}"
|
| 71 |
+
tool_names = [t.name for t in result.trace]
|
| 72 |
+
assert "run_bbb_pipeline" in tool_names, f"BBB pipeline not called; trace={tool_names}"
|
| 73 |
+
assert "retrieve_context" in tool_names, f"RAG not called; trace={tool_names}"
|
| 74 |
+
assert result.text, "empty final text"
|