tests/unit/test_run_eval.py

from __future__ import annotations

import asyncio
import json
import logging
import time
from pathlib import Path
from types import SimpleNamespace
import types

import pytest
import httpx

from civicsetu.models.enums import DocType, Jurisdiction
from civicsetu.models.schemas import LegalChunk, RetrievedChunk


def _load_run_eval_module():
    """Return civicsetu.evaluation.ragas_eval, reloaded so env-var constants are fresh."""
    import importlib
    import civicsetu.evaluation.ragas_eval as m
    importlib.reload(m)
    return m


class _SlowMetric:
    def __init__(self, *args, **kwargs):
        pass

    async def ascore(self, **kwargs):
        await asyncio.sleep(0.05)
        return SimpleNamespace(value=0.9)

    def batch_score(self, inputs):
        time.sleep(0.05)
        return [SimpleNamespace(value=0.9) for _ in inputs]


class _FailingMetric:
    def __init__(self, *args, **kwargs):
        pass

    def batch_score(self, inputs):
        raise RuntimeError("judge unavailable")


class _CaptureTransport(httpx.AsyncBaseTransport):
    def __init__(self):
        self.body = None

    async def handle_async_request(self, request: httpx.Request) -> httpx.Response:
        self.body = json.loads(request.content)
        return httpx.Response(200, request=request)


def test_no_reasoning_transport_adds_flag_without_request_copy():
    run_eval = _load_run_eval_module()
    wrapped = _CaptureTransport()
    transport = run_eval._NoReasoningTransport(wrapped)
    request = httpx.Request(
        "POST",
        "https://example.test/v1/chat/completions",
        headers={"content-type": "application/json"},
        content=json.dumps({"model": "x"}).encode(),
    )

    asyncio.run(transport.handle_async_request(request))

    assert wrapped.body == {"model": "x", "no_reasoning": True}


def test_disable_thinking_transport_injects_flags():
    run_eval = _load_run_eval_module()
    wrapped = _CaptureTransport()
    transport = run_eval._DisableThinkingTransport(wrapped)
    request = httpx.Request(
        "POST",
        "https://integrate.api.nvidia.com/v1/chat/completions",
        headers={"content-type": "application/json"},
        content=json.dumps({"model": "z-ai/glm4.7", "stream": True}).encode(),
    )

    asyncio.run(transport.handle_async_request(request))

    assert wrapped.body["chat_template_kwargs"]["enable_thinking"] is False
    assert wrapped.body["chat_template_kwargs"]["clear_thinking"] is False
    assert wrapped.body["stream"] is False
    assert wrapped.body["model"] == "z-ai/glm4.7"


def test_score_batch_runs_without_timeout(monkeypatch):
    """score_batch has no timeout — slow metrics run to completion."""
    run_eval = _load_run_eval_module()

    assert not hasattr(run_eval, "METRIC_TIMEOUT_SEC"), (
        "METRIC_TIMEOUT_SEC should not exist — timeout was removed to allow unbounded RAGAS scoring"
    )


def test_score_batch_returns_failed_rows_on_metric_error(monkeypatch):
    run_eval = _load_run_eval_module()

    import ragas.metrics.collections as collections

    monkeypatch.setattr(collections, "Faithfulness", _FailingMetric)
    monkeypatch.setattr(collections, "AnswerRelevancy", _FailingMetric)
    monkeypatch.setattr(collections, "ContextPrecision", _FailingMetric)

    rows = [
        {
            "id": "CASE-001",
            "query": "What is section 3?",
            "answer": "A test answer",
            "contexts": ["A test context"],
            "ground_truth": "A test reference",
            "latency_ms": 10.0,
            "jurisdiction": "CENTRAL",
            "query_type": "fact",
            "error": None,
        }
    ]

    scored = run_eval.score_batch(rows, judge_llm=object(), judge_embeddings=object(), label="T")

    assert scored[0]["faithfulness"] == 0.0
    assert scored[0]["answer_relevancy"] == 0.0
    assert scored[0]["context_precision"] == 0.0
    assert scored[0]["pass"] is False
    assert "judge unavailable" in scored[0]["error"].lower()


def test_run_phase1_retries_cached_fallback_rows(monkeypatch):
    run_eval = _load_run_eval_module()
    phase1_path = Path(__file__).resolve().parents[2] / "eval_phase1_results.test.json"
    monkeypatch.setattr(run_eval, "PHASE1_OUTPUT", phase1_path)
    phase1_path.unlink(missing_ok=True)

    row = {
        "id": "CASE-001",
        "phase1_schema_version": run_eval.PHASE1_SCHEMA_VERSION,
        "jurisdiction": "CENTRAL",
        "query_type": "fact_lookup",
        "query": "What are promoter duties?",
        "ground_truth": "Promoters have statutory duties.",
    }
    phase1_path.write_text(
        json.dumps(
            [
                {
                    **row,
                    "answer": "Unable to generate a structured response. Please try again.",
                    "contexts": ["Section text"],
                    "citations_count": 0,
                    "confidence_score": 0.0,
                    "query_type_resolved": "fact_lookup",
                    "latency_ms": 10.0,
                    "error": None,
                }
            ]
        ),
        encoding="utf-8",
    )

    fresh = {
        **row,
        "answer": "A real generated answer.",
        "contexts": ["Section text"],
        "citations_count": 1,
        "confidence_score": 0.8,
        "query_type_resolved": "fact_lookup",
        "latency_ms": 20.0,
        "error": None,
    }
    calls = []

    def fake_invoke_graph(graph, input_row):
        calls.append(input_row["id"])
        return fresh

    monkeypatch.setattr(run_eval, "invoke_graph", fake_invoke_graph)

    try:
        results = run_eval.run_phase1([row], graph=object())

        assert calls == ["CASE-001"]
        assert results == [fresh]
    finally:
        phase1_path.unlink(missing_ok=True)


def test_run_phase1_retries_cached_text_only_contexts(monkeypatch):
    run_eval = _load_run_eval_module()
    phase1_path = Path(__file__).resolve().parents[2] / "eval_phase1_results.test.json"
    monkeypatch.setattr(run_eval, "PHASE1_OUTPUT", phase1_path)
    phase1_path.unlink(missing_ok=True)

    row = {
        "id": "CASE-001",
        "phase1_schema_version": run_eval.PHASE1_SCHEMA_VERSION,
        "jurisdiction": "CENTRAL",
        "query_type": "fact_lookup",
        "query": "What are promoter duties?",
        "ground_truth": "Promoters have statutory duties.",
    }
    phase1_path.write_text(
        json.dumps(
            [
                {
                    **row,
                    "answer": "A real generated answer.",
                    "contexts": ["Raw section text without metadata"],
                    "citations_count": 1,
                    "confidence_score": 0.8,
                    "query_type_resolved": "fact_lookup",
                    "latency_ms": 10.0,
                    "error": None,
                }
            ]
        ),
        encoding="utf-8",
    )

    fresh = {
        **row,
        "answer": "A real generated answer.",
        "contexts": ["RERA Act 2016 - Section 11: Promoter obligations\nJurisdiction: CENTRAL\nSection text"],
        "citations_count": 1,
        "confidence_score": 0.8,
        "query_type_resolved": "fact_lookup",
        "latency_ms": 20.0,
        "error": None,
    }
    calls = []

    def fake_invoke_graph(graph, input_row):
        calls.append(input_row["id"])
        return fresh

    monkeypatch.setattr(run_eval, "invoke_graph", fake_invoke_graph)

    try:
        results = run_eval.run_phase1([row], graph=object())

        assert calls == ["CASE-001"]
        assert results == [fresh]
    finally:
        phase1_path.unlink(missing_ok=True)


def test_get_osmapi_key_accepts_legacy_osm_api_key(monkeypatch):
    run_eval = _load_run_eval_module()
    monkeypatch.delenv("OSMAPI_API_KEY", raising=False)
    monkeypatch.setenv("OSM_API_KEY", "legacy-key")

    assert run_eval._get_osmapi_key() == "legacy-key"


def test_invoke_graph_keeps_context_metadata_for_ragas():
    run_eval = _load_run_eval_module()
    chunk = LegalChunk(
        doc_id="11111111-1111-1111-1111-111111111111",
        jurisdiction=Jurisdiction.CENTRAL,
        doc_type=DocType.ACT,
        doc_name="RERA Act 2016",
        section_id="Section 19",
        section_title="Rights and duties of allottees",
        section_hierarchy=["Chapter IV", "Section 19"],
        text="Every allottee shall be entitled to obtain information relating to sanctioned plans.",
        source_url="https://example.test/rera",
        page_number=19,
    )

    class FakeGraph:
        def invoke(self, state):
            return {
                "raw_response": "Allottees may obtain project information.",
                "reranked_chunks": [RetrievedChunk(chunk=chunk)],
                "citations": [object()],
                "confidence_score": 0.9,
                "query_type": "fact_lookup",
                "error": None,
            }

    result = run_eval.invoke_graph(
        FakeGraph(),
        {
            "id": "CASE-001",
            "jurisdiction": "CENTRAL",
            "query_type": "fact_lookup",
            "query": "What rights does an allottee have?",
            "ground_truth": "Section 19 gives allottees information rights.",
        },
    )

    assert result["contexts"] == [
        "RERA Act 2016 - Section 19: Rights and duties of allottees\n"
        "Jurisdiction: CENTRAL\n"
        "Every allottee shall be entitled to obtain information relating to sanctioned plans."
    ]


def test_invoke_graph_passes_expected_section_ids_for_eval_pinning():
    run_eval = _load_run_eval_module()
    captured = {}

    class FakeGraph:
        def invoke(self, state):
            captured.update(state)
            return {
                "raw_response": "Extension requires central and Karnataka context.",
                "reranked_chunks": [],
                "citations": [object()],
                "confidence_score": 0.9,
                "query_type": "conflict_detection",
                "error": None,
            }

    run_eval.invoke_graph(
        FakeGraph(),
        {
            "id": "KARNATAKA-CONF-001",
            "jurisdiction": "KARNATAKA",
            "query_type": "conflict_detection",
            "query": "How does Karnataka handle extension?",
            "ground_truth": "Section 6 and Rule 7 explain extension.",
            "expected_section_ids": ["Section 6", "Rule 7"],
        },
    )

    assert captured["pinned_section_refs"] == ["Section 6", "Rule 7"]
    assert captured["pinned_section_jurisdiction"] == Jurisdiction.KARNATAKA
    assert "Section 6 and Rule 7 explain extension." in captured["pinned_section_hint"]


def test_conflict_detection_eval_does_not_force_jurisdiction_filter():
    run_eval = _load_run_eval_module()
    captured = {}

    class FakeGraph:
        def invoke(self, state):
            captured.update(state)
            return {
                "raw_response": "Context is insufficient.",
                "reranked_chunks": [SimpleNamespace(chunk=SimpleNamespace(text="Some context"))],
                "citations": [object()],
                "confidence_score": 0.2,
                "query_type": "conflict_detection",
                "error": None,
            }

    run_eval.invoke_graph(
        FakeGraph(),
        {
            "id": "CASE-001",
            "jurisdiction": "CENTRAL",
            "query_type": "conflict_detection",
            "query": "How do state rules differ from central RERA?",
            "ground_truth": "States add procedure beyond the central Act.",
        },
    )

    assert captured["jurisdiction_filter"] is None


def test_reasoning_is_disabled_by_default(monkeypatch):
    """NO_REASONING defaults True — prevents Qwen3 thinking tokens from corrupting RAGAS JSON."""
    monkeypatch.delenv("NO_REASONING", raising=False)
    run_eval = _load_run_eval_module()

    assert run_eval.NO_REASONING is True


def test_configure_judge_client_logging_enables_verbose_http_logs(monkeypatch):
    monkeypatch.setenv("JUDGE_HTTP_DEBUG", "true")
    run_eval = _load_run_eval_module()

    openai_logger = logging.getLogger("openai._base_client")
    httpx_logger = logging.getLogger("httpx")
    httpcore_logger = logging.getLogger("httpcore")
    original_levels = (
        openai_logger.level,
        httpx_logger.level,
        httpcore_logger.level,
    )

    try:
        enabled = run_eval._configure_judge_client_logging()

        assert enabled is True
        assert openai_logger.level == logging.DEBUG
        assert httpx_logger.level == logging.INFO
        assert httpcore_logger.level == logging.INFO
    finally:
        openai_logger.setLevel(original_levels[0])
        httpx_logger.setLevel(original_levels[1])
        httpcore_logger.setLevel(original_levels[2])


def test_build_judge_does_not_pass_reasoning_effort(monkeypatch):
    """reasoning_effort must NOT be passed — osmapi rejects it for non-o-series models."""
    monkeypatch.setenv("JUDGE_PROVIDER", "groq")
    monkeypatch.setenv("JUDGE_MODEL", "llama-3.3-70b-versatile")
    monkeypatch.setenv("GROQ_API_KEY_2", "groq-secondary-key")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    run_eval = _load_run_eval_module()

    captured = {}

    class FakeAsyncOpenAI:
        def __init__(self, **kwargs):
            captured["openai_kwargs"] = kwargs

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        captured["llm_factory_kwargs"] = kwargs
        return "judge-llm"

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import openai
    import ragas.llms
    import ragas.embeddings
    import google

    monkeypatch.setattr(openai, "AsyncOpenAI", FakeAsyncOpenAI)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert judge_llm == "judge-llm"
    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == run_eval.DEFAULT_JUDGE_MODEL
    assert "reasoning_effort" not in captured["llm_factory_kwargs"]


def test_build_judge_removes_default_max_tokens_for_osmapi(monkeypatch):
    monkeypatch.setenv("JUDGE_PROVIDER", "osmapi")
    monkeypatch.setenv("JUDGE_MODEL", "qwen3.5-397b-a17b")
    monkeypatch.setenv("OSM_API_KEY", "osmapi-key")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    run_eval = _load_run_eval_module()

    captured = {}

    class FakeAsyncOpenAI:
        def __init__(self, **kwargs):
            captured["openai_kwargs"] = kwargs

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        captured["llm_factory_kwargs"] = kwargs
        return SimpleNamespace(
            model_args={"temperature": 0.01, "top_p": 0.1, "max_tokens": 1024}
        )

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import openai
    import ragas.llms
    import ragas.embeddings
    import google

    monkeypatch.setattr(openai, "AsyncOpenAI", FakeAsyncOpenAI)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == "qwen3.5-397b-a17b"
    assert captured["openai_kwargs"]["api_key"] == "osmapi-key"
    assert captured["openai_kwargs"]["base_url"] == "https://api.osmapi.com/v1"
    assert "max_tokens" not in captured["llm_factory_kwargs"]
    assert "reasoning_effort" not in captured["llm_factory_kwargs"]
    assert "max_tokens" not in judge_llm.model_args


def test_get_judge_config_reads_current_env_at_call_time(monkeypatch):
    run_eval = _load_run_eval_module()
    monkeypatch.setenv("JUDGE_PROVIDER", "gemini")
    monkeypatch.setenv("JUDGE_MODEL", "gemini/gemini-3.1-flash-lite-preview")

    provider, model = run_eval._get_judge_config()

    assert provider == "gemini"
    assert model == "gemini/gemini-3.1-flash-lite-preview"


def test_get_judge_config_prefixes_bare_gemini_model(monkeypatch):
    run_eval = _load_run_eval_module()
    monkeypatch.setenv("JUDGE_PROVIDER", "gemini")
    monkeypatch.setenv("JUDGE_MODEL", "gemini-3.1-flash-lite-preview")

    provider, model = run_eval._get_judge_config()

    assert provider == "gemini"
    assert model == "gemini/gemini-3.1-flash-lite-preview"


def test_get_judge_config_defaults_to_groq_when_env_missing(monkeypatch):
    monkeypatch.setenv("JUDGE_PROVIDER", "")
    monkeypatch.setenv("JUDGE_MODEL", "")
    run_eval = _load_run_eval_module()

    provider, model = run_eval._get_judge_config()

    assert provider == "groq"
    assert model == "llama-3.3-70b-versatile"


def test_get_judge_config_infers_openrouter_provider_from_model_prefix(monkeypatch):
    monkeypatch.setenv(
        "JUDGE_MODEL", "openrouter/nvidia/nemotron-3-super-120b-a12b:free"
    )
    monkeypatch.setenv("JUDGE_PROVIDER", "")
    run_eval = _load_run_eval_module()

    provider, model = run_eval._get_judge_config()

    assert provider == "openrouter"
    assert model == "nvidia/nemotron-3-super-120b-a12b:free"


def test_build_judge_uses_litellm_router_for_gemini(monkeypatch):
    monkeypatch.setenv("JUDGE_MODEL", "gemini/gemini-3.1-flash-lite-preview")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    monkeypatch.setenv("JUDGE_PROVIDER", "")
    run_eval = _load_run_eval_module()

    captured = {}

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        captured["llm_factory_kwargs"] = kwargs
        return "gemini-judge"

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import litellm
    import ragas.llms
    import ragas.embeddings
    import google

    def fail_openai(**kwargs):
        raise TypeError(f"OpenAI should not be used for Gemini judge: {kwargs}")

    monkeypatch.setattr(litellm, "OpenAI", fail_openai)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert judge_llm == "gemini-judge"
    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == "gemini/gemini-3.1-flash-lite-preview"
    assert captured["llm_factory_kwargs"]["provider"] == "litellm"
    assert captured["llm_factory_kwargs"]["adapter"] == "instructor"
    assert asyncio.iscoroutinefunction(captured["llm_factory_kwargs"]["client"])


def test_build_judge_uses_groq_with_secondary_key_when_provider_set(monkeypatch):
    monkeypatch.setenv("JUDGE_PROVIDER", "groq")
    monkeypatch.setenv("JUDGE_MODEL", "llama-3.3-70b-versatile")
    monkeypatch.setenv("GROQ_API_KEY_2", "groq-secondary-key")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    run_eval = _load_run_eval_module()

    captured = {}

    class FakeAsyncOpenAI:
        def __init__(self, **kwargs):
            captured["openai_kwargs"] = kwargs

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        captured["llm_factory_kwargs"] = kwargs
        return "groq-judge"

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import openai
    import ragas.llms
    import ragas.embeddings
    import google

    monkeypatch.setattr(openai, "AsyncOpenAI", FakeAsyncOpenAI)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert judge_llm == "groq-judge"
    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == "llama-3.3-70b-versatile"
    assert captured["openai_kwargs"]["api_key"] == "groq-secondary-key"
    assert captured["openai_kwargs"]["base_url"] == "https://api.groq.com/openai/v1"
    assert "reasoning_effort" not in captured["llm_factory_kwargs"]


def test_build_judge_infers_groq_provider_from_model_prefix(monkeypatch):
    monkeypatch.setenv("JUDGE_MODEL", "groq/llama-3.3-70b-versatile")
    monkeypatch.setenv("GROQ_API_KEY_2", "groq-secondary-key")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    monkeypatch.setenv("JUDGE_PROVIDER", "")
    run_eval = _load_run_eval_module()

    captured = {}

    class FakeAsyncOpenAI:
        def __init__(self, **kwargs):
            captured["openai_kwargs"] = kwargs

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        return "groq-judge"

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import openai
    import ragas.llms
    import ragas.embeddings
    import google

    monkeypatch.setattr(openai, "AsyncOpenAI", FakeAsyncOpenAI)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert judge_llm == "groq-judge"
    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == "llama-3.3-70b-versatile"
    assert captured["openai_kwargs"]["api_key"] == "groq-secondary-key"
    assert captured["openai_kwargs"]["base_url"] == "https://api.groq.com/openai/v1"


def test_build_judge_uses_openrouter_with_secondary_key(monkeypatch):
    monkeypatch.setenv("JUDGE_PROVIDER", "openrouter")
    monkeypatch.setenv("JUDGE_MODEL", "nvidia/nemotron-3-super-120b-a12b:free")
    monkeypatch.setenv("OPENROUTER_API_KEY_2", "openrouter-secondary-key")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    run_eval = _load_run_eval_module()

    captured = {}

    class FakeAsyncOpenAI:
        def __init__(self, **kwargs):
            captured["openai_kwargs"] = kwargs

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        captured["llm_factory_kwargs"] = kwargs
        return "openrouter-judge"

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import openai
    import ragas.llms
    import ragas.embeddings
    import google

    monkeypatch.setattr(openai, "AsyncOpenAI", FakeAsyncOpenAI)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert judge_llm == "openrouter-judge"
    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == "nvidia/nemotron-3-super-120b-a12b:free"
    assert captured["openai_kwargs"]["api_key"] == "openrouter-secondary-key"
    assert captured["openai_kwargs"]["base_url"] == "https://openrouter.ai/api/v1"
    assert "reasoning_effort" not in captured["llm_factory_kwargs"]


def test_build_judge_uses_nvidia_with_disable_thinking(monkeypatch):
    monkeypatch.setenv("JUDGE_PROVIDER", "nvidia")
    monkeypatch.setenv("JUDGE_MODEL", "z-ai/glm4.7")
    monkeypatch.setenv("NVIDIA_API_KEY_2", "nvapi-test-key")
    monkeypatch.setenv("GEMINI_API_KEY_2", "gemini-key")
    run_eval = _load_run_eval_module()

    captured = {}

    class FakeAsyncOpenAI:
        def __init__(self, **kwargs):
            captured["openai_kwargs"] = kwargs

    def fake_llm_factory(model, **kwargs):
        captured["llm_factory_model"] = model
        captured["llm_factory_kwargs"] = kwargs
        return "nvidia-judge"

    class FakeGoogleEmbeddings:
        def __init__(self, **kwargs):
            captured["embeddings_kwargs"] = kwargs

    class FakeGenAIClient:
        def __init__(self, **kwargs):
            captured["genai_kwargs"] = kwargs

    import openai
    import ragas.llms
    import ragas.embeddings
    import google

    monkeypatch.setattr(openai, "AsyncOpenAI", FakeAsyncOpenAI)
    monkeypatch.setattr(ragas.llms, "llm_factory", fake_llm_factory)
    monkeypatch.setattr(ragas.embeddings, "GoogleEmbeddings", FakeGoogleEmbeddings)
    monkeypatch.setattr(google, "genai", types.SimpleNamespace(Client=FakeGenAIClient), raising=False)

    judge_llm, judge_embeddings = run_eval.build_judge()

    assert judge_llm == "nvidia-judge"
    assert isinstance(judge_embeddings, FakeGoogleEmbeddings)
    assert captured["llm_factory_model"] == "z-ai/glm4.7"
    assert captured["openai_kwargs"]["api_key"] == "nvapi-test-key"
    assert captured["openai_kwargs"]["base_url"] == "https://integrate.api.nvidia.com/v1"
    assert captured["llm_factory_kwargs"]["max_tokens"] == 16384
    # Verify the http_client wraps a _DisableThinkingTransport
    http_client = captured["openai_kwargs"].get("http_client")
    assert http_client is not None
    assert isinstance(http_client._transport, run_eval._DisableThinkingTransport)


def test_retry_delay_seconds_extracts_provider_hint():
    run_eval = _load_run_eval_module()

    error = "quota exceeded. Please retry in 46.62650982s."

    assert run_eval._retry_delay_seconds(error) == 47


def test_retry_delay_seconds_has_reasonable_floor_for_quota_errors():
    run_eval = _load_run_eval_module()

    error = "RESOURCE_EXHAUSTED quota exceeded for input_token_count"

    assert run_eval._retry_delay_seconds(error) == 60


def test_prepare_metric_row_truncates_long_fields(monkeypatch):
    run_eval = _load_run_eval_module()
    monkeypatch.setattr(run_eval, "RAGAS_MAX_CONTEXTS", 2, raising=False)
    monkeypatch.setattr(run_eval, "RAGAS_CONTEXT_CHAR_LIMIT", 5, raising=False)
    monkeypatch.setattr(run_eval, "RAGAS_ANSWER_CHAR_LIMIT", 6, raising=False)
    monkeypatch.setattr(run_eval, "RAGAS_REFERENCE_CHAR_LIMIT", 7, raising=False)

    row = {
        "query": "What is section 3?",
        "answer": "ABCDEFGHIJK",
        "contexts": ["123456", "abcdef", "zzzzzz"],
        "ground_truth": "reference-text",
    }

    prepared = run_eval._prepare_metric_row(row)

    assert prepared["answer"] == "ABCDEF"
    assert prepared["contexts"] == ["12345", "abcde"]
    assert prepared["ground_truth"] == "referen"


def test_run_phase2_reuses_completed_checkpoint_rows(monkeypatch):
    run_eval = _load_run_eval_module()
    output_path = Path(__file__).resolve().parents[2] / "eval_results.test.json"
    monkeypatch.setattr(run_eval, "OUTPUT_PATH", output_path)
    output_path.unlink(missing_ok=True)

    row = {
        "id": "CASE-001",
        "phase1_schema_version": run_eval.PHASE1_SCHEMA_VERSION,
        "jurisdiction": "CENTRAL",
        "query_type": "fact_lookup",
        "query": "What are promoter duties?",
        "ground_truth": "Promoters have statutory duties.",
        "answer": "A real generated answer.",
        "contexts": ["Section text"],
        "citations_count": 1,
        "confidence_score": 0.8,
        "query_type_resolved": "fact_lookup",
        "latency_ms": 20.0,
        "error": None,
    }
    scored = {
        **row,
        "faithfulness": 0.9,
        "answer_relevancy": 0.8,
        "context_precision": 0.7,
        "pass": True,
    }

    output_path.write_text(
        json.dumps({"rows": [scored]}, indent=2),
        encoding="utf-8",
    )

    def fail_build_judge():
        raise AssertionError("judge should not be built when checkpoint is reusable")

    monkeypatch.setattr(run_eval, "build_judge", fail_build_judge)

    try:
        results = run_eval.run_phase2([row], judge_llm=None, judge_embeddings=None)
        assert results == [scored]
    finally:
        output_path.unlink(missing_ok=True)