scripts/run_judge_with_trigger_log.py

from __future__ import annotations

import json
import os
import sys
import time
from pathlib import Path
from typing import Any
from urllib import error as urlerror
from urllib import request as urlrequest

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
import judge_simulator as js


if hasattr(sys.stdout, "reconfigure"):
    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
if hasattr(sys.stderr, "reconfigure"):
    sys.stderr.reconfigure(encoding="utf-8", errors="replace")


class RotatingOpenRouter(js.LLMProvider):
    def __init__(self, keys: list[str], model: str):
        self.keys = keys
        self.model = model
        self.index = 0
        self.rotations = 0

    def name(self) -> str:
        return f"OpenRouter rotating ({self.model})"

    def complete(self, prompt: str, system: str = None) -> str:
        messages: list[dict[str, str]] = []
        if system:
            messages.append({"role": "system", "content": system})
        messages.append({"role": "user", "content": prompt})
        body = json.dumps(
            {
                "model": self.model,
                "messages": messages,
                "temperature": 0.2,
                "max_tokens": 1500,
            }
        ).encode("utf-8")

        last_error: Exception | None = None
        for _ in range(max(1, len(self.keys)) * 2):
            key = self.keys[self.index % len(self.keys)]
            req = urlrequest.Request(
                "https://openrouter.ai/api/v1/chat/completions",
                data=body,
                headers={
                    "Authorization": f"Bearer {key}",
                    "Content-Type": "application/json",
                    "HTTP-Referer": "https://magicpin.com",
                    "X-Title": "Vera Judge Trigger Logger",
                },
            )
            try:
                with urlrequest.urlopen(req, timeout=js.TIMEOUT_LLM) as resp:
                    data = json.loads(resp.read().decode("utf-8"))
                content = data["choices"][0]["message"].get("content")
                if not isinstance(content, str) or not content.strip():
                    raise RuntimeError("OpenRouter returned empty message content")
                return content
            except urlerror.HTTPError as exc:
                last_error = exc
                if exc.code in {401, 403, 408, 409, 429, 500, 502, 503, 504}:
                    self.index += 1
                    self.rotations += 1
                    time.sleep(1)
                    continue
                raise
            except Exception as exc:
                last_error = exc
                self.index += 1
                self.rotations += 1
                time.sleep(1)
                continue
        raise RuntimeError(f"OpenRouter rotation exhausted: {last_error}")


def _env_keys() -> list[str]:
    raw = os.getenv("OPENROUTER_API_KEYS") or os.getenv("OPENROUTER_API_KEY") or ""
    keys = [part.strip() for part in raw.split(",") if part.strip()]
    if not keys:
        raise SystemExit("Set OPENROUTER_API_KEY or comma-separated OPENROUTER_API_KEYS first.")
    return keys


def _score_row(action: dict[str, Any], score: Any) -> dict[str, Any]:
    return {
        "trigger_id": action.get("trigger_id", ""),
        "merchant_id": action.get("merchant_id", ""),
        "customer_id": action.get("customer_id"),
        "total": score.total,
        "specificity": score.specificity,
        "category_fit": score.category_fit,
        "merchant_fit": score.merchant_fit,
        "decision_quality": score.decision_quality,
        "engagement": score.engagement_compulsion,
        "hint": score.hint,
        "body": action.get("body", ""),
    }


def main() -> None:
    bot_url = os.getenv("BOT_URL", "https://mokshak-vera-rubric-decision-engine.hf.space")
    scenario = os.getenv("JUDGE_SCENARIO", "all")
    model = os.getenv("OPENROUTER_MODEL", "poolside/laguna-xs.2:free")
    output_path = os.getenv("JUDGE_TRIGGER_LOG_JSON", "")

    js.BOT_URL = bot_url
    provider = RotatingOpenRouter(_env_keys(), model)
    judge = js.JudgeSimulator(provider)
    rows: list[dict[str, Any]] = []

    original = judge._score_and_display

    def wrapped(action: dict[str, Any], verbose: bool = True):
        before = len(judge.all_scores)
        original(action, verbose)
        if len(judge.all_scores) > before:
            rows.append(_score_row(action, judge.all_scores[-1]))

    judge._score_and_display = wrapped  # type: ignore[method-assign]
    ok = judge.run(scenario)

    print("\n=== FLOOR TRIGGERS ===")
    for row in sorted(rows, key=lambda item: item["total"])[:12]:
        print(
            f"{row['total']:>2}/50  {row['trigger_id']}  "
            f"S:{row['specificity']} C:{row['category_fit']} M:{row['merchant_fit']} "
            f"D:{row['decision_quality']} E:{row['engagement']}  {row['hint']}"
        )

    print(f"\nROTATIONS_USED={provider.rotations}")
    if output_path:
        Path(output_path).write_text(json.dumps(rows, indent=2), encoding="utf-8")
        print(f"Wrote {output_path}")
    raise SystemExit(0 if ok else 1)


if __name__ == "__main__":
    main()