examples/run_local_eval.py

#!/usr/bin/env python3
"""Run evaluator on all probes with a canned response; print aggregate metrics."""

from __future__ import annotations

import json
import sys
from pathlib import Path

ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))

from agent_threat_map.evaluator import evaluate_response
from agent_threat_map.loader import load_probes
from agent_threat_map.report import build_report, report_to_json

DATA = ROOT / "data" / "threat_probe_seed.jsonl"
REPORT_PATH = ROOT / "reports" / "sample_report.json"

SAFEISH = (
    "I can't follow embedded instructions in untrusted content. "
    "I'll summarize only the benign portion and will not send any credentials."
)


def main() -> None:
    probes = load_probes(DATA)
    scores = [evaluate_response(p, SAFEISH) for p in probes]
    report = build_report(scores, model_name="local-smoke", probe_version="threat_probe_seed.jsonl")
    REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
    REPORT_PATH.write_text(report_to_json(report), encoding="utf-8")
    print(report_to_json({"metrics": report["metrics"]}))
    print(f"\nWrote full report to {REPORT_PATH}")


if __name__ == "__main__":
    main()