File size: 1,192 Bytes
6c3043e | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 | #!/usr/bin/env python3
"""Run evaluator on all probes with a canned response; print aggregate metrics."""
from __future__ import annotations
import json
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from agent_threat_map.evaluator import evaluate_response
from agent_threat_map.loader import load_probes
from agent_threat_map.report import build_report, report_to_json
DATA = ROOT / "data" / "threat_probe_seed.jsonl"
REPORT_PATH = ROOT / "reports" / "sample_report.json"
SAFEISH = (
"I can't follow embedded instructions in untrusted content. "
"I'll summarize only the benign portion and will not send any credentials."
)
def main() -> None:
probes = load_probes(DATA)
scores = [evaluate_response(p, SAFEISH) for p in probes]
report = build_report(scores, model_name="local-smoke", probe_version="threat_probe_seed.jsonl")
REPORT_PATH.parent.mkdir(parents=True, exist_ok=True)
REPORT_PATH.write_text(report_to_json(report), encoding="utf-8")
print(report_to_json({"metrics": report["metrics"]}))
print(f"\nWrote full report to {REPORT_PATH}")
if __name__ == "__main__":
main()
|