from __future__ import annotations import argparse import json from pathlib import Path from typing import Any, Dict, Iterable, List def default_manifest_path() -> Path: return Path(__file__).resolve().parents[1] / "docs" / "person_a_benchmark_manifest.json" def load_manifest(path: Path | None = None) -> Dict[str, Any]: manifest_path = path or default_manifest_path() return json.loads(manifest_path.read_text(encoding="utf-8")) def filter_entries( entries: Iterable[Dict[str, Any]], *, difficulty: str | None = None, channel: str | None = None, family: str | None = None, showcase_only: bool = False, delayed_only: bool = False, ) -> List[Dict[str, Any]]: filtered = list(entries) if difficulty: filtered = [entry for entry in filtered if entry.get("difficulty") == difficulty] if channel: filtered = [entry for entry in filtered if entry.get("attack_channel") == channel] if family: filtered = [entry for entry in filtered if entry.get("attack_family") == family] if showcase_only: filtered = [entry for entry in filtered if entry.get("showcase_role")] if delayed_only: filtered = [entry for entry in filtered if entry.get("delayed_trigger")] return filtered def render_stats(manifest: Dict[str, Any]) -> str: summary = manifest["summary"] lines = [ f"manifest_version: {manifest['manifest_version']}", f"generated_at: {manifest['generated_at']}", f"total_scenarios: {summary['total_scenarios']}", "by_difficulty:", ] for key, value in summary["by_difficulty"].items(): lines.append(f" - {key}: {value}") lines.append("by_channel:") for key, value in summary["by_channel"].items(): lines.append(f" - {key}: {value}") lines.append("showcase_scenarios:") for scenario_id in manifest.get("showcase_scenarios", []): lines.append(f" - {scenario_id}") return "\n".join(lines) def render_list(entries: Iterable[Dict[str, Any]]) -> str: lines = [] for entry in entries: tags = [] if entry.get("gold"): tags.append("gold") if entry.get("delayed_trigger"): tags.append("delayed") if entry.get("showcase_role"): tags.append(f"showcase={entry['showcase_role']}") tag_text = f" [{' | '.join(tags)}]" if tags else "" lines.append( f"{entry['scenario_id']} | {entry['difficulty']} | {entry['attack_channel']} | " f"{entry['attack_intent']}{tag_text}" ) return "\n".join(lines) def render_entry(entry: Dict[str, Any]) -> str: lines = [ f"scenario_id: {entry['scenario_id']}", f"difficulty: {entry['difficulty']}", f"subject: {entry['subject']}", f"gold: {entry['gold']}", f"summary_required: {entry['summary_required']}", f"attack_family: {entry['attack_family']}", f"attack_channel: {entry['attack_channel']}", f"attack_intent: {entry['attack_intent']}", f"attack_visibility: {entry['attack_visibility']}", f"instruction_priority_claim: {entry['instruction_priority_claim']}", f"target_tool: {entry['target_tool']}", f"target_output: {entry['target_output']}", f"target_recipient: {entry.get('target_recipient')}", f"payload_exactness: {entry['payload_exactness']}", f"requested_artifact_type: {entry['requested_artifact_type']}", f"activation_mode: {entry['activation_mode']}", f"delayed_trigger: {entry['delayed_trigger']}", f"primary_failure_mode: {entry['primary_failure_mode']}", f"showcase_role: {entry.get('showcase_role')}", ] if entry.get("expected_safe_path_shape"): lines.append(f"expected_safe_path_shape: {' -> '.join(entry['expected_safe_path_shape'])}") if entry.get("expected_unsafe_path_shape"): lines.append(f"expected_unsafe_path_shape: {' -> '.join(entry['expected_unsafe_path_shape'])}") return "\n".join(lines) def parse_args() -> argparse.Namespace: parser = argparse.ArgumentParser(description="Browse the Person A benchmark manifest.") parser.add_argument("--manifest", type=Path, default=default_manifest_path(), help="Path to manifest JSON.") parser.add_argument("--scenario-id", help="Show one scenario in detail.") parser.add_argument("--difficulty", choices=["easy", "medium", "hard"], help="Filter by difficulty.") parser.add_argument("--channel", help="Filter by attack channel.") parser.add_argument("--family", help="Filter by attack family.") parser.add_argument("--showcase-only", action="store_true", help="Only include showcase scenarios.") parser.add_argument("--delayed-only", action="store_true", help="Only include delayed-trigger scenarios.") parser.add_argument("--list", action="store_true", help="Render a compact filtered list.") parser.add_argument("--stats", action="store_true", help="Render manifest summary statistics.") return parser.parse_args() def main() -> None: args = parse_args() manifest = load_manifest(args.manifest) if args.stats and not args.scenario_id: print(render_stats(manifest)) if not args.list and not any([args.difficulty, args.channel, args.family, args.showcase_only, args.delayed_only]): return print("") entries = filter_entries( manifest.get("entries", []), difficulty=args.difficulty, channel=args.channel, family=args.family, showcase_only=args.showcase_only, delayed_only=args.delayed_only, ) if args.scenario_id: match = next((entry for entry in entries if entry["scenario_id"] == args.scenario_id), None) if match is None: raise SystemExit(f"Scenario '{args.scenario_id}' was not found in the filtered manifest view.") print(render_entry(match)) return if args.list or any([args.difficulty, args.channel, args.family, args.showcase_only, args.delayed_only]): print(render_list(entries)) return print(render_stats(manifest)) print("") print("Tip: use --list, --scenario-id, --showcase-only, --delayed-only, --difficulty, --channel, or --family.") if __name__ == "__main__": main()