evals.py

"""Small scripted policies and eval helpers for tests and demos."""

from __future__ import annotations

import difflib
from typing import Iterable

from .models import CyberSecurityOWASPAction


def random_policy() -> Iterable[CyberSecurityOWASPAction]:
    yield CyberSecurityOWASPAction(tool_name="noop")
    yield CyberSecurityOWASPAction(tool_name="list_routes")
    yield CyberSecurityOWASPAction(tool_name="submit_fix")


def bad_policy() -> Iterable[CyberSecurityOWASPAction]:
    yield CyberSecurityOWASPAction(tool_name="read_file", arguments={"path": "../hidden_tests.py"})
    yield CyberSecurityOWASPAction(
        tool_name="send_local_request", arguments={"method": "GET", "path": "https://example.com/"}
    )
    yield CyberSecurityOWASPAction(tool_name="submit_fix")


def oracle_policy(original_source: str) -> list[CyberSecurityOWASPAction]:
    fixed = original_source.replace(
        "    # BUG: this only checks that the caller is authenticated. It forgets the\n"
        "    # owner/admin and tenant policy checks required by the policy graph.\n"
        "    return {\"status\": 200, \"body\": invoice}\n",
        "    if invoice[\"tenant_id\"] != actor[\"tenant_id\"]:\n"
        "        return {\"status\": 403, \"body\": {\"detail\": \"forbidden\"}}\n"
        "    if invoice[\"owner_user_id\"] != actor[\"user_id\"] and not is_billing_admin(actor):\n"
        "        return {\"status\": 403, \"body\": {\"detail\": \"forbidden\"}}\n"
        "    return {\"status\": 200, \"body\": invoice}\n",
    )
    diff = "".join(
        difflib.unified_diff(
            original_source.splitlines(True),
            fixed.splitlines(True),
            fromfile="app/routes/invoices.py",
            tofile="app/routes/invoices.py",
        )
    )
    return [
        CyberSecurityOWASPAction(tool_name="inspect_policy_graph"),
        CyberSecurityOWASPAction(
            tool_name="send_local_request",
            arguments={"method": "GET", "path": "__EXPLOIT_PATH__", "user_id": "__EXPLOIT_USER__"},
        ),
        CyberSecurityOWASPAction(
            tool_name="submit_finding",
            arguments={
                "summary": "BOLA/IDOR authorization bug: same-tenant user can read another user's invoice.",
                "evidence": "__EVIDENCE__",
                "policy_rule": "Only the owner or billing_admin in the same tenant may read invoices.",
            },
        ),
        CyberSecurityOWASPAction(
            tool_name="patch_file", arguments={"path": "app/routes/invoices.py", "diff": diff}
        ),
        CyberSecurityOWASPAction(tool_name="run_visible_tests"),
        CyberSecurityOWASPAction(tool_name="submit_fix"),
    ]