feat: introduce reward ablation configurations for enhanced training flexibility, implement YAML loading with extends support, and add reward variant tracking in training scripts

reward_config.py

@@ -74,7 +74,7 @@ def load_reward_settings(path: str | Path | None = None) -> RewardSettings:
         or os.getenv("CYBERSECURITY_OWASP_REWARD_CONFIG", "")
         or DEFAULT_GRPO_CONFIG_PATH
     )
-    raw = yaml.safe_load(configured_path.read_text(encoding="utf-8")) or {}
+    raw = _load_yaml_with_extends(configured_path)
     reward = dict(raw.get("reward") or {})
     mode = os.getenv("CYBERSECURITY_OWASP_REWARD_MODE", str(reward.get("mode", "sparse_eval")))
     training_mode = str(reward.get("training_mode", "dense_train"))
@@ -90,6 +90,44 @@ def load_reward_settings(path: str | Path | None = None) -> RewardSettings:
     return settings
 
 
+def _load_yaml_with_extends(path: Path, seen: set[Path] | None = None) -> dict[str, Any]:
+    """Load a YAML file, recursively merging an optional relative `extends` file."""
+
+    resolved_path = path.expanduser().resolve()
+    seen = seen or set()
+    if resolved_path in seen:
+        chain = " -> ".join(str(item) for item in [*seen, resolved_path])
+        raise ValueError(f"reward config extends cycle detected: {chain}")
+    seen.add(resolved_path)
+
+    raw = yaml.safe_load(resolved_path.read_text(encoding="utf-8")) or {}
+    if not isinstance(raw, dict):
+        raise ValueError(f"reward config must be a YAML mapping: {resolved_path}")
+
+    extends = raw.get("extends")
+    if not extends:
+        return raw
+    if not isinstance(extends, str):
+        raise ValueError("reward config extends must be a string path")
+
+    base_path = Path(extends)
+    if not base_path.is_absolute():
+        base_path = resolved_path.parent / base_path
+    child = {key: value for key, value in raw.items() if key != "extends"}
+    return _deep_merge(_load_yaml_with_extends(base_path, seen), child)
+
+
+def _deep_merge(base: dict[str, Any], override: dict[str, Any]) -> dict[str, Any]:
+    merged = dict(base)
+    for key, value in override.items():
+        base_value = merged.get(key)
+        if isinstance(base_value, dict) and isinstance(value, dict):
+            merged[key] = _deep_merge(base_value, value)
+        else:
+            merged[key] = value
+    return merged
+
+
 def flatten_reward_config(
     settings: RewardSettings | None = None,
 ) -> list[dict[str, Any]]:
@@ -175,6 +213,7 @@ def reward_config_run_config(settings: RewardSettings | None = None) -> dict[str
         "reward_config_hash": summary["reward_config_hash"],
         "reward_config_source": summary["reward_config_source"],
         "reward_config_source_name": summary["reward_config_source_name"],
+        "reward_variant": os.getenv("CYBERSECURITY_OWASP_REWARD_VARIANT", "default") or "default",
         "reward_mode": summary["reward_mode"],
         "reward_training_mode": summary["reward_training_mode"],
         "reward_stage": summary["reward_stage"],

scripts/generate_sft_dataset.py

@@ -0,0 +1,753 @@
+"""Generate verifier-gated SFT data for CyberSecurity_OWASP.
+
+The default path asks a larger Hugging Face-hosted teacher model for one JSON
+action at a time, executes those actions in the real environment, and keeps
+only trajectories that pass the local deterministic verifier.  The
+``--dry-run-oracle`` path is intentionally network-free and exists for CI and
+smoke tests.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import statistics
+import subprocess
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, Iterable
+
+from CyberSecurity_OWASP.models import CyberSecurityOWASPAction, CyberSecurityOWASPObservation
+from CyberSecurity_OWASP.server.CyberSecurity_OWASP_environment import (
+    CybersecurityOwaspEnvironment,
+)
+from CyberSecurity_OWASP.validators import detect_cheating
+
+
+DEFAULT_TEACHER_MODEL = "deepseek-ai/DeepSeek-V4-Pro"
+DEFAULT_TARGET_MODEL = "unsloth/gemma-4-E2B-it"
+TRAINING_SYSTEM_PROMPT = (
+    "You are a defensive AppSec repair agent in the local CyberSecurity_OWASP "
+    "OpenEnv environment. Use only the listed local tools. Do not target real "
+    "systems. Work step by step: inspect policy and generated code, reproduce "
+    "the authorization issue locally, submit a policy-tied diagnosis, patch the "
+    "generated app, run visible tests, then submit the fix. Return exactly one "
+    "JSON action object and no markdown."
+)
+BANNED_PROMPT_MARKERS = (
+    "hidden_facts",
+    "oracle_hidden_focus",
+    "reward_engine",
+    "validators.py",
+    "rewards.py",
+    "tests/hidden",
+    "hidden tests",
+    ".git",
+)
+RISKY_ARGUMENT_MARKERS = (
+    "hidden",
+    "oracle",
+    "reward_engine",
+    "validators.py",
+    "rewards.py",
+    ".git",
+    "..",
+)
+
+
+@dataclass
+class DatasetConfig:
+    teacher_model: str = DEFAULT_TEACHER_MODEL
+    target_model: str = DEFAULT_TARGET_MODEL
+    split: str = "train"
+    difficulty: int = 0
+    seed_start: int = 0
+    episodes: int = 100
+    validation_episodes: int = 0
+    out_dir: Path = Path("outputs/sft")
+    max_steps: int = 40
+    max_teacher_retries: int = 2
+    max_tokens: int = 768
+    temperature: float = 0.2
+    top_p: float = 0.95
+    dry_run_oracle: bool = False
+
+
+class HuggingFaceTeacher:
+    """Small wrapper around Hugging Face chat completion."""
+
+    def __init__(
+        self,
+        *,
+        model: str,
+        token: str,
+        max_tokens: int,
+        temperature: float,
+        top_p: float,
+    ) -> None:
+        try:
+            from huggingface_hub import InferenceClient
+        except ImportError as exc:  # pragma: no cover - dependency smoke checked separately
+            raise RuntimeError(
+                "huggingface_hub is required for teacher generation. Install project "
+                "dependencies or use --dry-run-oracle for local CI."
+            ) from exc
+
+        self.model = model
+        self.max_tokens = int(max_tokens)
+        self.temperature = float(temperature)
+        self.top_p = float(top_p)
+        self.client = InferenceClient(token=token)
+
+    def complete(self, messages: list[dict[str, str]]) -> str:
+        response = self.client.chat_completion(
+            model=self.model,
+            messages=messages,
+            max_tokens=self.max_tokens,
+            temperature=self.temperature,
+            top_p=self.top_p,
+        )
+        return _chat_response_content(response)
+
+
+def _chat_response_content(response: Any) -> str:
+    choices = getattr(response, "choices", None)
+    if choices:
+        message = getattr(choices[0], "message", None)
+        content = getattr(message, "content", None)
+        if content is not None:
+            return str(content)
+    if isinstance(response, dict):
+        choices = response.get("choices") or []
+        if choices:
+            message = choices[0].get("message") or {}
+            return str(message.get("content", ""))
+    return str(response)
+
+
+def extract_first_json_object(text: str) -> dict[str, Any] | None:
+    """Extract the first JSON object from raw teacher text."""
+
+    stripped = text.strip()
+    candidates = [stripped]
+    if "```" in stripped:
+        for part in stripped.split("```"):
+            candidate = part.strip()
+            if candidate.startswith("json"):
+                candidate = candidate[4:].strip()
+            candidates.append(candidate)
+
+    for candidate in candidates:
+        try:
+            loaded = json.loads(candidate)
+        except Exception:
+            continue
+        if isinstance(loaded, dict):
+            return loaded
+
+    start = stripped.find("{")
+    while start >= 0:
+        depth = 0
+        in_string = False
+        escaped = False
+        for index in range(start, len(stripped)):
+            char = stripped[index]
+            if in_string:
+                if escaped:
+                    escaped = False
+                elif char == "\\":
+                    escaped = True
+                elif char == '"':
+                    in_string = False
+                continue
+            if char == '"':
+                in_string = True
+            elif char == "{":
+                depth += 1
+            elif char == "}":
+                depth -= 1
+                if depth == 0:
+                    try:
+                        loaded = json.loads(stripped[start : index + 1])
+                    except Exception:
+                        break
+                    if isinstance(loaded, dict):
+                        return loaded
+        start = stripped.find("{", start + 1)
+    return None
+
+
+def parse_action_text(text: str) -> CyberSecurityOWASPAction:
+    data = extract_first_json_object(text)
+    if data is None:
+        raise ValueError("teacher did not return a JSON object")
+    return CyberSecurityOWASPAction(**data)
+
+
+def action_to_json(action: CyberSecurityOWASPAction) -> str:
+    return json.dumps(action.model_dump(), separators=(",", ":"), sort_keys=True)
+
+
+def _safe_observation_payload(
+    observation: CyberSecurityOWASPObservation,
+    recent_actions: list[dict[str, Any]],
+) -> dict[str, Any]:
+    return {
+        "phase": observation.phase,
+        "task_brief": observation.task_brief,
+        "scenario_prompt": observation.scenario_prompt,
+        "available_actions": observation.available_actions,
+        "last_tool_result": observation.last_tool_result,
+        "last_action_valid": observation.last_action_valid,
+        "last_action_error": observation.last_action_error,
+        "visible_test_result": observation.visible_test_result,
+        "done_reason": observation.done_reason,
+        "recent_actions": recent_actions[-8:],
+    }
+
+
+def build_user_prompt(
+    observation: CyberSecurityOWASPObservation,
+    recent_actions: list[dict[str, Any]],
+    retry_error: str | None = None,
+) -> str:
+    payload = _safe_observation_payload(observation, recent_actions)
+    prompt = (
+        "Current CyberSecurity_OWASP observation, containing only information "
+        "available to the agent:\n"
+        f"{json.dumps(payload, indent=2, sort_keys=True)}\n\n"
+        "Choose the next action. Output exactly one JSON object with keys "
+        "`tool_name` and `arguments`. Do not include markdown or commentary."
+    )
+    if retry_error:
+        prompt += f"\nPrevious candidate was rejected safely: {retry_error}"
+    _assert_prompt_is_safe(prompt)
+    return prompt
+
+
+def _assert_prompt_is_safe(prompt: str) -> None:
+    lowered = prompt.lower()
+    leaked = [marker for marker in BANNED_PROMPT_MARKERS if marker.lower() in lowered]
+    if leaked:
+        raise ValueError(f"prompt contains blocked marker(s): {', '.join(leaked)}")
+
+
+def build_chat_messages(
+    observation: CyberSecurityOWASPObservation,
+    recent_actions: list[dict[str, Any]],
+    retry_error: str | None = None,
+) -> list[dict[str, str]]:
+    return [
+        {"role": "system", "content": TRAINING_SYSTEM_PROMPT},
+        {"role": "user", "content": build_user_prompt(observation, recent_actions, retry_error)},
+    ]
+
+
+def make_chat_row(
+    *,
+    messages: list[dict[str, str]],
+    action: CyberSecurityOWASPAction,
+    metadata: dict[str, Any],
+) -> dict[str, Any]:
+    return {
+        "messages": [
+            *messages,
+            {"role": "assistant", "content": action_to_json(action)},
+        ],
+        "metadata": metadata,
+    }
+
+
+def preflight_action(
+    env: CybersecurityOwaspEnvironment,
+    observation: CyberSecurityOWASPObservation,
+    action: CyberSecurityOWASPAction,
+) -> tuple[bool, str]:
+    if action.tool_name not in observation.available_actions:
+        return False, f"{action.tool_name} is not allowed during {observation.phase}"
+    args = action.arguments or {}
+    flags = detect_cheating(env.state, action)
+    if flags:
+        return False, f"action triggered safety flags: {', '.join(flags)}"
+    arg_text = json.dumps(args, sort_keys=True, default=str).lower()
+    if any(marker in arg_text for marker in RISKY_ARGUMENT_MARKERS):
+        return False, "arguments reference blocked files or paths"
+    if action.tool_name == "read_file" and not args.get("path"):
+        return False, "read_file requires path"
+    if action.tool_name == "search_code" and not args.get("query"):
+        return False, "search_code requires query"
+    if action.tool_name == "patch_file":
+        path = str(args.get("path", ""))
+        if not path:
+            return False, "patch_file requires path"
+        if path.replace("\\", "/").startswith("tests/"):
+            return False, "patch_file cannot modify tests"
+        if not args.get("content") and not args.get("diff"):
+            return False, "patch_file requires content or diff"
+    if action.tool_name == "send_local_request":
+        path = str(args.get("path", ""))
+        if not path.startswith("/"):
+            return False, "send_local_request requires a local route path"
+    if action.tool_name == "compare_identities":
+        path = str(args.get("path", ""))
+        if not path.startswith("/"):
+            return False, "compare_identities requires a local route path"
+        if not args.get("first_user_id") or not args.get("second_user_id"):
+            return False, "compare_identities requires two user ids"
+    if action.tool_name == "submit_diagnosis":
+        required = ("bug_class", "route", "violated_policy_rule", "evidence_trace_ids", "fix_plan")
+        missing = [key for key in required if not args.get(key)]
+        if missing:
+            return False, f"submit_diagnosis missing: {', '.join(missing)}"
+    return True, ""
+
+
+def _trace_id_from_observation(observation: CyberSecurityOWASPObservation) -> str:
+    try:
+        payload = json.loads(observation.last_tool_result)
+    except Exception:
+        return "req_001"
+    return str(payload.get("trace_id", "req_001"))
+
+
+def _secure_invoice_source(env: CybersecurityOwaspEnvironment) -> str:
+    source = (Path(env.state.hidden_facts["workspace"]) / "app/routes/invoices.py").read_text(
+        encoding="utf-8"
+    )
+    return source.replace(
+        "    # BUG: this only checks that the caller is authenticated. It forgets the\n"
+        "    # owner/admin and tenant policy checks required by the policy graph.\n"
+        "    return {\"status\": 200, \"body\": invoice}\n",
+        "    if invoice[\"tenant_id\"] != actor[\"tenant_id\"]:\n"
+        "        return {\"status\": 403, \"body\": {\"detail\": \"forbidden\"}}\n"
+        "    if invoice[\"owner_user_id\"] != actor[\"user_id\"] and not is_billing_admin(actor):\n"
+        "        return {\"status\": 403, \"body\": {\"detail\": \"forbidden\"}}\n"
+        "    return {\"status\": 200, \"body\": invoice}\n",
+    )
+
+
+def oracle_actions_for_state(
+    env: CybersecurityOwaspEnvironment,
+    evidence_trace_id: str | None = None,
+) -> list[CyberSecurityOWASPAction]:
+    hidden = env.state.hidden_facts
+    trace_id = evidence_trace_id or "req_001"
+    return [
+        CyberSecurityOWASPAction(tool_name="inspect_policy_graph", arguments={}),
+        CyberSecurityOWASPAction(tool_name="list_routes", arguments={}),
+        CyberSecurityOWASPAction(
+            tool_name="read_file",
+            arguments={"path": "app/routes/invoices.py"},
+        ),
+        CyberSecurityOWASPAction(
+            tool_name="send_local_request",
+            arguments={
+                "method": "GET",
+                "path": f"/invoices/{hidden['other_invoice_id']}",
+                "user_id": hidden["owner_user_id"],
+            },
+        ),
+        CyberSecurityOWASPAction(
+            tool_name="submit_diagnosis",
+            arguments={
+                "bug_class": "idor_ownership_bug",
+                "route": "GET /invoices/{invoice_id}",
+                "violated_policy_rule": "Only the owner or a billing_admin in the same tenant may read invoices.",
+                "evidence_trace_ids": [trace_id],
+                "fix_plan": "Add tenant and owner/admin checks before returning invoice data.",
+            },
+        ),
+        CyberSecurityOWASPAction(
+            tool_name="patch_file",
+            arguments={"path": "app/routes/invoices.py", "content": _secure_invoice_source(env)},
+        ),
+        CyberSecurityOWASPAction(tool_name="run_visible_tests", arguments={}),
+        CyberSecurityOWASPAction(tool_name="submit_fix", arguments={}),
+    ]
+
+
+def _teacher_action(
+    *,
+    teacher: HuggingFaceTeacher,
+    env: CybersecurityOwaspEnvironment,
+    observation: CyberSecurityOWASPObservation,
+    recent_actions: list[dict[str, Any]],
+    config: DatasetConfig,
+) -> tuple[CyberSecurityOWASPAction, list[dict[str, str]]]:
+    retry_error: str | None = None
+    for _ in range(config.max_teacher_retries + 1):
+        messages = build_chat_messages(observation, recent_actions, retry_error)
+        raw = teacher.complete(messages)
+        try:
+            action = parse_action_text(raw)
+        except Exception as exc:
+            retry_error = str(exc)
+            continue
+        ok, error = preflight_action(env, observation, action)
+        if ok:
+            return action, messages
+        retry_error = error
+    raise ValueError(retry_error or "teacher did not produce a usable action")
+
+
+def _oracle_action(
+    *,
+    env: CybersecurityOwaspEnvironment,
+    observation: CyberSecurityOWASPObservation,
+    recent_actions: list[dict[str, Any]],
+    oracle_actions: list[CyberSecurityOWASPAction],
+    step_index: int,
+) -> tuple[CyberSecurityOWASPAction, list[dict[str, str]]]:
+    action = oracle_actions[step_index]
+    messages = build_chat_messages(observation, recent_actions)
+    ok, error = preflight_action(env, observation, action)
+    if not ok:
+        raise ValueError(error)
+    return action, messages
+
+
+def _terminal_checks_passed(env: CybersecurityOwaspEnvironment) -> bool:
+    verifier = env.state.verification_summary or {}
+    required = ("visible", "security", "regression", "public_routes", "patch_quality")
+    return all(bool((verifier.get(key) or {}).get("passed", False)) for key in required)
+
+
+def _episode_reward(env: CybersecurityOwaspEnvironment) -> float:
+    if env.state.reward_history:
+        return float(env.state.reward_history[-1].get("terminal_total", 0.0))
+    return 0.0
+
+
+def run_episode(
+    *,
+    seed: int,
+    split: str,
+    difficulty: int,
+    config: DatasetConfig,
+    teacher: HuggingFaceTeacher | None,
+) -> dict[str, Any]:
+    env = CybersecurityOwaspEnvironment()
+    rows: list[dict[str, Any]] = []
+    trajectory_steps: list[dict[str, Any]] = []
+    recent_actions: list[dict[str, Any]] = []
+    try:
+        observation = env.reset(seed=seed, split=split, difficulty=difficulty)
+        oracle_actions = oracle_actions_for_state(env) if config.dry_run_oracle else []
+        for step_index in range(config.max_steps):
+            if observation.done:
+                break
+            if config.dry_run_oracle:
+                if step_index >= len(oracle_actions):
+                    raise ValueError("oracle action script ended before terminal state")
+                if step_index == 4 and env.state.request_trace:
+                    trace_id = _trace_id_from_observation(observation)
+                    oracle_actions = oracle_actions_for_state(env, evidence_trace_id=trace_id)
+                action, messages = _oracle_action(
+                    env=env,
+                    observation=observation,
+                    recent_actions=recent_actions,
+                    oracle_actions=oracle_actions,
+                    step_index=step_index,
+                )
+            else:
+                if teacher is None:
+                    raise RuntimeError("teacher is required unless --dry-run-oracle is set")
+                action, messages = _teacher_action(
+                    teacher=teacher,
+                    env=env,
+                    observation=observation,
+                    recent_actions=recent_actions,
+                    config=config,
+                )
+
+            step_number = step_index + 1
+            action_record = action.model_dump()
+            row = make_chat_row(
+                messages=messages,
+                action=action,
+                metadata={
+                    "target_model": config.target_model,
+                    "teacher_model": config.teacher_model,
+                    "seed": seed,
+                    "split": split,
+                    "difficulty": difficulty,
+                    "step": step_number,
+                    "tool_name": action.tool_name,
+                    "task_id": env.state.task_id,
+                    "episode_id": env.state.episode_id,
+                    "scenario_hash": env.state.scenario_hash,
+                },
+            )
+            next_observation = env.step(action)
+            trajectory_steps.append(
+                {
+                    "step": step_number,
+                    "prompt_messages": messages,
+                    "action": action_record,
+                    "observation": next_observation.model_dump(),
+                    "reward_breakdown": dict(next_observation.reward_breakdown or {}),
+                }
+            )
+            if not next_observation.last_action_valid:
+                raise ValueError(next_observation.last_action_error or "invalid action")
+            if env.state.anti_cheat_flags:
+                raise ValueError(f"anti-cheat flags: {env.state.anti_cheat_flags}")
+            rows.append(row)
+            recent_actions.append(action_record)
+            observation = next_observation
+            if observation.done:
+                break
+
+        if not env.state.done:
+            raise ValueError("episode did not reach a terminal state")
+        if not env.state.success:
+            raise ValueError(env.state.failure_reason or "terminal verifier failed")
+        if env.state.step_count > config.max_steps:
+            raise ValueError("episode exceeded max steps")
+        if env.state.anti_cheat_flags:
+            raise ValueError("episode has anti-cheat flags")
+        if not _terminal_checks_passed(env):
+            raise ValueError("terminal verifier checks did not all pass")
+
+        final_reward = _episode_reward(env)
+        final_breakdown = dict(env.state.reward_history[-1]) if env.state.reward_history else {}
+        for row in rows:
+            row["metadata"].update(
+                {
+                    "final_success": True,
+                    "terminal_total": final_reward,
+                    "total_reward": float(env.state.accumulated_reward),
+                    "anti_cheat_flags": list(env.state.anti_cheat_flags),
+                    "final_reward_breakdown": final_breakdown,
+                }
+            )
+        return {
+            "accepted": True,
+            "seed": seed,
+            "split": split,
+            "difficulty": difficulty,
+            "rows": rows,
+            "trajectory": {
+                "episode_id": env.state.episode_id,
+                "task_id": env.state.task_id,
+                "seed": seed,
+                "split": split,
+                "difficulty": difficulty,
+                "domain": env.state.domain,
+                "bug_family": env.state.bug_family,
+                "scenario_hash": env.state.scenario_hash,
+                "actions": [step["action"] for step in trajectory_steps],
+                "steps": trajectory_steps,
+                "reward_breakdown_by_step": list(env.state.reward_history),
+                "final_reward_breakdown": final_breakdown,
+                "total_reward": float(env.state.accumulated_reward),
+                "terminal_total": final_reward,
+                "success": True,
+                "failure_reason": None,
+                "anti_cheat_flags": list(env.state.anti_cheat_flags),
+                "verification_summary": env.state.verification_summary,
+            },
+        }
+    except Exception as exc:
+        return {
+            "accepted": False,
+            "seed": seed,
+            "split": split,
+            "difficulty": difficulty,
+            "reason": str(exc),
+            "rows": [],
+            "trajectory": {
+                "seed": seed,
+                "split": split,
+                "difficulty": difficulty,
+                "steps": trajectory_steps,
+                "actions": [step["action"] for step in trajectory_steps],
+                "success": bool(env.state.success),
+                "failure_reason": env.state.failure_reason or str(exc),
+                "anti_cheat_flags": list(env.state.anti_cheat_flags),
+            },
+        }
+    finally:
+        env.close()
+
+
+def write_jsonl(path: Path, rows: Iterable[dict[str, Any]]) -> None:
+    path.parent.mkdir(parents=True, exist_ok=True)
+    with path.open("w", encoding="utf-8") as handle:
+        for row in rows:
+            handle.write(json.dumps(row, sort_keys=True, default=str) + "\n")
+
+
+def _write_trajectory(out_dir: Path, trajectory: dict[str, Any]) -> Path:
+    traj_dir = out_dir / "trajectories"
+    traj_dir.mkdir(parents=True, exist_ok=True)
+    name = (
+        f"{trajectory.get('split', 'train')}_seed{trajectory.get('seed', 0)}_"
+        f"{str(trajectory.get('episode_id', 'rejected'))[:12]}.json"
+    )
+    path = traj_dir / name
+    path.write_text(json.dumps(trajectory, indent=2, sort_keys=True, default=str), encoding="utf-8")
+    return path
+
+
+def _git_sha() -> str:
+    root = Path(__file__).resolve().parents[1]
+    try:
+        return subprocess.check_output(
+            [
+                "git",
+                "-c",
+                f"safe.directory={root.as_posix()}",
+                "rev-parse",
+                "HEAD",
+            ],
+            cwd=root,
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except Exception:
+        return "nogit"
+
+
+def _reward_summary(values: list[float]) -> dict[str, float]:
+    if not values:
+        return {"mean": 0.0, "min": 0.0, "max": 0.0, "p50": 0.0}
+    sorted_values = sorted(values)
+    return {
+        "mean": float(statistics.mean(values)),
+        "min": float(min(values)),
+        "max": float(max(values)),
+        "p50": float(sorted_values[len(sorted_values) // 2]),
+    }
+
+
+def generate_dataset(config: DatasetConfig) -> dict[str, Any]:
+    config.out_dir.mkdir(parents=True, exist_ok=True)
+    teacher = None
+    if not config.dry_run_oracle:
+        token = os.getenv("HF_TOKEN")
+        if not token:
+            raise RuntimeError("HF_TOKEN is required unless --dry-run-oracle is set")
+        teacher = HuggingFaceTeacher(
+            model=config.teacher_model,
+            token=token,
+            max_tokens=config.max_tokens,
+            temperature=config.temperature,
+            top_p=config.top_p,
+        )
+
+    split_jobs = [(config.split, config.episodes, config.seed_start)]
+    if config.validation_episodes:
+        split_jobs.append(("validation", config.validation_episodes, config.seed_start + config.episodes))
+
+    rows_by_split: dict[str, list[dict[str, Any]]] = {"train": [], "validation": []}
+    attempts: list[dict[str, Any]] = []
+    rewards: list[float] = []
+    accepted = 0
+    attempted = 0
+    for split, episodes, seed_start in split_jobs:
+        for offset in range(int(episodes)):
+            seed = int(seed_start) + offset
+            attempted += 1
+            result = run_episode(
+                seed=seed,
+                split=split,
+                difficulty=config.difficulty,
+                config=config,
+                teacher=teacher,
+            )
+            attempts.append(
+                {
+                    "seed": seed,
+                    "split": split,
+                    "accepted": bool(result["accepted"]),
+                    "reason": result.get("reason", ""),
+                    "trajectory_path": str(_write_trajectory(config.out_dir, result["trajectory"])),
+                }
+            )
+            if result["accepted"]:
+                accepted += 1
+                rows = list(result["rows"])
+                rows_by_split.setdefault(split, []).extend(rows)
+                rewards.append(float(result["trajectory"].get("terminal_total", 0.0)))
+
+    for split_name in ("train", "validation", config.split):
+        write_jsonl(config.out_dir / f"{split_name}.jsonl", rows_by_split.get(split_name, []))
+
+    manifest = {
+        "teacher_model": config.teacher_model,
+        "target_model": config.target_model,
+        "split": config.split,
+        "difficulty": config.difficulty,
+        "seed_start": config.seed_start,
+        "episodes_attempted": attempted,
+        "episodes_accepted": accepted,
+        "acceptance_rate": accepted / attempted if attempted else 0.0,
+        "rows_by_split": {key: len(value) for key, value in sorted(rows_by_split.items())},
+        "reward_summary": _reward_summary(rewards),
+        "git_sha": _git_sha(),
+        "verifier_version": "verifier_v1",
+        "dry_run_oracle": config.dry_run_oracle,
+        "attempts": attempts,
+    }
+    manifest_path = config.out_dir / "manifest.json"
+    manifest_path.write_text(
+        json.dumps(manifest, indent=2, sort_keys=True, default=str),
+        encoding="utf-8",
+    )
+    return manifest
+
+
+def build_arg_parser() -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(description=__doc__)
+    parser.add_argument("--teacher-model", default=DEFAULT_TEACHER_MODEL)
+    parser.add_argument("--target-model", default=DEFAULT_TARGET_MODEL)
+    parser.add_argument("--split", default="train", choices=["train", "validation", "hidden_eval"])
+    parser.add_argument("--difficulty", type=int, default=0)
+    parser.add_argument("--seed-start", type=int, default=0)
+    parser.add_argument("--episodes", type=int, default=100)
+    parser.add_argument("--validation-episodes", type=int, default=0)
+    parser.add_argument("--out-dir", type=Path, default=Path("outputs/sft"))
+    parser.add_argument("--max-steps", type=int, default=40)
+    parser.add_argument("--max-teacher-retries", type=int, default=2)
+    parser.add_argument("--max-tokens", type=int, default=768)
+    parser.add_argument("--temperature", type=float, default=0.2)
+    parser.add_argument("--top-p", type=float, default=0.95)
+    parser.add_argument(
+        "--dry-run-oracle",
+        action="store_true",
+        help="Generate deterministic oracle data without calling the HF API.",
+    )
+    return parser
+
+
+def config_from_args(args: argparse.Namespace) -> DatasetConfig:
+    return DatasetConfig(
+        teacher_model=args.teacher_model,
+        target_model=args.target_model,
+        split=args.split,
+        difficulty=args.difficulty,
+        seed_start=args.seed_start,
+        episodes=args.episodes,
+        validation_episodes=args.validation_episodes,
+        out_dir=args.out_dir,
+        max_steps=args.max_steps,
+        max_teacher_retries=args.max_teacher_retries,
+        max_tokens=args.max_tokens,
+        temperature=args.temperature,
+        top_p=args.top_p,
+        dry_run_oracle=args.dry_run_oracle,
+    )
+
+
+def main(argv: list[str] | None = None) -> int:
+    parser = build_arg_parser()
+    args = parser.parse_args(argv)
+    manifest = generate_dataset(config_from_args(args))
+    print(json.dumps(manifest, indent=2, sort_keys=True))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

scripts/launch_reward_ablations.ps1

@@ -0,0 +1,59 @@
+param(
+    [switch]$AllowActive
+)
+
+$ErrorActionPreference = "Stop"
+$env:PYTHONIOENCODING = "utf-8"
+$env:PYTHONUTF8 = "1"
+
+$appList = uv run --extra modal modal app list | Out-String
+Write-Host $appList
+if (-not $AllowActive -and $appList -match "CyberSecur" -and $appList -match "ephemeral") {
+    throw "Active CyberSecurity_OWASP Modal apps are present. Re-run with -AllowActive only if overlapping L4 jobs are intentional."
+}
+
+$runs = @(
+    @{
+        Variant = "abl-a0-sparse"
+        Config = "training/configs/reward_ablations/A0_sparse_terminal_only.yaml"
+        Seed = 110000
+    },
+    @{
+        Variant = "abl-a2-shape035"
+        Config = "training/configs/reward_ablations/A2_reduced_shaping.yaml"
+        Seed = 120000
+    },
+    @{
+        Variant = "abl-a6-visgate"
+        Config = "training/configs/reward_ablations/A6_visible_gate.yaml"
+        Seed = 130000
+    },
+    @{
+        Variant = "abl-a7-evid045"
+        Config = "training/configs/reward_ablations/A7_evidence045.yaml"
+        Seed = 140000
+    },
+    @{
+        Variant = "abl-a3-nospeed"
+        Config = "training/configs/reward_ablations/A3_no_speed_token.yaml"
+        Seed = 150000
+    }
+)
+
+foreach ($run in $runs) {
+    Write-Host "Launching $($run.Variant) with $($run.Config) seed $($run.Seed)"
+    uv run --extra modal modal run --detach scripts/modal_train_grpo.py `
+        --mode train `
+        --max-steps 60 `
+        --dataset-size 32 `
+        --num-generations 4 `
+        --max-completion-length 768 `
+        --difficulty 0 `
+        --split train `
+        --source-mode local `
+        --trace-log-every 5 `
+        --seed-start $run.Seed `
+        --reward-config $run.Config `
+        --reward-variant $run.Variant `
+        --detach
+}

scripts/modal_train_grpo.py

@@ -210,6 +210,24 @@ def _configure_scenario_cache_env(*, required: bool = True) -> dict[str, str]:
     return values
 
 
+def _configure_reward_env(
+    *,
+    reward_config: str = "",
+    reward_variant: str = "",
+    reward_mode: str = "",
+) -> dict[str, str]:
+    values: dict[str, str] = {}
+    if reward_config:
+        values["CYBERSECURITY_OWASP_REWARD_CONFIG"] = reward_config
+    if reward_variant:
+        values["CYBERSECURITY_OWASP_REWARD_VARIANT"] = reward_variant
+    if reward_mode:
+        values["CYBERSECURITY_OWASP_REWARD_MODE"] = reward_mode
+    for key, value in values.items():
+        os.environ[key] = value
+    return values
+
+
 def _print_image_startup_notice() -> None:
     global _IMAGE_NOTICE_PRINTED
     if _IMAGE_NOTICE_PRINTED:
@@ -583,6 +601,8 @@ def run_cybersecurity_owasp_baseline(
     source_mode: str = "local",
     repo_url: str = PUBLIC_REPO_URL,
     repo_branch: str = PUBLIC_REPO_BRANCH,
+    reward_config: str = "",
+    reward_variant: str = "",
 ) -> dict[str, str | int | float]:
     import statistics
     import time
@@ -627,8 +647,14 @@ def run_cybersecurity_owasp_baseline(
 
     os.environ["TRACKIO_SPACE_ID"] = trackio_space_id
     os.environ["TRACKIO_PROJECT"] = trackio_project
+    reward_env = _configure_reward_env(
+        reward_config=reward_config,
+        reward_variant=reward_variant,
+    )
     reward_settings = load_reward_settings()
     reward_tracking_config = reward_config_trackio_config(reward_settings)
+    reward_tracking_config["reward_variant"] = reward_variant or "default"
+    reward_tracking_config["reward_config_path"] = reward_config or reward_settings.source_path
     run_name = run_name or "baseline"
     output_dir = RUNS_DIR / run_name
     output_dir.mkdir(parents=True, exist_ok=True)
@@ -673,6 +699,10 @@ def run_cybersecurity_owasp_baseline(
     print(f"Trackio Project: {trackio_project}")
     print(f"Reward config: {reward_tracking_config['reward_config_id']}")
     print(f"Reward config hash: {reward_tracking_config['reward_config_hash']}")
+    print(f"Reward variant: {reward_tracking_config['reward_variant']}")
+    print(f"Reward config path: {reward_tracking_config['reward_config_path']}")
+    if reward_env:
+        print(f"Reward env overrides: {reward_env}")
     print(f"Scenario cache dir: {scenario_cache_env['CYBERSECURITY_OWASP_SCENARIO_CACHE_DIR']}")
     print(f"Scenario cache coverage: {coverage}")
     print(
@@ -818,6 +848,7 @@ def run_cybersecurity_owasp_baseline(
         "num_generations": num_generations,
         "max_completion_length": max_completion_length,
         "git_sha": git_sha,
+        "reward_variant": reward_tracking_config["reward_variant"],
         **reward_tracking_config,
     }
 
@@ -998,6 +1029,8 @@ def run_cybersecurity_owasp_baseline(
 def train_cybersecurity_owasp_grpo(
     env_repo_id: str = "",
     output_repo_id: str = "",
+    initial_adapter_path: str = "",
+    initial_adapter_repo_id: str = "",
     max_steps: int = 10,
     dataset_size: int = 16,
     difficulty: int = 0,
@@ -1021,6 +1054,8 @@ def train_cybersecurity_owasp_grpo(
     repo_url: str = PUBLIC_REPO_URL,
     repo_branch: str = PUBLIC_REPO_BRANCH,
     push_to_hub: bool = False,
+    reward_config: str = "",
+    reward_variant: str = "",
 ) -> dict[str, str | int | float]:
     import inspect
     import statistics
@@ -1050,6 +1085,7 @@ def train_cybersecurity_owasp_grpo(
     import transformers.utils.hub as transformers_hub
     from datasets import Dataset
     from huggingface_hub import snapshot_download, whoami
+    from peft import PeftModel
     from transformers import TrainerCallback
     from trl import GRPOConfig, GRPOTrainer, clone_chat_template
     from trl.chat_template_utils import add_response_schema
@@ -1110,14 +1146,22 @@ def train_cybersecurity_owasp_grpo(
 
     os.environ["TRACKIO_SPACE_ID"] = trackio_space_id
     os.environ["TRACKIO_PROJECT"] = trackio_project
-    os.environ.setdefault("CYBERSECURITY_OWASP_REWARD_MODE", "dense_train")
+    reward_env = _configure_reward_env(
+        reward_config=reward_config,
+        reward_variant=reward_variant,
+        reward_mode="dense_train",
+    )
     reward_settings = load_reward_settings()
     reward_tracking_config = reward_config_trackio_config(reward_settings)
+    reward_tracking_config["reward_variant"] = reward_variant or "default"
+    reward_tracking_config["reward_config_path"] = reward_config or reward_settings.source_path
 
     model_slug = model_name.replace("/", "-")
     stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
     run_name = run_name or (
-        f"CyberSecurity_OWASP-{model_slug}-grpo-level{difficulty}-{stamp}-{git_sha[:8]}"
+        f"CyberSecurity_OWASP-{model_slug}-grpo-level{difficulty}-"
+        f"{reward_tracking_config['reward_variant']}-steps{max_steps}-seed{seed_start}-"
+        f"{stamp}-{git_sha[:8]}"
     )
     output_dir = RUNS_DIR / run_name
     output_dir.mkdir(parents=True, exist_ok=True)
@@ -1253,6 +1297,7 @@ def train_cybersecurity_owasp_grpo(
                     "reward_config_hash": reward_tracking_config["reward_config_hash"],
                     "reward_stage": reward_tracking_config["reward_stage"],
                     "reward_mode": reward_tracking_config["reward_mode"],
+                    "reward_variant": reward_tracking_config["reward_variant"],
                 }
             )
             return obs.scenario_prompt
@@ -1613,6 +1658,7 @@ def train_cybersecurity_owasp_grpo(
                         "reward_config_hash": reward_tracking_config["reward_config_hash"],
                         "reward_stage": reward_tracking_config["reward_stage"],
                         "reward_mode": reward_tracking_config["reward_mode"],
+                        "reward_variant": reward_tracking_config["reward_variant"],
                     }
                 )
                 try:
@@ -1704,6 +1750,9 @@ def train_cybersecurity_owasp_grpo(
     print(f"Run name: {run_name}")
     print(f"Reward config: {reward_tracking_config['reward_config_id']}")
     print(f"Reward config hash: {reward_tracking_config['reward_config_hash']}")
+    print(f"Reward variant: {reward_tracking_config['reward_variant']}")
+    print(f"Reward config path: {reward_tracking_config['reward_config_path']}")
+    print(f"Reward env overrides: {reward_env}")
     print(f"Model cache volume: {CACHE_VOLUME_NAME}")
     print(f"Scenario cache volume: {SCENARIO_CACHE_VOLUME_NAME}")
     print(f"Scenario cache dir: {scenario_cache_env['CYBERSECURITY_OWASP_SCENARIO_CACHE_DIR']}")
@@ -1715,6 +1764,10 @@ def train_cybersecurity_owasp_grpo(
     print(f"Unsloth cache: {cache_env['UNSLOTH_CACHE_DIR']}")
     print(f"Triton cache: {cache_env['TRITON_CACHE_DIR']}")
     print(f"Hub push enabled: {push_to_hub}")
+    if initial_adapter_path:
+        print(f"Initial SFT adapter path: {initial_adapter_path}")
+    if initial_adapter_repo_id:
+        print(f"Initial SFT adapter repo: https://huggingface.co/{initial_adapter_repo_id}")
     print(
         "GRPO throughput config: "
         f"per_device_train_batch_size={per_device_train_batch_size}, "
@@ -1801,25 +1854,40 @@ def train_cybersecurity_owasp_grpo(
             f"{exc!r}"
         )
 
-    model = model_api.get_peft_model(
-        model,
-        r=lora_rank,
-        target_modules=[
-            "q_proj",
-            "k_proj",
-            "v_proj",
-            "o_proj",
-            "gate_proj",
-            "up_proj",
-            "down_proj",
-        ],
-        lora_alpha=lora_rank * 2,
-        use_gradient_checkpointing="unsloth",
-        random_state=3407,
-    )
+    adapter_source = initial_adapter_path
+    if initial_adapter_repo_id:
+        print(f"Downloading initial SFT adapter: {initial_adapter_repo_id}")
+        adapter_source = snapshot_download(
+            repo_id=initial_adapter_repo_id,
+            cache_dir=str(HF_HUB_CACHE_DIR),
+            token=hf_token,
+        )
+        cache_volume.commit()
+    if adapter_source:
+        print(f"Loading initial SFT adapter for trainable GRPO continuation: {adapter_source}")
+        model = PeftModel.from_pretrained(model, adapter_source, is_trainable=True)
+        if hasattr(model, "print_trainable_parameters"):
+            model.print_trainable_parameters()
+    else:
+        model = model_api.get_peft_model(
+            model,
+            r=lora_rank,
+            target_modules=[
+                "q_proj",
+                "k_proj",
+                "v_proj",
+                "o_proj",
+                "gate_proj",
+                "up_proj",
+                "down_proj",
+            ],
+            lora_alpha=lora_rank * 2,
+            use_gradient_checkpointing="unsloth",
+            random_state=3407,
+        )
     if hasattr(model_api, "for_training"):
         model_api.for_training(model)
-    print("LoRA adapter attached and model switched to training mode.")
+    print("LoRA adapter ready and model switched to training mode.")
 
     grpo_config_values = {
         "temperature": 1.0,
@@ -1942,6 +2010,8 @@ def train_cybersecurity_owasp_grpo(
         "difficulty": difficulty,
         "split": split,
         "model_name": model_name,
+        "initial_adapter_path": initial_adapter_path,
+        "initial_adapter_repo_id": initial_adapter_repo_id,
         "max_completion_length": max_completion_length,
         "num_generations": num_generations,
         "per_device_train_batch_size": per_device_train_batch_size,
@@ -1956,6 +2026,7 @@ def train_cybersecurity_owasp_grpo(
         "push_to_hub": push_to_hub,
         "scenario_cache_volume": SCENARIO_CACHE_VOLUME_NAME,
         "scenario_cache_mode": "require",
+        "reward_variant": reward_tracking_config["reward_variant"],
         **reward_tracking_config,
     }
 
@@ -1965,6 +2036,8 @@ def main(
     mode: str = "train",
     env_repo_id: str = "",
     output_repo_id: str = "",
+    initial_adapter_path: str = "",
+    initial_adapter_repo_id: str = "",
     max_steps: int = 10,
     dataset_size: int = 16,
     difficulty: int = 0,
@@ -1989,6 +2062,8 @@ def main(
     repo_branch: str = PUBLIC_REPO_BRANCH,
     detach: bool = False,
     push_to_hub: bool = False,
+    reward_config: str = "",
+    reward_variant: str = "",
     cache_seed_start: int = 0,
     cache_difficulty_buckets: int = 0,
     cache_train_per_bucket: int = 0,
@@ -2042,6 +2117,8 @@ def main(
             source_mode=source_mode,
             repo_url=repo_url,
             repo_branch=repo_branch,
+            reward_config=reward_config,
+            reward_variant=reward_variant,
         )
         if detach:
             call = run_cybersecurity_owasp_baseline.spawn(**kwargs)
@@ -2100,7 +2177,13 @@ def main(
     if git_sha == "nogit":
         try:
             git_sha = subprocess.check_output(
-                ["git", "rev-parse", "HEAD"],
+                [
+                    "git",
+                    "-c",
+                    f"safe.directory={PROJECT_ROOT.as_posix()}",
+                    "rev-parse",
+                    "HEAD",
+                ],
                 cwd=PROJECT_ROOT,
                 text=True,
                 stderr=subprocess.DEVNULL,
@@ -2110,12 +2193,15 @@ def main(
 
     model_slug = model_name.replace("/", "-")
     local_stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    variant_tag = reward_variant or "default"
     run_name = run_name or (
         f"CyberSecurity_OWASP-{model_slug}-grpo-level{difficulty}-"
-        f"{local_stamp}-{git_sha[:8]}"
+        f"{variant_tag}-steps{max_steps}-seed{seed_start}-{local_stamp}-{git_sha[:8]}"
     )
 
     print(f"Run name: {run_name}")
+    print(f"Reward variant: {variant_tag}")
+    print(f"Reward config path: {reward_config or '(default training/configs/grpo_small.yaml)'}")
     print(f"Source mode: {source_mode}")
     if source_mode == "public":
         print(f"Public repo: {repo_url}@{repo_branch}")
@@ -2131,6 +2217,10 @@ def main(
             f"<hf-user>/CyberSecurity_OWASP-{_model_repo_slug(model_name)}-grpo-lora"
         )
     print(f"Hub push enabled: {push_to_hub}")
+    if initial_adapter_path:
+        print(f"Initial SFT adapter path: {initial_adapter_path}")
+    if initial_adapter_repo_id:
+        print(f"Initial SFT adapter repo: https://huggingface.co/{initial_adapter_repo_id}")
     print(f"Model cache volume: {CACHE_VOLUME_NAME}")
     print(f"Scenario cache volume: {SCENARIO_CACHE_VOLUME_NAME}")
     print(
@@ -2164,6 +2254,8 @@ def main(
     kwargs = dict(
         env_repo_id=env_repo_id,
         output_repo_id=output_repo_id,
+        initial_adapter_path=initial_adapter_path,
+        initial_adapter_repo_id=initial_adapter_repo_id,
         max_steps=max_steps,
         dataset_size=dataset_size,
         difficulty=difficulty,
@@ -2187,6 +2279,8 @@ def main(
         repo_url=repo_url,
         repo_branch=repo_branch,
         push_to_hub=push_to_hub,
+        reward_config=reward_config,
+        reward_variant=reward_variant,
     )
     preflight = verify_modal_scenario_cache_for_training.remote(
         split=split,

scripts/modal_train_sft.py

@@ -0,0 +1,442 @@
+"""Modal SFT launcher for CyberSecurity_OWASP action JSON data.
+
+This trains a LoRA adapter on chat JSONL generated by
+``scripts/generate_sft_dataset.py``.  It intentionally mirrors the repo's Modal
+training pattern: local execution only launches remote jobs, while training runs
+inside Modal and saves adapters to the persistent run volume.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import pathlib
+import subprocess
+from datetime import datetime, timezone
+from typing import Any
+
+import modal
+
+
+APP_NAME = "CyberSecurity_OWASP-sft"
+VOLUME_NAME = "CyberSecurity_OWASP-grpo-runs"
+CACHE_VOLUME_NAME = "CyberSecurity_OWASP-model-cache"
+SECRET_NAME = "CyberSecurity_OWASP-secrets"
+RUNS_DIR = pathlib.Path("/runs")
+CACHE_DIR = pathlib.Path("/cache")
+HF_HOME_DIR = CACHE_DIR / "huggingface"
+HF_HUB_CACHE_DIR = HF_HOME_DIR / "hub"
+TORCH_HOME_DIR = CACHE_DIR / "torch"
+XDG_CACHE_DIR = CACHE_DIR / "xdg"
+UNSLOTH_CACHE_DIR = CACHE_DIR / "unsloth"
+TRITON_CACHE_DIR = CACHE_DIR / "triton"
+REMOTE_PROJECT = "/root/CyberSecurity_OWASP"
+PROJECT_ROOT = pathlib.Path(__file__).resolve().parents[1]
+DEFAULT_GEMMA_MODEL = "unsloth/gemma-4-E2B-it"
+PUBLIC_REPO_URL = "https://github.com/humandotlearning/CyberSecurity_OWASP.git"
+PUBLIC_REPO_BRANCH = "master"
+
+
+def _ensure_gemma4_model(model_name: str) -> str:
+    if model_name != DEFAULT_GEMMA_MODEL:
+        raise ValueError(
+            "CyberSecurity_OWASP SFT is pinned to "
+            f"{DEFAULT_GEMMA_MODEL}; received {model_name!r}."
+        )
+    return model_name
+
+
+def _model_repo_slug(model_name: str) -> str:
+    return model_name.replace("/", "-").replace("_", "-").replace(".", "-").lower()
+
+
+def _configure_modal_cache_env() -> dict[str, str]:
+    values = {
+        "HF_HOME": str(HF_HOME_DIR),
+        "HF_HUB_CACHE": str(HF_HUB_CACHE_DIR),
+        "TRANSFORMERS_CACHE": str(HF_HUB_CACHE_DIR),
+        "TORCH_HOME": str(TORCH_HOME_DIR),
+        "XDG_CACHE_HOME": str(XDG_CACHE_DIR),
+        "UNSLOTH_CACHE_DIR": str(UNSLOTH_CACHE_DIR),
+        "UNSLOTH_COMPILE_CACHE": str(UNSLOTH_CACHE_DIR / "compile"),
+        "TRITON_CACHE_DIR": str(TRITON_CACHE_DIR),
+    }
+    for key, value in values.items():
+        os.environ[key] = value
+    for path in {
+        CACHE_DIR,
+        HF_HOME_DIR,
+        HF_HUB_CACHE_DIR,
+        TORCH_HOME_DIR,
+        XDG_CACHE_DIR,
+        UNSLOTH_CACHE_DIR,
+        UNSLOTH_CACHE_DIR / "compile",
+        TRITON_CACHE_DIR,
+    }:
+        path.mkdir(parents=True, exist_ok=True)
+    return values
+
+
+def _cli_arg_value(name: str, default: str = "") -> str:
+    import sys
+
+    args = sys.argv[1:]
+    flag = f"--{name}"
+    for index, arg in enumerate(args):
+        if arg == flag and index + 1 < len(args):
+            return args[index + 1]
+        if arg.startswith(f"{flag}="):
+            return arg.split("=", 1)[1]
+    return default
+
+
+def _source_mode() -> str:
+    return _cli_arg_value("source-mode", os.environ.get("MODAL_SOURCE_MODE", "local"))
+
+
+def _training_image() -> modal.Image:
+    image = (
+        modal.Image.from_registry(
+            "nvidia/cuda:12.8.0-devel-ubuntu22.04",
+            add_python="3.11",
+        )
+        .apt_install("git", "build-essential", "curl")
+        .uv_pip_install(
+            "torch==2.10.0",
+            "triton>=3.4.0",
+            "torchvision==0.25.0",
+            "bitsandbytes",
+            "accelerate",
+            "datasets",
+            "huggingface_hub",
+            "peft",
+            "tokenizers",
+            "trackio>=0.25.0",
+            "transformers>=5.5.0",
+            "trl>=0.28.0",
+        )
+        .uv_pip_install(
+            "unsloth_zoo[base] @ git+https://github.com/unslothai/unsloth-zoo",
+            "unsloth[base] @ git+https://github.com/unslothai/unsloth",
+        )
+        .uv_pip_install("timm", extra_options="--no-deps")
+        .uv_pip_install("pydantic==2.10.6")
+    )
+    if _source_mode() == "public":
+        repo_url = _cli_arg_value("repo-url", PUBLIC_REPO_URL)
+        repo_branch = _cli_arg_value("repo-branch", PUBLIC_REPO_BRANCH)
+        image = image.run_commands(
+            f"git clone --depth 1 --branch {repo_branch} {repo_url} {REMOTE_PROJECT}",
+            f"python -m pip install --no-deps -e {REMOTE_PROJECT}",
+        )
+    else:
+        image = image.add_local_dir(
+            PROJECT_ROOT,
+            remote_path=REMOTE_PROJECT,
+            copy=True,
+            ignore=[
+                ".git",
+                ".venv",
+                ".env",
+                ".env.*",
+                "__pycache__",
+                ".pytest_cache",
+                "outputs",
+                "*.pyc",
+            ],
+        )
+        image = image.run_commands(f"python -m pip install --no-deps -e {REMOTE_PROJECT}")
+    return image.workdir(REMOTE_PROJECT)
+
+
+app = modal.App(APP_NAME)
+volume = modal.Volume.from_name(VOLUME_NAME, create_if_missing=True)
+cache_volume = modal.Volume.from_name(CACHE_VOLUME_NAME, create_if_missing=True)
+training_image = _training_image()
+secrets = [modal.Secret.from_name(SECRET_NAME, required_keys=["HF_TOKEN"])]
+
+
+@app.function(
+    image=modal.Image.debian_slim(python_version="3.11"),
+    timeout=60 * 20,
+    volumes={RUNS_DIR: volume},
+)
+def upload_sft_jsonl(relative_path: str, content: str) -> str:
+    target = RUNS_DIR / relative_path
+    target.parent.mkdir(parents=True, exist_ok=True)
+    target.write_text(content, encoding="utf-8")
+    volume.commit()
+    return str(target)
+
+
+@app.function(
+    image=training_image,
+    gpu="L4",
+    timeout=12 * 60 * 60,
+    volumes={RUNS_DIR: volume, CACHE_DIR: cache_volume},
+    secrets=secrets,
+)
+def train_cybersecurity_owasp_sft(
+    train_jsonl: str = "/runs/sft/train.jsonl",
+    validation_jsonl: str = "/runs/sft/validation.jsonl",
+    output_repo_id: str = "",
+    model_name: str = DEFAULT_GEMMA_MODEL,
+    run_name: str = "",
+    max_seq_length: int = 4096,
+    max_steps: int = 100,
+    num_train_epochs: float = 1.0,
+    per_device_train_batch_size: int = 1,
+    gradient_accumulation_steps: int = 16,
+    learning_rate: float = 2e-5,
+    lora_rank: int = 32,
+    trackio_space_id: str = "Humanlearning/CyberSecurity_OWASP-trackio",
+    trackio_project: str = "CyberSecurity_OWASP-sft",
+    push_to_hub: bool = False,
+) -> dict[str, Any]:
+    import inspect
+
+    from datasets import load_dataset
+    from huggingface_hub import snapshot_download, whoami
+    from trl import SFTConfig, SFTTrainer
+    from trl.chat_template_utils import add_response_schema
+    from unsloth import FastVisionModel
+
+    model_name = _ensure_gemma4_model(model_name)
+    cache_env = _configure_modal_cache_env()
+    hf_token = os.environ.get("HF_TOKEN")
+    if not hf_token:
+        raise RuntimeError(f"HF_TOKEN is missing from the Modal secret {SECRET_NAME}.")
+
+    user = whoami(token=hf_token)["name"]
+    output_repo_id = output_repo_id or (
+        f"{user}/CyberSecurity_OWASP-{_model_repo_slug(model_name)}-sft-lora"
+    )
+    stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+    run_name = run_name or f"CyberSecurity_OWASP-{_model_repo_slug(model_name)}-sft-{stamp}"
+    output_dir = RUNS_DIR / run_name
+    adapter_dir = output_dir / "sft_adapter"
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    data_files = {"train": train_jsonl}
+    validation_path = pathlib.Path(validation_jsonl)
+    has_validation = validation_path.exists() and validation_path.stat().st_size > 0
+    if has_validation:
+        data_files["validation"] = validation_jsonl
+    dataset = load_dataset("json", data_files=data_files)
+
+    print(f"SFT run name: {run_name}")
+    print(f"Model: {model_name}")
+    print(f"Train JSONL: {train_jsonl}")
+    print(f"Validation JSONL: {validation_jsonl if has_validation else '(none)'}")
+    print(f"Output adapter dir: {adapter_dir}")
+    print(f"Output repo: https://huggingface.co/{output_repo_id}")
+    print(f"Trackio Space: https://huggingface.co/spaces/{trackio_space_id}")
+    print(f"HF_HUB_CACHE: {cache_env['HF_HUB_CACHE']}")
+
+    try:
+        snapshot_download(repo_id=model_name, cache_dir=str(HF_HUB_CACHE_DIR), token=hf_token)
+        cache_volume.commit()
+    except Exception as exc:
+        print(f"Model snapshot prefetch skipped; loader will retry directly. Error: {exc!r}")
+
+    model_api = FastVisionModel
+    model, tokenizer = model_api.from_pretrained(
+        model_name=model_name,
+        max_seq_length=max_seq_length,
+        load_in_4bit=False,
+        fast_inference=False,
+        cache_dir=str(HF_HUB_CACHE_DIR),
+        token=hf_token,
+    )
+    try:
+        tokenizer = add_response_schema(tokenizer)
+    except Exception as exc:
+        print(f"Tokenizer response schema add skipped: {exc!r}")
+
+    model = model_api.get_peft_model(
+        model,
+        r=lora_rank,
+        target_modules=[
+            "q_proj",
+            "k_proj",
+            "v_proj",
+            "o_proj",
+            "gate_proj",
+            "up_proj",
+            "down_proj",
+        ],
+        lora_alpha=lora_rank * 2,
+        use_gradient_checkpointing="unsloth",
+        random_state=3407,
+    )
+    if hasattr(model_api, "for_training"):
+        model_api.for_training(model)
+
+    sft_values = {
+        "output_dir": str(output_dir),
+        "max_seq_length": max_seq_length,
+        "max_steps": max_steps,
+        "num_train_epochs": num_train_epochs,
+        "per_device_train_batch_size": per_device_train_batch_size,
+        "gradient_accumulation_steps": gradient_accumulation_steps,
+        "learning_rate": learning_rate,
+        "logging_steps": 1,
+        "save_steps": max(10, max_steps),
+        "report_to": "trackio",
+        "project": trackio_project,
+        "trackio_space_id": trackio_space_id,
+        "run_name": run_name,
+        "assistant_only_loss": True,
+        "packing": False,
+        "gradient_checkpointing": True,
+        "gradient_checkpointing_kwargs": {"use_reentrant": False},
+        "push_to_hub": push_to_hub,
+        "hub_model_id": output_repo_id,
+        "hub_private_repo": True,
+    }
+    sft_parameters = set(inspect.signature(SFTConfig).parameters)
+    skipped = sorted(set(sft_values) - sft_parameters)
+    if skipped:
+        print(f"Skipping unsupported SFTConfig keys: {skipped}")
+    training_args = SFTConfig(
+        **{key: value for key, value in sft_values.items() if key in sft_parameters}
+    )
+
+    trainer_values = {
+        "model": model,
+        "processing_class": tokenizer,
+        "args": training_args,
+        "train_dataset": dataset["train"],
+        "eval_dataset": dataset["validation"] if has_validation else None,
+    }
+    trainer_parameters = set(inspect.signature(SFTTrainer).parameters)
+    skipped_trainer = sorted(
+        key for key, value in trainer_values.items() if key not in trainer_parameters and value is not None
+    )
+    if skipped_trainer:
+        print(f"Skipping unsupported SFTTrainer keys: {skipped_trainer}")
+    trainer = SFTTrainer(
+        **{
+            key: value
+            for key, value in trainer_values.items()
+            if value is not None and key in trainer_parameters
+        }
+    )
+    trainer.train()
+    trainer.save_model(str(adapter_dir))
+    if push_to_hub:
+        trainer.push_to_hub()
+    volume.commit()
+    cache_volume.commit()
+    return {
+        "run_name": run_name,
+        "model_name": model_name,
+        "adapter_dir": str(adapter_dir),
+        "output_repo_id": output_repo_id,
+        "train_jsonl": train_jsonl,
+        "validation_jsonl": validation_jsonl if has_validation else "",
+        "max_steps": max_steps,
+        "push_to_hub": push_to_hub,
+        "trackio_space_id": trackio_space_id,
+        "trackio_project": trackio_project,
+    }
+
+
+def _git_sha(default: str = "nogit") -> str:
+    try:
+        return subprocess.check_output(
+            [
+                "git",
+                "-c",
+                f"safe.directory={PROJECT_ROOT.as_posix()}",
+                "rev-parse",
+                "HEAD",
+            ],
+            cwd=PROJECT_ROOT,
+            text=True,
+            stderr=subprocess.DEVNULL,
+        ).strip()
+    except Exception:
+        return default
+
+
+@app.local_entrypoint()
+def main(
+    mode: str = "train",
+    local_train_path: str = "outputs/sft/train.jsonl",
+    local_validation_path: str = "outputs/sft/validation.jsonl",
+    train_jsonl: str = "/runs/sft/train.jsonl",
+    validation_jsonl: str = "/runs/sft/validation.jsonl",
+    output_repo_id: str = "",
+    model_name: str = DEFAULT_GEMMA_MODEL,
+    run_name: str = "",
+    max_seq_length: int = 4096,
+    max_steps: int = 100,
+    num_train_epochs: float = 1.0,
+    per_device_train_batch_size: int = 1,
+    gradient_accumulation_steps: int = 16,
+    learning_rate: float = 2e-5,
+    lora_rank: int = 32,
+    trackio_space_id: str = "Humanlearning/CyberSecurity_OWASP-trackio",
+    trackio_project: str = "CyberSecurity_OWASP-sft",
+    source_mode: str = "local",
+    repo_url: str = PUBLIC_REPO_URL,
+    repo_branch: str = PUBLIC_REPO_BRANCH,
+    detach: bool = False,
+    push_to_hub: bool = False,
+) -> None:
+    del source_mode, repo_url, repo_branch  # consumed during image construction
+    model_name = _ensure_gemma4_model(model_name)
+    if mode not in {"upload", "train"}:
+        raise ValueError("mode must be 'upload' or 'train'")
+
+    local_train = pathlib.Path(local_train_path)
+    local_validation = pathlib.Path(local_validation_path)
+    if local_train.exists():
+        uploaded = upload_sft_jsonl.remote(
+            "sft/train.jsonl",
+            local_train.read_text(encoding="utf-8"),
+        )
+        print(f"Uploaded train JSONL: {uploaded}")
+        train_jsonl = uploaded
+    if local_validation.exists():
+        uploaded_validation = upload_sft_jsonl.remote(
+            "sft/validation.jsonl",
+            local_validation.read_text(encoding="utf-8"),
+        )
+        print(f"Uploaded validation JSONL: {uploaded_validation}")
+        validation_jsonl = uploaded_validation
+    if mode == "upload":
+        return
+
+    if not run_name:
+        stamp = datetime.now(timezone.utc).strftime("%Y%m%d-%H%M%S")
+        run_name = f"CyberSecurity_OWASP-{_model_repo_slug(model_name)}-sft-{stamp}-{_git_sha()[:8]}"
+
+    kwargs = dict(
+        train_jsonl=train_jsonl,
+        validation_jsonl=validation_jsonl,
+        output_repo_id=output_repo_id,
+        model_name=model_name,
+        run_name=run_name,
+        max_seq_length=max_seq_length,
+        max_steps=max_steps,
+        num_train_epochs=num_train_epochs,
+        per_device_train_batch_size=per_device_train_batch_size,
+        gradient_accumulation_steps=gradient_accumulation_steps,
+        learning_rate=learning_rate,
+        lora_rank=lora_rank,
+        trackio_space_id=trackio_space_id,
+        trackio_project=trackio_project,
+        push_to_hub=push_to_hub,
+    )
+    print(f"SFT run name: {run_name}")
+    print(f"Train JSONL: {train_jsonl}")
+    print(f"Validation JSONL: {validation_jsonl}")
+    print(f"Hub push enabled: {push_to_hub}")
+    if detach:
+        call = train_cybersecurity_owasp_sft.spawn(**kwargs)
+        print(f"Spawned Modal SFT call: {call.object_id}")
+    else:
+        result = train_cybersecurity_owasp_sft.remote(**kwargs)
+        print(json.dumps(result, indent=2, sort_keys=True))

tests/test_reward_config.py

@@ -68,6 +68,45 @@ def test_reward_config_hash_and_flattened_values_are_deterministic(monkeypatch):
     assert rows["hidden_file_probe"]["terminate"] is True
 
 
+def test_reward_ablation_configs_extend_default_and_have_unique_hashes(monkeypatch):
+    monkeypatch.setenv("CYBERSECURITY_OWASP_REWARD_MODE", "dense_train")
+    paths = [
+        Path("training/configs/reward_ablations/A0_sparse_terminal_only.yaml"),
+        Path("training/configs/reward_ablations/A2_reduced_shaping.yaml"),
+        Path("training/configs/reward_ablations/A6_visible_gate.yaml"),
+        Path("training/configs/reward_ablations/A7_evidence045.yaml"),
+        Path("training/configs/reward_ablations/A3_no_speed_token.yaml"),
+    ]
+
+    settings_by_name = {path.name: load_reward_settings(path) for path in paths}
+    hashes = {reward_config_hash(settings) for settings in settings_by_name.values()}
+
+    assert len(hashes) == len(paths)
+    assert settings_by_name["A0_sparse_terminal_only.yaml"].shaping_weight == 0.0
+    assert settings_by_name["A0_sparse_terminal_only.yaml"].value("progressive_cap") == 0.0
+    assert settings_by_name["A0_sparse_terminal_only.yaml"].value("terminal_cap") == 12.0
+    assert settings_by_name["A2_reduced_shaping.yaml"].shaping_weight == 0.35
+    assert settings_by_name["A2_reduced_shaping.yaml"].value("progressive_cap") == 2.5
+    assert settings_by_name["A6_visible_gate.yaml"].value("visible_tests_improved") == 0.0
+    assert settings_by_name["A6_visible_gate.yaml"].value("app_boots_after_patch") == 0.10
+    assert settings_by_name["A7_evidence045.yaml"].value("local_evidence_found") == 0.45
+    assert settings_by_name["A3_no_speed_token.yaml"].value("speed_bonus") == 0.0
+    assert compute_token_penalty(850, settings_by_name["A3_no_speed_token.yaml"]) == 0.0
+
+
+def test_reward_config_run_config_includes_variant(monkeypatch):
+    monkeypatch.setenv("CYBERSECURITY_OWASP_REWARD_MODE", "dense_train")
+    monkeypatch.setenv("CYBERSECURITY_OWASP_REWARD_VARIANT", "abl-a2-shape035")
+
+    config = reward_config_run_config(
+        load_reward_settings("training/configs/reward_ablations/A2_reduced_shaping.yaml")
+    )
+
+    assert config["reward_variant"] == "abl-a2-shape035"
+    assert config["reward_config_source_name"] == "A2_reduced_shaping.yaml"
+    assert config["reward_config__shaping_weight__stage_value"] == 0.35
+
+
 def test_reward_config_rejects_missing_descriptions(monkeypatch):
     config_path = Path("outputs/test_reward_config_bad.yaml")
     config_path.parent.mkdir(parents=True, exist_ok=True)

tests/test_sft_dataset_generation.py

@@ -0,0 +1,142 @@
+import importlib.util
+import json
+import os
+import sys
+import uuid
+from pathlib import Path
+
+from CyberSecurity_OWASP.models import CyberSecurityOWASPAction
+from CyberSecurity_OWASP.server.CyberSecurity_OWASP_environment import (
+    CybersecurityOwaspEnvironment,
+)
+
+
+MODULE_PATH = Path(__file__).resolve().parents[1] / "scripts" / "generate_sft_dataset.py"
+SPEC = importlib.util.spec_from_file_location("generate_sft_dataset", MODULE_PATH)
+generate_sft_dataset = importlib.util.module_from_spec(SPEC)
+assert SPEC.loader is not None
+sys.modules[SPEC.name] = generate_sft_dataset
+SPEC.loader.exec_module(generate_sft_dataset)
+
+
+def _isolated_out_dir(label: str) -> Path:
+    root = Path("outputs") / "sft_dataset_tests" / f"{label}_{uuid.uuid4().hex[:8]}"
+    workspace_root = root / "workspaces"
+    workspace_root.mkdir(parents=True, exist_ok=True)
+    os.environ["CYBERSECURITY_OWASP_WORKSPACE_ROOT"] = str(workspace_root)
+    return root / "sft"
+
+
+def test_extracts_and_validates_action_json():
+    action = generate_sft_dataset.parse_action_text(
+        '```json\n{"tool_name":"inspect_policy_graph","arguments":{}}\n```'
+    )
+
+    assert isinstance(action, CyberSecurityOWASPAction)
+    assert action.tool_name == "inspect_policy_graph"
+
+
+def test_prompt_uses_visible_observation_only():
+    _isolated_out_dir("prompt")
+    env = CybersecurityOwaspEnvironment()
+    try:
+        obs = env.reset(seed=501, split="train", difficulty=0)
+        prompt = generate_sft_dataset.build_user_prompt(obs, [])
+    finally:
+        env.close()
+
+    lowered = prompt.lower()
+    assert "hidden_facts" not in lowered
+    assert "oracle_hidden_focus" not in lowered
+    assert "reward_engine" not in lowered
+    assert "validators.py" not in lowered
+    assert "tests/hidden" not in lowered
+    assert "hidden tests" not in lowered
+
+
+def test_chat_row_matches_conversational_sft_shape():
+    _isolated_out_dir("chat_row")
+    env = CybersecurityOwaspEnvironment()
+    try:
+        obs = env.reset(seed=502, split="train", difficulty=0)
+        messages = generate_sft_dataset.build_chat_messages(obs, [])
+        action = CyberSecurityOWASPAction(tool_name="inspect_policy_graph", arguments={})
+        row = generate_sft_dataset.make_chat_row(
+            messages=messages,
+            action=action,
+            metadata={
+                "target_model": generate_sft_dataset.DEFAULT_TARGET_MODEL,
+                "teacher_model": generate_sft_dataset.DEFAULT_TEACHER_MODEL,
+                "seed": 502,
+            },
+        )
+    finally:
+        env.close()
+
+    assert [message["role"] for message in row["messages"]] == [
+        "system",
+        "user",
+        "assistant",
+    ]
+    assert json.loads(row["messages"][-1]["content"]) == action.model_dump()
+    assert row["metadata"]["target_model"] == "unsloth/gemma-4-E2B-it"
+
+
+def test_dry_run_oracle_creates_chat_jsonl_without_network():
+    out_dir = _isolated_out_dir("dry_run")
+    manifest = generate_sft_dataset.generate_dataset(
+        generate_sft_dataset.DatasetConfig(
+            episodes=2,
+            validation_episodes=1,
+            out_dir=out_dir,
+            dry_run_oracle=True,
+        )
+    )
+
+    assert manifest["episodes_attempted"] == 3
+    assert manifest["episodes_accepted"] == 3
+    assert (out_dir / "train.jsonl").exists()
+    assert (out_dir / "validation.jsonl").exists()
+    train_rows = [
+        json.loads(line)
+        for line in (out_dir / "train.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    validation_rows = [
+        json.loads(line)
+        for line in (out_dir / "validation.jsonl").read_text(encoding="utf-8").splitlines()
+        if line.strip()
+    ]
+    assert train_rows
+    assert validation_rows
+    assert all(row["messages"][-1]["role"] == "assistant" for row in train_rows)
+
+
+def test_saved_oracle_trajectory_replays_to_success():
+    out_dir = _isolated_out_dir("replay")
+    generate_sft_dataset.generate_dataset(
+        generate_sft_dataset.DatasetConfig(
+            episodes=1,
+            out_dir=out_dir,
+            dry_run_oracle=True,
+        )
+    )
+    trajectory_path = next((out_dir / "trajectories").glob("train_seed*.json"))
+    trajectory = json.loads(trajectory_path.read_text(encoding="utf-8"))
+
+    env = CybersecurityOwaspEnvironment()
+    try:
+        env.reset(
+            seed=int(trajectory["seed"]),
+            split=trajectory["split"],
+            difficulty=int(trajectory["difficulty"]),
+        )
+        final = None
+        for action_data in trajectory["actions"]:
+            final = env.step(CyberSecurityOWASPAction(**action_data))
+        assert final is not None
+        assert final.done is True
+        assert env.state.success is True
+        assert not env.state.anti_cheat_flags
+    finally:
+        env.close()

tests/test_trackio_utils.py

@@ -39,6 +39,10 @@ def test_canonical_tracking_fields_exist_and_are_numeric_where_expected():
         assert isinstance(fields["reward/hidden_authz_pass_rate"], float)
         assert isinstance(fields["reward/normal_flow_pass_rate"], float)
         assert isinstance(fields["reward/public_hidden_gap"], float)
+        assert isinstance(fields["reward/dense_to_terminal_ratio"], float)
+        assert isinstance(fields["episode/time_to_first_patch"], float)
+        assert isinstance(fields["episode/repeated_action_rate"], float)
+        assert isinstance(fields["episode/patch_to_hidden_success_conversion_rate"], float)
         assert isinstance(fields["skill/exploit_to_patch_alignment"], float)
 
         metrics = aggregate_episode_metrics([record])
@@ -156,11 +160,13 @@ def test_log_reward_config_emits_scalar_values_and_table(monkeypatch):
     monkeypatch.setitem(sys.modules, "trackio", fake_trackio)
     monkeypatch.setenv("CYBERSECURITY_OWASP_REWARD_MODE", "dense_train")
     monkeypatch.setenv("CYBERSECURITY_OWASP_REWARD_STAGE", "early")
+    monkeypatch.setenv("CYBERSECURITY_OWASP_REWARD_VARIANT", "abl-test")
 
     settings = load_reward_settings()
     summary = log_reward_config(settings, step=0)
 
     assert fake_trackio.config["reward_config_hash"] == summary["reward_config_hash"]
+    assert fake_trackio.config["reward_variant"] == "abl-test"
     assert fake_trackio.config["reward_config_values"]["policy_inspected"]["value"] == 0.30
     assert fake_trackio.config["reward_config__policy_inspected__value"] == 0.30
     scalar_payload = next(payload for payload, _step in logged if "reward_config/policy_inspected/value" in payload)

training/configs/reward_ablations/A0_sparse_terminal_only.yaml

@@ -0,0 +1,97 @@
+extends: ../grpo_small.yaml
+reward:
+  stage: early
+  terminal_cap:
+    value: 12.0
+  progressive_cap:
+    value: 0.0
+  penalty_floor:
+    value: -10.0
+  train_cap:
+    value: 12.0
+  shaping_weight:
+    early: 0.0
+    middle: 0.0
+    late: 0.0
+    final: 0.0
+  policy_inspected:
+    value: 0.0
+  route_map_inspected:
+    value: 0.0
+    cap: 0.0
+  relevant_file_inspected:
+    value: 0.0
+    cap: 0.0
+  local_evidence_found:
+    value: 0.0
+    cap: 0.0
+  diagnosis_correct:
+    value: 0.0
+  patch_applies:
+    value: 0.0
+  app_boots_after_patch:
+    value: 0.0
+  visible_tests_improved:
+    value: 0.0
+    cap: 0.0
+  public_routes_visible_pass:
+    value: 0.0
+  step_penalty:
+    early: 0.0
+    middle: 0.0
+    late: 0.0
+    final: 0.0
+    cap: 0.0
+  speed_bonus:
+    value: 0.0
+  token_penalty:
+    early: 0.0
+    middle: 0.0
+    late: 0.0
+    final: 0.0
+    cap: 0.0
+  invalid_action:
+    value: 0.0
+  repeated_invalid_action:
+    value: 0.0
+  repeated_low_value_action:
+    value: 0.0
+  no_progress_action:
+    value: 0.0
+  noop_action:
+    value: 0.0
+  repeated_file_read:
+    value: 0.0
+  repeated_local_request:
+    value: 0.0
+  repeated_visible_tests:
+    value: 0.0
+  patch_before_policy:
+    value: 0.0
+  submit_without_patch:
+    value: 0.0
+  submit_without_visible_tests:
+    value: 0.0
+  excessive_patch_attempt:
+    value: 0.0
+  too_many_files_changed:
+    value: 0.0
+  oversized_patch:
+    value: 0.0
+    severe_value: 0.0
+  hidden_file_probe:
+    value: -10.0
+    terminate: true
+  test_file_edit:
+    value: -10.0
+    terminate: true
+  host_path_escape:
+    value: -10.0
+    terminate: true
+  external_url_attempt:
+    value: -10.0
+    terminate: true
+  hardcoded_identifier:
+    value: -4.0
+  deny_all_patch:
+    value: -5.0

training/configs/reward_ablations/A2_reduced_shaping.yaml

@@ -0,0 +1,12 @@
+extends: ../grpo_small.yaml
+reward:
+  stage: early
+  progressive_cap:
+    value: 2.5
+  train_cap:
+    value: 18.0
+  shaping_weight:
+    early: 0.35
+    middle: 0.35
+    late: 0.35
+    final: 0.35

training/configs/reward_ablations/A3_no_speed_token.yaml

@@ -0,0 +1,17 @@
+extends: ../grpo_small.yaml
+reward:
+  stage: early
+  step_penalty:
+    early: -0.002
+    middle: -0.002
+    late: -0.002
+    final: -0.002
+    cap: -0.25
+  speed_bonus:
+    value: 0.0
+  token_penalty:
+    early: 0.0
+    middle: 0.0
+    late: 0.0
+    final: 0.0
+    cap: 0.0

training/configs/reward_ablations/A6_visible_gate.yaml

@@ -0,0 +1,10 @@
+extends: ../grpo_small.yaml
+reward:
+  stage: early
+  app_boots_after_patch:
+    value: 0.10
+  visible_tests_improved:
+    value: 0.0
+    cap: 0.20
+  public_routes_visible_pass:
+    value: 0.10

training/configs/reward_ablations/A7_evidence045.yaml

@@ -0,0 +1,6 @@
+extends: ../grpo_small.yaml
+reward:
+  stage: early
+  local_evidence_found:
+    value: 0.45
+    cap: 0.45

training/trackio_utils.py

@@ -17,6 +17,7 @@ RUN_SCENARIO_FIELDS = (
     "run/base_model",
     "run/algo",
     "run/reward_version",
+    "run/reward_variant",
     "run/env_version",
     "scenario/seed",
     "scenario/template_id",
@@ -136,6 +137,16 @@ CANONICAL_TRACKIO_SIGNALS = tuple(
 
 DERIVED_TRACKIO_METRICS = (
     "reward/public_hidden_gap",
+    "reward/visible_hidden_gap",
+    "reward/dense_total",
+    "reward/dense_to_terminal_ratio",
+    "episode/time_to_first_evidence",
+    "episode/time_to_first_patch",
+    "episode/repeated_action_rate",
+    "episode/submit_without_evidence_rate",
+    "episode/hardcoded_identifier_rate",
+    "episode/deny_all_patch_rate",
+    "episode/patch_to_hidden_success_conversion_rate",
     "cheat/score",
 )
 
@@ -545,6 +556,7 @@ def episode_record_from_state(
         "run/reward_config_hash": context.get("reward_config_hash", ""),
         "run/reward_mode": context.get("reward_mode", ""),
         "run/reward_stage": context.get("reward_stage", ""),
+        "run/reward_variant": context.get("reward_variant", ""),
         "run/env_version": context.get("env_version", "0.1.0"),
         "episode_id": getattr(state, "episode_id", ""),
         "task_id": getattr(state, "task_id", ""),
@@ -663,6 +675,16 @@ def episode_to_tracking_fields(episode: Any) -> dict[str, Any]:
         -_float(final_reward.get("safety")),
     )
     fields["reward/public_hidden_gap"] = visible_rate - hidden_rate
+    fields["reward/visible_hidden_gap"] = fields["reward/public_hidden_gap"]
+    fields["reward/dense_total"] = (
+        fields["reward/total"] - fields["reward/terminal_15"]
+    )
+    terminal_denominator = abs(fields["reward/terminal_15"])
+    fields["reward/dense_to_terminal_ratio"] = (
+        fields["reward/dense_total"] / terminal_denominator
+        if terminal_denominator > 1e-9
+        else fields["reward/dense_total"]
+    )
 
     fields["skill/valid_action_rate"] = max(0.0, min(1.0, valid_action_rate))
     fields["skill/discovery_success"] = fields["reward/bug_classification_correct"]
@@ -698,11 +720,33 @@ def episode_to_tracking_fields(episode: Any) -> dict[str, Any]:
     fields["episode/patch_attempt_count"] = _float(
         record.get("patch_attempt_count", tool_names.count("patch_file"))
     )
+    fields["episode/time_to_first_evidence"] = fields["skill/first_valid_exploit_step"]
+    fields["episode/time_to_first_patch"] = _first_tool_step(
+        actions,
+        {"patch_file"},
+        observations,
+    )
+    action_keys = [
+        json.dumps(action, sort_keys=True, default=str)
+        for action in actions
+    ]
+    repeated_actions = len(action_keys) - len(set(action_keys))
+    fields["episode/repeated_action_rate"] = repeated_actions / max(1.0, float(len(action_keys)))
     fields["episode/submit_without_test_rate"] = 1.0 if (
         "submit_fix" in tool_names and "patch_file" in tool_names and "run_visible_tests" not in tool_names
     ) else 0.0
+    fields["episode/submit_without_evidence_rate"] = 1.0 if (
+        "submit_fix" in tool_names and fields["episode/time_to_first_evidence"] < 0.0
+    ) else 0.0
     fields["episode/cheat_attempt_rate"] = 1.0 if anti_flags else 0.0
     fields["episode/oversecure_rate"] = 1.0 if bool(quality.get("deny_all")) else 0.0
+    fields["episode/hardcoded_identifier_rate"] = 1.0 if (
+        "hardcoded_hidden_identifier" in anti_flags or bool(quality.get("hardcoded"))
+    ) else 0.0
+    fields["episode/deny_all_patch_rate"] = fields["episode/oversecure_rate"]
+    fields["episode/patch_to_hidden_success_conversion_rate"] = (
+        hidden_rate if "patch_file" in tool_names else 0.0
+    )
 
     fields["cheat/hidden_file_read_attempt"] = 1.0 if (
         "blocked_file_probe" in anti_flags and "hidden" in path_text
@@ -802,6 +846,15 @@ def train_metric_aliases(metrics: Mapping[str, Any]) -> dict[str, float]:
         "train/reward_token_penalty_mean": _float(metrics.get("reward/token_penalty")),
         "train/reward_speed_bonus_mean": _float(metrics.get("reward/speed_bonus")),
         "train/reward_behavior_penalty_mean": _float(metrics.get("reward/behavior_penalty")),
+        "train/dense_to_terminal_ratio": _float(metrics.get("reward/dense_to_terminal_ratio")),
+        "train/visible_hidden_gap": _float(metrics.get("reward/visible_hidden_gap")),
+        "train/repeated_action_rate": _float(metrics.get("episode/repeated_action_rate")),
+        "train/submit_without_evidence_rate": _float(metrics.get("episode/submit_without_evidence_rate")),
+        "train/hardcoded_identifier_rate": _float(metrics.get("episode/hardcoded_identifier_rate")),
+        "train/deny_all_patch_rate": _float(metrics.get("episode/deny_all_patch_rate")),
+        "train/patch_to_hidden_success_conversion_rate": _float(
+            metrics.get("episode/patch_to_hidden_success_conversion_rate")
+        ),
         "train/success_rate": _float(metrics.get("skill/patch_success")),
         "train/exploit_block_rate": _float(metrics.get("reward/hidden_authz_pass_rate")),
         "train/regression_preservation_rate": _float(metrics.get("reward/normal_flow_pass_rate")),