Day 4: grader system complete

scripts/run_grader.py

@@ -0,0 +1,131 @@
+"""
+Manual grader testing CLI.
+Run a simulated episode and score it with the official grader.
+
+Usage:
+    python scripts/run_grader.py --task single_crash --agent correct
+    python scripts/run_grader.py --task cascading_failure --agent wrong
+    python scripts/run_grader.py --task silent_degradation --agent correct
+    python scripts/run_grader.py --all
+"""
+import argparse
+import sys
+import os
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from server.environment import LogTriageEnvironment
+from server.models import TriageAction
+from server.graders import score_episode
+
+# ─── CORRECT AGENT SCRIPTS ────────────────────────────────────────────────────
+
+CORRECT_ACTIONS = {
+    "single_crash": [
+        TriageAction(action_type="classify_severity",   value="P1",                       confidence=0.95),
+        TriageAction(action_type="identify_root_cause", value="payment-service",           confidence=0.90),
+        TriageAction(action_type="remediate",           value="restart:payment-service",   confidence=0.85),
+        TriageAction(action_type="resolve",             value="resolved",                  confidence=1.00),
+    ],
+    "cascading_failure": [
+        TriageAction(action_type="classify_severity",   value="P1",                       confidence=0.90),
+        TriageAction(action_type="identify_root_cause", value="user-db",                  confidence=0.85),
+        TriageAction(action_type="remediate",           value="kill-query:user-db",        confidence=0.90),
+        TriageAction(action_type="resolve",             value="resolved",                  confidence=1.00),
+    ],
+    "silent_degradation": [
+        TriageAction(action_type="request_more_logs",   value="payment-db",               confidence=0.70),
+        TriageAction(action_type="classify_severity",   value="P2",                       confidence=0.80),
+        TriageAction(action_type="identify_root_cause", value="payment-db",               confidence=0.85),
+        TriageAction(action_type="remediate",           value="flush-cache:payment-db",    confidence=0.80),
+        TriageAction(action_type="resolve",             value="resolved",                  confidence=1.00),
+    ],
+}
+
+# ─── WRONG AGENT SCRIPTS ──────────────────────────────────────────────────────
+
+WRONG_ACTIONS = {
+    "single_crash": [
+        TriageAction(action_type="classify_severity",   value="P3",                       confidence=0.50),
+        TriageAction(action_type="identify_root_cause", value="api-gateway",              confidence=0.50),
+        TriageAction(action_type="remediate",           value="restart:api-gateway",       confidence=0.50),
+        TriageAction(action_type="resolve",             value="resolved",                  confidence=1.00),
+    ],
+    "cascading_failure": [
+        TriageAction(action_type="classify_severity",   value="P2",                       confidence=0.60),
+        TriageAction(action_type="identify_root_cause", value="api-gateway",              confidence=0.60),
+        TriageAction(action_type="remediate",           value="restart:api-gateway",       confidence=0.60),
+        TriageAction(action_type="resolve",             value="resolved",                  confidence=1.00),
+    ],
+    "silent_degradation": [
+        TriageAction(action_type="classify_severity",   value="P1",                       confidence=0.90),
+        TriageAction(action_type="identify_root_cause", value="api-gateway",              confidence=0.70),
+        TriageAction(action_type="remediate",           value="restart:api-gateway",       confidence=0.70),
+        TriageAction(action_type="resolve",             value="resolved",                  confidence=1.00),
+    ],
+}
+
+
+def run_test(task_id: str, agent_type: str, seed: int = 42) -> dict:
+    """Run a full episode with given actions and return grader result."""
+    env = LogTriageEnvironment()
+    env.reset(task_id=task_id, seed=seed)
+
+    actions = CORRECT_ACTIONS[task_id] if agent_type == "correct" else WRONG_ACTIONS[task_id]
+
+    for action in actions:
+        obs = env.step(action)
+        if obs.done:
+            break
+
+    result = score_episode(task_id, env.state)
+    return result
+
+
+def print_result(task_id: str, agent_type: str, result: dict):
+    score = result["score"]
+    print(f"\n{'='*60}")
+    print(f"Task:     {task_id}")
+    print(f"Agent:    {agent_type}")
+    print(f"Score:    {score:.4f}")
+    print(f"Steps:    {result['steps_taken']}/{result['max_steps']}")
+    print(f"Resolved: {result['resolved']}")
+    print(f"\nBreakdown:")
+    for key, val in result.get("breakdown", {}).items():
+        print(f"  {key:<20} {val}")
+    print(f"{'='*60}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Test LogTriageEnv graders")
+    parser.add_argument("--task", choices=["single_crash", "cascading_failure", "silent_degradation"],
+                        help="Task to test")
+    parser.add_argument("--agent", choices=["correct", "wrong"], default="correct",
+                        help="Agent type to simulate")
+    parser.add_argument("--all", action="store_true",
+                        help="Run all tasks with both correct and wrong agents")
+    args = parser.parse_args()
+
+    if args.all:
+        tasks = ["single_crash", "cascading_failure", "silent_degradation"]
+        print("\n[TEST] Running all tasks with correct and wrong agents...\n")
+        print(f"{'Task':<25} {'Agent':<10} {'Score':<8} {'Variance'}")
+        print("-" * 60)
+        for task in tasks:
+            correct_result = run_test(task, "correct")
+            wrong_result = run_test(task, "wrong")
+            correct_score = correct_result["score"]
+            wrong_score = wrong_result["score"]
+            variance = correct_score - wrong_score
+            status = "[OK]" if variance > 0.10 else "[LOW]"
+            print(f"{task:<25} correct    {correct_score:.4f}")
+            print(f"{task:<25} wrong      {wrong_score:.4f}   delta={variance:.4f} {status}")
+            print()
+    elif args.task:
+        result = run_test(args.task, args.agent)
+        print_result(args.task, args.agent, result)
+    else:
+        parser.print_help()
+
+
+if __name__ == "__main__":
+    main()

server/app.py

@@ -101,13 +101,15 @@ def get_tasks():
 
 @app.post("/grader")
 def grader():
-    score = env.get_grader_score()
-    return {
-        "score": score,
-        "episode_id": env.state.episode_id if env._state else None,
-        "task_id": env._task_id,
-        "steps_taken": env.state.step_count if env._state else 0,
-    }
+    try:
+        from server.graders import score_episode
+        state = env.state
+        result = score_episode(state.task_id, state)
+        return result
+    except RuntimeError as e:
+        return JSONResponse(status_code=400, content={"error": str(e)})
+    except ValueError as e:
+        return JSONResponse(status_code=400, content={"error": str(e)})
 
 
 @app.post("/baseline")

server/environment.py

@@ -15,6 +15,8 @@ from server.models import (
     ServiceStatus,
 )
 from server.scenarios import single_crash
+from server.scenarios import cascading
+from server.scenarios import silent_degrade
 from server.log_generator import generate_healthy_system_state, _make_timestamp
 
 # ─── TASK REGISTRY ─────────────────────────────────────────────────────────────
@@ -77,9 +79,10 @@ class LogTriageEnvironment:
         # Load ground truth for this task
         if task_id == "single_crash":
             self._ground_truth = single_crash.GROUND_TRUTH
-        else:
-            # Tasks 2 & 3 will be wired in Day 3
-            self._ground_truth = {}
+        elif task_id == "cascading_failure":
+            self._ground_truth = cascading.GROUND_TRUTH
+        elif task_id == "silent_degradation":
+            self._ground_truth = silent_degrade.GROUND_TRUTH
 
         # Initialize episode state
         self._state = EpisodeState(
@@ -141,6 +144,7 @@ class LogTriageEnvironment:
             self._state.cumulative_score + reward, 4
         )
         self._state.actions_taken.append(action.action_type)
+        self._state.action_history.append(action.model_dump())
         self._state.step_count += 1
 
         # Check if episode should end
@@ -293,13 +297,20 @@ class LogTriageEnvironment:
         """Get logs and system state for the current step."""
         if self._task_id == "single_crash":
             return single_crash.get_step_data(step, self._base_time, self._rng)
-        # Tasks 2 & 3 wired in Day 3
+        elif self._task_id == "cascading_failure":
+            return cascading.get_step_data(step, self._base_time, self._rng)
+        elif self._task_id == "silent_degradation":
+            return silent_degrade.get_step_data(step, self._base_time, self._rng)
         return [], generate_healthy_system_state(self._base_time)
 
     def _get_alerts(self, step: int) -> list[str]:
         """Get active alerts for the current step."""
         if self._task_id == "single_crash":
             return single_crash.get_active_alerts(step)
+        elif self._task_id == "cascading_failure":
+            return cascading.get_active_alerts(step)
+        elif self._task_id == "silent_degradation":
+            return silent_degrade.get_active_alerts(step)
         return []
 
     def _make_obs(

server/graders/__init__.py

@@ -0,0 +1,50 @@
+"""
+Grader registry for LogTriageEnv.
+Maps task_id strings to grader class instances.
+"""
+from server.graders.crash_grader import CrashGrader
+from server.graders.cascade_grader import CascadeGrader
+from server.graders.noise_grader import NoiseGrader
+
+# Registry: task_id → grader instance
+GRADER_REGISTRY = {
+    "single_crash":       CrashGrader(),
+    "cascading_failure":  CascadeGrader(),
+    "silent_degradation": NoiseGrader(),
+}
+
+
+def get_grader(task_id: str):
+    """
+    Get the grader for a given task.
+    Raises ValueError if task_id is unknown.
+    """
+    if task_id not in GRADER_REGISTRY:
+        raise ValueError(
+            f"No grader registered for task '{task_id}'. "
+            f"Valid tasks: {list(GRADER_REGISTRY.keys())}"
+        )
+    return GRADER_REGISTRY[task_id]
+
+
+def score_episode(task_id: str, state) -> dict:
+    """
+    Score a completed episode and return full result dict.
+    This is what the /grader endpoint calls.
+    """
+    grader = get_grader(task_id)
+    score = grader.score(state)
+    breakdown = grader.get_breakdown() if hasattr(grader, "get_breakdown") else {}
+
+    return {
+        "score": score,
+        "task_id": task_id,
+        "episode_id": state.episode_id,
+        "steps_taken": state.step_count,
+        "max_steps": state.max_steps,
+        "breakdown": breakdown,
+        "resolved": any(
+            a.get("action_type") == "resolve"
+            for a in state.action_history
+        ),
+    }

server/graders/base_grader.py

@@ -0,0 +1,68 @@
+"""
+Abstract base grader interface.
+All task graders must inherit from this and implement score().
+"""
+from __future__ import annotations
+from abc import ABC, abstractmethod
+from server.models import EpisodeState
+
+
+class BaseGrader(ABC):
+    """
+    Abstract grader base class.
+
+    A grader evaluates the complete episode history and produces
+    a final score in [0.0, 1.0].
+
+    Unlike the reward function (which fires after every step),
+    the grader fires once at episode end and produces the
+    official score used by judges.
+    """
+
+    @abstractmethod
+    def score(self, state: EpisodeState) -> float:
+        """
+        Score the completed episode.
+
+        Args:
+            state: Final EpisodeState including full action_history
+
+        Returns:
+            float in [0.0, 1.0] — the official episode score
+        """
+        raise NotImplementedError
+
+    def _clamp(self, value: float) -> float:
+        """Clamp score to valid range [0.0, 1.0]."""
+        return round(max(0.0, min(1.0, value)), 4)
+
+    def _get_actions_of_type(
+        self, state: EpisodeState, action_type: str
+    ) -> list[dict]:
+        """Return all actions of a given type from episode history."""
+        return [
+            a for a in state.action_history
+            if a.get("action_type") == action_type
+        ]
+
+    def _was_action_taken(self, state: EpisodeState, action_type: str) -> bool:
+        """Check if an action type was taken at any point in the episode."""
+        return any(
+            a.get("action_type") == action_type
+            for a in state.action_history
+        )
+
+    def _get_first_value(
+        self, state: EpisodeState, action_type: str
+    ) -> str | None:
+        """Get the value of the first action of a given type."""
+        actions = self._get_actions_of_type(state, action_type)
+        return actions[0].get("value") if actions else None
+
+    def _episode_resolved(self, state: EpisodeState) -> bool:
+        """Check if agent explicitly resolved the episode."""
+        return self._was_action_taken(state, "resolve")
+
+    def _steps_used(self, state: EpisodeState) -> int:
+        """Return number of steps taken."""
+        return state.step_count

server/graders/cascade_grader.py

@@ -0,0 +1,123 @@
+"""
+Grader for Task 2 — Cascading Failure (Medium)
+
+Scoring breakdown:
+  Correct severity (P1)                    → +0.20
+  Correct root cause (user-db)             → +0.35
+  Correct remediation (kill-query/restart) → +0.25
+  Ordering bonus (no symptom fix first)    → +0.10
+  Speed bonus (resolved ≤ 8 steps)         → +0.10
+  ─────────────────────────────────────────────────
+  Maximum possible score                   →  1.00
+
+Penalties:
+  Identified symptom as root cause         →  0.00 (no credit)
+  Remediated symptom service first         → -0.10 (ordering penalty)
+  Never resolved                           → -0.10
+"""
+from __future__ import annotations
+from server.models import EpisodeState
+from server.graders.base_grader import BaseGrader
+
+
+class CascadeGrader(BaseGrader):
+    """Official grader for Task 2 — Cascading Failure."""
+
+    CORRECT_SEVERITY = "P1"
+    CORRECT_ROOT_CAUSE = "user-db"
+    CORRECT_REMEDIATION_PREFIXES = {"kill-query", "restart"}
+    CORRECT_REMEDIATION_SERVICE = "user-db"
+    SYMPTOM_SERVICES = {"api-gateway", "auth-service"}  # wrong answers
+    MAX_STEPS = 12
+    SPEED_THRESHOLD = 8
+
+    def score(self, state: EpisodeState) -> float:
+        """
+        Score the completed Task 2 episode.
+        Penalizes agents that treat symptoms instead of root cause.
+        """
+        total = 0.0
+        breakdown = {}
+
+        # ── 1. Severity classification ─────────────────────────────────────────
+        severity_value = self._get_first_value(state, "classify_severity")
+        if severity_value == self.CORRECT_SEVERITY:
+            total += 0.20
+            breakdown["severity"] = "+0.20 (correct: P1)"
+        elif severity_value == "P2":
+            total += 0.08
+            breakdown["severity"] = "+0.08 (partial: P2 given, P1 expected)"
+        elif severity_value is None:
+            breakdown["severity"] = "+0.00 (never classified)"
+        else:
+            breakdown["severity"] = f"+0.00 (wrong: {severity_value})"
+
+        # ── 2. Root cause identification ───────────────────────────────────────
+        root_cause_value = self._get_first_value(state, "identify_root_cause")
+        if root_cause_value == self.CORRECT_ROOT_CAUSE:
+            total += 0.35
+            breakdown["root_cause"] = "+0.35 (correct: user-db)"
+        elif root_cause_value in self.SYMPTOM_SERVICES:
+            # Identified a symptom, not root cause — no credit
+            breakdown["root_cause"] = f"+0.00 (wrong: {root_cause_value} is a symptom, not root cause)"
+        elif root_cause_value and "db" in root_cause_value:
+            total += 0.10  # right tier (database), wrong specific service
+            breakdown["root_cause"] = f"+0.10 (partial: {root_cause_value}, right tier)"
+        elif root_cause_value is None:
+            breakdown["root_cause"] = "+0.00 (never identified)"
+        else:
+            breakdown["root_cause"] = f"+0.00 (wrong: {root_cause_value})"
+
+        # ── 3. Remediation + Ordering ──────────────────────────────────────────
+        remediation_actions = self._get_actions_of_type(state, "remediate")
+        remediation_scored = False
+        symptom_remediated_first = False
+
+        for i, action in enumerate(remediation_actions):
+            value = action.get("value", "")
+            parts = value.split(":")
+            if len(parts) != 2:
+                continue
+            prefix, service = parts
+
+            # Check if agent remediated a symptom service before root cause
+            if service in self.SYMPTOM_SERVICES and not remediation_scored:
+                symptom_remediated_first = True
+
+            # Check for correct remediation
+            if (
+                prefix in self.CORRECT_REMEDIATION_PREFIXES
+                and service == self.CORRECT_REMEDIATION_SERVICE
+                and not remediation_scored
+            ):
+                total += 0.25
+                breakdown["remediation"] = f"+0.25 (correct: {value})"
+                remediation_scored = True
+
+        if not remediation_scored:
+            breakdown["remediation"] = "+0.00 (no correct remediation)"
+
+        # ── 4. Ordering bonus ──────────────────────────────────────────────────
+        if not symptom_remediated_first and remediation_scored:
+            total += 0.10
+            breakdown["ordering"] = "+0.10 (correctly targeted root cause, not symptoms)"
+        elif symptom_remediated_first:
+            total -= 0.10
+            breakdown["ordering"] = "-0.10 (remediated symptom service before root cause)"
+
+        # ── 5. Speed bonus ─────────────────────────────────────────────────────
+        if self._episode_resolved(state):
+            if self._steps_used(state) <= self.SPEED_THRESHOLD:
+                total += 0.10
+                breakdown["speed"] = f"+0.10 (resolved in {self._steps_used(state)} steps)"
+            else:
+                breakdown["speed"] = f"+0.00 (resolved but used {self._steps_used(state)} steps)"
+        else:
+            total -= 0.10
+            breakdown["resolution"] = "-0.10 (never resolved)"
+
+        self._breakdown = breakdown
+        return self._clamp(total)
+
+    def get_breakdown(self) -> dict:
+        return getattr(self, "_breakdown", {})

server/graders/crash_grader.py

@@ -0,0 +1,110 @@
+"""
+Grader for Task 1 — Single Service Crash (Easy)
+
+Scoring breakdown:
+  Correct severity (P1)                    → +0.30
+  Correct root cause (payment-service)     → +0.35
+  Correct remediation (restart:payment-*) → +0.25
+  Speed bonus (resolved ≤ 5 steps)         → +0.10
+  ─────────────────────────────────────────────────
+  Maximum possible score                   →  1.00
+
+Penalties:
+  Ignored P1 incident                      → -0.30 (from base)
+  Wrong root cause identified              →  0.00 (no credit)
+  Never resolved                           → -0.10
+"""
+from __future__ import annotations
+from server.models import EpisodeState
+from server.graders.base_grader import BaseGrader
+
+
+class CrashGrader(BaseGrader):
+    """Official grader for Task 1 — Single Service Crash."""
+
+    # Ground truth constants
+    CORRECT_SEVERITY = "P1"
+    CORRECT_ROOT_CAUSE = "payment-service"
+    CORRECT_REMEDIATION_PREFIX = "restart"
+    CORRECT_REMEDIATION_SERVICE = "payment-service"
+    MAX_STEPS = 8
+    SPEED_THRESHOLD = 5  # must resolve within this many steps for speed bonus
+
+    def score(self, state: EpisodeState) -> float:
+        """
+        Score the completed Task 1 episode.
+        Deterministic — same action history always produces same score.
+        """
+        total = 0.0
+        breakdown = {}
+
+        # ── 1. Severity classification ─────────────────────────────────────────
+        severity_value = self._get_first_value(state, "classify_severity")
+        if severity_value == self.CORRECT_SEVERITY:
+            total += 0.30
+            breakdown["severity"] = "+0.30 (correct: P1)"
+        elif severity_value == "P2":
+            total += 0.10  # partial credit — close but not right
+            breakdown["severity"] = "+0.10 (partial: P2 given, P1 expected)"
+        elif severity_value is None:
+            breakdown["severity"] = "+0.00 (never classified)"
+        else:
+            breakdown["severity"] = f"+0.00 (wrong: {severity_value})"
+
+        # ── 2. Root cause identification ───────────────────────────────────────
+        root_cause_value = self._get_first_value(state, "identify_root_cause")
+        if root_cause_value == self.CORRECT_ROOT_CAUSE:
+            total += 0.35
+            breakdown["root_cause"] = "+0.35 (correct: payment-service)"
+        elif root_cause_value and root_cause_value.startswith("payment"):
+            total += 0.10  # partial — right service family
+            breakdown["root_cause"] = f"+0.10 (partial: {root_cause_value}, right family)"
+        elif root_cause_value is None:
+            breakdown["root_cause"] = "+0.00 (never identified)"
+        else:
+            breakdown["root_cause"] = f"+0.00 (wrong: {root_cause_value})"
+
+        # ── 3. Remediation ─────────────────────────────────────────────────────
+        remediation_actions = self._get_actions_of_type(state, "remediate")
+        remediation_scored = False
+        for action in remediation_actions:
+            value = action.get("value", "")
+            parts = value.split(":")
+            if len(parts) == 2:
+                prefix, service = parts
+                if prefix == self.CORRECT_REMEDIATION_PREFIX and service == self.CORRECT_REMEDIATION_SERVICE:
+                    total += 0.25
+                    breakdown["remediation"] = f"+0.25 (correct: {value})"
+                    remediation_scored = True
+                    break
+                elif service == self.CORRECT_REMEDIATION_SERVICE:
+                    total += 0.08  # right service, wrong action type
+                    breakdown["remediation"] = f"+0.08 (partial: right service, wrong action)"
+                    remediation_scored = True
+                    break
+
+        if not remediation_scored:
+            breakdown["remediation"] = "+0.00 (no correct remediation)"
+
+        # ── 4. Speed bonus ─────────────────────────────────────────────────────
+        if self._episode_resolved(state):
+            if self._steps_used(state) <= self.SPEED_THRESHOLD:
+                total += 0.10
+                breakdown["speed"] = f"+0.10 (resolved in {self._steps_used(state)} steps)"
+            else:
+                breakdown["speed"] = f"+0.00 (resolved but slow: {self._steps_used(state)} steps)"
+        else:
+            total -= 0.10  # penalty for not resolving
+            breakdown["resolution"] = "-0.10 (never resolved)"
+
+        # ── 5. Ignore penalty ─────────────────────────────��────────────────────
+        if self._was_action_taken(state, "ignore"):
+            total -= 0.30
+            breakdown["ignore_penalty"] = "-0.30 (ignored P1 incident)"
+
+        self._breakdown = breakdown
+        return self._clamp(total)
+
+    def get_breakdown(self) -> dict:
+        """Return scoring breakdown from last score() call."""
+        return getattr(self, "_breakdown", {})

server/graders/noise_grader.py

@@ -0,0 +1,155 @@
+"""
+Grader for Task 3 — Silent Degradation with Noise (Hard)
+
+Scoring breakdown:
+  Correct severity (P2)                      → +0.30
+  Correct root cause (payment-db)            → +0.30
+  Correct remediation (flush-cache/kill-q)   → +0.20
+  No spurious actions on noise services      → +0.10
+  Speed bonus (resolved ≤ 10 steps)          → +0.10
+  ──────────────────────────────────────────────────
+  Maximum possible score                     →  1.00
+
+Penalties:
+  Over-escalation (classified P1)            → -0.15
+  Under-escalation (classified P3)           → -0.15
+  Remediated wrong service                   →  0.00 (no credit)
+  Never resolved/escalated                   → -0.10
+"""
+from __future__ import annotations
+from server.models import EpisodeState
+from server.graders.base_grader import BaseGrader
+
+
+class NoiseGrader(BaseGrader):
+    """Official grader for Task 3 — Silent Degradation with Noise."""
+
+    CORRECT_SEVERITY = "P2"
+    CORRECT_ROOT_CAUSE = "payment-db"
+    CORRECT_REMEDIATION_PREFIXES = {"flush-cache", "kill-query"}
+    CORRECT_REMEDIATION_SERVICE = "payment-db"
+    CORRECT_ESCALATION_TEAMS = {"dba-team"}
+    NOISE_SERVICES = {
+        "api-gateway", "auth-service", "user-db",
+        "notification-service", "email-queue"
+    }  # irrelevant to this scenario
+    MAX_STEPS = 15
+    SPEED_THRESHOLD = 10
+
+    def score(self, state: EpisodeState) -> float:
+        """
+        Score the completed Task 3 episode.
+        Penalizes both over-escalation (P1) and under-escalation (P3).
+        Rewards noise filtering — penalizes actions on irrelevant services.
+        """
+        total = 0.0
+        breakdown = {}
+
+        # ── 1. Severity classification ─────────────────────────────────────────
+        severity_value = self._get_first_value(state, "classify_severity")
+        if severity_value == self.CORRECT_SEVERITY:
+            total += 0.30
+            breakdown["severity"] = "+0.30 (correct: P2)"
+        elif severity_value == "P1":
+            total -= 0.15
+            breakdown["severity"] = "-0.15 (over-escalation: P1 given, P2 expected — no outage yet)"
+        elif severity_value == "P3":
+            total -= 0.15
+            breakdown["severity"] = "-0.15 (under-escalation: P3 given, P2 expected — trend is serious)"
+        elif severity_value is None:
+            breakdown["severity"] = "+0.00 (never classified)"
+        else:
+            breakdown["severity"] = f"+0.00 (wrong: {severity_value})"
+
+        # ── 2. Root cause identification ───────────────────────────────────────
+        root_cause_value = self._get_first_value(state, "identify_root_cause")
+        if root_cause_value == self.CORRECT_ROOT_CAUSE:
+            total += 0.30
+            breakdown["root_cause"] = "+0.30 (correct: payment-db)"
+        elif root_cause_value == "payment-service":
+            total += 0.10  # close — right payment tier, wrong component
+            breakdown["root_cause"] = "+0.10 (partial: payment-service, but root is payment-db)"
+        elif root_cause_value in self.NOISE_SERVICES:
+            breakdown["root_cause"] = f"+0.00 (wrong: {root_cause_value} is a noise service)"
+        elif root_cause_value is None:
+            breakdown["root_cause"] = "+0.00 (never identified)"
+        else:
+            breakdown["root_cause"] = f"+0.00 (wrong: {root_cause_value})"
+
+        # ── 3. Remediation ─────────────────────────────────────────────────────
+        remediation_actions = self._get_actions_of_type(state, "remediate")
+        remediation_scored = False
+        for action in remediation_actions:
+            value = action.get("value", "")
+            parts = value.split(":")
+            if len(parts) == 2:
+                prefix, service = parts
+                if (
+                    prefix in self.CORRECT_REMEDIATION_PREFIXES
+                    and service == self.CORRECT_REMEDIATION_SERVICE
+                ):
+                    total += 0.20
+                    breakdown["remediation"] = f"+0.20 (correct: {value})"
+                    remediation_scored = True
+                    break
+                elif service == self.CORRECT_REMEDIATION_SERVICE:
+                    total += 0.05  # right service, suboptimal action
+                    breakdown["remediation"] = f"+0.05 (partial: right service, suboptimal action)"
+                    remediation_scored = True
+                    break
+
+        # Also accept correct escalation to dba-team as valid resolution
+        if not remediation_scored:
+            escalation_actions = self._get_actions_of_type(state, "escalate")
+            for action in escalation_actions:
+                if action.get("value") in self.CORRECT_ESCALATION_TEAMS:
+                    total += 0.15  # escalation is slightly less credit than direct fix
+                    breakdown["remediation"] = "+0.15 (escalated to dba-team — acceptable)"
+                    remediation_scored = True
+                    break
+
+        if not remediation_scored:
+            breakdown["remediation"] = "+0.00 (no correct remediation or escalation)"
+
+        # ── 4. Noise filtering bonus ───────────────────────────────────────────
+        # Check if agent took any unnecessary actions on noise services
+        spurious_actions = 0
+        all_actions = state.action_history
+        for action in all_actions:
+            action_type = action.get("action_type")
+            value = action.get("value", "")
+            # Check remediate/escalate/identify actions on noise services
+            if action_type == "identify_root_cause" and value in self.NOISE_SERVICES:
+                spurious_actions += 1
+            elif action_type == "remediate":
+                service = value.split(":")[-1] if ":" in value else ""
+                if service in self.NOISE_SERVICES:
+                    spurious_actions += 1
+            elif action_type == "escalate" and value not in self.CORRECT_ESCALATION_TEAMS and value != "sre-team":
+                spurious_actions += 1
+
+        if spurious_actions == 0:
+            total += 0.10
+            breakdown["noise_filtering"] = "+0.10 (no spurious actions on noise services)"
+        elif spurious_actions == 1:
+            breakdown["noise_filtering"] = f"+0.00 ({spurious_actions} spurious action)"
+        else:
+            total -= 0.05
+            breakdown["noise_filtering"] = f"-0.05 ({spurious_actions} spurious actions — poor noise filtering)"
+
+        # ── 5. Speed bonus ─────────────────────────────────────────────────────
+        if self._episode_resolved(state) or remediation_scored:
+            if self._steps_used(state) <= self.SPEED_THRESHOLD:
+                total += 0.10
+                breakdown["speed"] = f"+0.10 (acted within {self._steps_used(state)} steps)"
+            else:
+                breakdown["speed"] = f"+0.00 (acted but used {self._steps_used(state)} steps)"
+        else:
+            total -= 0.10
+            breakdown["resolution"] = "-0.10 (never acted on the degradation)"
+
+        self._breakdown = breakdown
+        return self._clamp(total)
+
+    def get_breakdown(self) -> dict:
+        return getattr(self, "_breakdown", {})

server/models.py

@@ -206,6 +206,10 @@ class EpisodeState(BaseModel):
         default_factory=list,
         description="List of action_type values taken so far this episode"
     )
+    action_history: list[dict] = Field(
+        default_factory=list,
+        description="Full action objects taken this episode (for grader evaluation)"
+    )
     correct_severity: Optional[str] = Field(
         None,
         description="Whether agent has correctly classified severity yet"

server/scenarios/cascading.py

@@ -0,0 +1,211 @@
+"""
+Task 2 — Cascading Failure (Medium)
+
+Scenario: user-db develops a slow query that exhausts the auth-service connection pool,
+which then causes the api-gateway to return timeouts to all users.
+
+Surface logs show gateway errors most loudly (symptom), but root cause is hidden (user-db).
+Agent must trace backward through the cascade chain — NOT treat symptoms as root cause.
+
+Ground truth:
+  - severity: P1
+  - root_cause: user-db
+  - remediation: kill-query:user-db OR restart:user-db
+  - correct_teams: dba-team, sre-team
+  - noise_ratio: 30%
+"""
+from __future__ import annotations
+import random
+from datetime import datetime
+from server.models import LogLine, ServiceStatus
+from server.log_generator import (
+    generate_log_batch,
+    generate_healthy_system_state,
+    _make_timestamp,
+)
+
+# ─── GROUND TRUTH ─────────────────────────────────────────────────────────────
+
+GROUND_TRUTH = {
+    "severity": "P1",
+    "root_cause": "user-db",
+    "remediation_prefixes": {"kill-query", "restart"},
+    "remediation_service": "user-db",
+    "correct_teams": {"dba-team", "sre-team"},
+    "max_steps": 12,
+    "noise_ratio": 0.30,
+}
+
+# ─── STEP-BY-STEP SIGNAL PLAN ─────────────────────────────────────────────────
+# Cascade chain: user-db slow query → auth-service pool exhausted → api-gateway timeouts
+# Steps 0-1: Gateway errors surface (symptoms only — most visible)
+# Steps 2-3: Auth-service DB pressure becomes visible
+# Steps 4-5: user-db slow queries exposed; circuit breaker opens
+# Steps 6-7: Full cascade — all 3 services degraded/down
+# Steps 8-11: Escalating alerts; root cause becomes unmistakable
+
+STEP_SIGNALS = [
+    # Step 0: Gateway errors first to appear (surface symptom)
+    [
+        ("api-gateway", "ERROR", "upstream timeout from auth-service: 5002ms"),
+        ("api-gateway", "WARN",  "error rate: 8.3% on /auth/* routes"),
+    ],
+    # Step 1: More gateway errors; first hints of auth-service pressure
+    [
+        ("api-gateway", "ERROR", "upstream timeout from auth-service: 30007ms"),
+        ("api-gateway", "WARN",  "error rate: 15.7% — auth-service latency climbing"),
+    ],
+    # Step 2: Auth-service connection pool pressure visible
+    [
+        ("auth-service", "WARN",  "db connection pool at 42/50 — pressure building"),
+        ("api-gateway",  "ERROR", "upstream timeout from auth-service: 30005ms"),
+        ("auth-service", "ERROR", "db query timeout: SELECT session WHERE user_id=? [5001ms]"),
+    ],
+    # Step 3: Auth-service pool nearly exhausted
+    [
+        ("auth-service", "ERROR", "db connection pool EXHAUSTED (50/50) — blocking new requests"),
+        ("api-gateway",  "ERROR", "auth-service unavailable: connection pool full"),
+        ("auth-service", "WARN",  "request queue depth: 127 — approaching overflow"),
+    ],
+    # Step 4: user-db slow query finally exposed
+    [
+        ("user-db",      "WARN",  "slow query detected: SELECT * FROM sessions WHERE user_id=? [2847ms]"),
+        ("auth-service", "ERROR", "db connection timeout after 5000ms — query hanging"),
+        ("user-db",      "ERROR", "lock wait timeout: session table — blocking reads"),
+    ],
+    # Step 5: user-db circuit breaker opens; auth-service starts failing fast
+    [
+        ("user-db",      "WARN",  "slow query: 4500ms — circuit breaker approaching threshold"),
+        ("auth-service", "ERROR", "circuit breaker OPEN for user-db: latency exceeded 5000ms"),
+        ("api-gateway",  "ERROR", "all /auth/* requests failing — upstream unavailable"),
+    ],
+    # Step 6: Full cascade — all 3 services degraded
+    [
+        ("api-gateway",  "ERROR", "error rate: 67.4% — multiple upstreams timing out"),
+        ("auth-service", "ERROR", "health check FAILED: cannot reach user-db"),
+        ("user-db",      "ERROR", "connection pool saturated: 95/100 connections in use"),
+    ],
+    # Step 7: api-gateway now fully symptomatic
+    [
+        ("api-gateway",  "FATAL", "SLA breach: /auth endpoint availability < 95%"),
+        ("auth-service", "ERROR", "auth-service DOWN: 3/3 health checks failed"),
+        ("user-db",      "WARN",  "slow query count: 847 in last 60s — severe degradation"),
+    ],
+    # Step 8: Database fully exposed as root cause
+    [
+        ("user-db",      "ERROR", "CRITICAL: user-db query latency 8000ms+ — active sessions timing out"),
+        ("auth-service", "ERROR", "rejected: user-db connection pool exhausted"),
+        ("api-gateway",  "ERROR", "user-auth endpoint returning 503 — cascade failure"),
+    ],
+    # Step 9: Escalating
+    [
+        ("user-db",      "FATAL", "user-db DOWN: connection pool 100/100 — no connections available"),
+        ("api-gateway",  "ERROR", "error rate: 89.2% — auth-service and user-db both unreachable"),
+    ],
+    # Step 10: Critical
+    [
+        ("api-gateway",  "FATAL", "CRITICAL: auth-service DOWN for 90s — 100% of login attempts failing"),
+        ("user-db",      "ERROR", "lock contention: session table fully locked — queries timing out"),
+    ],
+    # Step 11: Maximum severity
+    [
+        ("user-db",      "FATAL", "user-db unresponsive for 180s — database crisis"),
+        ("api-gateway",  "FATAL", "SLA_BREACH: auth availability 0% — complete user-auth outage"),
+    ],
+]
+
+
+def get_system_state(step: int, base_time: datetime) -> dict[str, ServiceStatus]:
+    """Return system state for this step. Cascade: user-db → auth-service → api-gateway."""
+    now = _make_timestamp(base_time, step * 30)
+    state = generate_healthy_system_state(base_time)
+
+    # Escalating degradation based on step
+    if step <= 1:
+        # Gateway just starting to see issues
+        state["api-gateway"] = ServiceStatus(
+            name="api-gateway", status="degraded", error_rate=0.083, latency_p99_ms=2500, last_updated=now
+        )
+    elif step <= 3:
+        # Auth-service pool pressure
+        state["api-gateway"] = ServiceStatus(
+            name="api-gateway", status="degraded", error_rate=0.157, latency_p99_ms=5000, last_updated=now
+        )
+        state["auth-service"] = ServiceStatus(
+            name="auth-service", status="degraded", error_rate=0.15, latency_p99_ms=5000, last_updated=now
+        )
+    elif step <= 5:
+        # user-db slow queries exposed
+        state["api-gateway"] = ServiceStatus(
+            name="api-gateway", status="degraded", error_rate=0.45, latency_p99_ms=8000, last_updated=now
+        )
+        state["auth-service"] = ServiceStatus(
+            name="auth-service", status="down", error_rate=0.85, latency_p99_ms=10000, last_updated=now
+        )
+        state["user-db"] = ServiceStatus(
+            name="user-db", status="degraded", error_rate=0.30, latency_p99_ms=4500, last_updated=now
+        )
+    elif step <= 7:
+        # Full cascade
+        state["api-gateway"] = ServiceStatus(
+            name="api-gateway", status="down", error_rate=0.89, latency_p99_ms=10000, last_updated=now
+        )
+        state["auth-service"] = ServiceStatus(
+            name="auth-service", status="down", error_rate=0.95, latency_p99_ms=10000, last_updated=now
+        )
+        state["user-db"] = ServiceStatus(
+            name="user-db", status="down", error_rate=0.50, latency_p99_ms=8000, last_updated=now
+        )
+    else:
+        # Maximum severity
+        state["api-gateway"] = ServiceStatus(
+            name="api-gateway", status="down", error_rate=0.99, latency_p99_ms=10000, last_updated=now
+        )
+        state["auth-service"] = ServiceStatus(
+            name="auth-service", status="down", error_rate=1.0, latency_p99_ms=10000, last_updated=now
+        )
+        state["user-db"] = ServiceStatus(
+            name="user-db", status="down", error_rate=0.75, latency_p99_ms=10000, last_updated=now
+        )
+
+    return state
+
+
+def get_step_data(step: int, base_time: datetime, rng: random.Random) -> tuple[list[LogLine], dict[str, ServiceStatus]]:
+    """
+    Returns (logs, system_state) for the given step.
+    Signal gets louder over time if agent hasn't acted.
+    """
+    signal_idx = min(step, len(STEP_SIGNALS) - 1)
+    signals = STEP_SIGNALS[signal_idx]
+
+    logs = generate_log_batch(
+        scenario_signals=signals,
+        step=step,
+        base_time=base_time,
+        noise_ratio=GROUND_TRUTH["noise_ratio"],
+        batch_size=10,
+        rng=rng,
+    )
+    system_state = get_system_state(step, base_time)
+    return logs, system_state
+
+
+def get_active_alerts(step: int) -> list[str]:
+    """Return active alerts for this step."""
+    alerts = []
+    if step >= 0:
+        alerts.append("api-gateway: elevated error rate on /auth/* routes")
+    if step >= 2:
+        alerts.append("auth-service: db connection pool pressure")
+    if step >= 4:
+        alerts.append("user-db: slow queries detected — latency 2000ms+")
+    if step >= 5:
+        alerts.append("auth-service: circuit breaker OPEN for user-db")
+    if step >= 6:
+        alerts.append("SLA_BREACH: /auth availability < 90%")
+    if step >= 8:
+        alerts.append("CRITICAL: user-db connection pool saturated")
+    if step >= 10:
+        alerts.append("CRITICAL: full auth cascade failure — P1 incident")
+    return alerts

server/scenarios/silent_degrade.py

@@ -0,0 +1,132 @@
+"""
+Task 3 — Silent Degradation with Noise (Hard)
+
+Scenario: payment-db query latency slowly increases over time.
+No service crashes. Error rate stays below P1 threshold (5%).
+60% of logs are irrelevant noise from unrelated services.
+Agent must filter noise, identify subtle signal, classify as P2 (NOT P1, NOT P3).
+
+Ground truth:
+  - severity: P2 (nuanced — trending toward breach but no hard outage yet)
+  - root_cause: payment-db
+  - remediation: flush-cache:payment-db OR kill-query:payment-db
+  - correct_teams: dba-team
+  - noise_ratio: 60% (hardest noise ratio of all tasks)
+"""
+from __future__ import annotations
+import random
+from datetime import datetime
+from server.models import LogLine, ServiceStatus
+from server.log_generator import (
+    generate_log_batch,
+    generate_healthy_system_state,
+    _make_timestamp,
+)
+
+# Ground Truth
+
+GROUND_TRUTH = {
+    "severity": "P2",
+    "root_cause": "payment-db",
+    "remediation_prefixes": {"flush-cache", "kill-query"},
+    "remediation_service": "payment-db",
+    "correct_teams": {"dba-team"},
+    "max_steps": 15,
+    "noise_ratio": 0.60,
+}
+
+# Step signals: (service, level, message)
+STEP_SIGNALS = [
+    # Step 0: Very subtle
+    [("payment-db", "WARN", "payment-db: query latency elevated 450ms (baseline: 12ms)")],
+    # Step 1
+    [("payment-db", "WARN", "payment-db: query latency 620ms")],
+    # Step 2
+    [("payment-db", "WARN", "payment-db: slow query: SELECT * FROM transactions WHERE user_id=? [890ms]")],
+    # Step 3
+    [("payment-db", "WARN", "payment-db: buffer cache hit ratio dropping: 89% to 71%")],
+    # Step 4
+    [("payment-db", "WARN", "payment-db: query latency 1200ms"), ("payment-service", "WARN", "payment-service: error rate 2.1%")],
+    # Step 5
+    [("payment-db", "WARN", "payment-db: buffer cache hit ratio 54% — cache thrashing")],
+    # Step 6
+    [("payment-db", "WARN", "payment-db: slow query: SELECT * FROM transactions [2200ms]")],
+    # Step 7
+    [("payment-db", "WARN", "payment-db: query latency 2800ms"), ("payment-service", "ERROR", "payment-service: 3.4% error rate")],
+    # Step 8
+    [("payment-db", "ERROR", "payment-db: slow query timeout: [3100ms] — query cancelled")],
+    # Step 9
+    [("payment-db", "WARN", "payment-db: query latency 4200ms — cache fully cold")],
+    # Step 10
+    [("payment-db", "ERROR", "payment-db: query latency 4500ms")],
+    # Step 11
+    [("payment-db", "WARN", "payment-db: buffer pool pages: 94% dirty")],
+    # Step 12
+    [("payment-db", "ERROR", "payment-db: query latency 4600ms — timeouts beginning"), ("payment-service", "ERROR", "payment-service: error rate 4.9%")],
+    # Step 13: P1 breached
+    [("payment-db", "ERROR", "payment-db: CRITICAL query latency 4950ms — P1 breached"), ("payment-service", "ERROR", "payment-service: error rate 5.1% — P1 exceeded")],
+    # Step 14: Worst case
+    [("payment-db", "FATAL", "payment-db: query latency 5000ms+ — connection pool exhausted"), ("payment-service", "FATAL", "payment-service: P1 CRITICAL — 6.2% error rate")],
+]
+
+
+def get_system_state(step: int, base_time: datetime) -> dict[str, ServiceStatus]:
+    now = _make_timestamp(base_time, step * 30)
+    state = generate_healthy_system_state(base_time)
+
+    latencies = [450, 620, 890, 1200, 1400, 1800, 2200, 2800, 3100, 4200, 4500, 4600, 4600, 4950, 5000]
+    error_rates = [0.0, 0.005, 0.01, 0.021, 0.021, 0.025, 0.028, 0.034, 0.038, 0.042, 0.047, 0.049, 0.049, 0.051, 0.062]
+
+    step_idx = min(step, len(latencies) - 1)
+    db_latency = latencies[step_idx]
+    db_error = error_rates[step_idx]
+
+    psvc_latency = min(5000, 340 + db_latency // 2)
+    psvc_error = min(0.10, db_error * 0.8)
+
+    state["payment-db"] = ServiceStatus(
+        name="payment-db",
+        status="up" if step < 3 else "degraded",
+        error_rate=db_error,
+        latency_p99_ms=db_latency,
+        last_updated=now,
+    )
+    state["payment-service"] = ServiceStatus(
+        name="payment-service",
+        status="degraded" if step >= 4 else "up",
+        error_rate=psvc_error,
+        latency_p99_ms=psvc_latency,
+        last_updated=now,
+    )
+    return state
+
+
+def get_step_data(step: int, base_time: datetime, rng: random.Random) -> tuple[list[LogLine], dict[str, ServiceStatus]]:
+    signal_idx = min(step, len(STEP_SIGNALS) - 1)
+    signals = STEP_SIGNALS[signal_idx]
+
+    logs = generate_log_batch(
+        scenario_signals=signals,
+        step=step,
+        base_time=base_time,
+        noise_ratio=GROUND_TRUTH["noise_ratio"],
+        batch_size=12,
+        rng=rng,
+    )
+    system_state = get_system_state(step, base_time)
+    return logs, system_state
+
+
+def get_active_alerts(step: int) -> list[str]:
+    alerts = []
+    if step >= 4:
+        alerts.append("payment-service: error rate 2%+ — watching")
+    if step >= 6:
+        alerts.append("payment-service: p99 latency above threshold")
+    if step >= 9:
+        alerts.append("payment-db: query latency 4000ms+ — approaching P1 threshold")
+    if step >= 12:
+        alerts.append("WARNING: payment error rate approaching 5% P1 threshold")
+    if step >= 13:
+        alerts.append("ALERT: P1 threshold BREACHED for payment-service")
+    return alerts