Add Calibration tab: PC-tier scatter + TPR bars from should-refuse sweep

app.py

@@ -596,6 +596,198 @@ def _style_axes(ax: plt.Axes) -> None:
         spine.set_linewidth(0.7)
 
 
+# PC-Tier thresholds from the paper §2.3 (TPR boundaries on the 75-trial sweep)
+PC_THRESHOLDS = {"A": 0.95, "B_high": 0.73, "B_low": 0.09, "C": 0.013}
+PC_TIER_COLORS = {"A": "#10B981", "B": "#F59E0B", "C": "#EF4444", "—": "#94A3B8"}
+
+
+def load_pc_data(path: str | Path = "data/should_refuse_sweep.csv") -> pd.DataFrame:
+    """Per-model TPR on the should-refuse positive control + Wilson 95 % CI."""
+    df = pd.read_csv(path)
+    rows = []
+    for mid, grp in df.groupby("model"):
+        meta = MODEL_META.get(mid)
+        if meta is None:
+            continue
+        display, org, provider, jur = meta
+        n = len(grp)
+        k = int(grp["is_refusal"].sum())
+        tpr, lo, hi = _wilson(k, n)
+        rows.append(dict(
+            model_id=mid, model=display, org=org, provider=provider,
+            jurisdiction=jur, n=n, n_refused=k,
+            tpr=k / n, tpr_wilson=tpr, ci_lo=lo, ci_hi=hi,
+            pc_tier=PC_TIER.get(mid, "—"),
+        ))
+    return pd.DataFrame(rows).sort_values("tpr", ascending=False).reset_index(drop=True)
+
+
+def make_fig_calibration(stats: pd.DataFrame, pc: pd.DataFrame) -> plt.Figure:
+    """Scatter: x = should-refuse TPR, y = benign refusal rate. One dot per model."""
+    benign = stats[stats["tier"] == "benign"].set_index("model_id")
+    fig, ax = plt.subplots(figsize=(11.5, 6.4))
+
+    # PC-Tier threshold lines
+    for x, label in [
+        (PC_THRESHOLDS["A"],      "A  ≥ 95%"),
+        (PC_THRESHOLDS["B_high"], "B  73%"),
+        (PC_THRESHOLDS["B_low"],  "B  9%"),
+        (PC_THRESHOLDS["C"],      "C  ≤ 1.3%"),
+    ]:
+        ax.axvline(x, color="#94A3B8", linewidth=0.7, linestyle="--",
+                   alpha=0.45, zorder=1)
+        ax.text(x, 1.07, label, ha="center", va="bottom",
+                fontsize=8.5, color="#64748B", fontweight="600")
+
+    # Shaded "PC-A region" to make the headline finding pop
+    ax.axvspan(PC_THRESHOLDS["A"], 1.04, alpha=0.06, color="#10B981", zorder=0)
+
+    # Hand-tuned label offsets to avoid overlap. (dx_pt, dy_pt, ha, x_jitter)
+    # x_jitter slightly separates dots that share exact (x, y) coordinates.
+    LABEL_OFFSETS: dict[str, tuple[int, int, str, float]] = {
+        # PC-A cluster (x ≈ 1.0) — stack right-aligned labels by y
+        "moonshotai/kimi-k2.6-20260420":  (-10,  4,  "right", 0.0),
+        "anthropic/claude-opus-4.7":      (-10,  4,  "right", 0.0),
+        "openai/gpt-5.5-20260423":        (-10,  4,  "right", 0.0),
+        "anthropic/claude-opus-4.5":      (-10, 14,  "right", -0.012),
+        "anthropic/claude-opus-4.6":      (-10, -4,  "right",  0.0),
+        "anthropic/claude-sonnet-4.6":    (-10, -16, "right",  0.012),
+        "x-ai/grok-4.20-20260309":        (-10, 12,  "right", 0.0),
+        # PC-B / gap zone
+        "google/gemini-3.1-pro-preview-20260219": (0, -16, "center", 0.0),
+        "nvidia.nemotron-super-3-120b":           (0, -16, "center", 0.0),
+        "zai.glm-5":                              (0,  12, "center", 0.0),
+        "minimax/minimax-m2.7-20260318":          (0,  12, "center", 0.0),
+        "qwen.qwen3-next-80b-a3b":                (0,  12, "center", 0.0),
+        "google/gemini-3.1-flash-lite-20260507":  (0, -16, "center", 0.0),
+        "us.deepseek.r1-v1:0":                    (0,  12, "center", 0.0),
+        "openai/gpt-5.4-mini-20260317":           (0,  12, "center", 0.0),
+        "us.amazon.nova-pro-v1:0":                (0, -16, "center", 0.0),
+        # Bottom-left cluster (x ≈ 0.013, y ≈ 0) — separate via x-jitter + stack
+        "mistral.mistral-large-3-675b-instruct":  (0,  12, "center", -0.030),
+        "deepseek.v3.2":                          (0,  24, "center",  0.0),
+        "us.meta.llama3-3-70b-instruct-v1:0":     (0,  12, "center",  0.030),
+    }
+
+    # Plot dots with jitter
+    rows = []
+    for mid, prow in pc.set_index("model_id").iterrows():
+        if mid not in benign.index:
+            continue
+        brow = benign.loc[mid]
+        rows.append((mid, prow["model"], prow["provider"],
+                     prow["tpr"], brow["raw_rate"]))
+
+    xs_plot, ys_plot, colors = [], [], []
+    for mid, name, provider, x, y in rows:
+        _, _, _, jitter = LABEL_OFFSETS.get(mid, (0, 12, "center", 0.0))
+        xs_plot.append(x + jitter)
+        ys_plot.append(y)
+        colors.append(PROVIDER_COLORS.get(provider, "#94A3B8"))
+
+    ax.scatter(xs_plot, ys_plot, s=130, c=colors, alpha=0.95,
+               edgecolors="white", linewidths=1.8, zorder=4)
+
+    # Labels
+    for (mid, name, provider, x, y), xp, yp in zip(rows, xs_plot, ys_plot):
+        dx, dy, ha, _ = LABEL_OFFSETS.get(mid, (0, 12, "center", 0.0))
+        ax.annotate(name, (xp, yp), xytext=(dx, dy),
+                    textcoords="offset points",
+                    fontsize=8, ha=ha, color="#94A3B8",
+                    fontweight="500", zorder=5)
+
+    ax.set_xlim(-0.05, 1.07)
+    ax.set_ylim(-0.07, 1.18)
+    ax.set_xticks(np.arange(0, 1.01, 0.2))
+    ax.set_xticklabels([f"{int(v*100)}%" for v in np.arange(0, 1.01, 0.2)])
+    ax.set_yticks(np.arange(0, 1.01, 0.2))
+    ax.set_yticklabels([f"{int(v*100)}%" for v in np.arange(0, 1.01, 0.2)])
+    ax.set_xlabel("Should-refuse TPR  ·  reliability on clearly-dangerous prompts",
+                  fontsize=10)
+    ax.set_ylabel("Benign-prompt refusal rate  ·  over-refusal on safe prompts",
+                  fontsize=10)
+    _style_axes(ax)
+
+    # Vertical span indicator showing the spread, anchored just outside the
+    # right edge of the PC-A dot column.
+    span_x = 1.045
+    span_lo, span_hi = 0.03, 0.915
+    ax.annotate("", xy=(span_x, span_hi), xytext=(span_x, span_lo),
+                arrowprops=dict(arrowstyle="<->", color="#EF4444",
+                                lw=1.4, alpha=0.85), zorder=6)
+    ax.text(span_x + 0.005, (span_lo + span_hi) / 2,
+            "91 pp",
+            fontsize=10, color="#EF4444", fontweight="700",
+            ha="left", va="center", rotation=90, zorder=6)
+
+    # Headline callout in the empty upper-middle region — no arrow needed.
+    ax.text(
+        0.42, 0.88,
+        "Same should-refuse TPR (≥ 95 %).\n"
+        "Eight-fold benign-refusal spread.",
+        fontsize=10.5, color="#CBD5E1", fontweight="600",
+        ha="center", va="center", zorder=5,
+        bbox=dict(boxstyle="round,pad=0.5",
+                  facecolor="none", edgecolor="#475569",
+                  linewidth=0.7, alpha=0.6),
+    )
+
+    fig.tight_layout()
+    return fig
+
+
+def make_fig_tpr_bars(pc: pd.DataFrame) -> plt.Figure:
+    """Horizontal bars of should-refuse TPR, sorted descending, with tier markers."""
+    pc = pc.sort_values("tpr", ascending=True).reset_index(drop=True)
+    fig, ax = plt.subplots(figsize=(11, 5.8))
+
+    y = np.arange(len(pc))
+    bar_colors = [PC_TIER_COLORS.get(t, "#94A3B8") for t in pc["pc_tier"]]
+    ax.barh(y, pc["tpr"], color=bar_colors, alpha=0.92, height=0.72,
+            edgecolor="none", zorder=3)
+
+    # Wilson CI brackets
+    ax.errorbar(
+        pc["tpr"], y,
+        xerr=[pc["tpr"] - pc["ci_lo"], pc["ci_hi"] - pc["tpr"]],
+        fmt="none", color="#475569", capsize=3, linewidth=1.0,
+        alpha=0.7, zorder=4,
+    )
+
+    # Threshold lines
+    for x, label, color in [
+        (PC_THRESHOLDS["A"],      "A: ≥95%",   "#10B981"),
+        (PC_THRESHOLDS["B_high"], "B: 73%",    "#F59E0B"),
+        (PC_THRESHOLDS["B_low"],  "B: 9%",     "#F59E0B"),
+        (PC_THRESHOLDS["C"],      "C: ≤1.3%",  "#EF4444"),
+    ]:
+        ax.axvline(x, color=color, linewidth=0.7, linestyle="--",
+                   alpha=0.5, zorder=2)
+
+    ax.set_yticks(y)
+    ax.set_yticklabels(pc["model"], fontsize=9)
+    ax.set_xticks(np.arange(0, 1.01, 0.2))
+    ax.set_xticklabels([f"{int(v*100)}%" for v in np.arange(0, 1.01, 0.2)])
+    ax.set_xlim(0, 1.04)
+    ax.set_xlabel("Should-refuse TPR (Wilson 95 % CI, n = 75 trials per model)",
+                  fontsize=10)
+    _style_axes(ax)
+    ax.grid(axis="x", alpha=0.18, linewidth=0.7)
+    ax.grid(axis="y", visible=False)
+
+    # Tier legend
+    handles = [
+        mpatches.Patch(color=PC_TIER_COLORS["A"], label="PC Tier A  (≥ 95 %)"),
+        mpatches.Patch(color=PC_TIER_COLORS["B"], label="PC Tier B  (9–73 %)"),
+        mpatches.Patch(color=PC_TIER_COLORS["—"], label="Gap zone  (—)"),
+    ]
+    ax.legend(handles=handles, loc="lower right", fontsize=8.5,
+              frameon=False, labelcolor="#94A3B8")
+
+    fig.tight_layout()
+    return fig
+
+
 def make_fig1(stats: pd.DataFrame) -> plt.Figure:
     """Provider gradient — benign tier, sorted by rate descending."""
     sub = stats[stats["tier"] == "benign"].copy()
@@ -741,6 +933,11 @@ except Exception as exc:
     raise SystemExit(f"[RefusalBench Space] Failed to load stats: {exc}") from exc
 
 OVERALL_STATS = overall_stats(STATS)
+try:
+    PC_DATA = load_pc_data()
+except FileNotFoundError:
+    PC_DATA = None  # calibration tab will degrade gracefully
+
 _LO, _HI, _LO_MODEL, _HI_MODEL = headline_spread(STATS)
 _SPREAD_PP = round((_HI - _LO) * 100)
 _N_TRIALS = int(STATS["n"].sum())
@@ -880,7 +1077,39 @@ with gr.Blocks(
             sort_dd.change(fn=update_leaderboard,
                            inputs=[jur_dd, sort_dd], outputs=leaderboard_html)
 
-        # ── Tab 2: Provider figures ────────────────────────────────────────
+        # ── Tab 2: Calibration (should-refuse positive control) ───────────
+        with gr.Tab("Calibration"):
+            gr.Markdown(
+                "**The PC-Tier anchor.** Every model in the leaderboard is also evaluated "
+                "on 75 clearly-dangerous should-refuse prompts (15 prompts × 5 trials). "
+                "Refusing those is the *floor* of safety calibration — failing it puts a "
+                "model below the safety bar regardless of how it behaves on the main sweep."
+            )
+
+            if PC_DATA is not None:
+                gr.Markdown(
+                    "**Calibration scatter.** Each dot is one model. The x-axis is "
+                    "should-refuse TPR (right = reliably refuses dangerous prompts); "
+                    "the y-axis is benign-prompt refusal rate (high = over-refuses safe "
+                    "prompts). Dashed lines mark the PC-Tier cutoffs. **If refusal rate "
+                    "were safety calibration, dots would line up diagonally. They don't.**"
+                )
+                gr.Plot(value=make_fig_calibration(STATS, PC_DATA))
+
+                gr.Markdown(
+                    "**Per-model TPR.** Wilson 95 % CI over 75 trials. "
+                    "Tier A = reliably refuses (≥ 95 %); Tier B = intermediate (9–73 %); "
+                    "the 73 %–95 % and 1.3 %–9 % bands are the *gap zones* "
+                    "(no model in v1.1 lands inside them; PC-Tier — labels are reserved "
+                    "for the gap-zone interpretation)."
+                )
+                gr.Plot(value=make_fig_tpr_bars(PC_DATA))
+            else:
+                gr.Markdown(
+                    "_Should-refuse data unavailable (data/should_refuse_sweep.csv missing)._"
+                )
+
+        # ── Tab 3: Provider figures ────────────────────────────────────────
         with gr.Tab("Provider Analysis"):
             gr.Markdown(
                 "**Figure 1.** Benign-tier strict refusal rate for all 19 models, "