Upload folder using huggingface_hub

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/Q_shared_layer10_seed123.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:906481c53730b1c5b8c1aabc43e90570c76385cd57e7a2a1a4ea6b81d5d5788c
+size 2723968

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/Q_shared_layer10_seed456.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ab2ecc1600f61fefe53f12f2b1c8014973c7c4b92cd815d6d3e19024457673c5
+size 2422912

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/_summary/alpha_sweep.csv

@@ -0,0 +1,18 @@
+file,task,layer,seed,alpha,n,flip_rate,ablated_acc,pred_change_rate,mean_margin,mean_delta_margin_vs_baseline
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.0,64,0.0,1.0,0.0,2.773576259613037,0.0
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.02,64,0.0,1.0,0.0,2.7706217765808105,-0.002954694442451
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.05,64,0.015625,0.984375,0.015625,2.7653543949127197,-0.008222113363444805
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.1,64,0.03125,0.96875,0.03125,2.7537312507629395,-0.019845208153128624
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.2,64,0.046875,0.953125,0.046875,2.7078418731689453,-0.0657346323132515
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.3,64,0.0625,0.9375,0.0625,2.611804962158203,-0.1617715060710907
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.5,64,0.15625,0.84375,0.15625,2.0545363426208496,-0.7190403342247009
+aqua_alpha_sweep_seed123.json,aqua,10,123,0.75,64,0.375,0.625,0.375,1.0690979957580566,-1.7044786214828491
+aqua_alpha_sweep_seed123.json,aqua,10,123,1.0,64,1.0,0.0,1.0,-2.415497064590454,-5.18907356262207
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.0,65,0.0,1.0,0.0,2.5807881355285645,0.0
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.05,65,0.0,1.0,0.0,2.557753562927246,-0.023034870624542236
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.1,65,0.0,1.0,0.0,2.530287265777588,-0.050501130521297455
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.2,65,0.015384615384615385,0.9846153846153847,0.015384615384615385,2.4566214084625244,-0.12416719645261765
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.3,65,0.09230769230769231,0.9076923076923077,0.09230769230769231,2.3547489643096924,-0.22603949904441833
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.5,65,0.2153846153846154,0.7846153846153846,0.2153846153846154,1.979431390762329,-0.601357102394104
+aqua_alpha_sweep_seed456.json,aqua,10,456,0.75,65,0.3230769230769231,0.676923076923077,0.3230769230769231,1.2684406042099,-1.3123478889465332
+aqua_alpha_sweep_seed456.json,aqua,10,456,1.0,65,1.0,0.0,1.0,-2.193070411682129,-4.773859024047852

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/_summary/alpha_sweep.md

@@ -0,0 +1,19 @@
+| task | file | alpha | n | flip_rate | ablated_acc | mean_delta_margin_vs_baseline |
+| --- | --- | --- | --- | --- | --- | --- |
+| aqua | aqua_alpha_sweep_seed123.json | 0.0 | 64 | 0.0 | 100.0 | 0.000 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.02 | 64 | 0.0 | 100.0 | -0.003 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.05 | 64 | 1.6 | 98.4 | -0.008 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.1 | 64 | 3.1 | 96.9 | -0.020 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.2 | 64 | 4.7 | 95.3 | -0.066 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.3 | 64 | 6.2 | 93.8 | -0.162 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.5 | 64 | 15.6 | 84.4 | -0.719 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.75 | 64 | 37.5 | 62.5 | -1.704 |
+| aqua | aqua_alpha_sweep_seed123.json | 1.0 | 64 | 100.0 | 0.0 | -5.189 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.0 | 65 | 0.0 | 100.0 | 0.000 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.05 | 65 | 0.0 | 100.0 | -0.023 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.1 | 65 | 0.0 | 100.0 | -0.051 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.2 | 65 | 1.5 | 98.5 | -0.124 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.3 | 65 | 9.2 | 90.8 | -0.226 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.5 | 65 | 21.5 | 78.5 | -0.601 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.75 | 65 | 32.3 | 67.7 | -1.312 |
+| aqua | aqua_alpha_sweep_seed456.json | 1.0 | 65 | 100.0 | 0.0 | -4.774 |

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/_summary/analyze_qwen_results.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+SUMMARY_CSV = os.environ.get("SUMMARY_CSV", "summary.csv")
+ALPHA_CSV = os.environ.get("ALPHA_CSV", "alpha_sweep.csv")
+OUT_MD = os.environ.get("OUT_MD", "qwen_report.md")
+OUT_TEX = os.environ.get("OUT_TEX", "qwen_tables.tex")
+PLOT_DIR = os.environ.get("PLOT_DIR", "plots")
+
+os.makedirs(PLOT_DIR, exist_ok=True)
+
+df = pd.read_csv(SUMMARY_CSV)
+alpha_df = pd.read_csv(ALPHA_CSV) if os.path.exists(ALPHA_CSV) else pd.DataFrame()
+
+# Helpers
+def f(x, nd=3):
+    if pd.isna(x): return ""
+    if isinstance(x, (int, np.integer)): return str(int(x))
+    if isinstance(x, (float, np.floating)): return f"{float(x):.{nd}f}"
+    return str(x)
+
+def pct(x, nd=1):
+    if pd.isna(x): return ""
+    return f"{float(x):.{nd}f}"
+
+# Split by kind
+mc = df[df["kind"] == "subspace_mc"].copy()
+oa = df[df["kind"] == "openanswer"].copy()
+fs = df[df["kind"] == "flipset"].copy()
+
+# Choose representative columns
+def select_cols(kind_df, cols):
+    keep = [c for c in cols if c in kind_df.columns]
+    return kind_df[keep].copy()
+
+mc_cols = [
+    "task","eval_mode","seed","base_acc_scan","ablt_acc_scan","flips_scan",
+    "patched_0_rescued_pct","patched_full_rescued_pct",
+    "control_time_shuffled_rescued_pct","control_shared_randvec_rescued_pct",
+    "control_rand_subspace_rescued_pct","control_patch_nonshared_rescued_pct",
+]
+oa_cols = [
+    "task","eval_mode","seed","base_acc_scan","ablt_acc_scan","flips_scan",
+    "patched_self_rescued_pct","control_time_shuffled_rescued_pct",
+    "control_shared_randvec_rescued_pct","control_rand_subspace_rescued_pct",
+    "control_patch_nonshared_rescued_pct",
+]
+fs_cols = [
+    "file","seed","task","base_acc_scan","ablt_acc_scan","flips_scan",
+    "patched_self_rescued_pct","patched_transfer_rescued_pct",
+]
+
+mc_tbl = select_cols(mc, mc_cols).sort_values(["task","eval_mode","seed"])
+oa_tbl = select_cols(oa, oa_cols).sort_values(["task","eval_mode","seed"])
+fs_tbl = select_cols(fs, fs_cols).sort_values(["seed","file"])
+
+# ---- Plot 1: MC patched_0 rescue by task ----
+if len(mc_tbl) > 0 and "patched_0_rescued_pct" in mc_tbl.columns:
+    mc_plot = mc_tbl.groupby("task", as_index=False)["patched_0_rescued_pct"].mean()
+    plt.figure()
+    plt.bar(mc_plot["task"], mc_plot["patched_0_rescued_pct"])
+    plt.xticks(rotation=45, ha="right")
+    plt.ylabel("Rescue% on flips (patched_0)")
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "mc_patched0_rescue.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 2: MC controls gap (patched_0 vs rand-in-shared vs nonshared) ----
+if len(mc_tbl) > 0 and "patched_0_rescued_pct" in mc_tbl.columns:
+    tmp = mc_tbl.groupby("task", as_index=False).agg({
+        "patched_0_rescued_pct":"mean",
+        "control_shared_randvec_rescued_pct":"mean",
+        "control_patch_nonshared_rescued_pct":"mean",
+        "control_rand_subspace_rescued_pct":"mean",
+    })
+    plt.figure()
+    x = np.arange(len(tmp))
+    w = 0.2
+    plt.bar(x - 1.5*w, tmp["patched_0_rescued_pct"], width=w, label="patched_0")
+    plt.bar(x - 0.5*w, tmp["control_shared_randvec_rescued_pct"], width=w, label="rand vec in shared")
+    plt.bar(x + 0.5*w, tmp["control_rand_subspace_rescued_pct"], width=w, label="rand subspace")
+    plt.bar(x + 1.5*w, tmp["control_patch_nonshared_rescued_pct"], width=w, label="nonshared patch")
+    plt.xticks(x, tmp["task"], rotation=45, ha="right")
+    plt.ylabel("Rescue% on flips")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "mc_controls_gap.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 3: Alpha sweep flip_rate curves ----
+if len(alpha_df) > 0 and {"alpha","flip_rate","seed"}.issubset(alpha_df.columns):
+    alpha_df2 = alpha_df.copy()
+    alpha_df2["alpha"] = pd.to_numeric(alpha_df2["alpha"], errors="coerce")
+    alpha_df2 = alpha_df2.dropna(subset=["alpha"])
+    plt.figure()
+    for seed in sorted(alpha_df2["seed"].dropna().unique()):
+        sub = alpha_df2[alpha_df2["seed"] == seed].sort_values("alpha")
+        plt.plot(sub["alpha"], sub["flip_rate"]*100.0, marker="o", label=f"seed={int(seed)}")
+    plt.xlabel("alpha")
+    plt.ylabel("Flip rate on flip-set (%)")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "alpha_sweep_fliprate.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 4: Alpha sweep mean delta margin curves ----
+if len(alpha_df) > 0 and {"alpha","mean_delta_margin_vs_baseline","seed"}.issubset(alpha_df.columns):
+    alpha_df2 = alpha_df.copy()
+    alpha_df2["alpha"] = pd.to_numeric(alpha_df2["alpha"], errors="coerce")
+    alpha_df2 = alpha_df2.dropna(subset=["alpha"])
+    plt.figure()
+    for seed in sorted(alpha_df2["seed"].dropna().unique()):
+        sub = alpha_df2[alpha_df2["seed"] == seed].sort_values("alpha")
+        plt.plot(sub["alpha"], sub["mean_delta_margin_vs_baseline"], marker="o", label=f"seed={int(seed)}")
+    plt.xlabel("alpha")
+    plt.ylabel("Mean Δmargin vs baseline (on flip-set)")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "alpha_sweep_deltam.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 5: Open-answer patched_self rescue (pair_logprob vs gen) ----
+if len(oa_tbl) > 0 and "patched_self_rescued_pct" in oa_tbl.columns:
+    oa_plot = oa_tbl.copy()
+    oa_plot["label"] = oa_plot["task"].astype(str) + ":" + oa_plot["eval_mode"].astype(str)
+    plt.figure()
+    plt.bar(oa_plot["label"], oa_plot["patched_self_rescued_pct"])
+    plt.xticks(rotation=45, ha="right")
+    plt.ylabel("Rescue% on flips (patched_self)")
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "openanswer_patchedself_rescue.pdf"), dpi=300)
+    plt.close()
+
+# ---- Markdown report ----
+def df_to_md_table(dfx: pd.DataFrame, max_rows: int = 30) -> str:
+    if dfx is None or len(dfx) == 0:
+        return "_(none)_"
+    d = dfx.copy()
+    if len(d) > max_rows:
+        d = d.head(max_rows)
+    return d.to_markdown(index=False)
+
+lines = []
+lines.append(f"# Qwen subspace patching + flipset report\n")
+lines.append(f"Generated from `{os.path.basename(SUMMARY_CSV)}` and `{os.path.basename(ALPHA_CSV)}`.\n")
+lines.append("## Overview\n")
+lines.append(f"- Runs: {len(df)} total JSON summaries\n")
+lines.append(f"- MC runs: {len(mc)}; Open-answer runs: {len(oa)}; Flipset runs: {len(fs)}\n")
+lines.append("## Key plots (PDF, dpi=300)\n")
+for fn in [
+    "mc_patched0_rescue.pdf",
+    "mc_controls_gap.pdf",
+    "alpha_sweep_fliprate.pdf",
+    "alpha_sweep_deltam.pdf",
+    "openanswer_patchedself_rescue.pdf",
+]:
+    p = os.path.join(PLOT_DIR, fn)
+    if os.path.exists(p):
+        lines.append(f"- `{fn}`")
+lines.append("\n")
+
+lines.append("## Multiple-choice patchback (subspace_mc)\n")
+lines.append(df_to_md_table(mc_tbl))
+lines.append("\n")
+
+lines.append("## Open-answer patchback (openanswer)\n")
+lines.append(df_to_md_table(oa_tbl))
+lines.append("\n")
+
+lines.append("## Flipset transfer patching (flipset)\n")
+lines.append(df_to_md_table(fs_tbl))
+lines.append("\n")
+
+if len(alpha_df) > 0:
+    lines.append("## Alpha sweep (flip-set)\n")
+    # show a compact subset (alpha=0,0.5,0.75,1.0) if present
+    a = alpha_df.copy()
+    a["alpha"] = pd.to_numeric(a["alpha"], errors="coerce")
+    a = a.dropna(subset=["alpha"])
+    keep = a[a["alpha"].isin([0.0, 0.5, 0.75, 1.0])].copy()
+    if len(keep) == 0:
+        keep = a
+    keep = keep.sort_values(["seed","alpha"])
+    cols = [c for c in ["file","seed","alpha","n","flip_rate","ablated_acc","mean_delta_margin_vs_baseline"] if c in keep.columns]
+    lines.append(df_to_md_table(keep[cols], max_rows=60))
+    lines.append("\n")
+
+with open(OUT_MD, "w", encoding="utf-8") as f:
+    f.write("\n".join(lines))
+
+# ---- LaTeX tables (quick export) ----
+# We write two compact tables: MC summary + Open-answer summary
+def to_latex_table(dfx: pd.DataFrame, caption: str, label: str) -> str:
+    if dfx is None or len(dfx) == 0:
+        return f"% {caption}\n% (empty)\n"
+    return dfx.to_latex(index=False, escape=True, caption=caption, label=label)
+
+tex_lines = []
+tex_lines.append("% Auto-generated LaTeX tables for Qwen results\n")
+tex_lines.append("% Requires \\usepackage{booktabs}\n\n")
+
+if len(mc_tbl) > 0:
+    tex_lines.append(to_latex_table(
+        mc_tbl,
+        caption="Qwen: multiple-choice (subspace\\Apatch) summary.",
+        label="tab:qwen_mc_summary"
+    ))
+    tex_lines.append("\n")
+
+if len(oa_tbl) > 0:
+    tex_lines.append(to_latex_table(
+        oa_tbl,
+        caption="Qwen: open-answer (openanswer\\_subspace\\_patching) summary.",
+        label="tab:qwen_openanswer_summary"
+    ))
+    tex_lines.append("\n")
+
+if len(alpha_df) > 0:
+    tex_lines.append(to_latex_table(
+        alpha_df.sort_values(["seed","alpha"]).head(40),
+        caption="Qwen: alpha sweep (first 40 rows shown).",
+        label="tab:qwen_alpha_sweep_head"
+    ))
+    tex_lines.append("\n")
+
+with open(OUT_TEX, "w", encoding="utf-8") as f:
+    f.write("\n".join(tex_lines))
+
+print(f"[OK] Wrote report: {OUT_MD}")
+print(f"[OK] Wrote LaTeX tables: {OUT_TEX}")
+print(f"[OK] Plots in: {PLOT_DIR}")

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/_summary/paper_table.md

@@ -0,0 +1,17 @@
+| kind | task | eval_mode | base_acc_scan | ablt_acc_scan | flips_scan | Patched@0 (rescue%, Δm) | Patched@full (rescue%, Δm) | Patched(self) (rescue%, Δm) | Patched(transfer) (rescue%, Δm) | Cross-example donor (rescue%, Δm) | Donor mismatch (rescue%, Δm) | Shared coeff permute (rescue%, Δm) | Shared coeff signflip (rescue%, Δm) | Rand vec in shared (rescue%, Δm) | Rand subspace (rescue%, Δm) | Nonshared patch (rescue%, Δm) |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| flipset | aqua |  | 0.413 | 0.276 | 64 |  |  |  |  |  |  |  |  |  |  |  |
+| flipset | aqua |  | 0.402 | 0.291 | 65 |  |  |  |  |  |  |  |  |  |  |  |
+| flipset | aqua |  | 0.413 | 0.276 | 64 |  |  | 100.0%, 5.189 | 95.3%, 5.191 |  |  |  |  |  |  |  |
+| flipset | aqua |  | 0.413 | 0.276 | 64 |  |  | 100.0%, 5.189 | 96.9%, 5.143 |  |  |  |  |  |  |  |
+| openanswer | gsm8k | gen_math | 0.035 | 0.059 | 7 |  |  | 14.3%, - |  | 14.3%, - |  |  |  | 14.3%, - | 0.0%, - | 42.9%, - |
+| openanswer | gsm8k | pair_logprob | 0.801 | 0.625 | 53 |  |  | 98.1%, 10.441 |  | 94.3%, 10.053 |  |  |  | 20.8%, 0.094 | 18.9%, 0.630 | 5.7%, 0.223 |
+| openanswer | humaneval | gen_code_compile |  |  | 0 |  |  | 10.5%, - |  | 10.5%, - |  |  |  | 15.8%, - | 26.3%, - | 0.0%, - |
+| openanswer | humaneval | pair_logprob | 0.683 | 0.695 | 5 |  |  | 80.0%, 3.919 |  | 80.0%, 2.955 |  |  |  | 0.0%, -0.728 | 20.0%, -0.119 | 0.0%, -0.053 |
+| subspace_mc | aqua |  | 0.413 | 0.276 | 64 | 100.0%, 5.189 | 100.0%, 5.189 |  |  | 98.4%, 5.171 |  |  |  | 20.3%, 0.773 | 35.9%, 1.421 | 0.0%, 0.000 |
+| subspace_mc | arc_challenge |  | 0.906 | 0.514 | 108 | 100.0%, 9.373 | 100.0%, 9.373 |  |  | 100.0%, 9.361 |  |  |  | 36.1%, 1.780 | 60.2%, 2.989 | 0.0%, -0.000 |
+| subspace_mc | commonsenseqa |  | 0.867 | 0.598 | 77 | 100.0%, 8.117 | 100.0%, 8.117 |  |  | 100.0%, 8.102 |  |  |  | 39.0%, 1.555 | 55.8%, 2.602 | 0.0%, -0.000 |
+| subspace_mc | logiqa |  | 0.473 | 0.395 | 45 | 100.0%, 5.974 | 100.0%, 5.974 |  |  | 100.0%, 6.011 |  |  |  | 35.6%, 0.588 | 40.0%, 0.821 | 0.0%, 0.000 |
+| subspace_mc | openbookqa |  | 0.859 | 0.457 | 107 | 100.0%, 8.290 | 100.0%, 8.290 |  |  | 99.1%, 8.249 |  |  |  | 33.6%, 1.499 | 47.7%, 2.272 | 0.0%, 0.000 |
+| subspace_mc | piqa |  | 0.871 | 0.773 | 37 | 100.0%, 5.723 | 100.0%, 5.723 |  |  | 100.0%, 5.689 |  |  |  | 56.8%, 1.174 | 48.6%, 1.417 | 0.0%, -0.000 |
+| subspace_mc | qasc |  | 0.809 | 0.402 | 110 | 100.0%, 8.833 | 100.0%, 8.833 |  |  | 100.0%, 8.808 |  |  |  | 39.1%, 1.842 | 50.0%, 2.400 | 0.0%, -0.000 |

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/_summary/summary.csv

@@ -0,0 +1,16 @@
+kind,file,task,eval_mode,layer,seed,hf_id,hf_split,candidate_labels,Qs_shape,patch_desc,donor_source,donor_tasks,donor_pick,n_donor_bank,scan_effective,scan_skipped,base_acc_scan,ablt_acc_scan,flips_scan,anti_flips_scan,both_correct_scan,both_wrong_scan,patched_primary_method,patched_primary_rescued_pct,patched_primary_mean_dmargin,diff_time_shuffled_minus_patched_primary_rescued_pct,diff_patched_primary_minus_shared_randvec_rescued_pct,patched_0_rescued,patched_0_n,patched_0_rescued_pct,patched_0_mean_dmargin,patched_01_rescued,patched_01_n,patched_01_rescued_pct,patched_01_mean_dmargin,patched_full_rescued,patched_full_n,patched_full_rescued_pct,patched_full_mean_dmargin,patched_self_rescued,patched_self_n,patched_self_rescued_pct,patched_self_mean_dmargin,patched_transfer_rescued,patched_transfer_n,patched_transfer_rescued_pct,patched_transfer_mean_dmargin,control_time_shuffled_rescued,control_time_shuffled_n,control_time_shuffled_rescued_pct,control_time_shuffled_mean_dmargin,control_shared_mismatch_rescued,control_shared_mismatch_n,control_shared_mismatch_rescued_pct,control_shared_mismatch_mean_dmargin,control_shared_perm_rescued,control_shared_perm_n,control_shared_perm_rescued_pct,control_shared_perm_mean_dmargin,control_shared_signflip_rescued,control_shared_signflip_n,control_shared_signflip_rescued_pct,control_shared_signflip_mean_dmargin,control_shared_randvec_rescued,control_shared_randvec_n,control_shared_randvec_rescued_pct,control_shared_randvec_mean_dmargin,control_rand_subspace_rescued,control_rand_subspace_n,control_rand_subspace_rescued_pct,control_rand_subspace_mean_dmargin,control_patch_nonshared_rescued,control_patch_nonshared_n,control_patch_nonshared_rescued_pct,control_patch_nonshared_mean_dmargin
+flipset,aqua_alpha_sweep_seed123.json,aqua,,10,123,,,ABCDE,3584x190,steps=0,,,,,254,0,0.41338582677165353,0.2755905511811024,64,29,41,120,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flipset,aqua_alpha_sweep_seed456.json,aqua,,10,456,,,ABCDE,3584x169,steps=0,,,,,254,0,0.4015748031496063,0.29133858267716534,65,37,37,115,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flipset,aqua_transfer_cross_mc_baselinecorrect_seed123.json,aqua,,10,123,,,ABCDE,3584x190,steps=0,cross_task_eval,"commonsenseqa,openbookqa",random,256,254,0,0.41338582677165353,0.2755905511811024,64,29,41,120,patched_self,100.0,5.189074993133545,,,,,,,,,,,,,,,64,64,100.0,5.189074993133545,61,64,95.3125,5.190681457519531,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flipset,aqua_transfer_same_task_seed123.json,aqua,,10,123,,,ABCDE,3584x190,steps=0,same_task_eval,aqua,random,254,254,0,0.41338582677165353,0.2755905511811024,64,29,41,120,patched_self,100.0,5.189074993133545,,,,,,,,,,,,,,,64,64,100.0,5.189074993133545,62,64,96.875,5.143482208251953,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+openanswer,gsm8k_genmath.json,gsm8k,gen_math,10,123,gsm8k/main,,,3584x190,"steps=0,1,2,3",,,,,256,0,0.03515625,0.05859375,7,13,2,234,patched_self,14.285714285714286,,0.0,0.0,,,,,,,,,,,,,1,7,14.285714285714286,,,,,,1,7,14.285714285714286,,,,,,,,,,,,,,1,7,14.285714285714286,,0,7,0.0,,3,7,42.857142857142854,
+openanswer,gsm8k_pairlogprob.json,gsm8k,pair_logprob,10,123,gsm8k/main,,,3584x190,"steps=0,1,2,3",,,,,256,0,0.80078125,0.625,53,8,152,43,patched_self,98.11320754716981,10.441205978393555,-3.773584905660371,77.35849056603773,,,,,,,,,,,,,52,53,98.11320754716981,10.441205978393555,,,,,50,53,94.33962264150944,10.053207397460938,,,,,,,,,,,,,11,53,20.754716981132077,0.09415686130523682,10,53,18.867924528301888,0.6303381323814392,3,53,5.660377358490566,0.2226167768239975
+openanswer,humaneval_gencode_compile.json,humaneval,gen_code_compile,10,123,openai_humaneval,test,,3584x190,"steps=0,1,2,3",,,,,0,164,,,0,0,0,0,patched_self,10.526315789473685,,0.0,-5.263157894736841,,,,,,,,,,,,,2,19,10.526315789473685,,,,,,2,19,10.526315789473685,,,,,,,,,,,,,,3,19,15.789473684210526,,5,19,26.31578947368421,,0,19,0.0,
+openanswer,humaneval_pairlogprob.json,humaneval,pair_logprob,10,123,openai_humaneval,test,,3584x190,"steps=0,1,2,3",,,,,164,0,0.6829268292682927,0.6951219512195121,5,7,107,45,patched_self,80.0,3.919372081756592,0.0,80.0,,,,,,,,,,,,,4,5,80.0,3.919372081756592,,,,,4,5,80.0,2.954876661300659,,,,,,,,,,,,,0,5,0.0,-0.7276772260665894,1,5,20.0,-0.11909005790948868,0,5,0.0,-0.052741408348083496
+subspace_mc,aqua.json,aqua,,10,123,,,ABCDE,3584x190,,,,,,254,0,0.41338582677165353,0.2755905511811024,64,29,41,120,patched_0,100.0,5.1890749065205455,-1.5625,79.6875,64,64,100.0,5.1890749065205455,64,64,100.0,5.1890749065205455,64,64,100.0,5.1890749065205455,,,,,,,,,63,64,98.4375,5.171447346918285,,,,,,,,,,,,,13,64,20.3125,0.7729076333343983,23,64,35.9375,1.4208112843334675,0,64,0.0,2.1606683731079102e-07
+subspace_mc,arc_challenge.json,arc_challenge,,10,123,,,ABCD,3584x190,,,,,,255,1,0.9058823529411765,0.5137254901960784,108,8,123,16,patched_0,100.0,9.372807964682579,0.0,63.888888888888886,108,108,100.0,9.372807964682579,108,108,100.0,9.372807964682579,108,108,100.0,9.372807964682579,,,,,,,,,108,108,100.0,9.361393508535844,,,,,,,,,,,,,39,108,36.111111111111114,1.7799473873994969,65,108,60.18518518518518,2.988849757446183,0,108,0.0,-5.342342235423901e-07
+subspace_mc,commonsenseqa.json,commonsenseqa,,10,123,,,ABCDE,3584x190,,,,,,256,0,0.8671875,0.59765625,77,8,145,26,patched_0,100.0,8.11723030232764,0.0,61.03896103896104,77,77,100.0,8.11723030232764,77,77,100.0,8.11723030232764,77,77,100.0,8.11723030232764,,,,,,,,,77,77,100.0,8.101711891688309,,,,,,,,,,,,,30,77,38.96103896103896,1.5545239510474267,43,77,55.84415584415584,2.602022914143352,0,77,0.0,-9.103254838423296e-07
+subspace_mc,logiqa.json,logiqa,,10,123,,,ABCD,3584x190,,,,,,256,0,0.47265625,0.39453125,45,25,76,110,patched_0,100.0,5.974016189575195,0.0,64.44444444444444,45,45,100.0,5.974016189575195,45,45,100.0,5.974016189575195,45,45,100.0,5.974016189575195,,,,,,,,,45,45,100.0,6.01071400642395,,,,,,,,,,,,,16,45,35.55555555555556,0.588188378016154,18,45,40.0,0.8205998208787706,0,45,0.0,6.092919243706597e-07
+subspace_mc,openbookqa.json,openbookqa,,10,123,,,ABCD,3584x190,,,,,,256,0,0.859375,0.45703125,107,4,113,32,patched_0,100.0,8.289876717830372,-0.9345794392523317,66.35514018691589,107,107,100.0,8.289876717830372,107,107,100.0,8.289876717830372,107,107,100.0,8.289876717830372,,,,,,,,,106,107,99.06542056074767,8.248568821733244,,,,,,,,,,,,,36,107,33.64485981308411,1.498662706847503,51,107,47.66355140186916,2.2723147490314233,0,107,0.0,5.214013785959404e-07
+subspace_mc,piqa.json,piqa,,10,123,,,AB,3584x190,,,,,,256,0,0.87109375,0.7734375,37,12,186,21,patched_0,100.0,5.722948335312508,0.0,43.24324324324324,37,37,100.0,5.722948335312508,37,37,100.0,5.722948335312508,37,37,100.0,5.722948335312508,,,,,,,,,37,37,100.0,5.689002320573136,,,,,,,,,,,,,21,37,56.75675675675676,1.1743023846600507,18,37,48.648648648648646,1.417075691996394,0,37,0.0,-3.2218726905616553e-07
+subspace_mc,qasc.json,qasc,,10,123,,,ABCDEFGH,3584x190,,,,,,256,0,0.80859375,0.40234375,110,6,97,43,patched_0,100.0,8.833465647697448,0.0,60.90909090909091,110,110,100.0,8.833465647697448,110,110,100.0,8.833465647697448,110,110,100.0,8.833465647697448,,,,,,,,,110,110,100.0,8.807955605333502,,,,,,,,,,,,,43,110,39.09090909090909,1.8416864731095053,55,110,50.0,2.400287093899467,0,110,0.0,-6.198883056640625e-07

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/_summary/summary.md

@@ -0,0 +1,17 @@
+| kind | task | eval_mode | file | base_acc_scan | ablt_acc_scan | flips_scan | patched_primary_method | patched_primary_rescued_pct | control_time_shuffled_rescued_pct | control_shared_randvec_rescued_pct | control_rand_subspace_rescued_pct | control_patch_nonshared_rescued_pct |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| flipset | aqua |  | aqua_alpha_sweep_seed123.json | 0.413 | 0.276 | 64 |  |  |  |  |  |  |
+| flipset | aqua |  | aqua_alpha_sweep_seed456.json | 0.402 | 0.291 | 65 |  |  |  |  |  |  |
+| flipset | aqua |  | aqua_transfer_cross_mc_baselinecorrect_seed123.json | 0.413 | 0.276 | 64 | patched_self | 100.0 |  |  |  |  |
+| flipset | aqua |  | aqua_transfer_same_task_seed123.json | 0.413 | 0.276 | 64 | patched_self | 100.0 |  |  |  |  |
+| openanswer | gsm8k | gen_math | gsm8k_genmath.json | 0.035 | 0.059 | 7 | patched_self | 14.3 | 14.3 | 14.3 | 0.0 | 42.9 |
+| openanswer | gsm8k | pair_logprob | gsm8k_pairlogprob.json | 0.801 | 0.625 | 53 | patched_self | 98.1 | 94.3 | 20.8 | 18.9 | 5.7 |
+| openanswer | humaneval | gen_code_compile | humaneval_gencode_compile.json |  |  | 0 | patched_self | 10.5 | 10.5 | 15.8 | 26.3 | 0.0 |
+| openanswer | humaneval | pair_logprob | humaneval_pairlogprob.json | 0.683 | 0.695 | 5 | patched_self | 80.0 | 80.0 | 0.0 | 20.0 | 0.0 |
+| subspace_mc | aqua |  | aqua.json | 0.413 | 0.276 | 64 | patched_0 | 100.0 | 98.4 | 20.3 | 35.9 | 0.0 |
+| subspace_mc | arc_challenge |  | arc_challenge.json | 0.906 | 0.514 | 108 | patched_0 | 100.0 | 100.0 | 36.1 | 60.2 | 0.0 |
+| subspace_mc | commonsenseqa |  | commonsenseqa.json | 0.867 | 0.598 | 77 | patched_0 | 100.0 | 100.0 | 39.0 | 55.8 | 0.0 |
+| subspace_mc | logiqa |  | logiqa.json | 0.473 | 0.395 | 45 | patched_0 | 100.0 | 100.0 | 35.6 | 40.0 | 0.0 |
+| subspace_mc | openbookqa |  | openbookqa.json | 0.859 | 0.457 | 107 | patched_0 | 100.0 | 99.1 | 33.6 | 47.7 | 0.0 |
+| subspace_mc | piqa |  | piqa.json | 0.871 | 0.773 | 37 | patched_0 | 100.0 | 100.0 | 56.8 | 48.6 | 0.0 |
+| subspace_mc | qasc |  | qasc.json | 0.809 | 0.402 | 110 | patched_0 | 100.0 | 100.0 | 39.1 | 50.0 | 0.0 |

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/openanswer_seed123/gsm8k_genmath.json

@@ -0,0 +1,3888 @@
+{
+  "meta": {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "device": "cuda",
+    "dtype": "fp32",
+    "layer": 10,
+    "layers_path": "model.layers",
+    "seed": 123,
+    "task": "gsm8k",
+    "eval_mode": "gen_math",
+    "eval_meta": {
+      "subspace_split": null,
+      "eval_split": "test",
+      "available_splits": [
+        "train",
+        "test"
+      ],
+      "hf_id": "gsm8k/main"
+    },
+    "n_eval_loaded": 256,
+    "n_scanned": 256,
+    "base_acc_scan": 0.03515625,
+    "ablt_acc_scan": 0.05859375,
+    "flips_total": 7,
+    "flips_used": 7,
+    "patch_steps": [
+      0,
+      1,
+      2,
+      3
+    ],
+    "patch_n_steps": 4,
+    "Qs_path": "/home/zs89/decodeshare/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/Q_shared_layer10_seed123.npy",
+    "Qs_shape": [
+      3584,
+      190
+    ],
+    "gold_text_prefix": " ",
+    "dist_text_prefix": " ",
+    "gold_max_tokens": 0,
+    "distractor_mode": "next_gold",
+    "answer_prefix_effective": "\nLet's think step by step.\nFinal answer (number only):",
+    "max_new_tokens_effective": 64,
+    "run_coeff_controls": false,
+    "use_benchmark_loader": true,
+    "hf_id": "",
+    "hf_split": "test"
+  },
+  "summary_on_flips": {
+    "patched_self": {
+      "n": 7,
+      "rescued": 1,
+      "rescued_pct": 14.285714285714286
+    },
+    "control_time_shuffled": {
+      "n": 7,
+      "rescued": 1,
+      "rescued_pct": 14.285714285714286
+    },
+    "control_shared_randvec": {
+      "n": 7,
+      "rescued": 1,
+      "rescued_pct": 14.285714285714286
+    },
+    "control_rand_subspace": {
+      "n": 7,
+      "rescued": 0,
+      "rescued_pct": 0.0
+    },
+    "control_patch_nonshared": {
+      "n": 7,
+      "rescued": 3,
+      "rescued_pct": 42.857142857142854
+    }
+  },
+  "scan_rows": [
+    {
+      "ex_id": "gsm8k-test-0",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "140",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "70",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-1",
+      "gold_raw": "80",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-2",
+      "gold_raw": "12",
+      "baseline": {
+        "pred_answer": "7",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-3",
+      "gold_raw": "140",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "33",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-4",
+      "gold_raw": "36",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-5",
+      "gold_raw": "3200",
+      "baseline": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4800.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-6",
+      "gold_raw": "38",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "41",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-7",
+      "gold_raw": "32",
+      "baseline": {
+        "pred_answer": "15",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-8",
+      "gold_raw": "92",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-9",
+      "gold_raw": "16",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-10",
+      "gold_raw": "45",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "21.5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-11",
+      "gold_raw": "270",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "0.44704",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-12",
+      "gold_raw": "100",
+      "baseline": {
+        "pred_answer": "60",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-13",
+      "gold_raw": "25",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-14",
+      "gold_raw": "800",
+      "baseline": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-15",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "65",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-16",
+      "gold_raw": "7000",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-17",
+      "gold_raw": "25",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "42.15",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-18",
+      "gold_raw": "3",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-19",
+      "gold_raw": "3430",
+      "baseline": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2147483",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-20",
+      "gold_raw": "106",
+      "baseline": {
+        "pred_answer": "0",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-21",
+      "gold_raw": "80",
+      "baseline": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-22",
+      "gold_raw": "26",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-23",
+      "gold_raw": "750",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "59",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-24",
+      "gold_raw": "9",
+      "baseline": {
+        "pred_answer": "17",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "17",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-25",
+      "gold_raw": "40",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-26",
+      "gold_raw": "14",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "24.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-27",
+      "gold_raw": "160",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-28",
+      "gold_raw": "6",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-29",
+      "gold_raw": "132",
+      "baseline": {
+        "pred_answer": "16",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "16",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-30",
+      "gold_raw": "8",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5.000000000000000000000000000000000000000000000000000000000000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-31",
+      "gold_raw": "68",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-32",
+      "gold_raw": "31",
+      "baseline": {
+        "pred_answer": "0.02",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-33",
+      "gold_raw": "100",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-34",
+      "gold_raw": "1509",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-35",
+      "gold_raw": "480",
+      "baseline": {
+        "pred_answer": "35",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-36",
+      "gold_raw": "520",
+      "baseline": {
+        "pred_answer": "1.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3.00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-37",
+      "gold_raw": "3",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3.0",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-38",
+      "gold_raw": "33",
+      "baseline": {
+        "pred_answer": "32",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "28",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-39",
+      "gold_raw": "120",
+      "baseline": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "280",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-40",
+      "gold_raw": "14",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-41",
+      "gold_raw": "20",
+      "baseline": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "60",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-42",
+      "gold_raw": "95200",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-43",
+      "gold_raw": "77",
+      "baseline": {
+        "pred_answer": "11",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "42",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-44",
+      "gold_raw": "81",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "81",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-45",
+      "gold_raw": "310",
+      "baseline": {
+        "pred_answer": "24",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "255",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-46",
+      "gold_raw": "100",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-47",
+      "gold_raw": "160",
+      "baseline": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-48",
+      "gold_raw": "25",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-49",
+      "gold_raw": "1400",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-50",
+      "gold_raw": "120",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "120",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-51",
+      "gold_raw": "48",
+      "baseline": {
+        "pred_answer": "48",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-52",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-53",
+      "gold_raw": "15400",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1500",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-54",
+      "gold_raw": "80",
+      "baseline": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-55",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-56",
+      "gold_raw": "14",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-57",
+      "gold_raw": "31",
+      "baseline": {
+        "pred_answer": "7",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "7",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-58",
+      "gold_raw": "36",
+      "baseline": {
+        "pred_answer": "24",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-59",
+      "gold_raw": "144",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-60",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "33.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-61",
+      "gold_raw": "750",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "180",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-62",
+      "gold_raw": "38",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "48",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-63",
+      "gold_raw": "48",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-64",
+      "gold_raw": "655",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-65",
+      "gold_raw": "800",
+      "baseline": {
+        "pred_answer": "500",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-66",
+      "gold_raw": "7300",
+      "baseline": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-67",
+      "gold_raw": "48",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-68",
+      "gold_raw": "4",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-69",
+      "gold_raw": "15",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-70",
+      "gold_raw": "23",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-71",
+      "gold_raw": "225",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "25.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-72",
+      "gold_raw": "15",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "13",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-73",
+      "gold_raw": "82",
+      "baseline": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-74",
+      "gold_raw": "1218",
+      "baseline": {
+        "pred_answer": "48",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1024",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-75",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "2.00",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-76",
+      "gold_raw": "36",
+      "baseline": {
+        "pred_answer": "192",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "15",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-77",
+      "gold_raw": "13",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-78",
+      "gold_raw": "11",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-79",
+      "gold_raw": "8",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "8",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-80",
+      "gold_raw": "440",
+      "baseline": {
+        "pred_answer": "0.7",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "560",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-81",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-82",
+      "gold_raw": "45",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "9",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-83",
+      "gold_raw": "54",
+      "baseline": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "35",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-84",
+      "gold_raw": "6",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "9",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-85",
+      "gold_raw": "240",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12600.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-86",
+      "gold_raw": "428",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "278",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-87",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-88",
+      "gold_raw": "255",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "120",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-89",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-90",
+      "gold_raw": "9",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-91",
+      "gold_raw": "157",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-92",
+      "gold_raw": "56",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-93",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "0.5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-94",
+      "gold_raw": "144",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-95",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3600",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-96",
+      "gold_raw": "4",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-97",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "70",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-98",
+      "gold_raw": "42",
+      "baseline": {
+        "pred_answer": "10.00",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "30.00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-99",
+      "gold_raw": "7",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-100",
+      "gold_raw": "250",
+      "baseline": {
+        "pred_answer": "11",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-101",
+      "gold_raw": "12",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "29",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-102",
+      "gold_raw": "7",
+      "baseline": {
+        "pred_answer": "7",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "7.0",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-103",
+      "gold_raw": "8",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-104",
+      "gold_raw": "26",
+      "baseline": {
+        "pred_answer": "75",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "19",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-105",
+      "gold_raw": "42",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-106",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "15",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-107",
+      "gold_raw": "14400",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3640",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-108",
+      "gold_raw": "400",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "15",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-109",
+      "gold_raw": "40",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-110",
+      "gold_raw": "83",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-111",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-112",
+      "gold_raw": "80",
+      "baseline": {
+        "pred_answer": "50",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-113",
+      "gold_raw": "180",
+      "baseline": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5.00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-114",
+      "gold_raw": "1450000",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-115",
+      "gold_raw": "15",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-116",
+      "gold_raw": "1000",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1000",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-117",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-118",
+      "gold_raw": "15",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-119",
+      "gold_raw": "100",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "250",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-120",
+      "gold_raw": "335",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-121",
+      "gold_raw": "60",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-122",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-123",
+      "gold_raw": "9500",
+      "baseline": {
+        "pred_answer": "000",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3400",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-124",
+      "gold_raw": "160",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-125",
+      "gold_raw": "1050",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "14",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-126",
+      "gold_raw": "91",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-127",
+      "gold_raw": "21",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "11844329550533333923277761503275729703614298252421958262785",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-128",
+      "gold_raw": "20",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "24.0000000000000000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-129",
+      "gold_raw": "36",
+      "baseline": {
+        "pred_answer": "38",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "400",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-130",
+      "gold_raw": "36",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-131",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-132",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-133",
+      "gold_raw": "32",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "25",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-134",
+      "gold_raw": "18",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-135",
+      "gold_raw": "4",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-136",
+      "gold_raw": "48",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-137",
+      "gold_raw": "8",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-138",
+      "gold_raw": "21",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "21",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-139",
+      "gold_raw": "25",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-140",
+      "gold_raw": "3000",
+      "baseline": {
+        "pred_answer": "000",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "7000.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-141",
+      "gold_raw": "40",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-142",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "28",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "34",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-143",
+      "gold_raw": "90",
+      "baseline": {
+        "pred_answer": "75",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-144",
+      "gold_raw": "23",
+      "baseline": {
+        "pred_answer": "70",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-145",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "50",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": ". \n:\nTo find the the number of watermelons, we need to know the the the the number of watermelons, we need to know the the the number of watermelons, we need to know the the the number of watermelons, we need to know the the the number of watermel",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-146",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-147",
+      "gold_raw": "122",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-148",
+      "gold_raw": "300",
+      "baseline": {
+        "pred_answer": "400",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "700",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-149",
+      "gold_raw": "448",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-150",
+      "gold_raw": "2450",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "250",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-151",
+      "gold_raw": "803",
+      "baseline": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-152",
+      "gold_raw": "16",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-153",
+      "gold_raw": "280",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-154",
+      "gold_raw": "13",
+      "baseline": {
+        "pred_answer": "1.5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-155",
+      "gold_raw": "20",
+      "baseline": {
+        "pred_answer": "90",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-156",
+      "gold_raw": "14",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-157",
+      "gold_raw": "32",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-158",
+      "gold_raw": "105",
+      "baseline": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-159",
+      "gold_raw": "71",
+      "baseline": {
+        "pred_answer": "50",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "00.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-160",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-161",
+      "gold_raw": "30",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-162",
+      "gold_raw": "95",
+      "baseline": {
+        "pred_answer": "60",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "900.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-163",
+      "gold_raw": "147",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-164",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1.20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-165",
+      "gold_raw": "40000",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-166",
+      "gold_raw": "12",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "24.0000000000000000000000000000000000000000000000000000000024",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-167",
+      "gold_raw": "129200",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-168",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "10",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-169",
+      "gold_raw": "45",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "45",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-170",
+      "gold_raw": "20",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-171",
+      "gold_raw": "1170",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-172",
+      "gold_raw": "192",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "24",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-173",
+      "gold_raw": "14",
+      "baseline": {
+        "pred_answer": "0.5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "10.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-174",
+      "gold_raw": "144",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-175",
+      "gold_raw": "350",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-176",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "25",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-177",
+      "gold_raw": "7",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-178",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "40.000000000000000000000000000000000000000000000000000000001",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-179",
+      "gold_raw": "8",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-180",
+      "gold_raw": "3160",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-181",
+      "gold_raw": "80",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "7",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-182",
+      "gold_raw": "50",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "25",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-183",
+      "gold_raw": "40",
+      "baseline": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-184",
+      "gold_raw": "78",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-185",
+      "gold_raw": "273",
+      "baseline": {
+        "pred_answer": "364",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "147.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-186",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "11",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-187",
+      "gold_raw": "195",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-188",
+      "gold_raw": "1128",
+      "baseline": {
+        "pred_answer": "72",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-189",
+      "gold_raw": "172",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-190",
+      "gold_raw": "30",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "35",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-191",
+      "gold_raw": "30",
+      "baseline": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "19",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-192",
+      "gold_raw": "92",
+      "baseline": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-193",
+      "gold_raw": "20",
+      "baseline": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-194",
+      "gold_raw": "540",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-195",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-196",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "40",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "14",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-197",
+      "gold_raw": "38",
+      "baseline": {
+        "pred_answer": "14",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "14",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-198",
+      "gold_raw": "4000",
+      "baseline": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-199",
+      "gold_raw": "594",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "200",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-200",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-201",
+      "gold_raw": "142",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "17",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-202",
+      "gold_raw": "9",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-203",
+      "gold_raw": "6",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4.000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-204",
+      "gold_raw": "100",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-205",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "21.000000000000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-206",
+      "gold_raw": "15",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-207",
+      "gold_raw": "22",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "21",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-208",
+      "gold_raw": "16",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-209",
+      "gold_raw": "16",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "21.00000000000000000000000000000000000000000000000000000000000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-210",
+      "gold_raw": "5",
+      "baseline": {
+        "pred_answer": "1.5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-211",
+      "gold_raw": "23",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-212",
+      "gold_raw": "30",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-213",
+      "gold_raw": "14000",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6400",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-214",
+      "gold_raw": "60",
+      "baseline": {
+        "pred_answer": "16",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-215",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "100",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-216",
+      "gold_raw": "3",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-217",
+      "gold_raw": "30",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "123",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-218",
+      "gold_raw": "1920",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-219",
+      "gold_raw": "84",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "124",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-220",
+      "gold_raw": "8",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3.1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-221",
+      "gold_raw": "12",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "18",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-222",
+      "gold_raw": "260",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "120",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-223",
+      "gold_raw": "288",
+      "baseline": {
+        "pred_answer": "30",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "270",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-224",
+      "gold_raw": "3",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-225",
+      "gold_raw": "1596",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "176",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-226",
+      "gold_raw": "81",
+      "baseline": {
+        "pred_answer": "8",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-227",
+      "gold_raw": "56",
+      "baseline": {
+        "pred_answer": "38",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-228",
+      "gold_raw": "1490",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1200",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-229",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": true,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-230",
+      "gold_raw": "20",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "14",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-231",
+      "gold_raw": "11",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-232",
+      "gold_raw": "120",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-233",
+      "gold_raw": "45",
+      "baseline": {
+        "pred_answer": "60",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-234",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-235",
+      "gold_raw": "9",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "0.25",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-236",
+      "gold_raw": "33",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-237",
+      "gold_raw": "150",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "250.00",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-238",
+      "gold_raw": "60",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-239",
+      "gold_raw": "4",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-240",
+      "gold_raw": "7",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-241",
+      "gold_raw": "3140",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-242",
+      "gold_raw": "19",
+      "baseline": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "25262525",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-243",
+      "gold_raw": "6",
+      "baseline": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-244",
+      "gold_raw": "90",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-245",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "1.5",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "12.0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-246",
+      "gold_raw": "130000",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-247",
+      "gold_raw": "10",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-248",
+      "gold_raw": "525",
+      "baseline": {
+        "pred_answer": "31",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "31",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-249",
+      "gold_raw": "180",
+      "baseline": {
+        "pred_answer": "130",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "20",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-250",
+      "gold_raw": "1200",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "1200",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-251",
+      "gold_raw": "25",
+      "baseline": {
+        "pred_answer": "45",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-252",
+      "gold_raw": "21",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "4.00000000000",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-253",
+      "gold_raw": "2304",
+      "baseline": {
+        "pred_answer": "12",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "0",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-254",
+      "gold_raw": "2325",
+      "baseline": {
+        "pred_answer": "4",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2325",
+        "correct": true,
+        "n_gen_tokens": 64
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-255",
+      "gold_raw": "15",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": false,
+        "n_gen_tokens": 64
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false,
+        "n_gen_tokens": 64
+      }
+    }
+  ],
+  "flip_rows": [
+    {
+      "ex_id": "gsm8k-test-51",
+      "gold_raw": "48",
+      "baseline": {
+        "pred_answer": "48",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "12",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "5",
+        "correct": false
+      },
+      "control_time_shuffled": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_shared_randvec": {
+        "pred_answer": "2.0",
+        "correct": false
+      },
+      "control_rand_subspace": {
+        "pred_answer": "12",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "1",
+        "correct": false
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-75",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "2.00",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "6",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "2.00",
+        "correct": true
+      },
+      "control_time_shuffled": {
+        "pred_answer": "2",
+        "correct": true
+      },
+      "control_shared_randvec": {
+        "pred_answer": "2023",
+        "correct": false
+      },
+      "control_rand_subspace": {
+        "pred_answer": "10.00000000000000000000000000000000000001231029995651998697525",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "6",
+        "correct": false
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-84",
+      "gold_raw": "6",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "9",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "9",
+        "correct": false
+      },
+      "control_time_shuffled": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_shared_randvec": {
+        "pred_answer": "27",
+        "correct": false
+      },
+      "control_rand_subspace": {
+        "pred_answer": "9",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "0.0",
+        "correct": false
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-117",
+      "gold_raw": "2",
+      "baseline": {
+        "pred_answer": "2",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "10",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_time_shuffled": {
+        "pred_answer": "10",
+        "correct": false
+      },
+      "control_shared_randvec": {
+        "pred_answer": "4",
+        "correct": false
+      },
+      "control_rand_subspace": {
+        "pred_answer": "00",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "2.00",
+        "correct": true
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-203",
+      "gold_raw": "6",
+      "baseline": {
+        "pred_answer": "6",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "4.000",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "3",
+        "correct": false
+      },
+      "control_time_shuffled": {
+        "pred_answer": "3",
+        "correct": false
+      },
+      "control_shared_randvec": {
+        "pred_answer": "6",
+        "correct": true
+      },
+      "control_rand_subspace": {
+        "pred_answer": "4.041666667",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "3",
+        "correct": false
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-216",
+      "gold_raw": "3",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "2",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_time_shuffled": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_shared_randvec": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_rand_subspace": {
+        "pred_answer": "1.5",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "3",
+        "correct": true
+      }
+    },
+    {
+      "ex_id": "gsm8k-test-224",
+      "gold_raw": "3",
+      "baseline": {
+        "pred_answer": "3",
+        "correct": true
+      },
+      "ablated": {
+        "pred_answer": "5",
+        "correct": false
+      },
+      "patched_self": {
+        "pred_answer": "4",
+        "correct": false
+      },
+      "control_time_shuffled": {
+        "pred_answer": "4.00",
+        "correct": false
+      },
+      "control_shared_randvec": {
+        "pred_answer": "12",
+        "correct": false
+      },
+      "control_rand_subspace": {
+        "pred_answer": "1",
+        "correct": false
+      },
+      "control_patch_nonshared": {
+        "pred_answer": "3.0",
+        "correct": true
+      }
+    }
+  ]
+}

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/openanswer_seed123/humaneval_gencode_compile.json

@@ -0,0 +1,2336 @@
+{
+  "meta": {
+    "model": "Qwen/Qwen2.5-7B-Instruct",
+    "device": "cuda",
+    "dtype": "fp32",
+    "layer": 10,
+    "layers_path": "model.layers",
+    "seed": 123,
+    "task": "humaneval",
+    "eval_mode": "gen_code_compile",
+    "eval_meta": {
+      "hf_id": "openai_humaneval",
+      "split": "test",
+      "n_total": 164
+    },
+    "n_eval_loaded": 164,
+    "n_scanned": 164,
+    "base_acc_scan": 0.13414634146341464,
+    "ablt_acc_scan": 0.23780487804878048,
+    "flips_total": 19,
+    "flips_used": 19,
+    "patch_steps": [
+      0,
+      1,
+      2,
+      3
+    ],
+    "patch_n_steps": 4,
+    "Qs_path": "/home/zs89/decodeshare/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer10/Q_shared_layer10_seed123.npy",
+    "Qs_shape": [
+      3584,
+      190
+    ],
+    "gold_text_prefix": " ",
+    "dist_text_prefix": " ",
+    "gold_max_tokens": 0,
+    "distractor_mode": "next_gold",
+    "answer_prefix_effective": "\nFinal answer:",
+    "max_new_tokens_effective": 256,
+    "run_coeff_controls": false,
+    "use_benchmark_loader": false,
+    "hf_id": "openai_humaneval",
+    "hf_split": "test"
+  },
+  "summary_on_flips": {
+    "patched_self": {
+      "n": 19,
+      "rescued": 2,
+      "rescued_pct": 10.526315789473685
+    },
+    "control_time_shuffled": {
+      "n": 19,
+      "rescued": 2,
+      "rescued_pct": 10.526315789473685
+    },
+    "control_shared_randvec": {
+      "n": 19,
+      "rescued": 3,
+      "rescued_pct": 15.789473684210526
+    },
+    "control_rand_subspace": {
+      "n": 19,
+      "rescued": 5,
+      "rescued_pct": 26.31578947368421
+    },
+    "control_patch_nonshared": {
+      "n": 19,
+      "rescued": 0,
+      "rescued_pct": 0.0
+    }
+  },
+  "scan_rows": [
+    {
+      "ex_id": "openai_humaneval-test-18",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-31",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-158",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-43",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-39",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-15",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-151",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-101",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-34",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-52",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-113",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-119",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-83",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-116",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-56",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-131",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-1",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-159",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-123",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-23",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-124",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-38",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-84",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-41",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-134",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-2",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-80",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-74",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-162",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-138",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-87",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-145",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-54",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-109",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-102",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-62",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-129",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-110",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-4",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-8",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-97",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-137",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-96",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-65",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-114",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-117",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-155",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-37",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-115",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-86",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-85",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-53",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-130",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-139",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-81",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-19",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-69",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-122",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-108",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-48",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-17",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-95",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-91",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-79",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-14",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-77",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-3",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-146",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-57",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-143",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-0",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-106",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-50",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-58",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-147",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-160",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-67",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-55",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-118",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-154",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-25",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-36",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-63",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-132",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-21",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-9",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-72",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-128",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-126",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-70",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-40",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-66",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-71",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-107",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-32",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-11",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-47",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-22",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-140",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-30",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-10",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-112",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-111",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-28",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-135",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-94",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-13",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-7",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-157",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-49",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-120",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-89",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-98",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-100",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-64",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-103",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-125",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-51",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-90",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-152",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-24",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-20",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-148",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-92",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-26",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-16",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-142",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-99",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-127",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-156",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-141",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-78",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-68",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-93",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-60",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-82",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-59",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-149",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-42",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-163",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-133",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-161",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-29",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-27",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-61",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-104",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-45",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-75",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-6",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-136",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-33",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-44",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-88",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-12",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-105",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-121",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-144",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-73",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-76",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-5",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-46",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-150",
+      "baseline": {
+        "compile_ok": true,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-35",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-153",
+      "baseline": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      },
+      "ablated": {
+        "compile_ok": false,
+        "n_gen_tokens": 256
+      }
+    }
+  ],
+  "flip_rows": [
+    {
+      "ex_id": "openai_humaneval-test-74",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-87",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-14",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": true
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-58",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-47",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": true
+      },
+      "control_time_shuffled": {
+        "compile_ok": true
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-30",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-112",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-111",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-152",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": true
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-24",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-142",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-156",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": true
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-141",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": true
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-42",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": true
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-27",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": true
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-45",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-73",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": true
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-46",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": false
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": false
+      },
+      "control_rand_subspace": {
+        "compile_ok": false
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    },
+    {
+      "ex_id": "openai_humaneval-test-150",
+      "baseline": {
+        "compile_ok": true
+      },
+      "ablated": {
+        "compile_ok": false
+      },
+      "patched_self": {
+        "compile_ok": true
+      },
+      "control_time_shuffled": {
+        "compile_ok": false
+      },
+      "control_shared_randvec": {
+        "compile_ok": true
+      },
+      "control_rand_subspace": {
+        "compile_ok": true
+      },
+      "control_patch_nonshared": {
+        "compile_ok": false
+      }
+    }
+  ]
+}

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/Q_shared_layer24_seed123.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7a9018935fb6662a113283f4c171f976832435c2079df0e63e642751545ed5ae
+size 3197056

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/Q_shared_layer24_seed456.npy

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fad986416ec27af928bd2970b60b8a182807b2aea22f2685073b2b215f8e50e8
+size 3168384

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/alpha_sweep.csv

@@ -0,0 +1,18 @@
+file,task,layer,seed,alpha,n,flip_rate,ablated_acc,pred_change_rate,mean_margin,mean_delta_margin_vs_baseline
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.0,61,0.0,1.0,0.0,2.8609182834625244,0.0
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.02,61,0.01639344262295082,0.9836065573770492,0.01639344262295082,2.83201265335083,-0.028905684128403664
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.05,61,0.01639344262295082,0.9836065573770492,0.01639344262295082,2.7874269485473633,-0.0734911859035492
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.1,61,0.03278688524590164,0.9672131147540983,0.03278688524590164,2.709939956665039,-0.15097840130329132
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.2,61,0.04918032786885246,0.9508196721311475,0.04918032786885246,2.5366058349609375,-0.3243127763271332
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.3,61,0.11475409836065574,0.8852459016393442,0.11475409836065574,2.32297682762146,-0.5379418730735779
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.5,61,0.16393442622950818,0.8360655737704918,0.16393442622950818,1.717830777168274,-1.1430877447128296
+aqua_alpha_sweep_seed123.json,aqua,24,123,0.75,61,0.4098360655737705,0.5901639344262295,0.4098360655737705,0.5222399234771729,-2.3386785984039307
+aqua_alpha_sweep_seed123.json,aqua,24,123,1.0,61,1.0,0.0,1.0,-1.7653703689575195,-4.6262898445129395
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.0,45,0.0,1.0,0.0,2.238931179046631,0.0
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.05,45,0.0,1.0,0.0,2.1714553833007812,-0.06747613847255707
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.1,45,0.022222222222222223,0.9777777777777777,0.022222222222222223,2.0972023010253906,-0.14172926545143127
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.2,45,0.022222222222222223,0.9777777777777777,0.022222222222222223,1.9297974109649658,-0.30913400650024414
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.3,45,0.06666666666666667,0.9333333333333333,0.06666666666666667,1.7361918687820435,-0.5027394890785217
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.5,45,0.2222222222222222,0.7777777777777778,0.2222222222222222,1.229968547821045,-1.0089629888534546
+aqua_alpha_sweep_seed456.json,aqua,24,456,0.75,45,0.4444444444444444,0.5555555555555556,0.4444444444444444,0.25290924310684204,-1.9860222339630127
+aqua_alpha_sweep_seed456.json,aqua,24,456,1.0,45,1.0,0.0,1.0,-1.6927522420883179,-3.9316842555999756

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/alpha_sweep.md

@@ -0,0 +1,19 @@
+| task | file | alpha | n | flip_rate | ablated_acc | mean_delta_margin_vs_baseline |
+| --- | --- | --- | --- | --- | --- | --- |
+| aqua | aqua_alpha_sweep_seed123.json | 0.0 | 61 | 0.0 | 100.0 | 0.000 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.02 | 61 | 1.6 | 98.4 | -0.029 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.05 | 61 | 1.6 | 98.4 | -0.073 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.1 | 61 | 3.3 | 96.7 | -0.151 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.2 | 61 | 4.9 | 95.1 | -0.324 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.3 | 61 | 11.5 | 88.5 | -0.538 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.5 | 61 | 16.4 | 83.6 | -1.143 |
+| aqua | aqua_alpha_sweep_seed123.json | 0.75 | 61 | 41.0 | 59.0 | -2.339 |
+| aqua | aqua_alpha_sweep_seed123.json | 1.0 | 61 | 100.0 | 0.0 | -4.626 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.0 | 45 | 0.0 | 100.0 | 0.000 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.05 | 45 | 0.0 | 100.0 | -0.067 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.1 | 45 | 2.2 | 97.8 | -0.142 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.2 | 45 | 2.2 | 97.8 | -0.309 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.3 | 45 | 6.7 | 93.3 | -0.503 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.5 | 45 | 22.2 | 77.8 | -1.009 |
+| aqua | aqua_alpha_sweep_seed456.json | 0.75 | 45 | 44.4 | 55.6 | -1.986 |
+| aqua | aqua_alpha_sweep_seed456.json | 1.0 | 45 | 100.0 | 0.0 | -3.932 |

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/analyze_qwen_results.py

@@ -0,0 +1,236 @@
+#!/usr/bin/env python3
+# -*- coding: utf-8 -*-
+
+import os
+import pandas as pd
+import numpy as np
+import matplotlib.pyplot as plt
+
+SUMMARY_CSV = os.environ.get("SUMMARY_CSV", "summary.csv")
+ALPHA_CSV = os.environ.get("ALPHA_CSV", "alpha_sweep.csv")
+OUT_MD = os.environ.get("OUT_MD", "qwen_report.md")
+OUT_TEX = os.environ.get("OUT_TEX", "qwen_tables.tex")
+PLOT_DIR = os.environ.get("PLOT_DIR", "plots")
+
+os.makedirs(PLOT_DIR, exist_ok=True)
+
+df = pd.read_csv(SUMMARY_CSV)
+alpha_df = pd.read_csv(ALPHA_CSV) if os.path.exists(ALPHA_CSV) else pd.DataFrame()
+
+# Helpers
+def f(x, nd=3):
+    if pd.isna(x): return ""
+    if isinstance(x, (int, np.integer)): return str(int(x))
+    if isinstance(x, (float, np.floating)): return f"{float(x):.{nd}f}"
+    return str(x)
+
+def pct(x, nd=1):
+    if pd.isna(x): return ""
+    return f"{float(x):.{nd}f}"
+
+# Split by kind
+mc = df[df["kind"] == "subspace_mc"].copy()
+oa = df[df["kind"] == "openanswer"].copy()
+fs = df[df["kind"] == "flipset"].copy()
+
+# Choose representative columns
+def select_cols(kind_df, cols):
+    keep = [c for c in cols if c in kind_df.columns]
+    return kind_df[keep].copy()
+
+mc_cols = [
+    "task","eval_mode","seed","base_acc_scan","ablt_acc_scan","flips_scan",
+    "patched_0_rescued_pct","patched_full_rescued_pct",
+    "control_time_shuffled_rescued_pct","control_shared_randvec_rescued_pct",
+    "control_rand_subspace_rescued_pct","control_patch_nonshared_rescued_pct",
+]
+oa_cols = [
+    "task","eval_mode","seed","base_acc_scan","ablt_acc_scan","flips_scan",
+    "patched_self_rescued_pct","control_time_shuffled_rescued_pct",
+    "control_shared_randvec_rescued_pct","control_rand_subspace_rescued_pct",
+    "control_patch_nonshared_rescued_pct",
+]
+fs_cols = [
+    "file","seed","task","base_acc_scan","ablt_acc_scan","flips_scan",
+    "patched_self_rescued_pct","patched_transfer_rescued_pct",
+]
+
+mc_tbl = select_cols(mc, mc_cols).sort_values(["task","eval_mode","seed"])
+oa_tbl = select_cols(oa, oa_cols).sort_values(["task","eval_mode","seed"])
+fs_tbl = select_cols(fs, fs_cols).sort_values(["seed","file"])
+
+# ---- Plot 1: MC patched_0 rescue by task ----
+if len(mc_tbl) > 0 and "patched_0_rescued_pct" in mc_tbl.columns:
+    mc_plot = mc_tbl.groupby("task", as_index=False)["patched_0_rescued_pct"].mean()
+    plt.figure()
+    plt.bar(mc_plot["task"], mc_plot["patched_0_rescued_pct"])
+    plt.xticks(rotation=45, ha="right")
+    plt.ylabel("Rescue% on flips (patched_0)")
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "mc_patched0_rescue.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 2: MC controls gap (patched_0 vs rand-in-shared vs nonshared) ----
+if len(mc_tbl) > 0 and "patched_0_rescued_pct" in mc_tbl.columns:
+    tmp = mc_tbl.groupby("task", as_index=False).agg({
+        "patched_0_rescued_pct":"mean",
+        "control_shared_randvec_rescued_pct":"mean",
+        "control_patch_nonshared_rescued_pct":"mean",
+        "control_rand_subspace_rescued_pct":"mean",
+    })
+    plt.figure()
+    x = np.arange(len(tmp))
+    w = 0.2
+    plt.bar(x - 1.5*w, tmp["patched_0_rescued_pct"], width=w, label="patched_0")
+    plt.bar(x - 0.5*w, tmp["control_shared_randvec_rescued_pct"], width=w, label="rand vec in shared")
+    plt.bar(x + 0.5*w, tmp["control_rand_subspace_rescued_pct"], width=w, label="rand subspace")
+    plt.bar(x + 1.5*w, tmp["control_patch_nonshared_rescued_pct"], width=w, label="nonshared patch")
+    plt.xticks(x, tmp["task"], rotation=45, ha="right")
+    plt.ylabel("Rescue% on flips")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "mc_controls_gap.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 3: Alpha sweep flip_rate curves ----
+if len(alpha_df) > 0 and {"alpha","flip_rate","seed"}.issubset(alpha_df.columns):
+    alpha_df2 = alpha_df.copy()
+    alpha_df2["alpha"] = pd.to_numeric(alpha_df2["alpha"], errors="coerce")
+    alpha_df2 = alpha_df2.dropna(subset=["alpha"])
+    plt.figure()
+    for seed in sorted(alpha_df2["seed"].dropna().unique()):
+        sub = alpha_df2[alpha_df2["seed"] == seed].sort_values("alpha")
+        plt.plot(sub["alpha"], sub["flip_rate"]*100.0, marker="o", label=f"seed={int(seed)}")
+    plt.xlabel("alpha")
+    plt.ylabel("Flip rate on flip-set (%)")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "alpha_sweep_fliprate.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 4: Alpha sweep mean delta margin curves ----
+if len(alpha_df) > 0 and {"alpha","mean_delta_margin_vs_baseline","seed"}.issubset(alpha_df.columns):
+    alpha_df2 = alpha_df.copy()
+    alpha_df2["alpha"] = pd.to_numeric(alpha_df2["alpha"], errors="coerce")
+    alpha_df2 = alpha_df2.dropna(subset=["alpha"])
+    plt.figure()
+    for seed in sorted(alpha_df2["seed"].dropna().unique()):
+        sub = alpha_df2[alpha_df2["seed"] == seed].sort_values("alpha")
+        plt.plot(sub["alpha"], sub["mean_delta_margin_vs_baseline"], marker="o", label=f"seed={int(seed)}")
+    plt.xlabel("alpha")
+    plt.ylabel("Mean Δmargin vs baseline (on flip-set)")
+    plt.legend()
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "alpha_sweep_deltam.pdf"), dpi=300)
+    plt.close()
+
+# ---- Plot 5: Open-answer patched_self rescue (pair_logprob vs gen) ----
+if len(oa_tbl) > 0 and "patched_self_rescued_pct" in oa_tbl.columns:
+    oa_plot = oa_tbl.copy()
+    oa_plot["label"] = oa_plot["task"].astype(str) + ":" + oa_plot["eval_mode"].astype(str)
+    plt.figure()
+    plt.bar(oa_plot["label"], oa_plot["patched_self_rescued_pct"])
+    plt.xticks(rotation=45, ha="right")
+    plt.ylabel("Rescue% on flips (patched_self)")
+    plt.tight_layout()
+    plt.savefig(os.path.join(PLOT_DIR, "openanswer_patchedself_rescue.pdf"), dpi=300)
+    plt.close()
+
+# ---- Markdown report ----
+def df_to_md_table(dfx: pd.DataFrame, max_rows: int = 30) -> str:
+    if dfx is None or len(dfx) == 0:
+        return "_(none)_"
+    d = dfx.copy()
+    if len(d) > max_rows:
+        d = d.head(max_rows)
+    return d.to_markdown(index=False)
+
+lines = []
+lines.append(f"# Qwen subspace patching + flipset report\n")
+lines.append(f"Generated from `{os.path.basename(SUMMARY_CSV)}` and `{os.path.basename(ALPHA_CSV)}`.\n")
+lines.append("## Overview\n")
+lines.append(f"- Runs: {len(df)} total JSON summaries\n")
+lines.append(f"- MC runs: {len(mc)}; Open-answer runs: {len(oa)}; Flipset runs: {len(fs)}\n")
+lines.append("## Key plots (PDF, dpi=300)\n")
+for fn in [
+    "mc_patched0_rescue.pdf",
+    "mc_controls_gap.pdf",
+    "alpha_sweep_fliprate.pdf",
+    "alpha_sweep_deltam.pdf",
+    "openanswer_patchedself_rescue.pdf",
+]:
+    p = os.path.join(PLOT_DIR, fn)
+    if os.path.exists(p):
+        lines.append(f"- `{fn}`")
+lines.append("\n")
+
+lines.append("## Multiple-choice patchback (subspace_mc)\n")
+lines.append(df_to_md_table(mc_tbl))
+lines.append("\n")
+
+lines.append("## Open-answer patchback (openanswer)\n")
+lines.append(df_to_md_table(oa_tbl))
+lines.append("\n")
+
+lines.append("## Flipset transfer patching (flipset)\n")
+lines.append(df_to_md_table(fs_tbl))
+lines.append("\n")
+
+if len(alpha_df) > 0:
+    lines.append("## Alpha sweep (flip-set)\n")
+    # show a compact subset (alpha=0,0.5,0.75,1.0) if present
+    a = alpha_df.copy()
+    a["alpha"] = pd.to_numeric(a["alpha"], errors="coerce")
+    a = a.dropna(subset=["alpha"])
+    keep = a[a["alpha"].isin([0.0, 0.5, 0.75, 1.0])].copy()
+    if len(keep) == 0:
+        keep = a
+    keep = keep.sort_values(["seed","alpha"])
+    cols = [c for c in ["file","seed","alpha","n","flip_rate","ablated_acc","mean_delta_margin_vs_baseline"] if c in keep.columns]
+    lines.append(df_to_md_table(keep[cols], max_rows=60))
+    lines.append("\n")
+
+with open(OUT_MD, "w", encoding="utf-8") as f:
+    f.write("\n".join(lines))
+
+# ---- LaTeX tables (quick export) ----
+# We write two compact tables: MC summary + Open-answer summary
+def to_latex_table(dfx: pd.DataFrame, caption: str, label: str) -> str:
+    if dfx is None or len(dfx) == 0:
+        return f"% {caption}\n% (empty)\n"
+    return dfx.to_latex(index=False, escape=True, caption=caption, label=label)
+
+tex_lines = []
+tex_lines.append("% Auto-generated LaTeX tables for Qwen results\n")
+tex_lines.append("% Requires \\usepackage{booktabs}\n\n")
+
+if len(mc_tbl) > 0:
+    tex_lines.append(to_latex_table(
+        mc_tbl,
+        caption="Qwen: multiple-choice (subspace\\Apatch) summary.",
+        label="tab:qwen_mc_summary"
+    ))
+    tex_lines.append("\n")
+
+if len(oa_tbl) > 0:
+    tex_lines.append(to_latex_table(
+        oa_tbl,
+        caption="Qwen: open-answer (openanswer\\_subspace\\_patching) summary.",
+        label="tab:qwen_openanswer_summary"
+    ))
+    tex_lines.append("\n")
+
+if len(alpha_df) > 0:
+    tex_lines.append(to_latex_table(
+        alpha_df.sort_values(["seed","alpha"]).head(40),
+        caption="Qwen: alpha sweep (first 40 rows shown).",
+        label="tab:qwen_alpha_sweep_head"
+    ))
+    tex_lines.append("\n")
+
+with open(OUT_TEX, "w", encoding="utf-8") as f:
+    f.write("\n".join(tex_lines))
+
+print(f"[OK] Wrote report: {OUT_MD}")
+print(f"[OK] Wrote LaTeX tables: {OUT_TEX}")
+print(f"[OK] Plots in: {PLOT_DIR}")

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/paper_table.md

@@ -0,0 +1,17 @@
+| kind | task | eval_mode | base_acc_scan | ablt_acc_scan | flips_scan | Patched@0 (rescue%, Δm) | Patched@full (rescue%, Δm) | Patched(self) (rescue%, Δm) | Patched(transfer) (rescue%, Δm) | Cross-example donor (rescue%, Δm) | Donor mismatch (rescue%, Δm) | Shared coeff permute (rescue%, Δm) | Shared coeff signflip (rescue%, Δm) | Rand vec in shared (rescue%, Δm) | Rand subspace (rescue%, Δm) | Nonshared patch (rescue%, Δm) |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| flipset | aqua |  | 0.413 | 0.236 | 61 |  |  |  |  |  |  |  |  |  |  |  |
+| flipset | aqua |  | 0.402 | 0.299 | 45 |  |  |  |  |  |  |  |  |  |  |  |
+| flipset | aqua |  | 0.413 | 0.236 | 61 |  |  | 100.0%, 4.626 | 18.0%, -2.384 |  |  |  |  |  |  |  |
+| flipset | aqua |  | 0.413 | 0.236 | 61 |  |  | 100.0%, 4.626 | 31.1%, 0.919 |  |  |  |  |  |  |  |
+| openanswer | gsm8k | gen_math | 0.035 | 0.039 | 8 |  |  | 0.0%, - |  | 12.5%, - |  |  |  | 12.5%, - | 0.0%, - | 0.0%, - |
+| openanswer | gsm8k | pair_logprob | 0.801 | 0.762 | 15 |  |  | 93.3%, 4.365 |  | 73.3%, 3.870 |  |  |  | 20.0%, 0.357 | 13.3%, 0.209 | 6.7%, 0.067 |
+| openanswer | humaneval | gen_code_compile |  |  | 0 |  |  | 5.3%, - |  | 5.3%, - |  |  |  | 5.3%, - | 0.0%, - | 0.0%, - |
+| openanswer | humaneval | pair_logprob | 0.683 | 0.665 | 12 |  |  | 83.3%, 2.557 |  | 50.0%, 1.671 |  |  |  | 0.0%, -0.240 | 0.0%, -0.123 | 0.0%, -0.140 |
+| subspace_mc | aqua |  | 0.413 | 0.236 | 61 | 100.0%, 4.626 | 100.0%, 4.626 |  |  | 27.9%, 0.058 |  |  |  | 32.8%, -0.133 | 31.1%, 0.133 | 0.0%, 0.000 |
+| subspace_mc | arc_challenge |  | 0.906 | 0.608 | 83 | 100.0%, 8.554 | 100.0%, 8.554 |  |  | 59.0%, 3.809 |  |  |  | 27.7%, 0.258 | 33.7%, 0.832 | 0.0%, -0.000 |
+| subspace_mc | commonsenseqa |  | 0.867 | 0.645 | 64 | 100.0%, 7.356 | 100.0%, 7.356 |  |  | 46.9%, 0.967 |  |  |  | 35.9%, 0.356 | 26.6%, -0.094 | 0.0%, -0.000 |
+| subspace_mc | logiqa |  | 0.473 | 0.418 | 34 | 100.0%, 6.486 | 100.0%, 6.486 |  |  | 47.1%, 1.852 |  |  |  | 29.4%, 0.513 | 38.2%, 1.663 | 0.0%, -0.000 |
+| subspace_mc | openbookqa |  | 0.859 | 0.613 | 69 | 100.0%, 7.372 | 100.0%, 7.372 |  |  | 44.9%, 0.974 |  |  |  | 31.9%, 0.118 | 40.6%, 0.760 | 0.0%, 0.000 |
+| subspace_mc | piqa |  | 0.871 | 0.707 | 53 | 100.0%, 6.291 | 100.0%, 6.291 |  |  | 92.5%, 5.667 |  |  |  | 32.1%, -0.274 | 26.4%, -0.256 | 0.0%, -0.000 |
+| subspace_mc | qasc |  | 0.809 | 0.492 | 89 | 100.0%, 9.979 | 100.0%, 9.979 |  |  | 51.7%, 4.258 |  |  |  | 61.8%, 4.160 | 66.3%, 4.673 | 0.0%, 0.000 |

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/qwen_report.md

@@ -0,0 +1,65 @@
+# Qwen subspace patching + flipset report
+
+Generated from `summary.csv` and `alpha_sweep.csv`.
+
+## Overview
+
+- Runs: 15 total JSON summaries
+
+- MC runs: 7; Open-answer runs: 4; Flipset runs: 4
+
+## Key plots (PDF, dpi=300)
+
+- `mc_patched0_rescue.pdf`
+- `mc_controls_gap.pdf`
+- `alpha_sweep_fliprate.pdf`
+- `alpha_sweep_deltam.pdf`
+- `openanswer_patchedself_rescue.pdf`
+
+
+## Multiple-choice patchback (subspace_mc)
+
+| task          |   eval_mode |   seed |   base_acc_scan |   ablt_acc_scan |   flips_scan |   patched_0_rescued_pct |   patched_full_rescued_pct |   control_time_shuffled_rescued_pct |   control_shared_randvec_rescued_pct |   control_rand_subspace_rescued_pct |   control_patch_nonshared_rescued_pct |
+|:--------------|------------:|-------:|----------------:|----------------:|-------------:|------------------------:|---------------------------:|------------------------------------:|-------------------------------------:|------------------------------------:|--------------------------------------:|
+| aqua          |         nan |    123 |        0.413386 |        0.23622  |           61 |                     100 |                        100 |                             27.8689 |                              32.7869 |                             31.1475 |                                     0 |
+| arc_challenge |         nan |    123 |        0.905882 |        0.607843 |           83 |                     100 |                        100 |                             59.0361 |                              27.7108 |                             33.7349 |                                     0 |
+| commonsenseqa |         nan |    123 |        0.867188 |        0.644531 |           64 |                     100 |                        100 |                             46.875  |                              35.9375 |                             26.5625 |                                     0 |
+| logiqa        |         nan |    123 |        0.472656 |        0.417969 |           34 |                     100 |                        100 |                             47.0588 |                              29.4118 |                             38.2353 |                                     0 |
+| openbookqa    |         nan |    123 |        0.859375 |        0.613281 |           69 |                     100 |                        100 |                             44.9275 |                              31.8841 |                             40.5797 |                                     0 |
+| piqa          |         nan |    123 |        0.871094 |        0.707031 |           53 |                     100 |                        100 |                             92.4528 |                              32.0755 |                             26.4151 |                                     0 |
+| qasc          |         nan |    123 |        0.808594 |        0.492188 |           89 |                     100 |                        100 |                             51.6854 |                              61.7978 |                             66.2921 |                                     0 |
+
+
+## Open-answer patchback (openanswer)
+
+| task      | eval_mode        |   seed |   base_acc_scan |   ablt_acc_scan |   flips_scan |   patched_self_rescued_pct |   control_time_shuffled_rescued_pct |   control_shared_randvec_rescued_pct |   control_rand_subspace_rescued_pct |   control_patch_nonshared_rescued_pct |
+|:----------|:-----------------|-------:|----------------:|----------------:|-------------:|---------------------------:|------------------------------------:|-------------------------------------:|------------------------------------:|--------------------------------------:|
+| gsm8k     | gen_math         |    123 |       0.0351562 |       0.0390625 |            8 |                    0       |                            12.5     |                             12.5     |                              0      |                               0       |
+| gsm8k     | pair_logprob     |    123 |       0.800781  |       0.761719  |           15 |                   93.3333  |                            73.3333  |                             20       |                             13.3333 |                               6.66667 |
+| humaneval | gen_code_compile |    123 |     nan         |     nan         |            0 |                    5.26316 |                             5.26316 |                              5.26316 |                              0      |                               0       |
+| humaneval | pair_logprob     |    123 |       0.682927  |       0.664634  |           12 |                   83.3333  |                            50       |                              0       |                              0      |                               0       |
+
+
+## Flipset transfer patching (flipset)
+
+| file                                                |   seed | task   |   base_acc_scan |   ablt_acc_scan |   flips_scan |   patched_self_rescued_pct |   patched_transfer_rescued_pct |
+|:----------------------------------------------------|-------:|:-------|----------------:|----------------:|-------------:|---------------------------:|-------------------------------:|
+| aqua_alpha_sweep_seed123.json                       |    123 | aqua   |        0.413386 |        0.23622  |           61 |                        nan |                       nan      |
+| aqua_transfer_cross_mc_baselinecorrect_seed123.json |    123 | aqua   |        0.413386 |        0.23622  |           61 |                        100 |                        18.0328 |
+| aqua_transfer_same_task_seed123.json                |    123 | aqua   |        0.413386 |        0.23622  |           61 |                        100 |                        31.1475 |
+| aqua_alpha_sweep_seed456.json                       |    456 | aqua   |        0.401575 |        0.299213 |           45 |                        nan |                       nan      |
+
+
+## Alpha sweep (flip-set)
+
+| file                          |   seed |   alpha |   n |   flip_rate |   ablated_acc |   mean_delta_margin_vs_baseline |
+|:------------------------------|-------:|--------:|----:|------------:|--------------:|--------------------------------:|
+| aqua_alpha_sweep_seed123.json |    123 |    0    |  61 |    0        |      1        |                         0       |
+| aqua_alpha_sweep_seed123.json |    123 |    0.5  |  61 |    0.163934 |      0.836066 |                        -1.14309 |
+| aqua_alpha_sweep_seed123.json |    123 |    0.75 |  61 |    0.409836 |      0.590164 |                        -2.33868 |
+| aqua_alpha_sweep_seed123.json |    123 |    1    |  61 |    1        |      0        |                        -4.62629 |
+| aqua_alpha_sweep_seed456.json |    456 |    0    |  45 |    0        |      1        |                         0       |
+| aqua_alpha_sweep_seed456.json |    456 |    0.5  |  45 |    0.222222 |      0.777778 |                        -1.00896 |
+| aqua_alpha_sweep_seed456.json |    456 |    0.75 |  45 |    0.444444 |      0.555556 |                        -1.98602 |
+| aqua_alpha_sweep_seed456.json |    456 |    1    |  45 |    1        |      0        |                        -3.93168 |
+

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/qwen_tables.tex

@@ -0,0 +1,71 @@
+% Auto-generated LaTeX tables for Qwen results
+
+% Requires \usepackage{booktabs}
+
+
+\begin{table}
+\caption{Qwen: multiple-choice (subspace\Apatch) summary.}
+\label{tab:qwen_mc_summary}
+\begin{tabular}{llrrrrrrrrrr}
+\toprule
+task & eval\_mode & seed & base\_acc\_scan & ablt\_acc\_scan & flips\_scan & patched\_0\_rescued\_pct & patched\_full\_rescued\_pct & control\_time\_shuffled\_rescued\_pct & control\_shared\_randvec\_rescued\_pct & control\_rand\_subspace\_rescued\_pct & control\_patch\_nonshared\_rescued\_pct \\
+\midrule
+aqua & NaN & 123 & 0.413386 & 0.236220 & 61 & 100.000000 & 100.000000 & 27.868852 & 32.786885 & 31.147541 & 0.000000 \\
+arc\_challenge & NaN & 123 & 0.905882 & 0.607843 & 83 & 100.000000 & 100.000000 & 59.036145 & 27.710843 & 33.734940 & 0.000000 \\
+commonsenseqa & NaN & 123 & 0.867188 & 0.644531 & 64 & 100.000000 & 100.000000 & 46.875000 & 35.937500 & 26.562500 & 0.000000 \\
+logiqa & NaN & 123 & 0.472656 & 0.417969 & 34 & 100.000000 & 100.000000 & 47.058824 & 29.411765 & 38.235294 & 0.000000 \\
+openbookqa & NaN & 123 & 0.859375 & 0.613281 & 69 & 100.000000 & 100.000000 & 44.927536 & 31.884058 & 40.579710 & 0.000000 \\
+piqa & NaN & 123 & 0.871094 & 0.707031 & 53 & 100.000000 & 100.000000 & 92.452830 & 32.075472 & 26.415094 & 0.000000 \\
+qasc & NaN & 123 & 0.808594 & 0.492188 & 89 & 100.000000 & 100.000000 & 51.685393 & 61.797753 & 66.292135 & 0.000000 \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+
+
+\begin{table}
+\caption{Qwen: open-answer (openanswer\_subspace\_patching) summary.}
+\label{tab:qwen_openanswer_summary}
+\begin{tabular}{llrrrrrrrrr}
+\toprule
+task & eval\_mode & seed & base\_acc\_scan & ablt\_acc\_scan & flips\_scan & patched\_self\_rescued\_pct & control\_time\_shuffled\_rescued\_pct & control\_shared\_randvec\_rescued\_pct & control\_rand\_subspace\_rescued\_pct & control\_patch\_nonshared\_rescued\_pct \\
+\midrule
+gsm8k & gen\_math & 123 & 0.035156 & 0.039062 & 8 & 0.000000 & 12.500000 & 12.500000 & 0.000000 & 0.000000 \\
+gsm8k & pair\_logprob & 123 & 0.800781 & 0.761719 & 15 & 93.333333 & 73.333333 & 20.000000 & 13.333333 & 6.666667 \\
+humaneval & gen\_code\_compile & 123 & NaN & NaN & 0 & 5.263158 & 5.263158 & 5.263158 & 0.000000 & 0.000000 \\
+humaneval & pair\_logprob & 123 & 0.682927 & 0.664634 & 12 & 83.333333 & 50.000000 & 0.000000 & 0.000000 & 0.000000 \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+
+
+\begin{table}
+\caption{Qwen: alpha sweep (first 40 rows shown).}
+\label{tab:qwen_alpha_sweep_head}
+\begin{tabular}{llrrrrrrrrr}
+\toprule
+file & task & layer & seed & alpha & n & flip\_rate & ablated\_acc & pred\_change\_rate & mean\_margin & mean\_delta\_margin\_vs\_baseline \\
+\midrule
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.000000 & 61 & 0.000000 & 1.000000 & 0.000000 & 2.860918 & 0.000000 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.020000 & 61 & 0.016393 & 0.983607 & 0.016393 & 2.832013 & -0.028906 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.050000 & 61 & 0.016393 & 0.983607 & 0.016393 & 2.787427 & -0.073491 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.100000 & 61 & 0.032787 & 0.967213 & 0.032787 & 2.709940 & -0.150978 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.200000 & 61 & 0.049180 & 0.950820 & 0.049180 & 2.536606 & -0.324313 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.300000 & 61 & 0.114754 & 0.885246 & 0.114754 & 2.322977 & -0.537942 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.500000 & 61 & 0.163934 & 0.836066 & 0.163934 & 1.717831 & -1.143088 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 0.750000 & 61 & 0.409836 & 0.590164 & 0.409836 & 0.522240 & -2.338679 \\
+aqua\_alpha\_sweep\_seed123.json & aqua & 24 & 123 & 1.000000 & 61 & 1.000000 & 0.000000 & 1.000000 & -1.765370 & -4.626290 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.000000 & 45 & 0.000000 & 1.000000 & 0.000000 & 2.238931 & 0.000000 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.050000 & 45 & 0.000000 & 1.000000 & 0.000000 & 2.171455 & -0.067476 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.100000 & 45 & 0.022222 & 0.977778 & 0.022222 & 2.097202 & -0.141729 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.200000 & 45 & 0.022222 & 0.977778 & 0.022222 & 1.929797 & -0.309134 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.300000 & 45 & 0.066667 & 0.933333 & 0.066667 & 1.736192 & -0.502739 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.500000 & 45 & 0.222222 & 0.777778 & 0.222222 & 1.229969 & -1.008963 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 0.750000 & 45 & 0.444444 & 0.555556 & 0.444444 & 0.252909 & -1.986022 \\
+aqua\_alpha\_sweep\_seed456.json & aqua & 24 & 456 & 1.000000 & 45 & 1.000000 & 0.000000 & 1.000000 & -1.692752 & -3.931684 \\
+\bottomrule
+\end{tabular}
+\end{table}
+
+

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/summary.csv

@@ -0,0 +1,16 @@
+kind,file,task,eval_mode,layer,seed,hf_id,hf_split,candidate_labels,Qs_shape,patch_desc,donor_source,donor_tasks,donor_pick,n_donor_bank,scan_effective,scan_skipped,base_acc_scan,ablt_acc_scan,flips_scan,anti_flips_scan,both_correct_scan,both_wrong_scan,patched_primary_method,patched_primary_rescued_pct,patched_primary_mean_dmargin,diff_time_shuffled_minus_patched_primary_rescued_pct,diff_patched_primary_minus_shared_randvec_rescued_pct,patched_0_rescued,patched_0_n,patched_0_rescued_pct,patched_0_mean_dmargin,patched_01_rescued,patched_01_n,patched_01_rescued_pct,patched_01_mean_dmargin,patched_full_rescued,patched_full_n,patched_full_rescued_pct,patched_full_mean_dmargin,patched_self_rescued,patched_self_n,patched_self_rescued_pct,patched_self_mean_dmargin,patched_transfer_rescued,patched_transfer_n,patched_transfer_rescued_pct,patched_transfer_mean_dmargin,control_time_shuffled_rescued,control_time_shuffled_n,control_time_shuffled_rescued_pct,control_time_shuffled_mean_dmargin,control_shared_mismatch_rescued,control_shared_mismatch_n,control_shared_mismatch_rescued_pct,control_shared_mismatch_mean_dmargin,control_shared_perm_rescued,control_shared_perm_n,control_shared_perm_rescued_pct,control_shared_perm_mean_dmargin,control_shared_signflip_rescued,control_shared_signflip_n,control_shared_signflip_rescued_pct,control_shared_signflip_mean_dmargin,control_shared_randvec_rescued,control_shared_randvec_n,control_shared_randvec_rescued_pct,control_shared_randvec_mean_dmargin,control_rand_subspace_rescued,control_rand_subspace_n,control_rand_subspace_rescued_pct,control_rand_subspace_mean_dmargin,control_patch_nonshared_rescued,control_patch_nonshared_n,control_patch_nonshared_rescued_pct,control_patch_nonshared_mean_dmargin
+flipset,aqua_alpha_sweep_seed123.json,aqua,,24,123,,,ABCDE,3584x223,steps=0,,,,,254,0,0.41338582677165353,0.23622047244094488,61,16,44,133,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flipset,aqua_alpha_sweep_seed456.json,aqua,,24,456,,,ABCDE,3584x221,steps=0,,,,,254,0,0.4015748031496063,0.2992125984251969,45,19,57,133,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flipset,aqua_transfer_cross_mc_baselinecorrect_seed123.json,aqua,,24,123,,,ABCDE,3584x223,steps=0,cross_task_eval,"commonsenseqa,openbookqa",random,256,254,0,0.41338582677165353,0.23622047244094488,61,16,44,133,patched_self,100.0,4.6262898445129395,,,,,,,,,,,,,,,61,61,100.0,4.6262898445129395,11,61,18.0327868852459,-2.3844683170318604,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+flipset,aqua_transfer_same_task_seed123.json,aqua,,24,123,,,ABCDE,3584x223,steps=0,same_task_eval,aqua,random,254,254,0,0.41338582677165353,0.23622047244094488,61,16,44,133,patched_self,100.0,4.6262898445129395,,,,,,,,,,,,,,,61,61,100.0,4.6262898445129395,19,61,31.147540983606557,0.9193577766418457,,,,,,,,,,,,,,,,,,,,,,,,,,,,
+openanswer,gsm8k_genmath.json,gsm8k,gen_math,24,123,gsm8k/main,,,3584x223,"steps=0,1,2,3",,,,,256,0,0.03515625,0.0390625,8,9,1,238,patched_self,0.0,,12.5,-12.5,,,,,,,,,,,,,0,8,0.0,,,,,,1,8,12.5,,,,,,,,,,,,,,1,8,12.5,,0,8,0.0,,0,8,0.0,
+openanswer,gsm8k_pairlogprob.json,gsm8k,pair_logprob,24,123,gsm8k/main,,,3584x223,"steps=0,1,2,3",,,,,256,0,0.80078125,0.76171875,15,5,190,46,patched_self,93.33333333333333,4.364808082580566,-20.0,73.33333333333333,,,,,,,,,,,,,14,15,93.33333333333333,4.364808082580566,,,,,11,15,73.33333333333333,3.8701834678649902,,,,,,,,,,,,,3,15,20.0,0.35714292526245117,2,15,13.333333333333334,0.2093137800693512,1,15,6.666666666666667,0.06698532402515411
+openanswer,humaneval_gencode_compile.json,humaneval,gen_code_compile,24,123,openai_humaneval,test,,3584x223,"steps=0,1,2,3",,,,,0,164,,,0,0,0,0,patched_self,5.2631578947368425,,0.0,0.0,,,,,,,,,,,,,1,19,5.2631578947368425,,,,,,1,19,5.2631578947368425,,,,,,,,,,,,,,1,19,5.2631578947368425,,0,19,0.0,,0,19,0.0,
+openanswer,humaneval_pairlogprob.json,humaneval,pair_logprob,24,123,openai_humaneval,test,,3584x223,"steps=0,1,2,3",,,,,164,0,0.6829268292682927,0.6646341463414634,12,9,100,43,patched_self,83.33333333333333,2.5565900802612305,-33.33333333333333,83.33333333333333,,,,,,,,,,,,,10,12,83.33333333333333,2.5565900802612305,,,,,6,12,50.0,1.6708040237426758,,,,,,,,,,,,,0,12,0.0,-0.2395162135362625,0,12,0.0,-0.12262799590826035,0,12,0.0,-0.13950853049755096
+subspace_mc,aqua.json,aqua,,24,123,,,ABCDE,3584x223,,,,,,254,0,0.41338582677165353,0.23622047244094488,61,16,44,133,patched_0,100.0,4.626289450731433,-72.1311475409836,67.21311475409837,61,61,100.0,4.626289450731433,61,61,100.0,4.626289450731433,61,61,100.0,4.626289450731433,,,,,,,,,17,61,27.868852459016395,0.05754191758202725,,,,,,,,,,,,,20,61,32.78688524590164,-0.13294745664127539,19,61,31.147540983606557,0.1326175615435741,0,61,0.0,1.0748378566054047e-07
+subspace_mc,arc_challenge.json,arc_challenge,,24,123,,,ABCD,3584x223,,,,,,255,1,0.9058823529411765,0.6078431372549019,83,7,148,17,patched_0,100.0,8.554246582180621,-40.963855421686745,72.28915662650603,83,83,100.0,8.554246582180621,83,83,100.0,8.554246582180621,83,83,100.0,8.554246582180621,,,,,,,,,49,83,59.036144578313255,3.808587062789733,,,,,,,,,,,,,23,83,27.710843373493976,0.2576807780438159,28,83,33.734939759036145,0.8321906486189509,0,83,0.0,-1.2179455125188252e-06
+subspace_mc,commonsenseqa.json,commonsenseqa,,24,123,,,ABCDE,3584x223,,,,,,256,0,0.8671875,0.64453125,64,7,158,27,patched_0,100.0,7.355989446863532,-53.125,64.0625,64,64,100.0,7.355989446863532,64,64,100.0,7.355989446863532,64,64,100.0,7.355989446863532,,,,,,,,,30,64,46.875,0.9671716894954443,,,,,,,,,,,,,23,64,35.9375,0.356116883456707,17,64,26.5625,-0.09415088221430779,0,64,0.0,-4.6566128730773926e-07
+subspace_mc,logiqa.json,logiqa,,24,123,,,ABCD,3584x223,,,,,,256,0,0.47265625,0.41796875,34,20,87,115,patched_0,100.0,6.486427613917519,-52.94117647058823,70.58823529411765,34,34,100.0,6.486427613917519,34,34,100.0,6.486427613917519,34,34,100.0,6.486427613917519,,,,,,,,,16,34,47.05882352941177,1.8523672047783346,,,,,,,,,,,,,10,34,29.41176470588235,0.5126809863483205,13,34,38.23529411764706,1.6626762362087475,0,34,0.0,-6.731818704044118e-07
+subspace_mc,openbookqa.json,openbookqa,,24,123,,,ABCD,3584x223,,,,,,256,0,0.859375,0.61328125,69,6,151,30,patched_0,100.0,7.371976157893306,-55.072463768115945,68.1159420289855,69,69,100.0,7.371976157893306,69,69,100.0,7.371976157893306,69,69,100.0,7.371976157893306,,,,,,,,,31,69,44.927536231884055,0.9741012505863024,,,,,,,,,,,,,22,69,31.884057971014492,0.11762981483901756,28,69,40.57971014492754,0.7597268353337827,0,69,0.0,1.603278560914855e-06
+subspace_mc,piqa.json,piqa,,24,123,,,AB,3584x223,,,,,,256,0,0.87109375,0.70703125,53,11,170,22,patched_0,100.0,6.2906723629753545,-7.547169811320757,67.9245283018868,53,53,100.0,6.2906723629753545,53,53,100.0,6.2906723629753545,53,53,100.0,6.2906723629753545,,,,,,,,,49,53,92.45283018867924,5.667159054639204,,,,,,,,,,,,,17,53,32.075471698113205,-0.2743093382637456,14,53,26.41509433962264,-0.2557831530301076,0,53,0.0,-1.4934899672022407e-06
+subspace_mc,qasc.json,qasc,,24,123,,,ABCDEFGH,3584x223,,,,,,256,0,0.80859375,0.4921875,89,8,118,41,patched_0,100.0,9.979466939240359,-48.31460674157304,38.20224719101124,89,89,100.0,9.979466939240359,89,89,100.0,9.979466939240359,89,89,100.0,9.979466939240359,,,,,,,,,46,89,51.68539325842696,4.2575247676185,,,,,,,,,,,,,55,89,61.79775280898876,4.159921297866307,59,89,66.29213483146067,4.673449364940772,0,89,0.0,5.598818318227704e-07

artifacts/patch_back/results/Qwen/Qwen2.5-7B-Instruct/layer24/_summary/summary.md

@@ -0,0 +1,17 @@
+| kind | task | eval_mode | file | base_acc_scan | ablt_acc_scan | flips_scan | patched_primary_method | patched_primary_rescued_pct | control_time_shuffled_rescued_pct | control_shared_randvec_rescued_pct | control_rand_subspace_rescued_pct | control_patch_nonshared_rescued_pct |
+| --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- | --- |
+| flipset | aqua |  | aqua_alpha_sweep_seed123.json | 0.413 | 0.236 | 61 |  |  |  |  |  |  |
+| flipset | aqua |  | aqua_alpha_sweep_seed456.json | 0.402 | 0.299 | 45 |  |  |  |  |  |  |
+| flipset | aqua |  | aqua_transfer_cross_mc_baselinecorrect_seed123.json | 0.413 | 0.236 | 61 | patched_self | 100.0 |  |  |  |  |
+| flipset | aqua |  | aqua_transfer_same_task_seed123.json | 0.413 | 0.236 | 61 | patched_self | 100.0 |  |  |  |  |
+| openanswer | gsm8k | gen_math | gsm8k_genmath.json | 0.035 | 0.039 | 8 | patched_self | 0.0 | 12.5 | 12.5 | 0.0 | 0.0 |
+| openanswer | gsm8k | pair_logprob | gsm8k_pairlogprob.json | 0.801 | 0.762 | 15 | patched_self | 93.3 | 73.3 | 20.0 | 13.3 | 6.7 |
+| openanswer | humaneval | gen_code_compile | humaneval_gencode_compile.json |  |  | 0 | patched_self | 5.3 | 5.3 | 5.3 | 0.0 | 0.0 |
+| openanswer | humaneval | pair_logprob | humaneval_pairlogprob.json | 0.683 | 0.665 | 12 | patched_self | 83.3 | 50.0 | 0.0 | 0.0 | 0.0 |
+| subspace_mc | aqua |  | aqua.json | 0.413 | 0.236 | 61 | patched_0 | 100.0 | 27.9 | 32.8 | 31.1 | 0.0 |
+| subspace_mc | arc_challenge |  | arc_challenge.json | 0.906 | 0.608 | 83 | patched_0 | 100.0 | 59.0 | 27.7 | 33.7 | 0.0 |
+| subspace_mc | commonsenseqa |  | commonsenseqa.json | 0.867 | 0.645 | 64 | patched_0 | 100.0 | 46.9 | 35.9 | 26.6 | 0.0 |
+| subspace_mc | logiqa |  | logiqa.json | 0.473 | 0.418 | 34 | patched_0 | 100.0 | 47.1 | 29.4 | 38.2 | 0.0 |
+| subspace_mc | openbookqa |  | openbookqa.json | 0.859 | 0.613 | 69 | patched_0 | 100.0 | 44.9 | 31.9 | 40.6 | 0.0 |
+| subspace_mc | piqa |  | piqa.json | 0.871 | 0.707 | 53 | patched_0 | 100.0 | 92.5 | 32.1 | 26.4 | 0.0 |
+| subspace_mc | qasc |  | qasc.json | 0.809 | 0.492 | 89 | patched_0 | 100.0 | 51.7 | 61.8 | 66.3 | 0.0 |