{
  "meta": {
    "model": "meta-llama/Llama-2-7b-chat-hf",
    "device": "cuda",
    "dtype": "fp32",
    "layer": 10,
    "layers_path": "model.layers",
    "seed": 123,
    "task": "gsm8k",
    "eval_mode": "pair_logprob",
    "eval_meta": {
      "subspace_split": null,
      "eval_split": "test",
      "available_splits": [
        "train",
        "test"
      ],
      "hf_id": "gsm8k/main"
    },
    "n_eval_loaded": 256,
    "n_scanned": 256,
    "base_acc_scan": 0.625,
    "ablt_acc_scan": 0.59375,
    "flips_total": 31,
    "flips_used": 31,
    "patch_steps": [
      0
    ],
    "patch_n_steps": 1,
    "Qs_path": "Q_shared_layer10.npy",
    "Qs_shape": [
      4096,
      97
    ],
    "gold_text_prefix": " ",
    "dist_text_prefix": " ",
    "gold_max_tokens": 0,
    "distractor_mode": "next_gold",
    "answer_prefix_effective": "\nFinal answer:",
    "max_new_tokens_effective": 64,
    "run_coeff_controls": false,
    "use_benchmark_loader": true,
    "hf_id": "",
    "hf_split": "test"
  },
  "summary_on_flips": {
    "patched_self": {
      "n": 31,
      "rescued": 11,
      "rescued_pct": 35.483870967741936,
      "mean_delta_margin_vs_ablated": 1.5506091117858887,
      "median_delta_margin_vs_ablated": 1.8901087045669556
    },
    "control_time_shuffled": {
      "n": 31,
      "rescued": 11,
      "rescued_pct": 35.483870967741936,
      "mean_delta_margin_vs_ablated": 1.536723256111145,
      "median_delta_margin_vs_ablated": 1.8725682497024536
    },
    "control_shared_randvec": {
      "n": 31,
      "rescued": 1,
      "rescued_pct": 3.225806451612903,
      "mean_delta_margin_vs_ablated": -0.34435272216796875,
      "median_delta_margin_vs_ablated": -0.03446388244628906
    },
    "control_rand_subspace": {
      "n": 31,
      "rescued": 0,
      "rescued_pct": 0.0,
      "mean_delta_margin_vs_ablated": -0.26384562253952026,
      "median_delta_margin_vs_ablated": -0.18717603385448456
    },
    "control_patch_nonshared": {
      "n": 31,
      "rescued": 0,
      "rescued_pct": 0.0,
      "mean_delta_margin_vs_ablated": 8.61082810388325e-07,
      "median_delta_margin_vs_ablated": 1.0728836059570312e-06
    }
  },
  "scan_rows": [
    {
      "ex_id": "gsm8k-test-0",
      "gold_norm": "50",
      "dist_norm": "80",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0485178977251053,
        "lp_gold": -13.375523149967194,
        "lp_dist": -15.424041047692299,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2319180071353912,
        "lp_gold": -6.67494124174118,
        "lp_dist": -8.906859248876572,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-1",
      "gold_norm": "80",
      "dist_norm": "12",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.218225084245205,
        "lp_gold": -16.316218174993992,
        "lp_dist": -17.534443259239197,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6342043727636337,
        "lp_gold": -18.493512138724327,
        "lp_dist": -17.859307765960693,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-2",
      "gold_norm": "12",
      "dist_norm": "140",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.582239151000977,
        "lp_gold": -19.479307651519775,
        "lp_dist": -29.061546802520752,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.06528377532959,
        "lp_gold": -16.74149775505066,
        "lp_dist": -20.80678153038025,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-3",
      "gold_norm": "140",
      "dist_norm": "36",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.5653446912765503,
        "lp_gold": -19.955466985702515,
        "lp_dist": -17.390122294425964,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.37339717149734497,
        "lp_gold": -14.124846756458282,
        "lp_dist": -13.751449584960938,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-4",
      "gold_norm": "36",
      "dist_norm": "3200",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 17.496737867593765,
        "lp_gold": -13.73099598288536,
        "lp_dist": -31.227733850479126,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 9.695431232452393,
        "lp_gold": -7.723996877670288,
        "lp_dist": -17.41942811012268,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-5",
      "gold_norm": "3200",
      "dist_norm": "38",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.075981711270288,
        "lp_gold": -15.808944131014869,
        "lp_dist": -19.884925842285156,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.112908275797963,
        "lp_gold": -17.281133087351918,
        "lp_dist": -15.168224811553955,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-6",
      "gold_norm": "38",
      "dist_norm": "32",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.0733052492141724,
        "lp_gold": -17.57793438434601,
        "lp_dist": -15.504629135131836,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.1486949920654297,
        "lp_gold": -20.525099754333496,
        "lp_dist": -20.376404762268066,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-7",
      "gold_norm": "32",
      "dist_norm": "92",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.4189205169677734,
        "lp_gold": -16.66067409515381,
        "lp_dist": -20.079594612121582,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.4474713802337646,
        "lp_gold": -15.954271793365479,
        "lp_dist": -18.401743173599243,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-8",
      "gold_norm": "92",
      "dist_norm": "16",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.1885854713618755,
        "lp_gold": -20.10318946838379,
        "lp_dist": -16.914603997021914,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.5822286009788513,
        "lp_gold": -15.157714128494263,
        "lp_dist": -12.575485527515411,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-9",
      "gold_norm": "16",
      "dist_norm": "45",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.206469178199768,
        "lp_gold": -20.85190773010254,
        "lp_dist": -17.64543855190277,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.272299289703369,
        "lp_gold": -11.194756746292114,
        "lp_dist": -13.467056035995483,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-10",
      "gold_norm": "45",
      "dist_norm": "270",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 11.154298067092896,
        "lp_gold": -17.49683403968811,
        "lp_dist": -28.651132106781006,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.5754551887512207,
        "lp_gold": -13.103037357330322,
        "lp_dist": -16.678492546081543,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-11",
      "gold_norm": "270",
      "dist_norm": "100",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.8893778324127197,
        "lp_gold": -21.884052515029907,
        "lp_dist": -17.994674682617188,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.458778917789459,
        "lp_gold": -14.477847814559937,
        "lp_dist": -10.019068896770477,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-12",
      "gold_norm": "100",
      "dist_norm": "25",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.500066578388214,
        "lp_gold": -11.581663310527802,
        "lp_dist": -17.081729888916016,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.39761683344841,
        "lp_gold": -9.308580189943314,
        "lp_dist": -11.706197023391724,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-13",
      "gold_norm": "25",
      "dist_norm": "800",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 20.992703570984304,
        "lp_gold": -13.314849936403334,
        "lp_dist": -34.30755350738764,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.717362227849662,
        "lp_gold": -11.016974148340523,
        "lp_dist": -18.734336376190186,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-14",
      "gold_norm": "800",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.7909989710897207,
        "lp_gold": -10.428668463602662,
        "lp_dist": -13.219667434692383,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.28849396109581,
        "lp_gold": -15.488610118627548,
        "lp_dist": -9.200116157531738,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-15",
      "gold_norm": "2",
      "dist_norm": "7000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 14.264829635620117,
        "lp_gold": -12.60490345954895,
        "lp_dist": -26.869733095169067,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 10.93172574043274,
        "lp_gold": -10.274073839187622,
        "lp_dist": -21.20579957962036,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-16",
      "gold_norm": "7000",
      "dist_norm": "25",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.534815393853933,
        "lp_gold": -21.196847282815725,
        "lp_dist": -16.662031888961792,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.092950224876404,
        "lp_gold": -21.74782168865204,
        "lp_dist": -14.654871463775635,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-17",
      "gold_norm": "25",
      "dist_norm": "3",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.14834189414978,
        "lp_gold": -15.3827223777771,
        "lp_dist": -11.23438048362732,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8233906030654907,
        "lp_gold": -9.256547331809998,
        "lp_dist": -8.433156728744507,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-18",
      "gold_norm": "3",
      "dist_norm": "3430",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 31.8187518119812,
        "lp_gold": -10.239798672497272,
        "lp_dist": -42.058550484478474,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 16.154653549194336,
        "lp_gold": -7.938319206237793,
        "lp_dist": -24.09297275543213,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-19",
      "gold_norm": "3430",
      "dist_norm": "106",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.379680693149567,
        "lp_gold": -23.033769607543945,
        "lp_dist": -18.65408891439438,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -9.992487907409668,
        "lp_gold": -21.179072380065918,
        "lp_dist": -11.18658447265625,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-20",
      "gold_norm": "106",
      "dist_norm": "80",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.92336449585855,
        "lp_gold": -15.76830449141562,
        "lp_dist": -20.69166898727417,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6747859213501215,
        "lp_gold": -17.988985607400537,
        "lp_dist": -17.314199686050415,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-21",
      "gold_norm": "80",
      "dist_norm": "26",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.6582831740379333,
        "lp_gold": -10.863374054431915,
        "lp_dist": -13.521657228469849,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.6402748823165894,
        "lp_gold": -11.24216091632843,
        "lp_dist": -12.88243579864502,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-22",
      "gold_norm": "26",
      "dist_norm": "750",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.5539721846580505,
        "lp_gold": -21.11834144592285,
        "lp_dist": -25.672313630580902,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.063919067382812,
        "lp_gold": -12.24570107460022,
        "lp_dist": -20.309620141983032,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-23",
      "gold_norm": "750",
      "dist_norm": "9",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.4596693105995655,
        "lp_gold": -14.162512499839067,
        "lp_dist": -13.702843189239502,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3531132936477661,
        "lp_gold": -11.305097699165344,
        "lp_dist": -10.951984405517578,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-24",
      "gold_norm": "9",
      "dist_norm": "40",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.006132304668426514,
        "lp_gold": -15.839151382446289,
        "lp_dist": -15.845283687114716,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.110446274280548,
        "lp_gold": -12.25863265991211,
        "lp_dist": -10.148186385631561,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-25",
      "gold_norm": "40",
      "dist_norm": "14",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.5176091194152832,
        "lp_gold": -16.009315252304077,
        "lp_dist": -15.491706132888794,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.6056453585624695,
        "lp_gold": -14.140560686588287,
        "lp_dist": -15.746206045150757,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-26",
      "gold_norm": "14",
      "dist_norm": "160",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 13.092049598693848,
        "lp_gold": -12.284036666154861,
        "lp_dist": -25.37608626484871,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.250454902648926,
        "lp_gold": -11.187321424484253,
        "lp_dist": -22.43777632713318,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-27",
      "gold_norm": "160",
      "dist_norm": "6",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.597537249326706,
        "lp_gold": -12.841732293367386,
        "lp_dist": -14.439269542694092,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.338132083415985,
        "lp_gold": -17.455387771129608,
        "lp_dist": -10.117255687713623,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-28",
      "gold_norm": "6",
      "dist_norm": "132",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.4012770652771,
        "lp_gold": -12.933898210525513,
        "lp_dist": -18.335175275802612,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.1544095277786255,
        "lp_gold": -9.050714015960693,
        "lp_dist": -16.20512354373932,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-29",
      "gold_norm": "132",
      "dist_norm": "8",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -8.112765461206436,
        "lp_gold": -18.76314067840576,
        "lp_dist": -10.650375217199326,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.735072135925293,
        "lp_gold": -16.513195633888245,
        "lp_dist": -7.778123497962952,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-30",
      "gold_norm": "8",
      "dist_norm": "68",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2700021266937256,
        "lp_gold": -11.72844409942627,
        "lp_dist": -13.998446226119995,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.666181564331055,
        "lp_gold": -7.928534984588623,
        "lp_dist": -20.594716548919678,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-31",
      "gold_norm": "68",
      "dist_norm": "31",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.820281505584717,
        "lp_gold": -15.837103843688965,
        "lp_dist": -19.65738534927368,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.25214481353759766,
        "lp_gold": -12.841001033782959,
        "lp_dist": -12.588856220245361,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-32",
      "gold_norm": "31",
      "dist_norm": "100",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.4860659539699554,
        "lp_gold": -13.60796919465065,
        "lp_dist": -20.094035148620605,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.465068936347961,
        "lp_gold": -14.221534967422485,
        "lp_dist": -18.686603903770447,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-33",
      "gold_norm": "100",
      "dist_norm": "1509",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 13.144955581985414,
        "lp_gold": -13.722247913479805,
        "lp_dist": -26.86720349546522,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.218970347195864,
        "lp_gold": -9.529480028897524,
        "lp_dist": -22.748450376093388,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-34",
      "gold_norm": "1509",
      "dist_norm": "480",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.574819326400757,
        "lp_gold": -23.19943141937256,
        "lp_dist": -16.6246120929718,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.709552764892578,
        "lp_gold": -18.728264808654785,
        "lp_dist": -11.018712043762207,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-35",
      "gold_norm": "480",
      "dist_norm": "520",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.5111888945102692,
        "lp_gold": -12.033819317817688,
        "lp_dist": -13.545008212327957,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8960548639297485,
        "lp_gold": -19.025392055511475,
        "lp_dist": -16.129337191581726,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-36",
      "gold_norm": "520",
      "dist_norm": "3",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.531530350446701,
        "lp_gold": -14.413595885038376,
        "lp_dist": -11.882065534591675,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.166505575180054,
        "lp_gold": -12.240307569503784,
        "lp_dist": -8.07380199432373,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-37",
      "gold_norm": "3",
      "dist_norm": "33",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.900035858154297,
        "lp_gold": -12.652887344360352,
        "lp_dist": -20.55292320251465,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.076467990875244,
        "lp_gold": -7.961295485496521,
        "lp_dist": -14.037763476371765,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-38",
      "gold_norm": "33",
      "dist_norm": "120",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 15.355147242546082,
        "lp_gold": -12.304473280906677,
        "lp_dist": -27.65962052345276,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 10.39077889919281,
        "lp_gold": -11.524258255958557,
        "lp_dist": -21.915037155151367,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-39",
      "gold_norm": "120",
      "dist_norm": "14",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.22332683950662613,
        "lp_gold": -14.252640329301357,
        "lp_dist": -14.475967168807983,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.3626268804073334,
        "lp_gold": -10.953217655420303,
        "lp_dist": -13.315844535827637,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-40",
      "gold_norm": "14",
      "dist_norm": "20",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -7.754596941173077,
        "lp_gold": -17.770805835723877,
        "lp_dist": -10.0162088945508,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.5667039155960083,
        "lp_gold": -8.350147247314453,
        "lp_dist": -7.783443331718445,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-41",
      "gold_norm": "20",
      "dist_norm": "95200",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 33.85109522007406,
        "lp_gold": -16.739925840869546,
        "lp_dist": -50.5910210609436,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 23.679821968078613,
        "lp_gold": -9.72859787940979,
        "lp_dist": -33.4084198474884,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      }
    },
    {
      "ex_id": "gsm8k-test-42",
      "gold_norm": "95200",
      "dist_norm": "77",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.7502023852430284,
        "lp_gold": -19.08837911253795,
        "lp_dist": -16.338176727294922,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.2610103897750378,
        "lp_gold": -21.381718140095472,
        "lp_dist": -18.120707750320435,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-43",
      "gold_norm": "77",
      "dist_norm": "81",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.041521549224853516,
        "lp_gold": -20.968489170074463,
        "lp_dist": -20.92696762084961,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2143611907958984,
        "lp_gold": -14.122482776641846,
        "lp_dist": -16.336843967437744,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-44",
      "gold_norm": "81",
      "dist_norm": "310",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.942630738019943,
        "lp_gold": -12.305748492479324,
        "lp_dist": -21.248379230499268,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.2553623914718628,
        "lp_gold": -13.983943223953247,
        "lp_dist": -14.23930561542511,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-45",
      "gold_norm": "310",
      "dist_norm": "100",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9456039071083069,
        "lp_gold": -16.172270894050598,
        "lp_dist": -17.117874801158905,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.856696009635925,
        "lp_gold": -16.36608850955963,
        "lp_dist": -10.509392499923706,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-46",
      "gold_norm": "100",
      "dist_norm": "160",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.3894251135061495,
        "lp_gold": -12.40682859485969,
        "lp_dist": -15.796253708365839,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0833441019058228,
        "lp_gold": -16.92129546403885,
        "lp_dist": -19.00463956594467,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-47",
      "gold_norm": "160",
      "dist_norm": "25",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.824264804366976,
        "lp_gold": -14.298029144760221,
        "lp_dist": -15.122293949127197,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.2988634258508682,
        "lp_gold": -13.753293856978416,
        "lp_dist": -14.052157282829285,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-48",
      "gold_norm": "25",
      "dist_norm": "1400",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.125034153461456,
        "lp_gold": -16.447975158691406,
        "lp_dist": -23.573009312152863,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.08089584112167358,
        "lp_gold": -13.430449962615967,
        "lp_dist": -13.349554121494293,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-49",
      "gold_norm": "1400",
      "dist_norm": "120",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.9659185571945272,
        "lp_gold": -15.535773673269432,
        "lp_dist": -12.569855116074905,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.700849339365959,
        "lp_gold": -12.968689993023872,
        "lp_dist": -9.267840653657913,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-50",
      "gold_norm": "120",
      "dist_norm": "48",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.5417392253875732,
        "lp_gold": -22.00163245201111,
        "lp_dist": -25.54337167739868,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.251087546348572,
        "lp_gold": -15.896643280982971,
        "lp_dist": -21.147730827331543,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-51",
      "gold_norm": "48",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9340271949768066,
        "lp_gold": -11.738685846328735,
        "lp_dist": -12.672713041305542,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.7094523906707764,
        "lp_gold": -6.564473628997803,
        "lp_dist": -8.273926019668579,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-52",
      "gold_norm": "50",
      "dist_norm": "15400",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 19.066895127296448,
        "lp_gold": -13.457320094108582,
        "lp_dist": -32.52421522140503,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.942362904548645,
        "lp_gold": -15.37247109413147,
        "lp_dist": -22.314833998680115,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      }
    },
    {
      "ex_id": "gsm8k-test-53",
      "gold_norm": "15400",
      "dist_norm": "80",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8634248977759853,
        "lp_gold": -20.78267443238292,
        "lp_dist": -18.919249534606934,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.060255475342274,
        "lp_gold": -20.97295517474413,
        "lp_dist": -16.912699699401855,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-54",
      "gold_norm": "80",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1981298923492432,
        "lp_gold": -18.350556135177612,
        "lp_dist": -17.15242624282837,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.7748947478830814,
        "lp_gold": -7.663371529430151,
        "lp_dist": -11.438266277313232,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-55",
      "gold_norm": "5",
      "dist_norm": "14",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.197003062348813,
        "lp_gold": -18.496329307556152,
        "lp_dist": -12.29932624520734,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.144104599952698,
        "lp_gold": -7.03524386882782,
        "lp_dist": -14.179348468780518,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-56",
      "gold_norm": "14",
      "dist_norm": "31",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.266514074697625,
        "lp_gold": -18.428703057870734,
        "lp_dist": -26.69521713256836,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.1581010818481445,
        "lp_gold": -12.810563087463379,
        "lp_dist": -16.968664169311523,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-57",
      "gold_norm": "31",
      "dist_norm": "36",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7361334562301636,
        "lp_gold": -14.841211199760437,
        "lp_dist": -13.105077743530273,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6261711120605469,
        "lp_gold": -13.683454990386963,
        "lp_dist": -13.057283878326416,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-58",
      "gold_norm": "36",
      "dist_norm": "144",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.37734442949295044,
        "lp_gold": -15.673691511154175,
        "lp_dist": -16.051035940647125,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.521233081817627,
        "lp_gold": -17.215554237365723,
        "lp_dist": -20.73678731918335,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-59",
      "gold_norm": "144",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.0488511323928833,
        "lp_gold": -15.628765225410461,
        "lp_dist": -13.579914093017578,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.597236156463623,
        "lp_gold": -16.582991123199463,
        "lp_dist": -13.98575496673584,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-60",
      "gold_norm": "5",
      "dist_norm": "750",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 11.9340181350708,
        "lp_gold": -15.854983806610107,
        "lp_dist": -27.789001941680908,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.577134609222412,
        "lp_gold": -16.538414001464844,
        "lp_dist": -24.115548610687256,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-61",
      "gold_norm": "750",
      "dist_norm": "38",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7815818190574646,
        "lp_gold": -18.73184484243393,
        "lp_dist": -17.950263023376465,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.5384882092475891,
        "lp_gold": -11.338704288005829,
        "lp_dist": -11.877192497253418,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-62",
      "gold_norm": "38",
      "dist_norm": "48",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.48445200920105,
        "lp_gold": -17.717634916305542,
        "lp_dist": -11.233182907104492,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.8313806354999542,
        "lp_gold": -12.223527193069458,
        "lp_dist": -8.392146557569504,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-63",
      "gold_norm": "48",
      "dist_norm": "655",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 19.08035659790039,
        "lp_gold": -14.92322301864624,
        "lp_dist": -34.00357961654663,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.06407618522644,
        "lp_gold": -10.739889144897461,
        "lp_dist": -23.8039653301239,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-64",
      "gold_norm": "655",
      "dist_norm": "800",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.8698419332504272,
        "lp_gold": -17.930187582969666,
        "lp_dist": -19.800029516220093,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4062272310256958,
        "lp_gold": -14.47088611125946,
        "lp_dist": -13.064658880233765,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-65",
      "gold_norm": "800",
      "dist_norm": "7300",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 11.858935464173555,
        "lp_gold": -13.705154906958342,
        "lp_dist": -25.564090371131897,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.260221555829048,
        "lp_gold": -14.01822917163372,
        "lp_dist": -25.27845072746277,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-66",
      "gold_norm": "7300",
      "dist_norm": "48",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.354404352605343,
        "lp_gold": -20.02805521339178,
        "lp_dist": -17.673650860786438,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.008578598499298096,
        "lp_gold": -13.163566768169403,
        "lp_dist": -13.172145366668701,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-67",
      "gold_norm": "48",
      "dist_norm": "4",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.25957900285720825,
        "lp_gold": -17.536937534809113,
        "lp_dist": -17.277358531951904,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7365117073059082,
        "lp_gold": -10.189218521118164,
        "lp_dist": -8.452706813812256,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-68",
      "gold_norm": "4",
      "dist_norm": "15",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.0612575560808182,
        "lp_gold": -15.63377046585083,
        "lp_dist": -14.572512909770012,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.9296765327453613,
        "lp_gold": -9.090213418006897,
        "lp_dist": -12.019889950752258,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-69",
      "gold_norm": "15",
      "dist_norm": "23",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.69707328081131,
        "lp_gold": -14.628733813762665,
        "lp_dist": -23.325807094573975,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.665110111236572,
        "lp_gold": -12.951319694519043,
        "lp_dist": -17.616429805755615,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-70",
      "gold_norm": "23",
      "dist_norm": "225",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 11.066251754760742,
        "lp_gold": -16.185874462127686,
        "lp_dist": -27.252126216888428,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.176498889923096,
        "lp_gold": -14.897132635116577,
        "lp_dist": -20.073631525039673,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-71",
      "gold_norm": "225",
      "dist_norm": "15",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.3921156525611877,
        "lp_gold": -16.729829609394073,
        "lp_dist": -13.337713956832886,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.107092618942261,
        "lp_gold": -17.290175914764404,
        "lp_dist": -11.183083295822144,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-72",
      "gold_norm": "15",
      "dist_norm": "82",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.4555931091308594,
        "lp_gold": -13.475011110305786,
        "lp_dist": -16.930604219436646,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 9.554435849189758,
        "lp_gold": -11.591153025627136,
        "lp_dist": -21.145588874816895,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-73",
      "gold_norm": "82",
      "dist_norm": "1218",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.144330263137817,
        "lp_gold": -14.399481773376465,
        "lp_dist": -24.543812036514282,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.632067203521729,
        "lp_gold": -11.898912191390991,
        "lp_dist": -25.53097939491272,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-74",
      "gold_norm": "1218",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -14.016261577606201,
        "lp_gold": -28.183964252471924,
        "lp_dist": -14.167702674865723,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -9.392110228538513,
        "lp_gold": -21.020013689994812,
        "lp_dist": -11.627903461456299,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-75",
      "gold_norm": "2",
      "dist_norm": "36",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.396540880203247,
        "lp_gold": -13.256061553955078,
        "lp_dist": -15.652602434158325,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.963439464569092,
        "lp_gold": -9.20676326751709,
        "lp_dist": -16.17020273208618,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-76",
      "gold_norm": "36",
      "dist_norm": "13",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.540708303451538,
        "lp_gold": -18.379968881607056,
        "lp_dist": -14.839260578155518,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6864776611328125,
        "lp_gold": -14.157576084136963,
        "lp_dist": -13.47109842300415,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-77",
      "gold_norm": "13",
      "dist_norm": "11",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.3661365509033203,
        "lp_gold": -15.502496480941772,
        "lp_dist": -15.868633031845093,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.4615020751953125,
        "lp_gold": -14.546976089477539,
        "lp_dist": -15.008478164672852,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-78",
      "gold_norm": "11",
      "dist_norm": "8",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.7840418815612793,
        "lp_gold": -21.872905254364014,
        "lp_dist": -18.088863372802734,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.002429008483887,
        "lp_gold": -13.64483380317688,
        "lp_dist": -9.642404794692993,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-79",
      "gold_norm": "8",
      "dist_norm": "440",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.767229557037354,
        "lp_gold": -17.545647621154785,
        "lp_dist": -34.31287717819214,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.454898834228516,
        "lp_gold": -12.977333545684814,
        "lp_dist": -24.43223237991333,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-80",
      "gold_norm": "440",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.289784381631762,
        "lp_gold": -15.706766793970019,
        "lp_dist": -13.416982412338257,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.212664246559143,
        "lp_gold": -17.012100338935852,
        "lp_dist": -13.799436092376709,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-81",
      "gold_norm": "2",
      "dist_norm": "45",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.003484487533569,
        "lp_gold": -13.458641052246094,
        "lp_dist": -20.462125539779663,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.808353066444397,
        "lp_gold": -8.184542536735535,
        "lp_dist": -13.992895603179932,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-82",
      "gold_norm": "45",
      "dist_norm": "54",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.24556124210357666,
        "lp_gold": -15.131654500961304,
        "lp_dist": -14.886093258857727,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.5171573162078857,
        "lp_gold": -10.597485780715942,
        "lp_dist": -12.114643096923828,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-83",
      "gold_norm": "54",
      "dist_norm": "6",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.36360502243042,
        "lp_gold": -13.119836330413818,
        "lp_dist": -10.756231307983398,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.9401865005493164,
        "lp_gold": -13.909927368164062,
        "lp_dist": -10.969740867614746,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-84",
      "gold_norm": "6",
      "dist_norm": "240",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.525604009628296,
        "lp_gold": -14.586916446685791,
        "lp_dist": -31.112520456314087,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.351407527923584,
        "lp_gold": -8.294451236724854,
        "lp_dist": -20.645858764648438,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-85",
      "gold_norm": "240",
      "dist_norm": "428",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.200618744827807,
        "lp_gold": -10.596241324208677,
        "lp_dist": -15.796860069036484,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.575641840696335,
        "lp_gold": -10.782865315675735,
        "lp_dist": -16.35850715637207,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-86",
      "gold_norm": "428",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -11.445145592093468,
        "lp_gold": -21.372050523757935,
        "lp_dist": -9.926904931664467,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8857234716415405,
        "lp_gold": -13.641488909721375,
        "lp_dist": -10.755765438079834,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-87",
      "gold_norm": "5",
      "dist_norm": "255",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 13.83140754699707,
        "lp_gold": -11.654325008392334,
        "lp_dist": -25.485732555389404,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 15.030074834823608,
        "lp_gold": -10.351794719696045,
        "lp_dist": -25.381869554519653,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-88",
      "gold_norm": "255",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.704339981079102,
        "lp_gold": -22.688746690750122,
        "lp_dist": -17.98440670967102,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.790473118424416,
        "lp_gold": -23.528611078858376,
        "lp_dist": -18.73813796043396,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-89",
      "gold_norm": "10",
      "dist_norm": "9",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6458263397216797,
        "lp_gold": -14.022311687469482,
        "lp_dist": -12.376485347747803,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.548365592956543,
        "lp_gold": -11.53894329071045,
        "lp_dist": -9.990577697753906,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-90",
      "gold_norm": "9",
      "dist_norm": "157",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 11.838926374912262,
        "lp_gold": -14.729028940200806,
        "lp_dist": -26.567955315113068,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.449068486690521,
        "lp_gold": -13.700207710266113,
        "lp_dist": -18.149276196956635,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-91",
      "gold_norm": "157",
      "dist_norm": "56",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.06636106967926025,
        "lp_gold": -13.999522089958191,
        "lp_dist": -13.93316102027893,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.18358194828033447,
        "lp_gold": -12.361805081367493,
        "lp_dist": -12.178223133087158,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-92",
      "gold_norm": "56",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.014044851064682007,
        "lp_gold": -13.622624963521957,
        "lp_dist": -13.608580112457275,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.375391006469727,
        "lp_gold": -15.001872539520264,
        "lp_dist": -9.626481533050537,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-93",
      "gold_norm": "5",
      "dist_norm": "144",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.107487440109253,
        "lp_gold": -21.01281452178955,
        "lp_dist": -31.120301961898804,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.960463047027588,
        "lp_gold": -8.431816339492798,
        "lp_dist": -20.392279386520386,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-94",
      "gold_norm": "144",
      "dist_norm": "50",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8007860428187996,
        "lp_gold": -16.124470019945875,
        "lp_dist": -15.323683977127075,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.586140275001526,
        "lp_gold": -13.455833077430725,
        "lp_dist": -10.8696928024292,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-95",
      "gold_norm": "50",
      "dist_norm": "4",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.431166723370552,
        "lp_gold": -14.887599676847458,
        "lp_dist": -10.456432953476906,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.6977656185626984,
        "lp_gold": -7.156943529844284,
        "lp_dist": -8.854709148406982,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-96",
      "gold_norm": "4",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.549594163894653,
        "lp_gold": -12.547298669815063,
        "lp_dist": -19.096892833709717,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.145283490419388,
        "lp_gold": -8.248348951339722,
        "lp_dist": -10.39363244175911,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-97",
      "gold_norm": "50",
      "dist_norm": "42",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1262907013297081,
        "lp_gold": -12.730935551226139,
        "lp_dist": -12.857226252555847,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.2466068267822266,
        "lp_gold": -9.356587171554565,
        "lp_dist": -12.603193998336792,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-98",
      "gold_norm": "42",
      "dist_norm": "7",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -5.333731412887573,
        "lp_gold": -18.121748208999634,
        "lp_dist": -12.78801679611206,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.454429030418396,
        "lp_gold": -11.41723620891571,
        "lp_dist": -10.962807178497314,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-99",
      "gold_norm": "7",
      "dist_norm": "250",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.109447717666626,
        "lp_gold": -17.65127396583557,
        "lp_dist": -27.760721683502197,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.160389855504036,
        "lp_gold": -12.602290153503418,
        "lp_dist": -15.762680009007454,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-100",
      "gold_norm": "250",
      "dist_norm": "12",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.937235951423645,
        "lp_gold": -16.930358290672302,
        "lp_dist": -18.867594242095947,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8317363262176514,
        "lp_gold": -17.415368795394897,
        "lp_dist": -15.583632469177246,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-101",
      "gold_norm": "12",
      "dist_norm": "7",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.4091547727584839,
        "lp_gold": -11.118706822395325,
        "lp_dist": -12.527861595153809,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.3088250160217285,
        "lp_gold": -9.039770126342773,
        "lp_dist": -11.348595142364502,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-102",
      "gold_norm": "7",
      "dist_norm": "8",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.3529720306396484,
        "lp_gold": -18.84420108795166,
        "lp_dist": -17.49122905731201,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1846144199371338,
        "lp_gold": -10.204635620117188,
        "lp_dist": -9.020021200180054,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-103",
      "gold_norm": "8",
      "dist_norm": "26",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.383567810058594,
        "lp_gold": -15.868620872497559,
        "lp_dist": -26.252188682556152,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.597115993499756,
        "lp_gold": -10.359461784362793,
        "lp_dist": -18.95657777786255,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-104",
      "gold_norm": "26",
      "dist_norm": "42",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.550231754779816,
        "lp_gold": -16.079154193401337,
        "lp_dist": -20.629385948181152,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.743180751800537,
        "lp_gold": -20.936619758605957,
        "lp_dist": -17.19343900680542,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-105",
      "gold_norm": "42",
      "dist_norm": "5",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2448320388793945,
        "lp_gold": -17.29369354248047,
        "lp_dist": -19.538525581359863,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.48531031608581543,
        "lp_gold": -14.891574144363403,
        "lp_dist": -14.406263828277588,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-106",
      "gold_norm": "5",
      "dist_norm": "14400",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 17.918405055999756,
        "lp_gold": -16.223863124847412,
        "lp_dist": -34.14226818084717,
        "n_tokens_gold": 2,
        "n_tokens_dist": 6
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 18.689422607421875,
        "lp_gold": -8.31445324420929,
        "lp_dist": -27.003875851631165,
        "n_tokens_gold": 2,
        "n_tokens_dist": 6
      }
    },
    {
      "ex_id": "gsm8k-test-107",
      "gold_norm": "14400",
      "dist_norm": "400",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8998411595821381,
        "lp_gold": -19.079706698656082,
        "lp_dist": -19.97954785823822,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.72913409024477,
        "lp_gold": -22.703229255974293,
        "lp_dist": -20.974095165729523,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-108",
      "gold_norm": "400",
      "dist_norm": "40",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.00030357998912222683,
        "lp_gold": -12.683453394594835,
        "lp_dist": -12.683149814605713,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.014066597446799278,
        "lp_gold": -7.615564605221152,
        "lp_dist": -7.601498007774353,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-109",
      "gold_norm": "40",
      "dist_norm": "83",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.819165468215942,
        "lp_gold": -15.469541311264038,
        "lp_dist": -21.28870677947998,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.161367028951645,
        "lp_gold": -13.052747160196304,
        "lp_dist": -21.21411418914795,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-110",
      "gold_norm": "83",
      "dist_norm": "10",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9284783601760864,
        "lp_gold": -14.152065396308899,
        "lp_dist": -15.080543756484985,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.6964447498321533,
        "lp_gold": -11.795601606369019,
        "lp_dist": -8.099156856536865,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-111",
      "gold_norm": "10",
      "dist_norm": "80",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.51949143409729,
        "lp_gold": -16.031707048416138,
        "lp_dist": -22.551198482513428,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.987100124359131,
        "lp_gold": -14.24100637435913,
        "lp_dist": -20.22810649871826,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-112",
      "gold_norm": "80",
      "dist_norm": "180",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.0028108435217291,
        "lp_gold": -14.749124482274055,
        "lp_dist": -15.751935325795785,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.4700966998934746,
        "lp_gold": -9.443349197506905,
        "lp_dist": -11.91344589740038,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-113",
      "gold_norm": "180",
      "dist_norm": "1450000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 21.25386095046997,
        "lp_gold": -27.976417541503906,
        "lp_dist": -49.23027849197388,
        "n_tokens_gold": 4,
        "n_tokens_dist": 8
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.907071352005005,
        "lp_gold": -15.60246878862381,
        "lp_dist": -29.509540140628815,
        "n_tokens_gold": 4,
        "n_tokens_dist": 8
      }
    },
    {
      "ex_id": "gsm8k-test-114",
      "gold_norm": "1450000",
      "dist_norm": "15",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -5.66898221289739,
        "lp_gold": -18.9738236120902,
        "lp_dist": -13.30484139919281,
        "n_tokens_gold": 8,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.122156334575266,
        "lp_gold": -18.38626338308677,
        "lp_dist": -10.264107048511505,
        "n_tokens_gold": 8,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-115",
      "gold_norm": "15",
      "dist_norm": "1000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.356005743145943,
        "lp_gold": -21.951110124588013,
        "lp_dist": -30.307115867733955,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.199884116649628,
        "lp_gold": -11.521326780319214,
        "lp_dist": -19.72121089696884,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-116",
      "gold_norm": "1000",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.19174530085001606,
        "lp_gold": -13.59133626993571,
        "lp_dist": -13.399590969085693,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.19964181631803513,
        "lp_gold": -10.637855164706707,
        "lp_dist": -10.438213348388672,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-117",
      "gold_norm": "2",
      "dist_norm": "15",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.722217559814453,
        "lp_gold": -18.274237632751465,
        "lp_dist": -24.996455192565918,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.5563411712646484,
        "lp_gold": -12.917238712310791,
        "lp_dist": -14.47357988357544,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-118",
      "gold_norm": "15",
      "dist_norm": "100",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.2469024658203125,
        "lp_gold": -14.20986533164978,
        "lp_dist": -17.456767797470093,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.755454957485199,
        "lp_gold": -7.566788613796234,
        "lp_dist": -16.322243571281433,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-119",
      "gold_norm": "100",
      "dist_norm": "335",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.725587379769422,
        "lp_gold": -11.31663225905504,
        "lp_dist": -28.042219638824463,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.39561602845788,
        "lp_gold": -11.48399594053626,
        "lp_dist": -19.87961196899414,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-120",
      "gold_norm": "335",
      "dist_norm": "60",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.718258023262024,
        "lp_gold": -16.84885323047638,
        "lp_dist": -18.567111253738403,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.495100736618042,
        "lp_gold": -16.78837823867798,
        "lp_dist": -13.293277502059937,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-121",
      "gold_norm": "60",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.7624173164367676,
        "lp_gold": -16.792863368988037,
        "lp_dist": -14.03044605255127,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.886390924453735,
        "lp_gold": -13.833235025405884,
        "lp_dist": -7.946844100952148,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-122",
      "gold_norm": "5",
      "dist_norm": "9500",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.83846201375127,
        "lp_gold": -13.696074485778809,
        "lp_dist": -30.534536499530077,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 15.658583138138056,
        "lp_gold": -8.453100323677063,
        "lp_dist": -24.11168346181512,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-123",
      "gold_norm": "9500",
      "dist_norm": "160",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3972220839932561,
        "lp_gold": -20.279614341445267,
        "lp_dist": -19.88239225745201,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.62424199283123,
        "lp_gold": -20.69040386378765,
        "lp_dist": -18.06616187095642,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-124",
      "gold_norm": "160",
      "dist_norm": "1050",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.60233561617497,
        "lp_gold": -18.274476603444782,
        "lp_dist": -34.87681221961975,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.274514463730156,
        "lp_gold": -13.8317128745839,
        "lp_dist": -22.106227338314056,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-125",
      "gold_norm": "1050",
      "dist_norm": "91",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.0837584948167205,
        "lp_gold": -23.45734657999128,
        "lp_dist": -21.37358808517456,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.869788646697998,
        "lp_gold": -20.34303617477417,
        "lp_dist": -17.473247528076172,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-126",
      "gold_norm": "91",
      "dist_norm": "21",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.658190071582794,
        "lp_gold": -20.631014347076416,
        "lp_dist": -13.972824275493622,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.4234659671783447,
        "lp_gold": -17.770225048065186,
        "lp_dist": -15.34675908088684,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-127",
      "gold_norm": "21",
      "dist_norm": "20",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.925290822982788,
        "lp_gold": -16.08545808121562,
        "lp_dist": -14.160167258232832,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.1263790130615234,
        "lp_gold": -13.073553562164307,
        "lp_dist": -10.947174549102783,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-128",
      "gold_norm": "20",
      "dist_norm": "36",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.3922419548034668,
        "lp_gold": -19.574571132659912,
        "lp_dist": -19.96681308746338,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.179409921169281,
        "lp_gold": -8.625289022922516,
        "lp_dist": -10.804698944091797,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-129",
      "gold_norm": "36",
      "dist_norm": "36",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -12.731476545333862,
        "lp_dist": -12.731476545333862,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -11.376878261566162,
        "lp_dist": -11.376878261566162,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-130",
      "gold_norm": "36",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.552075147628784,
        "lp_gold": -14.53696346282959,
        "lp_dist": -11.984888315200806,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -11.998991012573242,
        "lp_gold": -20.990919589996338,
        "lp_dist": -8.991928577423096,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-131",
      "gold_norm": "10",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -8.991529874503613,
        "lp_gold": -16.921989023685455,
        "lp_dist": -7.930459149181843,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.6315575018525124,
        "lp_gold": -6.705006085336208,
        "lp_dist": -8.33656358718872,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-132",
      "gold_norm": "5",
      "dist_norm": "32",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.572678565979004,
        "lp_gold": -12.411277294158936,
        "lp_dist": -18.98395586013794,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.04911994934082,
        "lp_gold": -8.343387603759766,
        "lp_dist": -12.392507553100586,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-133",
      "gold_norm": "32",
      "dist_norm": "18",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6729400157928467,
        "lp_gold": -15.0261971950531,
        "lp_dist": -14.353257179260254,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8697174787521362,
        "lp_gold": -10.817826390266418,
        "lp_dist": -11.687543869018555,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-134",
      "gold_norm": "18",
      "dist_norm": "4",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.9239641074091196,
        "lp_gold": -15.171829616650939,
        "lp_dist": -19.09579372406006,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7061721086502075,
        "lp_gold": -9.731460690498352,
        "lp_dist": -9.025288581848145,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-135",
      "gold_norm": "4",
      "dist_norm": "48",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.912032127380371,
        "lp_gold": -11.661430835723877,
        "lp_dist": -21.573462963104248,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 10.949398040771484,
        "lp_gold": -13.34396743774414,
        "lp_dist": -24.293365478515625,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-136",
      "gold_norm": "48",
      "dist_norm": "8",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2137904167175293,
        "lp_gold": -13.246366620063782,
        "lp_dist": -12.032576203346252,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.785146713256836,
        "lp_gold": -15.54097604751587,
        "lp_dist": -10.755829334259033,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-137",
      "gold_norm": "8",
      "dist_norm": "21",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.050054788589478,
        "lp_gold": -14.328342199325562,
        "lp_dist": -23.37839698791504,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.801905870437622,
        "lp_gold": -9.647645235061646,
        "lp_dist": -13.449551105499268,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-138",
      "gold_norm": "21",
      "dist_norm": "25",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.6713391542434692,
        "lp_gold": -17.788984179496765,
        "lp_dist": -20.460323333740234,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8663175106048584,
        "lp_gold": -14.334570407867432,
        "lp_dist": -13.468252897262573,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-139",
      "gold_norm": "25",
      "dist_norm": "3000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.303999066352844,
        "lp_gold": -13.958717346191406,
        "lp_dist": -24.26271641254425,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.2307329177856445,
        "lp_gold": -7.678596615791321,
        "lp_dist": -13.909329533576965,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-140",
      "gold_norm": "3000",
      "dist_norm": "40",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8300985433161259,
        "lp_gold": -17.700348053127527,
        "lp_dist": -15.870249509811401,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.4543883726000786,
        "lp_gold": -14.831104047596455,
        "lp_dist": -15.285492420196533,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-141",
      "gold_norm": "40",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.7375423088669777,
        "lp_gold": -13.080543011426926,
        "lp_dist": -14.818085320293903,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.48419055342674255,
        "lp_gold": -11.210194662213326,
        "lp_dist": -11.694385215640068,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-142",
      "gold_norm": "50",
      "dist_norm": "90",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.239281177520752,
        "lp_gold": -13.6494460105896,
        "lp_dist": -12.410164833068848,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.5999003648757935,
        "lp_gold": -9.939468264579773,
        "lp_dist": -11.539368629455566,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-143",
      "gold_norm": "90",
      "dist_norm": "23",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3710329532623291,
        "lp_gold": -20.11902666091919,
        "lp_dist": -19.74799370765686,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.965712070465088,
        "lp_gold": -15.742670059204102,
        "lp_dist": -18.70838212966919,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-144",
      "gold_norm": "23",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.9211039543151855,
        "lp_gold": -17.69726538658142,
        "lp_dist": -12.776161432266235,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.012661933898926,
        "lp_gold": -12.401761054992676,
        "lp_dist": -8.38909912109375,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-145",
      "gold_norm": "2",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2492438331246376,
        "lp_gold": -13.602060556411743,
        "lp_dist": -14.85130438953638,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.6407327204942703,
        "lp_gold": -12.194403648376465,
        "lp_dist": -9.553670927882195,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-146",
      "gold_norm": "50",
      "dist_norm": "122",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.681281805038452,
        "lp_gold": -16.079622983932495,
        "lp_dist": -20.760904788970947,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.314260721206665,
        "lp_gold": -11.10973858833313,
        "lp_dist": -18.423999309539795,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-147",
      "gold_norm": "122",
      "dist_norm": "300",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.33387154340744,
        "lp_gold": -18.671439349651337,
        "lp_dist": -25.005310893058777,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.5978607535362244,
        "lp_gold": -15.271158814430237,
        "lp_dist": -15.869019567966461,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-148",
      "gold_norm": "300",
      "dist_norm": "448",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 13.451393851355533,
        "lp_gold": -17.04763673870184,
        "lp_dist": -30.499030590057373,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.197498982015532,
        "lp_gold": -14.669522101816256,
        "lp_dist": -27.867021083831787,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-149",
      "gold_norm": "448",
      "dist_norm": "2450",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.949738264083862,
        "lp_gold": -25.570088386535645,
        "lp_dist": -31.519826650619507,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.753702878952026,
        "lp_gold": -20.34923005104065,
        "lp_dist": -26.102932929992676,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-150",
      "gold_norm": "2450",
      "dist_norm": "803",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.90772854257375,
        "lp_gold": -16.1460417015478,
        "lp_dist": -22.05377024412155,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.6608445048332214,
        "lp_gold": -14.72765988111496,
        "lp_dist": -15.388504385948181,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-151",
      "gold_norm": "803",
      "dist_norm": "16",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.22600507736206055,
        "lp_gold": -20.624857425689697,
        "lp_dist": -20.850862503051758,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.131483793258667,
        "lp_gold": -19.228359699249268,
        "lp_dist": -14.0968759059906,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-152",
      "gold_norm": "16",
      "dist_norm": "280",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.489606261253357,
        "lp_gold": -13.729230046272278,
        "lp_dist": -23.218836307525635,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.312919616699219,
        "lp_gold": -9.783522605895996,
        "lp_dist": -23.096442222595215,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-153",
      "gold_norm": "280",
      "dist_norm": "13",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.694299184717238,
        "lp_gold": -15.244264638982713,
        "lp_dist": -18.93856382369995,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8231047093868256,
        "lp_gold": -12.64960965514183,
        "lp_dist": -11.826504945755005,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-154",
      "gold_norm": "13",
      "dist_norm": "20",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0602927803993225,
        "lp_gold": -15.215918719768524,
        "lp_dist": -17.276211500167847,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.633309006690979,
        "lp_gold": -12.038846015930176,
        "lp_dist": -11.405537009239197,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-155",
      "gold_norm": "20",
      "dist_norm": "14",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.590712457895279,
        "lp_gold": -18.284266233444214,
        "lp_dist": -15.693553775548935,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.9870389699935913,
        "lp_gold": -10.093660473823547,
        "lp_dist": -12.080699443817139,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-156",
      "gold_norm": "14",
      "dist_norm": "32",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.239875316619873,
        "lp_gold": -20.827781200408936,
        "lp_dist": -25.06765651702881,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.260110855102539,
        "lp_gold": -13.14831280708313,
        "lp_dist": -15.408423662185669,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-157",
      "gold_norm": "32",
      "dist_norm": "105",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 12.336161613464355,
        "lp_gold": -20.509262084960938,
        "lp_dist": -32.84542369842529,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.297417879104614,
        "lp_gold": -9.766870260238647,
        "lp_dist": -16.06428813934326,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-158",
      "gold_norm": "105",
      "dist_norm": "71",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.14784783124923706,
        "lp_gold": -21.096359431743622,
        "lp_dist": -20.948511600494385,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4936403930187225,
        "lp_gold": -13.522361606359482,
        "lp_dist": -12.02872121334076,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-159",
      "gold_norm": "71",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2031426429748535,
        "lp_gold": -17.497971057891846,
        "lp_dist": -16.294828414916992,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.283137083053589,
        "lp_gold": -15.376394033432007,
        "lp_dist": -12.093256950378418,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-160",
      "gold_norm": "5",
      "dist_norm": "30",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.20781135559082,
        "lp_gold": -9.617193222045898,
        "lp_dist": -14.825004577636719,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.284235119819641,
        "lp_gold": -6.052669286727905,
        "lp_dist": -9.336904406547546,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-161",
      "gold_norm": "30",
      "dist_norm": "95",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.308123826980591,
        "lp_gold": -12.008728742599487,
        "lp_dist": -18.316852569580078,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.064014196395874,
        "lp_gold": -11.074568510055542,
        "lp_dist": -16.138582706451416,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-162",
      "gold_norm": "95",
      "dist_norm": "147",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 14.487586110830307,
        "lp_gold": -13.062050491571426,
        "lp_dist": -27.549636602401733,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.7768335342407227,
        "lp_gold": -18.569175243377686,
        "lp_dist": -22.346008777618408,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-163",
      "gold_norm": "147",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -5.9637865126132965,
        "lp_gold": -13.891173975542188,
        "lp_dist": -7.927387462928891,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.2743232250213623,
        "lp_gold": -11.51265001296997,
        "lp_dist": -11.238326787948608,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-164",
      "gold_norm": "10",
      "dist_norm": "40000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.0207504555583,
        "lp_gold": -12.818441897630692,
        "lp_dist": -28.83919235318899,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 15.683642621152103,
        "lp_gold": -6.558068131096661,
        "lp_dist": -22.241710752248764,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      }
    },
    {
      "ex_id": "gsm8k-test-165",
      "gold_norm": "40000",
      "dist_norm": "12",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.0927228182554245,
        "lp_gold": -21.754829093813896,
        "lp_dist": -15.662106275558472,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.87526479922235,
        "lp_gold": -16.18092787824571,
        "lp_dist": -9.305663079023361,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-166",
      "gold_norm": "12",
      "dist_norm": "129200",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 17.634711742401123,
        "lp_gold": -20.60856056213379,
        "lp_dist": -38.24327230453491,
        "n_tokens_gold": 3,
        "n_tokens_dist": 7
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 16.27542757987976,
        "lp_gold": -20.810616493225098,
        "lp_dist": -37.08604407310486,
        "n_tokens_gold": 3,
        "n_tokens_dist": 7
      }
    },
    {
      "ex_id": "gsm8k-test-167",
      "gold_norm": "129200",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -8.816860965336673,
        "lp_gold": -24.884800246800296,
        "lp_dist": -16.067939281463623,
        "n_tokens_gold": 7,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -12.447648953646421,
        "lp_gold": -26.88768095895648,
        "lp_dist": -14.440032005310059,
        "n_tokens_gold": 7,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-168",
      "gold_norm": "5",
      "dist_norm": "45",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.262725353240967,
        "lp_gold": -11.162125587463379,
        "lp_dist": -18.424850940704346,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.129469871520996,
        "lp_gold": -6.1789350509643555,
        "lp_dist": -18.30840492248535,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-169",
      "gold_norm": "45",
      "dist_norm": "20",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.7432926744222641,
        "lp_gold": -13.45964826643467,
        "lp_dist": -14.202940940856934,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.6613135635852814,
        "lp_gold": -9.49636921286583,
        "lp_dist": -12.15768277645111,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-170",
      "gold_norm": "20",
      "dist_norm": "1170",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 28.943727374076843,
        "lp_gold": -13.921928644180298,
        "lp_dist": -42.86565601825714,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.730559349060059,
        "lp_gold": -15.503687143325806,
        "lp_dist": -28.234246492385864,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-171",
      "gold_norm": "1170",
      "dist_norm": "192",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.4793071039021015,
        "lp_gold": -25.866613794118166,
        "lp_dist": -23.387306690216064,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.7092228829860687,
        "lp_gold": -17.528388172388077,
        "lp_dist": -20.237611055374146,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-172",
      "gold_norm": "192",
      "dist_norm": "14",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.4512736797332764,
        "lp_gold": -20.52132660150528,
        "lp_dist": -18.070052921772003,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.270252227783203,
        "lp_gold": -18.500007390975952,
        "lp_dist": -10.229755163192749,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-173",
      "gold_norm": "14",
      "dist_norm": "144",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.012117385864258,
        "lp_gold": -19.01213574409485,
        "lp_dist": -25.024253129959106,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.6395363807678223,
        "lp_gold": -17.207820653915405,
        "lp_dist": -20.847357034683228,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-174",
      "gold_norm": "144",
      "dist_norm": "350",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.220242428360507,
        "lp_gold": -20.075438094558194,
        "lp_dist": -29.2956805229187,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.203243670635857,
        "lp_gold": -15.825250687426887,
        "lp_dist": -24.028494358062744,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-175",
      "gold_norm": "350",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9250896275043488,
        "lp_gold": -22.99281856417656,
        "lp_dist": -23.917908191680908,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.236621737480164,
        "lp_gold": -17.58333122730255,
        "lp_dist": -10.346709489822388,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-176",
      "gold_norm": "50",
      "dist_norm": "7",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.0019221305847167969,
        "lp_gold": -14.026922941207886,
        "lp_dist": -14.025000810623169,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7933419942855835,
        "lp_gold": -10.8789883852005,
        "lp_dist": -10.085646390914917,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-177",
      "gold_norm": "7",
      "dist_norm": "50",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.36652660369873047,
        "lp_gold": -14.853872776031494,
        "lp_dist": -14.487346172332764,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.34556770324707,
        "lp_gold": -7.91382908821106,
        "lp_dist": -13.25939679145813,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-178",
      "gold_norm": "50",
      "dist_norm": "8",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.387412190437317,
        "lp_gold": -12.106852412223816,
        "lp_dist": -15.494264602661133,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1638234257698059,
        "lp_gold": -11.002189338207245,
        "lp_dist": -11.16601276397705,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-179",
      "gold_norm": "8",
      "dist_norm": "3160",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 27.79468995332718,
        "lp_gold": -10.52354496717453,
        "lp_dist": -38.31823492050171,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 26.891671419143677,
        "lp_gold": -6.7627270221710205,
        "lp_dist": -33.6543984413147,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-180",
      "gold_norm": "3160",
      "dist_norm": "80",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -14.208902917802334,
        "lp_gold": -23.272248081862926,
        "lp_dist": -9.063345164060593,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.284509412944317,
        "lp_gold": -18.512411706149578,
        "lp_dist": -10.227902293205261,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-181",
      "gold_norm": "80",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.666141152381897,
        "lp_gold": -11.426292300224304,
        "lp_dist": -18.0924334526062,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.4361144304275513,
        "lp_gold": -8.623531460762024,
        "lp_dist": -12.059645891189575,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-182",
      "gold_norm": "50",
      "dist_norm": "40",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3921025022864342,
        "lp_gold": -16.256853722035885,
        "lp_dist": -15.86475121974945,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.5148345530033112,
        "lp_gold": -11.546109974384308,
        "lp_dist": -11.031275421380997,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-183",
      "gold_norm": "40",
      "dist_norm": "78",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.310973192565143,
        "lp_gold": -11.783679460175335,
        "lp_dist": -22.09465265274048,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.1819764897227287,
        "lp_gold": -8.697102136909962,
        "lp_dist": -11.87907862663269,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-184",
      "gold_norm": "78",
      "dist_norm": "273",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.903703212738037,
        "lp_gold": -21.98734474182129,
        "lp_dist": -31.891047954559326,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.26316213607788,
        "lp_gold": -13.972403526306152,
        "lp_dist": -25.235565662384033,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-185",
      "gold_norm": "273",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.715305805206299,
        "lp_gold": -18.186378479003906,
        "lp_dist": -13.471072673797607,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.78067421913147,
        "lp_gold": -15.708798170089722,
        "lp_dist": -9.928123950958252,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-186",
      "gold_norm": "2",
      "dist_norm": "195",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 12.730733886361122,
        "lp_gold": -12.38149118423462,
        "lp_dist": -25.11222507059574,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.67141306400299,
        "lp_gold": -8.928462505340576,
        "lp_dist": -21.599875569343567,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-187",
      "gold_norm": "195",
      "dist_norm": "1128",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 15.125296980142593,
        "lp_gold": -20.79087921977043,
        "lp_dist": -35.916176199913025,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.027642607688904,
        "lp_gold": -17.632879853248596,
        "lp_dist": -29.6605224609375,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-188",
      "gold_norm": "1128",
      "dist_norm": "172",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.5262241810560226,
        "lp_gold": -21.482525154948235,
        "lp_dist": -18.956300973892212,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.822476863861084,
        "lp_gold": -22.20119798183441,
        "lp_dist": -14.378721117973328,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-189",
      "gold_norm": "172",
      "dist_norm": "30",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -10.2972651720047,
        "lp_gold": -25.187148094177246,
        "lp_dist": -14.889882922172546,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.863584116101265,
        "lp_gold": -19.34168529510498,
        "lp_dist": -11.478101179003716,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-190",
      "gold_norm": "30",
      "dist_norm": "30",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.69141697883606,
        "lp_dist": -20.69141697883606,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -14.99114179611206,
        "lp_dist": -14.99114179611206,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-191",
      "gold_norm": "30",
      "dist_norm": "92",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 14.041085667908192,
        "lp_gold": -11.936107210814953,
        "lp_dist": -25.977192878723145,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 10.562521874904633,
        "lp_gold": -8.070083677768707,
        "lp_dist": -18.63260555267334,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-192",
      "gold_norm": "92",
      "dist_norm": "20",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.352982759475708,
        "lp_gold": -19.694137811660767,
        "lp_dist": -19.34115505218506,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.613435983657837,
        "lp_gold": -16.650076866149902,
        "lp_dist": -9.036640882492065,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-193",
      "gold_norm": "20",
      "dist_norm": "540",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.857290744781494,
        "lp_gold": -12.000582933425903,
        "lp_dist": -17.857873678207397,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.146737933158875,
        "lp_gold": -8.929409623146057,
        "lp_dist": -20.07614755630493,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-194",
      "gold_norm": "540",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.4153643026947975,
        "lp_gold": -17.50879267603159,
        "lp_dist": -17.093428373336792,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.476008802652359,
        "lp_gold": -18.04153409600258,
        "lp_dist": -11.56552529335022,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-195",
      "gold_norm": "10",
      "dist_norm": "10",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -13.743456363677979,
        "lp_dist": -13.743456363677979,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -11.619784355163574,
        "lp_dist": -11.619784355163574,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-196",
      "gold_norm": "10",
      "dist_norm": "38",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.615695416927338,
        "lp_gold": -13.448404610157013,
        "lp_dist": -21.06410002708435,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.2176668643951416,
        "lp_gold": -12.037200689315796,
        "lp_dist": -12.254867553710938,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-197",
      "gold_norm": "38",
      "dist_norm": "4000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 19.051328860223293,
        "lp_gold": -18.077466011047363,
        "lp_dist": -37.12879487127066,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.882668375968933,
        "lp_gold": -18.516667366027832,
        "lp_dist": -27.399335741996765,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-198",
      "gold_norm": "4000",
      "dist_norm": "594",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 10.496505833114497,
        "lp_gold": -20.328653597389348,
        "lp_dist": -30.825159430503845,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.5506546348333359,
        "lp_gold": -16.97495509684086,
        "lp_dist": -18.525609731674194,
        "n_tokens_gold": 5,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-199",
      "gold_norm": "594",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -13.668475985527039,
        "lp_gold": -24.507625102996826,
        "lp_dist": -10.839149117469788,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -12.574656009674072,
        "lp_gold": -20.77769374847412,
        "lp_dist": -8.203037738800049,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-200",
      "gold_norm": "2",
      "dist_norm": "142",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 20.30094861984253,
        "lp_gold": -13.631542205810547,
        "lp_dist": -33.932490825653076,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.00571084022522,
        "lp_gold": -11.859813690185547,
        "lp_dist": -24.865524530410767,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-201",
      "gold_norm": "142",
      "dist_norm": "9",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.474197149276733,
        "lp_gold": -17.457672357559204,
        "lp_dist": -12.98347520828247,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.4977235794067383,
        "lp_gold": -15.609914779663086,
        "lp_dist": -13.112191200256348,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-202",
      "gold_norm": "9",
      "dist_norm": "6",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.3756675720214844,
        "lp_gold": -12.738621711730957,
        "lp_dist": -11.362954139709473,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2889900207519531,
        "lp_gold": -12.575027465820312,
        "lp_dist": -11.28603744506836,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-203",
      "gold_norm": "6",
      "dist_norm": "100",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.780631840229034,
        "lp_gold": -15.693442344665527,
        "lp_dist": -22.47407418489456,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.30402946472168,
        "lp_gold": -9.650990724563599,
        "lp_dist": -16.95502018928528,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-204",
      "gold_norm": "100",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.23692995309829712,
        "lp_gold": -17.014364540576935,
        "lp_dist": -16.777434587478638,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6824193000793457,
        "lp_gold": -11.578391790390015,
        "lp_dist": -10.895972490310669,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-205",
      "gold_norm": "10",
      "dist_norm": "15",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3394050598144531,
        "lp_gold": -18.575839042663574,
        "lp_dist": -18.23643398284912,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.5095596313476562,
        "lp_gold": -18.252729892730713,
        "lp_dist": -18.76228952407837,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-206",
      "gold_norm": "15",
      "dist_norm": "22",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2520769834518433,
        "lp_gold": -16.89211142063141,
        "lp_dist": -18.144188404083252,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.4231153726577759,
        "lp_gold": -14.139848232269287,
        "lp_dist": -15.562963604927063,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-207",
      "gold_norm": "22",
      "dist_norm": "16",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.621975004673004,
        "lp_gold": -11.219844043254852,
        "lp_dist": -13.841819047927856,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.833777904510498,
        "lp_gold": -10.41726541519165,
        "lp_dist": -9.583487510681152,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-208",
      "gold_norm": "16",
      "dist_norm": "16",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -11.36221162811853,
        "lp_dist": -11.36221162811853,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -13.241074323654175,
        "lp_dist": -13.241074323654175,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-209",
      "gold_norm": "16",
      "dist_norm": "5",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -7.475744724273682,
        "lp_gold": -19.41911506652832,
        "lp_dist": -11.943370342254639,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.438729763031006,
        "lp_gold": -15.213366270065308,
        "lp_dist": -10.774636507034302,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-210",
      "gold_norm": "5",
      "dist_norm": "23",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.465651273727417,
        "lp_gold": -15.02237606048584,
        "lp_dist": -21.488027334213257,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.424657344818115,
        "lp_gold": -8.115961074829102,
        "lp_dist": -15.540618419647217,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-211",
      "gold_norm": "23",
      "dist_norm": "30",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.4010072350502014,
        "lp_gold": -15.865891933441162,
        "lp_dist": -15.46488469839096,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.2698206305503845,
        "lp_gold": -18.3642840385437,
        "lp_dist": -13.094463407993317,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-212",
      "gold_norm": "30",
      "dist_norm": "14000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 23.520719528198242,
        "lp_gold": -19.58930778503418,
        "lp_dist": -43.11002731323242,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 18.009515285491943,
        "lp_gold": -13.2671217918396,
        "lp_dist": -31.276637077331543,
        "n_tokens_gold": 3,
        "n_tokens_dist": 6
      }
    },
    {
      "ex_id": "gsm8k-test-213",
      "gold_norm": "14000",
      "dist_norm": "60",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.45991032384336,
        "lp_gold": -21.998028149828315,
        "lp_dist": -15.538117825984955,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.547004419262521,
        "lp_gold": -14.710521432454698,
        "lp_dist": -13.163517013192177,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-214",
      "gold_norm": "60",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -8.948975563049316,
        "lp_gold": -23.77088451385498,
        "lp_dist": -14.821908950805664,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.54961109161377,
        "lp_gold": -19.638930320739746,
        "lp_dist": -11.089319229125977,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-215",
      "gold_norm": "2",
      "dist_norm": "3",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.387197434902191,
        "lp_gold": -9.424871981143951,
        "lp_dist": -12.812069416046143,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5975170135498047,
        "lp_gold": -9.744040250778198,
        "lp_dist": -8.146523237228394,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-216",
      "gold_norm": "3",
      "dist_norm": "30",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.872649192810059,
        "lp_gold": -14.82950735092163,
        "lp_dist": -20.70215654373169,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.080531597137451,
        "lp_gold": -11.414220809936523,
        "lp_dist": -13.494752407073975,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-217",
      "gold_norm": "30",
      "dist_norm": "1920",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 21.697412703186274,
        "lp_gold": -13.986085917800665,
        "lp_dist": -35.68349862098694,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 12.313387870788574,
        "lp_gold": -12.878417491912842,
        "lp_dist": -25.191805362701416,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-218",
      "gold_norm": "1920",
      "dist_norm": "84",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6947197169065475,
        "lp_gold": -25.65590851008892,
        "lp_dist": -23.961188793182373,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.878614127635956,
        "lp_gold": -21.65609782934189,
        "lp_dist": -14.777483701705933,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-219",
      "gold_norm": "84",
      "dist_norm": "8",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1246260404586792,
        "lp_gold": -13.33847463130951,
        "lp_dist": -12.21384859085083,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.7234134674072266,
        "lp_gold": -12.977782487869263,
        "lp_dist": -10.254369020462036,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-220",
      "gold_norm": "8",
      "dist_norm": "12",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.304245948791504,
        "lp_gold": -15.825139999389648,
        "lp_dist": -14.520894050598145,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1817718744277954,
        "lp_gold": -12.667408466339111,
        "lp_dist": -12.849180340766907,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-221",
      "gold_norm": "12",
      "dist_norm": "260",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.42927235364914,
        "lp_gold": -9.642007768154144,
        "lp_dist": -18.071280121803284,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.9336384534835815,
        "lp_gold": -7.166749358177185,
        "lp_dist": -14.100387811660767,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-222",
      "gold_norm": "260",
      "dist_norm": "288",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.7796518057584763,
        "lp_gold": -18.89673836529255,
        "lp_dist": -22.676390171051025,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0456210374832153,
        "lp_gold": -16.525109887123108,
        "lp_dist": -18.570730924606323,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-223",
      "gold_norm": "288",
      "dist_norm": "3",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -19.030277393758297,
        "lp_gold": -26.646236896514893,
        "lp_dist": -7.615959502756596,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -15.758692368865013,
        "lp_gold": -20.088956594467163,
        "lp_dist": -4.33026422560215,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-224",
      "gold_norm": "3",
      "dist_norm": "1596",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.595462799072266,
        "lp_gold": -14.905784606933594,
        "lp_dist": -31.50124740600586,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 16.13599407672882,
        "lp_gold": -13.950559616088867,
        "lp_dist": -30.086553692817688,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-225",
      "gold_norm": "1596",
      "dist_norm": "81",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.9876238331198692,
        "lp_gold": -17.49847326427698,
        "lp_dist": -19.48609709739685,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.1633647084236145,
        "lp_gold": -24.1348779797554,
        "lp_dist": -17.971513271331787,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-226",
      "gold_norm": "81",
      "dist_norm": "56",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.605985850095749,
        "lp_gold": -15.771342545747757,
        "lp_dist": -19.377328395843506,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6331486701965332,
        "lp_gold": -16.669665813446045,
        "lp_dist": -15.036517143249512,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-227",
      "gold_norm": "56",
      "dist_norm": "1490",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.239468812942505,
        "lp_gold": -14.987546801567078,
        "lp_dist": -31.227015614509583,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.551798105239868,
        "lp_gold": -13.551477909088135,
        "lp_dist": -25.103276014328003,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-228",
      "gold_norm": "1490",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -19.20861628651619,
        "lp_gold": -30.783629894256592,
        "lp_dist": -11.575013607740402,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -13.095399379730225,
        "lp_gold": -24.58754062652588,
        "lp_dist": -11.492141246795654,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-229",
      "gold_norm": "2",
      "dist_norm": "20",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.965806007385254,
        "lp_gold": -11.508173823356628,
        "lp_dist": -17.473979830741882,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.32473722100257874,
        "lp_gold": -11.093923568725586,
        "lp_dist": -11.418660789728165,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-230",
      "gold_norm": "20",
      "dist_norm": "11",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1644073724746704,
        "lp_gold": -14.290618896484375,
        "lp_dist": -13.126211524009705,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.0560493469238281,
        "lp_gold": -12.166522026062012,
        "lp_dist": -13.22257137298584,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-231",
      "gold_norm": "11",
      "dist_norm": "120",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.219505786895752,
        "lp_gold": -14.914972305297852,
        "lp_dist": -13.6954665184021,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.7893390655517578,
        "lp_gold": -11.986905813217163,
        "lp_dist": -13.776244878768921,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-232",
      "gold_norm": "120",
      "dist_norm": "45",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.419019672088325,
        "lp_gold": -15.604393211193383,
        "lp_dist": -20.023412883281708,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.708264172077179,
        "lp_gold": -12.799233138561249,
        "lp_dist": -13.507497310638428,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-233",
      "gold_norm": "45",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5151035785675049,
        "lp_gold": -11.359116911888123,
        "lp_dist": -9.844013333320618,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.095129489898682,
        "lp_gold": -14.39400601387024,
        "lp_dist": -6.298876523971558,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-234",
      "gold_norm": "10",
      "dist_norm": "9",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.05903661251068115,
        "lp_gold": -13.167555451393127,
        "lp_dist": -13.226592063903809,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.7660583406686783,
        "lp_gold": -6.627322778105736,
        "lp_dist": -10.393381118774414,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-235",
      "gold_norm": "9",
      "dist_norm": "33",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.191148281097412,
        "lp_gold": -14.321090459823608,
        "lp_dist": -21.51223874092102,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.510578155517578,
        "lp_gold": -13.556029319763184,
        "lp_dist": -16.06660747528076,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-236",
      "gold_norm": "33",
      "dist_norm": "150",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.496721982955933,
        "lp_gold": -19.231878995895386,
        "lp_dist": -28.72860097885132,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.684651017189026,
        "lp_gold": -13.231264114379883,
        "lp_dist": -18.91591513156891,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-237",
      "gold_norm": "150",
      "dist_norm": "60",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.496449222322553,
        "lp_gold": -12.51727462792769,
        "lp_dist": -20.013723850250244,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.5342855900526047,
        "lp_gold": -11.47604425251484,
        "lp_dist": -12.010329842567444,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-238",
      "gold_norm": "60",
      "dist_norm": "4",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.38430750370025635,
        "lp_gold": -13.078525424003601,
        "lp_dist": -13.462832927703857,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5117335319519043,
        "lp_gold": -10.930900573730469,
        "lp_dist": -9.419167041778564,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-239",
      "gold_norm": "4",
      "dist_norm": "7",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.3680305480957031,
        "lp_gold": -12.839935302734375,
        "lp_dist": -13.207965850830078,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.0732803344726562,
        "lp_gold": -11.00877571105957,
        "lp_dist": -12.082056045532227,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-240",
      "gold_norm": "7",
      "dist_norm": "3140",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 22.65280568599701,
        "lp_gold": -12.292606830596924,
        "lp_dist": -34.94541251659393,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 18.48963165283203,
        "lp_gold": -9.809606552124023,
        "lp_dist": -28.299238204956055,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-241",
      "gold_norm": "3140",
      "dist_norm": "19",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -8.30290687084198,
        "lp_gold": -22.36732530593872,
        "lp_dist": -14.06441843509674,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -13.02580738067627,
        "lp_gold": -25.325818061828613,
        "lp_dist": -12.300010681152344,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-242",
      "gold_norm": "19",
      "dist_norm": "6",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.181618273258209,
        "lp_gold": -12.157159745693207,
        "lp_dist": -17.338778018951416,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4070416688919067,
        "lp_gold": -10.997576355934143,
        "lp_dist": -9.590534687042236,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-243",
      "gold_norm": "6",
      "dist_norm": "90",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.3495291471481323,
        "lp_gold": -19.138280868530273,
        "lp_dist": -22.487810015678406,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.812922954559326,
        "lp_gold": -11.337668418884277,
        "lp_dist": -17.150591373443604,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-244",
      "gold_norm": "90",
      "dist_norm": "10",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6097938957027509,
        "lp_gold": -13.304834717731865,
        "lp_dist": -11.695040822029114,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.5438682280946523,
        "lp_gold": -7.949862555367872,
        "lp_dist": -10.493730783462524,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-245",
      "gold_norm": "10",
      "dist_norm": "130000",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 26.08018460869789,
        "lp_gold": -8.254620164632797,
        "lp_dist": -34.33480477333069,
        "n_tokens_gold": 3,
        "n_tokens_dist": 7
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.28964650630951,
        "lp_gold": -7.219025731086731,
        "lp_dist": -20.50867223739624,
        "n_tokens_gold": 3,
        "n_tokens_dist": 7
      }
    },
    {
      "ex_id": "gsm8k-test-246",
      "gold_norm": "130000",
      "dist_norm": "10",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.715070237376494,
        "lp_gold": -12.472108629561262,
        "lp_dist": -17.187178866937757,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.156454911455512,
        "lp_gold": -13.647521084174514,
        "lp_dist": -6.491066172719002,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-247",
      "gold_norm": "10",
      "dist_norm": "525",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 17.677427016198635,
        "lp_gold": -15.68656424432993,
        "lp_dist": -33.363991260528564,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.708948612213135,
        "lp_gold": -11.453617930412292,
        "lp_dist": -23.162566542625427,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-248",
      "gold_norm": "525",
      "dist_norm": "180",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.852982550859451,
        "lp_gold": -11.824598759412766,
        "lp_dist": -18.677581310272217,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.6725820302963257,
        "lp_gold": -11.64935302734375,
        "lp_dist": -12.321935057640076,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-249",
      "gold_norm": "180",
      "dist_norm": "1200",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 13.544742852449417,
        "lp_gold": -8.964889764785767,
        "lp_dist": -22.509632617235184,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 15.23933944106102,
        "lp_gold": -7.498180732131004,
        "lp_dist": -22.737520173192024,
        "n_tokens_gold": 4,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-250",
      "gold_norm": "1200",
      "dist_norm": "25",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.737206868827343,
        "lp_gold": -9.13471419364214,
        "lp_dist": -18.871921062469482,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.1686492152512074,
        "lp_gold": -8.043186407536268,
        "lp_dist": -10.211835622787476,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-251",
      "gold_norm": "25",
      "dist_norm": "21",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8187389373779297,
        "lp_gold": -12.140745043754578,
        "lp_dist": -12.959483981132507,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.511936068534851,
        "lp_gold": -6.530247092247009,
        "lp_dist": -10.04218316078186,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-252",
      "gold_norm": "21",
      "dist_norm": "2304",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 16.503239154815674,
        "lp_gold": -17.229759454727173,
        "lp_dist": -33.73299860954285,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 14.275152683258057,
        "lp_gold": -13.177631378173828,
        "lp_dist": -27.452784061431885,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-253",
      "gold_norm": "2304",
      "dist_norm": "2325",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.856137990951538,
        "lp_gold": -25.891463041305542,
        "lp_dist": -30.74760103225708,
        "n_tokens_gold": 5,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9878709018230438,
        "lp_gold": -25.637850552797318,
        "lp_dist": -26.62572145462036,
        "n_tokens_gold": 5,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-254",
      "gold_norm": "2325",
      "dist_norm": "15",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.660225659608841,
        "lp_gold": -13.980357348918915,
        "lp_dist": -11.320131689310074,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.668605744838715,
        "lp_gold": -16.375044524669647,
        "lp_dist": -11.706438779830933,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-255",
      "gold_norm": "15",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.416126787662506,
        "lp_gold": -14.69802612066269,
        "lp_dist": -21.114152908325195,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.287871062755585,
        "lp_gold": -10.296239674091339,
        "lp_dist": -17.584110736846924,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    }
  ],
  "flip_rows": [
    {
      "ex_id": "gsm8k-test-1",
      "gold_norm": "80",
      "dist_norm": "12",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.218225084245205,
        "lp_gold": -16.316218174993992,
        "lp_dist": -17.534443259239197,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6342043727636337,
        "lp_gold": -18.493512138724327,
        "lp_dist": -17.859307765960693,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -2.4292978644371033,
        "lp_gold": -17.579256772994995,
        "lp_dist": -15.149958908557892,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -2.5465927198529243,
        "lp_gold": -17.71218091994524,
        "lp_dist": -15.165588200092316,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1809745579957962,
        "lp_gold": -14.873322412371635,
        "lp_dist": -15.054296970367432,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3117252141237259,
        "lp_gold": -17.119899585843086,
        "lp_dist": -16.80817437171936,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6342088878154755,
        "lp_gold": -18.49351069331169,
        "lp_dist": -17.859301805496216,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-5",
      "gold_norm": "3200",
      "dist_norm": "38",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.075981711270288,
        "lp_gold": -15.808944131014869,
        "lp_dist": -19.884925842285156,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.112908275797963,
        "lp_gold": -17.281133087351918,
        "lp_dist": -15.168224811553955,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.29585185274481773,
        "lp_gold": -18.88566479459405,
        "lp_dist": -19.181516647338867,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.4162398586049676,
        "lp_gold": -18.945031284354627,
        "lp_dist": -19.361271142959595,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -0.9830470234155655,
        "lp_gold": -17.90198041498661,
        "lp_dist": -16.918933391571045,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.479109164327383,
        "lp_gold": -18.568949338048697,
        "lp_dist": -16.089840173721313,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -2.112905891612172,
        "lp_gold": -17.28112688846886,
        "lp_dist": -15.16822099685669,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-14",
      "gold_norm": "800",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.7909989710897207,
        "lp_gold": -10.428668463602662,
        "lp_dist": -13.219667434692383,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.28849396109581,
        "lp_gold": -15.488610118627548,
        "lp_dist": -9.200116157531738,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -3.221886307001114,
        "lp_gold": -17.41419091820717,
        "lp_dist": -14.192304611206055,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -3.1399324536323547,
        "lp_gold": -16.989762604236603,
        "lp_dist": -13.849830150604248,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -5.316341996192932,
        "lp_gold": -16.00817358493805,
        "lp_dist": -10.691831588745117,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -6.094054877758026,
        "lp_gold": -17.299417197704315,
        "lp_dist": -11.205362319946289,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -6.288493096828461,
        "lp_gold": -15.488613307476044,
        "lp_dist": -9.200120210647583,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-20",
      "gold_norm": "106",
      "dist_norm": "80",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.92336449585855,
        "lp_gold": -15.76830449141562,
        "lp_dist": -20.69166898727417,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6747859213501215,
        "lp_gold": -17.988985607400537,
        "lp_dist": -17.314199686050415,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -3.9746298789978027,
        "lp_gold": -22.760347604751587,
        "lp_dist": -18.785717725753784,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -3.769364595413208,
        "lp_gold": -22.492818355560303,
        "lp_dist": -18.723453760147095,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8601390519179404,
        "lp_gold": -22.44402221823111,
        "lp_dist": -19.58388316631317,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.127611498348415,
        "lp_gold": -16.912333111278713,
        "lp_dist": -14.784721612930298,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6747779380530119,
        "lp_gold": -17.988986684009433,
        "lp_dist": -17.31420874595642,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-24",
      "gold_norm": "9",
      "dist_norm": "40",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.006132304668426514,
        "lp_gold": -15.839151382446289,
        "lp_dist": -15.845283687114716,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.110446274280548,
        "lp_gold": -12.25863265991211,
        "lp_dist": -10.148186385631561,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.04043316841125488,
        "lp_gold": -14.208327531814575,
        "lp_dist": -14.24876070022583,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.17713472247123718,
        "lp_gold": -14.100894212722778,
        "lp_dist": -14.278028935194016,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.736970603466034,
        "lp_gold": -13.546704292297363,
        "lp_dist": -11.80973368883133,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5443955063819885,
        "lp_gold": -14.465348243713379,
        "lp_dist": -12.92095273733139,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -2.110443741083145,
        "lp_gold": -12.258633613586426,
        "lp_dist": -10.14818987250328,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-27",
      "gold_norm": "160",
      "dist_norm": "6",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.597537249326706,
        "lp_gold": -12.841732293367386,
        "lp_dist": -14.439269542694092,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.338132083415985,
        "lp_gold": -17.455387771129608,
        "lp_dist": -10.117255687713623,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -2.827625960111618,
        "lp_gold": -15.895989626646042,
        "lp_dist": -13.068363666534424,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -2.9387161433696747,
        "lp_gold": -16.01883837580681,
        "lp_dist": -13.080122232437134,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -8.034337878227234,
        "lp_gold": -23.63157594203949,
        "lp_dist": -15.597238063812256,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -7.914303302764893,
        "lp_gold": -19.091363430023193,
        "lp_dist": -11.1770601272583,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -7.338137567043304,
        "lp_gold": -17.455391585826874,
        "lp_dist": -10.11725401878357,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-31",
      "gold_norm": "68",
      "dist_norm": "31",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.820281505584717,
        "lp_gold": -15.837103843688965,
        "lp_dist": -19.65738534927368,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.25214481353759766,
        "lp_gold": -12.841001033782959,
        "lp_dist": -12.588856220245361,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 4.767996072769165,
        "lp_gold": -13.063536882400513,
        "lp_dist": -17.831532955169678,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 4.778961181640625,
        "lp_gold": -12.704540252685547,
        "lp_dist": -17.483501434326172,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -0.2866086959838867,
        "lp_gold": -13.741567134857178,
        "lp_dist": -13.454958438873291,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.2309262752532959,
        "lp_gold": -13.31624436378479,
        "lp_dist": -13.085318088531494,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.2521398067474365,
        "lp_gold": -12.840993165969849,
        "lp_dist": -12.588853359222412,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-35",
      "gold_norm": "480",
      "dist_norm": "520",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.5111888945102692,
        "lp_gold": -12.033819317817688,
        "lp_dist": -13.545008212327957,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8960548639297485,
        "lp_gold": -19.025392055511475,
        "lp_dist": -16.129337191581726,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.173134446144104,
        "lp_gold": -16.326287806034088,
        "lp_dist": -16.499422252178192,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.15610426664352417,
        "lp_gold": -16.418395936489105,
        "lp_dist": -16.57450020313263,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.612240791320801,
        "lp_gold": -17.892987489700317,
        "lp_dist": -15.280746698379517,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -3.1868066787719727,
        "lp_gold": -17.928332090377808,
        "lp_dist": -14.741525411605835,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8960519433021545,
        "lp_gold": -19.0253963470459,
        "lp_dist": -16.129344403743744,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-45",
      "gold_norm": "310",
      "dist_norm": "100",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9456039071083069,
        "lp_gold": -16.172270894050598,
        "lp_dist": -17.117874801158905,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.856696009635925,
        "lp_gold": -16.36608850955963,
        "lp_dist": -10.509392499923706,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7440242022275925,
        "lp_gold": -17.919243693351746,
        "lp_dist": -16.175219491124153,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6509404331445694,
        "lp_gold": -17.9195556640625,
        "lp_dist": -16.26861523091793,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -3.5654631853103638,
        "lp_gold": -15.380214095115662,
        "lp_dist": -11.814750909805298,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -6.163789927959442,
        "lp_gold": -17.801445245742798,
        "lp_dist": -11.637655317783356,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -5.856698274612427,
        "lp_gold": -16.366087794303894,
        "lp_dist": -10.509389519691467,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-48",
      "gold_norm": "25",
      "dist_norm": "1400",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.125034153461456,
        "lp_gold": -16.447975158691406,
        "lp_dist": -23.573009312152863,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.08089584112167358,
        "lp_gold": -13.430449962615967,
        "lp_dist": -13.349554121494293,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 4.025165379047394,
        "lp_gold": -15.498599290847778,
        "lp_dist": -19.523764669895172,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 3.622505933046341,
        "lp_gold": -15.329206466674805,
        "lp_dist": -18.951712399721146,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6573466360569,
        "lp_gold": -16.551159858703613,
        "lp_dist": -14.893813222646713,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2268932834267616,
        "lp_gold": -15.390444993972778,
        "lp_dist": -14.163551710546017,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.08089244365692139,
        "lp_gold": -13.43044662475586,
        "lp_dist": -13.349554181098938,
        "n_tokens_gold": 3,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "gsm8k-test-64",
      "gold_norm": "655",
      "dist_norm": "800",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.8698419332504272,
        "lp_gold": -17.930187582969666,
        "lp_dist": -19.800029516220093,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4062272310256958,
        "lp_gold": -14.47088611125946,
        "lp_dist": -13.064658880233765,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -0.005745887756347656,
        "lp_gold": -18.56607985496521,
        "lp_dist": -18.560333967208862,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -0.33770978450775146,
        "lp_gold": -18.740556836128235,
        "lp_dist": -18.402847051620483,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1954535841941833,
        "lp_gold": -16.162434339523315,
        "lp_dist": -14.966980755329132,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.086472988128662,
        "lp_gold": -14.544449806213379,
        "lp_dist": -13.457976818084717,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4062250852584839,
        "lp_gold": -14.470888018608093,
        "lp_dist": -13.06466293334961,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-100",
      "gold_norm": "250",
      "dist_norm": "12",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.937235951423645,
        "lp_gold": -16.930358290672302,
        "lp_dist": -18.867594242095947,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8317363262176514,
        "lp_gold": -17.415368795394897,
        "lp_dist": -15.583632469177246,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7292950004339218,
        "lp_gold": -17.88250456750393,
        "lp_dist": -16.153209567070007,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.9416460394859314,
        "lp_gold": -17.420925438404083,
        "lp_dist": -15.479279398918152,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7523078918457031,
        "lp_gold": -15.994086980819702,
        "lp_dist": -14.241779088973999,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.070462703704834,
        "lp_gold": -15.625900983810425,
        "lp_dist": -14.55543828010559,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8317327499389648,
        "lp_gold": -17.415366888046265,
        "lp_dist": -15.5836341381073,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-104",
      "gold_norm": "26",
      "dist_norm": "42",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.550231754779816,
        "lp_gold": -16.079154193401337,
        "lp_dist": -20.629385948181152,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.743180751800537,
        "lp_gold": -20.936619758605957,
        "lp_dist": -17.19343900680542,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.501150131225586,
        "lp_gold": -20.479767084121704,
        "lp_dist": -18.978616952896118,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.586869239807129,
        "lp_gold": -20.261481761932373,
        "lp_dist": -18.674612522125244,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.488091826438904,
        "lp_gold": -20.40666627883911,
        "lp_dist": -17.918574452400208,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.9733150005340576,
        "lp_gold": -20.461706161499023,
        "lp_dist": -17.488391160964966,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -3.7431836128234863,
        "lp_gold": -20.936622619628906,
        "lp_dist": -17.19343900680542,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-105",
      "gold_norm": "42",
      "dist_norm": "5",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2448320388793945,
        "lp_gold": -17.29369354248047,
        "lp_dist": -19.538525581359863,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.48531031608581543,
        "lp_gold": -14.891574144363403,
        "lp_dist": -14.406263828277588,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6142416000366211,
        "lp_gold": -20.099190711975098,
        "lp_dist": -19.484949111938477,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -0.2368631362915039,
        "lp_gold": -19.605250358581543,
        "lp_dist": -19.36838722229004,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.025758981704712,
        "lp_gold": -16.692798852920532,
        "lp_dist": -15.66703987121582,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.37449169158935547,
        "lp_gold": -14.766992568969727,
        "lp_dist": -14.392500877380371,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.48531413078308105,
        "lp_gold": -14.891570806503296,
        "lp_dist": -14.406256675720215,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-107",
      "gold_norm": "14400",
      "dist_norm": "400",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8998411595821381,
        "lp_gold": -19.079706698656082,
        "lp_dist": -19.97954785823822,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.72913409024477,
        "lp_gold": -22.703229255974293,
        "lp_dist": -20.974095165729523,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.19682685285806656,
        "lp_gold": -22.464857898652554,
        "lp_dist": -22.66168475151062,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.14343415200710297,
        "lp_gold": -22.713250055909157,
        "lp_dist": -22.85668420791626,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -3.5759153068065643,
        "lp_gold": -27.662475764751434,
        "lp_dist": -24.08656045794487,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.206382006406784,
        "lp_gold": -25.311469167470932,
        "lp_dist": -23.105087161064148,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7291278392076492,
        "lp_gold": -22.70323248207569,
        "lp_dist": -20.974104642868042,
        "n_tokens_gold": 6,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "gsm8k-test-110",
      "gold_norm": "83",
      "dist_norm": "10",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9284783601760864,
        "lp_gold": -14.152065396308899,
        "lp_dist": -15.080543756484985,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.6964447498321533,
        "lp_gold": -11.795601606369019,
        "lp_dist": -8.099156856536865,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5404622554779053,
        "lp_gold": -14.069255948066711,
        "lp_dist": -12.528793692588806,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4889593124389648,
        "lp_gold": -14.175909280776978,
        "lp_dist": -12.686949968338013,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8712440729141235,
        "lp_gold": -14.255087852478027,
        "lp_dist": -11.383843779563904,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -3.085860013961792,
        "lp_gold": -12.02087950706482,
        "lp_dist": -8.935019493103027,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -3.696447730064392,
        "lp_gold": -11.795600891113281,
        "lp_dist": -8.09915316104889,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-120",
      "gold_norm": "335",
      "dist_norm": "60",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.718258023262024,
        "lp_gold": -16.84885323047638,
        "lp_dist": -18.567111253738403,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.495100736618042,
        "lp_gold": -16.78837823867798,
        "lp_dist": -13.293277502059937,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -6.745102643966675,
        "lp_gold": -21.85818600654602,
        "lp_dist": -15.113083362579346,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -6.640980243682861,
        "lp_gold": -21.43053102493286,
        "lp_dist": -14.78955078125,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -4.621345937252045,
        "lp_gold": -18.077104091644287,
        "lp_dist": -13.455758154392242,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -4.602914571762085,
        "lp_gold": -19.734163284301758,
        "lp_dist": -15.131248712539673,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -3.4950921535491943,
        "lp_gold": -16.78837251663208,
        "lp_dist": -13.293280363082886,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-134",
      "gold_norm": "18",
      "dist_norm": "4",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.9239641074091196,
        "lp_gold": -15.171829616650939,
        "lp_dist": -19.09579372406006,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7061721086502075,
        "lp_gold": -9.731460690498352,
        "lp_dist": -9.025288581848145,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.6682674884796143,
        "lp_gold": -13.212559461593628,
        "lp_dist": -13.880826950073242,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.691381573677063,
        "lp_gold": -13.084003806114197,
        "lp_dist": -13.77538537979126,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -3.0653414726257324,
        "lp_gold": -11.70475959777832,
        "lp_dist": -8.639418125152588,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6070647239685059,
        "lp_gold": -11.957983255386353,
        "lp_dist": -10.350918531417847,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7061715722084045,
        "lp_gold": -9.731466829776764,
        "lp_dist": -9.02529525756836,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-138",
      "gold_norm": "21",
      "dist_norm": "25",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.6713391542434692,
        "lp_gold": -17.788984179496765,
        "lp_dist": -20.460323333740234,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8663175106048584,
        "lp_gold": -14.334570407867432,
        "lp_dist": -13.468252897262573,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.3505210876464844,
        "lp_gold": -21.58456540107727,
        "lp_dist": -21.935086488723755,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.2903881072998047,
        "lp_gold": -22.286190509796143,
        "lp_dist": -22.576578617095947,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -0.9333076477050781,
        "lp_gold": -15.113465785980225,
        "lp_dist": -14.180158138275146,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7268631458282471,
        "lp_gold": -13.463084697723389,
        "lp_dist": -12.736221551895142,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8663196563720703,
        "lp_gold": -14.334563970565796,
        "lp_dist": -13.468244314193726,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-145",
      "gold_norm": "2",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2492438331246376,
        "lp_gold": -13.602060556411743,
        "lp_dist": -14.85130438953638,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.6407327204942703,
        "lp_gold": -12.194403648376465,
        "lp_dist": -9.553670927882195,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.6734669059514999,
        "lp_gold": -13.396984100341797,
        "lp_dist": -14.070451006293297,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8539667278528214,
        "lp_gold": -13.266191005706787,
        "lp_dist": -14.120157733559608,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.3427896350622177,
        "lp_gold": -10.751940488815308,
        "lp_dist": -9.40915085375309,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.827908754348755,
        "lp_gold": -14.305924892425537,
        "lp_dist": -11.478016138076782,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -2.640733018517494,
        "lp_gold": -12.194400548934937,
        "lp_dist": -9.553667530417442,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-151",
      "gold_norm": "803",
      "dist_norm": "16",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.22600507736206055,
        "lp_gold": -20.624857425689697,
        "lp_dist": -20.850862503051758,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.131483793258667,
        "lp_gold": -19.228359699249268,
        "lp_dist": -14.0968759059906,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -3.2413750886917114,
        "lp_gold": -19.84936547279358,
        "lp_dist": -16.607990384101868,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -3.131054997444153,
        "lp_gold": -19.767980694770813,
        "lp_dist": -16.63692569732666,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -4.6819539070129395,
        "lp_gold": -18.495857000350952,
        "lp_dist": -13.813903093338013,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -4.957081317901611,
        "lp_gold": -20.852898836135864,
        "lp_dist": -15.895817518234253,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -5.131482243537903,
        "lp_gold": -19.228361129760742,
        "lp_dist": -14.09687888622284,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-153",
      "gold_norm": "280",
      "dist_norm": "13",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.694299184717238,
        "lp_gold": -15.244264638982713,
        "lp_dist": -18.93856382369995,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8231047093868256,
        "lp_gold": -12.64960965514183,
        "lp_dist": -11.826504945755005,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8308207541704178,
        "lp_gold": -15.758303448557854,
        "lp_dist": -16.58912420272827,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.7265715599060059,
        "lp_gold": -16.042329788208008,
        "lp_dist": -16.768901348114014,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1412931680679321,
        "lp_gold": -14.319903492927551,
        "lp_dist": -13.17861032485962,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4246297478675842,
        "lp_gold": -15.16783195734024,
        "lp_dist": -13.743202209472656,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8231084495782852,
        "lp_gold": -12.649619355797768,
        "lp_dist": -11.826510906219482,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-154",
      "gold_norm": "13",
      "dist_norm": "20",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0602927803993225,
        "lp_gold": -15.215918719768524,
        "lp_dist": -17.276211500167847,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.633309006690979,
        "lp_gold": -12.038846015930176,
        "lp_dist": -11.405537009239197,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6033051013946533,
        "lp_gold": -17.432909965515137,
        "lp_dist": -16.829604864120483,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6372992992401123,
        "lp_gold": -17.39775514602661,
        "lp_dist": -16.7604558467865,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.48968505859375,
        "lp_gold": -13.174037456512451,
        "lp_dist": -11.684352397918701,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2143868207931519,
        "lp_gold": -12.643470287322998,
        "lp_dist": -11.429083466529846,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6333088874816895,
        "lp_gold": -12.038848400115967,
        "lp_dist": -11.405539512634277,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-175",
      "gold_norm": "350",
      "dist_norm": "50",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9250896275043488,
        "lp_gold": -22.99281856417656,
        "lp_dist": -23.917908191680908,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.236621737480164,
        "lp_gold": -17.58333122730255,
        "lp_dist": -10.346709489822388,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -2.966416835784912,
        "lp_gold": -20.354339838027954,
        "lp_dist": -17.387923002243042,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -3.5315762758255005,
        "lp_gold": -21.129161953926086,
        "lp_dist": -17.597585678100586,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -7.064225733280182,
        "lp_gold": -18.191003382205963,
        "lp_dist": -11.126777648925781,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -7.836197316646576,
        "lp_gold": -21.282770097255707,
        "lp_dist": -13.44657278060913,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -7.236626446247101,
        "lp_gold": -17.583332121372223,
        "lp_dist": -10.346705675125122,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-207",
      "gold_norm": "22",
      "dist_norm": "16",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.621975004673004,
        "lp_gold": -11.219844043254852,
        "lp_dist": -13.841819047927856,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.833777904510498,
        "lp_gold": -10.41726541519165,
        "lp_dist": -9.583487510681152,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4578208923339844,
        "lp_gold": -15.501156091690063,
        "lp_dist": -14.043335199356079,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.725459337234497,
        "lp_gold": -15.87945008277893,
        "lp_dist": -14.153990745544434,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1624937057495117,
        "lp_gold": -15.793409585952759,
        "lp_dist": -14.630915880203247,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6316690444946289,
        "lp_gold": -10.619104146957397,
        "lp_dist": -9.987435102462769,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8337790966033936,
        "lp_gold": -10.417269945144653,
        "lp_dist": -9.58349084854126,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-215",
      "gold_norm": "2",
      "dist_norm": "3",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.387197434902191,
        "lp_gold": -9.424871981143951,
        "lp_dist": -12.812069416046143,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5975170135498047,
        "lp_gold": -9.744040250778198,
        "lp_dist": -8.146523237228394,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -3.2604313492774963,
        "lp_gold": -12.714868068695068,
        "lp_dist": -9.454436719417572,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -3.084445893764496,
        "lp_gold": -12.945753574371338,
        "lp_dist": -9.861307680606842,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5732927322387695,
        "lp_gold": -12.206480503082275,
        "lp_dist": -10.633187770843506,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1921443939208984,
        "lp_gold": -10.054778575897217,
        "lp_dist": -8.862634181976318,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5975122451782227,
        "lp_gold": -9.744039058685303,
        "lp_dist": -8.14652681350708,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-225",
      "gold_norm": "1596",
      "dist_norm": "81",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.9876238331198692,
        "lp_gold": -17.49847326427698,
        "lp_dist": -19.48609709739685,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.1633647084236145,
        "lp_gold": -24.1348779797554,
        "lp_dist": -17.971513271331787,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7931787371635437,
        "lp_gold": -18.82551997900009,
        "lp_dist": -17.032341241836548,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -2.224741965532303,
        "lp_gold": -19.164868861436844,
        "lp_dist": -16.94012689590454,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -10.08569085597992,
        "lp_gold": -30.44549548625946,
        "lp_dist": -20.35980463027954,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -9.3379967212677,
        "lp_gold": -30.265968084335327,
        "lp_dist": -20.927971363067627,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -6.163362383842468,
        "lp_gold": -24.134878516197205,
        "lp_dist": -17.971516132354736,
        "n_tokens_gold": 5,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-226",
      "gold_norm": "81",
      "dist_norm": "56",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.605985850095749,
        "lp_gold": -15.771342545747757,
        "lp_dist": -19.377328395843506,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6331486701965332,
        "lp_gold": -16.669665813446045,
        "lp_dist": -15.036517143249512,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.214464545249939,
        "lp_gold": -16.49272656440735,
        "lp_dist": -15.27826201915741,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -0.9139895439147949,
        "lp_gold": -16.460952043533325,
        "lp_dist": -15.54696249961853,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7367334365844727,
        "lp_gold": -19.27399492263794,
        "lp_dist": -18.537261486053467,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8308906555175781,
        "lp_gold": -16.610596179962158,
        "lp_dist": -14.77970552444458,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6331486701965332,
        "lp_gold": -16.66966152191162,
        "lp_dist": -15.036512851715088,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "gsm8k-test-238",
      "gold_norm": "60",
      "dist_norm": "4",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.38430750370025635,
        "lp_gold": -13.078525424003601,
        "lp_dist": -13.462832927703857,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5117335319519043,
        "lp_gold": -10.930900573730469,
        "lp_dist": -9.419167041778564,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7834737300872803,
        "lp_gold": -15.81191873550415,
        "lp_dist": -14.02844500541687,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.6578322649002075,
        "lp_gold": -15.757366299629211,
        "lp_dist": -14.099534034729004,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -5.469817161560059,
        "lp_gold": -15.78016185760498,
        "lp_dist": -10.310344696044922,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.3997215032577515,
        "lp_gold": -11.266274809837341,
        "lp_dist": -9.86655330657959,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5117324590682983,
        "lp_gold": -10.930898785591125,
        "lp_dist": -9.419166326522827,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-242",
      "gold_norm": "19",
      "dist_norm": "6",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.181618273258209,
        "lp_gold": -12.157159745693207,
        "lp_dist": -17.338778018951416,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4070416688919067,
        "lp_gold": -10.997576355934143,
        "lp_dist": -9.590534687042236,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 1.4950295239686966,
        "lp_gold": -14.810317918658257,
        "lp_dist": -16.305347442626953,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 1.47235550545156,
        "lp_gold": -14.627300599589944,
        "lp_dist": -16.099656105041504,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.7640156745910645,
        "lp_gold": -15.07509469985962,
        "lp_dist": -12.311079025268555,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.5993056297302246,
        "lp_gold": -12.642723798751831,
        "lp_dist": -10.043418169021606,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4070385694503784,
        "lp_gold": -10.997580409049988,
        "lp_dist": -9.59054183959961,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "gsm8k-test-246",
      "gold_norm": "130000",
      "dist_norm": "10",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.715070237376494,
        "lp_gold": -12.472108629561262,
        "lp_dist": -17.187178866937757,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -7.156454911455512,
        "lp_gold": -13.647521084174514,
        "lp_dist": -6.491066172719002,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -5.767381154000759,
        "lp_gold": -21.409174405038357,
        "lp_dist": -15.641793251037598,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -5.478326896904036,
        "lp_gold": -22.25712094712071,
        "lp_dist": -16.778794050216675,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -7.735332287847996,
        "lp_gold": -17.688330195844173,
        "lp_dist": -9.952997907996178,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -6.886516407132149,
        "lp_gold": -16.19584783911705,
        "lp_dist": -9.309331431984901,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -7.15645333006978,
        "lp_gold": -13.64751996472478,
        "lp_dist": -6.491066634654999,
        "n_tokens_gold": 7,
        "n_tokens_dist": 3
      }
    }
  ]
}