{
  "meta": {
    "model": "meta-llama/Llama-2-7b-chat-hf",
    "device": "cuda",
    "dtype": "fp32",
    "layer": 10,
    "layers_path": "model.layers",
    "seed": 123,
    "task": "humaneval",
    "eval_mode": "pair_logprob",
    "eval_meta": {
      "hf_id": "openai_humaneval",
      "split": "test",
      "n_total": 164
    },
    "n_eval_loaded": 164,
    "n_scanned": 164,
    "base_acc_scan": 0.6585365853658537,
    "ablt_acc_scan": 0.6402439024390244,
    "flips_total": 8,
    "flips_used": 8,
    "patch_steps": [
      0,
      1,
      2,
      3
    ],
    "patch_n_steps": 4,
    "Qs_path": "Q_shared_layer10.npy",
    "Qs_shape": [
      4096,
      97
    ],
    "gold_text_prefix": " ",
    "dist_text_prefix": " ",
    "gold_max_tokens": 128,
    "distractor_mode": "next_gold",
    "answer_prefix_effective": "\nFinal answer:",
    "max_new_tokens_effective": 64,
    "run_coeff_controls": false,
    "use_benchmark_loader": false,
    "hf_id": "openai_humaneval",
    "hf_split": "test"
  },
  "summary_on_flips": {
    "patched_self": {
      "n": 8,
      "rescued": 6,
      "rescued_pct": 75.0,
      "mean_delta_margin_vs_ablated": 2.1062567234039307,
      "median_delta_margin_vs_ablated": 1.8823415040969849
    },
    "control_time_shuffled": {
      "n": 8,
      "rescued": 5,
      "rescued_pct": 62.5,
      "mean_delta_margin_vs_ablated": 1.611517310142517,
      "median_delta_margin_vs_ablated": 1.736205816268921
    },
    "control_shared_randvec": {
      "n": 8,
      "rescued": 0,
      "rescued_pct": 0.0,
      "mean_delta_margin_vs_ablated": -0.5279055237770081,
      "median_delta_margin_vs_ablated": -0.47387340664863586
    },
    "control_rand_subspace": {
      "n": 8,
      "rescued": 1,
      "rescued_pct": 12.5,
      "mean_delta_margin_vs_ablated": -0.11733363568782806,
      "median_delta_margin_vs_ablated": -0.24908697605133057
    },
    "control_patch_nonshared": {
      "n": 8,
      "rescued": 1,
      "rescued_pct": 12.5,
      "mean_delta_margin_vs_ablated": 0.048883724957704544,
      "median_delta_margin_vs_ablated": 0.0663357526063919
    }
  },
  "scan_rows": [
    {
      "ex_id": "openai_humaneval-test-18",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.234334945678711,
        "lp_gold": -19.061742782592773,
        "lp_dist": -21.296077728271484,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2801809310913086,
        "lp_gold": -21.540138244628906,
        "lp_dist": -23.820319175720215,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-31",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -24.834365844726562,
        "lp_dist": -24.834365844726562,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -26.02178192138672,
        "lp_dist": -26.02178192138672,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-158",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.071807861328125,
        "lp_dist": -20.071807861328125,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.830044746398926,
        "lp_dist": -20.830044746398926,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-43",
      "gold_norm": "0",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9933905601501465,
        "lp_gold": -19.265485286712646,
        "lp_dist": -20.258875846862793,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1291828155517578,
        "lp_gold": -20.913329124450684,
        "lp_dist": -21.04251194000244,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-39",
      "gold_norm": "-1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.44877076148986816,
        "lp_gold": -19.808196783065796,
        "lp_dist": -20.256967544555664,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.15900945663452148,
        "lp_gold": -22.082778453826904,
        "lp_dist": -21.923768997192383,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-15",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.9036493301391602,
        "lp_gold": -21.704959869384766,
        "lp_dist": -23.608609199523926,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.597311019897461,
        "lp_gold": -22.718390464782715,
        "lp_dist": -24.315701484680176,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-151",
      "gold_norm": "0",
      "dist_norm": "if not s:\n        return []\n\n    s_list = []\n\n    for letter in s:\n        if letter == ',':\n            s_list.append(' ')\n        else:\n            s_list.append(letter)\n\n    s_list = \"\".join(s_list)\n    return s_list.split()",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 121.7853691404589,
        "lp_gold": -18.80201482772827,
        "lp_dist": -140.58738396818717,
        "n_tokens_gold": 2,
        "n_tokens_dist": 79
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 121.62771476514445,
        "lp_gold": -21.097187042236328,
        "lp_dist": -142.72490180738077,
        "n_tokens_gold": 2,
        "n_tokens_dist": 79
      }
    },
    {
      "ex_id": "openai_humaneval-test-101",
      "gold_norm": "if not s:\n        return []\n\n    s_list = []\n\n    for letter in s:\n        if letter == ',':\n            s_list.append(' ')\n        else:\n            s_list.append(letter)\n\n    s_list = \"\".join(s_list)\n    return s_list.split()",
      "dist_norm": "return sorted(list(set(l)))",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -31.890593631385258,
        "lp_gold": -82.64533315385356,
        "lp_dist": -50.7547395224683,
        "n_tokens_gold": 79,
        "n_tokens_dist": 10
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -36.78753035544207,
        "lp_gold": -92.11622255231043,
        "lp_dist": -55.32869219686836,
        "n_tokens_gold": 79,
        "n_tokens_dist": 10
      }
    },
    {
      "ex_id": "openai_humaneval-test-34",
      "gold_norm": "return sorted(list(set(l)))",
      "dist_norm": "for e in l:\n        if e >= t:\n            return False\n    return True",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 49.12246614873311,
        "lp_gold": -11.745759465690753,
        "lp_dist": -60.868225614423864,
        "n_tokens_gold": 10,
        "n_tokens_dist": 21
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 45.176009765720664,
        "lp_gold": -15.221526360244752,
        "lp_dist": -60.397536125965416,
        "n_tokens_gold": 10,
        "n_tokens_dist": 21
      }
    },
    {
      "ex_id": "openai_humaneval-test-52",
      "gold_norm": "for e in l:\n        if e >= t:\n            return False\n    return True",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -18.150727199346875,
        "lp_gold": -34.72359650018916,
        "lp_dist": -16.572869300842285,
        "n_tokens_gold": 21,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -18.006402769460692,
        "lp_gold": -34.587387361898436,
        "lp_dist": -16.580984592437744,
        "n_tokens_gold": 21,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-113",
      "gold_norm": "1",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.4202613830566406,
        "lp_gold": -14.915795803070068,
        "lp_dist": -17.33605718612671,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.1670427322387695,
        "lp_gold": -17.457444190979004,
        "lp_dist": -19.624486923217773,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-119",
      "gold_norm": "2",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -19.728981971740723,
        "lp_dist": -19.728981971740723,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.550307273864746,
        "lp_dist": -20.550307273864746,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-83",
      "gold_norm": "2",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.055248260498047,
        "lp_gold": -21.746171951293945,
        "lp_dist": -19.6909236907959,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8670825958251953,
        "lp_gold": -22.857107162475586,
        "lp_dist": -20.99002456665039,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-116",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.5492916107177734,
        "lp_gold": -13.82323694229126,
        "lp_dist": -16.372528553009033,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.914440155029297,
        "lp_gold": -15.276045799255371,
        "lp_dist": -19.190485954284668,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-56",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.711745262145996,
        "lp_dist": -20.711745262145996,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.994271278381348,
        "lp_dist": -22.994271278381348,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-131",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.21437931060791,
        "lp_dist": -20.21437931060791,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.886200428009033,
        "lp_dist": -21.886200428009033,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-1",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -23.005115509033203,
        "lp_dist": -23.005115509033203,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.74118995666504,
        "lp_dist": -22.74118995666504,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-159",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -4.005736351013184,
        "lp_gold": -21.52419900894165,
        "lp_dist": -17.518462657928467,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.665071487426758,
        "lp_gold": -20.96841859817505,
        "lp_dist": -17.30334711074829,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-123",
      "gold_norm": "1",
      "dist_norm": "return len(string)",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 22.28865046799183,
        "lp_gold": -18.46173858642578,
        "lp_dist": -40.75038905441761,
        "n_tokens_gold": 2,
        "n_tokens_dist": 6
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 21.772113933227956,
        "lp_gold": -18.439892768859863,
        "lp_dist": -40.21200670208782,
        "n_tokens_gold": 2,
        "n_tokens_dist": 6
      }
    },
    {
      "ex_id": "openai_humaneval-test-23",
      "gold_norm": "return len(string)",
      "dist_norm": "29",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 12.790524657903006,
        "lp_gold": -15.688145462336252,
        "lp_dist": -28.478670120239258,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 13.356130968688376,
        "lp_gold": -16.101557362915628,
        "lp_dist": -29.457688331604004,
        "n_tokens_gold": 6,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-124",
      "gold_norm": "29",
      "dist_norm": "return encode_cyclic(encode_cyclic(s))",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 50.18577869143337,
        "lp_gold": -21.59093189239502,
        "lp_dist": -71.77671058382839,
        "n_tokens_gold": 3,
        "n_tokens_dist": 14
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 56.64552312903106,
        "lp_gold": -22.290241479873657,
        "lp_dist": -78.93576460890472,
        "n_tokens_gold": 3,
        "n_tokens_dist": 14
      }
    },
    {
      "ex_id": "openai_humaneval-test-38",
      "gold_norm": "return encode_cyclic(encode_cyclic(s))",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.7194257393894077,
        "lp_gold": -29.536041797739017,
        "lp_dist": -26.81661605834961,
        "n_tokens_gold": 14,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.8930526294279844,
        "lp_gold": -29.922911218600348,
        "lp_dist": -26.029858589172363,
        "n_tokens_gold": 14,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-84",
      "gold_norm": "2",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -19.775970458984375,
        "lp_dist": -19.775970458984375,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.496947288513184,
        "lp_dist": -20.496947288513184,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-41",
      "gold_norm": "2",
      "dist_norm": "122",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.278376579284668,
        "lp_gold": -22.390517234802246,
        "lp_dist": -30.668893814086914,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 9.729191303253174,
        "lp_gold": -21.189680099487305,
        "lp_dist": -30.91887140274048,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-134",
      "gold_norm": "122",
      "dist_norm": "1.0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8403654620051384,
        "lp_gold": -27.5423321723938,
        "lp_dist": -28.382697634398937,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.457980029284954,
        "lp_gold": -29.874002933502197,
        "lp_dist": -27.416022904217243,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-2",
      "gold_norm": "1.0",
      "dist_norm": "+2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.533992663025856,
        "lp_gold": -19.524202451109886,
        "lp_dist": -25.058195114135742,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.357768684625626,
        "lp_gold": -17.571494430303574,
        "lp_dist": -25.9292631149292,
        "n_tokens_gold": 4,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-80",
      "gold_norm": "+2",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -5.821111679077148,
        "lp_gold": -24.557270050048828,
        "lp_dist": -18.73615837097168,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.8031325340271,
        "lp_gold": -26.246718406677246,
        "lp_dist": -20.443585872650146,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-74",
      "gold_norm": "2",
      "dist_norm": "5",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2735700607299805,
        "lp_gold": -17.82392930984497,
        "lp_dist": -19.09749937057495,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.1347217559814453,
        "lp_gold": -18.707550048828125,
        "lp_dist": -19.84227180480957,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-162",
      "gold_norm": "5",
      "dist_norm": "8",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.4634990692138672,
        "lp_gold": -19.24656867980957,
        "lp_dist": -19.710067749023438,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.9875469207763672,
        "lp_gold": -21.240734100341797,
        "lp_dist": -22.228281021118164,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-138",
      "gold_norm": "8",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1303815841674805,
        "lp_gold": -21.265185356140137,
        "lp_dist": -20.134803771972656,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.3591747283935547,
        "lp_gold": -23.056735038757324,
        "lp_dist": -21.69756031036377,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-87",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.202168941497803,
        "lp_dist": -21.202168941497803,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.12106418609619,
        "lp_dist": -22.12106418609619,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-145",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.991208076477051,
        "lp_gold": -22.26430034637451,
        "lp_dist": -19.27309226989746,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.47486686706543,
        "lp_gold": -24.317991256713867,
        "lp_dist": -19.843124389648438,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-54",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8734645843505859,
        "lp_gold": -22.476731300354004,
        "lp_dist": -23.35019588470459,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.6492033004760742,
        "lp_gold": -24.899346351623535,
        "lp_dist": -26.54854965209961,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-109",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.480426788330078,
        "lp_gold": -19.18559741973877,
        "lp_dist": -16.70517063140869,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.790494918823242,
        "lp_gold": -21.397884845733643,
        "lp_dist": -16.6073899269104,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-102",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -19.423691272735596,
        "lp_dist": -19.423691272735596,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.474623680114746,
        "lp_dist": -20.474623680114746,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-62",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -17.160552978515625,
        "lp_dist": -17.160552978515625,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -15.657040119171143,
        "lp_dist": -15.657040119171143,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-129",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -19.168509483337402,
        "lp_dist": -19.168509483337402,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -19.092866897583008,
        "lp_dist": -19.092866897583008,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-110",
      "gold_norm": "1",
      "dist_norm": "mean = sum(numbers) / len(numbers)\n    return sum(abs(x - mean) for x in numbers) / len(numbers)",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 56.64819990084652,
        "lp_gold": -12.325343608856201,
        "lp_dist": -68.97354350970272,
        "n_tokens_gold": 2,
        "n_tokens_dist": 33
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 60.25500547605043,
        "lp_gold": -11.27043867111206,
        "lp_dist": -71.52544414716249,
        "n_tokens_gold": 2,
        "n_tokens_dist": 33
      }
    },
    {
      "ex_id": "openai_humaneval-test-4",
      "gold_norm": "mean = sum(numbers) / len(numbers)\n    return sum(abs(x - mean) for x in numbers) / len(numbers)",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.45850569046490364,
        "lp_gold": -22.989404618372987,
        "lp_dist": -23.44791030883789,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.605282360978833,
        "lp_gold": -30.1749826020677,
        "lp_dist": -21.569700241088867,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-8",
      "gold_norm": "1",
      "dist_norm": "10",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.0166356563568115,
        "lp_gold": -18.3596510887146,
        "lp_dist": -21.37628674507141,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.0787312984466553,
        "lp_gold": -18.83952569961548,
        "lp_dist": -21.918256998062134,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-97",
      "gold_norm": "10",
      "dist_norm": "temp_a, temp_b = a, b\n    if isinstance(temp_a, str): temp_a = temp_a.replace(',','.')\n    if isinstance(temp_b, str): temp_b = temp_b.replace(',','.')\n    if float(temp_a) == float(temp_b): return None\n    return a if float(temp_a) > float(temp_b) else b",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 94.84300347900216,
        "lp_gold": -21.25941014289856,
        "lp_dist": -116.10241362190072,
        "n_tokens_gold": 3,
        "n_tokens_dist": 100
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 98.10631249527887,
        "lp_gold": -23.950812816619873,
        "lp_dist": -122.05712531189874,
        "n_tokens_gold": 3,
        "n_tokens_dist": 100
      }
    },
    {
      "ex_id": "openai_humaneval-test-137",
      "gold_norm": "temp_a, temp_b = a, b\n    if isinstance(temp_a, str): temp_a = temp_a.replace(',','.')\n    if isinstance(temp_b, str): temp_b = temp_b.replace(',','.')\n    if float(temp_a) == float(temp_b): return None\n    return a if float(temp_a) > float(temp_b) else b",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -67.42598734535835,
        "lp_gold": -83.42734728493355,
        "lp_dist": -16.001359939575195,
        "n_tokens_gold": 100,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -67.17782856585814,
        "lp_gold": -82.82997252108885,
        "lp_dist": -15.652143955230713,
        "n_tokens_gold": 100,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-96",
      "gold_norm": "0",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.6102466583251953,
        "lp_gold": -18.234556198120117,
        "lp_dist": -21.844802856445312,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.688672065734863,
        "lp_gold": -19.75603199005127,
        "lp_dist": -24.444704055786133,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-65",
      "gold_norm": "-1",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.8471782207489014,
        "lp_gold": -22.299251317977905,
        "lp_dist": -21.452073097229004,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.24910449981689453,
        "lp_gold": -23.086091995239258,
        "lp_dist": -22.836987495422363,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-114",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8158721923828125,
        "lp_gold": -22.96446418762207,
        "lp_dist": -21.148591995239258,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.1736278533935547,
        "lp_gold": -25.378070831298828,
        "lp_dist": -22.204442977905273,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-117",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -15.196922779083252,
        "lp_dist": -15.196922779083252,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -15.9734787940979,
        "lp_dist": -15.9734787940979,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-155",
      "gold_norm": "1",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.606046199798584,
        "lp_gold": -18.941298484802246,
        "lp_dist": -21.54734468460083,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.5056796073913574,
        "lp_gold": -20.002915382385254,
        "lp_dist": -23.50859498977661,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-37",
      "gold_norm": "-1",
      "dist_norm": "return sum([math.ceil(sum(arr)/capacity) for arr in grid])",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 86.6937904068327,
        "lp_gold": -20.384744882583618,
        "lp_dist": -107.07853528941632,
        "n_tokens_gold": 3,
        "n_tokens_dist": 20
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 87.06271385207947,
        "lp_gold": -21.526390075683594,
        "lp_dist": -108.58910392776306,
        "n_tokens_gold": 3,
        "n_tokens_dist": 20
      }
    },
    {
      "ex_id": "openai_humaneval-test-115",
      "gold_norm": "return sum([math.ceil(sum(arr)/capacity) for arr in grid])",
      "dist_norm": "return ' '.join([''.join(sorted(list(i))) for i in s.split(' ')])",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 24.88441585241617,
        "lp_gold": -46.10199257756449,
        "lp_dist": -70.98640842998066,
        "n_tokens_gold": 20,
        "n_tokens_dist": 24
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 29.905949345506542,
        "lp_gold": -44.645553992972054,
        "lp_dist": -74.5515033384786,
        "n_tokens_gold": 20,
        "n_tokens_dist": 24
      }
    },
    {
      "ex_id": "openai_humaneval-test-86",
      "gold_norm": "return ' '.join([''.join(sorted(list(i))) for i in s.split(' ')])",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -14.466940372163663,
        "lp_gold": -36.1265668560809,
        "lp_dist": -21.659626483917236,
        "n_tokens_gold": 24,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -15.646357826058647,
        "lp_gold": -39.13540583020904,
        "lp_dist": -23.48904800415039,
        "n_tokens_gold": 24,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-85",
      "gold_norm": "0",
      "dist_norm": "return x + y",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.443425707519054,
        "lp_gold": -16.92650079727173,
        "lp_dist": -23.369926504790783,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.885157495737076,
        "lp_gold": -17.333487033843994,
        "lp_dist": -23.21864452958107,
        "n_tokens_gold": 2,
        "n_tokens_dist": 5
      }
    },
    {
      "ex_id": "openai_humaneval-test-53",
      "gold_norm": "return x + y",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 9.916433400284404,
        "lp_gold": -10.99827282987917,
        "lp_dist": -20.914706230163574,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 9.794849171276837,
        "lp_gold": -10.414248690966815,
        "lp_dist": -20.209097862243652,
        "n_tokens_gold": 5,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-130",
      "gold_norm": "2",
      "dist_norm": "+1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.369633674621582,
        "lp_gold": -20.579893589019775,
        "lp_dist": -22.949527263641357,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.291049003601074,
        "lp_gold": -21.44382381439209,
        "lp_dist": -24.734872817993164,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-139",
      "gold_norm": "+1",
      "dist_norm": "0.0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2652113437652588,
        "lp_gold": -22.48181676864624,
        "lp_dist": -23.7470281124115,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1343896389007568,
        "lp_gold": -23.847239017486572,
        "lp_dist": -22.712849378585815,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-81",
      "gold_norm": "0.0",
      "dist_norm": "9",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.3655529320240021,
        "lp_gold": -22.439337760210037,
        "lp_dist": -22.073784828186035,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.13648897409439087,
        "lp_gold": -23.279959738254547,
        "lp_dist": -23.143470764160156,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-19",
      "gold_norm": "9",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.07897377014160156,
        "lp_gold": -22.66366195678711,
        "lp_dist": -22.74263572692871,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4677810668945312,
        "lp_gold": -23.768733024597168,
        "lp_dist": -22.300951957702637,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-69",
      "gold_norm": "1",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.650723457336426,
        "lp_gold": -15.441932678222656,
        "lp_dist": -18.092656135559082,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.28917121887207,
        "lp_gold": -14.679984092712402,
        "lp_dist": -18.969155311584473,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-122",
      "gold_norm": "2",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.9139537811279297,
        "lp_gold": -18.55172109603882,
        "lp_dist": -17.63776731491089,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.36802101135253906,
        "lp_gold": -18.541946411132812,
        "lp_dist": -18.173925399780273,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-108",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.612565040588379,
        "lp_gold": -22.057893753051758,
        "lp_dist": -19.44532871246338,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.8640785217285156,
        "lp_gold": -23.09704303741455,
        "lp_dist": -19.232964515686035,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-48",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.96462059020996,
        "lp_dist": -21.96462059020996,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -24.449016571044922,
        "lp_dist": -24.449016571044922,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-17",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.34698486328125,
        "lp_gold": -21.55142116546631,
        "lp_dist": -22.89840602874756,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.0376691818237305,
        "lp_gold": -20.85892963409424,
        "lp_dist": -21.89659881591797,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-95",
      "gold_norm": "0",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8023662567138672,
        "lp_gold": -20.999281883239746,
        "lp_dist": -19.19691562652588,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.70697021484375,
        "lp_gold": -21.713737964630127,
        "lp_dist": -20.006767749786377,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-91",
      "gold_norm": "2",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.7028169631958,
        "lp_dist": -21.7028169631958,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.45036220550537,
        "lp_dist": -22.45036220550537,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-79",
      "gold_norm": "2",
      "dist_norm": "+1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.754400730133057,
        "lp_gold": -17.42607831954956,
        "lp_dist": -24.180479049682617,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.1651506423950195,
        "lp_gold": -19.892897605895996,
        "lp_dist": -26.058048248291016,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-14",
      "gold_norm": "+1",
      "dist_norm": "3",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.5707898139953613,
        "lp_gold": -29.604674816131592,
        "lp_dist": -26.03388500213623,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.903932571411133,
        "lp_gold": -27.936614990234375,
        "lp_dist": -22.032682418823242,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-77",
      "gold_norm": "3",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.08347511291503906,
        "lp_gold": -20.79250478744507,
        "lp_dist": -20.875979900360107,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.4646930694580078,
        "lp_gold": -21.741466522216797,
        "lp_dist": -22.206159591674805,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-3",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.1768722534179688,
        "lp_gold": -23.416447639465332,
        "lp_dist": -21.239575386047363,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.3070783615112305,
        "lp_gold": -20.640877723693848,
        "lp_dist": -16.333799362182617,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-146",
      "gold_norm": "1",
      "dist_norm": "if l == sorted(l) or l == sorted(l, reverse=True):\n        return True\n    return False",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 52.92330244462937,
        "lp_gold": -17.06475257873535,
        "lp_dist": -69.98805502336472,
        "n_tokens_gold": 2,
        "n_tokens_dist": 27
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 50.48234937642701,
        "lp_gold": -18.093742847442627,
        "lp_dist": -68.57609222386964,
        "n_tokens_gold": 2,
        "n_tokens_dist": 27
      }
    },
    {
      "ex_id": "openai_humaneval-test-57",
      "gold_norm": "if l == sorted(l) or l == sorted(l, reverse=True):\n        return True\n    return False",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -19.544386039720848,
        "lp_gold": -39.81637967680581,
        "lp_dist": -20.27199363708496,
        "n_tokens_gold": 27,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -23.215539787866874,
        "lp_gold": -47.03316483364324,
        "lp_dist": -23.817625045776367,
        "n_tokens_gold": 27,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-143",
      "gold_norm": "2",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -18.56871795654297,
        "lp_dist": -18.56871795654297,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -18.614916801452637,
        "lp_dist": -18.614916801452637,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-0",
      "gold_norm": "2",
      "dist_norm": "+1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.107102870941162,
        "lp_gold": -24.08918285369873,
        "lp_dist": -31.196285724639893,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.970984935760498,
        "lp_gold": -24.804469108581543,
        "lp_dist": -31.77545404434204,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-106",
      "gold_norm": "+1",
      "dist_norm": "26",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2219808101654053,
        "lp_gold": -21.646430253982544,
        "lp_dist": -23.86841106414795,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4264450073242188,
        "lp_gold": -24.843106269836426,
        "lp_dist": -23.416661262512207,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-50",
      "gold_norm": "26",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6608308553695679,
        "lp_gold": -26.048061728477478,
        "lp_dist": -25.38723087310791,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.0806753635406494,
        "lp_gold": -25.965544939041138,
        "lp_dist": -23.88486957550049,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-58",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0009937286376953,
        "lp_gold": -17.66666841506958,
        "lp_dist": -19.667662143707275,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.9970569610595703,
        "lp_gold": -19.63721752166748,
        "lp_dist": -21.63427448272705,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-147",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.277851104736328,
        "lp_gold": -21.182048797607422,
        "lp_dist": -17.904197692871094,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.9687089920043945,
        "lp_gold": -23.235244750976562,
        "lp_dist": -19.266535758972168,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-160",
      "gold_norm": "1",
      "dist_norm": "lis = list()\n    for i in s.split(' '):\n        if i.isdigit():\n            lis.append(int(i))\n    return n - sum(lis)",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 82.88829492655168,
        "lp_gold": -16.034985065460205,
        "lp_dist": -98.92327999201189,
        "n_tokens_gold": 2,
        "n_tokens_dist": 45
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 79.75362334529109,
        "lp_gold": -16.3989200592041,
        "lp_dist": -96.15254340449519,
        "n_tokens_gold": 2,
        "n_tokens_dist": 45
      }
    },
    {
      "ex_id": "openai_humaneval-test-67",
      "gold_norm": "lis = list()\n    for i in s.split(' '):\n        if i.isdigit():\n            lis.append(int(i))\n    return n - sum(lis)",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -49.84833694786221,
        "lp_gold": -66.66380525917202,
        "lp_dist": -16.815468311309814,
        "n_tokens_gold": 45,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -48.502373244005184,
        "lp_gold": -65.89947893782232,
        "lp_dist": -17.39710569381714,
        "n_tokens_gold": 45,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-55",
      "gold_norm": "2",
      "dist_norm": "-1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.25999921560287476,
        "lp_gold": -24.11659049987793,
        "lp_dist": -23.856591284275055,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.37139952182769775,
        "lp_gold": -24.66786289215088,
        "lp_dist": -25.039262413978577,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-118",
      "gold_norm": "-1",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.466064929962158,
        "lp_gold": -23.7993106842041,
        "lp_dist": -17.333245754241943,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.198596954345703,
        "lp_gold": -26.275648593902588,
        "lp_dist": -18.077051639556885,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-154",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -14.774529933929443,
        "lp_dist": -14.774529933929443,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -16.673503875732422,
        "lp_dist": -16.673503875732422,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-25",
      "gold_norm": "1",
      "dist_norm": "7",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.230724334716797,
        "lp_gold": -22.422992706298828,
        "lp_dist": -25.653717041015625,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.7021970748901367,
        "lp_gold": -19.818692207336426,
        "lp_dist": -23.520889282226562,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-36",
      "gold_norm": "7",
      "dist_norm": "3",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.6618576049804688,
        "lp_gold": -20.638415336608887,
        "lp_dist": -23.300272941589355,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.1126937866210938,
        "lp_gold": -22.601848602294922,
        "lp_dist": -25.714542388916016,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-63",
      "gold_norm": "3",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4981985092163086,
        "lp_gold": -23.173019409179688,
        "lp_dist": -21.67482089996338,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.069631576538086,
        "lp_gold": -24.268108367919922,
        "lp_dist": -23.198476791381836,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-132",
      "gold_norm": "2",
      "dist_norm": "min_number = min(numbers)\n    max_number = max(numbers)\n    return [(x - min_number) / (max_number - min_number) for x in numbers]",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 68.18116059236388,
        "lp_gold": -18.65871238708496,
        "lp_dist": -86.83987297944884,
        "n_tokens_gold": 2,
        "n_tokens_dist": 44
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 67.7937639305901,
        "lp_gold": -20.1024751663208,
        "lp_dist": -87.8962390969109,
        "n_tokens_gold": 2,
        "n_tokens_dist": 44
      }
    },
    {
      "ex_id": "openai_humaneval-test-21",
      "gold_norm": "min_number = min(numbers)\n    max_number = max(numbers)\n    return [(x - min_number) / (max_number - min_number) for x in numbers]",
      "dist_norm": "running_max = None\n    result = []\n\n    for n in numbers:\n        if running_max is None:\n            running_max = n\n        else:\n            running_max = max(running_max, n)\n\n        result.append(running_max)\n\n    return result",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 23.721003586428438,
        "lp_gold": -41.06687730719631,
        "lp_dist": -64.78788089362475,
        "n_tokens_gold": 44,
        "n_tokens_dist": 69
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 32.38431728373951,
        "lp_gold": -42.50547727979096,
        "lp_dist": -74.88979456353047,
        "n_tokens_gold": 44,
        "n_tokens_dist": 69
      }
    },
    {
      "ex_id": "openai_humaneval-test-9",
      "gold_norm": "running_max = None\n    result = []\n\n    for n in numbers:\n        if running_max is None:\n            running_max = n\n        else:\n            running_max = max(running_max, n)\n\n        result.append(running_max)\n\n    return result",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -29.805189722821147,
        "lp_gold": -50.413360232159526,
        "lp_dist": -20.60817050933838,
        "n_tokens_gold": 69,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -51.59206402557629,
        "lp_gold": -70.91829026001233,
        "lp_dist": -19.326226234436035,
        "n_tokens_gold": 69,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-72",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.30158805847168,
        "lp_gold": -13.809431552886963,
        "lp_dist": -19.111019611358643,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 4.650420188903809,
        "lp_gold": -15.45082139968872,
        "lp_dist": -20.10124158859253,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-128",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.34075927734375,
        "lp_gold": -21.099153995513916,
        "lp_dist": -17.758394718170166,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -4.7813920974731445,
        "lp_gold": -22.55471706390381,
        "lp_dist": -17.773324966430664,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-126",
      "gold_norm": "1",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 7.619829177856445,
        "lp_gold": -16.480793476104736,
        "lp_dist": -24.10062265396118,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 7.267373561859131,
        "lp_gold": -18.10063362121582,
        "lp_dist": -25.36800718307495,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-70",
      "gold_norm": "-1",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.621180534362793,
        "lp_gold": -23.137717247009277,
        "lp_dist": -20.516536712646484,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.558529853820801,
        "lp_gold": -25.226367950439453,
        "lp_dist": -22.667838096618652,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-40",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -18.633878707885742,
        "lp_dist": -18.633878707885742,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.059532165527344,
        "lp_dist": -21.059532165527344,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-66",
      "gold_norm": "0",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.13283920288085938,
        "lp_gold": -17.218024253845215,
        "lp_dist": -17.085185050964355,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.1121377944946289,
        "lp_gold": -18.084847927093506,
        "lp_dist": -17.972710132598877,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-71",
      "gold_norm": "2",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2154407501220703,
        "lp_gold": -18.285062789916992,
        "lp_dist": -17.069622039794922,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.0220775604248047,
        "lp_gold": -19.058651447296143,
        "lp_dist": -18.036573886871338,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-107",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.3218297958374023,
        "lp_gold": -17.986241340637207,
        "lp_dist": -20.30807113647461,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.979066848754883,
        "lp_gold": -16.702817916870117,
        "lp_dist": -19.681884765625,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-32",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.1487884521484375,
        "lp_gold": -22.8189640045166,
        "lp_dist": -24.96775245666504,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.087052345275879,
        "lp_gold": -22.50095844268799,
        "lp_dist": -24.588010787963867,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-11",
      "gold_norm": "1",
      "dist_norm": "2.0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 13.553829669952393,
        "lp_gold": -20.95840072631836,
        "lp_dist": -34.51223039627075,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 11.875772953033447,
        "lp_gold": -22.25577449798584,
        "lp_dist": -34.13154745101929,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-47",
      "gold_norm": "2.0",
      "dist_norm": "return [x for x in values if isinstance(x, int)]",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 22.397947419434786,
        "lp_gold": -22.972108960151672,
        "lp_dist": -45.37005637958646,
        "n_tokens_gold": 4,
        "n_tokens_dist": 16
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 21.156506193903624,
        "lp_gold": -25.384591817855835,
        "lp_dist": -46.54109801175946,
        "n_tokens_gold": 4,
        "n_tokens_dist": 16
      }
    },
    {
      "ex_id": "openai_humaneval-test-22",
      "gold_norm": "return [x for x in values if isinstance(x, int)]",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.9452141776912413,
        "lp_gold": -21.9394551262028,
        "lp_dist": -24.884669303894043,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.21686965267372216,
        "lp_gold": -23.520365059778214,
        "lp_dist": -23.303495407104492,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-140",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.853110790252686,
        "lp_dist": -22.853110790252686,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -24.441052436828613,
        "lp_dist": -24.441052436828613,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-30",
      "gold_norm": "0",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.6810879707336426,
        "lp_gold": -22.556360244750977,
        "lp_dist": -24.23744821548462,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2648732662200928,
        "lp_gold": -24.702096939086914,
        "lp_dist": -26.966970205307007,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-10",
      "gold_norm": "-1",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -26.95520782470703,
        "lp_dist": -26.95520782470703,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -24.403673887252808,
        "lp_dist": -24.403673887252808,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-112",
      "gold_norm": "-1",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -6.778575420379639,
        "lp_gold": -21.71489953994751,
        "lp_dist": -14.936324119567871,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.789752006530762,
        "lp_gold": -22.717198848724365,
        "lp_dist": -15.927446842193604,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-111",
      "gold_norm": "1",
      "dist_norm": "return ''.join(strings)",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 23.62041076645255,
        "lp_gold": -18.88068723678589,
        "lp_dist": -42.50109800323844,
        "n_tokens_gold": 2,
        "n_tokens_dist": 8
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 20.59145486354828,
        "lp_gold": -19.92290210723877,
        "lp_dist": -40.51435697078705,
        "n_tokens_gold": 2,
        "n_tokens_dist": 8
      }
    },
    {
      "ex_id": "openai_humaneval-test-28",
      "gold_norm": "return ''.join(strings)",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.083352681776887,
        "lp_gold": -25.75984155811966,
        "lp_dist": -24.676488876342773,
        "n_tokens_gold": 8,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2199663514859367,
        "lp_gold": -22.93246036817959,
        "lp_dist": -24.152426719665527,
        "n_tokens_gold": 8,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-135",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -16.229734420776367,
        "lp_dist": -16.229734420776367,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -16.821220874786377,
        "lp_dist": -16.821220874786377,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-94",
      "gold_norm": "1",
      "dist_norm": "while b:\n        a, b = b, a % b\n    return a",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 27.14890766143799,
        "lp_gold": -17.605591773986816,
        "lp_dist": -44.754499435424805,
        "n_tokens_gold": 2,
        "n_tokens_dist": 19
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 32.34489552024752,
        "lp_gold": -17.090249061584473,
        "lp_dist": -49.43514458183199,
        "n_tokens_gold": 2,
        "n_tokens_dist": 19
      }
    },
    {
      "ex_id": "openai_humaneval-test-13",
      "gold_norm": "while b:\n        a, b = b, a % b\n    return a",
      "dist_norm": "return [x for x in strings if substring in x]",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 27.657666564541046,
        "lp_gold": -24.580688443994973,
        "lp_dist": -52.23835500853602,
        "n_tokens_gold": 19,
        "n_tokens_dist": 13
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 26.75688247599828,
        "lp_gold": -29.810987717977696,
        "lp_dist": -56.56787019397598,
        "n_tokens_gold": 19,
        "n_tokens_dist": 13
      }
    },
    {
      "ex_id": "openai_humaneval-test-7",
      "gold_norm": "return [x for x in strings if substring in x]",
      "dist_norm": "return a*a == b*b + c*c or b*b == a*a + c*c or c*c == a*a + b*b",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 60.87681590977445,
        "lp_gold": -23.353178574070625,
        "lp_dist": -84.22999448384508,
        "n_tokens_gold": 13,
        "n_tokens_dist": 37
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 83.19848300620379,
        "lp_gold": -21.97879713297425,
        "lp_dist": -105.17728013917804,
        "n_tokens_gold": 13,
        "n_tokens_dist": 37
      }
    },
    {
      "ex_id": "openai_humaneval-test-157",
      "gold_norm": "return a*a == b*b + c*c or b*b == a*a + c*c or c*c == a*a + b*b",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -22.37864390958157,
        "lp_gold": -42.11520473111477,
        "lp_dist": -19.736560821533203,
        "n_tokens_gold": 37,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -21.383852594010023,
        "lp_gold": -42.70623838197389,
        "lp_dist": -21.322385787963867,
        "n_tokens_gold": 37,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-49",
      "gold_norm": "2",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 6.1286115646362305,
        "lp_gold": -14.368673324584961,
        "lp_dist": -20.49728488922119,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.370944976806641,
        "lp_gold": -15.46406078338623,
        "lp_dist": -21.83500576019287,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-120",
      "gold_norm": "0",
      "dist_norm": "26",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.207955360412598,
        "lp_gold": -19.637977600097656,
        "lp_dist": -23.845932960510254,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.25955867767334,
        "lp_gold": -19.60517692565918,
        "lp_dist": -21.86473560333252,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-89",
      "gold_norm": "26",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -7.005466938018799,
        "lp_gold": -25.530439853668213,
        "lp_dist": -18.524972915649414,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.288686752319336,
        "lp_gold": -25.664198875427246,
        "lp_dist": -19.37551212310791,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-98",
      "gold_norm": "1",
      "dist_norm": "2",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.9043750762939453,
        "lp_gold": -21.17328405380249,
        "lp_dist": -23.077659130096436,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2112646102905273,
        "lp_gold": -22.75312328338623,
        "lp_dist": -24.964387893676758,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-100",
      "gold_norm": "2",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.1701860427856445,
        "lp_gold": -22.101744651794434,
        "lp_dist": -19.93155860900879,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.7842912673950195,
        "lp_gold": -23.208542823791504,
        "lp_dist": -21.424251556396484,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-64",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -17.467191696166992,
        "lp_dist": -17.467191696166992,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -17.525324821472168,
        "lp_dist": -17.525324821472168,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-103",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.922170639038086,
        "lp_gold": -16.26321840286255,
        "lp_dist": -19.185389041900635,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.1872482299804688,
        "lp_gold": -18.55193328857422,
        "lp_dist": -21.739181518554688,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-125",
      "gold_norm": "0",
      "dist_norm": "return \"\".join([s for s in text if s.lower() not in [\"a\", \"e\", \"i\", \"o\", \"u\"]])",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 48.67979192888379,
        "lp_gold": -21.47081232070923,
        "lp_dist": -70.15060424959302,
        "n_tokens_gold": 2,
        "n_tokens_dist": 34
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 55.4912621024414,
        "lp_gold": -21.899943828582764,
        "lp_dist": -77.39120593102416,
        "n_tokens_gold": 2,
        "n_tokens_dist": 34
      }
    },
    {
      "ex_id": "openai_humaneval-test-51",
      "gold_norm": "return \"\".join([s for s in text if s.lower() not in [\"a\", \"e\", \"i\", \"o\", \"u\"]])",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -15.132140384576928,
        "lp_gold": -33.735738979242456,
        "lp_dist": -18.603598594665527,
        "n_tokens_gold": 34,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -17.125044157398406,
        "lp_gold": -38.15538339842624,
        "lp_dist": -21.030339241027832,
        "n_tokens_gold": 34,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-90",
      "gold_norm": "1",
      "dist_norm": "return [abs(x-y) for x,y in zip(game,guess)]",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 53.59280555654732,
        "lp_gold": -17.83990478515625,
        "lp_dist": -71.43271034170357,
        "n_tokens_gold": 2,
        "n_tokens_dist": 21
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 53.46250804614101,
        "lp_gold": -17.562668800354004,
        "lp_dist": -71.02517684649501,
        "n_tokens_gold": 2,
        "n_tokens_dist": 21
      }
    },
    {
      "ex_id": "openai_humaneval-test-152",
      "gold_norm": "return [abs(x-y) for x,y in zip(game,guess)]",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.4121496778377605,
        "lp_gold": -12.769094695483773,
        "lp_dist": -18.181244373321533,
        "n_tokens_gold": 21,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 6.846039920872499,
        "lp_gold": -11.968789428645323,
        "lp_dist": -18.814829349517822,
        "n_tokens_gold": 21,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-24",
      "gold_norm": "0",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.762319564819336,
        "lp_gold": -21.209729194641113,
        "lp_dist": -19.447409629821777,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.050973892211914,
        "lp_gold": -23.011981964111328,
        "lp_dist": -20.961008071899414,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-20",
      "gold_norm": "2",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.5170793533325195,
        "lp_gold": -21.059219360351562,
        "lp_dist": -18.542140007019043,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.842561721801758,
        "lp_gold": -24.164966583251953,
        "lp_dist": -20.322404861450195,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-148",
      "gold_norm": "1",
      "dist_norm": "if isinstance(x,int) and isinstance(y,int) and isinstance(z,int):\n        if (x+y==z) or (x+z==y) or (y+z==x):\n            return True\n        return False\n    return False",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 84.42804804586194,
        "lp_gold": -12.750839233398438,
        "lp_dist": -97.17888727926038,
        "n_tokens_gold": 2,
        "n_tokens_dist": 63
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 85.3895359868402,
        "lp_gold": -12.885719776153564,
        "lp_dist": -98.27525576299377,
        "n_tokens_gold": 2,
        "n_tokens_dist": 63
      }
    },
    {
      "ex_id": "openai_humaneval-test-92",
      "gold_norm": "if isinstance(x,int) and isinstance(y,int) and isinstance(z,int):\n        if (x+y==z) or (x+z==y) or (y+z==x):\n            return True\n        return False\n    return False",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -15.070005173284699,
        "lp_gold": -29.284181828100373,
        "lp_dist": -14.214176654815674,
        "n_tokens_gold": 63,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -16.731429186844252,
        "lp_gold": -33.610841837906264,
        "lp_dist": -16.87941265106201,
        "n_tokens_gold": 63,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-26",
      "gold_norm": "1",
      "dist_norm": "return len(set(string.lower()))",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 39.93833448708756,
        "lp_gold": -21.593355178833008,
        "lp_dist": -61.53168966592057,
        "n_tokens_gold": 2,
        "n_tokens_dist": 10
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 44.07288610804244,
        "lp_gold": -19.37473964691162,
        "lp_dist": -63.44762575495406,
        "n_tokens_gold": 2,
        "n_tokens_dist": 10
      }
    },
    {
      "ex_id": "openai_humaneval-test-16",
      "gold_norm": "return len(set(string.lower()))",
      "dist_norm": "3",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.23131456784904003,
        "lp_gold": -21.334825424477458,
        "lp_dist": -21.103510856628418,
        "n_tokens_gold": 10,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 1.4797544392640702,
        "lp_gold": -20.777993210882414,
        "lp_dist": -22.257747650146484,
        "n_tokens_gold": 10,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-142",
      "gold_norm": "3",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.5028867721557617,
        "lp_gold": -20.428752899169922,
        "lp_dist": -18.92586612701416,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.940119743347168,
        "lp_gold": -20.01767873764038,
        "lp_dist": -19.077558994293213,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-99",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.057826042175293,
        "lp_dist": -20.057826042175293,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.03865337371826,
        "lp_dist": -21.03865337371826,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-127",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.7136526107788086,
        "lp_gold": -19.18015956878662,
        "lp_dist": -15.466506958007812,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.9199934005737305,
        "lp_gold": -19.79235076904297,
        "lp_dist": -15.872357368469238,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-156",
      "gold_norm": "1",
      "dist_norm": "3",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.246551513671875,
        "lp_gold": -19.624220848083496,
        "lp_dist": -21.87077236175537,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.5017471313476562,
        "lp_gold": -21.236965656280518,
        "lp_dist": -23.738712787628174,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-141",
      "gold_norm": "3",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.184619903564453,
        "lp_gold": -20.60717535018921,
        "lp_dist": -17.422555446624756,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.860507011413574,
        "lp_gold": -21.143166542053223,
        "lp_dist": -18.28265953063965,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-78",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2415552139282227,
        "lp_gold": -19.249483585357666,
        "lp_dist": -21.49103879928589,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.3341121673583984,
        "lp_gold": -19.18357276916504,
        "lp_dist": -21.517684936523438,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-68",
      "gold_norm": "0",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2041006088256836,
        "lp_gold": -18.397984981536865,
        "lp_dist": -17.19388437271118,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.9792289733886719,
        "lp_gold": -17.703375339508057,
        "lp_dist": -16.724146366119385,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-93",
      "gold_norm": "2",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -2.300654411315918,
        "lp_gold": -20.44687557220459,
        "lp_dist": -18.146221160888672,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.002674102783203,
        "lp_gold": -19.50245952606201,
        "lp_dist": -17.49978542327881,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-60",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.0008726119995117,
        "lp_gold": -17.277544021606445,
        "lp_dist": -19.278416633605957,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 2.495342254638672,
        "lp_gold": -20.217015266418457,
        "lp_dist": -22.71235752105713,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-82",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -18.858778476715088,
        "lp_dist": -18.858778476715088,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.178369522094727,
        "lp_dist": -20.178369522094727,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-59",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -24.201942443847656,
        "lp_dist": -24.201942443847656,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -25.577584266662598,
        "lp_dist": -25.577584266662598,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-149",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.985135078430176,
        "lp_gold": -18.369908809661865,
        "lp_dist": -14.38477373123169,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -3.8884334564208984,
        "lp_gold": -19.89932155609131,
        "lp_dist": -16.01088809967041,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-42",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.361058235168457,
        "lp_gold": -16.945013999938965,
        "lp_dist": -20.306072235107422,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.650448799133301,
        "lp_gold": -18.96267032623291,
        "lp_dist": -22.61311912536621,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-163",
      "gold_norm": "0",
      "dist_norm": "2",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.24646282196044922,
        "lp_gold": -21.461342811584473,
        "lp_dist": -21.214879989624023,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1632556915283203,
        "lp_gold": -22.695805549621582,
        "lp_dist": -22.859061241149902,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-133",
      "gold_norm": "2",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.3052263259887695,
        "lp_gold": -19.276253700256348,
        "lp_dist": -24.581480026245117,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.072274208068848,
        "lp_gold": -20.15150022506714,
        "lp_dist": -25.223774433135986,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-161",
      "gold_norm": "-1",
      "dist_norm": "return [x for x in strings if x.startswith(prefix)]",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 34.15456820592226,
        "lp_gold": -21.663207292556763,
        "lp_dist": -55.81777549847902,
        "n_tokens_gold": 3,
        "n_tokens_dist": 16
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 39.88924138105904,
        "lp_gold": -23.96406078338623,
        "lp_dist": -63.85330216444527,
        "n_tokens_gold": 3,
        "n_tokens_dist": 16
      }
    },
    {
      "ex_id": "openai_humaneval-test-29",
      "gold_norm": "return [x for x in strings if x.startswith(prefix)]",
      "dist_norm": "return string.swapcase()",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 29.161806431533478,
        "lp_gold": -23.64715713548503,
        "lp_dist": -52.80896356701851,
        "n_tokens_gold": 16,
        "n_tokens_dist": 7
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 27.964344927147977,
        "lp_gold": -21.276650241537936,
        "lp_dist": -49.24099516868591,
        "n_tokens_gold": 16,
        "n_tokens_dist": 7
      }
    },
    {
      "ex_id": "openai_humaneval-test-27",
      "gold_norm": "return string.swapcase()",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.143668868753593,
        "lp_gold": -18.235226890828926,
        "lp_dist": -23.37889575958252,
        "n_tokens_gold": 7,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.050230901411851,
        "lp_gold": -19.54451664192311,
        "lp_dist": -22.59474754333496,
        "n_tokens_gold": 7,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-61",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.488422393798828,
        "lp_gold": -21.18766164779663,
        "lp_dist": -17.699239253997803,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.8497695922851562,
        "lp_gold": -23.119681358337402,
        "lp_dist": -20.269911766052246,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-104",
      "gold_norm": "1",
      "dist_norm": "2.0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 8.777350425720215,
        "lp_gold": -21.164722442626953,
        "lp_dist": -29.942072868347168,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 8.533888697624207,
        "lp_gold": -22.994476318359375,
        "lp_dist": -31.52836501598358,
        "n_tokens_gold": 2,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-45",
      "gold_norm": "2.0",
      "dist_norm": "101",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.062639269977808,
        "lp_gold": -26.36994906887412,
        "lp_dist": -31.43258833885193,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.08874404430389404,
        "lp_gold": -27.106158316135406,
        "lp_dist": -27.017414271831512,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-75",
      "gold_norm": "101",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -7.495836019515991,
        "lp_gold": -24.720885515213013,
        "lp_dist": -17.22504949569702,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -6.730545282363892,
        "lp_gold": -24.493883848190308,
        "lp_dist": -17.763338565826416,
        "n_tokens_gold": 4,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-6",
      "gold_norm": "1",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 3.3637475967407227,
        "lp_gold": -19.45144271850586,
        "lp_dist": -22.815190315246582,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 3.392610549926758,
        "lp_gold": -19.24812602996826,
        "lp_dist": -22.64073657989502,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-136",
      "gold_norm": "0",
      "dist_norm": "3",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -0.40959739685058594,
        "lp_gold": -18.25609064102173,
        "lp_dist": -17.846493244171143,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0858154296875,
        "lp_gold": -19.771966457366943,
        "lp_dist": -19.857781887054443,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-33",
      "gold_norm": "3",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.021329879760742188,
        "lp_gold": -19.846437454223633,
        "lp_dist": -19.867767333984375,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.7291717529296875,
        "lp_gold": -21.070161819458008,
        "lp_dist": -21.799333572387695,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-44",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.9867525100708,
        "lp_dist": -22.9867525100708,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.455985069274902,
        "lp_dist": -22.455985069274902,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-88",
      "gold_norm": "0",
      "dist_norm": "if not strings:\n        return None\n\n    maxlen = max(len(x) for x in strings)\n    for s in strings:\n        if len(s) == maxlen:\n            return s",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 91.28553534628554,
        "lp_gold": -19.742467880249023,
        "lp_dist": -111.02800322653457,
        "n_tokens_gold": 2,
        "n_tokens_dist": 48
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 100.06942167917623,
        "lp_gold": -20.66820240020752,
        "lp_dist": -120.73762407938375,
        "n_tokens_gold": 2,
        "n_tokens_dist": 48
      }
    },
    {
      "ex_id": "openai_humaneval-test-12",
      "gold_norm": "if not strings:\n        return None\n\n    maxlen = max(len(x) for x in strings)\n    for s in strings:\n        if len(s) == maxlen:\n            return s",
      "dist_norm": "9",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -26.818854912871643,
        "lp_gold": -51.12532578384332,
        "lp_dist": -24.30647087097168,
        "n_tokens_gold": 48,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -32.62358831267056,
        "lp_gold": -56.370979059740876,
        "lp_dist": -23.747390747070312,
        "n_tokens_gold": 48,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-105",
      "gold_norm": "9",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.5095605850219727,
        "lp_gold": -20.889235973358154,
        "lp_dist": -17.37967538833618,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.047847747802734,
        "lp_gold": -22.52873468399048,
        "lp_dist": -17.480886936187744,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-121",
      "gold_norm": "1",
      "dist_norm": "a, b = x.split(\"/\")\n    c, d = n.split(\"/\")\n    numerator = int(a) * int(c)\n    denom = int(b) * int(d)\n    if (numerator/denom == int(numerator/denom)):\n        return True\n    return False",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 97.35365189886329,
        "lp_gold": -20.428816318511963,
        "lp_dist": -117.78246821737525,
        "n_tokens_gold": 2,
        "n_tokens_dist": 75
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 109.46097758087126,
        "lp_gold": -22.744394302368164,
        "lp_dist": -132.20537188323942,
        "n_tokens_gold": 2,
        "n_tokens_dist": 75
      }
    },
    {
      "ex_id": "openai_humaneval-test-144",
      "gold_norm": "a, b = x.split(\"/\")\n    c, d = n.split(\"/\")\n    numerator = int(a) * int(c)\n    denom = int(b) * int(d)\n    if (numerator/denom == int(numerator/denom)):\n        return True\n    return False",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -39.772302772080536,
        "lp_gold": -56.47381558564865,
        "lp_dist": -16.701512813568115,
        "n_tokens_gold": 75,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -56.80937039689411,
        "lp_gold": -74.05506884888996,
        "lp_dist": -17.24569845199585,
        "n_tokens_gold": 75,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-73",
      "gold_norm": "1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -19.193764686584473,
        "lp_dist": -19.193764686584473,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -20.223108291625977,
        "lp_dist": -20.223108291625977,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-76",
      "gold_norm": "1",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 4.929764986038208,
        "lp_gold": -15.572730541229248,
        "lp_dist": -20.502495527267456,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 5.190648078918457,
        "lp_gold": -16.13015127182007,
        "lp_dist": -21.320799350738525,
        "n_tokens_gold": 2,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-5",
      "gold_norm": "-1",
      "dist_norm": "-1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -25.595508337020874,
        "lp_dist": -25.595508337020874,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -30.356321334838867,
        "lp_dist": -30.356321334838867,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-46",
      "gold_norm": "-1",
      "dist_norm": "0",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.3807437419891357,
        "lp_gold": -22.49609923362732,
        "lp_dist": -19.115355491638184,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.765854597091675,
        "lp_gold": -24.438928365707397,
        "lp_dist": -21.673073768615723,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-150",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.250157833099365,
        "lp_dist": -22.250157833099365,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -22.30048179626465,
        "lp_dist": -22.30048179626465,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-35",
      "gold_norm": "0",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -21.539960861206055,
        "lp_dist": -21.539960861206055,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "gold",
        "correct": true,
        "margin": 0.0,
        "lp_gold": -23.488576889038086,
        "lp_dist": -23.488576889038086,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-153",
      "gold_norm": "0",
      "dist_norm": "1",
      "baseline": {
        "pred": "dist",
        "correct": false,
        "margin": -3.9339113235473633,
        "lp_gold": -19.433120727539062,
        "lp_dist": -15.4992094039917,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -5.801613807678223,
        "lp_gold": -20.252619743347168,
        "lp_dist": -14.451005935668945,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    }
  ],
  "flip_rows": [
    {
      "ex_id": "openai_humaneval-test-39",
      "gold_norm": "-1",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.44877076148986816,
        "lp_gold": -19.808196783065796,
        "lp_dist": -20.256967544555664,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.15900945663452148,
        "lp_gold": -22.082778453826904,
        "lp_dist": -21.923768997192383,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.70906001329422,
        "lp_gold": -19.808191001415253,
        "lp_dist": -20.517251014709473,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.2600139379501343,
        "lp_gold": -20.45387899875641,
        "lp_dist": -20.713892936706543,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.896909236907959,
        "lp_gold": -22.914924144744873,
        "lp_dist": -21.018014907836914,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "gold",
        "correct": true,
        "margin": 0.4487643241882324,
        "lp_gold": -22.053846836090088,
        "lp_dist": -22.50261116027832,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.5229349136352539,
        "lp_gold": -22.082777976989746,
        "lp_dist": -21.559843063354492,
        "n_tokens_gold": 3,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-134",
      "gold_norm": "122",
      "dist_norm": "1.0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8403654620051384,
        "lp_gold": -27.5423321723938,
        "lp_dist": -28.382697634398937,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -2.457980029284954,
        "lp_gold": -29.874002933502197,
        "lp_dist": -27.416022904217243,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -2.6199172660708427,
        "lp_gold": -27.5423264503479,
        "lp_dist": -24.922409184277058,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.1696217805147171,
        "lp_gold": -28.275829792022705,
        "lp_dist": -28.445451572537422,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.7151931561529636,
        "lp_gold": -29.58313512802124,
        "lp_dist": -26.867941971868277,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -2.835217572748661,
        "lp_gold": -29.671478271484375,
        "lp_dist": -26.836260698735714,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -2.1110972091555595,
        "lp_gold": -29.87399911880493,
        "lp_dist": -27.762901909649372,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-4",
      "gold_norm": "mean = sum(numbers) / len(numbers)\n    return sum(abs(x - mean) for x in numbers) / len(numbers)",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.45850569046490364,
        "lp_gold": -22.989404618372987,
        "lp_dist": -23.44791030883789,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -8.605282360978833,
        "lp_gold": -30.1749826020677,
        "lp_dist": -21.569700241088867,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "dist",
        "correct": false,
        "margin": -6.219808316351077,
        "lp_gold": -29.551916813970706,
        "lp_dist": -23.33210849761963,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -6.893468835648207,
        "lp_gold": -30.730242708023695,
        "lp_dist": -23.83677387237549,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -8.074352583655013,
        "lp_gold": -29.26745828128685,
        "lp_dist": -21.193105697631836,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -9.070045394459783,
        "lp_gold": -29.0644835658465,
        "lp_dist": -19.99443817138672,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -8.754907058842889,
        "lp_gold": -30.174978660710565,
        "lp_dist": -21.420071601867676,
        "n_tokens_gold": 33,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-139",
      "gold_norm": "+1",
      "dist_norm": "0.0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 1.2652113437652588,
        "lp_gold": -22.48181676864624,
        "lp_dist": -23.7470281124115,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1343896389007568,
        "lp_gold": -23.847239017486572,
        "lp_dist": -22.712849378585815,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.8445468246936798,
        "lp_gold": -22.48181438446045,
        "lp_dist": -23.32636120915413,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -1.0516579747200012,
        "lp_gold": -24.160611152648926,
        "lp_dist": -23.108953177928925,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -2.6032419204711914,
        "lp_gold": -22.94077157974243,
        "lp_dist": -20.33752965927124,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.2553260326385498,
        "lp_gold": -24.39438009262085,
        "lp_dist": -23.1390540599823,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.7304326295852661,
        "lp_gold": -23.84722900390625,
        "lp_dist": -23.116796374320984,
        "n_tokens_gold": 3,
        "n_tokens_dist": 4
      }
    },
    {
      "ex_id": "openai_humaneval-test-19",
      "gold_norm": "9",
      "dist_norm": "1",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 0.07897377014160156,
        "lp_gold": -22.66366195678711,
        "lp_dist": -22.74263572692871,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4677810668945312,
        "lp_gold": -23.768733024597168,
        "lp_dist": -22.300951957702637,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.07897186279296875,
        "lp_gold": -22.663668632507324,
        "lp_dist": -22.742640495300293,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 0.2928171157836914,
        "lp_gold": -23.486221313476562,
        "lp_dist": -23.779038429260254,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.1306524276733398,
        "lp_gold": -22.043014526367188,
        "lp_dist": -20.912362098693848,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.335036277770996,
        "lp_gold": -23.414420127868652,
        "lp_dist": -22.079383850097656,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4677820205688477,
        "lp_gold": -23.768739700317383,
        "lp_dist": -22.300957679748535,
        "n_tokens_gold": 2,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-106",
      "gold_norm": "+1",
      "dist_norm": "26",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.2219808101654053,
        "lp_gold": -21.646430253982544,
        "lp_dist": -23.86841106414795,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -1.4264450073242188,
        "lp_gold": -24.843106269836426,
        "lp_dist": -23.416661262512207,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 0.3593015670776367,
        "lp_gold": -21.646427631378174,
        "lp_dist": -22.00572919845581,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_time_shuffled": {
        "pred": "dist",
        "correct": false,
        "margin": -2.295396327972412,
        "lp_gold": -24.159332752227783,
        "lp_dist": -21.86393642425537,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -1.9753856658935547,
        "lp_gold": -25.059080600738525,
        "lp_dist": -23.08369493484497,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -1.8524909019470215,
        "lp_gold": -25.18923282623291,
        "lp_dist": -23.33674192428589,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -1.216343879699707,
        "lp_gold": -24.843104362487793,
        "lp_dist": -23.626760482788086,
        "n_tokens_gold": 3,
        "n_tokens_dist": 3
      }
    },
    {
      "ex_id": "openai_humaneval-test-22",
      "gold_norm": "return [x for x in values if isinstance(x, int)]",
      "dist_norm": "0",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 2.9452141776912413,
        "lp_gold": -21.9394551262028,
        "lp_dist": -24.884669303894043,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.21686965267372216,
        "lp_gold": -23.520365059778214,
        "lp_dist": -23.303495407104492,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 2.3960064357215742,
        "lp_gold": -21.941916614205184,
        "lp_dist": -24.337923049926758,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 3.289606383860928,
        "lp_gold": -21.179310509144443,
        "lp_dist": -24.46891689300537,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -0.6156758015997639,
        "lp_gold": -24.370192403162264,
        "lp_dist": -23.7545166015625,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.07147166511367686,
        "lp_gold": -25.524140808790435,
        "lp_dist": -25.452669143676758,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      },
      "control_patch_nonshared": {
        "pred": "dist",
        "correct": false,
        "margin": -0.40586215297889794,
        "lp_gold": -23.520359337732316,
        "lp_dist": -23.114497184753418,
        "n_tokens_gold": 16,
        "n_tokens_dist": 2
      }
    },
    {
      "ex_id": "openai_humaneval-test-45",
      "gold_norm": "2.0",
      "dist_norm": "101",
      "baseline": {
        "pred": "gold",
        "correct": true,
        "margin": 5.062639269977808,
        "lp_gold": -26.36994906887412,
        "lp_dist": -31.43258833885193,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "ablated": {
        "pred": "dist",
        "correct": false,
        "margin": -0.08874404430389404,
        "lp_gold": -27.106158316135406,
        "lp_dist": -27.017414271831512,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "patched_self": {
        "pred": "gold",
        "correct": true,
        "margin": 5.745390687137842,
        "lp_gold": -26.369948115199804,
        "lp_dist": -32.11533880233765,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_time_shuffled": {
        "pred": "gold",
        "correct": true,
        "margin": 3.564100921154022,
        "lp_gold": -25.915751039981842,
        "lp_dist": -29.479851961135864,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_shared_randvec": {
        "pred": "dist",
        "correct": false,
        "margin": -0.768334724009037,
        "lp_gold": -28.143043376505375,
        "lp_dist": -27.374708652496338,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_rand_subspace": {
        "pred": "dist",
        "correct": false,
        "margin": -0.5243468135595322,
        "lp_gold": -28.490523919463158,
        "lp_dist": -27.966177105903625,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      },
      "control_patch_nonshared": {
        "pred": "gold",
        "correct": true,
        "margin": 0.04392840713262558,
        "lp_gold": -27.1061624661088,
        "lp_dist": -27.150090873241425,
        "n_tokens_gold": 4,
        "n_tokens_dist": 4
      }
    }
  ]
}